Support bundle deployment bind for DMS-recorded deployments

acceptance/bundle/deployment/bind/test.toml

@@ -1,2 +1,3 @@
-# Bind operations are not yet supported by the Deployment Metadata Service (DMS)
+# These tests cover the file-based bind, which stages its change for the next deploy. With the
+# Deployment Metadata Service (DMS) a bind is applied immediately, so it is covered by bundle/dms/bind.
 EnvMatrix.DMS = [""]

acceptance/bundle/dms/bind/databricks.yml.tmpl

@@ -0,0 +1,23 @@
+bundle:
+  name: dms-bind-$UNIQUE_NAME
+experimental:
+  deployment_history: true
+
+targets:
+  default:
+    default: true
+  first_job: {}
+
+resources:
+  schemas:
+    sc:
+      name: dms_bind_$UNIQUE_NAME
+      catalog_name: main
+  jobs:
+    foo:
+      name: foo
+      permissions:
+        - group_name: users
+          level: CAN_MANAGE
+    bar:
+      name: bar

acceptance/bundle/dms/bind/out.test.toml

@@ -0,0 +1,2 @@
+Cloud = true
+EnvMatrix.DMS = ["true"]

acceptance/bundle/dms/bind/output.txt

@@ -0,0 +1,167 @@
+
+>>> [CLI] schemas create dms_bind_[UNIQUE_NAME] main
+{
+  "full_name": "main.dms_bind_[UNIQUE_NAME]"
+}
+
+=== A missing resource cannot be bound
+>>> musterr [CLI] bundle deployment bind sc main.missing_[UNIQUE_NAME] --auto-approve
+Error: schema with an id 'main.missing_[UNIQUE_NAME]' is not found
+
+=== Bind sc, the first bind: it creates the deployment. The config already matches, so this is a plain bind that records the resource without touching it
+>>> [CLI] bundle deployment bind sc main.dms_bind_[UNIQUE_NAME]
+Successfully bound schema with an id 'main.dms_bind_[UNIQUE_NAME]'
+Bind changes have been applied to your workspace
+
+=== Version 1 records only sc, with the plain bind action type; the jobs are left untouched
+>>> print_requests.py --dms --method POST //versions --nostamp
+{
+  "method": "POST",
+  "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions",
+  "q": {
+    "version_id": "1"
+  },
+  "body": {
+    "cli_version": "[CLI_VERSION]",
+    "operations": [
+      {
+        "action_type": "OPERATION_ACTION_TYPE_BIND",
+        "resource_key": "schemas.sc"
+      }
+    ],
+    "version_type": "VERSION_TYPE_DEPLOY"
+  }
+}
+{
+  "method": "POST",
+  "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/1/complete",
+  "body": {
+    "completion_reason": "VERSION_COMPLETE_SUCCESS"
+  }
+}
+
+=== An update requires confirmation and leaves the workspace unchanged without it
+>>> [CLI] jobs get [FOO_ID]
+{
+  "name": "bind-target-[UNIQUE_NAME]"
+}
+
+=== Bind foo into the existing deployment: its config differs, so this is a bind-and-update that applies the config now
+>>> [CLI] bundle deployment bind foo [FOO_ID] --auto-approve
+Successfully bound job with an id '[FOO_ID]'
+Bind changes have been applied to your workspace
+
+=== Version 2 records only foo and its permissions (bar is untouched) and chains to version 1
+>>> print_requests.py --dms --method POST //versions --nostamp
+{
+  "method": "POST",
+  "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions",
+  "q": {
+    "version_id": "2"
+  },
+  "body": {
+    "cli_version": "[CLI_VERSION]",
+    "operations": [
+      {
+        "action_type": "OPERATION_ACTION_TYPE_BIND_AND_UPDATE",
+        "resource_key": "jobs.foo"
+      },
+      {
+        "action_type": "OPERATION_ACTION_TYPE_CREATE",
+        "resource_key": "jobs.foo.permissions"
+      }
+    ],
+    "previous_version_id": "1",
+    "version_type": "VERSION_TYPE_DEPLOY"
+  }
+}
+{
+  "method": "POST",
+  "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/2/complete",
+  "body": {
+    "completion_reason": "VERSION_COMPLETE_SUCCESS"
+  }
+}
+
+=== The bound job has already been updated
+>>> [CLI] jobs get [FOO_ID]
+{
+  "name": "foo"
+}
+
+=== An already-bound resource cannot be rebound
+>>> musterr [CLI] bundle deployment bind foo [FOO_ID] --auto-approve
+Error: resources.jobs.foo is already bound to ID "[FOO_ID]"; rebinding is not supported with deployment history
+
+
+=== sc and foo are now managed; bar is still an unmanaged create
+>>> [CLI] bundle plan
+create jobs.bar
+
+Plan: 1 to add, 0 to change, 0 to delete, 3 unchanged
+
+=== A job can be the first bind of a new deployment
+>>> [CLI] bundle deployment bind foo [FIRST_ID] -t first_job --auto-approve
+Successfully bound job with an id '[FIRST_ID]'
+Bind changes have been applied to your workspace
+
+>>> print_requests.py --dms --method POST //versions --nostamp
+{
+  "method": "POST",
+  "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions",
+  "q": {
+    "version_id": "1"
+  },
+  "body": {
+    "cli_version": "[CLI_VERSION]",
+    "operations": [
+      {
+        "action_type": "OPERATION_ACTION_TYPE_BIND_AND_UPDATE",
+        "resource_key": "jobs.foo"
+      },
+      {
+        "action_type": "OPERATION_ACTION_TYPE_CREATE",
+        "resource_key": "jobs.foo.permissions"
+      }
+    ],
+    "version_type": "VERSION_TYPE_DEPLOY"
+  }
+}
+{
+  "method": "POST",
+  "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/1/complete",
+  "body": {
+    "completion_reason": "VERSION_COMPLETE_SUCCESS"
+  }
+}
+
+=== The job points to the deployment that recorded its adoption, at version 1
+>>> [CLI] jobs get [FIRST_ID]
+{
+  "kind": "BUNDLE",
+  "version_id": "1"
+}
+
+>>> [CLI] bundle-deployments list-resources deployments/[DEPLOYMENT_ID] -o json
+true
+
+=== Test cleanup: the bound resources are now bundle-managed, so destroy removes them
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.jobs.foo
+  delete resources.schemas.sc
+
+This action will result in the deletion of the following UC schemas. Any underlying data may be lost:
+  delete resources.schemas.sc
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dms-bind-[UNIQUE_NAME]/default
+
+Destroy: 2 deleted
+
+>>> [CLI] bundle destroy -t first_job --auto-approve
+The following resources will be deleted:
+  delete resources.jobs.foo
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dms-bind-[UNIQUE_NAME]/first_job
+
+Destroy: 1 deleted

acceptance/bundle/dms/bind/script

@@ -0,0 +1,56 @@
+envsubst < databricks.yml.tmpl > databricks.yml
+
+# sc's config matches this existing schema exactly, so binding it changes nothing (a plain bind).
+trace $CLI schemas create "dms_bind_$UNIQUE_NAME" main | jq '{full_name}'
+# foo's config ("foo") differs from this existing job, so binding it applies an update.
+foo_id=$($CLI jobs create --json '{"name": "bind-target-'$UNIQUE_NAME'"}' | jq -r '.job_id')
+add_repl "$foo_id" FOO_ID
+
+cleanup() {
+    title "Test cleanup: the bound resources are now bundle-managed, so destroy removes them"
+    trace $CLI bundle destroy --auto-approve
+    if [[ -n "${first_id:-}" ]]; then
+        trace $CLI bundle destroy -t first_job --auto-approve
+    fi
+}
+trap cleanup EXIT
+
+title "A missing resource cannot be bound"
+trace musterr $CLI bundle deployment bind sc "main.missing_$UNIQUE_NAME" --auto-approve
+
+title "Bind sc, the first bind: it creates the deployment. The config already matches, so this is a plain bind that records the resource without touching it"
+trace $CLI bundle deployment bind sc "main.dms_bind_$UNIQUE_NAME"
+
+title "Version 1 records only sc, with the plain bind action type; the jobs are left untouched"
+trace print_requests.py --dms --method POST //versions --nostamp
+
+title "An update requires confirmation and leaves the workspace unchanged without it"
+trace musterr $CLI bundle deployment bind foo "$foo_id" &> LOG.confirm
+contains.py 'requires user confirmation' 'Plan: create resources.jobs.foo.permissions' < LOG.confirm > LOG.confirm.checked
+trace $CLI jobs get "$foo_id" | jq '{name: .settings.name}'
+
+title "Bind foo into the existing deployment: its config differs, so this is a bind-and-update that applies the config now"
+trace $CLI bundle deployment bind foo "$foo_id" --auto-approve
+
+title "Version 2 records only foo and its permissions (bar is untouched) and chains to version 1"
+trace print_requests.py --dms --method POST //versions --nostamp
+
+title "The bound job has already been updated"
+trace $CLI jobs get "$foo_id" | jq '{name: .settings.name}'
+
+title "An already-bound resource cannot be rebound"
+trace musterr $CLI bundle deployment bind foo "$foo_id" --auto-approve
+
+title "sc and foo are now managed; bar is still an unmanaged create"
+trace $CLI bundle plan
+
+title "A job can be the first bind of a new deployment"
+first_id=$($CLI jobs create --json '{"name": "first-bind-'$UNIQUE_NAME'"}' | jq -r '.job_id')
+add_repl "$first_id" FIRST_ID
+trace $CLI bundle deployment bind foo "$first_id" -t first_job --auto-approve
+trace print_requests.py --dms --method POST //versions --nostamp
+
+title "The job points to the deployment that recorded its adoption, at version 1"
+trace $CLI jobs get "$first_id" | jq '.settings.deployment | {kind, version_id}'
+deployment_id=$($CLI jobs get "$first_id" | jq -r '.settings.deployment.deployment_id')
+trace $CLI bundle-deployments list-resources "deployments/$deployment_id" -o json | jq -e --arg id "$first_id" 'any(.[]; .resource_key == "jobs.foo" and .resource_id == $id)'

acceptance/bundle/dms/record-op-fault/databricks.yml.tmpl

@@ -0,0 +1,9 @@
+bundle:
+  name: dms-record-op-fault-$UNIQUE_NAME
+experimental:
+  deployment_history: true
+
+resources:
+  jobs:
+    foo:
+      name: foo

acceptance/bundle/dms/record-op-fault/output.txt

@@ -0,0 +1,39 @@
+
+=== The service rejects the operation upload, so recording the bind fails
+>>> fault.py PATCH /api/2.0/bundle/* 500 0 1
+
+>>> musterr [CLI] bundle deployment bind foo [JOB_ID] --auto-approve
+Error: recording operation for resources.jobs.foo: Fault injected by test. (500 INJECTED)
+
+Endpoint: PATCH [DATABRICKS_URL]/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/1/operations/jobs.foo?update_mask=state%2Cerror_message%2Cresource_id%2Cstatus
+HTTP Status: 500 Internal Server Error
+API error_code: INJECTED
+API message: Fault injected by test.
+
+
+=== The version is still completed - with failure - rather than left holding its lease
+>>> print_requests.py --dms --method POST //versions
+{
+  "method": "POST",
+  "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions",
+  "q": {
+    "version_id": "1"
+  },
+  "body": {
+    "cli_version": "[CLI_VERSION]",
+    "operations": [
+      {
+        "action_type": "OPERATION_ACTION_TYPE_BIND_AND_UPDATE",
+        "resource_key": "jobs.foo"
+      }
+    ],
+    "version_type": "VERSION_TYPE_DEPLOY"
+  }
+}
+{
+  "method": "POST",
+  "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/1/complete",
+  "body": {
+    "completion_reason": "VERSION_COMPLETE_FAILURE"
+  }
+}

acceptance/bundle/dms/record-op-fault/script

@@ -0,0 +1,11 @@
+envsubst < databricks.yml.tmpl > databricks.yml
+
+job_id=$($CLI jobs create --json '{"name": "target-'$UNIQUE_NAME'"}' | jq -r '.job_id')
+add_repl "$job_id" JOB_ID
+
+title "The service rejects the operation upload, so recording the bind fails"
+trace fault.py "PATCH /api/2.0/bundle/*" 500 0 1
+trace musterr $CLI bundle deployment bind foo "$job_id" --auto-approve
+
+title "The version is still completed - with failure - rather than left holding its lease"
+trace print_requests.py --dms --method POST //versions

acceptance/bundle/dms/record-op-fault/test.toml

@@ -0,0 +1,3 @@
+# Local only: the case needs an injected failure (DMS must reject the operation upload once), and
+# fault.py registers its rule on the fake, which a cloud run does not have.
+Cloud = false

acceptance/bundle/dms/bind-unbind-unsupported/databricks.yml.tmpl → acceptance/bundle/dms/unbind-unsupported/databricks.yml.tmpl

@@ -1,5 +1,5 @@
 bundle:
-  name: dms-bind-unbind-$UNIQUE_NAME
+  name: dms-unbind-$UNIQUE_NAME
 experimental:
   deployment_history: true
 

acceptance/bundle/dms/unbind-unsupported/out.test.toml

@@ -0,0 +1,2 @@
+Cloud = false
+EnvMatrix.DMS = ["true"]

acceptance/bundle/dms/bind-unbind-unsupported/output.txt → acceptance/bundle/dms/unbind-unsupported/output.txt

@@ -1,12 +1,7 @@
 
-=== bind is refused while recording deployment history
->>> musterr [CLI] bundle deployment bind foo [JOB_ID]
-Error: bind is not supported for a bundle target that records deployment history
-
-
-=== Deploy so the bundle records state, then unbind is refused too
+=== Deploy so the bundle records state, then unbind is refused
 >>> [CLI] bundle deploy
-Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-bind-unbind-[UNIQUE_NAME]/default/files...
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-unbind-[UNIQUE_NAME]/default/files...
 Created jobs.foo
 Files: 2 uploaded, 0 deleted
 Resources: 1 created, 0 changed, 0 deleted, 0 unchanged

acceptance/bundle/dms/unbind-unsupported/script

@@ -0,0 +1,6 @@
+envsubst < databricks.yml.tmpl > databricks.yml
+
+# unbind refuses when it opens an existing recording state, so it needs a deployment first.
+title "Deploy so the bundle records state, then unbind is refused"
+trace $CLI bundle deploy
+trace musterr $CLI bundle deployment unbind foo

acceptance/bundle/dms/bind-unbind-unsupported/test.toml → acceptance/bundle/dms/unbind-unsupported/test.toml

@@ -1,4 +1,4 @@
 RecordRequests = false
-# bind/unbind refusal is a client-side gate on the recording config; it needs no real
+# The unbind refusal is a client-side gate on the recording config; it needs no real
 # workspace, so this stays local-only rather than inheriting the suite's cloud run.
 Cloud = false