Add support for breakglass for immutable folders

.nextchanges/bundles/immutable-breakglass.md

@@ -0,0 +1 @@
+* Immutable-folder deploys now refuse to deploy onto a snapshot that was modified outside the bundle (break glass); `bundle deploy --force` moves the deployment to a fresh snapshot. ([#6774](https://github.com/databricks/cli/pull/6774))

acceptance/bundle/deploy/immutable/basic/out.test.toml

@@ -0,0 +1,2 @@
+Cloud = true
+EnvMatrix.DMS = [""]

acceptance/bundle/deploy/immutable/output.txt → acceptance/bundle/deploy/immutable/basic/output.txt

@@ -15,7 +15,7 @@ Building python_artifact...
 [
   {
     "notebook_task": {
-      "notebook_path": "/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]/files/src/notebook"
+      "notebook_path": "/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]-[UUID]/snapshot/files/src/notebook"
     },
     "task_key": "notebook_task"
   },
@@ -30,7 +30,7 @@ Building python_artifact...
   {
     "environment_key": "env",
     "spark_python_task": {
-      "python_file": "/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]/files/src/main.py"
+      "python_file": "/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]-[UUID]/snapshot/files/src/main.py"
     },
     "task_key": "spark_python_task"
   }
@@ -42,7 +42,7 @@ Building python_artifact...
     "environment_key": "env",
     "spec": {
       "dependencies": [
-        "/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]/artifacts/.internal/immutable-0.0.1-py3-none-any.whl"
+        "/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]-[UUID]/snapshot/artifacts/.internal/immutable-0.0.1-py3-none-any.whl"
       ],
       "environment_version": "4"
     }
@@ -54,16 +54,21 @@ Building python_artifact...
   "action": "create",
   "new_state": {
     "value": {
-      "relative_path": "[SNAPSHOT_HASH]/[SNAPSHOT_HASH]",
-      "full_path": "/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]",
+      "relative_path": "[SNAPSHOT_HASH]/[SNAPSHOT_HASH]-[UUID]",
+      "full_path": "/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]-[UUID]/snapshot",
       "bundle_id": "[SNAPSHOT_HASH]",
       "acl": [
         {
           "user_name": "[USERNAME]",
           "permission_level": "CAN_READ"
         }
       ],
-      "zip_path": "[TEST_TMP_DIR]/.databricks/bundle/default/snapshots/[SNAPSHOT_HASH].zip"
+      "can_manage": [
+        {
+          "service_principal_name": "[USERNAME]"
+        }
+      ],
+      "zip_path": "[TEST_TMP_DIR]/.databricks/bundle/default/snapshots/[SNAPSHOT_HASH]-[UUID].zip"
     }
   }
 }
@@ -76,14 +81,14 @@ Files: 0 uploaded, 0 deleted
 Resources: 2 created, 0 changed, 0 deleted, 0 unchanged
 
 >>> jq .settings.tasks job.json
-"/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]/files/src/main.py"
+"/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]-[UUID]/snapshot/files/src/main.py"
 
 >>> jq .settings.tasks job.json
-"/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]/files/src/notebook"
+"/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]-[UUID]/snapshot/files/src/notebook"
 
 >>> jq .settings.environments[0].spec.dependencies job.json
 [
-  "/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]/artifacts/.internal/immutable-0.0.1-py3-none-any.whl"
+  "/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]-[UUID]/snapshot/artifacts/.internal/immutable-0.0.1-py3-none-any.whl"
 ]
 
 >>> title Check snapshot plan after deploy
@@ -97,21 +102,26 @@ Building python_artifact...
   "action": "recreate",
   "new_state": {
     "value": {
-      "relative_path": "[SNAPSHOT_HASH]/[SNAPSHOT_HASH]",
-      "full_path": "/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]",
+      "relative_path": "[SNAPSHOT_HASH]/[SNAPSHOT_HASH]-[UUID]",
+      "full_path": "/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]-[UUID]/snapshot",
       "bundle_id": "[SNAPSHOT_HASH]",
       "acl": [
         {
           "user_name": "[USERNAME]",
           "permission_level": "CAN_READ"
         }
       ],
-      "zip_path": "[TEST_TMP_DIR]/.databricks/bundle/default/snapshots/[SNAPSHOT_HASH].zip"
+      "can_manage": [
+        {
+          "service_principal_name": "[USERNAME]"
+        }
+      ],
+      "zip_path": "[TEST_TMP_DIR]/.databricks/bundle/default/snapshots/[SNAPSHOT_HASH]-[UUID].zip"
     }
   },
   "remote_state": {
-    "relative_path": "[SNAPSHOT_HASH]/[SNAPSHOT_HASH]",
-    "full_path": "/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]"
+    "relative_path": "[SNAPSHOT_HASH]/[SNAPSHOT_HASH]-[UUID]",
+    "full_path": "/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]-[UUID]/snapshot"
   },
   "changes": {
     "acl": {
@@ -138,25 +148,39 @@ Building python_artifact...
       "new": "[SNAPSHOT_HASH]",
       "remote": ""
     },
+    "can_manage": {
+      "action": "skip",
+      "reason": "missing_in_remote",
+      "old": [
+        {
+          "service_principal_name": "[USERNAME]"
+        }
+      ],
+      "new": [
+        {
+          "service_principal_name": "[USERNAME]"
+        }
+      ]
+    },
     "full_path": {
       "action": "recreate",
       "reason": "immutable",
-      "old": "/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]",
-      "new": "/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]",
-      "remote": "/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]"
+      "old": "/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]-[UUID]/snapshot",
+      "new": "/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]-[UUID]/snapshot",
+      "remote": "/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]-[UUID]/snapshot"
     },
     "relative_path": {
       "action": "recreate",
       "reason": "immutable",
-      "old": "[SNAPSHOT_HASH]/[SNAPSHOT_HASH]",
-      "new": "[SNAPSHOT_HASH]/[SNAPSHOT_HASH]",
-      "remote": "[SNAPSHOT_HASH]/[SNAPSHOT_HASH]"
+      "old": "[SNAPSHOT_HASH]/[SNAPSHOT_HASH]-[UUID]",
+      "new": "[SNAPSHOT_HASH]/[SNAPSHOT_HASH]-[UUID]",
+      "remote": "[SNAPSHOT_HASH]/[SNAPSHOT_HASH]-[UUID]"
     },
     "zip_path": {
       "action": "recreate",
       "reason": "immutable",
-      "old": "[TEST_TMP_DIR]/.databricks/bundle/default/snapshots/[SNAPSHOT_HASH].zip",
-      "new": "[TEST_TMP_DIR]/.databricks/bundle/default/snapshots/[SNAPSHOT_HASH].zip",
+      "old": "[TEST_TMP_DIR]/.databricks/bundle/default/snapshots/[SNAPSHOT_HASH]-[UUID].zip",
+      "new": "[TEST_TMP_DIR]/.databricks/bundle/default/snapshots/[SNAPSHOT_HASH]-[UUID].zip",
       "remote": ""
     }
   }

acceptance/bundle/deploy/immutable/basic/test.toml

@@ -0,0 +1,7 @@
+Ignore = [
+    "dist",
+    "build",
+    ".venv",
+    "*.pyc",
+    "src/*.egg-info",
+]

acceptance/bundle/deploy/immutable/breakglass/databricks.yml.tmpl

@@ -0,0 +1,19 @@
+bundle:
+  name: test-bundle-immutable-breakglass-$UNIQUE_NAME
+
+experimental:
+  immutable_folder: true
+
+resources:
+  jobs:
+    my_job:
+      name: my job
+      tasks:
+        - task_key: spark_python_task
+          spark_python_task:
+            python_file: ./src/main.py
+          environment_key: env
+      environments:
+        - environment_key: env
+          spec:
+            environment_version: "4"

acceptance/bundle/deploy/immutable/breakglass/out.test.toml

@@ -0,0 +1,2 @@
+Cloud = true
+EnvMatrix.DMS = [""]

acceptance/bundle/deploy/immutable/breakglass/output.txt

@@ -0,0 +1,42 @@
+
+=== Deploy the immutable snapshot
+>>> [CLI] bundle deploy
+Created internal_immutable_snapshots.immutable
+Created jobs.my_job
+Files: 0 uploaded, 0 deleted
+Resources: 2 created, 0 changed, 0 deleted, 0 unchanged
+First snapshot path: /Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]-[UUID]/snapshot/files/src/main.py
+
+=== Break the glass out of band
+>>> [CLI] api post /api/2.0/snapshots:breakGlass --json {"snapshot_content_path": "/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]-[UUID]/snapshot"}
+true
+
+=== Deploy without --force fails
+>>> musterr [CLI] bundle deploy
+Error: the last deployment was modified in the workspace. Changes might be overwritten by this deployment. To confirm it, use the --force flag
+
+Files: 0 uploaded, 0 deleted
+
+=== Deploy with --force moves to a new snapshot
+>>> [CLI] bundle deploy --force
+Recreated internal_immutable_snapshots.immutable
+Updated jobs.my_job
+Files: 0 uploaded, 0 deleted
+Resources: 1 created, 1 changed, 1 deleted, 0 unchanged
+Second snapshot path: /Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]-[UUID]/snapshot/files/src/main.py
+OK: --force created a new snapshot path
+
+=== Redeploy reuses the new snapshot
+>>> [CLI] bundle deploy
+Files: 0 uploaded, 0 deleted
+Resources: 0 created, 0 changed, 0 deleted, 2 unchanged
+Third snapshot path: /Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]-[UUID]/snapshot/files/src/main.py
+OK: new snapshot reused on redeploy
+
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.jobs.my_job
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-immutable-breakglass-[UNIQUE_NAME]/default
+
+Destroy: 1 deleted

acceptance/bundle/deploy/immutable/breakglass/script

@@ -0,0 +1,42 @@
+envsubst < databricks.yml.tmpl > databricks.yml
+
+cleanup() {
+    trace $CLI bundle destroy --auto-approve
+}
+trap cleanup EXIT
+
+title "Deploy the immutable snapshot"
+trace $CLI bundle deploy
+
+JOB_ID=$($CLI bundle summary -o json | jq -r '.resources.jobs.my_job.id')
+SNAPSHOT_PATH_1=$($CLI jobs get "$JOB_ID" | jq -r '.settings.tasks[0].spark_python_task.python_file')
+echo "First snapshot path: $SNAPSHOT_PATH_1"
+
+title "Break the glass out of band"
+# Breaking the glass grants write access to the snapshot content path (the folder holding files/
+# and artifacts/) and marks the snapshot modified (dirty) for good.
+CONTENT_PATH="${SNAPSHOT_PATH_1%/files/src/main.py}"
+trace $CLI api post /api/2.0/snapshots:breakGlass --json "{\"snapshot_content_path\": \"$CONTENT_PATH\"}" | jq .status.dirty
+
+title "Deploy without --force fails"
+trace musterr $CLI bundle deploy
+
+title "Deploy with --force moves to a new snapshot"
+trace $CLI bundle deploy --force
+SNAPSHOT_PATH_2=$($CLI jobs get "$JOB_ID" | jq -r '.settings.tasks[0].spark_python_task.python_file')
+echo "Second snapshot path: $SNAPSHOT_PATH_2"
+if [ "$SNAPSHOT_PATH_1" = "$SNAPSHOT_PATH_2" ]; then
+    echo "ERROR: snapshot path did not change after --force"
+    exit 1
+fi
+echo "OK: --force created a new snapshot path"
+
+title "Redeploy reuses the new snapshot"
+trace $CLI bundle deploy
+SNAPSHOT_PATH_3=$($CLI jobs get "$JOB_ID" | jq -r '.settings.tasks[0].spark_python_task.python_file')
+echo "Third snapshot path: $SNAPSHOT_PATH_3"
+if [ "$SNAPSHOT_PATH_2" != "$SNAPSHOT_PATH_3" ]; then
+    echo "ERROR: new snapshot path was not reused on redeploy"
+    exit 1
+fi
+echo "OK: new snapshot reused on redeploy"

acceptance/bundle/deploy/immutable/breakglass/test.toml

@@ -0,0 +1,4 @@
+# The test breaks the glass with "api post /api/2.0/snapshots:breakGlass". On Windows the script
+# runs under MSYS2, which rewrites leading-slash arguments to Windows paths before the CLI sees
+# them. Setting this environment variable disables that conversion.
+Env.MSYS_NO_PATHCONV = "1"

acceptance/bundle/deploy/immutable/can-manage/databricks.yml.tmpl

@@ -0,0 +1,23 @@
+bundle:
+  name: test-bundle-immutable-can-manage-$UNIQUE_NAME
+
+experimental:
+  immutable_folder: true
+
+permissions:
+  - level: CAN_MANAGE
+    user_name: $TEST_USER_EMAIL
+
+resources:
+  jobs:
+    my_job:
+      name: my job
+      tasks:
+        - task_key: spark_python_task
+          spark_python_task:
+            python_file: ./src/main.py
+          environment_key: env
+      environments:
+        - environment_key: env
+          spec:
+            environment_version: "4"

acceptance/bundle/deploy/immutable/can-manage/out.test.toml

@@ -0,0 +1,3 @@
+Cloud = true
+CloudEnvs.gcp = false
+EnvMatrix.DMS = [""]