From 211ba6c7c7ab05face8b931e55c3daf15bc02b6c Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Mon, 21 Sep 2026 16:37:42 +0200 Subject: [PATCH 01/16] Add support for breakglass for immutable folders --- .nextchanges/bundles/immutable-breakglass.md | 1 + .../immutable-breakglass/databricks.yml.tmpl | 19 +++ .../deploy/immutable-breakglass/out.test.toml | 3 + .../deploy/immutable-breakglass/output.txt | 40 +++++ .../bundle/deploy/immutable-breakglass/script | 42 +++++ .../deploy/immutable-breakglass/src/main.py | 1 + .../deploy/immutable-breakglass/test.toml | 14 ++ .../deploy/immutable-no-artifacts/output.txt | 16 +- .../immutable-permissions-change/output.txt | 6 +- acceptance/bundle/deploy/immutable/output.txt | 24 +-- .../resources/apps/immutable/output.txt | 2 +- bundle/config/experimental.go | 4 +- bundle/config/resources/snapshot.go | 27 +++- bundle/deploy/snapshot/upload.go | 72 +++++++++ bundle/deploy/snapshot/upload_warning_test.go | 33 ++++ bundle/direct/dresources/all_test.go | 2 +- bundle/direct/dresources/snapshot.go | 11 +- bundle/direct/dresources/type_test.go | 2 + libs/snapshot/client.go | 145 ++++++++++++++++-- libs/testserver/fake_workspace.go | 35 +++++ libs/testserver/handlers.go | 50 ++++-- 21 files changed, 489 insertions(+), 60 deletions(-) create mode 100644 .nextchanges/bundles/immutable-breakglass.md create mode 100644 acceptance/bundle/deploy/immutable-breakglass/databricks.yml.tmpl create mode 100644 acceptance/bundle/deploy/immutable-breakglass/out.test.toml create mode 100644 acceptance/bundle/deploy/immutable-breakglass/output.txt create mode 100644 acceptance/bundle/deploy/immutable-breakglass/script create mode 100644 acceptance/bundle/deploy/immutable-breakglass/src/main.py create mode 100644 acceptance/bundle/deploy/immutable-breakglass/test.toml diff --git a/.nextchanges/bundles/immutable-breakglass.md b/.nextchanges/bundles/immutable-breakglass.md new file mode 100644 index 00000000000..22d9a48288e --- /dev/null +++ b/.nextchanges/bundles/immutable-breakglass.md @@ -0,0 +1 @@ +* Immutable-folder deploys now detect a break-glassed snapshot and refuse to redeploy over it; `bundle deploy --force` recovers by creating a new snapshot at a suffixed path. diff --git a/acceptance/bundle/deploy/immutable-breakglass/databricks.yml.tmpl b/acceptance/bundle/deploy/immutable-breakglass/databricks.yml.tmpl new file mode 100644 index 00000000000..671725dcf5c --- /dev/null +++ b/acceptance/bundle/deploy/immutable-breakglass/databricks.yml.tmpl @@ -0,0 +1,19 @@ +bundle: + name: test-bundle-immutable-breakglass-$UNIQUE_NAME + +experimental: + immutable_folder: true + +resources: + jobs: + my_job: + name: my job + tasks: + - task_key: spark_python_task + spark_python_task: + python_file: ./src/main.py + environment_key: env + environments: + - environment_key: env + spec: + environment_version: "4" diff --git a/acceptance/bundle/deploy/immutable-breakglass/out.test.toml b/acceptance/bundle/deploy/immutable-breakglass/out.test.toml new file mode 100644 index 00000000000..59b56a2037c --- /dev/null +++ b/acceptance/bundle/deploy/immutable-breakglass/out.test.toml @@ -0,0 +1,3 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/deploy/immutable-breakglass/output.txt b/acceptance/bundle/deploy/immutable-breakglass/output.txt new file mode 100644 index 00000000000..3379c10d1f0 --- /dev/null +++ b/acceptance/bundle/deploy/immutable-breakglass/output.txt @@ -0,0 +1,40 @@ + +=== Deploy the immutable snapshot +>>> [CLI] bundle deploy +Created internal_immutable_snapshots.immutable +Created jobs.my_job +Files: 0 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged +First snapshot path: /Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]/snapshot/files/src/main.py + +=== Break the glass out of band +>>> [CLI] workspace mkdirs /Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]/snapshot/tampered + +=== Deploy without --force fails +Error: the previously deployed immutable snapshot at /Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]/snapshot was modified out of band (break-glass); re-run 'bundle deploy --force' to deploy a new snapshot + +Files: 0 uploaded, 0 deleted + +=== Deploy with --force recovers to a new snapshot +>>> [CLI] bundle deploy --force +Recreated internal_immutable_snapshots.immutable +Updated jobs.my_job +Files: 0 uploaded, 0 deleted +Resources: 1 created, 1 changed, 1 deleted, 0 unchanged +Second snapshot path: /Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]-1/snapshot/files/src/main.py +OK: recovery created a new snapshot path + +=== Redeploy reuses the recovered snapshot +>>> [CLI] bundle deploy +Files: 0 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 2 unchanged +Third snapshot path: /Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]-1/snapshot/files/src/main.py +OK: recovered snapshot reused on redeploy + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.my_job + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-immutable-breakglass-[UNIQUE_NAME]/default + +Destroy: 1 deleted diff --git a/acceptance/bundle/deploy/immutable-breakglass/script b/acceptance/bundle/deploy/immutable-breakglass/script new file mode 100644 index 00000000000..32ba1240ea6 --- /dev/null +++ b/acceptance/bundle/deploy/immutable-breakglass/script @@ -0,0 +1,42 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve +} +trap cleanup EXIT + +title "Deploy the immutable snapshot" +trace $CLI bundle deploy + +JOB_ID=$($CLI bundle summary -o json | jq -r '.resources.jobs.my_job.id') +SNAPSHOT_PATH_1=$($CLI jobs get "$JOB_ID" | jq -r '.settings.tasks[0].spark_python_task.python_file') +echo "First snapshot path: $SNAPSHOT_PATH_1" + +title "Break the glass out of band" +# The snapshot content path holds files/ and artifacts/; writing into it out of band +# marks the immutable snapshot as broken. +CONTENT_PATH="${SNAPSHOT_PATH_1%/files/src/main.py}" +trace $CLI workspace mkdirs "$CONTENT_PATH/tampered" + +title "Deploy without --force fails\n" +musterr $CLI bundle deploy + +title "Deploy with --force recovers to a new snapshot" +trace $CLI bundle deploy --force +SNAPSHOT_PATH_2=$($CLI jobs get "$JOB_ID" | jq -r '.settings.tasks[0].spark_python_task.python_file') +echo "Second snapshot path: $SNAPSHOT_PATH_2" +if [ "$SNAPSHOT_PATH_1" = "$SNAPSHOT_PATH_2" ]; then + echo "ERROR: snapshot path did not change after break-glass recovery" + exit 1 +fi +echo "OK: recovery created a new snapshot path" + +title "Redeploy reuses the recovered snapshot" +trace $CLI bundle deploy +SNAPSHOT_PATH_3=$($CLI jobs get "$JOB_ID" | jq -r '.settings.tasks[0].spark_python_task.python_file') +echo "Third snapshot path: $SNAPSHOT_PATH_3" +if [ "$SNAPSHOT_PATH_2" != "$SNAPSHOT_PATH_3" ]; then + echo "ERROR: recovered snapshot path was not reused on redeploy" + exit 1 +fi +echo "OK: recovered snapshot reused on redeploy" diff --git a/acceptance/bundle/deploy/immutable-breakglass/src/main.py b/acceptance/bundle/deploy/immutable-breakglass/src/main.py new file mode 100644 index 00000000000..6c285f7e2f5 --- /dev/null +++ b/acceptance/bundle/deploy/immutable-breakglass/src/main.py @@ -0,0 +1 @@ +print("Hello from Spark Python Task!") diff --git a/acceptance/bundle/deploy/immutable-breakglass/test.toml b/acceptance/bundle/deploy/immutable-breakglass/test.toml new file mode 100644 index 00000000000..88b225ea71f --- /dev/null +++ b/acceptance/bundle/deploy/immutable-breakglass/test.toml @@ -0,0 +1,14 @@ +Cloud = false # Temporary disable cloud tests until the API is fully available + +# immutable_folder only works with the direct engine. +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] + +Ignore = [ + "databricks.yml", + ".databricks", +] + +[[Repls]] +# Replace snapshot hash with SNAPSHOT_HASH +Old = "[0-9a-f]{64}" +New = "[SNAPSHOT_HASH]" diff --git a/acceptance/bundle/deploy/immutable-no-artifacts/output.txt b/acceptance/bundle/deploy/immutable-no-artifacts/output.txt index 2fd5d2c3ef1..023c77bbe3c 100644 --- a/acceptance/bundle/deploy/immutable-no-artifacts/output.txt +++ b/acceptance/bundle/deploy/immutable-no-artifacts/output.txt @@ -15,16 +15,16 @@ Validation OK! { "notebook_task": { "base_parameters": { - "path": "/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]/files/some_path" + "path": "/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]/snapshot/files/some_path" }, - "notebook_path": "/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]/files/src/notebook" + "notebook_path": "/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]/snapshot/files/src/notebook" }, "task_key": "notebook_task" }, { "environment_key": "env", "spark_python_task": { - "python_file": "/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]/files/src/main.py" + "python_file": "/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]/snapshot/files/src/main.py" }, "task_key": "spark_python_task" } @@ -36,7 +36,7 @@ Validation OK! "new_state": { "value": { "relative_path": "[SNAPSHOT_HASH]/[SNAPSHOT_HASH]", - "full_path": "/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]", + "full_path": "/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]/snapshot", "bundle_id": "[SNAPSHOT_HASH]", "acl": [ { @@ -56,13 +56,13 @@ Files: 0 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged >>> jq .settings.tasks job.json -"/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]/files/src/main.py" +"/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]/snapshot/files/src/main.py" >>> jq .settings.tasks job.json -"/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]/files/src/notebook" +"/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]/snapshot/files/src/notebook" >>> jq .settings.tasks job.json -"/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]/files/some_path" +"/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]/snapshot/files/some_path" >>> title Check snapshot plan after deploy: @@ -74,7 +74,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "action": "skip", "remote_state": { "relative_path": "[SNAPSHOT_HASH]/[SNAPSHOT_HASH]", - "full_path": "/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]" + "full_path": "/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]/snapshot" }, "changes": { "acl": { diff --git a/acceptance/bundle/deploy/immutable-permissions-change/output.txt b/acceptance/bundle/deploy/immutable-permissions-change/output.txt index 7e45089b269..740dc2a0f33 100644 --- a/acceptance/bundle/deploy/immutable-permissions-change/output.txt +++ b/acceptance/bundle/deploy/immutable-permissions-change/output.txt @@ -5,7 +5,7 @@ Created internal_immutable_snapshots.immutable Created jobs.my_job Files: 0 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged -First snapshot path: /Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]/files/src/main.py +First snapshot path: /Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]/snapshot/files/src/main.py === Add permissions and redeploy >>> [CLI] bundle deploy @@ -21,12 +21,12 @@ Consider using a adding a top-level permissions section such as the following: See https://docs.databricks.com/dev-tools/bundles/permissions.html to learn more about permission configuration. in databricks.yml:22:3 +Created jobs.my_job.permissions Recreated internal_immutable_snapshots.immutable Updated jobs.my_job -Created jobs.my_job.permissions Files: 0 uploaded, 0 deleted Resources: 2 created, 1 changed, 1 deleted, 0 unchanged -Second snapshot path: /Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]/files/src/main.py +Second snapshot path: /Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]/snapshot/files/src/main.py === Verify snapshot path changed after permissions change OK: snapshot path changed as expected diff --git a/acceptance/bundle/deploy/immutable/output.txt b/acceptance/bundle/deploy/immutable/output.txt index 56de6c28d57..26c9339e605 100644 --- a/acceptance/bundle/deploy/immutable/output.txt +++ b/acceptance/bundle/deploy/immutable/output.txt @@ -15,7 +15,7 @@ Building python_artifact... [ { "notebook_task": { - "notebook_path": "/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]/files/src/notebook" + "notebook_path": "/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]/snapshot/files/src/notebook" }, "task_key": "notebook_task" }, @@ -30,7 +30,7 @@ Building python_artifact... { "environment_key": "env", "spark_python_task": { - "python_file": "/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]/files/src/main.py" + "python_file": "/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]/snapshot/files/src/main.py" }, "task_key": "spark_python_task" } @@ -42,7 +42,7 @@ Building python_artifact... "environment_key": "env", "spec": { "dependencies": [ - "/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]/artifacts/.internal/immutable-0.0.1-py3-none-any.whl" + "/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]/snapshot/artifacts/.internal/immutable-0.0.1-py3-none-any.whl" ], "environment_version": "4" } @@ -55,7 +55,7 @@ Building python_artifact... "new_state": { "value": { "relative_path": "[SNAPSHOT_HASH]/[SNAPSHOT_HASH]", - "full_path": "/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]", + "full_path": "/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]/snapshot", "bundle_id": "[SNAPSHOT_HASH]", "acl": [ { @@ -76,14 +76,14 @@ Files: 0 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged >>> jq .settings.tasks job.json -"/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]/files/src/main.py" +"/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]/snapshot/files/src/main.py" >>> jq .settings.tasks job.json -"/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]/files/src/notebook" +"/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]/snapshot/files/src/notebook" >>> jq .settings.environments[0].spec.dependencies job.json [ - "/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]/artifacts/.internal/immutable-0.0.1-py3-none-any.whl" + "/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]/snapshot/artifacts/.internal/immutable-0.0.1-py3-none-any.whl" ] >>> title Check snapshot plan after deploy @@ -98,7 +98,7 @@ Building python_artifact... "new_state": { "value": { "relative_path": "[SNAPSHOT_HASH]/[SNAPSHOT_HASH]", - "full_path": "/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]", + "full_path": "/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]/snapshot", "bundle_id": "[SNAPSHOT_HASH]", "acl": [ { @@ -111,7 +111,7 @@ Building python_artifact... }, "remote_state": { "relative_path": "[SNAPSHOT_HASH]/[SNAPSHOT_HASH]", - "full_path": "/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]" + "full_path": "/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]/snapshot" }, "changes": { "acl": { @@ -141,9 +141,9 @@ Building python_artifact... "full_path": { "action": "recreate", "reason": "immutable", - "old": "/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]", - "new": "/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]", - "remote": "/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]" + "old": "/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]/snapshot", + "new": "/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]/snapshot", + "remote": "/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]/snapshot" }, "relative_path": { "action": "recreate", diff --git a/acceptance/bundle/resources/apps/immutable/output.txt b/acceptance/bundle/resources/apps/immutable/output.txt index c411677bca0..14b931e25f9 100644 --- a/acceptance/bundle/resources/apps/immutable/output.txt +++ b/acceptance/bundle/resources/apps/immutable/output.txt @@ -37,6 +37,6 @@ You can access the app at my-immutable-app-123.cloud.databricksapps.com "path": "/api/2.0/apps/my-immutable-app/deployments", "body": { "mode": "SNAPSHOT", - "source_code_path": "/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]/files/app" + "source_code_path": "/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]/snapshot/files/app" } } diff --git a/bundle/config/experimental.go b/bundle/config/experimental.go index babf44e2b3b..126b3aa6ac5 100644 --- a/bundle/config/experimental.go +++ b/bundle/config/experimental.go @@ -5,8 +5,8 @@ type Experimental struct { // ImmutableFolder specifies that bundle files and artifacts are uploaded as a // single immutable snapshot rather than being synced individually. When true, - // the deployment calls /api/2.0/repos/snapshots with a zip of all files and sets - // workspace.file_path and workspace.artifact_path to the returned content-addressed + // the deployment calls /api/2.0/snapshots with a zip of all files and sets + // workspace.file_path and workspace.artifact_path to the returned snapshot content // path. Only supported with the direct deployment engine. ImmutableFolder bool `json:"immutable_folder,omitempty"` diff --git a/bundle/config/resources/snapshot.go b/bundle/config/resources/snapshot.go index 95d98c89271..ca9476010cb 100644 --- a/bundle/config/resources/snapshot.go +++ b/bundle/config/resources/snapshot.go @@ -2,6 +2,7 @@ package resources import ( "context" + "fmt" "net/url" "path" "strings" @@ -47,6 +48,9 @@ type Snapshot struct { // ACL is the access control list applied to the uploaded snapshot, granting // CAN_READ to the deploying user and to every principal in bundle.permissions. ACL []snapshot.ACLEntry `json:"acl"` + // CanManage lists the principals allowed to break the glass on the snapshot, derived + // from top-level bundle.permissions entries with level CAN_MANAGE. + CanManage []snapshot.ManagePrincipal `json:"can_manage,omitempty"` // ZipPath is the local path of the bundle zip staged by the deploy pipeline. // The file is named ".zip", so its base name is the content hash used // as the last component of RelativePath. Storing the path (not the bytes) keeps @@ -59,22 +63,35 @@ type Snapshot struct { // Example: "/Workspace/Users//.snapshots". RemoteRoot string `json:"remote_root"` + // Generation is the break-glass generation. It is 0 for the initial snapshot and + // incremented each time "deploy --force" recovers from a broken (break-glassed) + // snapshot, appending a "-" suffix to the content hash so the recovered + // snapshot gets a distinct path. It is carried forward in state across deploys, so a + // recovered path keeps being reused until it too is broken. + Generation int `json:"generation,omitempty"` + Lifecycle Lifecycle `json:"-"` } // RelativePath is the snapshot's location under RemoteRoot: "/". // It is the resource ID; the last component is the zip content hash (the ZipPath base -// name), which makes the pre-computed path match the uploaded content. -// Example: "1a2b3c4d-.../e3b0c4...". +// name), which makes the pre-computed path match the uploaded content. A non-zero +// Generation appends a "-" suffix to the hash so a break-glass recovery +// uploads to a distinct path (e.g. "1a2b3c4d-.../e3b0c4...-1"). func (s *Snapshot) RelativePath() string { hash := strings.TrimSuffix(path.Base(s.ZipPath), ".zip") + if s.Generation > 0 { + hash = fmt.Sprintf("%s-%d", hash, s.Generation) + } return path.Join(s.BundleID, hash) } -// FullPath is the absolute workspace path of the snapshot: RemoteRoot + "/" + RelativePath. -// Example: "/Workspace/Users//.snapshots/1a2b3c4d-.../e3b0c4...". +// FullPath is the absolute workspace content path of the snapshot: +// RemoteRoot + "/" + RelativePath + "/" + ContentSubdir. The API stores the content under a +// fixed "snapshot" subfolder, and downstream resources / the inspect check reference it. +// Example: "/Workspace/Users//.snapshots/1a2b3c4d-.../e3b0c4.../snapshot". func (s *Snapshot) FullPath() string { - return path.Join(s.RemoteRoot, s.RelativePath()) + return path.Join(s.RemoteRoot, s.RelativePath(), snapshot.ContentSubdir) } func (s *Snapshot) Exists(ctx context.Context, w *databricks.WorkspaceClient, name string) (bool, error) { diff --git a/bundle/deploy/snapshot/upload.go b/bundle/deploy/snapshot/upload.go index df56e8fda5e..d69f73348ab 100644 --- a/bundle/deploy/snapshot/upload.go +++ b/bundle/deploy/snapshot/upload.go @@ -4,6 +4,7 @@ import ( "context" "crypto/sha256" "encoding/hex" + "encoding/json" "fmt" "os" "path/filepath" @@ -69,6 +70,13 @@ func (m *snapshotUpload) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagn return nil } + // Detect a break-glassed previous snapshot before doing any work, so a normal deploy + // over a broken snapshot fails fast and --force recovery gets the right generation. + generation, err := breakGlassGeneration(ctx, b, uploader) + if err != nil { + return diag.FromErr(err) + } + zipContent, fileCount, err := BundleZip(ctx, b) if err != nil { return diag.FromErr(fmt.Errorf("failed to build snapshot zip: %w", err)) @@ -91,8 +99,10 @@ func (m *snapshotUpload) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagn b.Config.Resources.Snapshots[resources.SnapshotResourceKey] = &resources.Snapshot{ BundleID: BundleID(b), ACL: BuildACL(b), + CanManage: BuildCanManage(b), RemoteRoot: remoteRoot, ZipPath: filepath.ToSlash(zipPath), + Generation: generation, } var diags diag.Diagnostics @@ -134,3 +144,65 @@ func BuildACL(b *bundle.Bundle) []snapshot.ACLEntry { } return acl } + +// BuildCanManage constructs can_manage_principals for the snapshot upload: every principal +// granted CAN_MANAGE in the top-level permissions section may break the glass on the snapshot. +func BuildCanManage(b *bundle.Bundle) []snapshot.ManagePrincipal { + var canManage []snapshot.ManagePrincipal + for _, p := range b.Config.Permissions { + if p.Level != "CAN_MANAGE" { + continue + } + canManage = append(canManage, snapshot.ManagePrincipal{ + UserName: p.UserName, + GroupName: p.GroupName, + ServicePrincipalName: p.ServicePrincipalName, + }) + } + return canManage +} + +// breakGlassGeneration reads the previously deployed snapshot from state and asks the backend +// whether it was modified out of band ("break-glassed"). It returns the generation to deploy: +// - the previous generation when the snapshot is intact, so a recovered path keeps being reused; +// - the previous generation + 1 when the snapshot is broken and --force was given (recovery); +// - an error when the snapshot is broken and --force was not given. +// +// On the first deploy (no prior snapshot) it returns 0. +func breakGlassGeneration(ctx context.Context, b *bundle.Bundle, uploader *snapshot.SnapshotClient) (int, error) { + // The deploy/plan pipeline opens the state DB for read before this runs, but callers that + // exercise PlanUpload in isolation (unit tests) may not. No open state means no prior + // snapshot to check, which is the first-deploy case: generation 0. + if !b.DeploymentBundle.StateDB.IsOpen() { + return 0, nil + } + + entry, ok := b.DeploymentBundle.StateDB.GetResourceEntry(resources.SnapshotKey) + if !ok || len(entry.State) == 0 { + return 0, nil + } + + // Only the content path and generation are needed; unmarshal a subset of the state. + var prev struct { + FullPath string `json:"full_path"` + Generation int `json:"generation"` + } + if err := json.Unmarshal(entry.State, &prev); err != nil { + return 0, fmt.Errorf("reading previous snapshot state: %w", err) + } + if prev.FullPath == "" { + return 0, nil + } + + status, err := uploader.InspectSnapshot(ctx, prev.FullPath) + if err != nil { + return 0, err + } + if !status.Dirty { + return prev.Generation, nil + } + if !b.Config.Bundle.Force { + return 0, fmt.Errorf("the previously deployed immutable snapshot at %s was modified out of band (break-glass); re-run 'bundle deploy --force' to deploy a new snapshot", prev.FullPath) + } + return prev.Generation + 1, nil +} diff --git a/bundle/deploy/snapshot/upload_warning_test.go b/bundle/deploy/snapshot/upload_warning_test.go index 8020405b211..79ac4b070e4 100644 --- a/bundle/deploy/snapshot/upload_warning_test.go +++ b/bundle/deploy/snapshot/upload_warning_test.go @@ -11,9 +11,11 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/logdiag" + "github.com/databricks/cli/libs/snapshot" "github.com/databricks/cli/libs/testserver" "github.com/databricks/cli/libs/vfs" "github.com/databricks/databricks-sdk-go" @@ -90,6 +92,37 @@ func TestUploadNoWarningBelowFileLimit(t *testing.T) { assert.True(t, diags.HasError() == false && len(diags) == 0, "expected no diagnostics") } +func TestBuildCanManage(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Permissions: []resources.Permission{ + {Level: "CAN_MANAGE", UserName: "manager@example.test"}, + {Level: "CAN_VIEW", UserName: "viewer@example.test"}, + {Level: "CAN_MANAGE", GroupName: "admins"}, + {Level: "CAN_MANAGE", ServicePrincipalName: "sp-1"}, + }, + }, + } + + // Only CAN_MANAGE principals become break-glass managers. + assert.Equal(t, []snapshot.ManagePrincipal{ + {UserName: "manager@example.test"}, + {GroupName: "admins"}, + {ServicePrincipalName: "sp-1"}, + }, BuildCanManage(b)) +} + +func TestBuildCanManageWithoutManagers(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Permissions: []resources.Permission{ + {Level: "CAN_VIEW", UserName: "viewer@example.test"}, + }, + }, + } + assert.Empty(t, BuildCanManage(b)) +} + func TestUploadReusesStagedZipWhenNotClean(t *testing.T) { // clean=false is the deploy --plan path: the plan already carries the resource and // its staged zip, so Apply must not rebuild, stage, or register anything (and must diff --git a/bundle/direct/dresources/all_test.go b/bundle/direct/dresources/all_test.go index 96bc471399f..66132a6e0c8 100644 --- a/bundle/direct/dresources/all_test.go +++ b/bundle/direct/dresources/all_test.go @@ -1112,7 +1112,7 @@ func testCRUD(t *testing.T, group string, adapter *Adapter, client *databricks.W require.NoError(t, os.WriteFile(zipPath, content, 0o600)) ss.ZipPath = filepath.ToSlash(zipPath) ss.RelativePath = ss.BundleID + "/" + hash - ss.FullPath = snap.RemoteRoot + "/" + ss.RelativePath + ss.FullPath = snap.RemoteRoot + "/" + ss.RelativePath + "/" + snapshot.ContentSubdir } ctx := t.Context() diff --git a/bundle/direct/dresources/snapshot.go b/bundle/direct/dresources/snapshot.go index 5c59e5110c8..1b97e99d9b0 100644 --- a/bundle/direct/dresources/snapshot.go +++ b/bundle/direct/dresources/snapshot.go @@ -23,11 +23,16 @@ type SnapshotState struct { FullPath string `json:"full_path"` BundleID string `json:"bundle_id"` ACL []snapshot.ACLEntry `json:"acl"` + // CanManage lists the principals allowed to break the glass on the snapshot. + CanManage []snapshot.ManagePrincipal `json:"can_manage,omitempty"` // ZipPath locates the bundle zip staged locally by the deploy pipeline. It is // small enough to round-trip through the plan file, so deploying from a plan // restores it directly (no re-injection needed). The file name is the content // hash, which is also the last component of RelativePath. ZipPath string `json:"zip_path"` + // Generation is the break-glass generation folded into RelativePath (see + // resources.Snapshot.Generation). Carried forward in state; bumped on --force recovery. + Generation int `json:"generation,omitempty"` } type SnapshotRemote struct { @@ -59,7 +64,9 @@ func (s *ResourceSnapshot) PrepareState(input *resources.Snapshot) *SnapshotStat FullPath: input.FullPath(), BundleID: input.BundleID, ACL: input.ACL, + CanManage: input.CanManage, ZipPath: input.ZipPath, + Generation: input.Generation, } } @@ -69,7 +76,9 @@ func (s *ResourceSnapshot) RemapState(remote *SnapshotRemote) *SnapshotState { FullPath: remote.FullPath, BundleID: "", ACL: nil, + CanManage: nil, ZipPath: "", + Generation: 0, } } @@ -91,7 +100,7 @@ func (s *ResourceSnapshot) DoCreate(ctx context.Context, state *SnapshotState) ( } path := state.RelativePath - info, err := s.uploader.Upload(ctx, path, state.BundleID, state.ACL, content) + info, err := s.uploader.Upload(ctx, path, state.ACL, state.CanManage, content) if err != nil { return "", nil, err } diff --git a/bundle/direct/dresources/type_test.go b/bundle/direct/dresources/type_test.go index 9d7f6770180..864c3e1db35 100644 --- a/bundle/direct/dresources/type_test.go +++ b/bundle/direct/dresources/type_test.go @@ -59,7 +59,9 @@ var knownMissingInRemoteType = map[string][]string{ "internal_immutable_snapshots": { "bundle_id", "acl", + "can_manage", "zip_path", + "generation", }, "job_runs": { // Local-only trigger fingerprints under lifecycle. diff --git a/libs/snapshot/client.go b/libs/snapshot/client.go index 8539cf0bda5..279ae1ba790 100644 --- a/libs/snapshot/client.go +++ b/libs/snapshot/client.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "encoding/json" + "errors" "fmt" "mime/multipart" "net/http" @@ -15,6 +16,11 @@ import ( databricksclient "github.com/databricks/databricks-sdk-go/client" ) +// ContentSubdir is the fixed subfolder the snapshot API appends to a caller-supplied +// relative_path to form the immutable content path (e.g. "//snapshot"). +// The content path is what the create API echoes back and what InspectSnapshot inspects. +const ContentSubdir = "snapshot" + // SnapshotInfo holds the result of a successful snapshot upload. type SnapshotInfo struct { // Path is the immutable workspace path for the uploaded snapshot content. @@ -30,13 +36,29 @@ type ACLEntry struct { PermissionLevel string `json:"permission_level"` } -// SnapshotClient implements the /api/2.0/repos/snapshots endpoint. +// ManagePrincipal is one element of can_manage_principals: a principal allowed to +// "break the glass" and modify an otherwise-immutable snapshot out of band. +type ManagePrincipal struct { + UserName string `json:"user_name,omitempty"` + GroupName string `json:"group_name,omitempty"` + ServicePrincipalName string `json:"service_principal_name,omitempty"` +} + +// SnapshotStatus is the break-glass status of a snapshot's content path, as reported by +// InspectSnapshot. Dirty is true when the immutable content was modified out of band. +type SnapshotStatus struct { + Dirty bool + Permissions []string +} + +// SnapshotClient implements the /api/2.0/snapshots endpoint. type SnapshotClient struct { workspaceClient *databricks.WorkspaceClient client *databricksclient.DatabricksClient } -// snapshotUploadResponse mirrors the /api/2.0/repos/snapshots response body. +// snapshotUploadResponse mirrors the /api/2.0/snapshots response body. Only the content +// path is used; the operation name, done flag and creation metadata are ignored. type snapshotUploadResponse struct { Snapshot struct { Path string `json:"path"` @@ -47,7 +69,52 @@ type snapshotRootPathResponse struct { Path string `json:"path"` } -// NewSnapshotClient creates a SnapshotClient backed by /api/2.0/repos/snapshots. +// graphqlRequest is the standard GraphQL POST envelope. +type graphqlRequest struct { + Query string `json:"query"` + Variables map[string]any `json:"variables"` +} + +// inspectSnapshotResponse mirrors the projectsInspectSnapshot GraphQL response. +type inspectSnapshotResponse struct { + Data struct { + ProjectsInspectSnapshot struct { + Status *struct { + SnapshotContentPath string `json:"snapshotContentPath"` + Dirty bool `json:"dirty"` + Permissions []string `json:"permissions"` + } `json:"status"` + APIError *struct { + Code string `json:"code"` + Message string `json:"message"` + } `json:"apiError"` + } `json:"projectsInspectSnapshot"` + } `json:"data"` + Errors []struct { + Message string `json:"message"` + } `json:"errors"` +} + +// inspectSnapshotQuery reads the break-glass status of a snapshot content path. +const inspectSnapshotQuery = `query InspectSnapshot($path: String!) { + projectsInspectSnapshot(input: {snapshotContentPath: $path}) { + status { + snapshotContentPath + dirty + permissions + } + apiError { + code + message + } + } +}` + +// graphqlPath is the workspace GraphQL gateway. InspectSnapshot is exposed there today; the +// same RPC also has a REST binding (GET /api/2.0/snapshots:inspect) it can be moved to later. +const graphqlPath = "/api/2.0/graphql" + +// NewSnapshotClient creates a SnapshotClient backed by /api/2.0/snapshots. func NewSnapshotClient(w *databricks.WorkspaceClient) (*SnapshotClient, error) { c, err := databricksclient.New(w.Config) if err != nil { @@ -56,19 +123,16 @@ func NewSnapshotClient(w *databricks.WorkspaceClient) (*SnapshotClient, error) { return &SnapshotClient{workspaceClient: w, client: c}, nil } -// Upload uploads zipContent as an immutable snapshot. The server derives the -// content-addressed path from the SHA-256 of the zip, and acl grants CAN_READ to -// each listed principal. path is reserved for forthcoming API support for a -// caller-specified upload location; it is not sent yet. -func (c *SnapshotClient) Upload(ctx context.Context, path, bundleID string, acl []ACLEntry, zipContent []byte) (*SnapshotInfo, error) { +// Upload uploads zipContent as an immutable snapshot at relativePath. The server stores the +// content under "//snapshot" and returns that content path. acl grants +// CAN_READ to each listed principal; canManage lists the principals allowed to break the +// glass on the snapshot. +func (c *SnapshotClient) Upload(ctx context.Context, relativePath string, acl []ACLEntry, canManage []ManagePrincipal, zipContent []byte) (*SnapshotInfo, error) { var body bytes.Buffer mw := multipart.NewWriter(&body) - if err := mw.WriteField("snapshot_id", HashFromContent(zipContent)); err != nil { - return nil, fmt.Errorf("failed to write snapshot_id: %w", err) - } - if err := mw.WriteField("bundle_id", bundleID); err != nil { - return nil, fmt.Errorf("failed to write bundle_id: %w", err) + if err := mw.WriteField("relative_path", relativePath); err != nil { + return nil, fmt.Errorf("failed to write relative_path: %w", err) } aclJSON, err := json.Marshal(acl) @@ -79,6 +143,18 @@ func (c *SnapshotClient) Upload(ctx context.Context, path, bundleID string, acl return nil, fmt.Errorf("failed to write access_control_list: %w", err) } + // Always send an array, never null, so "no break-glass managers" is unambiguous. + if canManage == nil { + canManage = []ManagePrincipal{} + } + canManageJSON, err := json.Marshal(canManage) + if err != nil { + return nil, fmt.Errorf("failed to marshal can_manage_principals: %w", err) + } + if err := mw.WriteField("can_manage_principals", string(canManageJSON)); err != nil { + return nil, fmt.Errorf("failed to write can_manage_principals: %w", err) + } + // Attach the zip with an explicit content-type so the server treats it as binary. fh := make(textproto.MIMEHeader) fh.Set("Content-Disposition", `form-data; name="file"; filename="snapshot.zip"`) @@ -103,7 +179,7 @@ func (c *SnapshotClient) Upload(ctx context.Context, path, bundleID string, acl headers["Content-Type"] = mw.FormDataContentType() var resp snapshotUploadResponse - err = c.client.Do(ctx, http.MethodPost, "/api/2.0/repos/snapshots", headers, nil, body.Bytes(), &resp) + err = c.client.Do(ctx, http.MethodPost, "/api/2.0/snapshots", headers, nil, body.Bytes(), &resp) if err != nil { return nil, fmt.Errorf("snapshot upload: %w", err) } @@ -111,12 +187,51 @@ func (c *SnapshotClient) Upload(ctx context.Context, path, bundleID string, acl return &SnapshotInfo{Path: resp.Snapshot.Path}, nil } +// InspectSnapshot reports the break-glass status of the snapshot content at contentPath. +// It is implemented today as a GraphQL projectsInspectSnapshot query; the method boundary +// hides that so it can later be swapped to the REST binding GET /api/2.0/snapshots:inspect +// without touching callers. +func (c *SnapshotClient) InspectSnapshot(ctx context.Context, contentPath string) (*SnapshotStatus, error) { + headers := auth.WorkspaceIDHeaders(c.client.Config) + if headers == nil { + headers = make(map[string]string) + } + headers["Content-Type"] = "application/json" + + req := graphqlRequest{ + Query: inspectSnapshotQuery, + Variables: map[string]any{"path": contentPath}, + } + + var resp inspectSnapshotResponse + err := c.client.Do(ctx, http.MethodPost, graphqlPath, headers, nil, req, &resp) + if err != nil { + return nil, fmt.Errorf("snapshot inspect: %w", err) + } + if len(resp.Errors) > 0 { + return nil, fmt.Errorf("snapshot inspect: %s", resp.Errors[0].Message) + } + + result := resp.Data.ProjectsInspectSnapshot + if result.APIError != nil { + return nil, fmt.Errorf("snapshot inspect: %s", result.APIError.Message) + } + if result.Status == nil { + return nil, errors.New("snapshot inspect: response contained no status") + } + + return &SnapshotStatus{ + Dirty: result.Status.Dirty, + Permissions: result.Status.Permissions, + }, nil +} + func (c *SnapshotClient) Get(ctx context.Context, snapshotRelativePath string) (*SnapshotInfo, error) { rootPath, err := c.GetSnapshotRootPath(ctx) if err != nil { return nil, fmt.Errorf("failed to get snapshot root path: %w", err) } - snapshotPath := path.Join(rootPath, snapshotRelativePath) + snapshotPath := path.Join(rootPath, snapshotRelativePath, ContentSubdir) resp, err := c.workspaceClient.Workspace.GetStatusByPath(ctx, snapshotPath) if err != nil { return nil, fmt.Errorf("snapshot get: %w", err) diff --git a/libs/testserver/fake_workspace.go b/libs/testserver/fake_workspace.go index 44d6ccd1889..a27d2c4399f 100644 --- a/libs/testserver/fake_workspace.go +++ b/libs/testserver/fake_workspace.go @@ -188,6 +188,11 @@ type FakeWorkspace struct { files map[string]FileEntry repoIdByPath map[string]int64 + // snapshots records immutable-snapshot content paths and whether each was modified out + // of band ("break-glassed"). A workspace write into a recorded path flips it dirty; + // RecordSnapshot resets it when the snapshot is (re)created by the snapshot API. + snapshots map[string]bool + Jobs map[int64]jobs.Job JobRuns map[int64]jobs.Run JobRunOutputs map[int64]jobs.RunOutput @@ -479,6 +484,7 @@ func NewFakeWorkspace(url, token string) *FakeWorkspace { }, files: make(map[string]FileEntry), repoIdByPath: make(map[string]int64), + snapshots: make(map[string]bool), Jobs: map[int64]jobs.Job{}, JobRuns: map[int64]jobs.Run{}, @@ -804,6 +810,31 @@ func (s *FakeWorkspace) FsDeleteFile(filePath string) Response { return Response{} } +// RecordSnapshot marks contentPath as a known, clean immutable snapshot. The snapshot create +// handler calls it after materializing the content directory, so the snapshot's own creation +// write does not count as tampering. +func (s *FakeWorkspace) RecordSnapshot(contentPath string) { + defer s.LockUnlock()() + s.snapshots[contentPath] = false +} + +// SnapshotDirty reports whether the immutable snapshot at contentPath was modified out of +// band. An unknown path is reported clean. +func (s *FakeWorkspace) SnapshotDirty(contentPath string) bool { + defer s.LockUnlock()() + return s.snapshots[contentPath] +} + +// markSnapshotDirty flips any recorded snapshot dirty when targetPath writes at or under its +// content path. The caller must already hold the lock. +func (s *FakeWorkspace) markSnapshotDirty(targetPath string) { + for contentPath := range s.snapshots { + if targetPath == contentPath || strings.HasPrefix(targetPath, contentPath+"/") { + s.snapshots[contentPath] = true + } + } +} + func (s *FakeWorkspace) WorkspaceMkdirs(request workspace.Mkdirs) { defer s.LockUnlock()() // The real mkdirs API creates all intermediate directories ("mkdir -p"), @@ -817,6 +848,7 @@ func (s *FakeWorkspace) WorkspaceMkdirs(request workspace.Mkdirs) { } } } + s.markSnapshotDirty(request.Path) } func (s *FakeWorkspace) WorkspaceExport(path string) []byte { @@ -853,6 +885,7 @@ func (s *FakeWorkspace) WorkspaceDelete(path string, recursive bool) Response { } } } + s.markSnapshotDirty(path) return Response{} } @@ -925,6 +958,7 @@ func (s *FakeWorkspace) WorkspaceFilesImportFile(filePath string, body []byte, o } } + s.markSnapshotDirty(workspacePath) return Response{} } @@ -982,6 +1016,7 @@ func (s *FakeWorkspace) WorkspaceImportNotebook(filePath string, body []byte, la Data: body, } + s.markSnapshotDirty(filePath) return Response{} } diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go index 15aa89e9bdf..f6ed2809bb0 100644 --- a/libs/testserver/handlers.go +++ b/libs/testserver/handlers.go @@ -724,7 +724,7 @@ func AddDefaultHandlers(server *Server) { } }) - server.Handle("POST", "/api/2.0/repos/snapshots", func(req Request) any { + server.Handle("POST", "/api/2.0/snapshots", func(req Request) any { contentType := req.Headers.Get("Content-Type") mediaType, params, err := mime.ParseMediaType(contentType) if err != nil || !strings.HasPrefix(mediaType, "multipart/") { @@ -732,7 +732,7 @@ func AddDefaultHandlers(server *Server) { } mr := multipart.NewReader(bytes.NewReader(req.Body), params["boundary"]) - var bundleID, snapshotID string + var relativePath string for { p, err := mr.NextPart() if err == io.EOF { @@ -745,21 +745,47 @@ func AddDefaultHandlers(server *Server) { if err != nil { return Response{StatusCode: http.StatusInternalServerError} } - switch p.FormName() { - case "bundle_id": - bundleID = string(data) - case "snapshot_id": - snapshotID = string(data) + if p.FormName() == "relative_path" { + relativePath = string(data) } } - // The real API uses the workspace user UUID (not email) in the snapshot path, + // The server stores content under a fixed "snapshot" subfolder of the caller's + // relative_path. The real API uses the workspace user UUID (not email) in the path, // matching service-principal identities used in cloud acceptance tests. - snapshotPath := fmt.Sprintf("/Workspace/Users/%s/.snapshots/%s/%s", TestUserSP.UserName, bundleID, snapshotID) - req.Workspace.WorkspaceMkdirs(workspace.Mkdirs{Path: snapshotPath}) + contentPath := fmt.Sprintf("/Workspace/Users/%s/.snapshots/%s/snapshot", TestUserSP.UserName, relativePath) + req.Workspace.WorkspaceMkdirs(workspace.Mkdirs{Path: contentPath}) + // Record after mkdirs so the snapshot's own creation write is not counted as tampering. + req.Workspace.RecordSnapshot(contentPath) return map[string]any{ - "snapshot": map[string]any{ - "path": snapshotPath, + "name": "workspaces/snapshotOperations/" + relativePath, + "done": true, + "snapshot": map[string]any{"path": contentPath}, + } + }) + + // GraphQL gateway. The only query modeled is projectsInspectSnapshot, which reports the + // break-glass status (dirty) of a snapshot content path. + server.Handle("POST", "/api/2.0/graphql", func(req Request) any { + var body struct { + Variables struct { + Path string `json:"path"` + } `json:"variables"` + } + if err := json.Unmarshal(req.Body, &body); err != nil { + return Response{StatusCode: http.StatusBadRequest} + } + contentPath := body.Variables.Path + return map[string]any{ + "data": map[string]any{ + "projectsInspectSnapshot": map[string]any{ + "status": map[string]any{ + "snapshotContentPath": contentPath, + "dirty": req.Workspace.SnapshotDirty(contentPath), + "permissions": []string{}, + }, + "apiError": nil, + }, }, } }) From 8c4166675790f1ef356db1b79481fc8142c91da3 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Mon, 21 Sep 2026 16:41:41 +0200 Subject: [PATCH 02/16] fix PR link --- .nextchanges/bundles/immutable-breakglass.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.nextchanges/bundles/immutable-breakglass.md b/.nextchanges/bundles/immutable-breakglass.md index 22d9a48288e..9a082a2a4e6 100644 --- a/.nextchanges/bundles/immutable-breakglass.md +++ b/.nextchanges/bundles/immutable-breakglass.md @@ -1 +1 @@ -* Immutable-folder deploys now detect a break-glassed snapshot and refuse to redeploy over it; `bundle deploy --force` recovers by creating a new snapshot at a suffixed path. +* Immutable-folder deploys now detect a break-glassed snapshot and refuse to redeploy over it; `bundle deploy --force` recovers by creating a new snapshot at a suffixed path. ([#6774](https://github.com/databricks/cli/pull/6774)) From 35ce9cb17008640dc1d154c351d3ba12fe81d499 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Tue, 22 Sep 2026 12:24:36 +0200 Subject: [PATCH 03/16] fix refschema --- acceptance/bundle/refschema/out.fields.txt | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index 5e774e1b580..7c8944d9cd4 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -908,7 +908,13 @@ resources.internal_immutable_snapshots.*.acl[*].permission_level string INPUT ST resources.internal_immutable_snapshots.*.acl[*].service_principal_name string INPUT STATE resources.internal_immutable_snapshots.*.acl[*].user_name string INPUT STATE resources.internal_immutable_snapshots.*.bundle_id string INPUT STATE +resources.internal_immutable_snapshots.*.can_manage []snapshot.ManagePrincipal INPUT STATE +resources.internal_immutable_snapshots.*.can_manage[*] snapshot.ManagePrincipal INPUT STATE +resources.internal_immutable_snapshots.*.can_manage[*].group_name string INPUT STATE +resources.internal_immutable_snapshots.*.can_manage[*].service_principal_name string INPUT STATE +resources.internal_immutable_snapshots.*.can_manage[*].user_name string INPUT STATE resources.internal_immutable_snapshots.*.full_path string REMOTE STATE +resources.internal_immutable_snapshots.*.generation int INPUT STATE resources.internal_immutable_snapshots.*.relative_path string REMOTE STATE resources.internal_immutable_snapshots.*.remote_root string INPUT resources.internal_immutable_snapshots.*.zip_path string INPUT STATE From f1b06de4555c06a4a680ba939471968858f8d23c Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Fri, 25 Sep 2026 12:11:23 +0200 Subject: [PATCH 04/16] use rest API + feedback --- .nextchanges/bundles/immutable-breakglass.md | 2 +- .../deploy/immutable-breakglass/output.txt | 13 ++- .../bundle/deploy/immutable-breakglass/script | 20 ++-- .../deploy/immutable-breakglass/test.toml | 7 ++ bundle/config/resources/snapshot.go | 14 +-- bundle/deploy/snapshot/upload.go | 42 ++++--- bundle/deploy/snapshot/upload_warning_test.go | 2 +- bundle/direct/dresources/snapshot.go | 4 +- libs/snapshot/client.go | 109 +++++++----------- libs/testserver/fake_workspace.go | 2 +- libs/testserver/handlers.go | 24 ++-- 11 files changed, 111 insertions(+), 128 deletions(-) diff --git a/.nextchanges/bundles/immutable-breakglass.md b/.nextchanges/bundles/immutable-breakglass.md index 9a082a2a4e6..586600d58c4 100644 --- a/.nextchanges/bundles/immutable-breakglass.md +++ b/.nextchanges/bundles/immutable-breakglass.md @@ -1 +1 @@ -* Immutable-folder deploys now detect a break-glassed snapshot and refuse to redeploy over it; `bundle deploy --force` recovers by creating a new snapshot at a suffixed path. ([#6774](https://github.com/databricks/cli/pull/6774)) +* Immutable-folder deploys now refuse to deploy onto a snapshot that was modified outside the bundle (break glass); `bundle deploy --force` moves the deployment to a fresh snapshot. ([#6774](https://github.com/databricks/cli/pull/6774)) diff --git a/acceptance/bundle/deploy/immutable-breakglass/output.txt b/acceptance/bundle/deploy/immutable-breakglass/output.txt index 3379c10d1f0..f538fa251cb 100644 --- a/acceptance/bundle/deploy/immutable-breakglass/output.txt +++ b/acceptance/bundle/deploy/immutable-breakglass/output.txt @@ -11,25 +11,28 @@ First snapshot path: /Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHO >>> [CLI] workspace mkdirs /Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]/snapshot/tampered === Deploy without --force fails -Error: the previously deployed immutable snapshot at /Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]/snapshot was modified out of band (break-glass); re-run 'bundle deploy --force' to deploy a new snapshot +>>> musterr [CLI] bundle deploy +Error: the immutable snapshot of the last deployment was modified outside of the bundle (break glass): + /Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]/snapshot +To deploy a new snapshot and point resources at it, use --force Files: 0 uploaded, 0 deleted -=== Deploy with --force recovers to a new snapshot +=== Deploy with --force moves to a new snapshot >>> [CLI] bundle deploy --force Recreated internal_immutable_snapshots.immutable Updated jobs.my_job Files: 0 uploaded, 0 deleted Resources: 1 created, 1 changed, 1 deleted, 0 unchanged Second snapshot path: /Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]-1/snapshot/files/src/main.py -OK: recovery created a new snapshot path +OK: --force created a new snapshot path -=== Redeploy reuses the recovered snapshot +=== Redeploy reuses the new snapshot >>> [CLI] bundle deploy Files: 0 uploaded, 0 deleted Resources: 0 created, 0 changed, 0 deleted, 2 unchanged Third snapshot path: /Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]-1/snapshot/files/src/main.py -OK: recovered snapshot reused on redeploy +OK: new snapshot reused on redeploy >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: diff --git a/acceptance/bundle/deploy/immutable-breakglass/script b/acceptance/bundle/deploy/immutable-breakglass/script index 32ba1240ea6..eafcfea63ec 100644 --- a/acceptance/bundle/deploy/immutable-breakglass/script +++ b/acceptance/bundle/deploy/immutable-breakglass/script @@ -13,30 +13,30 @@ SNAPSHOT_PATH_1=$($CLI jobs get "$JOB_ID" | jq -r '.settings.tasks[0].spark_pyth echo "First snapshot path: $SNAPSHOT_PATH_1" title "Break the glass out of band" -# The snapshot content path holds files/ and artifacts/; writing into it out of band -# marks the immutable snapshot as broken. +# The snapshot content path holds files/ and artifacts/; writing into it out of band is what +# the backend reports as a modified (dirty) snapshot. CONTENT_PATH="${SNAPSHOT_PATH_1%/files/src/main.py}" trace $CLI workspace mkdirs "$CONTENT_PATH/tampered" -title "Deploy without --force fails\n" -musterr $CLI bundle deploy +title "Deploy without --force fails" +trace musterr $CLI bundle deploy -title "Deploy with --force recovers to a new snapshot" +title "Deploy with --force moves to a new snapshot" trace $CLI bundle deploy --force SNAPSHOT_PATH_2=$($CLI jobs get "$JOB_ID" | jq -r '.settings.tasks[0].spark_python_task.python_file') echo "Second snapshot path: $SNAPSHOT_PATH_2" if [ "$SNAPSHOT_PATH_1" = "$SNAPSHOT_PATH_2" ]; then - echo "ERROR: snapshot path did not change after break-glass recovery" + echo "ERROR: snapshot path did not change after --force" exit 1 fi -echo "OK: recovery created a new snapshot path" +echo "OK: --force created a new snapshot path" -title "Redeploy reuses the recovered snapshot" +title "Redeploy reuses the new snapshot" trace $CLI bundle deploy SNAPSHOT_PATH_3=$($CLI jobs get "$JOB_ID" | jq -r '.settings.tasks[0].spark_python_task.python_file') echo "Third snapshot path: $SNAPSHOT_PATH_3" if [ "$SNAPSHOT_PATH_2" != "$SNAPSHOT_PATH_3" ]; then - echo "ERROR: recovered snapshot path was not reused on redeploy" + echo "ERROR: new snapshot path was not reused on redeploy" exit 1 fi -echo "OK: recovered snapshot reused on redeploy" +echo "OK: new snapshot reused on redeploy" diff --git a/acceptance/bundle/deploy/immutable-breakglass/test.toml b/acceptance/bundle/deploy/immutable-breakglass/test.toml index 88b225ea71f..064e7eeff9b 100644 --- a/acceptance/bundle/deploy/immutable-breakglass/test.toml +++ b/acceptance/bundle/deploy/immutable-breakglass/test.toml @@ -8,6 +8,13 @@ Ignore = [ ".databricks", ] +# This test passes the snapshot's absolute workspace path to "workspace mkdirs" to modify it out +# of band. On Windows the script runs under MSYS2, which rewrites such leading-slash arguments to +# Windows paths (e.g. C:/Program Files/Git/Workspace/...) before the CLI sees them, so the write +# would land outside the snapshot and leave it looking untouched. +# Setting this environment variable disables that conversion. +Env.MSYS_NO_PATHCONV = "1" + [[Repls]] # Replace snapshot hash with SNAPSHOT_HASH Old = "[0-9a-f]{64}" diff --git a/bundle/config/resources/snapshot.go b/bundle/config/resources/snapshot.go index ca9476010cb..921a944bbf2 100644 --- a/bundle/config/resources/snapshot.go +++ b/bundle/config/resources/snapshot.go @@ -63,11 +63,11 @@ type Snapshot struct { // Example: "/Workspace/Users//.snapshots". RemoteRoot string `json:"remote_root"` - // Generation is the break-glass generation. It is 0 for the initial snapshot and - // incremented each time "deploy --force" recovers from a broken (break-glassed) - // snapshot, appending a "-" suffix to the content hash so the recovered - // snapshot gets a distinct path. It is carried forward in state across deploys, so a - // recovered path keeps being reused until it too is broken. + // Generation is 0 for the initial snapshot and incremented each time "deploy --force" + // moves off a snapshot that was modified outside the bundle, appending a "-" + // suffix to the content hash so the new snapshot gets a distinct path. It is carried + // forward in state across deploys, so that path keeps being reused until it is modified + // in turn. Generation int `json:"generation,omitempty"` Lifecycle Lifecycle `json:"-"` @@ -76,8 +76,8 @@ type Snapshot struct { // RelativePath is the snapshot's location under RemoteRoot: "/". // It is the resource ID; the last component is the zip content hash (the ZipPath base // name), which makes the pre-computed path match the uploaded content. A non-zero -// Generation appends a "-" suffix to the hash so a break-glass recovery -// uploads to a distinct path (e.g. "1a2b3c4d-.../e3b0c4...-1"). +// Generation appends a "-" suffix to the hash so a snapshot replacing one that was +// modified outside the bundle uploads to a distinct path (e.g. "1a2b3c4d-.../e3b0c4...-1"). func (s *Snapshot) RelativePath() string { hash := strings.TrimSuffix(path.Base(s.ZipPath), ".zip") if s.Generation > 0 { diff --git a/bundle/deploy/snapshot/upload.go b/bundle/deploy/snapshot/upload.go index d69f73348ab..416ffdaf821 100644 --- a/bundle/deploy/snapshot/upload.go +++ b/bundle/deploy/snapshot/upload.go @@ -11,6 +11,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/direct/dresources" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/snapshot" ) @@ -70,9 +71,9 @@ func (m *snapshotUpload) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagn return nil } - // Detect a break-glassed previous snapshot before doing any work, so a normal deploy - // over a broken snapshot fails fast and --force recovery gets the right generation. - generation, err := breakGlassGeneration(ctx, b, uploader) + // Check the previous snapshot before doing any work, so a deploy onto a snapshot that was + // modified outside the bundle fails before building a zip. + generation, err := resolveGeneration(ctx, b, uploader) if err != nil { return diag.FromErr(err) } @@ -162,17 +163,25 @@ func BuildCanManage(b *bundle.Bundle) []snapshot.ManagePrincipal { return canManage } -// breakGlassGeneration reads the previously deployed snapshot from state and asks the backend -// whether it was modified out of band ("break-glassed"). It returns the generation to deploy: -// - the previous generation when the snapshot is intact, so a recovered path keeps being reused; -// - the previous generation + 1 when the snapshot is broken and --force was given (recovery); -// - an error when the snapshot is broken and --force was not given. +// resolveGeneration asks the backend whether the snapshot deployed last time was modified +// outside of the bundle, and returns the generation the new snapshot should use: +// - the previous generation while the snapshot is untouched, so a path reached by an earlier +// recovery keeps being reused; +// - the previous generation + 1 when it was modified and --force was given, which moves the +// deployment to a fresh path; +// - an error when it was modified and --force was not given. // -// On the first deploy (no prior snapshot) it returns 0. -func breakGlassGeneration(ctx context.Context, b *bundle.Bundle, uploader *snapshot.SnapshotClient) (int, error) { +// The check has to happen here rather than in the resource's DoRead: the generation decides +// the snapshot's path, and resources referencing ${...full_path} resolve it from the desired +// state computed before any resource is read (see LookupReferencePreDeploy). Erroring unless +// --force also has no equivalent in the resource layer. CheckDashboardsModifiedRemotely is the +// same shape: read state, ask the API, refuse unless forced. +// +// On the first deploy (no snapshot in state) the generation is 0. +func resolveGeneration(ctx context.Context, b *bundle.Bundle, uploader *snapshot.SnapshotClient) (int, error) { // The deploy/plan pipeline opens the state DB for read before this runs, but callers that - // exercise PlanUpload in isolation (unit tests) may not. No open state means no prior - // snapshot to check, which is the first-deploy case: generation 0. + // exercise PlanUpload in isolation (unit tests) may not. No open state means no previous + // snapshot to check, which is the first-deploy case. if !b.DeploymentBundle.StateDB.IsOpen() { return 0, nil } @@ -182,11 +191,8 @@ func breakGlassGeneration(ctx context.Context, b *bundle.Bundle, uploader *snaps return 0, nil } - // Only the content path and generation are needed; unmarshal a subset of the state. - var prev struct { - FullPath string `json:"full_path"` - Generation int `json:"generation"` - } + // Decode into the state type the engine persists, so the field names cannot drift apart. + var prev dresources.SnapshotState if err := json.Unmarshal(entry.State, &prev); err != nil { return 0, fmt.Errorf("reading previous snapshot state: %w", err) } @@ -202,7 +208,7 @@ func breakGlassGeneration(ctx context.Context, b *bundle.Bundle, uploader *snaps return prev.Generation, nil } if !b.Config.Bundle.Force { - return 0, fmt.Errorf("the previously deployed immutable snapshot at %s was modified out of band (break-glass); re-run 'bundle deploy --force' to deploy a new snapshot", prev.FullPath) + return 0, fmt.Errorf("the immutable snapshot of the last deployment was modified outside of the bundle (break glass):\n %s\nTo deploy a new snapshot and point resources at it, use --force", prev.FullPath) } return prev.Generation + 1, nil } diff --git a/bundle/deploy/snapshot/upload_warning_test.go b/bundle/deploy/snapshot/upload_warning_test.go index 79ac4b070e4..cc18c728274 100644 --- a/bundle/deploy/snapshot/upload_warning_test.go +++ b/bundle/deploy/snapshot/upload_warning_test.go @@ -104,7 +104,7 @@ func TestBuildCanManage(t *testing.T) { }, } - // Only CAN_MANAGE principals become break-glass managers. + // Only CAN_MANAGE principals may break the glass on the snapshot. assert.Equal(t, []snapshot.ManagePrincipal{ {UserName: "manager@example.test"}, {GroupName: "admins"}, diff --git a/bundle/direct/dresources/snapshot.go b/bundle/direct/dresources/snapshot.go index 1b97e99d9b0..88b6ac7c64a 100644 --- a/bundle/direct/dresources/snapshot.go +++ b/bundle/direct/dresources/snapshot.go @@ -30,8 +30,8 @@ type SnapshotState struct { // restores it directly (no re-injection needed). The file name is the content // hash, which is also the last component of RelativePath. ZipPath string `json:"zip_path"` - // Generation is the break-glass generation folded into RelativePath (see - // resources.Snapshot.Generation). Carried forward in state; bumped on --force recovery. + // Generation is folded into RelativePath (see resources.Snapshot.Generation). Carried + // forward in state; bumped when --force moves off a snapshot modified outside the bundle. Generation int `json:"generation,omitempty"` } diff --git a/libs/snapshot/client.go b/libs/snapshot/client.go index 279ae1ba790..1a64baa32de 100644 --- a/libs/snapshot/client.go +++ b/libs/snapshot/client.go @@ -6,6 +6,7 @@ import ( "encoding/json" "errors" "fmt" + "io" "mime/multipart" "net/http" "net/textproto" @@ -36,19 +37,18 @@ type ACLEntry struct { PermissionLevel string `json:"permission_level"` } -// ManagePrincipal is one element of can_manage_principals: a principal allowed to -// "break the glass" and modify an otherwise-immutable snapshot out of band. +// ManagePrincipal is one element of can_manage_principals: a principal allowed to break the +// glass on a snapshot, i.e. to modify otherwise-immutable content. type ManagePrincipal struct { UserName string `json:"user_name,omitempty"` GroupName string `json:"group_name,omitempty"` ServicePrincipalName string `json:"service_principal_name,omitempty"` } -// SnapshotStatus is the break-glass status of a snapshot's content path, as reported by -// InspectSnapshot. Dirty is true when the immutable content was modified out of band. +// SnapshotStatus is the status of a snapshot's content path, as reported by InspectSnapshot. +// Dirty is true when the immutable content was modified out of band. type SnapshotStatus struct { - Dirty bool - Permissions []string + Dirty bool } // SnapshotClient implements the /api/2.0/snapshots endpoint. @@ -69,51 +69,23 @@ type snapshotRootPathResponse struct { Path string `json:"path"` } -// graphqlRequest is the standard GraphQL POST envelope. -type graphqlRequest struct { - Query string `json:"query"` - Variables map[string]any `json:"variables"` +// inspectSnapshotPath reports whether a snapshot's content was modified out of band. +const inspectSnapshotPath = "/api/2.0/snapshots:inspect" + +// inspectSnapshotRequest is the body of an inspect call. +type inspectSnapshotRequest struct { + SnapshotContentPath string `json:"snapshot_content_path"` } -// inspectSnapshotResponse mirrors the projectsInspectSnapshot GraphQL response. +// inspectSnapshotResponse mirrors the /api/2.0/snapshots:inspect response body. The +// permissions the status also carries are not used. type inspectSnapshotResponse struct { - Data struct { - ProjectsInspectSnapshot struct { - Status *struct { - SnapshotContentPath string `json:"snapshotContentPath"` - Dirty bool `json:"dirty"` - Permissions []string `json:"permissions"` - } `json:"status"` - APIError *struct { - Code string `json:"code"` - Message string `json:"message"` - } `json:"apiError"` - } `json:"projectsInspectSnapshot"` - } `json:"data"` - Errors []struct { - Message string `json:"message"` - } `json:"errors"` + Status *struct { + SnapshotContentPath string `json:"snapshot_content_path"` + Dirty bool `json:"dirty"` + } `json:"status"` } -// inspectSnapshotQuery reads the break-glass status of a snapshot content path. -const inspectSnapshotQuery = `query InspectSnapshot($path: String!) { - projectsInspectSnapshot(input: {snapshotContentPath: $path}) { - status { - snapshotContentPath - dirty - permissions - } - apiError { - code - message - } - } -}` - -// graphqlPath is the workspace GraphQL gateway. InspectSnapshot is exposed there today; the -// same RPC also has a REST binding (GET /api/2.0/snapshots:inspect) it can be moved to later. -const graphqlPath = "/api/2.0/graphql" - // NewSnapshotClient creates a SnapshotClient backed by /api/2.0/snapshots. func NewSnapshotClient(w *databricks.WorkspaceClient) (*SnapshotClient, error) { c, err := databricksclient.New(w.Config) @@ -187,10 +159,8 @@ func (c *SnapshotClient) Upload(ctx context.Context, relativePath string, acl [] return &SnapshotInfo{Path: resp.Snapshot.Path}, nil } -// InspectSnapshot reports the break-glass status of the snapshot content at contentPath. -// It is implemented today as a GraphQL projectsInspectSnapshot query; the method boundary -// hides that so it can later be swapped to the REST binding GET /api/2.0/snapshots:inspect -// without touching callers. +// InspectSnapshot reports the status of the snapshot content at contentPath, which tells the +// caller whether the immutable content was modified out of band. func (c *SnapshotClient) InspectSnapshot(ctx context.Context, contentPath string) (*SnapshotStatus, error) { headers := auth.WorkspaceIDHeaders(c.client.Config) if headers == nil { @@ -198,32 +168,37 @@ func (c *SnapshotClient) InspectSnapshot(ctx context.Context, contentPath string } headers["Content-Type"] = "application/json" - req := graphqlRequest{ - Query: inspectSnapshotQuery, - Variables: map[string]any{"path": contentPath}, + payload, err := json.Marshal(inspectSnapshotRequest{SnapshotContentPath: contentPath}) + if err != nil { + return nil, fmt.Errorf("failed to marshal inspect request: %w", err) } var resp inspectSnapshotResponse - err := c.client.Do(ctx, http.MethodPost, graphqlPath, headers, nil, req, &resp) + err = c.client.Do(ctx, http.MethodGet, inspectSnapshotPath, headers, nil, nil, &resp, withJSONBody(payload)) if err != nil { return nil, fmt.Errorf("snapshot inspect: %w", err) } - if len(resp.Errors) > 0 { - return nil, fmt.Errorf("snapshot inspect: %s", resp.Errors[0].Message) - } - - result := resp.Data.ProjectsInspectSnapshot - if result.APIError != nil { - return nil, fmt.Errorf("snapshot inspect: %s", result.APIError.Message) - } - if result.Status == nil { + // A 200 without a status would otherwise read as "not modified", silently disabling the + // check, so treat it as an error instead. + if resp.Status == nil { return nil, errors.New("snapshot inspect: response contained no status") } - return &SnapshotStatus{ - Dirty: result.Status.Dirty, - Permissions: result.Status.Permissions, - }, nil + return &SnapshotStatus{Dirty: resp.Status.Dirty}, nil +} + +// withJSONBody returns a request visitor that sends payload as the request body. +// +// The inspect RPC is a GET that carries its arguments in a JSON body, which the SDK cannot +// express: for GET it serializes the request value into the query string and sends an empty +// body (see makeRequestBody in databricks-sdk-go/httpclient/request.go). Visitors run against +// a freshly built request on every attempt, so installing the body here is retry-safe. +func withJSONBody(payload []byte) func(*http.Request) error { + return func(r *http.Request) error { + r.Body = io.NopCloser(bytes.NewReader(payload)) + r.ContentLength = int64(len(payload)) + return nil + } } func (c *SnapshotClient) Get(ctx context.Context, snapshotRelativePath string) (*SnapshotInfo, error) { diff --git a/libs/testserver/fake_workspace.go b/libs/testserver/fake_workspace.go index a27d2c4399f..f3e7d99e25f 100644 --- a/libs/testserver/fake_workspace.go +++ b/libs/testserver/fake_workspace.go @@ -189,7 +189,7 @@ type FakeWorkspace struct { repoIdByPath map[string]int64 // snapshots records immutable-snapshot content paths and whether each was modified out - // of band ("break-glassed"). A workspace write into a recorded path flips it dirty; + // of band. A workspace write into a recorded path flips it dirty; // RecordSnapshot resets it when the snapshot is (re)created by the snapshot API. snapshots map[string]bool diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go index f6ed2809bb0..1344b18b6f7 100644 --- a/libs/testserver/handlers.go +++ b/libs/testserver/handlers.go @@ -764,28 +764,20 @@ func AddDefaultHandlers(server *Server) { } }) - // GraphQL gateway. The only query modeled is projectsInspectSnapshot, which reports the - // break-glass status (dirty) of a snapshot content path. - server.Handle("POST", "/api/2.0/graphql", func(req Request) any { + // Reports whether a snapshot's content was modified out of band. This is a GET that takes + // its argument in a JSON body, matching the real API. + server.Handle("GET", "/api/2.0/snapshots:inspect", func(req Request) any { var body struct { - Variables struct { - Path string `json:"path"` - } `json:"variables"` + SnapshotContentPath string `json:"snapshot_content_path"` } if err := json.Unmarshal(req.Body, &body); err != nil { return Response{StatusCode: http.StatusBadRequest} } - contentPath := body.Variables.Path return map[string]any{ - "data": map[string]any{ - "projectsInspectSnapshot": map[string]any{ - "status": map[string]any{ - "snapshotContentPath": contentPath, - "dirty": req.Workspace.SnapshotDirty(contentPath), - "permissions": []string{}, - }, - "apiError": nil, - }, + "status": map[string]any{ + "snapshot_content_path": body.SnapshotContentPath, + "dirty": req.Workspace.SnapshotDirty(body.SnapshotContentPath), + "permissions": []string{"SNAPSHOT_PERMISSION_CAN_BREAK_GLASS"}, }, } }) From 23db79566c8f039f9ce676aa07e181d332298121 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Mon, 28 Sep 2026 12:10:46 +0200 Subject: [PATCH 05/16] use rest api and set can_manage --- .../databricks.yml.tmpl | 19 +++++++ .../immutable-legacy-state/out.test.toml | 3 ++ .../deploy/immutable-legacy-state/output.txt | 35 ++++++++++++ .../deploy/immutable-legacy-state/script | 30 +++++++++++ .../deploy/immutable-legacy-state/src/main.py | 1 + .../deploy/immutable-legacy-state/test.toml | 25 +++++++++ .../databricks.yml.tmpl | 19 +++++++ .../immutable-missing-snapshot/out.test.toml | 3 ++ .../immutable-missing-snapshot/output.txt | 21 ++++++++ .../deploy/immutable-missing-snapshot/script | 20 +++++++ .../immutable-missing-snapshot/src/main.py | 1 + .../immutable-missing-snapshot/test.toml | 19 +++++++ .../deploy/immutable-no-artifacts/output.txt | 19 +++++++ acceptance/bundle/deploy/immutable/output.txt | 24 +++++++++ bundle/deploy/snapshot/upload.go | 38 +++++++++---- bundle/deploy/snapshot/upload_warning_test.go | 53 +++++++++++++------ libs/snapshot/client.go | 40 +++----------- libs/testserver/fake_workspace.go | 9 ++-- libs/testserver/handlers.go | 24 +++++---- 19 files changed, 332 insertions(+), 71 deletions(-) create mode 100644 acceptance/bundle/deploy/immutable-legacy-state/databricks.yml.tmpl create mode 100644 acceptance/bundle/deploy/immutable-legacy-state/out.test.toml create mode 100644 acceptance/bundle/deploy/immutable-legacy-state/output.txt create mode 100644 acceptance/bundle/deploy/immutable-legacy-state/script create mode 100644 acceptance/bundle/deploy/immutable-legacy-state/src/main.py create mode 100644 acceptance/bundle/deploy/immutable-legacy-state/test.toml create mode 100644 acceptance/bundle/deploy/immutable-missing-snapshot/databricks.yml.tmpl create mode 100644 acceptance/bundle/deploy/immutable-missing-snapshot/out.test.toml create mode 100644 acceptance/bundle/deploy/immutable-missing-snapshot/output.txt create mode 100644 acceptance/bundle/deploy/immutable-missing-snapshot/script create mode 100644 acceptance/bundle/deploy/immutable-missing-snapshot/src/main.py create mode 100644 acceptance/bundle/deploy/immutable-missing-snapshot/test.toml diff --git a/acceptance/bundle/deploy/immutable-legacy-state/databricks.yml.tmpl b/acceptance/bundle/deploy/immutable-legacy-state/databricks.yml.tmpl new file mode 100644 index 00000000000..202edfebf32 --- /dev/null +++ b/acceptance/bundle/deploy/immutable-legacy-state/databricks.yml.tmpl @@ -0,0 +1,19 @@ +bundle: + name: test-bundle-immutable-legacy-$UNIQUE_NAME + +experimental: + immutable_folder: true + +resources: + jobs: + my_job: + name: my job + tasks: + - task_key: spark_python_task + spark_python_task: + python_file: ./src/main.py + environment_key: env + environments: + - environment_key: env + spec: + environment_version: "4" diff --git a/acceptance/bundle/deploy/immutable-legacy-state/out.test.toml b/acceptance/bundle/deploy/immutable-legacy-state/out.test.toml new file mode 100644 index 00000000000..27ec2a7fcd6 --- /dev/null +++ b/acceptance/bundle/deploy/immutable-legacy-state/out.test.toml @@ -0,0 +1,3 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/deploy/immutable-legacy-state/output.txt b/acceptance/bundle/deploy/immutable-legacy-state/output.txt new file mode 100644 index 00000000000..103e796c576 --- /dev/null +++ b/acceptance/bundle/deploy/immutable-legacy-state/output.txt @@ -0,0 +1,35 @@ + +=== Deploy an immutable snapshot +>>> [CLI] bundle deploy +Created internal_immutable_snapshots.immutable +Created jobs.my_job +Files: 0 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +=== Rewrite the state the way a CLI without this feature wrote it +>>> update_file.py .databricks/bundle/default/resources.json /snapshot" " + +=== Deploying on top of that state succeeds and moves to the current layout +>>> [CLI] bundle deploy +Files: 0 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 2 unchanged +Snapshot path: /Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]/snapshot/files/src/main.py +OK: deployed to the current snapshot layout + +=== The check still applies to a deployment carried over from the older layout +>>> [CLI] workspace mkdirs /Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]/snapshot/tampered + +>>> musterr [CLI] bundle deploy +Error: the immutable snapshot of the last deployment was modified outside of the bundle (break glass): + /Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]/snapshot +To deploy a new snapshot and point resources at it, use --force + +Files: 0 uploaded, 0 deleted + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.my_job + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-immutable-legacy-[UNIQUE_NAME]/default + +Destroy: 1 deleted diff --git a/acceptance/bundle/deploy/immutable-legacy-state/script b/acceptance/bundle/deploy/immutable-legacy-state/script new file mode 100644 index 00000000000..2ba1ca06e76 --- /dev/null +++ b/acceptance/bundle/deploy/immutable-legacy-state/script @@ -0,0 +1,30 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve +} +trap cleanup EXIT + +title "Deploy an immutable snapshot" +trace $CLI bundle deploy + +title "Rewrite the state the way a CLI without this feature wrote it" +# Back then full_path had no trailing "/snapshot" segment. It is the only value ending in it, so +# stripping the segment together with the closing quote leaves the rest of the state untouched. +trace update_file.py .databricks/bundle/default/resources.json '/snapshot"' '"' + +title "Deploying on top of that state succeeds and moves to the current layout" +trace $CLI bundle deploy + +JOB_ID=$($CLI bundle summary -o json | jq -r '.resources.jobs.my_job.id') +SNAPSHOT_PATH=$($CLI jobs get "$JOB_ID" | jq -r '.settings.tasks[0].spark_python_task.python_file') +echo "Snapshot path: $SNAPSHOT_PATH" +case "$SNAPSHOT_PATH" in + */snapshot/files/src/main.py) echo "OK: deployed to the current snapshot layout" ;; + *) echo "ERROR: unexpected snapshot path"; exit 1 ;; +esac + +title "The check still applies to a deployment carried over from the older layout" +CONTENT_PATH="${SNAPSHOT_PATH%/files/src/main.py}" +trace $CLI workspace mkdirs "$CONTENT_PATH/tampered" +trace musterr $CLI bundle deploy diff --git a/acceptance/bundle/deploy/immutable-legacy-state/src/main.py b/acceptance/bundle/deploy/immutable-legacy-state/src/main.py new file mode 100644 index 00000000000..6c285f7e2f5 --- /dev/null +++ b/acceptance/bundle/deploy/immutable-legacy-state/src/main.py @@ -0,0 +1 @@ +print("Hello from Spark Python Task!") diff --git a/acceptance/bundle/deploy/immutable-legacy-state/test.toml b/acceptance/bundle/deploy/immutable-legacy-state/test.toml new file mode 100644 index 00000000000..1a22649e622 --- /dev/null +++ b/acceptance/bundle/deploy/immutable-legacy-state/test.toml @@ -0,0 +1,25 @@ +Cloud = false # Temporary disable cloud tests until the API is fully available + +# immutable_folder only works with the direct engine. +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] + +# The test rewrites the deployment state by hand, which the recording service refuses because it +# no longer matches the deployment it has recorded, so run without deployment history. +EnvMatrix.DMS = [""] + +Ignore = [ + "databricks.yml", + ".databricks", +] + +# This test passes the snapshot's absolute workspace path to "workspace mkdirs" to modify it out +# of band. On Windows the script runs under MSYS2, which rewrites such leading-slash arguments to +# Windows paths (e.g. C:/Program Files/Git/Workspace/...) before the CLI sees them, so the write +# would land outside the snapshot and leave it looking untouched. +# Setting this environment variable disables that conversion. +Env.MSYS_NO_PATHCONV = "1" + +[[Repls]] +# Replace snapshot hash with SNAPSHOT_HASH +Old = "[0-9a-f]{64}" +New = "[SNAPSHOT_HASH]" diff --git a/acceptance/bundle/deploy/immutable-missing-snapshot/databricks.yml.tmpl b/acceptance/bundle/deploy/immutable-missing-snapshot/databricks.yml.tmpl new file mode 100644 index 00000000000..e44ed8ec436 --- /dev/null +++ b/acceptance/bundle/deploy/immutable-missing-snapshot/databricks.yml.tmpl @@ -0,0 +1,19 @@ +bundle: + name: test-bundle-immutable-missing-$UNIQUE_NAME + +experimental: + immutable_folder: true + +resources: + jobs: + my_job: + name: my job + tasks: + - task_key: spark_python_task + spark_python_task: + python_file: ./src/main.py + environment_key: env + environments: + - environment_key: env + spec: + environment_version: "4" diff --git a/acceptance/bundle/deploy/immutable-missing-snapshot/out.test.toml b/acceptance/bundle/deploy/immutable-missing-snapshot/out.test.toml new file mode 100644 index 00000000000..27ec2a7fcd6 --- /dev/null +++ b/acceptance/bundle/deploy/immutable-missing-snapshot/out.test.toml @@ -0,0 +1,3 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/deploy/immutable-missing-snapshot/output.txt b/acceptance/bundle/deploy/immutable-missing-snapshot/output.txt new file mode 100644 index 00000000000..a6c2ea78641 --- /dev/null +++ b/acceptance/bundle/deploy/immutable-missing-snapshot/output.txt @@ -0,0 +1,21 @@ + +=== Deploy an immutable snapshot +>>> [CLI] bundle deploy +Created internal_immutable_snapshots.immutable +Created jobs.my_job +Files: 0 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +=== Point the state at a snapshot that is not there +=== Deploying still works instead of failing on the snapshot that is not there +>>> [CLI] bundle deploy +Files: 0 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 2 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.my_job + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-immutable-missing-[UNIQUE_NAME]/default + +Destroy: 1 deleted diff --git a/acceptance/bundle/deploy/immutable-missing-snapshot/script b/acceptance/bundle/deploy/immutable-missing-snapshot/script new file mode 100644 index 00000000000..d1a54836020 --- /dev/null +++ b/acceptance/bundle/deploy/immutable-missing-snapshot/script @@ -0,0 +1,20 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve +} +trap cleanup EXIT + +title "Deploy an immutable snapshot" +trace $CLI bundle deploy + +title "Point the state at a snapshot that is not there" +# This is what a deployment made by a CLI without this feature looks like: its content sits one +# level above the content subfolder, so nothing is stored where the snapshot is looked up now. +# The same shape arises when the snapshot folder is deleted out of band. +STATE=.databricks/bundle/default/resources.json +jq '.state["resources.internal_immutable_snapshots.immutable"].state.relative_path += "-missing"' "$STATE" > tmp.json +mv tmp.json "$STATE" + +title "Deploying still works instead of failing on the snapshot that is not there" +trace $CLI bundle deploy diff --git a/acceptance/bundle/deploy/immutable-missing-snapshot/src/main.py b/acceptance/bundle/deploy/immutable-missing-snapshot/src/main.py new file mode 100644 index 00000000000..6c285f7e2f5 --- /dev/null +++ b/acceptance/bundle/deploy/immutable-missing-snapshot/src/main.py @@ -0,0 +1 @@ +print("Hello from Spark Python Task!") diff --git a/acceptance/bundle/deploy/immutable-missing-snapshot/test.toml b/acceptance/bundle/deploy/immutable-missing-snapshot/test.toml new file mode 100644 index 00000000000..c191a87d736 --- /dev/null +++ b/acceptance/bundle/deploy/immutable-missing-snapshot/test.toml @@ -0,0 +1,19 @@ +Cloud = false # Temporary disable cloud tests until the API is fully available + +# immutable_folder only works with the direct engine. +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] + +# The test rewrites the deployment state by hand, which the recording service refuses because it +# no longer matches the deployment it has recorded, so run without deployment history. +EnvMatrix.DMS = [""] + +Ignore = [ + "databricks.yml", + ".databricks", + "tmp.*", +] + +[[Repls]] +# Replace snapshot hash with SNAPSHOT_HASH +Old = "[0-9a-f]{64}" +New = "[SNAPSHOT_HASH]" diff --git a/acceptance/bundle/deploy/immutable-no-artifacts/output.txt b/acceptance/bundle/deploy/immutable-no-artifacts/output.txt index 023c77bbe3c..60b2c41d7d9 100644 --- a/acceptance/bundle/deploy/immutable-no-artifacts/output.txt +++ b/acceptance/bundle/deploy/immutable-no-artifacts/output.txt @@ -44,6 +44,11 @@ Validation OK! "permission_level": "CAN_READ" } ], + "can_manage": [ + { + "user_name": "[USERNAME]" + } + ], "zip_path": "[TEST_TMP_DIR]/.databricks/bundle/default/snapshots/[SNAPSHOT_HASH].zip" } } @@ -101,6 +106,20 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "new": "[SNAPSHOT_HASH]", "remote": "" }, + "can_manage": { + "action": "skip", + "reason": "missing_in_remote", + "old": [ + { + "user_name": "[USERNAME]" + } + ], + "new": [ + { + "user_name": "[USERNAME]" + } + ] + }, "zip_path": { "action": "skip", "reason": "missing_in_remote", diff --git a/acceptance/bundle/deploy/immutable/output.txt b/acceptance/bundle/deploy/immutable/output.txt index 26c9339e605..acee49af72a 100644 --- a/acceptance/bundle/deploy/immutable/output.txt +++ b/acceptance/bundle/deploy/immutable/output.txt @@ -63,6 +63,11 @@ Building python_artifact... "permission_level": "CAN_READ" } ], + "can_manage": [ + { + "user_name": "[USERNAME]" + } + ], "zip_path": "[TEST_TMP_DIR]/.databricks/bundle/default/snapshots/[SNAPSHOT_HASH].zip" } } @@ -106,6 +111,11 @@ Building python_artifact... "permission_level": "CAN_READ" } ], + "can_manage": [ + { + "user_name": "[USERNAME]" + } + ], "zip_path": "[TEST_TMP_DIR]/.databricks/bundle/default/snapshots/[SNAPSHOT_HASH].zip" } }, @@ -138,6 +148,20 @@ Building python_artifact... "new": "[SNAPSHOT_HASH]", "remote": "" }, + "can_manage": { + "action": "skip", + "reason": "missing_in_remote", + "old": [ + { + "user_name": "[USERNAME]" + } + ], + "new": [ + { + "user_name": "[USERNAME]" + } + ] + }, "full_path": { "action": "recreate", "reason": "immutable", diff --git a/bundle/deploy/snapshot/upload.go b/bundle/deploy/snapshot/upload.go index 416ffdaf821..ef7bc7a9ea1 100644 --- a/bundle/deploy/snapshot/upload.go +++ b/bundle/deploy/snapshot/upload.go @@ -5,8 +5,10 @@ import ( "crypto/sha256" "encoding/hex" "encoding/json" + "errors" "fmt" "os" + "path" "path/filepath" "github.com/databricks/cli/bundle" @@ -14,6 +16,7 @@ import ( "github.com/databricks/cli/bundle/direct/dresources" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/snapshot" + "github.com/databricks/databricks-sdk-go/apierr" ) // fileLimitWarning is the file count above which immutable folder deployments may fail. @@ -73,7 +76,7 @@ func (m *snapshotUpload) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagn // Check the previous snapshot before doing any work, so a deploy onto a snapshot that was // modified outside the bundle fails before building a zip. - generation, err := resolveGeneration(ctx, b, uploader) + generation, err := resolveGeneration(ctx, b, uploader, remoteRoot) if err != nil { return diag.FromErr(err) } @@ -146,12 +149,17 @@ func BuildACL(b *bundle.Bundle) []snapshot.ACLEntry { return acl } -// BuildCanManage constructs can_manage_principals for the snapshot upload: every principal -// granted CAN_MANAGE in the top-level permissions section may break the glass on the snapshot. +// BuildCanManage constructs can_manage_principals for the snapshot upload: the principals that +// may break the glass on the snapshot. Those are the deploying identity plus everyone granted +// CAN_MANAGE in the top-level permissions section. The deploying identity is always included +// because the API rejects an empty list, and it is the one principal always available. The +// recommended permissions section names it with CAN_MANAGE as well, so skip it in the loop +// rather than sending it twice. func BuildCanManage(b *bundle.Bundle) []snapshot.ManagePrincipal { - var canManage []snapshot.ManagePrincipal + currentUser := b.Config.Workspace.CurrentUser.UserName + canManage := []snapshot.ManagePrincipal{{UserName: currentUser}} for _, p := range b.Config.Permissions { - if p.Level != "CAN_MANAGE" { + if p.Level != "CAN_MANAGE" || p.UserName == currentUser { continue } canManage = append(canManage, snapshot.ManagePrincipal{ @@ -178,7 +186,7 @@ func BuildCanManage(b *bundle.Bundle) []snapshot.ManagePrincipal { // same shape: read state, ask the API, refuse unless forced. // // On the first deploy (no snapshot in state) the generation is 0. -func resolveGeneration(ctx context.Context, b *bundle.Bundle, uploader *snapshot.SnapshotClient) (int, error) { +func resolveGeneration(ctx context.Context, b *bundle.Bundle, uploader *snapshot.SnapshotClient, remoteRoot string) (int, error) { // The deploy/plan pipeline opens the state DB for read before this runs, but callers that // exercise PlanUpload in isolation (unit tests) may not. No open state means no previous // snapshot to check, which is the first-deploy case. @@ -196,11 +204,23 @@ func resolveGeneration(ctx context.Context, b *bundle.Bundle, uploader *snapshot if err := json.Unmarshal(entry.State, &prev); err != nil { return 0, fmt.Errorf("reading previous snapshot state: %w", err) } - if prev.FullPath == "" { + if prev.RelativePath == "" { return 0, nil } - status, err := uploader.InspectSnapshot(ctx, prev.FullPath) + // Compose the content path from relative_path rather than reading full_path: relative_path + // already encodes the previous generation and means the same thing in every CLI version, + // while full_path from a CLI that predates the content subfolder points one level too high. + contentPath := path.Join(remoteRoot, prev.RelativePath, snapshot.ContentSubdir) + + status, err := uploader.InspectSnapshot(ctx, contentPath) + // Nothing to protect when no snapshot is there: the deployment predates the content + // subfolder, or the snapshot was deleted out of band, or an earlier deploy never created it. + // Failing here would also be a dead end, since this runs before the --force check and so + // could not be forced past. + if errors.Is(err, apierr.ErrNotFound) { + return prev.Generation, nil + } if err != nil { return 0, err } @@ -208,7 +228,7 @@ func resolveGeneration(ctx context.Context, b *bundle.Bundle, uploader *snapshot return prev.Generation, nil } if !b.Config.Bundle.Force { - return 0, fmt.Errorf("the immutable snapshot of the last deployment was modified outside of the bundle (break glass):\n %s\nTo deploy a new snapshot and point resources at it, use --force", prev.FullPath) + return 0, fmt.Errorf("the immutable snapshot of the last deployment was modified outside of the bundle (break glass):\n %s\nTo deploy a new snapshot and point resources at it, use --force", contentPath) } return prev.Generation + 1, nil } diff --git a/bundle/deploy/snapshot/upload_warning_test.go b/bundle/deploy/snapshot/upload_warning_test.go index cc18c728274..2ef7db23d59 100644 --- a/bundle/deploy/snapshot/upload_warning_test.go +++ b/bundle/deploy/snapshot/upload_warning_test.go @@ -92,20 +92,31 @@ func TestUploadNoWarningBelowFileLimit(t *testing.T) { assert.True(t, diags.HasError() == false && len(diags) == 0, "expected no diagnostics") } -func TestBuildCanManage(t *testing.T) { - b := &bundle.Bundle{ +// canManageBundle returns a bundle deployed by deployer@example.test with the given permissions. +func canManageBundle(permissions []resources.Permission) *bundle.Bundle { + return &bundle.Bundle{ Config: config.Root{ - Permissions: []resources.Permission{ - {Level: "CAN_MANAGE", UserName: "manager@example.test"}, - {Level: "CAN_VIEW", UserName: "viewer@example.test"}, - {Level: "CAN_MANAGE", GroupName: "admins"}, - {Level: "CAN_MANAGE", ServicePrincipalName: "sp-1"}, + Workspace: config.Workspace{ + CurrentUser: &config.User{ + User: &iam.User{UserName: "deployer@example.test"}, + }, }, + Permissions: permissions, }, } +} - // Only CAN_MANAGE principals may break the glass on the snapshot. +func TestBuildCanManage(t *testing.T) { + b := canManageBundle([]resources.Permission{ + {Level: "CAN_MANAGE", UserName: "manager@example.test"}, + {Level: "CAN_VIEW", UserName: "viewer@example.test"}, + {Level: "CAN_MANAGE", GroupName: "admins"}, + {Level: "CAN_MANAGE", ServicePrincipalName: "sp-1"}, + }) + + // The deploying identity, then only the CAN_MANAGE principals. assert.Equal(t, []snapshot.ManagePrincipal{ + {UserName: "deployer@example.test"}, {UserName: "manager@example.test"}, {GroupName: "admins"}, {ServicePrincipalName: "sp-1"}, @@ -113,14 +124,24 @@ func TestBuildCanManage(t *testing.T) { } func TestBuildCanManageWithoutManagers(t *testing.T) { - b := &bundle.Bundle{ - Config: config.Root{ - Permissions: []resources.Permission{ - {Level: "CAN_VIEW", UserName: "viewer@example.test"}, - }, - }, - } - assert.Empty(t, BuildCanManage(b)) + b := canManageBundle([]resources.Permission{ + {Level: "CAN_VIEW", UserName: "viewer@example.test"}, + }) + + // The API rejects an empty list, so the deploying identity is always there. + assert.Equal(t, []snapshot.ManagePrincipal{ + {UserName: "deployer@example.test"}, + }, BuildCanManage(b)) +} + +func TestBuildCanManageSkipsDuplicateDeployer(t *testing.T) { + b := canManageBundle([]resources.Permission{ + {Level: "CAN_MANAGE", UserName: "deployer@example.test"}, + }) + + assert.Equal(t, []snapshot.ManagePrincipal{ + {UserName: "deployer@example.test"}, + }, BuildCanManage(b)) } func TestUploadReusesStagedZipWhenNotClean(t *testing.T) { diff --git a/libs/snapshot/client.go b/libs/snapshot/client.go index 1a64baa32de..1fd05b3e708 100644 --- a/libs/snapshot/client.go +++ b/libs/snapshot/client.go @@ -6,7 +6,6 @@ import ( "encoding/json" "errors" "fmt" - "io" "mime/multipart" "net/http" "net/textproto" @@ -72,9 +71,11 @@ type snapshotRootPathResponse struct { // inspectSnapshotPath reports whether a snapshot's content was modified out of band. const inspectSnapshotPath = "/api/2.0/snapshots:inspect" -// inspectSnapshotRequest is the body of an inspect call. +// inspectSnapshotRequest carries the arguments of an inspect call. They travel in the query +// string: the endpoint is a GET, and the SDK builds a GET's query string from the request +// value's `url` tags. type inspectSnapshotRequest struct { - SnapshotContentPath string `json:"snapshot_content_path"` + SnapshotContentPath string `url:"snapshot_content_path"` } // inspectSnapshotResponse mirrors the /api/2.0/snapshots:inspect response body. The @@ -98,7 +99,7 @@ func NewSnapshotClient(w *databricks.WorkspaceClient) (*SnapshotClient, error) { // Upload uploads zipContent as an immutable snapshot at relativePath. The server stores the // content under "//snapshot" and returns that content path. acl grants // CAN_READ to each listed principal; canManage lists the principals allowed to break the -// glass on the snapshot. +// glass on the snapshot and must hold at least one, which the API requires. func (c *SnapshotClient) Upload(ctx context.Context, relativePath string, acl []ACLEntry, canManage []ManagePrincipal, zipContent []byte) (*SnapshotInfo, error) { var body bytes.Buffer mw := multipart.NewWriter(&body) @@ -115,10 +116,6 @@ func (c *SnapshotClient) Upload(ctx context.Context, relativePath string, acl [] return nil, fmt.Errorf("failed to write access_control_list: %w", err) } - // Always send an array, never null, so "no break-glass managers" is unambiguous. - if canManage == nil { - canManage = []ManagePrincipal{} - } canManageJSON, err := json.Marshal(canManage) if err != nil { return nil, fmt.Errorf("failed to marshal can_manage_principals: %w", err) @@ -162,19 +159,10 @@ func (c *SnapshotClient) Upload(ctx context.Context, relativePath string, acl [] // InspectSnapshot reports the status of the snapshot content at contentPath, which tells the // caller whether the immutable content was modified out of band. func (c *SnapshotClient) InspectSnapshot(ctx context.Context, contentPath string) (*SnapshotStatus, error) { - headers := auth.WorkspaceIDHeaders(c.client.Config) - if headers == nil { - headers = make(map[string]string) - } - headers["Content-Type"] = "application/json" - - payload, err := json.Marshal(inspectSnapshotRequest{SnapshotContentPath: contentPath}) - if err != nil { - return nil, fmt.Errorf("failed to marshal inspect request: %w", err) - } + req := inspectSnapshotRequest{SnapshotContentPath: contentPath} var resp inspectSnapshotResponse - err = c.client.Do(ctx, http.MethodGet, inspectSnapshotPath, headers, nil, nil, &resp, withJSONBody(payload)) + err := c.client.Do(ctx, http.MethodGet, inspectSnapshotPath, auth.WorkspaceIDHeaders(c.client.Config), nil, req, &resp) if err != nil { return nil, fmt.Errorf("snapshot inspect: %w", err) } @@ -187,20 +175,6 @@ func (c *SnapshotClient) InspectSnapshot(ctx context.Context, contentPath string return &SnapshotStatus{Dirty: resp.Status.Dirty}, nil } -// withJSONBody returns a request visitor that sends payload as the request body. -// -// The inspect RPC is a GET that carries its arguments in a JSON body, which the SDK cannot -// express: for GET it serializes the request value into the query string and sends an empty -// body (see makeRequestBody in databricks-sdk-go/httpclient/request.go). Visitors run against -// a freshly built request on every attempt, so installing the body here is retry-safe. -func withJSONBody(payload []byte) func(*http.Request) error { - return func(r *http.Request) error { - r.Body = io.NopCloser(bytes.NewReader(payload)) - r.ContentLength = int64(len(payload)) - return nil - } -} - func (c *SnapshotClient) Get(ctx context.Context, snapshotRelativePath string) (*SnapshotInfo, error) { rootPath, err := c.GetSnapshotRootPath(ctx) if err != nil { diff --git a/libs/testserver/fake_workspace.go b/libs/testserver/fake_workspace.go index f3e7d99e25f..00f88113982 100644 --- a/libs/testserver/fake_workspace.go +++ b/libs/testserver/fake_workspace.go @@ -818,11 +818,12 @@ func (s *FakeWorkspace) RecordSnapshot(contentPath string) { s.snapshots[contentPath] = false } -// SnapshotDirty reports whether the immutable snapshot at contentPath was modified out of -// band. An unknown path is reported clean. -func (s *FakeWorkspace) SnapshotDirty(contentPath string) bool { +// SnapshotDirty reports whether the immutable snapshot at contentPath was modified out of band, +// and whether a snapshot is recorded there at all. +func (s *FakeWorkspace) SnapshotDirty(contentPath string) (dirty, ok bool) { defer s.LockUnlock()() - return s.snapshots[contentPath] + dirty, ok = s.snapshots[contentPath] + return dirty, ok } // markSnapshotDirty flips any recorded snapshot dirty when targetPath writes at or under its diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go index 1344b18b6f7..92299f76b80 100644 --- a/libs/testserver/handlers.go +++ b/libs/testserver/handlers.go @@ -764,19 +764,25 @@ func AddDefaultHandlers(server *Server) { } }) - // Reports whether a snapshot's content was modified out of band. This is a GET that takes - // its argument in a JSON body, matching the real API. + // Reports whether a snapshot's content was modified out of band. server.Handle("GET", "/api/2.0/snapshots:inspect", func(req Request) any { - var body struct { - SnapshotContentPath string `json:"snapshot_content_path"` - } - if err := json.Unmarshal(req.Body, &body); err != nil { - return Response{StatusCode: http.StatusBadRequest} + contentPath := req.URL.Query().Get("snapshot_content_path") + dirty, ok := req.Workspace.SnapshotDirty(contentPath) + // A path that holds no snapshot is not inspectable, so the API reports NOT_FOUND + // rather than calling it clean. + if !ok { + return Response{ + StatusCode: http.StatusNotFound, + Body: map[string]string{ + "error_code": "RESOURCE_DOES_NOT_EXIST", + "message": "Snapshot not found: " + contentPath, + }, + } } return map[string]any{ "status": map[string]any{ - "snapshot_content_path": body.SnapshotContentPath, - "dirty": req.Workspace.SnapshotDirty(body.SnapshotContentPath), + "snapshot_content_path": contentPath, + "dirty": dirty, "permissions": []string{"SNAPSHOT_PERMISSION_CAN_BREAK_GLASS"}, }, } From 7d282102b64fd0a435763c5d835da5151dee959c Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Mon, 28 Sep 2026 17:30:16 +0200 Subject: [PATCH 06/16] fix test + include metadata --- .../deploy/immutable-legacy-state/output.txt | 2 -- .../deploy/immutable-legacy-state/script | 7 ++++--- bundle/deploy/metadata/compute.go | 7 +++++++ bundle/deploy/metadata/compute_test.go | 21 +++++++++++++++++++ bundle/metadata/metadata.go | 4 ++++ 5 files changed, 36 insertions(+), 5 deletions(-) diff --git a/acceptance/bundle/deploy/immutable-legacy-state/output.txt b/acceptance/bundle/deploy/immutable-legacy-state/output.txt index 103e796c576..e06269208cf 100644 --- a/acceptance/bundle/deploy/immutable-legacy-state/output.txt +++ b/acceptance/bundle/deploy/immutable-legacy-state/output.txt @@ -7,8 +7,6 @@ Files: 0 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged === Rewrite the state the way a CLI without this feature wrote it ->>> update_file.py .databricks/bundle/default/resources.json /snapshot" " - === Deploying on top of that state succeeds and moves to the current layout >>> [CLI] bundle deploy Files: 0 uploaded, 0 deleted diff --git a/acceptance/bundle/deploy/immutable-legacy-state/script b/acceptance/bundle/deploy/immutable-legacy-state/script index 2ba1ca06e76..9d0d47653d2 100644 --- a/acceptance/bundle/deploy/immutable-legacy-state/script +++ b/acceptance/bundle/deploy/immutable-legacy-state/script @@ -9,9 +9,10 @@ title "Deploy an immutable snapshot" trace $CLI bundle deploy title "Rewrite the state the way a CLI without this feature wrote it" -# Back then full_path had no trailing "/snapshot" segment. It is the only value ending in it, so -# stripping the segment together with the closing quote leaves the rest of the state untouched. -trace update_file.py .databricks/bundle/default/resources.json '/snapshot"' '"' +# Back then full_path had no trailing "/snapshot" segment. +STATE=.databricks/bundle/default/resources.json +jq '.state["resources.internal_immutable_snapshots.immutable"].state.full_path |= rtrimstr("/snapshot")' "$STATE" > tmp.json +mv tmp.json "$STATE" title "Deploying on top of that state succeeds and moves to the current layout" trace $CLI bundle deploy diff --git a/bundle/deploy/metadata/compute.go b/bundle/deploy/metadata/compute.go index 1fa4c3bf2d9..61b7029f35c 100644 --- a/bundle/deploy/metadata/compute.go +++ b/bundle/deploy/metadata/compute.go @@ -7,6 +7,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/metadata" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/log" @@ -127,6 +128,12 @@ func (m *compute) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics b.Metadata.Config.Presets.SourceLinkedDeployment = true } + // For immutable-folder deployments the content lives in a snapshot rather than a regular + // workspace directory. Record its path so downstream services (e.g. the jobs UI) can locate it. + if snap, ok := b.Config.Resources.Snapshots[resources.SnapshotResourceKey]; ok { + b.Metadata.Config.Workspace.SnapshotContentPath = snap.FullPath() + } + // Set the git folder path for deployments from the workspace if b.WorktreeRoot != nil && strings.HasPrefix(b.WorktreeRoot.Native(), "/Workspace/") { b.Metadata.Extra.GitFolderPath = b.WorktreeRoot.Native() diff --git a/bundle/deploy/metadata/compute_test.go b/bundle/deploy/metadata/compute_test.go index 58b8b390d35..74da859bbd9 100644 --- a/bundle/deploy/metadata/compute_test.go +++ b/bundle/deploy/metadata/compute_test.go @@ -221,3 +221,24 @@ func TestComputeMetadataMutatorGitFolderPath(t *testing.T) { assert.Equal(t, gitFolderPath, b.Metadata.Extra.GitFolderPath) } + +func TestComputeMetadataMutatorSnapshotContentPath(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + Snapshots: map[string]*resources.Snapshot{ + resources.SnapshotResourceKey: { + RemoteRoot: "/Workspace/Users/test@databricks.com/.snapshots", + BundleID: "bundle-id", + ZipPath: "/tmp/content-hash.zip", + }, + }, + }, + }, + } + + diags := bundle.Apply(t.Context(), b, Compute()) + require.NoError(t, diags.Error()) + + assert.Equal(t, "/Workspace/Users/test@databricks.com/.snapshots/bundle-id/content-hash/snapshot", b.Metadata.Config.Workspace.SnapshotContentPath) +} diff --git a/bundle/metadata/metadata.go b/bundle/metadata/metadata.go index 1c61cb093f0..16d2ba00901 100644 --- a/bundle/metadata/metadata.go +++ b/bundle/metadata/metadata.go @@ -15,6 +15,10 @@ type Bundle struct { type Workspace struct { FilePath string `json:"file_path"` + + // SnapshotContentPath is the workspace path of the immutable snapshot, present only when + // experimental.immutable_folder is true. The jobs UI uses it to display the file origin. + SnapshotContentPath string `json:"snapshot_content_path,omitempty"` } type Resource struct { From 512d62091887c74da5404f4d8fa15b5a3e6af3cd Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Mon, 28 Sep 2026 18:07:29 +0200 Subject: [PATCH 07/16] check that we send can_snapshot_manage --- .../immutable-can-manage/databricks.yml.tmpl | 23 ++++++++ .../deploy/immutable-can-manage/out.test.toml | 3 ++ .../deploy/immutable-can-manage/output.txt | 53 ++++++++++++++++++ .../bundle/deploy/immutable-can-manage/script | 20 +++++++ .../deploy/immutable-can-manage/src/main.py | 1 + .../deploy/immutable-can-manage/test.toml | 21 ++++++++ bundle/deploy/snapshot/upload.go | 16 ++++-- libs/testserver/fake_workspace.go | 54 ++++++++++++++----- libs/testserver/handlers.go | 29 +++++++++- 9 files changed, 201 insertions(+), 19 deletions(-) create mode 100644 acceptance/bundle/deploy/immutable-can-manage/databricks.yml.tmpl create mode 100644 acceptance/bundle/deploy/immutable-can-manage/out.test.toml create mode 100644 acceptance/bundle/deploy/immutable-can-manage/output.txt create mode 100644 acceptance/bundle/deploy/immutable-can-manage/script create mode 100644 acceptance/bundle/deploy/immutable-can-manage/src/main.py create mode 100644 acceptance/bundle/deploy/immutable-can-manage/test.toml diff --git a/acceptance/bundle/deploy/immutable-can-manage/databricks.yml.tmpl b/acceptance/bundle/deploy/immutable-can-manage/databricks.yml.tmpl new file mode 100644 index 00000000000..a06b94c3532 --- /dev/null +++ b/acceptance/bundle/deploy/immutable-can-manage/databricks.yml.tmpl @@ -0,0 +1,23 @@ +bundle: + name: test-bundle-immutable-can-manage-$UNIQUE_NAME + +experimental: + immutable_folder: true + +permissions: + - level: CAN_MANAGE + user_name: external-manager@databricks.com + +resources: + jobs: + my_job: + name: my job + tasks: + - task_key: spark_python_task + spark_python_task: + python_file: ./src/main.py + environment_key: env + environments: + - environment_key: env + spec: + environment_version: "4" diff --git a/acceptance/bundle/deploy/immutable-can-manage/out.test.toml b/acceptance/bundle/deploy/immutable-can-manage/out.test.toml new file mode 100644 index 00000000000..59b56a2037c --- /dev/null +++ b/acceptance/bundle/deploy/immutable-can-manage/out.test.toml @@ -0,0 +1,3 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/deploy/immutable-can-manage/output.txt b/acceptance/bundle/deploy/immutable-can-manage/output.txt new file mode 100644 index 00000000000..4870dc653bc --- /dev/null +++ b/acceptance/bundle/deploy/immutable-can-manage/output.txt @@ -0,0 +1,53 @@ + +=== Deploy with a CAN_MANAGE permission entry +>>> [CLI] bundle deploy +Recommendation: permissions section should explicitly include the current deployment identity '[USERNAME]' or one of its groups +If it is not included, CAN_MANAGE permissions are only applied if the present identity is used to deploy. + +Consider using a adding a top-level permissions section such as the following: + + permissions: + - user_name: [USERNAME] + level: CAN_MANAGE + +See https://docs.databricks.com/dev-tools/bundles/permissions.html to learn more about permission configuration. + in databricks.yml:8:3 + +Created internal_immutable_snapshots.immutable +Created jobs.my_job +Created jobs.my_job.permissions +Files: 0 uploaded, 0 deleted +Resources: 3 created, 0 changed, 0 deleted, 0 unchanged + +=== Check can_manage_principals sent to the snapshot create API +>>> [CLI] api get /api/2.0/snapshots/info?relative_path=[SNAPSHOT_HASH]/[SNAPSHOT_HASH] +{ + "can_manage_principals": [ + { + "user_name": "[USERNAME]" + }, + { + "user_name": "external-manager@databricks.com" + } + ] +} + +>>> [CLI] bundle destroy --auto-approve +Recommendation: permissions section should explicitly include the current deployment identity '[USERNAME]' or one of its groups +If it is not included, CAN_MANAGE permissions are only applied if the present identity is used to deploy. + +Consider using a adding a top-level permissions section such as the following: + + permissions: + - user_name: [USERNAME] + level: CAN_MANAGE + +See https://docs.databricks.com/dev-tools/bundles/permissions.html to learn more about permission configuration. + in databricks.yml:8:3 + +The following resources will be deleted: + delete resources.jobs.my_job + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-immutable-can-manage-[UNIQUE_NAME]/default + +Destroy: 1 deleted diff --git a/acceptance/bundle/deploy/immutable-can-manage/script b/acceptance/bundle/deploy/immutable-can-manage/script new file mode 100644 index 00000000000..99ab49414a1 --- /dev/null +++ b/acceptance/bundle/deploy/immutable-can-manage/script @@ -0,0 +1,20 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve +} +trap cleanup EXIT + +title "Deploy with a CAN_MANAGE permission entry" +trace $CLI bundle deploy + +title "Check can_manage_principals sent to the snapshot create API" +# The testserver stores parsed form fields per snapshot and exposes them via a query endpoint. +# The snapshot resource ID is its relative_path; read it from the post-deploy plan where the +# remote_state is populated and errors/warnings go to a log file to keep the output clean. +$CLI bundle plan -o json > plan.json 2> LOG.plan +RELATIVE_PATH=$(jq -r \ + '.plan["resources.internal_immutable_snapshots.immutable"].remote_state.relative_path' \ + plan.json) +trace $CLI api get "/api/2.0/snapshots/info?relative_path=$RELATIVE_PATH" | \ + jq '{can_manage_principals: .can_manage_principals}' diff --git a/acceptance/bundle/deploy/immutable-can-manage/src/main.py b/acceptance/bundle/deploy/immutable-can-manage/src/main.py new file mode 100644 index 00000000000..6c285f7e2f5 --- /dev/null +++ b/acceptance/bundle/deploy/immutable-can-manage/src/main.py @@ -0,0 +1 @@ +print("Hello from Spark Python Task!") diff --git a/acceptance/bundle/deploy/immutable-can-manage/test.toml b/acceptance/bundle/deploy/immutable-can-manage/test.toml new file mode 100644 index 00000000000..32b9edbb62b --- /dev/null +++ b/acceptance/bundle/deploy/immutable-can-manage/test.toml @@ -0,0 +1,21 @@ +Cloud = false # Temporary disable cloud tests until the API is fully available + +# immutable_folder only works with the direct engine. +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] + +Ignore = [ + "databricks.yml", + ".databricks", + "plan.json", + "LOG.*", +] + +# The test passes an absolute workspace path to "workspace mkdirs" and also queries +# /api/2.0/snapshots/info. On Windows MSYS2 rewrites leading-slash arguments to Windows +# paths before the CLI sees them, so disable that conversion. +Env.MSYS_NO_PATHCONV = "1" + +[[Repls]] +# Replace snapshot hash with SNAPSHOT_HASH +Old = "[0-9a-f]{64}" +New = "[SNAPSHOT_HASH]" diff --git a/bundle/deploy/snapshot/upload.go b/bundle/deploy/snapshot/upload.go index ef7bc7a9ea1..de7e0c6b58e 100644 --- a/bundle/deploy/snapshot/upload.go +++ b/bundle/deploy/snapshot/upload.go @@ -15,6 +15,7 @@ import ( "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/direct/dresources" "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/iamutil" "github.com/databricks/cli/libs/snapshot" "github.com/databricks/databricks-sdk-go/apierr" ) @@ -156,10 +157,19 @@ func BuildACL(b *bundle.Bundle) []snapshot.ACLEntry { // recommended permissions section names it with CAN_MANAGE as well, so skip it in the loop // rather than sending it twice. func BuildCanManage(b *bundle.Bundle) []snapshot.ManagePrincipal { - currentUser := b.Config.Workspace.CurrentUser.UserName - canManage := []snapshot.ManagePrincipal{{UserName: currentUser}} + currentUser := b.Config.Workspace.CurrentUser.User + var canManage []snapshot.ManagePrincipal + if iamutil.IsServicePrincipal(currentUser) { + canManage = append(canManage, snapshot.ManagePrincipal{ + ServicePrincipalName: currentUser.UserName, + }) + } else { + canManage = append(canManage, snapshot.ManagePrincipal{ + UserName: currentUser.UserName, + }) + } for _, p := range b.Config.Permissions { - if p.Level != "CAN_MANAGE" || p.UserName == currentUser { + if p.Level != "CAN_MANAGE" { continue } canManage = append(canManage, snapshot.ManagePrincipal{ diff --git a/libs/testserver/fake_workspace.go b/libs/testserver/fake_workspace.go index 00f88113982..104cf7eb621 100644 --- a/libs/testserver/fake_workspace.go +++ b/libs/testserver/fake_workspace.go @@ -171,6 +171,16 @@ type fakeDashboard struct { InputSerializedDashboard string `json:"-"` } +// snapshotRecord holds the create-request fields and the out-of-band modification flag for one +// immutable snapshot. Its JSON tags match what the accept test's info endpoint returns so the +// test script can call the endpoint and assert the fields with jq. +type snapshotRecord struct { + Dirty bool `json:"dirty"` + RelPath string `json:"relative_path"` + ACL []map[string]any `json:"access_control_list"` + CanManage []map[string]any `json:"can_manage_principals"` +} + // FakeWorkspace holds a state of a workspace for acceptance tests. type FakeWorkspace struct { mu sync.Mutex @@ -188,10 +198,10 @@ type FakeWorkspace struct { files map[string]FileEntry repoIdByPath map[string]int64 - // snapshots records immutable-snapshot content paths and whether each was modified out - // of band. A workspace write into a recorded path flips it dirty; - // RecordSnapshot resets it when the snapshot is (re)created by the snapshot API. - snapshots map[string]bool + // snapshots records immutable-snapshot content paths. A workspace write into a + // recorded path flips it dirty; RecordSnapshot resets it when the snapshot is + // (re)created by the snapshot API. + snapshots map[string]*snapshotRecord Jobs map[int64]jobs.Job JobRuns map[int64]jobs.Run @@ -484,7 +494,7 @@ func NewFakeWorkspace(url, token string) *FakeWorkspace { }, files: make(map[string]FileEntry), repoIdByPath: make(map[string]int64), - snapshots: make(map[string]bool), + snapshots: make(map[string]*snapshotRecord), Jobs: map[int64]jobs.Job{}, JobRuns: map[int64]jobs.Run{}, @@ -810,28 +820,44 @@ func (s *FakeWorkspace) FsDeleteFile(filePath string) Response { return Response{} } -// RecordSnapshot marks contentPath as a known, clean immutable snapshot. The snapshot create -// handler calls it after materializing the content directory, so the snapshot's own creation -// write does not count as tampering. -func (s *FakeWorkspace) RecordSnapshot(contentPath string) { +// RecordSnapshot stores a create-request record for the snapshot at contentPath. It is called +// after the content directory is materialized, so the snapshot's own creation write is not +// counted as tampering. +func (s *FakeWorkspace) RecordSnapshot(contentPath, relPath string, acl, canManage []map[string]any) { defer s.LockUnlock()() - s.snapshots[contentPath] = false + s.snapshots[contentPath] = &snapshotRecord{ + Dirty: false, + RelPath: relPath, + ACL: acl, + CanManage: canManage, + } } // SnapshotDirty reports whether the immutable snapshot at contentPath was modified out of band, // and whether a snapshot is recorded there at all. func (s *FakeWorkspace) SnapshotDirty(contentPath string) (dirty, ok bool) { defer s.LockUnlock()() - dirty, ok = s.snapshots[contentPath] - return dirty, ok + rec, ok := s.snapshots[contentPath] + if !ok { + return false, false + } + return rec.Dirty, true +} + +// SnapshotInfo returns the stored create-request record for the snapshot at contentPath. +// ok is false when no snapshot is recorded there. +func (s *FakeWorkspace) SnapshotInfo(contentPath string) (*snapshotRecord, bool) { + defer s.LockUnlock()() + rec, ok := s.snapshots[contentPath] + return rec, ok } // markSnapshotDirty flips any recorded snapshot dirty when targetPath writes at or under its // content path. The caller must already hold the lock. func (s *FakeWorkspace) markSnapshotDirty(targetPath string) { - for contentPath := range s.snapshots { + for contentPath, rec := range s.snapshots { if targetPath == contentPath || strings.HasPrefix(targetPath, contentPath+"/") { - s.snapshots[contentPath] = true + rec.Dirty = true } } } diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go index 92299f76b80..445ff5fc965 100644 --- a/libs/testserver/handlers.go +++ b/libs/testserver/handlers.go @@ -733,6 +733,7 @@ func AddDefaultHandlers(server *Server) { mr := multipart.NewReader(bytes.NewReader(req.Body), params["boundary"]) var relativePath string + var acl, canManage []map[string]any for { p, err := mr.NextPart() if err == io.EOF { @@ -745,8 +746,13 @@ func AddDefaultHandlers(server *Server) { if err != nil { return Response{StatusCode: http.StatusInternalServerError} } - if p.FormName() == "relative_path" { + switch p.FormName() { + case "relative_path": relativePath = string(data) + case "access_control_list": + _ = json.Unmarshal(data, &acl) + case "can_manage_principals": + _ = json.Unmarshal(data, &canManage) } } @@ -756,7 +762,7 @@ func AddDefaultHandlers(server *Server) { contentPath := fmt.Sprintf("/Workspace/Users/%s/.snapshots/%s/snapshot", TestUserSP.UserName, relativePath) req.Workspace.WorkspaceMkdirs(workspace.Mkdirs{Path: contentPath}) // Record after mkdirs so the snapshot's own creation write is not counted as tampering. - req.Workspace.RecordSnapshot(contentPath) + req.Workspace.RecordSnapshot(contentPath, relativePath, acl, canManage) return map[string]any{ "name": "workspaces/snapshotOperations/" + relativePath, "done": true, @@ -764,6 +770,25 @@ func AddDefaultHandlers(server *Server) { } }) + // Returns the create-request fields (ACL, can_manage_principals) stored for the snapshot + // at the given relative_path. Intended for acceptance test assertions. + server.Handle("GET", "/api/2.0/snapshots/info", func(req Request) any { + relPath := req.URL.Query().Get("relative_path") + // Build the content path the same way the create handler does. + contentPath := fmt.Sprintf("/Workspace/Users/%s/.snapshots/%s/snapshot", TestUserSP.UserName, relPath) + rec, ok := req.Workspace.SnapshotInfo(contentPath) + if !ok { + return Response{ + StatusCode: http.StatusNotFound, + Body: map[string]string{ + "error_code": "RESOURCE_DOES_NOT_EXIST", + "message": "Snapshot not found: " + relPath, + }, + } + } + return rec + }) + // Reports whether a snapshot's content was modified out of band. server.Handle("GET", "/api/2.0/snapshots:inspect", func(req Request) any { contentPath := req.URL.Query().Get("snapshot_content_path") From 535004e8c4cda5a741556fb231b6cf877d969d4e Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Tue, 13 Oct 2026 17:13:02 +0200 Subject: [PATCH 08/16] fixed the break glass message --- acceptance/bundle/deploy/immutable-breakglass/output.txt | 4 +--- .../bundle/deploy/immutable-legacy-state/output.txt | 4 +--- bundle/deploy/snapshot/upload.go | 8 ++++---- 3 files changed, 6 insertions(+), 10 deletions(-) diff --git a/acceptance/bundle/deploy/immutable-breakglass/output.txt b/acceptance/bundle/deploy/immutable-breakglass/output.txt index f538fa251cb..e661b297116 100644 --- a/acceptance/bundle/deploy/immutable-breakglass/output.txt +++ b/acceptance/bundle/deploy/immutable-breakglass/output.txt @@ -12,9 +12,7 @@ First snapshot path: /Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHO === Deploy without --force fails >>> musterr [CLI] bundle deploy -Error: the immutable snapshot of the last deployment was modified outside of the bundle (break glass): - /Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]/snapshot -To deploy a new snapshot and point resources at it, use --force +Error: the last deployment was modified in the workspace. Changes might be overwritten by this deployment. To confirm it, use the --force flag Files: 0 uploaded, 0 deleted diff --git a/acceptance/bundle/deploy/immutable-legacy-state/output.txt b/acceptance/bundle/deploy/immutable-legacy-state/output.txt index e06269208cf..ce0913375e9 100644 --- a/acceptance/bundle/deploy/immutable-legacy-state/output.txt +++ b/acceptance/bundle/deploy/immutable-legacy-state/output.txt @@ -18,9 +18,7 @@ OK: deployed to the current snapshot layout >>> [CLI] workspace mkdirs /Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]/snapshot/tampered >>> musterr [CLI] bundle deploy -Error: the immutable snapshot of the last deployment was modified outside of the bundle (break glass): - /Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]/snapshot -To deploy a new snapshot and point resources at it, use --force +Error: the last deployment was modified in the workspace. Changes might be overwritten by this deployment. To confirm it, use the --force flag Files: 0 uploaded, 0 deleted diff --git a/bundle/deploy/snapshot/upload.go b/bundle/deploy/snapshot/upload.go index 8d290cd1165..1541b23618a 100644 --- a/bundle/deploy/snapshot/upload.go +++ b/bundle/deploy/snapshot/upload.go @@ -86,8 +86,8 @@ func (m *snapshotUpload) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagn if err != nil { return diags.Extend(diag.FromErr(err)) } - - b.Config.Resources.Snapshots[resources.SnapshotResourceKey] = &resources.Snapshot{ + + b.Config.Resources.Snapshots[resources.SnapshotResourceKey] = &resources.Snapshot{ BundleID: BundleID(b), ACL: BuildACL(b), CanManage: BuildCanManage(b), @@ -117,7 +117,7 @@ func stageZip(ctx context.Context, b *bundle.Bundle) (string, diag.Diagnostics, } zipPath := filepath.Join(dir, snapshot.HashFromContent(zipContent)+".zip") if err := os.WriteFile(zipPath, zipContent, 0o600); err != nil { - return diag.FromErr(fmt.Errorf("failed to write snapshot zip: %w", err)) + return "", nil, fmt.Errorf("failed to write snapshot zip: %w", err) } var diags diag.Diagnostics @@ -260,7 +260,7 @@ func resolveGeneration(ctx context.Context, b *bundle.Bundle, uploader *snapshot return prev.Generation, nil } if !b.Config.Bundle.Force { - return 0, fmt.Errorf("the immutable snapshot of the last deployment was modified outside of the bundle (break glass):\n %s\nTo deploy a new snapshot and point resources at it, use --force", contentPath) + return 0, fmt.Errorf("the last deployment was modified in the workspace. Changes might be overwritten by this deployment. To confirm it, use the --force flag") } return prev.Generation + 1, nil } From 576a04e40624c7b5ef52e163b4542950b2832020 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Tue, 13 Oct 2026 17:19:35 +0200 Subject: [PATCH 09/16] remove test --- bundle/deploy/snapshot/upload_warning_test.go | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/bundle/deploy/snapshot/upload_warning_test.go b/bundle/deploy/snapshot/upload_warning_test.go index 166fbb46421..1a17c62a3a4 100644 --- a/bundle/deploy/snapshot/upload_warning_test.go +++ b/bundle/deploy/snapshot/upload_warning_test.go @@ -133,16 +133,6 @@ func TestBuildCanManageWithoutManagers(t *testing.T) { }, BuildCanManage(b)) } -func TestBuildCanManageSkipsDuplicateDeployer(t *testing.T) { - b := canManageBundle([]resources.Permission{ - {Level: "CAN_MANAGE", UserName: "deployer@example.test"}, - }) - - assert.Equal(t, []snapshot.ManagePrincipal{ - {UserName: "deployer@example.test"}, - }, BuildCanManage(b)) -} - func TestUploadReusesStagedZipWhenNotClean(t *testing.T) { // clean=false is the deploy --plan path: the zip staged when the plan was produced is // registered on the config as is, without rebuilding or warning again, even for a From 151a68f8e91be80d739a917402552b70a00dd9f0 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Wed, 14 Oct 2026 11:47:40 +0200 Subject: [PATCH 10/16] always use unique upload path --- .../deploy/immutable-breakglass/output.txt | 8 +- .../deploy/immutable-can-manage/output.txt | 2 +- .../databricks.yml.tmpl | 19 +++ .../immutable-destroy-redeploy/out.test.toml | 3 + .../immutable-destroy-redeploy/output.txt | 58 ++++++++ .../deploy/immutable-destroy-redeploy/script | 51 +++++++ .../immutable-destroy-redeploy/src/main.py | 1 + .../immutable-destroy-redeploy/test.toml | 14 ++ .../deploy/immutable-legacy-state/output.txt | 8 +- .../immutable-missing-snapshot/output.txt | 4 +- .../deploy/immutable-no-artifacts/output.txt | 26 ++-- .../immutable-permissions-change/output.txt | 4 +- acceptance/bundle/deploy/immutable/output.txt | 44 +++--- .../bundle/dms/immutable-folder/output.txt | 6 +- acceptance/bundle/refschema/out.fields.txt | 1 - .../resources/apps/immutable/output.txt | 2 +- bundle/config/resources/snapshot.go | 33 ++--- bundle/deploy/snapshot/upload.go | 130 +++++++++++------- bundle/deploy/snapshot/upload_warning_test.go | 60 ++++++++ bundle/direct/dresources/snapshot.go | 9 +- bundle/direct/dresources/type_test.go | 1 - libs/testserver/handlers.go | 10 ++ 22 files changed, 362 insertions(+), 132 deletions(-) create mode 100644 acceptance/bundle/deploy/immutable-destroy-redeploy/databricks.yml.tmpl create mode 100644 acceptance/bundle/deploy/immutable-destroy-redeploy/out.test.toml create mode 100644 acceptance/bundle/deploy/immutable-destroy-redeploy/output.txt create mode 100644 acceptance/bundle/deploy/immutable-destroy-redeploy/script create mode 100644 acceptance/bundle/deploy/immutable-destroy-redeploy/src/main.py create mode 100644 acceptance/bundle/deploy/immutable-destroy-redeploy/test.toml diff --git a/acceptance/bundle/deploy/immutable-breakglass/output.txt b/acceptance/bundle/deploy/immutable-breakglass/output.txt index e661b297116..f2e33387c06 100644 --- a/acceptance/bundle/deploy/immutable-breakglass/output.txt +++ b/acceptance/bundle/deploy/immutable-breakglass/output.txt @@ -5,10 +5,10 @@ Created internal_immutable_snapshots.immutable Created jobs.my_job Files: 0 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged -First snapshot path: /Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]/snapshot/files/src/main.py +First snapshot path: /Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]-[UUID]/snapshot/files/src/main.py === Break the glass out of band ->>> [CLI] workspace mkdirs /Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]/snapshot/tampered +>>> [CLI] workspace mkdirs /Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]-[UUID]/snapshot/tampered === Deploy without --force fails >>> musterr [CLI] bundle deploy @@ -22,14 +22,14 @@ Recreated internal_immutable_snapshots.immutable Updated jobs.my_job Files: 0 uploaded, 0 deleted Resources: 1 created, 1 changed, 1 deleted, 0 unchanged -Second snapshot path: /Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]-1/snapshot/files/src/main.py +Second snapshot path: /Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]-[UUID]/snapshot/files/src/main.py OK: --force created a new snapshot path === Redeploy reuses the new snapshot >>> [CLI] bundle deploy Files: 0 uploaded, 0 deleted Resources: 0 created, 0 changed, 0 deleted, 2 unchanged -Third snapshot path: /Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]-1/snapshot/files/src/main.py +Third snapshot path: /Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]-[UUID]/snapshot/files/src/main.py OK: new snapshot reused on redeploy >>> [CLI] bundle destroy --auto-approve diff --git a/acceptance/bundle/deploy/immutable-can-manage/output.txt b/acceptance/bundle/deploy/immutable-can-manage/output.txt index 4870dc653bc..b142c587a3b 100644 --- a/acceptance/bundle/deploy/immutable-can-manage/output.txt +++ b/acceptance/bundle/deploy/immutable-can-manage/output.txt @@ -20,7 +20,7 @@ Files: 0 uploaded, 0 deleted Resources: 3 created, 0 changed, 0 deleted, 0 unchanged === Check can_manage_principals sent to the snapshot create API ->>> [CLI] api get /api/2.0/snapshots/info?relative_path=[SNAPSHOT_HASH]/[SNAPSHOT_HASH] +>>> [CLI] api get /api/2.0/snapshots/info?relative_path=[SNAPSHOT_HASH]/[SNAPSHOT_HASH]-[UUID] { "can_manage_principals": [ { diff --git a/acceptance/bundle/deploy/immutable-destroy-redeploy/databricks.yml.tmpl b/acceptance/bundle/deploy/immutable-destroy-redeploy/databricks.yml.tmpl new file mode 100644 index 00000000000..a1729107bdb --- /dev/null +++ b/acceptance/bundle/deploy/immutable-destroy-redeploy/databricks.yml.tmpl @@ -0,0 +1,19 @@ +bundle: + name: test-bundle-immutable-redeploy-$UNIQUE_NAME + +experimental: + immutable_folder: true + +resources: + jobs: + my_job: + name: my job + tasks: + - task_key: spark_python_task + spark_python_task: + python_file: ./src/main.py + environment_key: env + environments: + - environment_key: env + spec: + environment_version: "4" diff --git a/acceptance/bundle/deploy/immutable-destroy-redeploy/out.test.toml b/acceptance/bundle/deploy/immutable-destroy-redeploy/out.test.toml new file mode 100644 index 00000000000..59b56a2037c --- /dev/null +++ b/acceptance/bundle/deploy/immutable-destroy-redeploy/out.test.toml @@ -0,0 +1,3 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/deploy/immutable-destroy-redeploy/output.txt b/acceptance/bundle/deploy/immutable-destroy-redeploy/output.txt new file mode 100644 index 00000000000..cf05c9e7e33 --- /dev/null +++ b/acceptance/bundle/deploy/immutable-destroy-redeploy/output.txt @@ -0,0 +1,58 @@ + +=== Deploy the immutable snapshot +>>> [CLI] bundle deploy +Created internal_immutable_snapshots.immutable +Created jobs.my_job +Files: 0 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged +First snapshot path: /Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]-[UUID]/snapshot/files/src/main.py + +=== Destroy the bundle +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.my_job + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-immutable-redeploy-[UNIQUE_NAME]/default + +Destroy: 1 deleted + +=== Deploy again moves to a new snapshot instead of the one left behind +>>> [CLI] bundle deploy +Created internal_immutable_snapshots.immutable +Created jobs.my_job +Files: 0 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged +Second snapshot path: /Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]-[UUID]/snapshot/files/src/main.py +OK: deployment moved to a new snapshot + +=== Redeploy reuses the new snapshot +>>> [CLI] bundle deploy +Files: 0 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 2 unchanged +Third snapshot path: /Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]-[UUID]/snapshot/files/src/main.py +OK: new snapshot reused on redeploy + +=== Destroy and deploy again moves to yet another snapshot +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.my_job + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-immutable-redeploy-[UNIQUE_NAME]/default + +Destroy: 1 deleted + +>>> [CLI] bundle deploy +Created internal_immutable_snapshots.immutable +Created jobs.my_job +Files: 0 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged +Fourth snapshot path: /Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]-[UUID]/snapshot/files/src/main.py +OK: deployment moved to a new snapshot + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.my_job + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-immutable-redeploy-[UNIQUE_NAME]/default + +Destroy: 1 deleted diff --git a/acceptance/bundle/deploy/immutable-destroy-redeploy/script b/acceptance/bundle/deploy/immutable-destroy-redeploy/script new file mode 100644 index 00000000000..43618ed3886 --- /dev/null +++ b/acceptance/bundle/deploy/immutable-destroy-redeploy/script @@ -0,0 +1,51 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve +} +trap cleanup EXIT + +snapshot_path() { + JOB_ID=$($CLI bundle summary -o json | jq -r '.resources.jobs.my_job.id') + $CLI jobs get "$JOB_ID" | jq -r '.settings.tasks[0].spark_python_task.python_file' +} + +title "Deploy the immutable snapshot" +trace $CLI bundle deploy +SNAPSHOT_PATH_1=$(snapshot_path) +echo "First snapshot path: $SNAPSHOT_PATH_1" + +title "Destroy the bundle" +# The snapshot is immutable and stays in the workspace; destroy only drops it from the state. +trace $CLI bundle destroy --auto-approve + +title "Deploy again moves to a new snapshot instead of the one left behind" +trace $CLI bundle deploy +SNAPSHOT_PATH_2=$(snapshot_path) +echo "Second snapshot path: $SNAPSHOT_PATH_2" +if [ "$SNAPSHOT_PATH_1" = "$SNAPSHOT_PATH_2" ]; then + echo "ERROR: deployment reused the snapshot left behind by destroy" + exit 1 +fi +echo "OK: deployment moved to a new snapshot" + +title "Redeploy reuses the new snapshot" +trace $CLI bundle deploy +SNAPSHOT_PATH_3=$(snapshot_path) +echo "Third snapshot path: $SNAPSHOT_PATH_3" +if [ "$SNAPSHOT_PATH_2" != "$SNAPSHOT_PATH_3" ]; then + echo "ERROR: new snapshot path was not reused on redeploy" + exit 1 +fi +echo "OK: new snapshot reused on redeploy" + +title "Destroy and deploy again moves to yet another snapshot" +trace $CLI bundle destroy --auto-approve +trace $CLI bundle deploy +SNAPSHOT_PATH_4=$(snapshot_path) +echo "Fourth snapshot path: $SNAPSHOT_PATH_4" +if [ "$SNAPSHOT_PATH_4" = "$SNAPSHOT_PATH_1" ] || [ "$SNAPSHOT_PATH_4" = "$SNAPSHOT_PATH_2" ]; then + echo "ERROR: deployment reused a snapshot left behind by destroy" + exit 1 +fi +echo "OK: deployment moved to a new snapshot" diff --git a/acceptance/bundle/deploy/immutable-destroy-redeploy/src/main.py b/acceptance/bundle/deploy/immutable-destroy-redeploy/src/main.py new file mode 100644 index 00000000000..6c285f7e2f5 --- /dev/null +++ b/acceptance/bundle/deploy/immutable-destroy-redeploy/src/main.py @@ -0,0 +1 @@ +print("Hello from Spark Python Task!") diff --git a/acceptance/bundle/deploy/immutable-destroy-redeploy/test.toml b/acceptance/bundle/deploy/immutable-destroy-redeploy/test.toml new file mode 100644 index 00000000000..88b225ea71f --- /dev/null +++ b/acceptance/bundle/deploy/immutable-destroy-redeploy/test.toml @@ -0,0 +1,14 @@ +Cloud = false # Temporary disable cloud tests until the API is fully available + +# immutable_folder only works with the direct engine. +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] + +Ignore = [ + "databricks.yml", + ".databricks", +] + +[[Repls]] +# Replace snapshot hash with SNAPSHOT_HASH +Old = "[0-9a-f]{64}" +New = "[SNAPSHOT_HASH]" diff --git a/acceptance/bundle/deploy/immutable-legacy-state/output.txt b/acceptance/bundle/deploy/immutable-legacy-state/output.txt index ce0913375e9..72c4f7f4dd1 100644 --- a/acceptance/bundle/deploy/immutable-legacy-state/output.txt +++ b/acceptance/bundle/deploy/immutable-legacy-state/output.txt @@ -9,13 +9,15 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged === Rewrite the state the way a CLI without this feature wrote it === Deploying on top of that state succeeds and moves to the current layout >>> [CLI] bundle deploy +Recreated internal_immutable_snapshots.immutable +Updated jobs.my_job Files: 0 uploaded, 0 deleted -Resources: 0 created, 0 changed, 0 deleted, 2 unchanged -Snapshot path: /Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]/snapshot/files/src/main.py +Resources: 1 created, 1 changed, 1 deleted, 0 unchanged +Snapshot path: /Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]-[UUID]/snapshot/files/src/main.py OK: deployed to the current snapshot layout === The check still applies to a deployment carried over from the older layout ->>> [CLI] workspace mkdirs /Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]/snapshot/tampered +>>> [CLI] workspace mkdirs /Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]-[UUID]/snapshot/tampered >>> musterr [CLI] bundle deploy Error: the last deployment was modified in the workspace. Changes might be overwritten by this deployment. To confirm it, use the --force flag diff --git a/acceptance/bundle/deploy/immutable-missing-snapshot/output.txt b/acceptance/bundle/deploy/immutable-missing-snapshot/output.txt index a6c2ea78641..c938ad1143f 100644 --- a/acceptance/bundle/deploy/immutable-missing-snapshot/output.txt +++ b/acceptance/bundle/deploy/immutable-missing-snapshot/output.txt @@ -9,8 +9,10 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged === Point the state at a snapshot that is not there === Deploying still works instead of failing on the snapshot that is not there >>> [CLI] bundle deploy +Recreated internal_immutable_snapshots.immutable +Updated jobs.my_job Files: 0 uploaded, 0 deleted -Resources: 0 created, 0 changed, 0 deleted, 2 unchanged +Resources: 1 created, 1 changed, 1 deleted, 0 unchanged >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: diff --git a/acceptance/bundle/deploy/immutable-no-artifacts/output.txt b/acceptance/bundle/deploy/immutable-no-artifacts/output.txt index 60b2c41d7d9..7222a27b940 100644 --- a/acceptance/bundle/deploy/immutable-no-artifacts/output.txt +++ b/acceptance/bundle/deploy/immutable-no-artifacts/output.txt @@ -15,16 +15,16 @@ Validation OK! { "notebook_task": { "base_parameters": { - "path": "/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]/snapshot/files/some_path" + "path": "/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]-[UUID]/snapshot/files/some_path" }, - "notebook_path": "/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]/snapshot/files/src/notebook" + "notebook_path": "/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]-[UUID]/snapshot/files/src/notebook" }, "task_key": "notebook_task" }, { "environment_key": "env", "spark_python_task": { - "python_file": "/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]/snapshot/files/src/main.py" + "python_file": "/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]-[UUID]/snapshot/files/src/main.py" }, "task_key": "spark_python_task" } @@ -35,8 +35,8 @@ Validation OK! "action": "create", "new_state": { "value": { - "relative_path": "[SNAPSHOT_HASH]/[SNAPSHOT_HASH]", - "full_path": "/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]/snapshot", + "relative_path": "[SNAPSHOT_HASH]/[SNAPSHOT_HASH]-[UUID]", + "full_path": "/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]-[UUID]/snapshot", "bundle_id": "[SNAPSHOT_HASH]", "acl": [ { @@ -49,7 +49,7 @@ Validation OK! "user_name": "[USERNAME]" } ], - "zip_path": "[TEST_TMP_DIR]/.databricks/bundle/default/snapshots/[SNAPSHOT_HASH].zip" + "zip_path": "[TEST_TMP_DIR]/.databricks/bundle/default/snapshots/[SNAPSHOT_HASH]-[UUID].zip" } } } @@ -61,13 +61,13 @@ Files: 0 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged >>> jq .settings.tasks job.json -"/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]/snapshot/files/src/main.py" +"/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]-[UUID]/snapshot/files/src/main.py" >>> jq .settings.tasks job.json -"/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]/snapshot/files/src/notebook" +"/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]-[UUID]/snapshot/files/src/notebook" >>> jq .settings.tasks job.json -"/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]/snapshot/files/some_path" +"/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]-[UUID]/snapshot/files/some_path" >>> title Check snapshot plan after deploy: @@ -78,8 +78,8 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged { "action": "skip", "remote_state": { - "relative_path": "[SNAPSHOT_HASH]/[SNAPSHOT_HASH]", - "full_path": "/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]/snapshot" + "relative_path": "[SNAPSHOT_HASH]/[SNAPSHOT_HASH]-[UUID]", + "full_path": "/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]-[UUID]/snapshot" }, "changes": { "acl": { @@ -123,8 +123,8 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "zip_path": { "action": "skip", "reason": "missing_in_remote", - "old": "[TEST_TMP_DIR]/.databricks/bundle/default/snapshots/[SNAPSHOT_HASH].zip", - "new": "[TEST_TMP_DIR]/.databricks/bundle/default/snapshots/[SNAPSHOT_HASH].zip", + "old": "[TEST_TMP_DIR]/.databricks/bundle/default/snapshots/[SNAPSHOT_HASH]-[UUID].zip", + "new": "[TEST_TMP_DIR]/.databricks/bundle/default/snapshots/[SNAPSHOT_HASH]-[UUID].zip", "remote": "" } } diff --git a/acceptance/bundle/deploy/immutable-permissions-change/output.txt b/acceptance/bundle/deploy/immutable-permissions-change/output.txt index 740dc2a0f33..4ed18185018 100644 --- a/acceptance/bundle/deploy/immutable-permissions-change/output.txt +++ b/acceptance/bundle/deploy/immutable-permissions-change/output.txt @@ -5,7 +5,7 @@ Created internal_immutable_snapshots.immutable Created jobs.my_job Files: 0 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged -First snapshot path: /Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]/snapshot/files/src/main.py +First snapshot path: /Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]-[UUID]/snapshot/files/src/main.py === Add permissions and redeploy >>> [CLI] bundle deploy @@ -26,7 +26,7 @@ Recreated internal_immutable_snapshots.immutable Updated jobs.my_job Files: 0 uploaded, 0 deleted Resources: 2 created, 1 changed, 1 deleted, 0 unchanged -Second snapshot path: /Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]/snapshot/files/src/main.py +Second snapshot path: /Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]-[UUID]/snapshot/files/src/main.py === Verify snapshot path changed after permissions change OK: snapshot path changed as expected diff --git a/acceptance/bundle/deploy/immutable/output.txt b/acceptance/bundle/deploy/immutable/output.txt index acee49af72a..39516596c77 100644 --- a/acceptance/bundle/deploy/immutable/output.txt +++ b/acceptance/bundle/deploy/immutable/output.txt @@ -15,7 +15,7 @@ Building python_artifact... [ { "notebook_task": { - "notebook_path": "/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]/snapshot/files/src/notebook" + "notebook_path": "/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]-[UUID]/snapshot/files/src/notebook" }, "task_key": "notebook_task" }, @@ -30,7 +30,7 @@ Building python_artifact... { "environment_key": "env", "spark_python_task": { - "python_file": "/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]/snapshot/files/src/main.py" + "python_file": "/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]-[UUID]/snapshot/files/src/main.py" }, "task_key": "spark_python_task" } @@ -42,7 +42,7 @@ Building python_artifact... "environment_key": "env", "spec": { "dependencies": [ - "/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]/snapshot/artifacts/.internal/immutable-0.0.1-py3-none-any.whl" + "/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]-[UUID]/snapshot/artifacts/.internal/immutable-0.0.1-py3-none-any.whl" ], "environment_version": "4" } @@ -54,8 +54,8 @@ Building python_artifact... "action": "create", "new_state": { "value": { - "relative_path": "[SNAPSHOT_HASH]/[SNAPSHOT_HASH]", - "full_path": "/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]/snapshot", + "relative_path": "[SNAPSHOT_HASH]/[SNAPSHOT_HASH]-[UUID]", + "full_path": "/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]-[UUID]/snapshot", "bundle_id": "[SNAPSHOT_HASH]", "acl": [ { @@ -68,7 +68,7 @@ Building python_artifact... "user_name": "[USERNAME]" } ], - "zip_path": "[TEST_TMP_DIR]/.databricks/bundle/default/snapshots/[SNAPSHOT_HASH].zip" + "zip_path": "[TEST_TMP_DIR]/.databricks/bundle/default/snapshots/[SNAPSHOT_HASH]-[UUID].zip" } } } @@ -81,14 +81,14 @@ Files: 0 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged >>> jq .settings.tasks job.json -"/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]/snapshot/files/src/main.py" +"/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]-[UUID]/snapshot/files/src/main.py" >>> jq .settings.tasks job.json -"/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]/snapshot/files/src/notebook" +"/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]-[UUID]/snapshot/files/src/notebook" >>> jq .settings.environments[0].spec.dependencies job.json [ - "/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]/snapshot/artifacts/.internal/immutable-0.0.1-py3-none-any.whl" + "/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]-[UUID]/snapshot/artifacts/.internal/immutable-0.0.1-py3-none-any.whl" ] >>> title Check snapshot plan after deploy @@ -102,8 +102,8 @@ Building python_artifact... "action": "recreate", "new_state": { "value": { - "relative_path": "[SNAPSHOT_HASH]/[SNAPSHOT_HASH]", - "full_path": "/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]/snapshot", + "relative_path": "[SNAPSHOT_HASH]/[SNAPSHOT_HASH]-[UUID]", + "full_path": "/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]-[UUID]/snapshot", "bundle_id": "[SNAPSHOT_HASH]", "acl": [ { @@ -116,12 +116,12 @@ Building python_artifact... "user_name": "[USERNAME]" } ], - "zip_path": "[TEST_TMP_DIR]/.databricks/bundle/default/snapshots/[SNAPSHOT_HASH].zip" + "zip_path": "[TEST_TMP_DIR]/.databricks/bundle/default/snapshots/[SNAPSHOT_HASH]-[UUID].zip" } }, "remote_state": { - "relative_path": "[SNAPSHOT_HASH]/[SNAPSHOT_HASH]", - "full_path": "/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]/snapshot" + "relative_path": "[SNAPSHOT_HASH]/[SNAPSHOT_HASH]-[UUID]", + "full_path": "/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]-[UUID]/snapshot" }, "changes": { "acl": { @@ -165,22 +165,22 @@ Building python_artifact... "full_path": { "action": "recreate", "reason": "immutable", - "old": "/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]/snapshot", - "new": "/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]/snapshot", - "remote": "/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]/snapshot" + "old": "/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]-[UUID]/snapshot", + "new": "/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]-[UUID]/snapshot", + "remote": "/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]-[UUID]/snapshot" }, "relative_path": { "action": "recreate", "reason": "immutable", - "old": "[SNAPSHOT_HASH]/[SNAPSHOT_HASH]", - "new": "[SNAPSHOT_HASH]/[SNAPSHOT_HASH]", - "remote": "[SNAPSHOT_HASH]/[SNAPSHOT_HASH]" + "old": "[SNAPSHOT_HASH]/[SNAPSHOT_HASH]-[UUID]", + "new": "[SNAPSHOT_HASH]/[SNAPSHOT_HASH]-[UUID]", + "remote": "[SNAPSHOT_HASH]/[SNAPSHOT_HASH]-[UUID]" }, "zip_path": { "action": "recreate", "reason": "immutable", - "old": "[TEST_TMP_DIR]/.databricks/bundle/default/snapshots/[SNAPSHOT_HASH].zip", - "new": "[TEST_TMP_DIR]/.databricks/bundle/default/snapshots/[SNAPSHOT_HASH].zip", + "old": "[TEST_TMP_DIR]/.databricks/bundle/default/snapshots/[SNAPSHOT_HASH]-[UUID].zip", + "new": "[TEST_TMP_DIR]/.databricks/bundle/default/snapshots/[SNAPSHOT_HASH]-[UUID].zip", "remote": "" } } diff --git a/acceptance/bundle/dms/immutable-folder/output.txt b/acceptance/bundle/dms/immutable-folder/output.txt index a45aa8ba189..53ff59a9f4e 100644 --- a/acceptance/bundle/dms/immutable-folder/output.txt +++ b/acceptance/bundle/dms/immutable-folder/output.txt @@ -10,7 +10,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged { "workspace_info": { "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-immutable-folder-[UNIQUE_NAME]/default", - "file_path": "/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]/files" + "file_path": "/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]-[UUID]/snapshot/files" } } @@ -25,7 +25,7 @@ Resources: 1 created, 1 changed, 1 deleted, 0 unchanged { "workspace_info": { "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-immutable-folder-[UNIQUE_NAME]/default", - "file_path": "/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]/files" + "file_path": "/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]-[UUID]/snapshot/files" } } @@ -38,6 +38,6 @@ Resources: 0 created, 0 changed, 0 deleted, 2 unchanged { "workspace_info": { "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-immutable-folder-[UNIQUE_NAME]/default", - "file_path": "/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]/files" + "file_path": "/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]-[UUID]/snapshot/files" } } diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index 615d21f6b71..92c95e74393 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -931,7 +931,6 @@ resources.internal_immutable_snapshots.*.can_manage[*].group_name string INPUT S resources.internal_immutable_snapshots.*.can_manage[*].service_principal_name string INPUT STATE resources.internal_immutable_snapshots.*.can_manage[*].user_name string INPUT STATE resources.internal_immutable_snapshots.*.full_path string REMOTE STATE -resources.internal_immutable_snapshots.*.generation int INPUT STATE resources.internal_immutable_snapshots.*.relative_path string REMOTE STATE resources.internal_immutable_snapshots.*.remote_root string INPUT resources.internal_immutable_snapshots.*.zip_path string INPUT STATE diff --git a/acceptance/bundle/resources/apps/immutable/output.txt b/acceptance/bundle/resources/apps/immutable/output.txt index 14b931e25f9..356636ef6f5 100644 --- a/acceptance/bundle/resources/apps/immutable/output.txt +++ b/acceptance/bundle/resources/apps/immutable/output.txt @@ -37,6 +37,6 @@ You can access the app at my-immutable-app-123.cloud.databricksapps.com "path": "/api/2.0/apps/my-immutable-app/deployments", "body": { "mode": "SNAPSHOT", - "source_code_path": "/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]/snapshot/files/app" + "source_code_path": "/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]-[UUID]/snapshot/files/app" } } diff --git a/bundle/config/resources/snapshot.go b/bundle/config/resources/snapshot.go index 921a944bbf2..7aee13a1d08 100644 --- a/bundle/config/resources/snapshot.go +++ b/bundle/config/resources/snapshot.go @@ -2,7 +2,6 @@ package resources import ( "context" - "fmt" "net/url" "path" "strings" @@ -52,38 +51,26 @@ type Snapshot struct { // from top-level bundle.permissions entries with level CAN_MANAGE. CanManage []snapshot.ManagePrincipal `json:"can_manage,omitempty"` // ZipPath is the local path of the bundle zip staged by the deploy pipeline. - // The file is named ".zip", so its base name is the content hash used - // as the last component of RelativePath. Storing the path (not the bytes) keeps - // the zip off the config, the state, and the plan file, and lets DoCreate stream - // it straight from disk at upload time. - // Example: "/.../.databricks/bundle/default/snapshots/e3b0c4...zip". + // The file is named "-.zip", and its base name is the last component + // of RelativePath. Storing the path (not the bytes) keeps the zip off the config, + // the state, and the plan file, and lets DoCreate stream it straight from disk at + // upload time. + // Example: "/.../.databricks/bundle/default/snapshots/e3b0c4...-0199e2c1-....zip". ZipPath string `json:"zip_path"` // RemoteRoot is the workspace directory that holds every snapshot for this user, // returned by the snapshot rootpath API. // Example: "/Workspace/Users//.snapshots". RemoteRoot string `json:"remote_root"` - // Generation is 0 for the initial snapshot and incremented each time "deploy --force" - // moves off a snapshot that was modified outside the bundle, appending a "-" - // suffix to the content hash so the new snapshot gets a distinct path. It is carried - // forward in state across deploys, so that path keeps being reused until it is modified - // in turn. - Generation int `json:"generation,omitempty"` - Lifecycle Lifecycle `json:"-"` } -// RelativePath is the snapshot's location under RemoteRoot: "/". -// It is the resource ID; the last component is the zip content hash (the ZipPath base -// name), which makes the pre-computed path match the uploaded content. A non-zero -// Generation appends a "-" suffix to the hash so a snapshot replacing one that was -// modified outside the bundle uploads to a distinct path (e.g. "1a2b3c4d-.../e3b0c4...-1"). +// RelativePath is the snapshot's location under RemoteRoot: "/". It is the +// resource ID. The name is the ZipPath base name, "-": every new snapshot gets a +// fresh one, because a path can be uploaded to only once (see snapshot.PlanUpload). +// Example: "1a2b3c4d-.../e3b0c4...-0199e2c1-...". func (s *Snapshot) RelativePath() string { - hash := strings.TrimSuffix(path.Base(s.ZipPath), ".zip") - if s.Generation > 0 { - hash = fmt.Sprintf("%s-%d", hash, s.Generation) - } - return path.Join(s.BundleID, hash) + return path.Join(s.BundleID, strings.TrimSuffix(path.Base(s.ZipPath), ".zip")) } // FullPath is the absolute workspace content path of the snapshot: diff --git a/bundle/deploy/snapshot/upload.go b/bundle/deploy/snapshot/upload.go index 1541b23618a..8ddc343203f 100644 --- a/bundle/deploy/snapshot/upload.go +++ b/bundle/deploy/snapshot/upload.go @@ -10,6 +10,8 @@ import ( "os" "path" "path/filepath" + "reflect" + "strings" "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config/resources" @@ -18,6 +20,7 @@ import ( "github.com/databricks/cli/libs/iamutil" "github.com/databricks/cli/libs/snapshot" "github.com/databricks/databricks-sdk-go/apierr" + "github.com/google/uuid" ) // fileLimitWarning is the file count above which immutable folder deployments may fail. @@ -25,9 +28,9 @@ const fileLimitWarning = 1000 type snapshotUpload struct { // clean is true for a fresh plan or deploy: the mutator discards zips staged by a - // previous run (so content-addressed ".zip" files don't accumulate) and - // stages the current one. It is false when applying a pre-existing plan - // (deploy --plan): the mutator reuses the zip staged when the plan was produced + // previous run (so they don't accumulate), decides the snapshot's path and stages the + // current zip. It is false when applying a pre-existing plan (deploy --plan): the mutator + // reuses the zip staged when the plan was produced, and with it the path the plan chose, // instead of rebuilding it. clean bool } @@ -38,11 +41,11 @@ type PlanUploadOptions struct { // PlanUpload returns a mutator that registers the immutable snapshot as an internal // resource. When Clean is true it discards zips staged by a previous run, builds the -// bundle zip, stages it under the bundle's local state directory as ".zip", and -// records the path on the resource; the staged file is uploaded when the resource is -// created on apply. Pass Clean=false when applying a pre-existing plan: the mutator reuses the -// zip staged when the plan was produced. Either way the resource is registered in the config, -// so its path is known to the rest of the deploy before the snapshot is uploaded. +// bundle zip, stages it under the bundle's local state directory as ".zip" (see +// snapshotName), and records the path on the resource; the staged file is uploaded when the +// resource is created on apply. Pass Clean=false when applying a pre-existing plan: the mutator +// reuses the zip staged when the plan was produced. Either way the resource is registered in the +// config, so its path is known to the rest of the deploy before the snapshot is uploaded. func PlanUpload(opts PlanUploadOptions) bundle.Mutator { return &snapshotUpload{clean: opts.Clean} } @@ -69,17 +72,17 @@ func (m *snapshotUpload) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagn return nil } - // Check the previous snapshot before doing any work, so a deploy onto a snapshot that was - // modified outside the bundle fails before building a zip. - generation, err := resolveGeneration(ctx, b, uploader, remoteRoot) - if err != nil { - return diag.FromErr(err) + snap := &resources.Snapshot{ + BundleID: BundleID(b), + ACL: BuildACL(b), + CanManage: BuildCanManage(b), + RemoteRoot: remoteRoot, } var zipPath string var diags diag.Diagnostics if m.clean { - zipPath, diags, err = stageZip(ctx, b) + zipPath, diags, err = stageZip(ctx, b, uploader, snap) } else { zipPath, err = stagedZipPath(ctx, b) } @@ -87,22 +90,23 @@ func (m *snapshotUpload) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagn return diags.Extend(diag.FromErr(err)) } - b.Config.Resources.Snapshots[resources.SnapshotResourceKey] = &resources.Snapshot{ - BundleID: BundleID(b), - ACL: BuildACL(b), - CanManage: BuildCanManage(b), - RemoteRoot: remoteRoot, - ZipPath: filepath.ToSlash(zipPath), - Generation: generation, - } + snap.ZipPath = filepath.ToSlash(zipPath) + b.Config.Resources.Snapshots[resources.SnapshotResourceKey] = snap return diags } // stageZip builds the bundle zip and stages it under the bundle's local state directory as -// ".zip", discarding zips staged by a previous run so content-addressed files don't +// ".zip" (see snapshotName), discarding zips staged by a previous run so they don't // accumulate. -func stageZip(ctx context.Context, b *bundle.Bundle) (string, diag.Diagnostics, error) { +func stageZip(ctx context.Context, b *bundle.Bundle, uploader *snapshot.SnapshotClient, snap *resources.Snapshot) (string, diag.Diagnostics, error) { + // Check the previous snapshot before doing any work, so a deploy onto a snapshot that was + // modified outside the bundle fails before building a zip. + prev, err := checkPrevSnapshot(ctx, b, uploader, snap.RemoteRoot) + if err != nil { + return "", nil, err + } + zipContent, fileCount, err := BundleZip(ctx, b) if err != nil { return "", nil, fmt.Errorf("failed to build snapshot zip: %w", err) @@ -115,7 +119,11 @@ func stageZip(ctx context.Context, b *bundle.Bundle) (string, diag.Diagnostics, if err != nil { return "", nil, err } - zipPath := filepath.Join(dir, snapshot.HashFromContent(zipContent)+".zip") + name, err := snapshotName(snap, prev, dir, snapshot.HashFromContent(zipContent)) + if err != nil { + return "", nil, err + } + zipPath := filepath.Join(dir, name+".zip") if err := os.WriteFile(zipPath, zipContent, 0o600); err != nil { return "", nil, fmt.Errorf("failed to write snapshot zip: %w", err) } @@ -130,6 +138,31 @@ func stageZip(ctx context.Context, b *bundle.Bundle) (string, diag.Diagnostics, return zipPath, diags, nil } +// snapshotName returns the name the zip is staged under, which is also the last component of +// the snapshot's path: "-". A snapshot cannot be deleted, so a path can be +// uploaded to only once and stays taken after "bundle destroy" or a deploy that moved on to other +// content. Every new snapshot therefore goes to a path of its own, made unique by a UUIDv7 +// (a timestamp followed by random bits). The previous snapshot's name is kept only when nothing +// about the snapshot would change, so the deploy leaves it in place instead of uploading it again. +func snapshotName(snap *resources.Snapshot, prev *dresources.SnapshotState, dir, hash string) (string, error) { + if prev != nil { + name := path.Base(prev.RelativePath) + reused := *snap + reused.ZipPath = filepath.ToSlash(filepath.Join(dir, name+".zip")) + // The name carries the hash of the content uploaded to it. The rest of the state has to + // match as well: any difference makes the engine recreate the snapshot, which needs a new path. + if strings.HasPrefix(name, hash+"-") && reflect.DeepEqual((&dresources.ResourceSnapshot{}).PrepareState(&reused), prev) { + return name, nil + } + } + + id, err := uuid.NewV7() + if err != nil { + return "", err + } + return hash + "-" + id.String(), nil +} + // stagedZipPath returns the zip staged when the plan being applied was produced. Each fresh // plan clears the directory before staging, so it holds exactly that one zip. func stagedZipPath(ctx context.Context, b *bundle.Bundle) (string, error) { @@ -203,46 +236,43 @@ func BuildCanManage(b *bundle.Bundle) []snapshot.ManagePrincipal { return canManage } -// resolveGeneration asks the backend whether the snapshot deployed last time was modified -// outside of the bundle, and returns the generation the new snapshot should use: -// - the previous generation while the snapshot is untouched, so a path reached by an earlier -// recovery keeps being reused; -// - the previous generation + 1 when it was modified and --force was given, which moves the -// deployment to a fresh path; +// checkPrevSnapshot asks the backend whether the snapshot deployed last time was modified outside +// of the bundle, and returns its state when the deployment may keep using it: +// - the previous state while the snapshot is untouched; +// - nil when there is no previous snapshot, or it was modified and --force was given, which +// moves the deployment to a new snapshot; // - an error when it was modified and --force was not given. // -// The check has to happen here rather than in the resource's DoRead: the generation decides -// the snapshot's path, and resources referencing ${...full_path} resolve it from the desired -// state computed before any resource is read (see LookupReferencePreDeploy). Erroring unless -// --force also has no equivalent in the resource layer. CheckDashboardsModifiedRemotely is the -// same shape: read state, ask the API, refuse unless forced. -// -// On the first deploy (no snapshot in state) the generation is 0. -func resolveGeneration(ctx context.Context, b *bundle.Bundle, uploader *snapshot.SnapshotClient, remoteRoot string) (int, error) { +// The check has to happen here rather than in the resource's DoRead: it decides the snapshot's +// path, and resources referencing ${...full_path} resolve it from the desired state computed +// before any resource is read (see LookupReferencePreDeploy). Erroring unless --force also has no +// equivalent in the resource layer. CheckDashboardsModifiedRemotely is the same shape: read state, +// ask the API, refuse unless forced. +func checkPrevSnapshot(ctx context.Context, b *bundle.Bundle, uploader *snapshot.SnapshotClient, remoteRoot string) (*dresources.SnapshotState, error) { // The deploy/plan pipeline opens the state DB for read before this runs, but callers that // exercise PlanUpload in isolation (unit tests) may not. No open state means no previous // snapshot to check, which is the first-deploy case. if !b.DeploymentBundle.StateDB.IsOpen() { - return 0, nil + return nil, nil } entry, ok := b.DeploymentBundle.StateDB.GetResourceEntry(resources.SnapshotKey) if !ok || len(entry.State) == 0 { - return 0, nil + return nil, nil } // Decode into the state type the engine persists, so the field names cannot drift apart. var prev dresources.SnapshotState if err := json.Unmarshal(entry.State, &prev); err != nil { - return 0, fmt.Errorf("reading previous snapshot state: %w", err) + return nil, fmt.Errorf("reading previous snapshot state: %w", err) } if prev.RelativePath == "" { - return 0, nil + return nil, nil } // Compose the content path from relative_path rather than reading full_path: relative_path - // already encodes the previous generation and means the same thing in every CLI version, - // while full_path from a CLI that predates the content subfolder points one level too high. + // means the same thing in every CLI version, while full_path from a CLI that predates the + // content subfolder points one level too high. contentPath := path.Join(remoteRoot, prev.RelativePath, snapshot.ContentSubdir) status, err := uploader.InspectSnapshot(ctx, contentPath) @@ -251,16 +281,16 @@ func resolveGeneration(ctx context.Context, b *bundle.Bundle, uploader *snapshot // Failing here would also be a dead end, since this runs before the --force check and so // could not be forced past. if errors.Is(err, apierr.ErrNotFound) { - return prev.Generation, nil + return nil, nil } if err != nil { - return 0, err + return nil, err } if !status.Dirty { - return prev.Generation, nil + return &prev, nil } if !b.Config.Bundle.Force { - return 0, fmt.Errorf("the last deployment was modified in the workspace. Changes might be overwritten by this deployment. To confirm it, use the --force flag") + return nil, errors.New("the last deployment was modified in the workspace. Changes might be overwritten by this deployment. To confirm it, use the --force flag") } - return prev.Generation + 1, nil + return nil, nil } diff --git a/bundle/deploy/snapshot/upload_warning_test.go b/bundle/deploy/snapshot/upload_warning_test.go index 1a17c62a3a4..1d66b30d17b 100644 --- a/bundle/deploy/snapshot/upload_warning_test.go +++ b/bundle/deploy/snapshot/upload_warning_test.go @@ -6,11 +6,13 @@ import ( "math" "os" "path/filepath" + "strings" "testing" "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/direct/dresources" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/logdiag" @@ -164,3 +166,61 @@ func TestUploadNotCleanWithoutStagedZip(t *testing.T) { require.True(t, diags.HasError()) assert.Contains(t, diags[0].Summary, "run 'bundle plan' again") } + +func TestSnapshotName(t *testing.T) { + const hash = "e3b0c4" + const prevName = hash + "-0199e2c1-7b1a-7c2d-8e3f-4a5b6c7d8e9f" + snap := &resources.Snapshot{ + BundleID: "bundle-id", + ACL: []snapshot.ACLEntry{{UserName: "deployer@example.test", PermissionLevel: "CAN_READ"}}, + CanManage: []snapshot.ManagePrincipal{{UserName: "deployer@example.test"}}, + RemoteRoot: "/Workspace/Users/sp/.snapshots", + } + unchanged := &dresources.SnapshotState{ + RelativePath: "bundle-id/" + prevName, + FullPath: "/Workspace/Users/sp/.snapshots/bundle-id/" + prevName + "/snapshot", + BundleID: "bundle-id", + ACL: []snapshot.ACLEntry{{UserName: "deployer@example.test", PermissionLevel: "CAN_READ"}}, + CanManage: []snapshot.ManagePrincipal{{UserName: "deployer@example.test"}}, + ZipPath: "/stage/" + prevName + ".zip", + } + otherManagers := *unchanged + otherManagers.CanManage = []snapshot.ManagePrincipal{{GroupName: "admins"}} + + tests := []struct { + name string + prev *dresources.SnapshotState + dir string + hash string + reused bool + }{ + {name: "no previous snapshot", prev: nil, dir: "/stage", hash: hash, reused: false}, + {name: "unchanged", prev: unchanged, dir: "/stage", hash: hash, reused: true}, + {name: "content changed", prev: unchanged, dir: "/stage", hash: "a1b2c3", reused: false}, + {name: "staged elsewhere", prev: unchanged, dir: "/other", hash: hash, reused: false}, + {name: "managers changed", prev: &otherManagers, dir: "/stage", hash: hash, reused: false}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + name, err := snapshotName(snap, tc.prev, tc.dir, tc.hash) + require.NoError(t, err) + if tc.reused { + assert.Equal(t, prevName, name) + return + } + assert.NotEqual(t, prevName, name) + assert.True(t, strings.HasPrefix(name, tc.hash+"-"), name) + }) + } +} + +func TestSnapshotNameIsUniquePerUpload(t *testing.T) { + snap := &resources.Snapshot{BundleID: "bundle-id"} + + first, err := snapshotName(snap, nil, "/stage", "e3b0c4") + require.NoError(t, err) + second, err := snapshotName(snap, nil, "/stage", "e3b0c4") + require.NoError(t, err) + + assert.NotEqual(t, first, second) +} diff --git a/bundle/direct/dresources/snapshot.go b/bundle/direct/dresources/snapshot.go index 88b6ac7c64a..02c562a0565 100644 --- a/bundle/direct/dresources/snapshot.go +++ b/bundle/direct/dresources/snapshot.go @@ -27,12 +27,9 @@ type SnapshotState struct { CanManage []snapshot.ManagePrincipal `json:"can_manage,omitempty"` // ZipPath locates the bundle zip staged locally by the deploy pipeline. It is // small enough to round-trip through the plan file, so deploying from a plan - // restores it directly (no re-injection needed). The file name is the content - // hash, which is also the last component of RelativePath. + // restores it directly (no re-injection needed). The file name is also the last + // component of RelativePath. ZipPath string `json:"zip_path"` - // Generation is folded into RelativePath (see resources.Snapshot.Generation). Carried - // forward in state; bumped when --force moves off a snapshot modified outside the bundle. - Generation int `json:"generation,omitempty"` } type SnapshotRemote struct { @@ -66,7 +63,6 @@ func (s *ResourceSnapshot) PrepareState(input *resources.Snapshot) *SnapshotStat ACL: input.ACL, CanManage: input.CanManage, ZipPath: input.ZipPath, - Generation: input.Generation, } } @@ -78,7 +74,6 @@ func (s *ResourceSnapshot) RemapState(remote *SnapshotRemote) *SnapshotState { ACL: nil, CanManage: nil, ZipPath: "", - Generation: 0, } } diff --git a/bundle/direct/dresources/type_test.go b/bundle/direct/dresources/type_test.go index 2d6d8133b8e..9fc611d245a 100644 --- a/bundle/direct/dresources/type_test.go +++ b/bundle/direct/dresources/type_test.go @@ -58,7 +58,6 @@ var knownMissingInRemoteType = map[string][]string{ "acl", "can_manage", "zip_path", - "generation", }, "job_runs": { // Local-only trigger fingerprints under lifecycle. diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go index 685ba778ef1..dc8b3c2de7a 100644 --- a/libs/testserver/handlers.go +++ b/libs/testserver/handlers.go @@ -932,6 +932,16 @@ func AddDefaultHandlers(server *Server) { // relative_path. The real API uses the workspace user UUID (not email) in the path, // matching service-principal identities used in cloud acceptance tests. contentPath := fmt.Sprintf("/Workspace/Users/%s/.snapshots/%s/snapshot", TestUserSP.UserName, relativePath) + // Snapshots are immutable, so a relative_path can be created only once. The real API + // answers with a 400 that carries the code in the message and sets no error_code. + if _, exists := req.Workspace.SnapshotInfo(contentPath); exists { + return Response{ + StatusCode: http.StatusBadRequest, + Body: map[string]string{ + "message": "RESOURCE_ALREADY_EXISTS: relative_path already exists", + }, + } + } req.Workspace.WorkspaceMkdirs(workspace.Mkdirs{Path: contentPath}) // Record after mkdirs so the snapshot's own creation write is not counted as tampering. req.Workspace.RecordSnapshot(contentPath, relativePath, acl, canManage) From 6ea080879ef2b05023ac9768140e239c672a5906 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Wed, 14 Oct 2026 13:51:24 +0200 Subject: [PATCH 11/16] fixed the test output --- acceptance/bundle/deploy/immutable-dev-mode/output.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/acceptance/bundle/deploy/immutable-dev-mode/output.txt b/acceptance/bundle/deploy/immutable-dev-mode/output.txt index 9d38d6c271e..51b14e79830 100644 --- a/acceptance/bundle/deploy/immutable-dev-mode/output.txt +++ b/acceptance/bundle/deploy/immutable-dev-mode/output.txt @@ -26,7 +26,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] jobs get [MY_JOB_ID] { "name": "[dev [USERNAME]] my job", - "python_file": "/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]/files/src/main.py" + "python_file": "/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]-[UUID]/snapshot/files/src/main.py" } >>> [CLI] bundle destroy -t dev --auto-approve From eb4f1921ca743aa1a075737a399bcfebe53d8227 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Thu, 15 Oct 2026 13:10:44 +0200 Subject: [PATCH 12/16] move and enable tests on Cloud --- .../deploy/immutable-breakglass/out.test.toml | 3 -- .../deploy/immutable-breakglass/test.toml | 21 -------- .../deploy/immutable-can-manage/out.test.toml | 3 -- .../bundle/deploy/immutable-can-manage/script | 20 -------- .../deploy/immutable-can-manage/test.toml | 21 -------- .../immutable-destroy-redeploy/out.test.toml | 3 -- .../immutable-destroy-redeploy/test.toml | 14 ----- .../deploy/immutable-dev-mode/test.toml | 14 ----- .../immutable-legacy-state/out.test.toml | 3 -- .../deploy/immutable-legacy-state/test.toml | 25 --------- .../immutable-missing-snapshot/out.test.toml | 3 -- .../immutable-missing-snapshot/test.toml | 19 ------- .../deploy/immutable-no-artifacts/test.toml | 23 --------- .../immutable-permissions-change/test.toml | 13 ----- .../immutable/{ => basic}/databricks.yml.tmpl | 0 .../basic}/out.test.toml | 2 +- .../deploy/immutable/{ => basic}/output.txt | 0 .../immutable/{ => basic}/pyproject.toml | 0 .../deploy/immutable/{ => basic}/script | 0 .../{ => basic}/src/immutable/__init__.py | 0 .../{ => basic}/src/immutable/main.py | 0 .../basic}/src/main.py | 0 .../basic}/src/notebook.py | 0 .../bundle/deploy/immutable/basic/test.toml | 7 +++ .../breakglass}/databricks.yml.tmpl | 0 .../immutable/{ => breakglass}/out.test.toml | 2 +- .../breakglass}/output.txt | 3 +- .../breakglass}/script | 6 +-- .../breakglass}/src/main.py | 0 .../deploy/immutable/breakglass/test.toml | 4 ++ .../can-manage}/databricks.yml.tmpl | 2 +- .../deploy/immutable/can-manage/out.test.toml | 3 ++ .../can-manage}/output.txt | 24 ++++----- .../bundle/deploy/immutable/can-manage/script | 17 +++++++ .../can-manage}/src/main.py | 0 .../deploy/immutable/can-manage/test.toml | 10 ++++ .../destroy-redeploy}/databricks.yml.tmpl | 0 .../destroy-redeploy}/out.test.toml | 2 +- .../destroy-redeploy}/output.txt | 0 .../destroy-redeploy}/script | 0 .../destroy-redeploy}/src/main.py | 0 .../dev-mode}/databricks.yml.tmpl | 0 .../deploy/immutable/dev-mode/out.test.toml | 2 + .../dev-mode}/output.txt | 0 .../dev-mode}/script | 0 .../dev-mode}/src/main.py | 0 .../legacy-state}/databricks.yml.tmpl | 0 .../immutable/legacy-state/out.test.toml | 2 + .../legacy-state}/output.txt | 3 +- .../legacy-state}/script | 2 +- .../legacy-state}/src/main.py | 0 .../deploy/immutable/legacy-state/test.toml | 8 +++ .../missing-snapshot}/databricks.yml.tmpl | 0 .../immutable/missing-snapshot/out.test.toml | 2 + .../missing-snapshot}/output.txt | 0 .../missing-snapshot}/script | 0 .../missing-snapshot}/src/main.py | 0 .../immutable/missing-snapshot/test.toml | 7 +++ .../no-artifacts}/.gitignore | 0 .../no-artifacts}/databricks.yml.tmpl | 0 .../no-artifacts}/out.test.toml | 2 +- .../no-artifacts}/output.txt | 0 .../no-artifacts}/script | 0 .../no-artifacts}/src/main.py | 0 .../{ => no-artifacts}/src/notebook.py | 0 .../deploy/immutable/no-artifacts/test.toml | 12 +++++ .../permissions-change}/databricks.yml.tmpl | 0 .../permissions-change/out.test.toml | 2 + .../permissions-change}/output.txt | 0 .../permissions-change}/script | 0 .../{ => permissions-change}/src/main.py | 0 acceptance/bundle/deploy/immutable/test.toml | 9 +--- libs/testserver/fake_workspace.go | 31 +++++------ libs/testserver/handlers.go | 51 ++++++++++++++----- 74 files changed, 152 insertions(+), 248 deletions(-) delete mode 100644 acceptance/bundle/deploy/immutable-breakglass/out.test.toml delete mode 100644 acceptance/bundle/deploy/immutable-breakglass/test.toml delete mode 100644 acceptance/bundle/deploy/immutable-can-manage/out.test.toml delete mode 100644 acceptance/bundle/deploy/immutable-can-manage/script delete mode 100644 acceptance/bundle/deploy/immutable-can-manage/test.toml delete mode 100644 acceptance/bundle/deploy/immutable-destroy-redeploy/out.test.toml delete mode 100644 acceptance/bundle/deploy/immutable-destroy-redeploy/test.toml delete mode 100644 acceptance/bundle/deploy/immutable-dev-mode/test.toml delete mode 100644 acceptance/bundle/deploy/immutable-legacy-state/out.test.toml delete mode 100644 acceptance/bundle/deploy/immutable-legacy-state/test.toml delete mode 100644 acceptance/bundle/deploy/immutable-missing-snapshot/out.test.toml delete mode 100644 acceptance/bundle/deploy/immutable-missing-snapshot/test.toml delete mode 100644 acceptance/bundle/deploy/immutable-no-artifacts/test.toml delete mode 100644 acceptance/bundle/deploy/immutable-permissions-change/test.toml rename acceptance/bundle/deploy/immutable/{ => basic}/databricks.yml.tmpl (100%) rename acceptance/bundle/deploy/{immutable-permissions-change => immutable/basic}/out.test.toml (67%) rename acceptance/bundle/deploy/immutable/{ => basic}/output.txt (100%) rename acceptance/bundle/deploy/immutable/{ => basic}/pyproject.toml (100%) rename acceptance/bundle/deploy/immutable/{ => basic}/script (100%) rename acceptance/bundle/deploy/immutable/{ => basic}/src/immutable/__init__.py (100%) rename acceptance/bundle/deploy/immutable/{ => basic}/src/immutable/main.py (100%) rename acceptance/bundle/deploy/{immutable-breakglass => immutable/basic}/src/main.py (100%) rename acceptance/bundle/deploy/{immutable-no-artifacts => immutable/basic}/src/notebook.py (100%) create mode 100644 acceptance/bundle/deploy/immutable/basic/test.toml rename acceptance/bundle/deploy/{immutable-breakglass => immutable/breakglass}/databricks.yml.tmpl (100%) rename acceptance/bundle/deploy/immutable/{ => breakglass}/out.test.toml (67%) rename acceptance/bundle/deploy/{immutable-breakglass => immutable/breakglass}/output.txt (90%) rename acceptance/bundle/deploy/{immutable-breakglass => immutable/breakglass}/script (82%) rename acceptance/bundle/deploy/{immutable-can-manage => immutable/breakglass}/src/main.py (100%) create mode 100644 acceptance/bundle/deploy/immutable/breakglass/test.toml rename acceptance/bundle/deploy/{immutable-can-manage => immutable/can-manage}/databricks.yml.tmpl (90%) create mode 100644 acceptance/bundle/deploy/immutable/can-manage/out.test.toml rename acceptance/bundle/deploy/{immutable-can-manage => immutable/can-manage}/output.txt (84%) create mode 100644 acceptance/bundle/deploy/immutable/can-manage/script rename acceptance/bundle/deploy/{immutable-destroy-redeploy => immutable/can-manage}/src/main.py (100%) create mode 100644 acceptance/bundle/deploy/immutable/can-manage/test.toml rename acceptance/bundle/deploy/{immutable-destroy-redeploy => immutable/destroy-redeploy}/databricks.yml.tmpl (100%) rename acceptance/bundle/deploy/{immutable-dev-mode => immutable/destroy-redeploy}/out.test.toml (67%) rename acceptance/bundle/deploy/{immutable-destroy-redeploy => immutable/destroy-redeploy}/output.txt (100%) rename acceptance/bundle/deploy/{immutable-destroy-redeploy => immutable/destroy-redeploy}/script (100%) rename acceptance/bundle/deploy/{immutable-legacy-state => immutable/destroy-redeploy}/src/main.py (100%) rename acceptance/bundle/deploy/{immutable-dev-mode => immutable/dev-mode}/databricks.yml.tmpl (100%) create mode 100644 acceptance/bundle/deploy/immutable/dev-mode/out.test.toml rename acceptance/bundle/deploy/{immutable-dev-mode => immutable/dev-mode}/output.txt (100%) rename acceptance/bundle/deploy/{immutable-dev-mode => immutable/dev-mode}/script (100%) rename acceptance/bundle/deploy/{immutable-dev-mode => immutable/dev-mode}/src/main.py (100%) rename acceptance/bundle/deploy/{immutable-legacy-state => immutable/legacy-state}/databricks.yml.tmpl (100%) create mode 100644 acceptance/bundle/deploy/immutable/legacy-state/out.test.toml rename acceptance/bundle/deploy/{immutable-legacy-state => immutable/legacy-state}/output.txt (87%) rename acceptance/bundle/deploy/{immutable-legacy-state => immutable/legacy-state}/script (90%) rename acceptance/bundle/deploy/{immutable-missing-snapshot => immutable/legacy-state}/src/main.py (100%) create mode 100644 acceptance/bundle/deploy/immutable/legacy-state/test.toml rename acceptance/bundle/deploy/{immutable-missing-snapshot => immutable/missing-snapshot}/databricks.yml.tmpl (100%) create mode 100644 acceptance/bundle/deploy/immutable/missing-snapshot/out.test.toml rename acceptance/bundle/deploy/{immutable-missing-snapshot => immutable/missing-snapshot}/output.txt (100%) rename acceptance/bundle/deploy/{immutable-missing-snapshot => immutable/missing-snapshot}/script (100%) rename acceptance/bundle/deploy/{immutable-no-artifacts => immutable/missing-snapshot}/src/main.py (100%) create mode 100644 acceptance/bundle/deploy/immutable/missing-snapshot/test.toml rename acceptance/bundle/deploy/{immutable-no-artifacts => immutable/no-artifacts}/.gitignore (100%) rename acceptance/bundle/deploy/{immutable-no-artifacts => immutable/no-artifacts}/databricks.yml.tmpl (100%) rename acceptance/bundle/deploy/{immutable-no-artifacts => immutable/no-artifacts}/out.test.toml (81%) rename acceptance/bundle/deploy/{immutable-no-artifacts => immutable/no-artifacts}/output.txt (100%) rename acceptance/bundle/deploy/{immutable-no-artifacts => immutable/no-artifacts}/script (100%) rename acceptance/bundle/deploy/{immutable-permissions-change => immutable/no-artifacts}/src/main.py (100%) rename acceptance/bundle/deploy/immutable/{ => no-artifacts}/src/notebook.py (100%) create mode 100644 acceptance/bundle/deploy/immutable/no-artifacts/test.toml rename acceptance/bundle/deploy/{immutable-permissions-change => immutable/permissions-change}/databricks.yml.tmpl (100%) create mode 100644 acceptance/bundle/deploy/immutable/permissions-change/out.test.toml rename acceptance/bundle/deploy/{immutable-permissions-change => immutable/permissions-change}/output.txt (100%) rename acceptance/bundle/deploy/{immutable-permissions-change => immutable/permissions-change}/script (100%) rename acceptance/bundle/deploy/immutable/{ => permissions-change}/src/main.py (100%) diff --git a/acceptance/bundle/deploy/immutable-breakglass/out.test.toml b/acceptance/bundle/deploy/immutable-breakglass/out.test.toml deleted file mode 100644 index 59b56a2037c..00000000000 --- a/acceptance/bundle/deploy/immutable-breakglass/out.test.toml +++ /dev/null @@ -1,3 +0,0 @@ -Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/deploy/immutable-breakglass/test.toml b/acceptance/bundle/deploy/immutable-breakglass/test.toml deleted file mode 100644 index 064e7eeff9b..00000000000 --- a/acceptance/bundle/deploy/immutable-breakglass/test.toml +++ /dev/null @@ -1,21 +0,0 @@ -Cloud = false # Temporary disable cloud tests until the API is fully available - -# immutable_folder only works with the direct engine. -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] - -Ignore = [ - "databricks.yml", - ".databricks", -] - -# This test passes the snapshot's absolute workspace path to "workspace mkdirs" to modify it out -# of band. On Windows the script runs under MSYS2, which rewrites such leading-slash arguments to -# Windows paths (e.g. C:/Program Files/Git/Workspace/...) before the CLI sees them, so the write -# would land outside the snapshot and leave it looking untouched. -# Setting this environment variable disables that conversion. -Env.MSYS_NO_PATHCONV = "1" - -[[Repls]] -# Replace snapshot hash with SNAPSHOT_HASH -Old = "[0-9a-f]{64}" -New = "[SNAPSHOT_HASH]" diff --git a/acceptance/bundle/deploy/immutable-can-manage/out.test.toml b/acceptance/bundle/deploy/immutable-can-manage/out.test.toml deleted file mode 100644 index 59b56a2037c..00000000000 --- a/acceptance/bundle/deploy/immutable-can-manage/out.test.toml +++ /dev/null @@ -1,3 +0,0 @@ -Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/deploy/immutable-can-manage/script b/acceptance/bundle/deploy/immutable-can-manage/script deleted file mode 100644 index 99ab49414a1..00000000000 --- a/acceptance/bundle/deploy/immutable-can-manage/script +++ /dev/null @@ -1,20 +0,0 @@ -envsubst < databricks.yml.tmpl > databricks.yml - -cleanup() { - trace $CLI bundle destroy --auto-approve -} -trap cleanup EXIT - -title "Deploy with a CAN_MANAGE permission entry" -trace $CLI bundle deploy - -title "Check can_manage_principals sent to the snapshot create API" -# The testserver stores parsed form fields per snapshot and exposes them via a query endpoint. -# The snapshot resource ID is its relative_path; read it from the post-deploy plan where the -# remote_state is populated and errors/warnings go to a log file to keep the output clean. -$CLI bundle plan -o json > plan.json 2> LOG.plan -RELATIVE_PATH=$(jq -r \ - '.plan["resources.internal_immutable_snapshots.immutable"].remote_state.relative_path' \ - plan.json) -trace $CLI api get "/api/2.0/snapshots/info?relative_path=$RELATIVE_PATH" | \ - jq '{can_manage_principals: .can_manage_principals}' diff --git a/acceptance/bundle/deploy/immutable-can-manage/test.toml b/acceptance/bundle/deploy/immutable-can-manage/test.toml deleted file mode 100644 index 32b9edbb62b..00000000000 --- a/acceptance/bundle/deploy/immutable-can-manage/test.toml +++ /dev/null @@ -1,21 +0,0 @@ -Cloud = false # Temporary disable cloud tests until the API is fully available - -# immutable_folder only works with the direct engine. -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] - -Ignore = [ - "databricks.yml", - ".databricks", - "plan.json", - "LOG.*", -] - -# The test passes an absolute workspace path to "workspace mkdirs" and also queries -# /api/2.0/snapshots/info. On Windows MSYS2 rewrites leading-slash arguments to Windows -# paths before the CLI sees them, so disable that conversion. -Env.MSYS_NO_PATHCONV = "1" - -[[Repls]] -# Replace snapshot hash with SNAPSHOT_HASH -Old = "[0-9a-f]{64}" -New = "[SNAPSHOT_HASH]" diff --git a/acceptance/bundle/deploy/immutable-destroy-redeploy/out.test.toml b/acceptance/bundle/deploy/immutable-destroy-redeploy/out.test.toml deleted file mode 100644 index 59b56a2037c..00000000000 --- a/acceptance/bundle/deploy/immutable-destroy-redeploy/out.test.toml +++ /dev/null @@ -1,3 +0,0 @@ -Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/deploy/immutable-destroy-redeploy/test.toml b/acceptance/bundle/deploy/immutable-destroy-redeploy/test.toml deleted file mode 100644 index 88b225ea71f..00000000000 --- a/acceptance/bundle/deploy/immutable-destroy-redeploy/test.toml +++ /dev/null @@ -1,14 +0,0 @@ -Cloud = false # Temporary disable cloud tests until the API is fully available - -# immutable_folder only works with the direct engine. -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] - -Ignore = [ - "databricks.yml", - ".databricks", -] - -[[Repls]] -# Replace snapshot hash with SNAPSHOT_HASH -Old = "[0-9a-f]{64}" -New = "[SNAPSHOT_HASH]" diff --git a/acceptance/bundle/deploy/immutable-dev-mode/test.toml b/acceptance/bundle/deploy/immutable-dev-mode/test.toml deleted file mode 100644 index 267c545f8e0..00000000000 --- a/acceptance/bundle/deploy/immutable-dev-mode/test.toml +++ /dev/null @@ -1,14 +0,0 @@ -Cloud = false # Temporary disable cloud tests until the API is fully available - -# immutable_folder only works with the direct engine. - -Ignore = [ - "databricks.yml", - ".databricks", -] - -# Normalize the content-addressed snapshot hash so it doesn't need to be -# hardcoded in output.txt and the test stays stable across file changes. -[[Repls]] -Old = '[0-9a-f]{64}' -New = '[SNAPSHOT_HASH]' diff --git a/acceptance/bundle/deploy/immutable-legacy-state/out.test.toml b/acceptance/bundle/deploy/immutable-legacy-state/out.test.toml deleted file mode 100644 index 27ec2a7fcd6..00000000000 --- a/acceptance/bundle/deploy/immutable-legacy-state/out.test.toml +++ /dev/null @@ -1,3 +0,0 @@ -Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/deploy/immutable-legacy-state/test.toml b/acceptance/bundle/deploy/immutable-legacy-state/test.toml deleted file mode 100644 index 1a22649e622..00000000000 --- a/acceptance/bundle/deploy/immutable-legacy-state/test.toml +++ /dev/null @@ -1,25 +0,0 @@ -Cloud = false # Temporary disable cloud tests until the API is fully available - -# immutable_folder only works with the direct engine. -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] - -# The test rewrites the deployment state by hand, which the recording service refuses because it -# no longer matches the deployment it has recorded, so run without deployment history. -EnvMatrix.DMS = [""] - -Ignore = [ - "databricks.yml", - ".databricks", -] - -# This test passes the snapshot's absolute workspace path to "workspace mkdirs" to modify it out -# of band. On Windows the script runs under MSYS2, which rewrites such leading-slash arguments to -# Windows paths (e.g. C:/Program Files/Git/Workspace/...) before the CLI sees them, so the write -# would land outside the snapshot and leave it looking untouched. -# Setting this environment variable disables that conversion. -Env.MSYS_NO_PATHCONV = "1" - -[[Repls]] -# Replace snapshot hash with SNAPSHOT_HASH -Old = "[0-9a-f]{64}" -New = "[SNAPSHOT_HASH]" diff --git a/acceptance/bundle/deploy/immutable-missing-snapshot/out.test.toml b/acceptance/bundle/deploy/immutable-missing-snapshot/out.test.toml deleted file mode 100644 index 27ec2a7fcd6..00000000000 --- a/acceptance/bundle/deploy/immutable-missing-snapshot/out.test.toml +++ /dev/null @@ -1,3 +0,0 @@ -Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/deploy/immutable-missing-snapshot/test.toml b/acceptance/bundle/deploy/immutable-missing-snapshot/test.toml deleted file mode 100644 index c191a87d736..00000000000 --- a/acceptance/bundle/deploy/immutable-missing-snapshot/test.toml +++ /dev/null @@ -1,19 +0,0 @@ -Cloud = false # Temporary disable cloud tests until the API is fully available - -# immutable_folder only works with the direct engine. -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] - -# The test rewrites the deployment state by hand, which the recording service refuses because it -# no longer matches the deployment it has recorded, so run without deployment history. -EnvMatrix.DMS = [""] - -Ignore = [ - "databricks.yml", - ".databricks", - "tmp.*", -] - -[[Repls]] -# Replace snapshot hash with SNAPSHOT_HASH -Old = "[0-9a-f]{64}" -New = "[SNAPSHOT_HASH]" diff --git a/acceptance/bundle/deploy/immutable-no-artifacts/test.toml b/acceptance/bundle/deploy/immutable-no-artifacts/test.toml deleted file mode 100644 index a2373c4f892..00000000000 --- a/acceptance/bundle/deploy/immutable-no-artifacts/test.toml +++ /dev/null @@ -1,23 +0,0 @@ -Cloud = false # Temporary disable cloud tests until the API is fully available -EnvMatrix.READPLAN = ["", "1"] - -# immutable_folder only works with the direct engine. - -Ignore = [ - "databricks.yml", - ".databricks", - ".venv", - "*.pyc", -] - -# Normalize the content-addressed snapshot hash so it doesn't need to be -# hardcoded in output.txt and the test stays stable across file changes. -[[Repls]] -Old = '[0-9a-f]{64}' -New = '[SNAPSHOT_HASH]' - -# When READPLAN=1, "bundle deploy" is called as "bundle deploy --plan plan.json". -# Normalize so both variants produce identical output. -[[Repls]] -Old = ' --plan plan.json' -New = '' diff --git a/acceptance/bundle/deploy/immutable-permissions-change/test.toml b/acceptance/bundle/deploy/immutable-permissions-change/test.toml deleted file mode 100644 index 0439d8243bc..00000000000 --- a/acceptance/bundle/deploy/immutable-permissions-change/test.toml +++ /dev/null @@ -1,13 +0,0 @@ -Cloud = false # Temporary disable cloud tests until the API is fully available - -# immutable_folder only works with the direct engine. - -Ignore = [ - "databricks.yml", - ".databricks", -] - -[[Repls]] -# Replace snapshot hash with SNAPSHOT_HASH -Old = "[0-9a-f]{64}" -New = "[SNAPSHOT_HASH]" diff --git a/acceptance/bundle/deploy/immutable/databricks.yml.tmpl b/acceptance/bundle/deploy/immutable/basic/databricks.yml.tmpl similarity index 100% rename from acceptance/bundle/deploy/immutable/databricks.yml.tmpl rename to acceptance/bundle/deploy/immutable/basic/databricks.yml.tmpl diff --git a/acceptance/bundle/deploy/immutable-permissions-change/out.test.toml b/acceptance/bundle/deploy/immutable/basic/out.test.toml similarity index 67% rename from acceptance/bundle/deploy/immutable-permissions-change/out.test.toml rename to acceptance/bundle/deploy/immutable/basic/out.test.toml index a927a5fbc06..0f167effef1 100644 --- a/acceptance/bundle/deploy/immutable-permissions-change/out.test.toml +++ b/acceptance/bundle/deploy/immutable/basic/out.test.toml @@ -1,2 +1,2 @@ -Cloud = false +Cloud = true EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/deploy/immutable/output.txt b/acceptance/bundle/deploy/immutable/basic/output.txt similarity index 100% rename from acceptance/bundle/deploy/immutable/output.txt rename to acceptance/bundle/deploy/immutable/basic/output.txt diff --git a/acceptance/bundle/deploy/immutable/pyproject.toml b/acceptance/bundle/deploy/immutable/basic/pyproject.toml similarity index 100% rename from acceptance/bundle/deploy/immutable/pyproject.toml rename to acceptance/bundle/deploy/immutable/basic/pyproject.toml diff --git a/acceptance/bundle/deploy/immutable/script b/acceptance/bundle/deploy/immutable/basic/script similarity index 100% rename from acceptance/bundle/deploy/immutable/script rename to acceptance/bundle/deploy/immutable/basic/script diff --git a/acceptance/bundle/deploy/immutable/src/immutable/__init__.py b/acceptance/bundle/deploy/immutable/basic/src/immutable/__init__.py similarity index 100% rename from acceptance/bundle/deploy/immutable/src/immutable/__init__.py rename to acceptance/bundle/deploy/immutable/basic/src/immutable/__init__.py diff --git a/acceptance/bundle/deploy/immutable/src/immutable/main.py b/acceptance/bundle/deploy/immutable/basic/src/immutable/main.py similarity index 100% rename from acceptance/bundle/deploy/immutable/src/immutable/main.py rename to acceptance/bundle/deploy/immutable/basic/src/immutable/main.py diff --git a/acceptance/bundle/deploy/immutable-breakglass/src/main.py b/acceptance/bundle/deploy/immutable/basic/src/main.py similarity index 100% rename from acceptance/bundle/deploy/immutable-breakglass/src/main.py rename to acceptance/bundle/deploy/immutable/basic/src/main.py diff --git a/acceptance/bundle/deploy/immutable-no-artifacts/src/notebook.py b/acceptance/bundle/deploy/immutable/basic/src/notebook.py similarity index 100% rename from acceptance/bundle/deploy/immutable-no-artifacts/src/notebook.py rename to acceptance/bundle/deploy/immutable/basic/src/notebook.py diff --git a/acceptance/bundle/deploy/immutable/basic/test.toml b/acceptance/bundle/deploy/immutable/basic/test.toml new file mode 100644 index 00000000000..dc5a1df1c80 --- /dev/null +++ b/acceptance/bundle/deploy/immutable/basic/test.toml @@ -0,0 +1,7 @@ +Ignore = [ + "dist", + "build", + ".venv", + "*.pyc", + "src/*.egg-info", +] diff --git a/acceptance/bundle/deploy/immutable-breakglass/databricks.yml.tmpl b/acceptance/bundle/deploy/immutable/breakglass/databricks.yml.tmpl similarity index 100% rename from acceptance/bundle/deploy/immutable-breakglass/databricks.yml.tmpl rename to acceptance/bundle/deploy/immutable/breakglass/databricks.yml.tmpl diff --git a/acceptance/bundle/deploy/immutable/out.test.toml b/acceptance/bundle/deploy/immutable/breakglass/out.test.toml similarity index 67% rename from acceptance/bundle/deploy/immutable/out.test.toml rename to acceptance/bundle/deploy/immutable/breakglass/out.test.toml index a927a5fbc06..0f167effef1 100644 --- a/acceptance/bundle/deploy/immutable/out.test.toml +++ b/acceptance/bundle/deploy/immutable/breakglass/out.test.toml @@ -1,2 +1,2 @@ -Cloud = false +Cloud = true EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/deploy/immutable-breakglass/output.txt b/acceptance/bundle/deploy/immutable/breakglass/output.txt similarity index 90% rename from acceptance/bundle/deploy/immutable-breakglass/output.txt rename to acceptance/bundle/deploy/immutable/breakglass/output.txt index f2e33387c06..ba71879f6f0 100644 --- a/acceptance/bundle/deploy/immutable-breakglass/output.txt +++ b/acceptance/bundle/deploy/immutable/breakglass/output.txt @@ -8,7 +8,8 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged First snapshot path: /Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]-[UUID]/snapshot/files/src/main.py === Break the glass out of band ->>> [CLI] workspace mkdirs /Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]-[UUID]/snapshot/tampered +>>> [CLI] api post /api/2.0/snapshots:breakGlass --json {"snapshot_content_path": "/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]-[UUID]/snapshot"} +true === Deploy without --force fails >>> musterr [CLI] bundle deploy diff --git a/acceptance/bundle/deploy/immutable-breakglass/script b/acceptance/bundle/deploy/immutable/breakglass/script similarity index 82% rename from acceptance/bundle/deploy/immutable-breakglass/script rename to acceptance/bundle/deploy/immutable/breakglass/script index eafcfea63ec..b504207ce10 100644 --- a/acceptance/bundle/deploy/immutable-breakglass/script +++ b/acceptance/bundle/deploy/immutable/breakglass/script @@ -13,10 +13,10 @@ SNAPSHOT_PATH_1=$($CLI jobs get "$JOB_ID" | jq -r '.settings.tasks[0].spark_pyth echo "First snapshot path: $SNAPSHOT_PATH_1" title "Break the glass out of band" -# The snapshot content path holds files/ and artifacts/; writing into it out of band is what -# the backend reports as a modified (dirty) snapshot. +# Breaking the glass grants write access to the snapshot content path (the folder holding files/ +# and artifacts/) and marks the snapshot modified (dirty) for good. CONTENT_PATH="${SNAPSHOT_PATH_1%/files/src/main.py}" -trace $CLI workspace mkdirs "$CONTENT_PATH/tampered" +trace $CLI api post /api/2.0/snapshots:breakGlass --json "{\"snapshot_content_path\": \"$CONTENT_PATH\"}" | jq .status.dirty title "Deploy without --force fails" trace musterr $CLI bundle deploy diff --git a/acceptance/bundle/deploy/immutable-can-manage/src/main.py b/acceptance/bundle/deploy/immutable/breakglass/src/main.py similarity index 100% rename from acceptance/bundle/deploy/immutable-can-manage/src/main.py rename to acceptance/bundle/deploy/immutable/breakglass/src/main.py diff --git a/acceptance/bundle/deploy/immutable/breakglass/test.toml b/acceptance/bundle/deploy/immutable/breakglass/test.toml new file mode 100644 index 00000000000..07b549c7195 --- /dev/null +++ b/acceptance/bundle/deploy/immutable/breakglass/test.toml @@ -0,0 +1,4 @@ +# The test breaks the glass with "api post /api/2.0/snapshots:breakGlass". On Windows the script +# runs under MSYS2, which rewrites leading-slash arguments to Windows paths before the CLI sees +# them. Setting this environment variable disables that conversion. +Env.MSYS_NO_PATHCONV = "1" diff --git a/acceptance/bundle/deploy/immutable-can-manage/databricks.yml.tmpl b/acceptance/bundle/deploy/immutable/can-manage/databricks.yml.tmpl similarity index 90% rename from acceptance/bundle/deploy/immutable-can-manage/databricks.yml.tmpl rename to acceptance/bundle/deploy/immutable/can-manage/databricks.yml.tmpl index a06b94c3532..96e57c53fe5 100644 --- a/acceptance/bundle/deploy/immutable-can-manage/databricks.yml.tmpl +++ b/acceptance/bundle/deploy/immutable/can-manage/databricks.yml.tmpl @@ -6,7 +6,7 @@ experimental: permissions: - level: CAN_MANAGE - user_name: external-manager@databricks.com + user_name: $TEST_USER_EMAIL resources: jobs: diff --git a/acceptance/bundle/deploy/immutable/can-manage/out.test.toml b/acceptance/bundle/deploy/immutable/can-manage/out.test.toml new file mode 100644 index 00000000000..669a6b593c4 --- /dev/null +++ b/acceptance/bundle/deploy/immutable/can-manage/out.test.toml @@ -0,0 +1,3 @@ +Cloud = true +CloudEnvs.gcp = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/deploy/immutable-can-manage/output.txt b/acceptance/bundle/deploy/immutable/can-manage/output.txt similarity index 84% rename from acceptance/bundle/deploy/immutable-can-manage/output.txt rename to acceptance/bundle/deploy/immutable/can-manage/output.txt index b142c587a3b..cdcca39708d 100644 --- a/acceptance/bundle/deploy/immutable-can-manage/output.txt +++ b/acceptance/bundle/deploy/immutable/can-manage/output.txt @@ -7,7 +7,7 @@ If it is not included, CAN_MANAGE permissions are only applied if the present id Consider using a adding a top-level permissions section such as the following: permissions: - - user_name: [USERNAME] + - service_principal_name: [USERNAME] level: CAN_MANAGE See https://docs.databricks.com/dev-tools/bundles/permissions.html to learn more about permission configuration. @@ -20,17 +20,15 @@ Files: 0 uploaded, 0 deleted Resources: 3 created, 0 changed, 0 deleted, 0 unchanged === Check can_manage_principals sent to the snapshot create API ->>> [CLI] api get /api/2.0/snapshots/info?relative_path=[SNAPSHOT_HASH]/[SNAPSHOT_HASH]-[UUID] -{ - "can_manage_principals": [ - { - "user_name": "[USERNAME]" - }, - { - "user_name": "external-manager@databricks.com" - } - ] -} +>>> print_requests.py //api/2.0/snapshots +[ + { + "service_principal_name": "[USERNAME]" + }, + { + "user_name": "deco-test-user@databricks.com" + } +] >>> [CLI] bundle destroy --auto-approve Recommendation: permissions section should explicitly include the current deployment identity '[USERNAME]' or one of its groups @@ -39,7 +37,7 @@ If it is not included, CAN_MANAGE permissions are only applied if the present id Consider using a adding a top-level permissions section such as the following: permissions: - - user_name: [USERNAME] + - service_principal_name: [USERNAME] level: CAN_MANAGE See https://docs.databricks.com/dev-tools/bundles/permissions.html to learn more about permission configuration. diff --git a/acceptance/bundle/deploy/immutable/can-manage/script b/acceptance/bundle/deploy/immutable/can-manage/script new file mode 100644 index 00000000000..1bf48588c08 --- /dev/null +++ b/acceptance/bundle/deploy/immutable/can-manage/script @@ -0,0 +1,17 @@ +if [ -z "$CLOUD_ENV" ]; then + export TEST_USER_EMAIL="deco-test-user@databricks.com" +fi + +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve + rm -f out.requests.txt +} +trap cleanup EXIT + +title "Deploy with a CAN_MANAGE permission entry" +trace $CLI bundle deploy + +title "Check can_manage_principals sent to the snapshot create API" +trace print_requests.py //api/2.0/snapshots | jq '.body.multipart_form.can_manage_principals | fromjson' diff --git a/acceptance/bundle/deploy/immutable-destroy-redeploy/src/main.py b/acceptance/bundle/deploy/immutable/can-manage/src/main.py similarity index 100% rename from acceptance/bundle/deploy/immutable-destroy-redeploy/src/main.py rename to acceptance/bundle/deploy/immutable/can-manage/src/main.py diff --git a/acceptance/bundle/deploy/immutable/can-manage/test.toml b/acceptance/bundle/deploy/immutable/can-manage/test.toml new file mode 100644 index 00000000000..003c144d006 --- /dev/null +++ b/acceptance/bundle/deploy/immutable/can-manage/test.toml @@ -0,0 +1,10 @@ +# The snapshot create request carries can_manage_principals, which the test reads back from the +# recorded request. +RecordRequests = true + +# The deploying identity is sent as a can_manage principal, keyed by its kind. Cloud tests deploy +# as a service principal, so do the same locally to get the same output. +IsServicePrincipal = true + +# The current user on GCP is a user and not a SP, so the deploying identity is sent as user_name. +CloudEnvs.gcp = false diff --git a/acceptance/bundle/deploy/immutable-destroy-redeploy/databricks.yml.tmpl b/acceptance/bundle/deploy/immutable/destroy-redeploy/databricks.yml.tmpl similarity index 100% rename from acceptance/bundle/deploy/immutable-destroy-redeploy/databricks.yml.tmpl rename to acceptance/bundle/deploy/immutable/destroy-redeploy/databricks.yml.tmpl diff --git a/acceptance/bundle/deploy/immutable-dev-mode/out.test.toml b/acceptance/bundle/deploy/immutable/destroy-redeploy/out.test.toml similarity index 67% rename from acceptance/bundle/deploy/immutable-dev-mode/out.test.toml rename to acceptance/bundle/deploy/immutable/destroy-redeploy/out.test.toml index a927a5fbc06..0f167effef1 100644 --- a/acceptance/bundle/deploy/immutable-dev-mode/out.test.toml +++ b/acceptance/bundle/deploy/immutable/destroy-redeploy/out.test.toml @@ -1,2 +1,2 @@ -Cloud = false +Cloud = true EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/deploy/immutable-destroy-redeploy/output.txt b/acceptance/bundle/deploy/immutable/destroy-redeploy/output.txt similarity index 100% rename from acceptance/bundle/deploy/immutable-destroy-redeploy/output.txt rename to acceptance/bundle/deploy/immutable/destroy-redeploy/output.txt diff --git a/acceptance/bundle/deploy/immutable-destroy-redeploy/script b/acceptance/bundle/deploy/immutable/destroy-redeploy/script similarity index 100% rename from acceptance/bundle/deploy/immutable-destroy-redeploy/script rename to acceptance/bundle/deploy/immutable/destroy-redeploy/script diff --git a/acceptance/bundle/deploy/immutable-legacy-state/src/main.py b/acceptance/bundle/deploy/immutable/destroy-redeploy/src/main.py similarity index 100% rename from acceptance/bundle/deploy/immutable-legacy-state/src/main.py rename to acceptance/bundle/deploy/immutable/destroy-redeploy/src/main.py diff --git a/acceptance/bundle/deploy/immutable-dev-mode/databricks.yml.tmpl b/acceptance/bundle/deploy/immutable/dev-mode/databricks.yml.tmpl similarity index 100% rename from acceptance/bundle/deploy/immutable-dev-mode/databricks.yml.tmpl rename to acceptance/bundle/deploy/immutable/dev-mode/databricks.yml.tmpl diff --git a/acceptance/bundle/deploy/immutable/dev-mode/out.test.toml b/acceptance/bundle/deploy/immutable/dev-mode/out.test.toml new file mode 100644 index 00000000000..0f167effef1 --- /dev/null +++ b/acceptance/bundle/deploy/immutable/dev-mode/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/deploy/immutable-dev-mode/output.txt b/acceptance/bundle/deploy/immutable/dev-mode/output.txt similarity index 100% rename from acceptance/bundle/deploy/immutable-dev-mode/output.txt rename to acceptance/bundle/deploy/immutable/dev-mode/output.txt diff --git a/acceptance/bundle/deploy/immutable-dev-mode/script b/acceptance/bundle/deploy/immutable/dev-mode/script similarity index 100% rename from acceptance/bundle/deploy/immutable-dev-mode/script rename to acceptance/bundle/deploy/immutable/dev-mode/script diff --git a/acceptance/bundle/deploy/immutable-dev-mode/src/main.py b/acceptance/bundle/deploy/immutable/dev-mode/src/main.py similarity index 100% rename from acceptance/bundle/deploy/immutable-dev-mode/src/main.py rename to acceptance/bundle/deploy/immutable/dev-mode/src/main.py diff --git a/acceptance/bundle/deploy/immutable-legacy-state/databricks.yml.tmpl b/acceptance/bundle/deploy/immutable/legacy-state/databricks.yml.tmpl similarity index 100% rename from acceptance/bundle/deploy/immutable-legacy-state/databricks.yml.tmpl rename to acceptance/bundle/deploy/immutable/legacy-state/databricks.yml.tmpl diff --git a/acceptance/bundle/deploy/immutable/legacy-state/out.test.toml b/acceptance/bundle/deploy/immutable/legacy-state/out.test.toml new file mode 100644 index 00000000000..f68b4a89c36 --- /dev/null +++ b/acceptance/bundle/deploy/immutable/legacy-state/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/deploy/immutable-legacy-state/output.txt b/acceptance/bundle/deploy/immutable/legacy-state/output.txt similarity index 87% rename from acceptance/bundle/deploy/immutable-legacy-state/output.txt rename to acceptance/bundle/deploy/immutable/legacy-state/output.txt index 72c4f7f4dd1..b9858f2b3c0 100644 --- a/acceptance/bundle/deploy/immutable-legacy-state/output.txt +++ b/acceptance/bundle/deploy/immutable/legacy-state/output.txt @@ -17,7 +17,8 @@ Snapshot path: /Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH OK: deployed to the current snapshot layout === The check still applies to a deployment carried over from the older layout ->>> [CLI] workspace mkdirs /Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]-[UUID]/snapshot/tampered +>>> [CLI] api post /api/2.0/snapshots:breakGlass --json {"snapshot_content_path": "/Workspace/Users/[UUID]/.snapshots/[SNAPSHOT_HASH]/[SNAPSHOT_HASH]-[UUID]/snapshot"} +true >>> musterr [CLI] bundle deploy Error: the last deployment was modified in the workspace. Changes might be overwritten by this deployment. To confirm it, use the --force flag diff --git a/acceptance/bundle/deploy/immutable-legacy-state/script b/acceptance/bundle/deploy/immutable/legacy-state/script similarity index 90% rename from acceptance/bundle/deploy/immutable-legacy-state/script rename to acceptance/bundle/deploy/immutable/legacy-state/script index 9d0d47653d2..8f6fba02120 100644 --- a/acceptance/bundle/deploy/immutable-legacy-state/script +++ b/acceptance/bundle/deploy/immutable/legacy-state/script @@ -27,5 +27,5 @@ esac title "The check still applies to a deployment carried over from the older layout" CONTENT_PATH="${SNAPSHOT_PATH%/files/src/main.py}" -trace $CLI workspace mkdirs "$CONTENT_PATH/tampered" +trace $CLI api post /api/2.0/snapshots:breakGlass --json "{\"snapshot_content_path\": \"$CONTENT_PATH\"}" | jq .status.dirty trace musterr $CLI bundle deploy diff --git a/acceptance/bundle/deploy/immutable-missing-snapshot/src/main.py b/acceptance/bundle/deploy/immutable/legacy-state/src/main.py similarity index 100% rename from acceptance/bundle/deploy/immutable-missing-snapshot/src/main.py rename to acceptance/bundle/deploy/immutable/legacy-state/src/main.py diff --git a/acceptance/bundle/deploy/immutable/legacy-state/test.toml b/acceptance/bundle/deploy/immutable/legacy-state/test.toml new file mode 100644 index 00000000000..5d348f4d12f --- /dev/null +++ b/acceptance/bundle/deploy/immutable/legacy-state/test.toml @@ -0,0 +1,8 @@ +# The test rewrites the deployment state by hand, which the recording service refuses because it +# no longer matches the deployment it has recorded, so run without deployment history. +EnvMatrix.DMS = [""] + +# The test breaks the glass with "api post /api/2.0/snapshots:breakGlass". On Windows the script +# runs under MSYS2, which rewrites leading-slash arguments to Windows paths before the CLI sees +# them. Setting this environment variable disables that conversion. +Env.MSYS_NO_PATHCONV = "1" diff --git a/acceptance/bundle/deploy/immutable-missing-snapshot/databricks.yml.tmpl b/acceptance/bundle/deploy/immutable/missing-snapshot/databricks.yml.tmpl similarity index 100% rename from acceptance/bundle/deploy/immutable-missing-snapshot/databricks.yml.tmpl rename to acceptance/bundle/deploy/immutable/missing-snapshot/databricks.yml.tmpl diff --git a/acceptance/bundle/deploy/immutable/missing-snapshot/out.test.toml b/acceptance/bundle/deploy/immutable/missing-snapshot/out.test.toml new file mode 100644 index 00000000000..f68b4a89c36 --- /dev/null +++ b/acceptance/bundle/deploy/immutable/missing-snapshot/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/deploy/immutable-missing-snapshot/output.txt b/acceptance/bundle/deploy/immutable/missing-snapshot/output.txt similarity index 100% rename from acceptance/bundle/deploy/immutable-missing-snapshot/output.txt rename to acceptance/bundle/deploy/immutable/missing-snapshot/output.txt diff --git a/acceptance/bundle/deploy/immutable-missing-snapshot/script b/acceptance/bundle/deploy/immutable/missing-snapshot/script similarity index 100% rename from acceptance/bundle/deploy/immutable-missing-snapshot/script rename to acceptance/bundle/deploy/immutable/missing-snapshot/script diff --git a/acceptance/bundle/deploy/immutable-no-artifacts/src/main.py b/acceptance/bundle/deploy/immutable/missing-snapshot/src/main.py similarity index 100% rename from acceptance/bundle/deploy/immutable-no-artifacts/src/main.py rename to acceptance/bundle/deploy/immutable/missing-snapshot/src/main.py diff --git a/acceptance/bundle/deploy/immutable/missing-snapshot/test.toml b/acceptance/bundle/deploy/immutable/missing-snapshot/test.toml new file mode 100644 index 00000000000..00611b6195e --- /dev/null +++ b/acceptance/bundle/deploy/immutable/missing-snapshot/test.toml @@ -0,0 +1,7 @@ +# The test rewrites the deployment state by hand, which the recording service refuses because it +# no longer matches the deployment it has recorded, so run without deployment history. +EnvMatrix.DMS = [""] + +Ignore = [ + "tmp.*", +] diff --git a/acceptance/bundle/deploy/immutable-no-artifacts/.gitignore b/acceptance/bundle/deploy/immutable/no-artifacts/.gitignore similarity index 100% rename from acceptance/bundle/deploy/immutable-no-artifacts/.gitignore rename to acceptance/bundle/deploy/immutable/no-artifacts/.gitignore diff --git a/acceptance/bundle/deploy/immutable-no-artifacts/databricks.yml.tmpl b/acceptance/bundle/deploy/immutable/no-artifacts/databricks.yml.tmpl similarity index 100% rename from acceptance/bundle/deploy/immutable-no-artifacts/databricks.yml.tmpl rename to acceptance/bundle/deploy/immutable/no-artifacts/databricks.yml.tmpl diff --git a/acceptance/bundle/deploy/immutable-no-artifacts/out.test.toml b/acceptance/bundle/deploy/immutable/no-artifacts/out.test.toml similarity index 81% rename from acceptance/bundle/deploy/immutable-no-artifacts/out.test.toml rename to acceptance/bundle/deploy/immutable/no-artifacts/out.test.toml index f7fafbcc731..7a62f726f25 100644 --- a/acceptance/bundle/deploy/immutable-no-artifacts/out.test.toml +++ b/acceptance/bundle/deploy/immutable/no-artifacts/out.test.toml @@ -1,3 +1,3 @@ -Cloud = false +Cloud = true EnvMatrix.DMS = ["", "true"] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/deploy/immutable-no-artifacts/output.txt b/acceptance/bundle/deploy/immutable/no-artifacts/output.txt similarity index 100% rename from acceptance/bundle/deploy/immutable-no-artifacts/output.txt rename to acceptance/bundle/deploy/immutable/no-artifacts/output.txt diff --git a/acceptance/bundle/deploy/immutable-no-artifacts/script b/acceptance/bundle/deploy/immutable/no-artifacts/script similarity index 100% rename from acceptance/bundle/deploy/immutable-no-artifacts/script rename to acceptance/bundle/deploy/immutable/no-artifacts/script diff --git a/acceptance/bundle/deploy/immutable-permissions-change/src/main.py b/acceptance/bundle/deploy/immutable/no-artifacts/src/main.py similarity index 100% rename from acceptance/bundle/deploy/immutable-permissions-change/src/main.py rename to acceptance/bundle/deploy/immutable/no-artifacts/src/main.py diff --git a/acceptance/bundle/deploy/immutable/src/notebook.py b/acceptance/bundle/deploy/immutable/no-artifacts/src/notebook.py similarity index 100% rename from acceptance/bundle/deploy/immutable/src/notebook.py rename to acceptance/bundle/deploy/immutable/no-artifacts/src/notebook.py diff --git a/acceptance/bundle/deploy/immutable/no-artifacts/test.toml b/acceptance/bundle/deploy/immutable/no-artifacts/test.toml new file mode 100644 index 00000000000..aff1c17e1c6 --- /dev/null +++ b/acceptance/bundle/deploy/immutable/no-artifacts/test.toml @@ -0,0 +1,12 @@ +EnvMatrix.READPLAN = ["", "1"] + +Ignore = [ + ".venv", + "*.pyc", +] + +# When READPLAN=1, "bundle deploy" is called as "bundle deploy --plan plan.json". +# Normalize so both variants produce identical output. +[[Repls]] +Old = ' --plan plan.json' +New = '' diff --git a/acceptance/bundle/deploy/immutable-permissions-change/databricks.yml.tmpl b/acceptance/bundle/deploy/immutable/permissions-change/databricks.yml.tmpl similarity index 100% rename from acceptance/bundle/deploy/immutable-permissions-change/databricks.yml.tmpl rename to acceptance/bundle/deploy/immutable/permissions-change/databricks.yml.tmpl diff --git a/acceptance/bundle/deploy/immutable/permissions-change/out.test.toml b/acceptance/bundle/deploy/immutable/permissions-change/out.test.toml new file mode 100644 index 00000000000..0f167effef1 --- /dev/null +++ b/acceptance/bundle/deploy/immutable/permissions-change/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/deploy/immutable-permissions-change/output.txt b/acceptance/bundle/deploy/immutable/permissions-change/output.txt similarity index 100% rename from acceptance/bundle/deploy/immutable-permissions-change/output.txt rename to acceptance/bundle/deploy/immutable/permissions-change/output.txt diff --git a/acceptance/bundle/deploy/immutable-permissions-change/script b/acceptance/bundle/deploy/immutable/permissions-change/script similarity index 100% rename from acceptance/bundle/deploy/immutable-permissions-change/script rename to acceptance/bundle/deploy/immutable/permissions-change/script diff --git a/acceptance/bundle/deploy/immutable/src/main.py b/acceptance/bundle/deploy/immutable/permissions-change/src/main.py similarity index 100% rename from acceptance/bundle/deploy/immutable/src/main.py rename to acceptance/bundle/deploy/immutable/permissions-change/src/main.py diff --git a/acceptance/bundle/deploy/immutable/test.toml b/acceptance/bundle/deploy/immutable/test.toml index 81910694737..618e742def3 100644 --- a/acceptance/bundle/deploy/immutable/test.toml +++ b/acceptance/bundle/deploy/immutable/test.toml @@ -1,15 +1,8 @@ -Cloud = false # Temporary disable cloud tests until the API is fully available - -# immutable_folder only works with the direct engine. +Cloud = true Ignore = [ - "dist", - "build", "databricks.yml", ".databricks", - ".venv", - "*.pyc", - "src/*.egg-info", ] [[Repls]] diff --git a/libs/testserver/fake_workspace.go b/libs/testserver/fake_workspace.go index 8f98ae194f8..fbbd3c70a22 100644 --- a/libs/testserver/fake_workspace.go +++ b/libs/testserver/fake_workspace.go @@ -200,9 +200,8 @@ type FakeWorkspace struct { files map[string]FileEntry repoIdByPath map[string]int64 - // snapshots records immutable-snapshot content paths. A workspace write into a - // recorded path flips it dirty; RecordSnapshot resets it when the snapshot is - // (re)created by the snapshot API. + // snapshots records immutable-snapshot content paths. Breaking the glass on a + // recorded path flips it dirty for good, as in the real API. snapshots map[string]*snapshotRecord Jobs map[int64]jobs.Job @@ -831,9 +830,7 @@ func (s *FakeWorkspace) FsDeleteFile(filePath string) Response { return Response{} } -// RecordSnapshot stores a create-request record for the snapshot at contentPath. It is called -// after the content directory is materialized, so the snapshot's own creation write is not -// counted as tampering. +// RecordSnapshot stores a create-request record for the snapshot at contentPath. func (s *FakeWorkspace) RecordSnapshot(contentPath, relPath string, acl, canManage []map[string]any) { defer s.LockUnlock()() s.snapshots[contentPath] = &snapshotRecord{ @@ -863,14 +860,16 @@ func (s *FakeWorkspace) SnapshotInfo(contentPath string) (*snapshotRecord, bool) return rec, ok } -// markSnapshotDirty flips any recorded snapshot dirty when targetPath writes at or under its -// content path. The caller must already hold the lock. -func (s *FakeWorkspace) markSnapshotDirty(targetPath string) { - for contentPath, rec := range s.snapshots { - if targetPath == contentPath || strings.HasPrefix(targetPath, contentPath+"/") { - rec.Dirty = true - } +// BreakGlassSnapshot marks the snapshot at contentPath dirty. The real API flips the flag when +// the glass is broken, not when a file changes: until then the content is read-only to callers. +// ok is false when no snapshot is recorded there. +func (s *FakeWorkspace) BreakGlassSnapshot(contentPath string) (ok bool) { + defer s.LockUnlock()() + rec, ok := s.snapshots[contentPath] + if ok { + rec.Dirty = true } + return ok } func (s *FakeWorkspace) WorkspaceMkdirs(request workspace.Mkdirs) { @@ -886,7 +885,6 @@ func (s *FakeWorkspace) WorkspaceMkdirs(request workspace.Mkdirs) { } } } - s.markSnapshotDirty(request.Path) } func (s *FakeWorkspace) WorkspaceExport(path string) []byte { @@ -923,7 +921,6 @@ func (s *FakeWorkspace) WorkspaceDelete(path string, recursive bool) Response { } } } - s.markSnapshotDirty(path) return Response{} } @@ -1126,8 +1123,6 @@ func (s *FakeWorkspace) importFile(filePath string, body []byte, overwrite, dete Data: body, } } - - s.markSnapshotDirty(workspacePath) return Response{} } @@ -1184,8 +1179,6 @@ func (s *FakeWorkspace) WorkspaceImportNotebook(filePath string, body []byte, la }, Data: body, } - - s.markSnapshotDirty(filePath) return Response{} } diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go index dc8b3c2de7a..d20bdfecdd9 100644 --- a/libs/testserver/handlers.go +++ b/libs/testserver/handlers.go @@ -166,6 +166,29 @@ func ParseImportForm(req Request) (ImportForm, error) { } } +// snapshotStatus is the body the snapshot inspect and break glass APIs answer with. +func snapshotStatus(contentPath string, dirty bool) map[string]any { + return map[string]any{ + "status": map[string]any{ + "snapshot_content_path": contentPath, + "dirty": dirty, + "permissions": []string{"SNAPSHOT_PERMISSION_CAN_BREAK_GLASS"}, + }, + } +} + +// snapshotNotFound is how the snapshot inspect and break glass APIs answer for a path that +// holds no snapshot. +func snapshotNotFound(contentPath string) Response { + return Response{ + StatusCode: http.StatusNotFound, + Body: map[string]string{ + "error_code": "RESOURCE_DOES_NOT_EXIST", + "message": "Snapshot not found: " + contentPath, + }, + } +} + func AddDefaultHandlers(server *Server) { server.Handle("GET", "/api/2.0/policies/clusters/list", func(req Request) any { return req.Workspace.ClusterPoliciesList(req) @@ -943,7 +966,6 @@ func AddDefaultHandlers(server *Server) { } } req.Workspace.WorkspaceMkdirs(workspace.Mkdirs{Path: contentPath}) - // Record after mkdirs so the snapshot's own creation write is not counted as tampering. req.Workspace.RecordSnapshot(contentPath, relativePath, acl, canManage) return map[string]any{ "name": "workspaces/snapshotOperations/" + relativePath, @@ -978,21 +1000,26 @@ func AddDefaultHandlers(server *Server) { // A path that holds no snapshot is not inspectable, so the API reports NOT_FOUND // rather than calling it clean. if !ok { + return snapshotNotFound(contentPath) + } + return snapshotStatus(contentPath, dirty) + }) + + // Breaks the glass on a snapshot: grants write access to its content and marks it dirty. + server.Handle("POST", "/api/2.0/snapshots:breakGlass", func(req Request) any { + var request struct { + SnapshotContentPath string `json:"snapshot_content_path"` + } + if err := json.Unmarshal(req.Body, &request); err != nil { return Response{ - StatusCode: http.StatusNotFound, - Body: map[string]string{ - "error_code": "RESOURCE_DOES_NOT_EXIST", - "message": "Snapshot not found: " + contentPath, - }, + Body: fmt.Sprintf("internal error: %s", err), + StatusCode: http.StatusInternalServerError, } } - return map[string]any{ - "status": map[string]any{ - "snapshot_content_path": contentPath, - "dirty": dirty, - "permissions": []string{"SNAPSHOT_PERMISSION_CAN_BREAK_GLASS"}, - }, + if !req.Workspace.BreakGlassSnapshot(request.SnapshotContentPath) { + return snapshotNotFound(request.SnapshotContentPath) } + return snapshotStatus(request.SnapshotContentPath, true) }) // SQL Warehouses: From cf7cf56d0dd980a0ca1ef4e838ece2491c6939d1 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Tue, 20 Oct 2026 20:53:27 +0200 Subject: [PATCH 13/16] skip DMS --- acceptance/bundle/deploy/immutable/basic/out.test.toml | 2 +- acceptance/bundle/deploy/immutable/breakglass/out.test.toml | 2 +- acceptance/bundle/deploy/immutable/can-manage/out.test.toml | 2 +- .../bundle/deploy/immutable/destroy-redeploy/out.test.toml | 2 +- acceptance/bundle/deploy/immutable/dev-mode/out.test.toml | 2 +- acceptance/bundle/deploy/immutable/no-artifacts/out.test.toml | 2 +- .../bundle/deploy/immutable/permissions-change/out.test.toml | 2 +- acceptance/bundle/deploy/immutable/test.toml | 3 +++ 8 files changed, 10 insertions(+), 7 deletions(-) diff --git a/acceptance/bundle/deploy/immutable/basic/out.test.toml b/acceptance/bundle/deploy/immutable/basic/out.test.toml index 0f167effef1..f68b4a89c36 100644 --- a/acceptance/bundle/deploy/immutable/basic/out.test.toml +++ b/acceptance/bundle/deploy/immutable/basic/out.test.toml @@ -1,2 +1,2 @@ Cloud = true -EnvMatrix.DMS = ["", "true"] +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/deploy/immutable/breakglass/out.test.toml b/acceptance/bundle/deploy/immutable/breakglass/out.test.toml index 0f167effef1..f68b4a89c36 100644 --- a/acceptance/bundle/deploy/immutable/breakglass/out.test.toml +++ b/acceptance/bundle/deploy/immutable/breakglass/out.test.toml @@ -1,2 +1,2 @@ Cloud = true -EnvMatrix.DMS = ["", "true"] +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/deploy/immutable/can-manage/out.test.toml b/acceptance/bundle/deploy/immutable/can-manage/out.test.toml index 669a6b593c4..3970ea98616 100644 --- a/acceptance/bundle/deploy/immutable/can-manage/out.test.toml +++ b/acceptance/bundle/deploy/immutable/can-manage/out.test.toml @@ -1,3 +1,3 @@ Cloud = true CloudEnvs.gcp = false -EnvMatrix.DMS = ["", "true"] +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/deploy/immutable/destroy-redeploy/out.test.toml b/acceptance/bundle/deploy/immutable/destroy-redeploy/out.test.toml index 0f167effef1..f68b4a89c36 100644 --- a/acceptance/bundle/deploy/immutable/destroy-redeploy/out.test.toml +++ b/acceptance/bundle/deploy/immutable/destroy-redeploy/out.test.toml @@ -1,2 +1,2 @@ Cloud = true -EnvMatrix.DMS = ["", "true"] +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/deploy/immutable/dev-mode/out.test.toml b/acceptance/bundle/deploy/immutable/dev-mode/out.test.toml index 0f167effef1..f68b4a89c36 100644 --- a/acceptance/bundle/deploy/immutable/dev-mode/out.test.toml +++ b/acceptance/bundle/deploy/immutable/dev-mode/out.test.toml @@ -1,2 +1,2 @@ Cloud = true -EnvMatrix.DMS = ["", "true"] +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/deploy/immutable/no-artifacts/out.test.toml b/acceptance/bundle/deploy/immutable/no-artifacts/out.test.toml index 7a62f726f25..de136de256d 100644 --- a/acceptance/bundle/deploy/immutable/no-artifacts/out.test.toml +++ b/acceptance/bundle/deploy/immutable/no-artifacts/out.test.toml @@ -1,3 +1,3 @@ Cloud = true -EnvMatrix.DMS = ["", "true"] +EnvMatrix.DMS = [""] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/deploy/immutable/permissions-change/out.test.toml b/acceptance/bundle/deploy/immutable/permissions-change/out.test.toml index 0f167effef1..f68b4a89c36 100644 --- a/acceptance/bundle/deploy/immutable/permissions-change/out.test.toml +++ b/acceptance/bundle/deploy/immutable/permissions-change/out.test.toml @@ -1,2 +1,2 @@ Cloud = true -EnvMatrix.DMS = ["", "true"] +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/deploy/immutable/test.toml b/acceptance/bundle/deploy/immutable/test.toml index 618e742def3..9803bf00957 100644 --- a/acceptance/bundle/deploy/immutable/test.toml +++ b/acceptance/bundle/deploy/immutable/test.toml @@ -9,3 +9,6 @@ Ignore = [ # Replace snapshot hash with SNAPSHOT_HASH Old = "[0-9a-f]{64}" New = "[SNAPSHOT_HASH]" + +# Skipping DMS tests for now because DMS do not support immutable resource yet +EnvMatrix.DMS = [""] From 779156fa973055a7761ff05555d4bd2fe05570e5 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Wed, 21 Oct 2026 10:28:01 +0200 Subject: [PATCH 14/16] move dms section --- acceptance/bundle/deploy/immutable/test.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/acceptance/bundle/deploy/immutable/test.toml b/acceptance/bundle/deploy/immutable/test.toml index 9803bf00957..bf968aad44f 100644 --- a/acceptance/bundle/deploy/immutable/test.toml +++ b/acceptance/bundle/deploy/immutable/test.toml @@ -1,5 +1,8 @@ Cloud = true +# Skipping DMS tests for now because DMS do not support immutable resource yet +EnvMatrix.DMS = [""] + Ignore = [ "databricks.yml", ".databricks", @@ -9,6 +12,3 @@ Ignore = [ # Replace snapshot hash with SNAPSHOT_HASH Old = "[0-9a-f]{64}" New = "[SNAPSHOT_HASH]" - -# Skipping DMS tests for now because DMS do not support immutable resource yet -EnvMatrix.DMS = [""] From a31d8d9383cf4c5542cae2ddadd967861571821b Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Thu, 22 Oct 2026 11:34:08 +0200 Subject: [PATCH 15/16] fix tests on cloud --- acceptance/bundle/deploy/immutable/basic/output.txt | 8 ++++---- .../bundle/deploy/immutable/can-manage/test.toml | 4 ---- .../bundle/deploy/immutable/no-artifacts/output.txt | 6 +++--- .../deploy/immutable/permissions-change/output.txt | 4 ++-- acceptance/bundle/deploy/immutable/test.toml | 4 ++++ libs/testserver/handlers.go | 13 +++++++++---- 6 files changed, 22 insertions(+), 17 deletions(-) diff --git a/acceptance/bundle/deploy/immutable/basic/output.txt b/acceptance/bundle/deploy/immutable/basic/output.txt index 39516596c77..0930c3d5e51 100644 --- a/acceptance/bundle/deploy/immutable/basic/output.txt +++ b/acceptance/bundle/deploy/immutable/basic/output.txt @@ -65,7 +65,7 @@ Building python_artifact... ], "can_manage": [ { - "user_name": "[USERNAME]" + "service_principal_name": "[USERNAME]" } ], "zip_path": "[TEST_TMP_DIR]/.databricks/bundle/default/snapshots/[SNAPSHOT_HASH]-[UUID].zip" @@ -113,7 +113,7 @@ Building python_artifact... ], "can_manage": [ { - "user_name": "[USERNAME]" + "service_principal_name": "[USERNAME]" } ], "zip_path": "[TEST_TMP_DIR]/.databricks/bundle/default/snapshots/[SNAPSHOT_HASH]-[UUID].zip" @@ -153,12 +153,12 @@ Building python_artifact... "reason": "missing_in_remote", "old": [ { - "user_name": "[USERNAME]" + "service_principal_name": "[USERNAME]" } ], "new": [ { - "user_name": "[USERNAME]" + "service_principal_name": "[USERNAME]" } ] }, diff --git a/acceptance/bundle/deploy/immutable/can-manage/test.toml b/acceptance/bundle/deploy/immutable/can-manage/test.toml index 003c144d006..9b8d29984a9 100644 --- a/acceptance/bundle/deploy/immutable/can-manage/test.toml +++ b/acceptance/bundle/deploy/immutable/can-manage/test.toml @@ -2,9 +2,5 @@ # recorded request. RecordRequests = true -# The deploying identity is sent as a can_manage principal, keyed by its kind. Cloud tests deploy -# as a service principal, so do the same locally to get the same output. -IsServicePrincipal = true - # The current user on GCP is a user and not a SP, so the deploying identity is sent as user_name. CloudEnvs.gcp = false diff --git a/acceptance/bundle/deploy/immutable/no-artifacts/output.txt b/acceptance/bundle/deploy/immutable/no-artifacts/output.txt index 7222a27b940..96f778441ac 100644 --- a/acceptance/bundle/deploy/immutable/no-artifacts/output.txt +++ b/acceptance/bundle/deploy/immutable/no-artifacts/output.txt @@ -46,7 +46,7 @@ Validation OK! ], "can_manage": [ { - "user_name": "[USERNAME]" + "service_principal_name": "[USERNAME]" } ], "zip_path": "[TEST_TMP_DIR]/.databricks/bundle/default/snapshots/[SNAPSHOT_HASH]-[UUID].zip" @@ -111,12 +111,12 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "reason": "missing_in_remote", "old": [ { - "user_name": "[USERNAME]" + "service_principal_name": "[USERNAME]" } ], "new": [ { - "user_name": "[USERNAME]" + "service_principal_name": "[USERNAME]" } ] }, diff --git a/acceptance/bundle/deploy/immutable/permissions-change/output.txt b/acceptance/bundle/deploy/immutable/permissions-change/output.txt index 4ed18185018..41d23e84549 100644 --- a/acceptance/bundle/deploy/immutable/permissions-change/output.txt +++ b/acceptance/bundle/deploy/immutable/permissions-change/output.txt @@ -15,7 +15,7 @@ If it is not included, CAN_MANAGE permissions are only applied if the present id Consider using a adding a top-level permissions section such as the following: permissions: - - user_name: [USERNAME] + - service_principal_name: [USERNAME] level: CAN_MANAGE See https://docs.databricks.com/dev-tools/bundles/permissions.html to learn more about permission configuration. @@ -38,7 +38,7 @@ If it is not included, CAN_MANAGE permissions are only applied if the present id Consider using a adding a top-level permissions section such as the following: permissions: - - user_name: [USERNAME] + - service_principal_name: [USERNAME] level: CAN_MANAGE See https://docs.databricks.com/dev-tools/bundles/permissions.html to learn more about permission configuration. diff --git a/acceptance/bundle/deploy/immutable/test.toml b/acceptance/bundle/deploy/immutable/test.toml index bf968aad44f..85e2650da1a 100644 --- a/acceptance/bundle/deploy/immutable/test.toml +++ b/acceptance/bundle/deploy/immutable/test.toml @@ -3,6 +3,10 @@ Cloud = true # Skipping DMS tests for now because DMS do not support immutable resource yet EnvMatrix.DMS = [""] +# The deploying identity shows in the output keyed by its kind (can_manage principals, permission +# hints). Cloud tests deploy as a service principal, so do the same locally to get the same output. +IsServicePrincipal = true + Ignore = [ "databricks.yml", ".databricks", diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go index e23e685e739..b03625edbc3 100644 --- a/libs/testserver/handlers.go +++ b/libs/testserver/handlers.go @@ -25,6 +25,11 @@ import ( // TestWorkspaceID is the id of the fake workspace. const TestWorkspaceID = 900800700600 +// snapshotRootUUID is the workspace user UUID the snapshot root is under. The real API uses a +// UUID that is not the deploying identity's user name (an email or a service principal's +// application ID), so snapshot paths never contain the user name. +const snapshotRootUUID = "3f9d2c1b-7a6e-4b5d-8c4f-0e1a2b3c4d5e" + var TestMetastore = catalog.MetastoreAssignment{ DefaultCatalogName: "hive_metastore", MetastoreId: "120efa64-9b68-46ba-be38-f319458430d2", @@ -930,7 +935,7 @@ func AddDefaultHandlers(server *Server) { server.Handle("GET", "/api/2.0/repos/snapshots/rootpath", func(req Request) any { return map[string]any{ - "path": "/Workspace/Users/" + TestUserSP.UserName + "/.snapshots/", + "path": "/Workspace/Users/" + snapshotRootUUID + "/.snapshots/", } }) @@ -968,8 +973,8 @@ func AddDefaultHandlers(server *Server) { // The server stores content under a fixed "snapshot" subfolder of the caller's // relative_path. The real API uses the workspace user UUID (not email) in the path, - // matching service-principal identities used in cloud acceptance tests. - contentPath := fmt.Sprintf("/Workspace/Users/%s/.snapshots/%s/snapshot", TestUserSP.UserName, relativePath) + // see snapshotRootUUID. + contentPath := fmt.Sprintf("/Workspace/Users/%s/.snapshots/%s/snapshot", snapshotRootUUID, relativePath) // Snapshots are immutable, so a relative_path can be created only once. The real API // answers with a 400 that carries the code in the message and sets no error_code. if _, exists := req.Workspace.SnapshotInfo(contentPath); exists { @@ -994,7 +999,7 @@ func AddDefaultHandlers(server *Server) { server.Handle("GET", "/api/2.0/snapshots/info", func(req Request) any { relPath := req.URL.Query().Get("relative_path") // Build the content path the same way the create handler does. - contentPath := fmt.Sprintf("/Workspace/Users/%s/.snapshots/%s/snapshot", TestUserSP.UserName, relPath) + contentPath := fmt.Sprintf("/Workspace/Users/%s/.snapshots/%s/snapshot", snapshotRootUUID, relPath) rec, ok := req.Workspace.SnapshotInfo(contentPath) if !ok { return Response{ From 600c1463a458d6fb67b6ac4216d9e7085ff0dac9 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Thu, 22 Oct 2026 11:58:50 +0200 Subject: [PATCH 16/16] fix test --- bundle/direct/dresources/all_test.go | 2 +- libs/testserver/handlers.go | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/bundle/direct/dresources/all_test.go b/bundle/direct/dresources/all_test.go index ae08611f0b1..140f3acf1e5 100644 --- a/bundle/direct/dresources/all_test.go +++ b/bundle/direct/dresources/all_test.go @@ -290,7 +290,7 @@ var testConfig map[string]any = map[string]any{ }, "internal_immutable_snapshots": &resources.Snapshot{ - RemoteRoot: "/Workspace/Users/" + testserver.TestUserSP.UserName + "/.snapshots", + RemoteRoot: "/Workspace/Users/" + testserver.SnapshotRootUUID + "/.snapshots", BundleID: "test-bundle-id", ACL: []snapshot.ACLEntry{{UserName: "user@example.com", PermissionLevel: "CAN_READ"}}, }, diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go index b03625edbc3..c8b54ec672b 100644 --- a/libs/testserver/handlers.go +++ b/libs/testserver/handlers.go @@ -25,10 +25,10 @@ import ( // TestWorkspaceID is the id of the fake workspace. const TestWorkspaceID = 900800700600 -// snapshotRootUUID is the workspace user UUID the snapshot root is under. The real API uses a +// SnapshotRootUUID is the workspace user UUID the snapshot root is under. The real API uses a // UUID that is not the deploying identity's user name (an email or a service principal's // application ID), so snapshot paths never contain the user name. -const snapshotRootUUID = "3f9d2c1b-7a6e-4b5d-8c4f-0e1a2b3c4d5e" +const SnapshotRootUUID = "3f9d2c1b-7a6e-4b5d-8c4f-0e1a2b3c4d5e" var TestMetastore = catalog.MetastoreAssignment{ DefaultCatalogName: "hive_metastore", @@ -935,7 +935,7 @@ func AddDefaultHandlers(server *Server) { server.Handle("GET", "/api/2.0/repos/snapshots/rootpath", func(req Request) any { return map[string]any{ - "path": "/Workspace/Users/" + snapshotRootUUID + "/.snapshots/", + "path": "/Workspace/Users/" + SnapshotRootUUID + "/.snapshots/", } }) @@ -973,8 +973,8 @@ func AddDefaultHandlers(server *Server) { // The server stores content under a fixed "snapshot" subfolder of the caller's // relative_path. The real API uses the workspace user UUID (not email) in the path, - // see snapshotRootUUID. - contentPath := fmt.Sprintf("/Workspace/Users/%s/.snapshots/%s/snapshot", snapshotRootUUID, relativePath) + // see SnapshotRootUUID. + contentPath := fmt.Sprintf("/Workspace/Users/%s/.snapshots/%s/snapshot", SnapshotRootUUID, relativePath) // Snapshots are immutable, so a relative_path can be created only once. The real API // answers with a 400 that carries the code in the message and sets no error_code. if _, exists := req.Workspace.SnapshotInfo(contentPath); exists { @@ -999,7 +999,7 @@ func AddDefaultHandlers(server *Server) { server.Handle("GET", "/api/2.0/snapshots/info", func(req Request) any { relPath := req.URL.Query().Get("relative_path") // Build the content path the same way the create handler does. - contentPath := fmt.Sprintf("/Workspace/Users/%s/.snapshots/%s/snapshot", snapshotRootUUID, relPath) + contentPath := fmt.Sprintf("/Workspace/Users/%s/.snapshots/%s/snapshot", SnapshotRootUUID, relPath) rec, ok := req.Workspace.SnapshotInfo(contentPath) if !ok { return Response{