From f3b45182a695b0df72f749fa0379ede2cd17a141 Mon Sep 17 00:00:00 2001 From: Ilya Kuznetsov Date: Mon, 24 Aug 2026 14:50:03 +0000 Subject: [PATCH 1/3] Add --state-path flag to bundle config-remote-sync Co-authored-by: Isaac --- .../state_path/databricks.yml.tmpl | 19 ++++++++++ .../state_path/out.direct.txt | 5 +++ .../state_path/out.terraform.txt | 5 +++ .../state_path/out.test.toml | 3 ++ .../config-remote-sync/state_path/output.txt | 36 ++++++++++++++++++ .../config-remote-sync/state_path/script | 38 +++++++++++++++++++ .../config-remote-sync/state_path/test.toml | 6 +++ cmd/bundle/config_remote_sync.go | 32 +++++++++++++++- 8 files changed, 143 insertions(+), 1 deletion(-) create mode 100644 acceptance/bundle/config-remote-sync/state_path/databricks.yml.tmpl create mode 100644 acceptance/bundle/config-remote-sync/state_path/out.direct.txt create mode 100644 acceptance/bundle/config-remote-sync/state_path/out.terraform.txt create mode 100644 acceptance/bundle/config-remote-sync/state_path/out.test.toml create mode 100644 acceptance/bundle/config-remote-sync/state_path/output.txt create mode 100755 acceptance/bundle/config-remote-sync/state_path/script create mode 100644 acceptance/bundle/config-remote-sync/state_path/test.toml diff --git a/acceptance/bundle/config-remote-sync/state_path/databricks.yml.tmpl b/acceptance/bundle/config-remote-sync/state_path/databricks.yml.tmpl new file mode 100644 index 00000000000..f0eb5edcb98 --- /dev/null +++ b/acceptance/bundle/config-remote-sync/state_path/databricks.yml.tmpl @@ -0,0 +1,19 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + jobs: + my_job: + max_concurrent_runs: 1 + tasks: + - task_key: main + notebook_task: + notebook_path: /Users/{{workspace_user_name}}/notebook + new_cluster: + spark_version: $DEFAULT_SPARK_VERSION + node_type_id: $NODE_TYPE_ID + num_workers: 1 + +targets: + default: + mode: development diff --git a/acceptance/bundle/config-remote-sync/state_path/out.direct.txt b/acceptance/bundle/config-remote-sync/state_path/out.direct.txt new file mode 100644 index 00000000000..1390dc66377 --- /dev/null +++ b/acceptance/bundle/config-remote-sync/state_path/out.direct.txt @@ -0,0 +1,5 @@ + +>>> [CLI] bundle config-remote-sync --state-path /Workspace/Users/[USERNAME]/.bundle/other-bundle/default/state --select-ids jobs:[MY_JOB_ID] +Error: no deployed jobs resource with id [MY_JOB_ID]; the deployment state contains no resources with ids (the bundle may not be deployed, or its resource state is missing) + +Exit code: 1 diff --git a/acceptance/bundle/config-remote-sync/state_path/out.terraform.txt b/acceptance/bundle/config-remote-sync/state_path/out.terraform.txt new file mode 100644 index 00000000000..d0e30f46836 --- /dev/null +++ b/acceptance/bundle/config-remote-sync/state_path/out.terraform.txt @@ -0,0 +1,5 @@ + +>>> [CLI] bundle config-remote-sync --state-path /Workspace/Users/[USERNAME]/.bundle/other-bundle/default/state --select-ids jobs:[MY_JOB_ID] +Error: state snapshot not available: resources state snapshot not found remotely at resources-config-sync-snapshot.json: state snapshot not found + +Exit code: 1 diff --git a/acceptance/bundle/config-remote-sync/state_path/out.test.toml b/acceptance/bundle/config-remote-sync/state_path/out.test.toml new file mode 100644 index 00000000000..a55b29471a4 --- /dev/null +++ b/acceptance/bundle/config-remote-sync/state_path/out.test.toml @@ -0,0 +1,3 @@ +Cloud = false +GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/state_path/output.txt b/acceptance/bundle/config-remote-sync/state_path/output.txt new file mode 100644 index 00000000000..c310b9e8c46 --- /dev/null +++ b/acceptance/bundle/config-remote-sync/state_path/output.txt @@ -0,0 +1,36 @@ +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Created jobs.my_job +Files: 5 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +=== Modify the job remotely + +=== --state-path at the deployment's own state folder detects the drift +Detected changes in 1 resource(s): + +Resource: resources.jobs.my_job + max_concurrent_runs: replace + + + +=== A relative --state-path is rejected before any state is read +>>> [CLI] bundle config-remote-sync --state-path relative/state +Error: --state-path must be an absolute workspace path, got "relative/state" + +Exit code: 1 + +=== A ~-relative --state-path is rejected too +>>> [CLI] bundle config-remote-sync --state-path ~/.bundle/test/default/state +Error: --state-path must be an absolute workspace path, got "~/.bundle/test/default/state" + +Exit code: 1 + +=== --state-path elsewhere does not see the deployment + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.my_job + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Destroy: 1 deleted diff --git a/acceptance/bundle/config-remote-sync/state_path/script b/acceptance/bundle/config-remote-sync/state_path/script new file mode 100755 index 00000000000..10d89390679 --- /dev/null +++ b/acceptance/bundle/config-remote-sync/state_path/script @@ -0,0 +1,38 @@ +#!/bin/bash + +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve +} +trap cleanup EXIT + +deployed_state="/Workspace/Users/${CURRENT_USER_NAME}/.bundle/test-bundle-${UNIQUE_NAME}/default/state" +other_state="/Workspace/Users/${CURRENT_USER_NAME}/.bundle/other-bundle/default/state" + +$CLI bundle deploy +job_id="$(read_id.py my_job)" + +title "Modify the job remotely" +echo +edit_resource.py jobs $job_id < out.$DATABRICKS_BUNDLE_ENGINE.txt 2>&1 diff --git a/acceptance/bundle/config-remote-sync/state_path/test.toml b/acceptance/bundle/config-remote-sync/state_path/test.toml new file mode 100644 index 00000000000..4f6a37ad901 --- /dev/null +++ b/acceptance/bundle/config-remote-sync/state_path/test.toml @@ -0,0 +1,6 @@ +RecordRequests = false +Ignore = [".databricks", "databricks.yml"] + +Env.DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_YAML_SYNC = "true" + +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/cmd/bundle/config_remote_sync.go b/cmd/bundle/config_remote_sync.go index f6c6e5cbe7f..329e0eca990 100644 --- a/cmd/bundle/config_remote_sync.go +++ b/cmd/bundle/config_remote_sync.go @@ -8,6 +8,7 @@ import ( "maps" "runtime" "slices" + "strings" "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/configsync" @@ -25,6 +26,7 @@ import ( func newConfigRemoteSyncCommand() *cobra.Command { var save bool var selectIDs []string + var statePath string cmd := &cobra.Command{ Use: "config-remote-sync", @@ -44,18 +46,26 @@ Examples: databricks bundle config-remote-sync --save # Restrict the sync to a single resource by its type and deployed resource ID - databricks bundle config-remote-sync --select-ids jobs:123456789 --save`, + databricks bundle config-remote-sync --select-ids jobs:123456789 --save + + # Read the deployment state from an explicit workspace location + databricks bundle config-remote-sync --state-path /Workspace/Shared/.bundle/my_bundle/dev/state`, Hidden: true, // Used by DABs in the Workspace only } cmd.Flags().BoolVar(&save, "save", false, "Write updated config files to disk") cmd.Flags().StringSliceVar(&selectIDs, "select-ids", nil, "Sync only the given resources, each as : (e.g. jobs:123456789). Can be repeated or comma-separated.") + cmd.Flags().StringVar(&statePath, "state-path", "", "Absolute workspace path of the deployment state folder to read, overriding workspace.state_path. Use when the state does not live under the path this command resolves by default, e.g. because the bundle was deployed by another user.") cmd.RunE = func(cmd *cobra.Command, args []string) error { if runtime.GOOS == "windows" { return errors.New("config-remote-sync command is not supported on Windows") } + if err := validateStatePathFlag(statePath); err != nil { + return err + } + stats := configsync.Stats{Save: save} // Emit telemetry on every exit path, including failures inside @@ -74,6 +84,13 @@ Examples: AlwaysPull: true, InitFunc: func(b *bundle.Bundle) { b.SkipLocalFileValidation = true + + // InitFunc runs before phases.Initialize, so this assignment takes + // precedence over DefineDefaultWorkspacePaths (which only fills an + // empty state_path) while still passing through PrependWorkspacePrefix. + if statePath != "" { + b.Config.Workspace.StatePath = statePath + } }, PostStateFunc: func(ctx context.Context, b *bundle.Bundle, stateDesc *statemgmt.StateDesc) error { stats.Engine = stateDesc.Engine @@ -181,3 +198,16 @@ Examples: return cmd } + +// validateStatePathFlag rejects a --state-path that would resolve against whoever runs +// the command. A caller-relative state folder is the failure this flag exists to +// override, so silently expanding "~" or a relative path here would reintroduce it. +func validateStatePathFlag(statePath string) error { + if statePath == "" { + return nil + } + if strings.HasPrefix(statePath, "~") || !strings.HasPrefix(statePath, "/") { + return fmt.Errorf("--state-path must be an absolute workspace path, got %q", statePath) + } + return nil +} From f55a90a4853e06c302d6e8c02b341a7178c866f2 Mon Sep 17 00:00:00 2001 From: Ilya Kuznetsov Date: Mon, 24 Aug 2026 16:17:24 +0000 Subject: [PATCH 2/3] Fix review findings: apply state path after target-mode validation, scope its cache Co-authored-by: Isaac --- .../state_path/other.yml.tmpl | 19 +++++ .../state_path/out.direct.txt | 6 +- .../state_path/out.terraform.txt | 6 +- .../config-remote-sync/state_path/output.txt | 47 ++++++++++--- .../config-remote-sync/state_path/script | 42 +++++++---- .../config-remote-sync/state_path/test.toml | 2 +- cmd/bundle/config_remote_sync.go | 69 +++++++++++++++---- 7 files changed, 153 insertions(+), 38 deletions(-) create mode 100644 acceptance/bundle/config-remote-sync/state_path/other.yml.tmpl diff --git a/acceptance/bundle/config-remote-sync/state_path/other.yml.tmpl b/acceptance/bundle/config-remote-sync/state_path/other.yml.tmpl new file mode 100644 index 00000000000..c487e491f01 --- /dev/null +++ b/acceptance/bundle/config-remote-sync/state_path/other.yml.tmpl @@ -0,0 +1,19 @@ +bundle: + name: other-bundle-$UNIQUE_NAME + +resources: + jobs: + other_job: + max_concurrent_runs: 1 + tasks: + - task_key: main + notebook_task: + notebook_path: /Users/{{workspace_user_name}}/notebook + new_cluster: + spark_version: $DEFAULT_SPARK_VERSION + node_type_id: $NODE_TYPE_ID + num_workers: 1 + +targets: + default: + mode: development diff --git a/acceptance/bundle/config-remote-sync/state_path/out.direct.txt b/acceptance/bundle/config-remote-sync/state_path/out.direct.txt index 1390dc66377..43525d99290 100644 --- a/acceptance/bundle/config-remote-sync/state_path/out.direct.txt +++ b/acceptance/bundle/config-remote-sync/state_path/out.direct.txt @@ -1,5 +1,9 @@ +Error: no deployed jobs resource with id [MY_JOB_ID]; deployed jobs ids in state: [NUMID] ->>> [CLI] bundle config-remote-sync --state-path /Workspace/Users/[USERNAME]/.bundle/other-bundle/default/state --select-ids jobs:[MY_JOB_ID] +Exit code: 1 +Error: no deployed jobs resource with id [MY_JOB_ID]; the deployment state contains no resources with ids (the bundle may not be deployed, or its resource state is missing) + +Exit code: 1 Error: no deployed jobs resource with id [MY_JOB_ID]; the deployment state contains no resources with ids (the bundle may not be deployed, or its resource state is missing) Exit code: 1 diff --git a/acceptance/bundle/config-remote-sync/state_path/out.terraform.txt b/acceptance/bundle/config-remote-sync/state_path/out.terraform.txt index d0e30f46836..e0824635ae6 100644 --- a/acceptance/bundle/config-remote-sync/state_path/out.terraform.txt +++ b/acceptance/bundle/config-remote-sync/state_path/out.terraform.txt @@ -1,5 +1,9 @@ +Error: no deployed jobs resource with id [MY_JOB_ID]; deployed jobs ids in state: [NUMID] ->>> [CLI] bundle config-remote-sync --state-path /Workspace/Users/[USERNAME]/.bundle/other-bundle/default/state --select-ids jobs:[MY_JOB_ID] +Exit code: 1 +Error: state snapshot not available: resources state snapshot not found remotely at resources-config-sync-snapshot.json: state snapshot not found + +Exit code: 1 Error: state snapshot not available: resources state snapshot not found remotely at resources-config-sync-snapshot.json: state snapshot not found Exit code: 1 diff --git a/acceptance/bundle/config-remote-sync/state_path/output.txt b/acceptance/bundle/config-remote-sync/state_path/output.txt index c310b9e8c46..a4ae30c164f 100644 --- a/acceptance/bundle/config-remote-sync/state_path/output.txt +++ b/acceptance/bundle/config-remote-sync/state_path/output.txt @@ -1,11 +1,15 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... Created jobs.my_job -Files: 5 uploaded, 0 deleted +Files: 7 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/other-bundle-[UNIQUE_NAME]/default/files... +Created jobs.other_job +Files: 1 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Modify the job remotely -=== --state-path at the deployment's own state folder detects the drift +=== --state-path at this deployment's own state folder detects the drift Detected changes in 1 resource(s): Resource: resources.jobs.my_job @@ -13,19 +17,44 @@ Resource: resources.jobs.my_job -=== A relative --state-path is rejected before any state is read +=== The same folder without a /Workspace prefix resolves identically +Detected changes in 1 resource(s): + +Resource: resources.jobs.my_job + max_concurrent_runs: replace + + + +=== Reading another deployment's state does not poison this bundle's own cache +sync against the other deployment's state failed as expected (its state has no jobs:[MY_JOB_ID]) +this bundle still reads its own state: +Detected changes in 1 resource(s): + +Resource: resources.jobs.my_job + max_concurrent_runs: replace + + + +=== A dev-mode target is not blocked from reading a state folder it does not own +neither foreign path was rejected by the dev-mode uniqueness check + +=== Rejected values >>> [CLI] bundle config-remote-sync --state-path relative/state Error: --state-path must be an absolute workspace path, got "relative/state" -Exit code: 1 - -=== A ~-relative --state-path is rejected too >>> [CLI] bundle config-remote-sync --state-path ~/.bundle/test/default/state -Error: --state-path must be an absolute workspace path, got "~/.bundle/test/default/state" +Error: --state-path must be an absolute workspace path, got "~/.bundle/test/default/state": pass the path of the deployment to sync, not one relative to the current user's home + +>>> [CLI] bundle config-remote-sync --state-path /Volumes/main/default/vol/state +Error: --state-path does not support Volumes paths, got "/Volumes/main/default/vol/state" -Exit code: 1 +>>> withdir other [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.other_job -=== --state-path elsewhere does not see the deployment +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/other-bundle-[UNIQUE_NAME]/default + +Destroy: 1 deleted >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: diff --git a/acceptance/bundle/config-remote-sync/state_path/script b/acceptance/bundle/config-remote-sync/state_path/script index 10d89390679..49cd499f6d5 100755 --- a/acceptance/bundle/config-remote-sync/state_path/script +++ b/acceptance/bundle/config-remote-sync/state_path/script @@ -1,16 +1,20 @@ #!/bin/bash envsubst < databricks.yml.tmpl > databricks.yml +mkdir -p other +envsubst < other.yml.tmpl > other/databricks.yml cleanup() { + trace withdir other $CLI bundle destroy --auto-approve trace $CLI bundle destroy --auto-approve } trap cleanup EXIT -deployed_state="/Workspace/Users/${CURRENT_USER_NAME}/.bundle/test-bundle-${UNIQUE_NAME}/default/state" -other_state="/Workspace/Users/${CURRENT_USER_NAME}/.bundle/other-bundle/default/state" +own_state="/Workspace/Users/${CURRENT_USER_NAME}/.bundle/test-bundle-${UNIQUE_NAME}/default/state" +other_state="/Workspace/Users/${CURRENT_USER_NAME}/.bundle/other-bundle-${UNIQUE_NAME}/default/state" $CLI bundle deploy +withdir other $CLI bundle deploy job_id="$(read_id.py my_job)" title "Modify the job remotely" @@ -19,20 +23,30 @@ edit_resource.py jobs $job_id < out.$DATABRICKS_BUNDLE_ENGINE.txt 2>&1 +echo "sync against the other deployment's state failed as expected (its state has no jobs:$job_id)" +echo "this bundle still reads its own state:" +$CLI bundle config-remote-sync -title "--state-path elsewhere does not see the deployment" +title "A dev-mode target is not blocked from reading a state folder it does not own" echo -rm -rf .databricks -errcode trace $CLI bundle config-remote-sync --state-path "$other_state" --select-ids "jobs:$job_id" > out.$DATABRICKS_BUNDLE_ENGINE.txt 2>&1 +errcode $CLI bundle config-remote-sync --state-path "/Workspace/Users/someone-else@example.com/.bundle/b/default/state" --select-ids "jobs:$job_id" 2>&1 | contains.py "!uniqueness when using 'mode: development'" >> out.$DATABRICKS_BUNDLE_ENGINE.txt +errcode $CLI bundle config-remote-sync --state-path "/Workspace/Shared/.bundle/b/default/state" --select-ids "jobs:$job_id" 2>&1 | contains.py "!uniqueness when using 'mode: development'" >> out.$DATABRICKS_BUNDLE_ENGINE.txt +echo "neither foreign path was rejected by the dev-mode uniqueness check" + +title "Rejected values" +musterr trace $CLI bundle config-remote-sync --state-path relative/state +musterr trace $CLI bundle config-remote-sync --state-path '~/.bundle/test/default/state' +musterr trace $CLI bundle config-remote-sync --state-path /Volumes/main/default/vol/state diff --git a/acceptance/bundle/config-remote-sync/state_path/test.toml b/acceptance/bundle/config-remote-sync/state_path/test.toml index 4f6a37ad901..c5e4b14d404 100644 --- a/acceptance/bundle/config-remote-sync/state_path/test.toml +++ b/acceptance/bundle/config-remote-sync/state_path/test.toml @@ -1,5 +1,5 @@ RecordRequests = false -Ignore = [".databricks", "databricks.yml"] +Ignore = [".databricks", "databricks.yml", "other"] Env.DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_YAML_SYNC = "true" diff --git a/cmd/bundle/config_remote_sync.go b/cmd/bundle/config_remote_sync.go index 329e0eca990..e9e99343b2b 100644 --- a/cmd/bundle/config_remote_sync.go +++ b/cmd/bundle/config_remote_sync.go @@ -2,20 +2,26 @@ package bundle import ( "context" + "crypto/sha256" + "encoding/hex" "encoding/json" "errors" "fmt" "maps" + "os" + "path/filepath" "runtime" "slices" "strings" "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/configsync" + "github.com/databricks/cli/bundle/env" "github.com/databricks/cli/bundle/statemgmt" "github.com/databricks/cli/cmd/bundle/utils" "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/cmdctx" + envlib "github.com/databricks/cli/libs/env" "github.com/databricks/cli/libs/flags" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/telemetry" @@ -66,6 +72,14 @@ Examples: return err } + // Scope the local state cache to the state folder being read. The cache is + // otherwise keyed on bundle root and target alone, so another deployment's + // state would land in the default location and be picked up as this bundle's + // own by later commands, including deploy. + if statePath != "" { + cmd.SetContext(envlib.Set(cmd.Context(), env.TempDirVariable, stateCacheDir(statePath))) + } + stats := configsync.Stats{Save: save} // Emit telemetry on every exit path, including failures inside @@ -84,13 +98,21 @@ Examples: AlwaysPull: true, InitFunc: func(b *bundle.Bundle) { b.SkipLocalFileValidation = true - - // InitFunc runs before phases.Initialize, so this assignment takes - // precedence over DefineDefaultWorkspacePaths (which only fills an - // empty state_path) while still passing through PrependWorkspacePrefix. - if statePath != "" { - b.Config.Workspace.StatePath = statePath + }, + // Applied after phases.Initialize so the dev-mode uniqueness check does not + // reject a state folder this command only reads: it belongs to the + // deployment being synced, not to whoever runs the command. + PostInitFunc: func(ctx context.Context, b *bundle.Bundle) error { + if statePath == "" { + return nil } + // Assigned through a mutator so the value also lands in the dynamic + // config tree. Later phases convert dyn->typed on entry, which would + // otherwise restore the default before the state snapshot is read. + bundle.ApplyFuncContext(ctx, b, func(context.Context, *bundle.Bundle) { + b.Config.Workspace.StatePath = normalizeStatePath(statePath) + }) + return nil }, PostStateFunc: func(ctx context.Context, b *bundle.Bundle, stateDesc *statemgmt.StateDesc) error { stats.Engine = stateDesc.Engine @@ -199,15 +221,38 @@ Examples: return cmd } -// validateStatePathFlag rejects a --state-path that would resolve against whoever runs -// the command. A caller-relative state folder is the failure this flag exists to -// override, so silently expanding "~" or a relative path here would reintroduce it. +// validateStatePathFlag rejects a --state-path this command cannot resolve to one +// deployment's state folder. "~" is refused rather than expanded: it resolves to the +// home of whoever runs the command, which is the resolution this flag exists to override. func validateStatePathFlag(statePath string) error { - if statePath == "" { + switch { + case statePath == "": return nil - } - if strings.HasPrefix(statePath, "~") || !strings.HasPrefix(statePath, "/") { + case strings.HasPrefix(statePath, "~"): + return fmt.Errorf("--state-path must be an absolute workspace path, got %q: pass the path of the deployment to sync, not one relative to the current user's home", statePath) + case !strings.HasPrefix(statePath, "/"): return fmt.Errorf("--state-path must be an absolute workspace path, got %q", statePath) + case strings.HasPrefix(statePath, "/Volumes/"): + return fmt.Errorf("--state-path does not support Volumes paths, got %q", statePath) } return nil } + +// stateCacheDir returns the local cache directory to use for an overridden state path. +// It is keyed on the state folder so state read from another deployment can never occupy +// the location this bundle caches its own state in, and stays outside the bundle tree so +// a plain deploy in the same directory cannot pick it up. Deterministic, so repeat runs +// against the same state folder still reuse the cache. +func stateCacheDir(statePath string) string { + sum := sha256.Sum256([]byte(normalizeStatePath(statePath))) + return filepath.Join(os.TempDir(), "databricks-bundle-state", hex.EncodeToString(sum[:8])) +} + +// normalizeStatePath applies the /Workspace prefixing that PrependWorkspacePrefix gives +// a configured state_path. That mutator has already run by the time the flag is applied. +func normalizeStatePath(statePath string) string { + if strings.HasPrefix(statePath, "/Workspace/") { + return statePath + } + return "/Workspace" + statePath +} From 3cfddf89cff884b30e304c4f6578b7330839d94c Mon Sep 17 00:00:00 2001 From: Ilya Kuznetsov Date: Mon, 24 Aug 2026 17:10:33 +0000 Subject: [PATCH 3/3] Read state into a scratch dir and recompute metadata_file_path Co-authored-by: Isaac --- .../state_path/other.yml.tmpl | 2 +- .../config-remote-sync/state_path/output.txt | 8 +++-- .../config-remote-sync/state_path/script | 7 ++++ cmd/bundle/config_remote_sync.go | 35 +++++++++---------- 4 files changed, 31 insertions(+), 21 deletions(-) diff --git a/acceptance/bundle/config-remote-sync/state_path/other.yml.tmpl b/acceptance/bundle/config-remote-sync/state_path/other.yml.tmpl index c487e491f01..838c49a3a3b 100644 --- a/acceptance/bundle/config-remote-sync/state_path/other.yml.tmpl +++ b/acceptance/bundle/config-remote-sync/state_path/other.yml.tmpl @@ -3,7 +3,7 @@ bundle: resources: jobs: - other_job: + my_job: max_concurrent_runs: 1 tasks: - task_key: main diff --git a/acceptance/bundle/config-remote-sync/state_path/output.txt b/acceptance/bundle/config-remote-sync/state_path/output.txt index a4ae30c164f..4045b8cc84c 100644 --- a/acceptance/bundle/config-remote-sync/state_path/output.txt +++ b/acceptance/bundle/config-remote-sync/state_path/output.txt @@ -3,7 +3,7 @@ Created jobs.my_job Files: 7 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/other-bundle-[UNIQUE_NAME]/default/files... -Created jobs.other_job +Created jobs.my_job Files: 1 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged @@ -25,6 +25,10 @@ Resource: resources.jobs.my_job +=== Reading another deployment's state reports no change of its own making +No changes detected. + + === Reading another deployment's state does not poison this bundle's own cache sync against the other deployment's state failed as expected (its state has no jobs:[MY_JOB_ID]) this bundle still reads its own state: @@ -50,7 +54,7 @@ Error: --state-path does not support Volumes paths, got "/Volumes/main/default/v >>> withdir other [CLI] bundle destroy --auto-approve The following resources will be deleted: - delete resources.jobs.other_job + delete resources.jobs.my_job All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/other-bundle-[UNIQUE_NAME]/default diff --git a/acceptance/bundle/config-remote-sync/state_path/script b/acceptance/bundle/config-remote-sync/state_path/script index 49cd499f6d5..2a0963ef607 100755 --- a/acceptance/bundle/config-remote-sync/state_path/script +++ b/acceptance/bundle/config-remote-sync/state_path/script @@ -33,6 +33,13 @@ title "The same folder without a /Workspace prefix resolves identically" echo $CLI bundle config-remote-sync --state-path "/Users/${CURRENT_USER_NAME}/.bundle/test-bundle-${UNIQUE_NAME}/default/state" --select-ids "jobs:$job_id" +title "Reading another deployment's state reports no change of its own making" +echo +# The other deployment declares the same resource with the same fields, so a clean +# diff is empty. Anything reported here is derived from this bundle's own defaults +# rather than from the state folder that was passed. +$CLI bundle config-remote-sync --state-path "$other_state" + title "Reading another deployment's state does not poison this bundle's own cache" echo errcode $CLI bundle config-remote-sync --state-path "$other_state" --select-ids "jobs:$job_id" > out.$DATABRICKS_BUNDLE_ENGINE.txt 2>&1 diff --git a/cmd/bundle/config_remote_sync.go b/cmd/bundle/config_remote_sync.go index e9e99343b2b..5ade75f5dec 100644 --- a/cmd/bundle/config_remote_sync.go +++ b/cmd/bundle/config_remote_sync.go @@ -2,20 +2,18 @@ package bundle import ( "context" - "crypto/sha256" - "encoding/hex" "encoding/json" "errors" "fmt" "maps" "os" - "path/filepath" "runtime" "slices" "strings" "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/configsync" + "github.com/databricks/cli/bundle/deploy/metadata" "github.com/databricks/cli/bundle/env" "github.com/databricks/cli/bundle/statemgmt" "github.com/databricks/cli/cmd/bundle/utils" @@ -72,12 +70,19 @@ Examples: return err } - // Scope the local state cache to the state folder being read. The cache is - // otherwise keyed on bundle root and target alone, so another deployment's - // state would land in the default location and be picked up as this bundle's - // own by later commands, including deploy. + // Materialize the state being read into a scratch directory: it has to be on + // disk to be opened, and the default location is keyed on bundle root and + // target alone, so another deployment's state would be left there for later + // commands, including deploy, to pick up as this bundle's own. Nothing needs + // it to persist, since the state is always re-read and never written back. if statePath != "" { - cmd.SetContext(envlib.Set(cmd.Context(), env.TempDirVariable, stateCacheDir(statePath))) + parent, _ := env.TempDir(cmd.Context()) + stateDir, err := os.MkdirTemp(parent, "databricks-bundle-state-") + if err != nil { + return err + } + defer os.RemoveAll(stateDir) + cmd.SetContext(envlib.Set(cmd.Context(), env.TempDirVariable, stateDir)) } stats := configsync.Stats{Save: save} @@ -112,6 +117,10 @@ Examples: bundle.ApplyFuncContext(ctx, b, func(context.Context, *bundle.Bundle) { b.Config.Workspace.StatePath = normalizeStatePath(statePath) }) + // deployment.metadata_file_path is derived from state_path at the end of + // phases.Initialize, which ran before this override. Recompute it, or the + // diff reports the default location as a change on every resource. + bundle.ApplySeqContext(ctx, b, metadata.AnnotateJobs(), metadata.AnnotatePipelines()) return nil }, PostStateFunc: func(ctx context.Context, b *bundle.Bundle, stateDesc *statemgmt.StateDesc) error { @@ -238,16 +247,6 @@ func validateStatePathFlag(statePath string) error { return nil } -// stateCacheDir returns the local cache directory to use for an overridden state path. -// It is keyed on the state folder so state read from another deployment can never occupy -// the location this bundle caches its own state in, and stays outside the bundle tree so -// a plain deploy in the same directory cannot pick it up. Deterministic, so repeat runs -// against the same state folder still reuse the cache. -func stateCacheDir(statePath string) string { - sum := sha256.Sum256([]byte(normalizeStatePath(statePath))) - return filepath.Join(os.TempDir(), "databricks-bundle-state", hex.EncodeToString(sum[:8])) -} - // normalizeStatePath applies the /Workspace prefixing that PrependWorkspacePrefix gives // a configured state_path. That mutator has already run by the time the flag is applied. func normalizeStatePath(statePath string) string {