Add --state-path flag to bundle config-remote-sync

acceptance/bundle/config-remote-sync/state_path/databricks.yml.tmpl

@@ -0,0 +1,19 @@
+bundle:
+  name: test-bundle-$UNIQUE_NAME
+
+resources:
+  jobs:
+    my_job:
+      max_concurrent_runs: 1
+      tasks:
+        - task_key: main
+          notebook_task:
+            notebook_path: /Users/{{workspace_user_name}}/notebook
+          new_cluster:
+            spark_version: $DEFAULT_SPARK_VERSION
+            node_type_id: $NODE_TYPE_ID
+            num_workers: 1
+
+targets:
+  default:
+    mode: development

acceptance/bundle/config-remote-sync/state_path/other.yml.tmpl

@@ -0,0 +1,19 @@
+bundle:
+  name: other-bundle-$UNIQUE_NAME
+
+resources:
+  jobs:
+    my_job:
+      max_concurrent_runs: 1
+      tasks:
+        - task_key: main
+          notebook_task:
+            notebook_path: /Users/{{workspace_user_name}}/notebook
+          new_cluster:
+            spark_version: $DEFAULT_SPARK_VERSION
+            node_type_id: $NODE_TYPE_ID
+            num_workers: 1
+
+targets:
+  default:
+    mode: development

acceptance/bundle/config-remote-sync/state_path/out.direct.txt

@@ -0,0 +1,9 @@
+Error: no deployed jobs resource with id [MY_JOB_ID]; deployed jobs ids in state: [NUMID]
+
+Exit code: 1
+Error: no deployed jobs resource with id [MY_JOB_ID]; the deployment state contains no resources with ids (the bundle may not be deployed, or its resource state is missing)
+
+Exit code: 1
+Error: no deployed jobs resource with id [MY_JOB_ID]; the deployment state contains no resources with ids (the bundle may not be deployed, or its resource state is missing)
+
+Exit code: 1

acceptance/bundle/config-remote-sync/state_path/out.terraform.txt

@@ -0,0 +1,9 @@
+Error: no deployed jobs resource with id [MY_JOB_ID]; deployed jobs ids in state: [NUMID]
+
+Exit code: 1
+Error: state snapshot not available: resources state snapshot not found remotely at resources-config-sync-snapshot.json: state snapshot not found
+
+Exit code: 1
+Error: state snapshot not available: resources state snapshot not found remotely at resources-config-sync-snapshot.json: state snapshot not found
+
+Exit code: 1

acceptance/bundle/config-remote-sync/state_path/output.txt

@@ -0,0 +1,69 @@
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files...
+Created jobs.my_job
+Files: 7 uploaded, 0 deleted
+Resources: 1 created, 0 changed, 0 deleted, 0 unchanged
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/other-bundle-[UNIQUE_NAME]/default/files...
+Created jobs.my_job
+Files: 1 uploaded, 0 deleted
+Resources: 1 created, 0 changed, 0 deleted, 0 unchanged
+
+=== Modify the job remotely
+
+=== --state-path at this deployment's own state folder detects the drift
+Detected changes in 1 resource(s):
+
+Resource: resources.jobs.my_job
+  max_concurrent_runs: replace
+
+
+
+=== The same folder without a /Workspace prefix resolves identically
+Detected changes in 1 resource(s):
+
+Resource: resources.jobs.my_job
+  max_concurrent_runs: replace
+
+
+
+=== Reading another deployment's state reports no change of its own making
+No changes detected.
+
+
+=== Reading another deployment's state does not poison this bundle's own cache
+sync against the other deployment's state failed as expected (its state has no jobs:[MY_JOB_ID])
+this bundle still reads its own state:
+Detected changes in 1 resource(s):
+
+Resource: resources.jobs.my_job
+  max_concurrent_runs: replace
+
+
+
+=== A dev-mode target is not blocked from reading a state folder it does not own
+neither foreign path was rejected by the dev-mode uniqueness check
+
+=== Rejected values
+>>> [CLI] bundle config-remote-sync --state-path relative/state
+Error: --state-path must be an absolute workspace path, got "relative/state"
+
+>>> [CLI] bundle config-remote-sync --state-path ~/.bundle/test/default/state
+Error: --state-path must be an absolute workspace path, got "~/.bundle/test/default/state": pass the path of the deployment to sync, not one relative to the current user's home
+
+>>> [CLI] bundle config-remote-sync --state-path /Volumes/main/default/vol/state
+Error: --state-path does not support Volumes paths, got "/Volumes/main/default/vol/state"
+
+>>> withdir other [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.jobs.my_job
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/other-bundle-[UNIQUE_NAME]/default
+
+Destroy: 1 deleted
+
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.jobs.my_job
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default
+
+Destroy: 1 deleted

acceptance/bundle/config-remote-sync/state_path/script

@@ -0,0 +1,59 @@
+#!/bin/bash
+
+envsubst < databricks.yml.tmpl > databricks.yml
+mkdir -p other
+envsubst < other.yml.tmpl > other/databricks.yml
+
+cleanup() {
+  trace withdir other $CLI bundle destroy --auto-approve
+  trace $CLI bundle destroy --auto-approve
+}
+trap cleanup EXIT
+
+own_state="/Workspace/Users/${CURRENT_USER_NAME}/.bundle/test-bundle-${UNIQUE_NAME}/default/state"
+other_state="/Workspace/Users/${CURRENT_USER_NAME}/.bundle/other-bundle-${UNIQUE_NAME}/default/state"
+
+$CLI bundle deploy
+withdir other $CLI bundle deploy
+job_id="$(read_id.py my_job)"
+
+title "Modify the job remotely"
+echo
+edit_resource.py jobs $job_id <<INNER
+r["max_concurrent_runs"] = 5
+INNER
+
+# Nothing below flushes .databricks: the deploy above populated it, and the override
+# has to win over that cache rather than be silently defeated by it.
+title "--state-path at this deployment's own state folder detects the drift"
+echo
+$CLI bundle config-remote-sync --state-path "$own_state" --select-ids "jobs:$job_id"
+
+title "The same folder without a /Workspace prefix resolves identically"
+echo
+$CLI bundle config-remote-sync --state-path "/Users/${CURRENT_USER_NAME}/.bundle/test-bundle-${UNIQUE_NAME}/default/state" --select-ids "jobs:$job_id"
+
+title "Reading another deployment's state reports no change of its own making"
+echo
+# The other deployment declares the same resource with the same fields, so a clean
+# diff is empty. Anything reported here is derived from this bundle's own defaults
+# rather than from the state folder that was passed.
+$CLI bundle config-remote-sync --state-path "$other_state"
+
+title "Reading another deployment's state does not poison this bundle's own cache"
+echo
+errcode $CLI bundle config-remote-sync --state-path "$other_state" --select-ids "jobs:$job_id" > out.$DATABRICKS_BUNDLE_ENGINE.txt 2>&1
+echo "sync against the other deployment's state failed as expected (its state has no jobs:$job_id)"
+echo "this bundle still reads its own state:"
+$CLI bundle config-remote-sync
+
+title "A dev-mode target is not blocked from reading a state folder it does not own"
+echo
+errcode $CLI bundle config-remote-sync --state-path "/Workspace/Users/someone-else@example.com/.bundle/b/default/state" --select-ids "jobs:$job_id" 2>&1 | contains.py "!uniqueness when using 'mode: development'" >> out.$DATABRICKS_BUNDLE_ENGINE.txt
+errcode $CLI bundle config-remote-sync --state-path "/Workspace/Shared/.bundle/b/default/state" --select-ids "jobs:$job_id" 2>&1 | contains.py "!uniqueness when using 'mode: development'" >> out.$DATABRICKS_BUNDLE_ENGINE.txt
+echo "neither foreign path was rejected by the dev-mode uniqueness check"
+
+title "Rejected values"
+musterr trace $CLI bundle config-remote-sync --state-path relative/state
+musterr trace $CLI bundle config-remote-sync --state-path '~/.bundle/test/default/state'
+musterr trace $CLI bundle config-remote-sync --state-path /Volumes/main/default/vol/state

acceptance/bundle/config-remote-sync/state_path/test.toml

@@ -0,0 +1,6 @@
+RecordRequests = false
+Ignore = [".databricks", "databricks.yml", "other"]
+
+Env.DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_YAML_SYNC = "true"
+
+EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"]

cmd/bundle/config_remote_sync.go

@@ -6,15 +6,20 @@ import (
 	"errors"
 	"fmt"
 	"maps"
+	"os"
 	"runtime"
 	"slices"
+	"strings"
 
 	"github.com/databricks/cli/bundle"
 	"github.com/databricks/cli/bundle/configsync"
+	"github.com/databricks/cli/bundle/deploy/metadata"
+	"github.com/databricks/cli/bundle/env"
 	"github.com/databricks/cli/bundle/statemgmt"
 	"github.com/databricks/cli/cmd/bundle/utils"
 	"github.com/databricks/cli/cmd/root"
 	"github.com/databricks/cli/libs/cmdctx"
+	envlib "github.com/databricks/cli/libs/env"
 	"github.com/databricks/cli/libs/flags"
 	"github.com/databricks/cli/libs/log"
 	"github.com/databricks/cli/libs/telemetry"
@@ -25,6 +30,7 @@ import (
 func newConfigRemoteSyncCommand() *cobra.Command {
 	var save bool
 	var selectIDs []string
+	var statePath string
 
 	cmd := &cobra.Command{
 		Use:   "config-remote-sync",
@@ -44,18 +50,41 @@ Examples:
   databricks bundle config-remote-sync --save
 
   # Restrict the sync to a single resource by its type and deployed resource ID
-  databricks bundle config-remote-sync --select-ids jobs:123456789 --save`,
+  databricks bundle config-remote-sync --select-ids jobs:123456789 --save
+
+  # Read the deployment state from an explicit workspace location
+  databricks bundle config-remote-sync --state-path /Workspace/Shared/.bundle/my_bundle/dev/state`,
 		Hidden: true, // Used by DABs in the Workspace only
 	}
 
 	cmd.Flags().BoolVar(&save, "save", false, "Write updated config files to disk")
 	cmd.Flags().StringSliceVar(&selectIDs, "select-ids", nil, "Sync only the given resources, each as <type>:<id> (e.g. jobs:123456789). Can be repeated or comma-separated.")
+	cmd.Flags().StringVar(&statePath, "state-path", "", "Absolute workspace path of the deployment state folder to read, overriding workspace.state_path. Use when the state does not live under the path this command resolves by default, e.g. because the bundle was deployed by another user.")
 
 	cmd.RunE = func(cmd *cobra.Command, args []string) error {
 		if runtime.GOOS == "windows" {
 			return errors.New("config-remote-sync command is not supported on Windows")
 		}
 
+		if err := validateStatePathFlag(statePath); err != nil {
+			return err
+		}
+
+		// Materialize the state being read into a scratch directory: it has to be on
+		// disk to be opened, and the default location is keyed on bundle root and
+		// target alone, so another deployment's state would be left there for later
+		// commands, including deploy, to pick up as this bundle's own. Nothing needs
+		// it to persist, since the state is always re-read and never written back.
+		if statePath != "" {
+			parent, _ := env.TempDir(cmd.Context())
+			stateDir, err := os.MkdirTemp(parent, "databricks-bundle-state-")
+			if err != nil {
+				return err
+			}
+			defer os.RemoveAll(stateDir)
+			cmd.SetContext(envlib.Set(cmd.Context(), env.TempDirVariable, stateDir))
+		}
+
 		stats := configsync.Stats{Save: save}
 
 		// Emit telemetry on every exit path, including failures inside
@@ -75,6 +104,25 @@ Examples:
 			InitFunc: func(b *bundle.Bundle) {
 				b.SkipLocalFileValidation = true
 			},
+			// Applied after phases.Initialize so the dev-mode uniqueness check does not
+			// reject a state folder this command only reads: it belongs to the
+			// deployment being synced, not to whoever runs the command.
+			PostInitFunc: func(ctx context.Context, b *bundle.Bundle) error {
+				if statePath == "" {
+					return nil
+				}
+				// Assigned through a mutator so the value also lands in the dynamic
+				// config tree. Later phases convert dyn->typed on entry, which would
+				// otherwise restore the default before the state snapshot is read.
+				bundle.ApplyFuncContext(ctx, b, func(context.Context, *bundle.Bundle) {
+					b.Config.Workspace.StatePath = normalizeStatePath(statePath)
+				})
+				// deployment.metadata_file_path is derived from state_path at the end of
+				// phases.Initialize, which ran before this override. Recompute it, or the
+				// diff reports the default location as a change on every resource.
+				bundle.ApplySeqContext(ctx, b, metadata.AnnotateJobs(), metadata.AnnotatePipelines())
+				return nil
+			},
 			PostStateFunc: func(ctx context.Context, b *bundle.Bundle, stateDesc *statemgmt.StateDesc) error {
 				stats.Engine = stateDesc.Engine
 				stats.CollectStateStats(stateDesc)
@@ -181,3 +229,29 @@ Examples:
 
 	return cmd
 }
+
+// validateStatePathFlag rejects a --state-path this command cannot resolve to one
+// deployment's state folder. "~" is refused rather than expanded: it resolves to the
+// home of whoever runs the command, which is the resolution this flag exists to override.
+func validateStatePathFlag(statePath string) error {
+	switch {
+	case statePath == "":
+		return nil
+	case strings.HasPrefix(statePath, "~"):
+		return fmt.Errorf("--state-path must be an absolute workspace path, got %q: pass the path of the deployment to sync, not one relative to the current user's home", statePath)
+	case !strings.HasPrefix(statePath, "/"):
+		return fmt.Errorf("--state-path must be an absolute workspace path, got %q", statePath)
+	case strings.HasPrefix(statePath, "/Volumes/"):
+		return fmt.Errorf("--state-path does not support Volumes paths, got %q", statePath)
+	}
+	return nil
+}
+
+// normalizeStatePath applies the /Workspace prefixing that PrependWorkspacePrefix gives
+// a configured state_path. That mutator has already run by the time the flag is applied.
+func normalizeStatePath(statePath string) string {
+	if strings.HasPrefix(statePath, "/Workspace/") {
+		return statePath
+	}
+	return "/Workspace" + statePath
+}