From 67338bb83a0f364a71574d8fb0d6ed567a83ce59 Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Fri, 16 Oct 2026 09:34:15 +0000 Subject: [PATCH] Port sync integration tests to acceptance tests Each --watch step becomes a separate `databricks sync` run, so the tests now also cover the snapshot being saved and reloaded between runs, and they run locally against the testserver as well as on cloud. TestSyncIncrementalFileSync stays in integration/ to cover --watch. Co-authored-by: Isaac --- .../file-replaces-folder/out.test.toml | 1 + .../file-replaces-folder/output.txt | 37 ++ .../file-replaces-folder/script | 18 + .../cmd/sync-incremental/files/out.test.toml | 1 + .../cmd/sync-incremental/files/output.txt | 63 +++ acceptance/cmd/sync-incremental/files/script | 33 ++ .../nested-folders/out.test.toml | 1 + .../nested-folders/output.txt | 58 +++ .../sync-incremental/nested-folders/script | 25 ++ .../nonempty-remote-dir/out.test.toml | 1 + .../nonempty-remote-dir/output.txt | 30 ++ .../nonempty-remote-dir/script | 13 + .../notebook-transitions/out.test.toml | 1 + .../notebook-transitions/output.txt | 67 ++++ .../notebook-transitions/script | 34 ++ .../cmd/sync-incremental/script.prepare | 39 ++ acceptance/cmd/sync-incremental/test.toml | 8 + acceptance/cmd/sync-remote-path/out.test.toml | 1 + acceptance/cmd/sync-remote-path/output.txt | 37 ++ acceptance/cmd/sync-remote-path/script | 30 ++ acceptance/cmd/sync-remote-path/test.toml | 7 + integration/cmd/sync/sync_test.go | 375 +----------------- 22 files changed, 508 insertions(+), 372 deletions(-) create mode 100644 acceptance/cmd/sync-incremental/file-replaces-folder/out.test.toml create mode 100644 acceptance/cmd/sync-incremental/file-replaces-folder/output.txt create mode 100644 acceptance/cmd/sync-incremental/file-replaces-folder/script create mode 100644 acceptance/cmd/sync-incremental/files/out.test.toml create mode 100644 acceptance/cmd/sync-incremental/files/output.txt create mode 100644 acceptance/cmd/sync-incremental/files/script create mode 100644 acceptance/cmd/sync-incremental/nested-folders/out.test.toml create mode 100644 acceptance/cmd/sync-incremental/nested-folders/output.txt create mode 100644 acceptance/cmd/sync-incremental/nested-folders/script create mode 100644 acceptance/cmd/sync-incremental/nonempty-remote-dir/out.test.toml create mode 100644 acceptance/cmd/sync-incremental/nonempty-remote-dir/output.txt create mode 100644 acceptance/cmd/sync-incremental/nonempty-remote-dir/script create mode 100644 acceptance/cmd/sync-incremental/notebook-transitions/out.test.toml create mode 100644 acceptance/cmd/sync-incremental/notebook-transitions/output.txt create mode 100644 acceptance/cmd/sync-incremental/notebook-transitions/script create mode 100644 acceptance/cmd/sync-incremental/script.prepare create mode 100644 acceptance/cmd/sync-incremental/test.toml create mode 100644 acceptance/cmd/sync-remote-path/out.test.toml create mode 100644 acceptance/cmd/sync-remote-path/output.txt create mode 100644 acceptance/cmd/sync-remote-path/script create mode 100644 acceptance/cmd/sync-remote-path/test.toml diff --git a/acceptance/cmd/sync-incremental/file-replaces-folder/out.test.toml b/acceptance/cmd/sync-incremental/file-replaces-folder/out.test.toml new file mode 100644 index 00000000000..c7c6f58ed6e --- /dev/null +++ b/acceptance/cmd/sync-incremental/file-replaces-folder/out.test.toml @@ -0,0 +1 @@ +Cloud = true diff --git a/acceptance/cmd/sync-incremental/file-replaces-folder/output.txt b/acceptance/cmd/sync-incremental/file-replaces-folder/output.txt new file mode 100644 index 00000000000..c2f120f1a6d --- /dev/null +++ b/acceptance/cmd/sync-incremental/file-replaces-folder/output.txt @@ -0,0 +1,37 @@ + +=== Create foo/bar.txt + +>>> [CLI] sync src /Users/[USERNAME]/sync-[UNIQUE_NAME] +Initial Sync Complete +Uploaded foo +Uploaded foo/bar.txt + +>>> remote_tree +DIRECTORY foo +FILE foo/bar.txt + +>>> snapshot_files +foo/bar.txt + +=== Delete foo/bar.txt and the foo folder + +>>> [CLI] sync src /Users/[USERNAME]/sync-[UNIQUE_NAME] +Deleted foo +Deleted foo/bar.txt +Initial Sync Complete + +>>> remote_tree + +>>> snapshot_files + +=== Create a file named foo where the folder used to be + +>>> [CLI] sync src /Users/[USERNAME]/sync-[UNIQUE_NAME] +Initial Sync Complete +Uploaded foo + +>>> remote_tree +FILE foo + +>>> snapshot_files +foo diff --git a/acceptance/cmd/sync-incremental/file-replaces-folder/script b/acceptance/cmd/sync-incremental/file-replaces-folder/script new file mode 100644 index 00000000000..f1d8ee703d0 --- /dev/null +++ b/acceptance/cmd/sync-incremental/file-replaces-folder/script @@ -0,0 +1,18 @@ +title "Create foo/bar.txt\n" +mkdir src/foo +touch src/foo/bar.txt +sync_once +trace remote_tree +trace snapshot_files + +title "Delete foo/bar.txt and the foo folder\n" +rm -r src/foo +sync_once +trace remote_tree +trace snapshot_files + +title "Create a file named foo where the folder used to be\n" +touch src/foo +sync_once +trace remote_tree +trace snapshot_files diff --git a/acceptance/cmd/sync-incremental/files/out.test.toml b/acceptance/cmd/sync-incremental/files/out.test.toml new file mode 100644 index 00000000000..c7c6f58ed6e --- /dev/null +++ b/acceptance/cmd/sync-incremental/files/out.test.toml @@ -0,0 +1 @@ +Cloud = true diff --git a/acceptance/cmd/sync-incremental/files/output.txt b/acceptance/cmd/sync-incremental/files/output.txt new file mode 100644 index 00000000000..0455713c529 --- /dev/null +++ b/acceptance/cmd/sync-incremental/files/output.txt @@ -0,0 +1,63 @@ + +=== Initial sync of an empty directory + +>>> [CLI] sync src /Users/[USERNAME]/sync-[UNIQUE_NAME] +Initial Sync Complete + +>>> remote_tree + +=== New file + +>>> [CLI] sync src /Users/[USERNAME]/sync-[UNIQUE_NAME] +Initial Sync Complete +Uploaded foo.txt + +>>> remote_tree +FILE foo.txt + +>>> remote_content foo.txt + +>>> snapshot_files +foo.txt + +=== Write to file + +>>> [CLI] sync src /Users/[USERNAME]/sync-[UNIQUE_NAME] +Initial Sync Complete +Uploaded foo.txt + +>>> remote_content foo.txt +{"statement": "Mi Gente"} + +=== Write again + +>>> [CLI] sync src /Users/[USERNAME]/sync-[UNIQUE_NAME] +Initial Sync Complete +Uploaded foo.txt + +>>> remote_content foo.txt +{"statement": "Young Dumb & Broke"} + +=== Sync without changes uploads nothing + +>>> [CLI] sync src /Users/[USERNAME]/sync-[UNIQUE_NAME] +Initial Sync Complete + +=== Full sync uploads unchanged files again + +>>> [CLI] sync src /Users/[USERNAME]/sync-[UNIQUE_NAME] --full +Initial Sync Complete +Uploaded foo.txt + +>>> remote_content foo.txt +{"statement": "Young Dumb & Broke"} + +=== Delete + +>>> [CLI] sync src /Users/[USERNAME]/sync-[UNIQUE_NAME] +Deleted foo.txt +Initial Sync Complete + +>>> remote_tree + +>>> snapshot_files diff --git a/acceptance/cmd/sync-incremental/files/script b/acceptance/cmd/sync-incremental/files/script new file mode 100644 index 00000000000..6b980ece213 --- /dev/null +++ b/acceptance/cmd/sync-incremental/files/script @@ -0,0 +1,33 @@ +title "Initial sync of an empty directory\n" +sync_once +trace remote_tree + +title "New file\n" +touch src/foo.txt +sync_once +trace remote_tree +trace remote_content foo.txt +trace snapshot_files + +title "Write to file\n" +echo '{"statement": "Mi Gente"}' > src/foo.txt +sync_once +trace remote_content foo.txt + +title "Write again\n" +echo '{"statement": "Young Dumb & Broke"}' > src/foo.txt +sync_once +trace remote_content foo.txt + +title "Sync without changes uploads nothing\n" +sync_once + +title "Full sync uploads unchanged files again\n" +sync_once --full +trace remote_content foo.txt + +title "Delete\n" +rm src/foo.txt +sync_once +trace remote_tree +trace snapshot_files diff --git a/acceptance/cmd/sync-incremental/nested-folders/out.test.toml b/acceptance/cmd/sync-incremental/nested-folders/out.test.toml new file mode 100644 index 00000000000..c7c6f58ed6e --- /dev/null +++ b/acceptance/cmd/sync-incremental/nested-folders/out.test.toml @@ -0,0 +1 @@ +Cloud = true diff --git a/acceptance/cmd/sync-incremental/nested-folders/output.txt b/acceptance/cmd/sync-incremental/nested-folders/output.txt new file mode 100644 index 00000000000..6c5c091968a --- /dev/null +++ b/acceptance/cmd/sync-incremental/nested-folders/output.txt @@ -0,0 +1,58 @@ + +=== New file in nested folders + +>>> [CLI] sync src /Users/[USERNAME]/sync-[UNIQUE_NAME] +Initial Sync Complete +Uploaded dir1/dir2/dir3 +Uploaded dir1/dir2/dir3/foo.txt + +>>> remote_tree +DIRECTORY dir1 +DIRECTORY dir1/dir2 +DIRECTORY dir1/dir2/dir3 +FILE dir1/dir2/dir3/foo.txt + +>>> snapshot_files +dir1/dir2/dir3/foo.txt + +=== Delete the file; the empty remote folders are removed too + +>>> [CLI] sync src /Users/[USERNAME]/sync-[UNIQUE_NAME] +Deleted dir1 +Deleted dir1/dir2 +Deleted dir1/dir2/dir3 +Deleted dir1/dir2/dir3/foo.txt +Initial Sync Complete + +>>> remote_tree + +>>> snapshot_files + +=== Spaces, plus and hash in folder and file names + +>>> [CLI] sync src /Users/[USERNAME]/sync-[UNIQUE_NAME] +Initial Sync Complete +Uploaded dir1/a b+c/c+d e +Uploaded dir1/a b+c/c+d e/e+f g#i.txt + +>>> remote_tree +DIRECTORY dir1 +DIRECTORY dir1/a b+c +DIRECTORY dir1/a b+c/c+d e +FILE dir1/a b+c/c+d e/e+f g#i.txt + +>>> snapshot_files +dir1/a b+c/c+d e/e+f g#i.txt + +=== Delete the file with special characters + +>>> [CLI] sync src /Users/[USERNAME]/sync-[UNIQUE_NAME] +Deleted dir1 +Deleted dir1/a b+c +Deleted dir1/a b+c/c+d e +Deleted dir1/a b+c/c+d e/e+f g#i.txt +Initial Sync Complete + +>>> remote_tree + +>>> snapshot_files diff --git a/acceptance/cmd/sync-incremental/nested-folders/script b/acceptance/cmd/sync-incremental/nested-folders/script new file mode 100644 index 00000000000..49a95888799 --- /dev/null +++ b/acceptance/cmd/sync-incremental/nested-folders/script @@ -0,0 +1,25 @@ +title "New file in nested folders\n" +mkdir -p src/dir1/dir2/dir3 +touch src/dir1/dir2/dir3/foo.txt +sync_once +trace remote_tree +trace snapshot_files + +title "Delete the file; the empty remote folders are removed too\n" +rm src/dir1/dir2/dir3/foo.txt +sync_once +trace remote_tree +trace snapshot_files + +title "Spaces, plus and hash in folder and file names\n" +mkdir -p "src/dir1/a b+c/c+d e" +touch "src/dir1/a b+c/c+d e/e+f g#i.txt" +sync_once +trace remote_tree +trace snapshot_files + +title "Delete the file with special characters\n" +rm "src/dir1/a b+c/c+d e/e+f g#i.txt" +sync_once +trace remote_tree +trace snapshot_files diff --git a/acceptance/cmd/sync-incremental/nonempty-remote-dir/out.test.toml b/acceptance/cmd/sync-incremental/nonempty-remote-dir/out.test.toml new file mode 100644 index 00000000000..c7c6f58ed6e --- /dev/null +++ b/acceptance/cmd/sync-incremental/nonempty-remote-dir/out.test.toml @@ -0,0 +1 @@ +Cloud = true diff --git a/acceptance/cmd/sync-incremental/nonempty-remote-dir/output.txt b/acceptance/cmd/sync-incremental/nonempty-remote-dir/output.txt new file mode 100644 index 00000000000..62e6ececd22 --- /dev/null +++ b/acceptance/cmd/sync-incremental/nonempty-remote-dir/output.txt @@ -0,0 +1,30 @@ + +=== New file in nested folders + +>>> [CLI] sync src /Users/[USERNAME]/sync-[UNIQUE_NAME] +Initial Sync Complete +Uploaded dir1/dir2/dir3 +Uploaded dir1/dir2/dir3/foo.txt + +>>> remote_tree +DIRECTORY dir1 +DIRECTORY dir1/dir2 +DIRECTORY dir1/dir2/dir3 +FILE dir1/dir2/dir3/foo.txt + +=== Write a file to dir1 directly in the workspace + +>>> [CLI] workspace import /Users/[USERNAME]/sync-[UNIQUE_NAME]/dir1/foo.txt --format RAW --content Y29udGVudHM= + +=== Delete the local file: sync removes dir2 and dir3 but keeps dir1, which is not empty + +>>> [CLI] sync src /Users/[USERNAME]/sync-[UNIQUE_NAME] +Deleted dir1 +Deleted dir1/dir2 +Deleted dir1/dir2/dir3 +Deleted dir1/dir2/dir3/foo.txt +Initial Sync Complete + +>>> remote_tree +DIRECTORY dir1 +FILE dir1/foo.txt diff --git a/acceptance/cmd/sync-incremental/nonempty-remote-dir/script b/acceptance/cmd/sync-incremental/nonempty-remote-dir/script new file mode 100644 index 00000000000..bd1914ebc7e --- /dev/null +++ b/acceptance/cmd/sync-incremental/nonempty-remote-dir/script @@ -0,0 +1,13 @@ +title "New file in nested folders\n" +mkdir -p src/dir1/dir2/dir3 +touch src/dir1/dir2/dir3/foo.txt +sync_once +trace remote_tree + +title "Write a file to dir1 directly in the workspace\n" +trace $CLI workspace import "$remote_dir/dir1/foo.txt" --format RAW --content "$(printf contents | base64)" + +title "Delete the local file: sync removes dir2 and dir3 but keeps dir1, which is not empty\n" +rm src/dir1/dir2/dir3/foo.txt +sync_once +trace remote_tree diff --git a/acceptance/cmd/sync-incremental/notebook-transitions/out.test.toml b/acceptance/cmd/sync-incremental/notebook-transitions/out.test.toml new file mode 100644 index 00000000000..c7c6f58ed6e --- /dev/null +++ b/acceptance/cmd/sync-incremental/notebook-transitions/out.test.toml @@ -0,0 +1 @@ +Cloud = true diff --git a/acceptance/cmd/sync-incremental/notebook-transitions/output.txt b/acceptance/cmd/sync-incremental/notebook-transitions/output.txt new file mode 100644 index 00000000000..7111fe9125d --- /dev/null +++ b/acceptance/cmd/sync-incremental/notebook-transitions/output.txt @@ -0,0 +1,67 @@ + +=== Python notebook is uploaded without its extension + +>>> [CLI] sync src /Users/[USERNAME]/sync-[UNIQUE_NAME] +Initial Sync Complete +Uploaded foo.py + +>>> remote_tree +NOTEBOOK foo PYTHON + +>>> snapshot_files +foo.py + +=== Turn the notebook into a plain Python file + +>>> [CLI] sync src /Users/[USERNAME]/sync-[UNIQUE_NAME] +Deleted foo +Initial Sync Complete +Uploaded foo.py + +>>> remote_tree +FILE foo.py + +>>> snapshot_files +foo.py + +=== Delete the plain Python file + +>>> [CLI] sync src /Users/[USERNAME]/sync-[UNIQUE_NAME] +Deleted foo.py +Initial Sync Complete + +>>> remote_tree + +>>> snapshot_files + +=== Plain Python file + +>>> [CLI] sync src /Users/[USERNAME]/sync-[UNIQUE_NAME] +Initial Sync Complete +Uploaded bar.py + +>>> remote_tree +FILE bar.py + +=== Turn the plain Python file into a notebook + +>>> [CLI] sync src /Users/[USERNAME]/sync-[UNIQUE_NAME] +Deleted bar.py +Initial Sync Complete +Uploaded bar.py + +>>> remote_tree +NOTEBOOK bar PYTHON + +>>> snapshot_files +bar.py + +=== Delete the notebook + +>>> [CLI] sync src /Users/[USERNAME]/sync-[UNIQUE_NAME] +Deleted bar +Initial Sync Complete + +>>> remote_tree + +>>> snapshot_files diff --git a/acceptance/cmd/sync-incremental/notebook-transitions/script b/acceptance/cmd/sync-incremental/notebook-transitions/script new file mode 100644 index 00000000000..6d2ee17b43b --- /dev/null +++ b/acceptance/cmd/sync-incremental/notebook-transitions/script @@ -0,0 +1,34 @@ +title "Python notebook is uploaded without its extension\n" +echo "# Databricks notebook source" > src/foo.py +sync_once +trace remote_tree +trace snapshot_files + +title "Turn the notebook into a plain Python file\n" +echo "# No longer a python notebook" > src/foo.py +sync_once +trace remote_tree +trace snapshot_files + +title "Delete the plain Python file\n" +rm src/foo.py +sync_once +trace remote_tree +trace snapshot_files + +title "Plain Python file\n" +touch src/bar.py +sync_once +trace remote_tree + +title "Turn the plain Python file into a notebook\n" +echo "# Databricks notebook source" > src/bar.py +sync_once +trace remote_tree +trace snapshot_files + +title "Delete the notebook\n" +rm src/bar.py +sync_once +trace remote_tree +trace snapshot_files diff --git a/acceptance/cmd/sync-incremental/script.prepare b/acceptance/cmd/sync-incremental/script.prepare new file mode 100644 index 00000000000..79f0a29a189 --- /dev/null +++ b/acceptance/cmd/sync-incremental/script.prepare @@ -0,0 +1,39 @@ +# Every test syncs ./src to its own remote directory, so concurrent cloud runs do not collide. +# Each sync_once call is a separate CLI process, so the snapshot in src/.databricks is saved +# and loaded again between steps. +remote_dir="/Users/$CURRENT_USER_NAME/sync-$UNIQUE_NAME" +mkdir -p src + +cleanup_remote_dir() { + $CLI workspace delete --recursive "$remote_dir" &>> LOG.cleanup || true +} +trap cleanup_remote_dir EXIT + +# Runs one sync pass and prints the files it uploaded and deleted. +# The "Action:" line lists files in non-deterministic order, so it is dropped. +sync_once() { + trace $CLI sync src "$remote_dir" "$@" | grep -v '^Action' | sort +} + +# Prints every object under the sync root as " []". +remote_tree() { + _list_remote "$remote_dir" | sort +} + +_list_remote() { + local entries + entries=$($CLI workspace list "$1" -o json) + echo "$entries" | jq -r --arg root "$remote_dir/" '.[] | [.object_type, (.path | ltrimstr($root)), (.language // empty)] | join(" ")' + echo "$entries" | jq -r '.[] | select(.object_type == "DIRECTORY") | .path' | while IFS= read -r dir; do + _list_remote "$dir" + done +} + +# Prints the local paths recorded in the sync snapshot. +snapshot_files() { + jq -r '.last_modified_times | keys[]' src/.databricks/sync-snapshots/*.json +} + +remote_content() { + $CLI workspace export "$remote_dir/$1" +} diff --git a/acceptance/cmd/sync-incremental/test.toml b/acceptance/cmd/sync-incremental/test.toml new file mode 100644 index 00000000000..966682ea7c9 --- /dev/null +++ b/acceptance/cmd/sync-incremental/test.toml @@ -0,0 +1,8 @@ +Cloud = true + +# Each test syncs ./src, which its script creates and edits step by step. +Ignore = ["src"] + +# Stops Git Bash on Windows from rewriting the /Users/... sync target into a +# Windows path (e.g. C:/Program Files/Git/Users/...) before the CLI sees it. +Env.MSYS_NO_PATHCONV = "1" diff --git a/acceptance/cmd/sync-remote-path/out.test.toml b/acceptance/cmd/sync-remote-path/out.test.toml new file mode 100644 index 00000000000..c7c6f58ed6e --- /dev/null +++ b/acceptance/cmd/sync-remote-path/out.test.toml @@ -0,0 +1 @@ +Cloud = true diff --git a/acceptance/cmd/sync-remote-path/output.txt b/acceptance/cmd/sync-remote-path/output.txt new file mode 100644 index 00000000000..b6ad2574e76 --- /dev/null +++ b/acceptance/cmd/sync-remote-path/output.txt @@ -0,0 +1,37 @@ + +=== A repo path must exist + +>>> musterr [CLI] sync src /Repos/[USERNAME]/doesnt-exist-[UNIQUE_NAME] +Error: /Repos/[USERNAME]/doesnt-exist-[UNIQUE_NAME] does not exist; please create it first + +=== So must the repo a nested path points into + +>>> musterr [CLI] sync src /Repos/[USERNAME]/doesnt-exist-[UNIQUE_NAME]/nested/directory +Error: /Repos/[USERNAME]/doesnt-exist-[UNIQUE_NAME] does not exist; please create it first + +=== Syncing into an existing repo works + +>>> [CLI] sync src /Repos/[USERNAME]/sync-repo-[UNIQUE_NAME] +Action: PUT: foo.txt +Initial Sync Complete +Uploaded foo.txt + +=== A missing folder nested under the repo is created + +>>> [CLI] sync src /Repos/[USERNAME]/sync-repo-[UNIQUE_NAME]/nested/directory +Action: PUT: foo.txt +Initial Sync Complete +Uploaded foo.txt + +>>> [CLI] workspace get-status /Repos/[USERNAME]/sync-repo-[UNIQUE_NAME]/nested/directory +DIRECTORY + +=== A missing workspace folder is created + +>>> [CLI] sync src /Users/[USERNAME]/sync-[UNIQUE_NAME] +Action: PUT: foo.txt +Initial Sync Complete +Uploaded foo.txt + +>>> [CLI] workspace get-status /Users/[USERNAME]/sync-[UNIQUE_NAME] +DIRECTORY diff --git a/acceptance/cmd/sync-remote-path/script b/acceptance/cmd/sync-remote-path/script new file mode 100644 index 00000000000..65e1a3b330f --- /dev/null +++ b/acceptance/cmd/sync-remote-path/script @@ -0,0 +1,30 @@ +# Output events arrive in non-deterministic order, so sync output is sorted. +mkdir src +touch src/foo.txt + +repo_path="/Repos/$CURRENT_USER_NAME/sync-repo-$UNIQUE_NAME" +workspace_path="/Users/$CURRENT_USER_NAME/sync-$UNIQUE_NAME" + +cleanup() { + $CLI repos delete "$repo_path" &>> LOG.cleanup || true + $CLI workspace delete --recursive "$workspace_path" &>> LOG.cleanup || true +} +trap cleanup EXIT + +title "A repo path must exist\n" +trace musterr $CLI sync src "/Repos/$CURRENT_USER_NAME/doesnt-exist-$UNIQUE_NAME" + +title "So must the repo a nested path points into\n" +trace musterr $CLI sync src "/Repos/$CURRENT_USER_NAME/doesnt-exist-$UNIQUE_NAME/nested/directory" + +title "Syncing into an existing repo works\n" +$CLI repos create https://github.com/databricks/databricks-empty-ide-project.git gitHub --path "$repo_path" > /dev/null +trace $CLI sync src "$repo_path" | sort + +title "A missing folder nested under the repo is created\n" +trace $CLI sync src "$repo_path/nested/directory" | sort +trace $CLI workspace get-status "$repo_path/nested/directory" | jq -r .object_type + +title "A missing workspace folder is created\n" +trace $CLI sync src "$workspace_path" | sort +trace $CLI workspace get-status "$workspace_path" | jq -r .object_type diff --git a/acceptance/cmd/sync-remote-path/test.toml b/acceptance/cmd/sync-remote-path/test.toml new file mode 100644 index 00000000000..b79cf28cd79 --- /dev/null +++ b/acceptance/cmd/sync-remote-path/test.toml @@ -0,0 +1,7 @@ +Cloud = true + +Ignore = ["src"] + +# Stops Git Bash on Windows from rewriting /Users/... and /Repos/... arguments +# into Windows paths (e.g. C:/Program Files/Git/Users/...) before the CLI sees them. +Env.MSYS_NO_PATHCONV = "1" diff --git a/integration/cmd/sync/sync_test.go b/integration/cmd/sync/sync_test.go index b58d00ca2d0..f7cfd2e7d31 100644 --- a/integration/cmd/sync/sync_test.go +++ b/integration/cmd/sync/sync_test.go @@ -3,12 +3,9 @@ package sync_test import ( "context" "encoding/json" - "fmt" "io" - "io/fs" "net/http" "os" - "os/exec" "path" "path/filepath" "strings" @@ -17,58 +14,21 @@ import ( "github.com/databricks/cli/integration/internal/acc" "github.com/databricks/cli/internal/testcli" - "github.com/databricks/cli/internal/testutil" - "github.com/databricks/cli/libs/filer" "github.com/databricks/cli/libs/sync" "github.com/databricks/cli/libs/testfile" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/client" - "github.com/databricks/databricks-sdk-go/service/iam" "github.com/databricks/databricks-sdk-go/service/workspace" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -var ( - repoUrl = "https://github.com/databricks/databricks-empty-ide-project.git" - repoFiles = []string{} -) - -// This test needs auth env vars to run. -// Please run using the deco env test or deco env shell -func setupRepo(t *testing.T, wsc *databricks.WorkspaceClient, ctx context.Context) (localRoot, remoteRoot string) { - me, err := wsc.CurrentUser.Me(ctx, iam.MeRequest{}) - require.NoError(t, err) - repoPath := fmt.Sprintf("/Repos/%s/%s", me.UserName, testutil.RandomName("empty-repo-sync-integration-")) - - repoInfo, err := wsc.Repos.Create(ctx, workspace.CreateRepoRequest{ - Path: repoPath, - Url: repoUrl, - Provider: "gitHub", - }) - require.NoError(t, err) - - t.Cleanup(func() { - err := wsc.Repos.DeleteByRepoId(context.WithoutCancel(ctx), repoInfo.Id) - assert.NoError(t, err) - }) - - tempDir := t.TempDir() - localRoot = filepath.Join(tempDir, "empty-repo") - remoteRoot = repoPath - - // clone public empty remote repo - cmd := exec.Command("git", "clone", repoUrl, localRoot) - err = cmd.Run() - require.NoError(t, err) - return localRoot, remoteRoot -} +var repoFiles = []string{} type syncTest struct { t *testing.T c *testcli.Runner w *databricks.WorkspaceClient - f filer.Filer localRoot string remoteRoot string } @@ -79,9 +39,6 @@ func setupSyncTest(t *testing.T, args ...string) (context.Context, *syncTest) { localRoot := t.TempDir() remoteRoot := acc.TemporaryWorkspaceDir(wt, "sync-") - f, err := filer.NewWorkspaceFilesClient(w, remoteRoot) - require.NoError(t, err) - // Prepend common arguments. args = append([]string{ "sync", @@ -98,7 +55,6 @@ func setupSyncTest(t *testing.T, args ...string) (context.Context, *syncTest) { t: t, c: c, w: w, - f: f, localRoot: localRoot, remoteRoot: remoteRoot, } @@ -166,45 +122,6 @@ func (a *syncTest) remoteFileContent(ctx context.Context, relativePath, expected }, 30*time.Second, 5*time.Second) } -func (a *syncTest) remoteNotExist(ctx context.Context, relativePath string) { - _, err := a.f.Stat(ctx, relativePath) - require.ErrorIs(a.t, err, fs.ErrNotExist) -} - -func (a *syncTest) remoteExists(ctx context.Context, relativePath string) { - _, err := a.f.Stat(ctx, relativePath) - require.NoError(a.t, err) -} - -func (a *syncTest) touchFile(ctx context.Context, path string) { - err := a.f.Write(ctx, path, strings.NewReader("contents"), filer.CreateParentDirectories) - require.NoError(a.t, err) -} - -func (a *syncTest) objectType(ctx context.Context, relativePath, expected string) { - path := path.Join(a.remoteRoot, relativePath) - - a.c.Eventually(func() bool { - metadata, err := a.w.Workspace.GetStatusByPath(ctx, path) - if err != nil { - return false - } - return metadata.ObjectType.String() == expected - }, 30*time.Second, 5*time.Second) -} - -func (a *syncTest) language(ctx context.Context, relativePath, expected string) { - path := path.Join(a.remoteRoot, relativePath) - - a.c.Eventually(func() bool { - metadata, err := a.w.Workspace.GetStatusByPath(ctx, path) - if err != nil { - return false - } - return metadata.Language.String() == expected - }, 30*time.Second, 5*time.Second) -} - func (a *syncTest) snapshotContains(files []string) { snapshotPath := filepath.Join(a.localRoot, ".databricks/sync-snapshots", sync.GetFileName(a.w.Config.Host, a.remoteRoot)) assert.FileExists(a.t, snapshotPath) @@ -228,36 +145,8 @@ func (a *syncTest) snapshotContains(files []string) { assert.Len(a.t, s.LastModifiedTimes, len(files), "files=%s s.LastModifiedTimes=%s", files, s.LastModifiedTimes) } -func TestSyncFullFileSync(t *testing.T) { - ctx, assertSync := setupSyncTest(t, "--full", "--watch") - - assertSync.waitForCompletionMarker() - assertSync.remoteDirContent(ctx, "", repoFiles) - - // New file - localFilePath := filepath.Join(assertSync.localRoot, "foo.txt") - f := testfile.CreateFile(t, localFilePath) - defer f.Close(t) - assertSync.waitForCompletionMarker() - assertSync.remoteDirContent(ctx, "", append(repoFiles, "foo.txt")) - assertSync.remoteFileContent(ctx, "foo.txt", "") - - // Write to file - f.Overwrite(t, `{"statement": "Mi Gente"}`) - assertSync.waitForCompletionMarker() - assertSync.remoteFileContent(ctx, "foo.txt", `{"statement": "Mi Gente"}`) - - // Write again - f.Overwrite(t, `{"statement": "Young Dumb & Broke"}`) - assertSync.waitForCompletionMarker() - assertSync.remoteFileContent(ctx, "foo.txt", `{"statement": "Young Dumb & Broke"}`) - - // delete - f.Remove(t) - assertSync.waitForCompletionMarker() - assertSync.remoteDirContent(ctx, "", repoFiles) -} - +// Other sync scenarios are acceptance tests under acceptance/cmd/sync-incremental, each step a +// separate CLI run. This test stays here to cover --watch, which runs as one long-lived process. func TestSyncIncrementalFileSync(t *testing.T) { ctx, assertSync := setupSyncTest(t, "--watch") @@ -289,261 +178,3 @@ func TestSyncIncrementalFileSync(t *testing.T) { assertSync.remoteDirContent(ctx, "", repoFiles) assertSync.snapshotContains(repoFiles) } - -func TestSyncNestedFolderSync(t *testing.T) { - ctx, assertSync := setupSyncTest(t, "--watch") - - assertSync.waitForCompletionMarker() - assertSync.remoteDirContent(ctx, "", repoFiles) - - // New file - localFilePath := filepath.Join(assertSync.localRoot, "dir1/dir2/dir3/foo.txt") - err := os.MkdirAll(filepath.Dir(localFilePath), 0o755) - assert.NoError(t, err) - f := testfile.CreateFile(t, localFilePath) - defer f.Close(t) - assertSync.waitForCompletionMarker() - assertSync.remoteDirContent(ctx, "", append(repoFiles, "dir1")) - assertSync.remoteDirContent(ctx, "dir1", []string{"dir2"}) - assertSync.remoteDirContent(ctx, "dir1/dir2", []string{"dir3"}) - assertSync.remoteDirContent(ctx, "dir1/dir2/dir3", []string{"foo.txt"}) - assertSync.snapshotContains(append(repoFiles, "dir1/dir2/dir3/foo.txt")) - - // delete - f.Remove(t) - assertSync.waitForCompletionMarker() - assertSync.remoteNotExist(ctx, "dir1") - assertSync.snapshotContains(repoFiles) -} - -func TestSyncNestedFolderDoesntFailOnNonEmptyDirectory(t *testing.T) { - ctx, assertSync := setupSyncTest(t, "--watch") - - assertSync.waitForCompletionMarker() - assertSync.remoteDirContent(ctx, "", repoFiles) - - // New file - localFilePath := filepath.Join(assertSync.localRoot, "dir1/dir2/dir3/foo.txt") - err := os.MkdirAll(filepath.Dir(localFilePath), 0o755) - assert.NoError(t, err) - f := testfile.CreateFile(t, localFilePath) - defer f.Close(t) - assertSync.waitForCompletionMarker() - assertSync.remoteDirContent(ctx, "dir1/dir2/dir3", []string{"foo.txt"}) - - // Add file to dir1 to simulate a user writing to the workspace directly. - assertSync.touchFile(ctx, "dir1/foo.txt") - - // Remove original file. - f.Remove(t) - assertSync.waitForCompletionMarker() - - // Sync should have removed these directories. - assertSync.remoteNotExist(ctx, "dir1/dir2/dir3") - assertSync.remoteNotExist(ctx, "dir1/dir2") - - // Sync should have ignored not being able to delete dir1. - assertSync.remoteExists(ctx, "dir1/foo.txt") - assertSync.remoteExists(ctx, "dir1") -} - -func TestSyncNestedSpacePlusAndHashAreEscapedSync(t *testing.T) { - ctx, assertSync := setupSyncTest(t, "--watch") - - assertSync.waitForCompletionMarker() - assertSync.remoteDirContent(ctx, "", repoFiles) - - // New file - localFilePath := filepath.Join(assertSync.localRoot, "dir1/a b+c/c+d e/e+f g#i.txt") - err := os.MkdirAll(filepath.Dir(localFilePath), 0o755) - assert.NoError(t, err) - f := testfile.CreateFile(t, localFilePath) - defer f.Close(t) - assertSync.waitForCompletionMarker() - assertSync.remoteDirContent(ctx, "", append(repoFiles, "dir1")) - assertSync.remoteDirContent(ctx, "dir1", []string{"a b+c"}) - assertSync.remoteDirContent(ctx, "dir1/a b+c", []string{"c+d e"}) - assertSync.remoteDirContent(ctx, "dir1/a b+c/c+d e", []string{"e+f g#i.txt"}) - assertSync.snapshotContains(append(repoFiles, "dir1/a b+c/c+d e/e+f g#i.txt")) - - // delete - f.Remove(t) - assertSync.waitForCompletionMarker() - assertSync.remoteNotExist(ctx, "dir1/a b+c/c+d e") - assertSync.snapshotContains(repoFiles) -} - -// This is a check for the edge case when a user does the following: -// -// 1. Add file foo/bar.txt -// 2. Delete foo/bar.txt (including the directory) -// 3. Add file foo -// -// In the above scenario sync should delete the empty folder and add foo to the remote -// file system -func TestSyncIncrementalFileOverwritesFolder(t *testing.T) { - ctx, assertSync := setupSyncTest(t, "--watch") - - // create foo/bar.txt - localFilePath := filepath.Join(assertSync.localRoot, "foo/bar.txt") - err := os.MkdirAll(filepath.Dir(localFilePath), 0o755) - assert.NoError(t, err) - f := testfile.CreateFile(t, localFilePath) - defer f.Close(t) - assertSync.waitForCompletionMarker() - assertSync.remoteDirContent(ctx, "", append(repoFiles, "foo")) - assertSync.remoteDirContent(ctx, "foo", []string{"bar.txt"}) - assertSync.snapshotContains(append(repoFiles, "foo/bar.txt")) - - // delete foo/bar.txt - f.Remove(t) - os.Remove(filepath.Join(assertSync.localRoot, "foo")) - assertSync.waitForCompletionMarker() - assertSync.remoteNotExist(ctx, "foo") - assertSync.snapshotContains(repoFiles) - - f2 := testfile.CreateFile(t, filepath.Join(assertSync.localRoot, "foo")) - defer f2.Close(t) - assertSync.waitForCompletionMarker() - assertSync.remoteDirContent(ctx, "", append(repoFiles, "foo")) - assertSync.objectType(ctx, "foo", "FILE") - assertSync.snapshotContains(append(repoFiles, "foo")) -} - -func TestSyncIncrementalSyncPythonNotebookToFile(t *testing.T) { - ctx, assertSync := setupSyncTest(t, "--watch") - - // create python notebook - localFilePath := filepath.Join(assertSync.localRoot, "foo.py") - f := testfile.CreateFile(t, localFilePath) - defer f.Close(t) - f.Overwrite(t, "# Databricks notebook source") - - // notebook was uploaded properly - assertSync.waitForCompletionMarker() - assertSync.remoteDirContent(ctx, "", append(repoFiles, "foo")) - assertSync.objectType(ctx, "foo", "NOTEBOOK") - assertSync.language(ctx, "foo", "PYTHON") - assertSync.snapshotContains(append(repoFiles, "foo.py")) - - // convert to vanilla python file - f.Overwrite(t, "# No longer a python notebook") - assertSync.waitForCompletionMarker() - assertSync.objectType(ctx, "foo.py", "FILE") - assertSync.remoteDirContent(ctx, "", append(repoFiles, "foo.py")) - assertSync.snapshotContains(append(repoFiles, "foo.py")) - - // delete the vanilla python file - f.Remove(t) - assertSync.waitForCompletionMarker() - assertSync.remoteDirContent(ctx, "", repoFiles) - assertSync.snapshotContains(repoFiles) -} - -func TestSyncIncrementalSyncFileToPythonNotebook(t *testing.T) { - ctx, assertSync := setupSyncTest(t, "--watch") - - // create vanilla python file - localFilePath := filepath.Join(assertSync.localRoot, "foo.py") - f := testfile.CreateFile(t, localFilePath) - defer f.Close(t) - assertSync.waitForCompletionMarker() - - // assert file upload - assertSync.remoteDirContent(ctx, "", append(repoFiles, "foo.py")) - assertSync.objectType(ctx, "foo.py", "FILE") - assertSync.snapshotContains(append(repoFiles, "foo.py")) - - // convert to notebook - f.Overwrite(t, "# Databricks notebook source") - assertSync.waitForCompletionMarker() - assertSync.objectType(ctx, "foo", "NOTEBOOK") - assertSync.language(ctx, "foo", "PYTHON") - assertSync.remoteDirContent(ctx, "", append(repoFiles, "foo")) - assertSync.snapshotContains(append(repoFiles, "foo.py")) -} - -func TestSyncIncrementalSyncPythonNotebookDelete(t *testing.T) { - ctx, assertSync := setupSyncTest(t, "--watch") - - // create python notebook - localFilePath := filepath.Join(assertSync.localRoot, "foo.py") - f := testfile.CreateFile(t, localFilePath) - defer f.Close(t) - f.Overwrite(t, "# Databricks notebook source") - assertSync.waitForCompletionMarker() - - // notebook was uploaded properly - assertSync.remoteDirContent(ctx, "", append(repoFiles, "foo")) - assertSync.objectType(ctx, "foo", "NOTEBOOK") - assertSync.language(ctx, "foo", "PYTHON") - - // Delete notebook - f.Remove(t) - assertSync.waitForCompletionMarker() - assertSync.remoteDirContent(ctx, "", repoFiles) -} - -func TestSyncEnsureRemotePathIsUsableIfRepoDoesntExist(t *testing.T) { - ctx, wt := acc.WorkspaceTest(t) - wsc := wt.W - - me, err := wsc.CurrentUser.Me(ctx, iam.MeRequest{}) - require.NoError(t, err) - - // Hypothetical repo path doesn't exist. - nonExistingRepoPath := fmt.Sprintf("/Repos/%s/%s", me.UserName, testutil.RandomName("doesnt-exist-")) - err = sync.EnsureRemotePathIsUsable(ctx, wsc, nonExistingRepoPath, nil, false) - assert.ErrorContains(t, err, " does not exist; please create it first") - - // Paths nested under a hypothetical repo path should yield the same error. - nestedPath := path.Join(nonExistingRepoPath, "nested/directory") - err = sync.EnsureRemotePathIsUsable(ctx, wsc, nestedPath, nil, false) - assert.ErrorContains(t, err, " does not exist; please create it first") -} - -func TestSyncEnsureRemotePathIsUsableIfRepoExists(t *testing.T) { - ctx, wt := acc.WorkspaceTest(t) - wsc := wt.W - - _, remoteRepoPath := setupRepo(t, wsc, ctx) - - // Repo itself is usable. - err := sync.EnsureRemotePathIsUsable(ctx, wsc, remoteRepoPath, nil, false) - assert.NoError(t, err) - - // Path nested under repo path is usable. - nestedPath := path.Join(remoteRepoPath, "nested/directory") - err = sync.EnsureRemotePathIsUsable(ctx, wsc, nestedPath, nil, false) - assert.NoError(t, err) - - // Verify that the directory has been created. - info, err := wsc.Workspace.GetStatusByPath(ctx, nestedPath) - require.NoError(t, err) - require.Equal(t, workspace.ObjectTypeDirectory, info.ObjectType) -} - -func TestSyncEnsureRemotePathIsUsableInWorkspace(t *testing.T) { - ctx, wt := acc.WorkspaceTest(t) - wsc := wt.W - - me, err := wsc.CurrentUser.Me(ctx, iam.MeRequest{}) - require.NoError(t, err) - - remotePath := fmt.Sprintf("/Users/%s/%s", me.UserName, testutil.RandomName("ensure-path-exists-test-")) - err = sync.EnsureRemotePathIsUsable(ctx, wsc, remotePath, me, false) - assert.NoError(t, err) - - // Clean up directory after test. - defer func() { - err := wsc.Workspace.Delete(ctx, workspace.Delete{ - Path: remotePath, - }) - assert.NoError(t, err) - }() - - // Verify that the directory has been created. - info, err := wsc.Workspace.GetStatusByPath(ctx, remotePath) - require.NoError(t, err) - require.Equal(t, workspace.ObjectTypeDirectory, info.ObjectType) -}