From 0d4f9e8bacfacac7a1427991bf21566e9844d456 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 14 Oct 2026 14:52:47 +0000 Subject: [PATCH 1/2] Skip redundant DMS resources.json uploads --- acceptance/bundle/dms/empty-grants/output.txt | 2 +- .../bundle/dms/existing-state/output.txt | 8 ++ acceptance/bundle/dms/existing-state/script | 15 ++++ .../dms/marker-writes/databricks.yml.tmpl | 14 ++++ .../bundle/dms/marker-writes/out.test.toml | 2 + .../bundle/dms/marker-writes/output.txt | 38 +++++++++ acceptance/bundle/dms/marker-writes/script | 33 ++++++++ acceptance/bundle/dms/no-drift/output.txt | 2 +- .../bundle/dms/stale-local-state/output.txt | 16 ++-- .../bundle/dms/stale-local-state/script | 17 ++-- .../bundle/dms/successful-recreate/output.txt | 2 +- bundle/bundle.go | 3 + bundle/phases/deploy.go | 1 + bundle/statemgmt/state_push.go | 9 +++ bundle/statemgmt/state_push_test.go | 80 +++++++++++++++++++ cmd/bundle/utils/process.go | 1 + cmd/bundle/utils/process_test.go | 4 + 17 files changed, 228 insertions(+), 19 deletions(-) create mode 100644 acceptance/bundle/dms/marker-writes/databricks.yml.tmpl create mode 100644 acceptance/bundle/dms/marker-writes/out.test.toml create mode 100644 acceptance/bundle/dms/marker-writes/output.txt create mode 100644 acceptance/bundle/dms/marker-writes/script create mode 100644 bundle/statemgmt/state_push_test.go diff --git a/acceptance/bundle/dms/empty-grants/output.txt b/acceptance/bundle/dms/empty-grants/output.txt index 0a6cbabcbce..a4928d401e5 100644 --- a/acceptance/bundle/dms/empty-grants/output.txt +++ b/acceptance/bundle/dms/empty-grants/output.txt @@ -161,7 +161,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged "state_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", - "serial": 2, + "serial": 1, "features": { "deployment_history": {} }, diff --git a/acceptance/bundle/dms/existing-state/output.txt b/acceptance/bundle/dms/existing-state/output.txt index 0001ca7a7b5..c6cc02ac929 100644 --- a/acceptance/bundle/dms/existing-state/output.txt +++ b/acceptance/bundle/dms/existing-state/output.txt @@ -16,6 +16,10 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "features": null } +=== Direct deployment still writes resources.json once +>>> print_requests.py --method POST //workspace/import //workspace-files/import-file +1 + === Config enables recording despite the disabled environment: register version 2, then deploy version 3 >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/files... @@ -36,6 +40,10 @@ Resources: 0 created, 1 changed, 0 deleted, 1 unchanged "state": {} } +=== Migration writes the marker once; the standalone deployment does not rewrite it +>>> print_requests.py --method POST //workspace/import //workspace-files/import-file +1 + === Inspect the remote backup's original pre-migration resource IDs, state, and dependencies >>> MSYS_NO_PATHCONV=1 [CLI] workspace export /Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/state/resources.json.backup --format RAW --file .databricks/resources.backup.first.json diff --git a/acceptance/bundle/dms/existing-state/script b/acceptance/bundle/dms/existing-state/script index dec0741aafc..cc27af4ffd7 100644 --- a/acceptance/bundle/dms/existing-state/script +++ b/acceptance/bundle/dms/existing-state/script @@ -1,4 +1,5 @@ envsubst < databricks.yml.tmpl > databricks.yml +state_path="/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-existing-state-${UNIQUE_NAME}/default/state" title "Config disables recording even when the environment enables it" trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true $CLI bundle deploy 2>&1 | contains.py '!Current Deployment Version:' '!Enabling deployment history' @@ -6,6 +7,13 @@ replace_ids.py trace cp .databricks/bundle/default/resources.json .databricks/resources.before-dms.json trace jq '{state_version, serial, features}' .databricks/bundle/default/resources.json +title "Direct deployment still writes resources.json once" +trace print_requests.py --method POST //workspace/import //workspace-files/import-file | + MSYS_NO_PATHCONV=1 jq -s --arg path "$state_path/resources.json" ' + map(select((.path == "/api/2.0/workspace/import" and (.body.multipart_form.path // .body.path) == $path) + or .path == ("/api/2.0/workspace-files/import-file" + $path))) | length | + if . == 1 then . else error("expected one direct state write") end' + title "Config enables recording despite the disabled environment: register version 2, then deploy version 3" export DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=false update_file.py databricks.yml 'deployment_history: false' 'deployment_history: true' @@ -14,6 +22,13 @@ trace $CLI bundle deploy 2>&1 | contains.py 'Enabling deployment history...' 'De trace find.py 'resources[.]json[.]wal' --expect 0 trace jq '{features, state}' .databricks/bundle/default/resources.json +title "Migration writes the marker once; the standalone deployment does not rewrite it" +trace print_requests.py --method POST //workspace/import //workspace-files/import-file | + MSYS_NO_PATHCONV=1 jq -s --arg path "$state_path/resources.json" ' + map(select((.path == "/api/2.0/workspace/import" and (.body.multipart_form.path // .body.path) == $path) + or .path == ("/api/2.0/workspace-files/import-file" + $path))) | length | + if . == 1 then . else error("expected one migration marker write") end' + title "Inspect the remote backup's original pre-migration resource IDs, state, and dependencies" trace MSYS_NO_PATHCONV=1 $CLI workspace export "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-existing-state-${UNIQUE_NAME}/default/state/resources.json.backup" --format RAW --file .databricks/resources.backup.first.json trace cat .databricks/resources.backup.first.json diff --git a/acceptance/bundle/dms/marker-writes/databricks.yml.tmpl b/acceptance/bundle/dms/marker-writes/databricks.yml.tmpl new file mode 100644 index 00000000000..bb7c81facf0 --- /dev/null +++ b/acceptance/bundle/dms/marker-writes/databricks.yml.tmpl @@ -0,0 +1,14 @@ +bundle: + name: dms-marker-writes-$UNIQUE_NAME + +experimental: + deployment_history: true + +sync: + exclude: + - LOG.* + +resources: + jobs: + foo: + name: foo diff --git a/acceptance/bundle/dms/marker-writes/out.test.toml b/acceptance/bundle/dms/marker-writes/out.test.toml new file mode 100644 index 00000000000..84b22ba6735 --- /dev/null +++ b/acceptance/bundle/dms/marker-writes/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/marker-writes/output.txt b/acceptance/bundle/dms/marker-writes/output.txt new file mode 100644 index 00000000000..cf32c690b0d --- /dev/null +++ b/acceptance/bundle/dms/marker-writes/output.txt @@ -0,0 +1,38 @@ + +=== 1. First recorded deploy: one remote marker write, version 1 +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-marker-writes-[UNIQUE_NAME]/default/files... +Created jobs.foo +Files: 2 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> resources_json_writes 1 +resources.json writes=1 + +>>> [CLI] bundle summary -o json +{"job_id":"[FOO_ID]","version":1} + +=== 2. Real job update: zero marker writes, same ID, version 2 +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-marker-writes-[UNIQUE_NAME]/default/files... +Updated jobs.foo +Files: 1 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 0 unchanged + +>>> resources_json_writes 0 +resources.json writes=0 + +>>> [CLI] bundle summary -o json +{"job_id":"[FOO_ID]","version":2} + +=== 3. No-op deploy: zero marker writes, still version 2 +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-marker-writes-[UNIQUE_NAME]/default/files... +Files: 0 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 1 unchanged + +>>> resources_json_writes 0 +resources.json writes=0 + +>>> [CLI] bundle summary -o json +{"job_id":"[FOO_ID]","version":2} diff --git a/acceptance/bundle/dms/marker-writes/script b/acceptance/bundle/dms/marker-writes/script new file mode 100644 index 00000000000..cd83ebef8bb --- /dev/null +++ b/acceptance/bundle/dms/marker-writes/script @@ -0,0 +1,33 @@ +envsubst < databricks.yml.tmpl > databricks.yml +state_path="/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-marker-writes-${UNIQUE_NAME}/default/state" + +# Import's target lives in its body; match the exact state path, not metadata or backup files. +resources_json_writes() { + print_requests.py --method POST //workspace/import //workspace-files/import-file | + MSYS_NO_PATHCONV=1 jq -sr --arg path "$state_path/resources.json" --argjson expected "$1" ' + map(select( + (.path == "/api/2.0/workspace/import" and (.body.multipart_form.path // .body.path) == $path) + or .path == ("/api/2.0/workspace-files/import-file" + $path) + )) | length | if . == $expected then "resources.json writes=\(.)" + else error("unexpected resources.json write count: \(.)") end' +} + +title "1. First recorded deploy: one remote marker write, version 1" +trace $CLI bundle deploy +replace_ids.py +trace resources_json_writes 1 +trace $CLI bundle summary -o json | jq -c '{job_id: .resources.jobs.foo.id, version: .bundle.deployment.latest_version_id}' + +title "2. Real job update: zero marker writes, same ID, version 2" +update_file.py databricks.yml 'name: foo' 'name: updated' +trace $CLI bundle deploy 2>&1 | contains.py 'Updated jobs.foo' '!Created ' +trace resources_json_writes 0 +trace $CLI bundle summary -o json | jq -c '{job_id: .resources.jobs.foo.id, version: .bundle.deployment.latest_version_id}' + +title "3. No-op deploy: zero marker writes, still version 2" +trace $CLI bundle deploy 2>&1 | contains.py 'Resources: 0 created, 0 changed, 0 deleted, 1 unchanged' +trace resources_json_writes 0 +trace $CLI bundle summary -o json | jq -c '{job_id: .resources.jobs.foo.id, version: .bundle.deployment.latest_version_id}' + +$CLI bundle destroy --auto-approve &> LOG.cleanup +rm -f "$OUT_REQUESTS" diff --git a/acceptance/bundle/dms/no-drift/output.txt b/acceptance/bundle/dms/no-drift/output.txt index c2fe06a747e..eec5ae39653 100644 --- a/acceptance/bundle/dms/no-drift/output.txt +++ b/acceptance/bundle/dms/no-drift/output.txt @@ -198,7 +198,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged "state_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", - "serial": 2, + "serial": 1, "features": { "deployment_history": {} }, diff --git a/acceptance/bundle/dms/stale-local-state/output.txt b/acceptance/bundle/dms/stale-local-state/output.txt index 32b359dbf87..12e400fc733 100644 --- a/acceptance/bundle/dms/stale-local-state/output.txt +++ b/acceptance/bundle/dms/stale-local-state/output.txt @@ -1,19 +1,19 @@ -=== Setup (LOG.setup): reconstruct full version 1 without a history marker, then deploy version 2 +=== Setup (LOG.setup): reconstruct version 1, then advance DMS to version 2 without rewriting its checkpoint >>> jq -c {serial, features} .databricks/resources.v1.json {"serial":1,"features":null} ->>> jq -c {serial, features} .databricks/resources.v2.json -{"serial":2,"features":{"deployment_history":{}}} +>>> jq -c {serial, features} .databricks/resources.checkpoint.json +{"serial":1,"features":{"deployment_history":{}}} -=== 1. Config false and env false: newer remote state replaces the local copy and reads DMS +=== 1. Config false and env false: remote DMS checkpoint replaces the direct snapshot despite equal serials >>> cp .databricks/resources.v1.json .databricks/bundle/default/resources.json >>> DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=false [CLI] bundle summary --force-pull -o json Warn: Deployment history setting (false) does not match the existing state (true). Using the existing state. -{"job_id":"[FOO_ID]"} +{"job_id":"[FOO_ID]","version":2} ->>> cmp .databricks/resources.v2.json .databricks/bundle/default/resources.json +>>> cmp .databricks/resources.checkpoint.json .databricks/bundle/default/resources.json >>> print_requests.py --dms //resources --get --oneline {"method": "GET", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/resources"} @@ -23,9 +23,9 @@ Warn: Deployment history setting (false) does not match the existing state (true >>> DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=false [CLI] bundle summary --force-pull -o json Warn: Deployment history setting (false) does not match the existing state (true). Using the existing state. -{"job_id":"[FOO_ID]"} +{"job_id":"[FOO_ID]","version":2} ->>> cmp .databricks/resources.v2.json .databricks/bundle/default/resources.json +>>> cmp .databricks/resources.checkpoint.json .databricks/bundle/default/resources.json >>> print_requests.py --dms //resources --get --oneline {"method": "GET", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/resources"} diff --git a/acceptance/bundle/dms/stale-local-state/script b/acceptance/bundle/dms/stale-local-state/script index ee16e686128..4dcb8dc9e94 100644 --- a/acceptance/bundle/dms/stale-local-state/script +++ b/acceptance/bundle/dms/stale-local-state/script @@ -1,32 +1,33 @@ envsubst < databricks.yml.tmpl > databricks.yml +state_path="/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-stale-local-state-${UNIQUE_NAME}/default/state" -title "Setup (LOG.setup): reconstruct full version 1 without a history marker, then deploy version 2" +title "Setup (LOG.setup): reconstruct version 1, then advance DMS to version 2 without rewriting its checkpoint" { $CLI bundle deploy DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true print_state.py > .databricks/resources.v1.json DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true replace_ids.py update_file.py databricks.yml 'name: foo' 'name: updated' $CLI bundle deploy - cp .databricks/bundle/default/resources.json .databricks/resources.v2.json + MSYS_NO_PATHCONV=1 $CLI workspace export "$state_path/resources.json" --format RAW --file .databricks/resources.checkpoint.json deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-stale-local-state-${UNIQUE_NAME}/default/state/resources.deployment.json" -o json | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') add_repl "$deployment_id" DEPLOYMENT_ID } &> LOG.setup trace jq -c '{serial, features}' .databricks/resources.v1.json -trace jq -c '{serial, features}' .databricks/resources.v2.json +trace jq -c '{serial, features}' .databricks/resources.checkpoint.json rm -f "$OUT_REQUESTS" -title "1. Config false and env false: newer remote state replaces the local copy and reads DMS" +title "1. Config false and env false: remote DMS checkpoint replaces the direct snapshot despite equal serials" update_file.py databricks.yml 'deployment_history: true' 'deployment_history: false' trace cp .databricks/resources.v1.json .databricks/bundle/default/resources.json -trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=false $CLI bundle summary --force-pull -o json | jq -c '{job_id: .resources.jobs.foo.id}' -trace cmp .databricks/resources.v2.json .databricks/bundle/default/resources.json +trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=false $CLI bundle summary --force-pull -o json | jq -c '{job_id: .resources.jobs.foo.id, version: .bundle.deployment.latest_version_id}' +trace cmp .databricks/resources.checkpoint.json .databricks/bundle/default/resources.json trace print_requests.py --dms //resources --get --oneline title "2. Config removed and env false: restore the same snapshot, then read remote DMS state" update_file.py databricks.yml $'experimental:\n deployment_history: false\n\n' '' trace cp .databricks/resources.v1.json .databricks/bundle/default/resources.json -trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=false $CLI bundle summary --force-pull -o json | jq -c '{job_id: .resources.jobs.foo.id}' -trace cmp .databricks/resources.v2.json .databricks/bundle/default/resources.json +trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=false $CLI bundle summary --force-pull -o json | jq -c '{job_id: .resources.jobs.foo.id, version: .bundle.deployment.latest_version_id}' +trace cmp .databricks/resources.checkpoint.json .databricks/bundle/default/resources.json trace print_requests.py --dms //resources --get --oneline DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true $CLI bundle destroy --auto-approve &> LOG.cleanup diff --git a/acceptance/bundle/dms/successful-recreate/output.txt b/acceptance/bundle/dms/successful-recreate/output.txt index f2ecc3f8cbc..2f67fd19da9 100644 --- a/acceptance/bundle/dms/successful-recreate/output.txt +++ b/acceptance/bundle/dms/successful-recreate/output.txt @@ -193,7 +193,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged "state_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", - "serial": 2, + "serial": 1, "features": { "deployment_history": {} }, diff --git a/bundle/bundle.go b/bundle/bundle.go index a22099cd236..aac79e89ebc 100644 --- a/bundle/bundle.go +++ b/bundle/bundle.go @@ -197,6 +197,9 @@ type Bundle struct { // MigratingToDMS registers the existing direct state before deploying with history enabled. MigratingToDMS bool + // NeedsDMSStateUpload creates or repairs the remote deployment-history marker. + NeedsDMSStateUpload bool + // AllowTerraformEngineConfig reports a terraform bundle.engine as a warning instead // of an error and skips the DATABRICKS_BUNDLE_ENGINE check. Set by read-only commands // (bundle summary) so users can still inspect a bundle that pins the removed engine. diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index e791b26b393..3f0f2a4e2c9 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -310,6 +310,7 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand return } logDeploymentVersion(ctx, b, "Deployment history enabled") + b.NeedsDMSStateUpload = true statemgmt.PushResourcesState(ctx, b) if logdiag.HasError(ctx) { return diff --git a/bundle/statemgmt/state_push.go b/bundle/statemgmt/state_push.go index 9df720f6157..79f25039a78 100644 --- a/bundle/statemgmt/state_push.go +++ b/bundle/statemgmt/state_push.go @@ -16,6 +16,11 @@ import ( // PushResourcesState uploads the local state file to the remote location. func PushResourcesState(ctx context.Context, b *bundle.Bundle) { + recorded := b.DeploymentBundle.StateDB.IsDeploymentMetadataService() + if recorded && !b.NeedsDMSStateUpload { + return + } + f, err := deploy.StateFiler(ctx, b) if err != nil { logdiag.LogError(ctx, err) @@ -41,6 +46,10 @@ func PushResourcesState(ctx context.Context, b *bundle.Bundle) { err = f.Write(ctx, remotePath, local, filer.CreateParentDirectories, filer.OverwriteIfExists) if err != nil { logdiag.LogError(ctx, err) + return + } + if recorded { + b.NeedsDMSStateUpload = false } } diff --git a/bundle/statemgmt/state_push_test.go b/bundle/statemgmt/state_push_test.go new file mode 100644 index 00000000000..a4d5e209e20 --- /dev/null +++ b/bundle/statemgmt/state_push_test.go @@ -0,0 +1,80 @@ +package statemgmt_test + +import ( + "os" + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/direct/dstate" + "github.com/databricks/cli/bundle/statemgmt" + "github.com/databricks/cli/libs/logdiag" + "github.com/databricks/cli/libs/testserver" + "github.com/databricks/databricks-sdk-go" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestPushResourcesStateMarkerWrites(t *testing.T) { + const recordedMarker = `{"state_version":3,"cli_version":"0.1.0","lineage":"recorded-lineage","serial":2,"features":{"deployment_history":{}},"state":{}}` + const directSnapshot = `{"state_version":3,"lineage":"direct-lineage","serial":99,"state":{"resources.jobs.foo":{"__id__":"456","state":{"name":"foo"}}}}` + tests := []struct { + name string + recorded bool + needsWrite bool + missing bool + failUpload bool + wantWrites int + wantFlag bool + }{ + {name: "direct state always uploads", wantWrites: 2}, + {name: "direct upload does not consume migration marker", needsWrite: true, wantWrites: 2, wantFlag: true}, + {name: "recorded state skips ordinary uploads", recorded: true}, + {name: "recorded marker uploads once", recorded: true, needsWrite: true, wantWrites: 1}, + {name: "missing local marker keeps upload pending", recorded: true, needsWrite: true, missing: true, wantFlag: true}, + {name: "failed upload keeps marker pending", recorded: true, needsWrite: true, failUpload: true, wantWrites: 2, wantFlag: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + server := testserver.New(t) + client, err := databricks.NewWorkspaceClient(&databricks.Config{Host: server.URL, Token: "testtoken"}) + require.NoError(t, err) + b := &bundle.Bundle{ + BundleRootPath: t.TempDir(), + Config: config.Root{ + Bundle: config.Bundle{Target: "test"}, + Workspace: config.Workspace{StatePath: "/Workspace/state"}, + }, + } + b.SetWorkpaceClient(client) + ctx := logdiag.InitContext(t.Context()) + logdiag.SetCollect(ctx, true) + _, localPath := b.StateFilenameDirect(ctx) + content := directSnapshot + if tt.recorded { + content = recordedMarker + } + writeLocalState(t, localPath, content) + require.NoError(t, b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(false), dstate.WithWrite(false), dstate.WithDeploymentHistory(tt.recorded), dstate.OpenDmsArgs{})) + _, err = b.DeploymentBundle.StateDB.Finalize(ctx) + require.NoError(t, err) + if tt.missing { + require.NoError(t, os.Remove(localPath)) + } + b.NeedsDMSStateUpload = tt.needsWrite + writes := 0 + server.Handle("POST", "/api/2.0/workspace/import", func(req testserver.Request) any { + writes++ + if tt.failUpload { + return testserver.Response{StatusCode: 403, Body: map[string]string{"error_code": "PERMISSION_DENIED", "message": "access denied"}} + } + return map[string]any{} + }) + statemgmt.PushResourcesState(ctx, b) + statemgmt.PushResourcesState(ctx, b) + assert.Equal(t, tt.wantWrites, writes) + assert.Equal(t, tt.wantFlag, b.NeedsDMSStateUpload) + assert.Equal(t, tt.failUpload, logdiag.HasError(ctx)) + }) + } +} diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index 04989740c69..489f1ad9685 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -675,6 +675,7 @@ func resolveDeploymentHistory(ctx context.Context, b *bundle.Bundle, stateDesc * if stateDesc.SourcePath == "" { if configured { stateDesc.Features = map[string]struct{}{dstate.FeatureDeploymentHistory: {}} + b.NeedsDMSStateUpload = true } return configured } diff --git a/cmd/bundle/utils/process_test.go b/cmd/bundle/utils/process_test.go index b216f1102d2..a5c82c29991 100644 --- a/cmd/bundle/utils/process_test.go +++ b/cmd/bundle/utils/process_test.go @@ -20,12 +20,16 @@ func TestResolveDeploymentHistory(t *testing.T) { newState := &statemgmt.StateDesc{} assert.True(t, resolveDeploymentHistory(t.Context(), b, newState)) assert.True(t, newState.IsDMS()) + assert.True(t, b.NeedsDMSStateUpload) + b.NeedsDMSStateUpload = false assert.False(t, resolveDeploymentHistory(t.Context(), b, &statemgmt.StateDesc{SourcePath: "resources.json"})) + assert.False(t, b.NeedsDMSStateUpload) b.Config.Experimental.DeploymentHistory = new(false) assert.True(t, resolveDeploymentHistory(t.Context(), b, &statemgmt.StateDesc{ SourcePath: "resources.json", Features: map[string]struct{}{dstate.FeatureDeploymentHistory: {}}, })) + assert.False(t, b.NeedsDMSStateUpload) } func TestIsNewerVersion(t *testing.T) { From 46c0cc0c5c6d3e0d67c4f438e4f15f6c60ae40a6 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 14 Oct 2026 15:37:10 +0000 Subject: [PATCH 2/2] Keep wheel upload assertions independent of DMS marker writes --- acceptance/bundle/artifacts/whl_change_version/output.txt | 3 --- acceptance/bundle/artifacts/whl_change_version/test.toml | 6 ++++-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/acceptance/bundle/artifacts/whl_change_version/output.txt b/acceptance/bundle/artifacts/whl_change_version/output.txt index 30134175168..ee120e1fac0 100644 --- a/acceptance/bundle/artifacts/whl_change_version/output.txt +++ b/acceptance/bundle/artifacts/whl_change_version/output.txt @@ -41,7 +41,6 @@ dist/my_test_code-0.1.0-py3-none-any.whl /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/STATE_FILENAME >>> update_file.py my_test_code/__init__.py 0.1.0 0.2.0 @@ -69,7 +68,6 @@ json[0].libraries[0].whl = "/Workspace/Users/[USERNAME]/.bundle/test-bundle/defa /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/STATE_FILENAME === Restore config to target old wheel >>> update_file.py databricks.yml ./dist/*.whl ./dist/my*0.1.0*.whl @@ -100,4 +98,3 @@ json[0].libraries[0].whl = "/Workspace/Users/[USERNAME]/.bundle/test-bundle/defa /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json -/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/STATE_FILENAME diff --git a/acceptance/bundle/artifacts/whl_change_version/test.toml b/acceptance/bundle/artifacts/whl_change_version/test.toml index fddb9ae512f..7ce7f2c14ea 100644 --- a/acceptance/bundle/artifacts/whl_change_version/test.toml +++ b/acceptance/bundle/artifacts/whl_change_version/test.toml @@ -1,3 +1,5 @@ +# This test asserts wheel uploads; DMS checkpoints are not rewritten on later deploys. +# Exact resources.json write counts are covered by bundle/dms/marker-writes. [[Repls]] -Old = '(resources.json|terraform.tfstate)' -New = 'STATE_FILENAME' +Old = '(?m)^/Workspace/Users/[^\r\n]+/\.bundle/test-bundle/default/state/(resources\.json|terraform\.tfstate)\r?\n' +New = ''