Skip redundant DMS resources.json uploads

acceptance/bundle/artifacts/whl_change_version/output.txt

@@ -41,7 +41,6 @@ dist/my_test_code-0.1.0-py3-none-any.whl
 /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock
 /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json
 /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json
-/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/STATE_FILENAME
 
 >>> update_file.py my_test_code/__init__.py 0.1.0 0.2.0
 
@@ -69,7 +68,6 @@ json[0].libraries[0].whl = "/Workspace/Users/[USERNAME]/.bundle/test-bundle/defa
 /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock
 /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json
 /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json
-/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/STATE_FILENAME
 
 === Restore config to target old wheel
 >>> update_file.py databricks.yml ./dist/*.whl ./dist/my*0.1.0*.whl
@@ -100,4 +98,3 @@ json[0].libraries[0].whl = "/Workspace/Users/[USERNAME]/.bundle/test-bundle/defa
 /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock
 /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json
 /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json
-/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/STATE_FILENAME

acceptance/bundle/artifacts/whl_change_version/test.toml

@@ -1,3 +1,5 @@
+# This test asserts wheel uploads; DMS checkpoints are not rewritten on later deploys.
+# Exact resources.json write counts are covered by bundle/dms/marker-writes.
 [[Repls]]
-Old = '(resources.json|terraform.tfstate)'
-New = 'STATE_FILENAME'
+Old = '(?m)^/Workspace/Users/[^\r\n]+/\.bundle/test-bundle/default/state/(resources\.json|terraform\.tfstate)\r?\n'
+New = ''

acceptance/bundle/dms/empty-grants/output.txt

@@ -161,7 +161,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged
  "state_version": 3,
  "cli_version": "[CLI_VERSION]",
  "lineage": "[UUID]",
- "serial": 2,
+ "serial": 1,
  "features": {
   "deployment_history": {}
  },

acceptance/bundle/dms/existing-state/output.txt

@@ -16,6 +16,10 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged
   "features": null
 }
 
+=== Direct deployment still writes resources.json once
+>>> print_requests.py --method POST //workspace/import //workspace-files/import-file
+1
+
 === Config enables recording despite the disabled environment: register version 2, then deploy version 3
 >>> [CLI] bundle deploy
 Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/files...
@@ -36,6 +40,10 @@ Resources: 0 created, 1 changed, 0 deleted, 1 unchanged
   "state": {}
 }
 
+=== Migration writes the marker once; the standalone deployment does not rewrite it
+>>> print_requests.py --method POST //workspace/import //workspace-files/import-file
+1
+
 === Inspect the remote backup's original pre-migration resource IDs, state, and dependencies
 >>> MSYS_NO_PATHCONV=1 [CLI] workspace export /Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/state/resources.json.backup --format RAW --file .databricks/resources.backup.first.json
 

acceptance/bundle/dms/existing-state/script

@@ -1,11 +1,19 @@
 envsubst < databricks.yml.tmpl > databricks.yml
+state_path="/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-existing-state-${UNIQUE_NAME}/default/state"
 
 title "Config disables recording even when the environment enables it"
 trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true $CLI bundle deploy 2>&1 | contains.py '!Current Deployment Version:' '!Enabling deployment history'
 replace_ids.py
 trace cp .databricks/bundle/default/resources.json .databricks/resources.before-dms.json
 trace jq '{state_version, serial, features}' .databricks/bundle/default/resources.json
 
+title "Direct deployment still writes resources.json once"
+trace print_requests.py --method POST //workspace/import //workspace-files/import-file |
+    MSYS_NO_PATHCONV=1 jq -s --arg path "$state_path/resources.json" '
+        map(select((.path == "/api/2.0/workspace/import" and (.body.multipart_form.path // .body.path) == $path)
+            or .path == ("/api/2.0/workspace-files/import-file" + $path))) | length |
+        if . == 1 then . else error("expected one direct state write") end'
+
 title "Config enables recording despite the disabled environment: register version 2, then deploy version 3"
 export DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=false
 update_file.py databricks.yml 'deployment_history: false' 'deployment_history: true'
@@ -14,6 +22,13 @@ trace $CLI bundle deploy 2>&1 | contains.py 'Enabling deployment history...' 'De
 trace find.py 'resources[.]json[.]wal' --expect 0
 trace jq '{features, state}' .databricks/bundle/default/resources.json
 
+title "Migration writes the marker once; the standalone deployment does not rewrite it"
+trace print_requests.py --method POST //workspace/import //workspace-files/import-file |
+    MSYS_NO_PATHCONV=1 jq -s --arg path "$state_path/resources.json" '
+        map(select((.path == "/api/2.0/workspace/import" and (.body.multipart_form.path // .body.path) == $path)
+            or .path == ("/api/2.0/workspace-files/import-file" + $path))) | length |
+        if . == 1 then . else error("expected one migration marker write") end'
+
 title "Inspect the remote backup's original pre-migration resource IDs, state, and dependencies"
 trace MSYS_NO_PATHCONV=1 $CLI workspace export "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-existing-state-${UNIQUE_NAME}/default/state/resources.json.backup" --format RAW --file .databricks/resources.backup.first.json
 trace cat .databricks/resources.backup.first.json

acceptance/bundle/dms/marker-writes/databricks.yml.tmpl

@@ -0,0 +1,14 @@
+bundle:
+  name: dms-marker-writes-$UNIQUE_NAME
+
+experimental:
+  deployment_history: true
+
+sync:
+  exclude:
+    - LOG.*
+
+resources:
+  jobs:
+    foo:
+      name: foo

acceptance/bundle/dms/marker-writes/out.test.toml

@@ -0,0 +1,2 @@
+Cloud = true
+EnvMatrix.DMS = ["true"]

acceptance/bundle/dms/marker-writes/output.txt

@@ -0,0 +1,38 @@
+
+=== 1. First recorded deploy: one remote marker write, version 1
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-marker-writes-[UNIQUE_NAME]/default/files...
+Created jobs.foo
+Files: 2 uploaded, 0 deleted
+Resources: 1 created, 0 changed, 0 deleted, 0 unchanged
+
+>>> resources_json_writes 1
+resources.json writes=1
+
+>>> [CLI] bundle summary -o json
+{"job_id":"[FOO_ID]","version":1}
+
+=== 2. Real job update: zero marker writes, same ID, version 2
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-marker-writes-[UNIQUE_NAME]/default/files...
+Updated jobs.foo
+Files: 1 uploaded, 0 deleted
+Resources: 0 created, 1 changed, 0 deleted, 0 unchanged
+
+>>> resources_json_writes 0
+resources.json writes=0
+
+>>> [CLI] bundle summary -o json
+{"job_id":"[FOO_ID]","version":2}
+
+=== 3. No-op deploy: zero marker writes, still version 2
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-marker-writes-[UNIQUE_NAME]/default/files...
+Files: 0 uploaded, 0 deleted
+Resources: 0 created, 0 changed, 0 deleted, 1 unchanged
+
+>>> resources_json_writes 0
+resources.json writes=0
+
+>>> [CLI] bundle summary -o json
+{"job_id":"[FOO_ID]","version":2}

acceptance/bundle/dms/marker-writes/script

@@ -0,0 +1,33 @@
+envsubst < databricks.yml.tmpl > databricks.yml
+state_path="/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-marker-writes-${UNIQUE_NAME}/default/state"
+
+# Import's target lives in its body; match the exact state path, not metadata or backup files.
+resources_json_writes() {
+    print_requests.py --method POST //workspace/import //workspace-files/import-file |
+        MSYS_NO_PATHCONV=1 jq -sr --arg path "$state_path/resources.json" --argjson expected "$1" '
+            map(select(
+                (.path == "/api/2.0/workspace/import" and (.body.multipart_form.path // .body.path) == $path)
+                or .path == ("/api/2.0/workspace-files/import-file" + $path)
+            )) | length | if . == $expected then "resources.json writes=\(.)"
+                else error("unexpected resources.json write count: \(.)") end'
+}
+
+title "1. First recorded deploy: one remote marker write, version 1"
+trace $CLI bundle deploy
+replace_ids.py
+trace resources_json_writes 1
+trace $CLI bundle summary -o json | jq -c '{job_id: .resources.jobs.foo.id, version: .bundle.deployment.latest_version_id}'
+
+title "2. Real job update: zero marker writes, same ID, version 2"
+update_file.py databricks.yml 'name: foo' 'name: updated'
+trace $CLI bundle deploy 2>&1 | contains.py 'Updated jobs.foo' '!Created '
+trace resources_json_writes 0
+trace $CLI bundle summary -o json | jq -c '{job_id: .resources.jobs.foo.id, version: .bundle.deployment.latest_version_id}'
+
+title "3. No-op deploy: zero marker writes, still version 2"
+trace $CLI bundle deploy 2>&1 | contains.py 'Resources: 0 created, 0 changed, 0 deleted, 1 unchanged'
+trace resources_json_writes 0
+trace $CLI bundle summary -o json | jq -c '{job_id: .resources.jobs.foo.id, version: .bundle.deployment.latest_version_id}'
+
+$CLI bundle destroy --auto-approve &> LOG.cleanup
+rm -f "$OUT_REQUESTS"

acceptance/bundle/dms/no-drift/output.txt

@@ -198,7 +198,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged
  "state_version": 3,
  "cli_version": "[CLI_VERSION]",
  "lineage": "[UUID]",
- "serial": 2,
+ "serial": 1,
  "features": {
   "deployment_history": {}
  },

acceptance/bundle/dms/stale-local-state/output.txt

@@ -1,19 +1,19 @@
 
-=== Setup (LOG.setup): reconstruct full version 1 without a history marker, then deploy version 2
+=== Setup (LOG.setup): reconstruct version 1, then advance DMS to version 2 without rewriting its checkpoint
 >>> jq -c {serial, features} .databricks/resources.v1.json
 {"serial":1,"features":null}
 
->>> jq -c {serial, features} .databricks/resources.v2.json
-{"serial":2,"features":{"deployment_history":{}}}
+>>> jq -c {serial, features} .databricks/resources.checkpoint.json
+{"serial":1,"features":{"deployment_history":{}}}
 
-=== 1. Config false and env false: newer remote state replaces the local copy and reads DMS
+=== 1. Config false and env false: remote DMS checkpoint replaces the direct snapshot despite equal serials
 >>> cp .databricks/resources.v1.json .databricks/bundle/default/resources.json
 
 >>> DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=false [CLI] bundle summary --force-pull -o json
 Warn: Deployment history setting (false) does not match the existing state (true). Using the existing state.
-{"job_id":"[FOO_ID]"}
+{"job_id":"[FOO_ID]","version":2}
 
->>> cmp .databricks/resources.v2.json .databricks/bundle/default/resources.json
+>>> cmp .databricks/resources.checkpoint.json .databricks/bundle/default/resources.json
 
 >>> print_requests.py --dms //resources --get --oneline
 {"method": "GET", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/resources"}
@@ -23,9 +23,9 @@ Warn: Deployment history setting (false) does not match the existing state (true
 
 >>> DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=false [CLI] bundle summary --force-pull -o json
 Warn: Deployment history setting (false) does not match the existing state (true). Using the existing state.
-{"job_id":"[FOO_ID]"}
+{"job_id":"[FOO_ID]","version":2}
 
->>> cmp .databricks/resources.v2.json .databricks/bundle/default/resources.json
+>>> cmp .databricks/resources.checkpoint.json .databricks/bundle/default/resources.json
 
 >>> print_requests.py --dms //resources --get --oneline
 {"method": "GET", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/resources"}

acceptance/bundle/dms/stale-local-state/script

@@ -1,32 +1,33 @@
 envsubst < databricks.yml.tmpl > databricks.yml
+state_path="/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-stale-local-state-${UNIQUE_NAME}/default/state"
 
-title "Setup (LOG.setup): reconstruct full version 1 without a history marker, then deploy version 2"
+title "Setup (LOG.setup): reconstruct version 1, then advance DMS to version 2 without rewriting its checkpoint"
 {
     $CLI bundle deploy
     DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true print_state.py > .databricks/resources.v1.json
     DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true replace_ids.py
     update_file.py databricks.yml 'name: foo' 'name: updated'
     $CLI bundle deploy
-    cp .databricks/bundle/default/resources.json .databricks/resources.v2.json
+    MSYS_NO_PATHCONV=1 $CLI workspace export "$state_path/resources.json" --format RAW --file .databricks/resources.checkpoint.json
     deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-stale-local-state-${UNIQUE_NAME}/default/state/resources.deployment.json" -o json | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])')
     add_repl "$deployment_id" DEPLOYMENT_ID
 } &> LOG.setup
 trace jq -c '{serial, features}' .databricks/resources.v1.json
-trace jq -c '{serial, features}' .databricks/resources.v2.json
+trace jq -c '{serial, features}' .databricks/resources.checkpoint.json
 rm -f "$OUT_REQUESTS"
 
-title "1. Config false and env false: newer remote state replaces the local copy and reads DMS"
+title "1. Config false and env false: remote DMS checkpoint replaces the direct snapshot despite equal serials"
 update_file.py databricks.yml 'deployment_history: true' 'deployment_history: false'
 trace cp .databricks/resources.v1.json .databricks/bundle/default/resources.json
-trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=false $CLI bundle summary --force-pull -o json | jq -c '{job_id: .resources.jobs.foo.id}'
-trace cmp .databricks/resources.v2.json .databricks/bundle/default/resources.json
+trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=false $CLI bundle summary --force-pull -o json | jq -c '{job_id: .resources.jobs.foo.id, version: .bundle.deployment.latest_version_id}'
+trace cmp .databricks/resources.checkpoint.json .databricks/bundle/default/resources.json
 trace print_requests.py --dms //resources --get --oneline
 
 title "2. Config removed and env false: restore the same snapshot, then read remote DMS state"
 update_file.py databricks.yml $'experimental:\n  deployment_history: false\n\n' ''
 trace cp .databricks/resources.v1.json .databricks/bundle/default/resources.json
-trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=false $CLI bundle summary --force-pull -o json | jq -c '{job_id: .resources.jobs.foo.id}'
-trace cmp .databricks/resources.v2.json .databricks/bundle/default/resources.json
+trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=false $CLI bundle summary --force-pull -o json | jq -c '{job_id: .resources.jobs.foo.id, version: .bundle.deployment.latest_version_id}'
+trace cmp .databricks/resources.checkpoint.json .databricks/bundle/default/resources.json
 trace print_requests.py --dms //resources --get --oneline
 
 DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true $CLI bundle destroy --auto-approve &> LOG.cleanup

acceptance/bundle/dms/successful-recreate/output.txt

@@ -193,7 +193,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged
  "state_version": 3,
  "cli_version": "[CLI_VERSION]",
  "lineage": "[UUID]",
- "serial": 2,
+ "serial": 1,
  "features": {
   "deployment_history": {}
  },

bundle/bundle.go

@@ -197,6 +197,9 @@ type Bundle struct {
 	// MigratingToDMS registers the existing direct state before deploying with history enabled.
 	MigratingToDMS bool
 
+	// NeedsDMSStateUpload creates or repairs the remote deployment-history marker.
+	NeedsDMSStateUpload bool
+
 	// AllowTerraformEngineConfig reports a terraform bundle.engine as a warning instead
 	// of an error and skips the DATABRICKS_BUNDLE_ENGINE check. Set by read-only commands
 	// (bundle summary) so users can still inspect a bundle that pins the removed engine.

bundle/phases/deploy.go

@@ -310,6 +310,7 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand
 			return
 		}
 		logDeploymentVersion(ctx, b, "Deployment history enabled")
+		b.NeedsDMSStateUpload = true
 		statemgmt.PushResourcesState(ctx, b)
 		if logdiag.HasError(ctx) {
 			return

bundle/statemgmt/state_push.go

@@ -16,6 +16,11 @@ import (
 
 // PushResourcesState uploads the local state file to the remote location.
 func PushResourcesState(ctx context.Context, b *bundle.Bundle) {
+	recorded := b.DeploymentBundle.StateDB.IsDeploymentMetadataService()
+	if recorded && !b.NeedsDMSStateUpload {
+		return
+	}
+
 	f, err := deploy.StateFiler(ctx, b)
 	if err != nil {
 		logdiag.LogError(ctx, err)
@@ -41,6 +46,10 @@ func PushResourcesState(ctx context.Context, b *bundle.Bundle) {
 	err = f.Write(ctx, remotePath, local, filer.CreateParentDirectories, filer.OverwriteIfExists)
 	if err != nil {
 		logdiag.LogError(ctx, err)
+		return
+	}
+	if recorded {
+		b.NeedsDMSStateUpload = false
 	}
 }