Recover corrupted DMS deployment-history markers

acceptance/bundle/dms/marker-writes/output.txt

@@ -36,3 +36,29 @@ resources.json writes=0
 
 >>> [CLI] bundle summary -o json
 {"job_id":"[FOO_ID]","version":2}
+
+=== 4. Corrupt remote marker: repair once, preserve ID, continue at version 3
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-marker-writes-[UNIQUE_NAME]/default/files...
+Updated jobs.foo
+Files: 1 uploaded, 0 deleted
+Resources: 0 created, 1 changed, 0 deleted, 0 unchanged
+
+>>> resources_json_writes 1
+resources.json writes=1
+
+>>> [CLI] bundle summary -o json
+{"job_id":"[FOO_ID]","version":3}
+
+=== 5. Missing remote marker: repair once, preserve ID, continue at version 4
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-marker-writes-[UNIQUE_NAME]/default/files...
+Updated jobs.foo
+Files: 1 uploaded, 0 deleted
+Resources: 0 created, 1 changed, 0 deleted, 0 unchanged
+
+>>> resources_json_writes 1
+resources.json writes=1
+
+>>> [CLI] bundle summary -o json
+{"job_id":"[FOO_ID]","version":4}

acceptance/bundle/dms/marker-writes/script

@@ -29,5 +29,21 @@ trace $CLI bundle deploy 2>&1 | contains.py 'Resources: 0 created, 0 changed, 0
 trace resources_json_writes 0
 trace $CLI bundle summary -o json | jq -c '{job_id: .resources.jobs.foo.id, version: .bundle.deployment.latest_version_id}'
 
+title "4. Corrupt remote marker: repair once, preserve ID, continue at version 3"
+MSYS_NO_PATHCONV=1 $CLI workspace import "$state_path/resources.json" --file databricks.yml --format RAW --overwrite &> LOG.corrupt
+rm -f "$OUT_REQUESTS"
+update_file.py databricks.yml 'name: updated' 'name: repaired'
+trace $CLI bundle deploy 2>&1 | contains.py 'Updated jobs.foo' '!Created ' '!Recovering deployment history state with a new lineage'
+trace resources_json_writes 1
+trace $CLI bundle summary -o json | jq -c '{job_id: .resources.jobs.foo.id, version: .bundle.deployment.latest_version_id}'
+
+title "5. Missing remote marker: repair once, preserve ID, continue at version 4"
+MSYS_NO_PATHCONV=1 $CLI workspace delete "$state_path/resources.json" &> LOG.missing
+rm -f "$OUT_REQUESTS"
+update_file.py databricks.yml 'name: repaired' 'name: restored'
+trace $CLI bundle deploy 2>&1 | contains.py 'Updated jobs.foo' '!Created ' '!Recovering deployment history state with a new lineage'
+trace resources_json_writes 1
+trace $CLI bundle summary -o json | jq -c '{job_id: .resources.jobs.foo.id, version: .bundle.deployment.latest_version_id}'
+
 $CLI bundle destroy --auto-approve &> LOG.cleanup
 rm -f "$OUT_REQUESTS"

acceptance/bundle/dms/state-resolution/output.txt

@@ -2,8 +2,8 @@
 === Setup (LOG.setup)
 Current: DMS enabled; remote reference snapshot saved.
 
-=== 1. Stale cache
-Expect: remote beats serial 999/wrong lineage; cache matches remote.
+=== 1. Stale local copy
+Expect: remote beats serial 999/wrong lineage; local copy matches remote.
 
 >>> jq {serial, lineage} .databricks/bundle/default/resources.json
 {
@@ -15,11 +15,11 @@ Expect: remote beats serial 999/wrong lineage; cache matches remote.
 resources.json: remote direct state serial=1 lineage="[UUID]"
 
 >>> cmp .databricks/resources.remote.json .databricks/bundle/default/resources.json
-PASS: stale cache replaced
+PASS: stale local copy replaced
 
-=== 2. Invalid cache
-Input: YAML in JSON cache paths.
-Expect: use remote state; overwrite broken JSON; remove Terraform cache; no resource changes.
+=== 2. Invalid local copies
+Input: YAML in JSON state paths.
+Expect: use remote state; overwrite broken JSON; remove Terraform copy; no resource changes.
 
 >>> cp databricks.yml .databricks/bundle/default/resources.json
 
@@ -33,18 +33,20 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged
 >>> test ! -e .databricks/bundle/default/terraform/terraform.tfstate
 
 >>> cmp .databricks/resources.remote.json .databricks/bundle/default/resources.json
-PASS: invalid caches handled
+PASS: invalid local copies handled
 
-=== 3. Missing remote
-Expect: bad JSON fails; local cache bytes remain unchanged.
+=== 3. Missing remote checkpoint and invalid local copy
+Expect: recover confirmed DMS state with a new lineage and invalidate old plans.
 
 >>> MSYS_NO_PATHCONV=1 [CLI] workspace delete /Workspace/Users/[USERNAME]/.bundle/dms-state-resolution-[UNIQUE_NAME]/default/state/resources.json
 
 >>> cp databricks.yml .databricks/bundle/default/resources.json
 
->>> musterr [CLI] bundle plan
-Error: invalid character 'b' looking for beginning of value
+>>> [CLI] bundle plan
+Warning: Recovering deployment history state with a new lineage; existing saved plans are no longer valid
 
+Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged
 
->>> cmp databricks.yml .databricks/bundle/default/resources.json
-PASS: failure preserved cache
+>>> jq -e --slurpfile original .databricks/resources.remote.json .lineage != $original[0].lineage and (.features | has("deployment_history")) and .state == {} .databricks/bundle/default/resources.json
+true
+PASS: DMS recovery replaced the invalid local copy

acceptance/bundle/dms/state-resolution/script

@@ -7,31 +7,31 @@ title "Setup (LOG.setup)\nCurrent: DMS enabled; remote reference snapshot saved.
     trace cp .databricks/bundle/default/resources.json .databricks/resources.remote.json
 } &> LOG.setup
 
-title "1. Stale cache\nExpect: remote beats serial 999/wrong lineage; cache matches remote.\n"
+title "1. Stale local copy\nExpect: remote beats serial 999/wrong lineage; local copy matches remote.\n"
 {
     trace update_file.py .databricks/bundle/default/resources.json '"serial": 1,' '"serial": 999,'
     trace update_file.py .databricks/bundle/default/resources.json '"lineage": "' '"lineage": "stale-'
-} &> LOG.stale-cache
+} &> LOG.stale-copy
 trace jq '{serial, lineage}' .databricks/bundle/default/resources.json
 trace $CLI bundle debug states --force-pull 2>&1 | contains.py 'remote direct state' '!local ' '!stale-' '!serial=999'
 trace cmp .databricks/resources.remote.json .databricks/bundle/default/resources.json
-echo "PASS: stale cache replaced"
+echo "PASS: stale local copy replaced"
 
-title "2. Invalid cache\nInput: YAML in JSON cache paths.\nExpect: use remote state; overwrite broken JSON; remove Terraform cache; no resource changes.\n"
+title "2. Invalid local copies\nInput: YAML in JSON state paths.\nExpect: use remote state; overwrite broken JSON; remove Terraform copy; no resource changes.\n"
 trace cp databricks.yml .databricks/bundle/default/resources.json
 trace mkdir -p .databricks/bundle/default/terraform
 trace cp databricks.yml .databricks/bundle/default/terraform/terraform.tfstate
 trace $CLI bundle plan | contains.py 'Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged'
 trace test ! -e .databricks/bundle/default/terraform/terraform.tfstate
 trace cmp .databricks/resources.remote.json .databricks/bundle/default/resources.json
-echo "PASS: invalid caches handled"
+echo "PASS: invalid local copies handled"
 
-title "3. Missing remote\nExpect: bad JSON fails; local cache bytes remain unchanged.\n"
+title "3. Missing remote checkpoint and invalid local copy\nExpect: recover confirmed DMS state with a new lineage and invalidate old plans.\n"
 trace MSYS_NO_PATHCONV=1 $CLI workspace delete "$state_path/resources.json"
 trace cp databricks.yml .databricks/bundle/default/resources.json
-trace musterr $CLI bundle plan 2>&1 | contains.py 'Error' 'invalid character'
-trace cmp databricks.yml .databricks/bundle/default/resources.json
-echo "PASS: failure preserved cache"
+trace $CLI bundle plan 2>&1 | contains.py 'Recovering deployment history state with a new lineage; existing saved plans are no longer valid' 'Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged'
+trace jq -e --slurpfile original .databricks/resources.remote.json '.lineage != $original[0].lineage and (.features | has("deployment_history")) and .state == {}' .databricks/bundle/default/resources.json
+echo "PASS: DMS recovery replaced the invalid local copy"
 
 {
     trace MSYS_NO_PATHCONV=1 $CLI workspace import "$state_path/resources.json" --file .databricks/resources.remote.json --format AUTO --overwrite

acceptance/bundle/dms/version-never-created/output.txt

@@ -12,9 +12,12 @@ API message: Internal error
 Files: 1 uploaded, 0 deleted
 
 >>> [CLI] bundle plan -o json
+Warning: Recovering deployment history state with a new lineage; existing saved plans are no longer valid
+
 {
   "plan_version": [PLAN_VERSION],
   "cli_version": "[CLI_VERSION]",
+  "lineage": "[UUID]",
   "features": {
     "deployment_history": {}
   }