diff --git a/acceptance/bin/print_state.py b/acceptance/bin/print_state.py index 6ad674342c3..9badc1e4d5e 100755 --- a/acceptance/bin/print_state.py +++ b/acceptance/bin/print_state.py @@ -174,23 +174,26 @@ def get_last_version_id(target): def print_recorded_state(filename, target): - """Print the state file with its resources filled in from the deployment metadata service. + """Print DMS resources using the state file's header. - While recording, the file itself carries only the header - the service holds the resources - so - printing it raw would show an empty state and differ from the same test's non-recording run. + This checks history uploads even though resources.json remains authoritative. + Use --no-dms to inspect the file state instead. """ data = json.loads(open(filename).read()) # Recording stamps each resource payload with the deployment and version; drop it here so the # printed state matches a non-recording run without every caller piping through nostamp. data["state"] = scrub(get_recorded_state(target)) - # The file's serial is a copy of the DMS version to ensure outdated resource state is not read. - # Rebuild in header order with the authoritative serial from the deployment. + # Pair DMS resources with the DMS version. The file serial can advance independently. last_version_id = get_last_version_id(target) serial = int(last_version_id) if last_version_id else 0 # Recording itself is not what these tests assert, so drop the feature that marks it. - features = {k: v for k, v in (data.get("features") or {}).items() if k != "deployment_history"} + features = { + k: v + for k, v in (data.get("features") or {}).items() + if k not in {"deployment_history", "deployment_history_shadow"} + } rebuilt = {} for key in ("state_version", "cli_version", "lineage"): @@ -213,7 +216,7 @@ def main(): "--no-dms", action="store_true", help="Print resources.json as-is without fetching state from DMS. " - "Use in bundle/dms tests to assert the tombstone content of the file.", + "Use in bundle/dms tests to assert the file state and its feature markers.", ) args = parser.parse_args() diff --git a/acceptance/bundle/deploy/readplan/lineage-mismatch/plan.dms.json b/acceptance/bundle/deploy/readplan/lineage-mismatch/plan.dms.json index 58e072b84f1..aa3f25e2d27 100644 --- a/acceptance/bundle/deploy/readplan/lineage-mismatch/plan.dms.json +++ b/acceptance/bundle/deploy/readplan/lineage-mismatch/plan.dms.json @@ -5,6 +5,6 @@ "serial": 0, "plan": {}, "features": { - "deployment_history": {} + "deployment_history_shadow": {} } } diff --git a/acceptance/bundle/dms/bind-unbind-unsupported/databricks.yml.tmpl b/acceptance/bundle/dms/bind-unbind-unsupported/databricks.yml.tmpl index e60f4605610..92a70710626 100644 --- a/acceptance/bundle/dms/bind-unbind-unsupported/databricks.yml.tmpl +++ b/acceptance/bundle/dms/bind-unbind-unsupported/databricks.yml.tmpl @@ -1,8 +1,5 @@ bundle: name: dms-bind-unbind-$UNIQUE_NAME -experimental: - deployment_history: true - resources: jobs: foo: diff --git a/acceptance/bundle/dms/bind-unbind-unsupported/output.txt b/acceptance/bundle/dms/bind-unbind-unsupported/output.txt index d2d0a366778..e2781fef6fb 100644 --- a/acceptance/bundle/dms/bind-unbind-unsupported/output.txt +++ b/acceptance/bundle/dms/bind-unbind-unsupported/output.txt @@ -4,13 +4,17 @@ Error: bind is not supported for a bundle target that records deployment history -=== Deploy so the bundle records state, then unbind is refused too +=== Unbind is allowed for file-backed history state >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-bind-unbind-[UNIQUE_NAME]/default/files... Created jobs.foo Files: 2 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged ->>> musterr [CLI] bundle deployment unbind foo -Error: unbind is not supported for a bundle target that records deployment history +>>> [CLI] bundle deployment unbind foo +>>> jq {features, state} .databricks/bundle/default/resources.json +{ + "features": null, + "state": {} +} diff --git a/acceptance/bundle/dms/bind-unbind-unsupported/script b/acceptance/bundle/dms/bind-unbind-unsupported/script index 03d9cf34a31..7437d908eb2 100644 --- a/acceptance/bundle/dms/bind-unbind-unsupported/script +++ b/acceptance/bundle/dms/bind-unbind-unsupported/script @@ -8,7 +8,8 @@ job_id=$($CLI jobs create --json '{"name": "bind-target-'$UNIQUE_NAME'"}' | jq - add_repl "$job_id" JOB_ID trace musterr $CLI bundle deployment bind foo "$job_id" -# unbind refuses when it opens an existing recording state, so it needs a deployment first. -title "Deploy so the bundle records state, then unbind is refused too" +# File state remains writable even when it carries the history marker. +title "Unbind is allowed for file-backed history state" trace $CLI bundle deploy -trace musterr $CLI bundle deployment unbind foo +trace $CLI bundle deployment unbind foo +trace jq '{features, state}' .databricks/bundle/default/resources.json diff --git a/acceptance/bundle/dms/declined-deploy/output.txt b/acceptance/bundle/dms/declined-deploy/output.txt index 3ef52219df5..02b3d5450f9 100644 --- a/acceptance/bundle/dms/declined-deploy/output.txt +++ b/acceptance/bundle/dms/declined-deploy/output.txt @@ -67,7 +67,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "lineage": "[UUID]", "serial": 1, "features": { - "deployment_history": {} + "deployment_history_shadow": {} } } @@ -79,18 +79,10 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged "method": "GET", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]" } -{ - "method": "GET", - "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/resources" -} { "method": "GET", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]" } -{ - "method": "GET", - "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/resources" -} >>> print_state.py --no-dms { @@ -99,9 +91,17 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged "lineage": "[UUID]", "serial": 1, "features": { - "deployment_history": {} + "deployment_history_shadow": {} }, - "state": {} + "state": { + "resources.schemas.foo": { + "__id__": "main.dms_declined_deploy_schema_[UNIQUE_NAME]", + "state": { + "catalog_name": "main", + "name": "dms_declined_deploy_schema_[UNIQUE_NAME]" + } + } + } } === A destructive change without --auto-approve is declined: this console cannot prompt @@ -124,10 +124,6 @@ Files: 2 uploaded, 0 deleted "method": "GET", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]" } -{ - "method": "GET", - "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/resources" -} === The recorded resources are unchanged by the decline: the schema is still at version 1 >>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[DEPLOYMENT_ID] -o json @@ -163,10 +159,6 @@ false "method": "GET", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]" } -{ - "method": "GET", - "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/resources" -} { "method": "POST", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions", diff --git a/acceptance/bundle/dms/depends-on/output.txt b/acceptance/bundle/dms/depends-on/output.txt index c1b9beb1dec..e77719a1b69 100644 --- a/acceptance/bundle/dms/depends-on/output.txt +++ b/acceptance/bundle/dms/depends-on/output.txt @@ -5,7 +5,7 @@ "plan_version": [PLAN_VERSION], "cli_version": "[CLI_VERSION]", "features": { - "deployment_history": {} + "deployment_history_shadow": {} } } @@ -61,7 +61,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "lineage": "[UUID]", "serial": 1, "features": { - "deployment_history": {} + "deployment_history_shadow": {} } } @@ -91,9 +91,53 @@ Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged "lineage": "[UUID]", "serial": 1, "features": { - "deployment_history": {} + "deployment_history_shadow": {} }, - "state": {} + "state": { + "resources.jobs.child": { + "__id__": "[CHILD_ID]", + "state": { + "deployment": { + "deployment_id": "[DEPLOYMENT_ID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-depends-on-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" + }, + "description": "depends on [PARENT_ID]", + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "child", + "queue": { + "enabled": true + } + }, + "depends_on": [ + { + "node": "resources.jobs.parent", + "label": "${resources.jobs.parent.id}" + } + ] + }, + "resources.jobs.parent": { + "__id__": "[PARENT_ID]", + "state": { + "deployment": { + "deployment_id": "[DEPLOYMENT_ID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-depends-on-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "parent", + "queue": { + "enabled": true + } + } + } + } } >>> [CLI] bundle destroy --auto-approve diff --git a/acceptance/bundle/dms/deployment-metadata-change/output.txt b/acceptance/bundle/dms/deployment-metadata-change/output.txt index 675d23ca70d..94f0a68a122 100644 --- a/acceptance/bundle/dms/deployment-metadata-change/output.txt +++ b/acceptance/bundle/dms/deployment-metadata-change/output.txt @@ -67,7 +67,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "lineage": "[UUID]", "serial": 1, "features": { - "deployment_history": {} + "deployment_history_shadow": {} } } @@ -128,7 +128,7 @@ A common practice is to use a username or principal name in this path, i.e. use "lineage": "[UUID]", "serial": 1, "features": { - "deployment_history": {} + "deployment_history_shadow": {} } } diff --git a/acceptance/bundle/dms/empty-grants/output.txt b/acceptance/bundle/dms/empty-grants/output.txt index 0a6cbabcbce..c12e022ced2 100644 --- a/acceptance/bundle/dms/empty-grants/output.txt +++ b/acceptance/bundle/dms/empty-grants/output.txt @@ -138,7 +138,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged "lineage": "[UUID]", "serial": 2, "features": { - "deployment_history": {} + "deployment_history_shadow": {} } } @@ -163,9 +163,17 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged "lineage": "[UUID]", "serial": 2, "features": { - "deployment_history": {} + "deployment_history_shadow": {} }, - "state": {} + "state": { + "resources.schemas.foo": { + "__id__": "main.dms_empty_grants_[UNIQUE_NAME]", + "state": { + "catalog_name": "main", + "name": "dms_empty_grants_[UNIQUE_NAME]" + } + } + } } >>> [CLI] bundle destroy --auto-approve diff --git a/acceptance/bundle/dms/existing-state/output.txt b/acceptance/bundle/dms/existing-state/output.txt index 4057525509b..c225a6b716b 100644 --- a/acceptance/bundle/dms/existing-state/output.txt +++ b/acceptance/bundle/dms/existing-state/output.txt @@ -73,6 +73,6 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged { "state_version": 3, "features": { - "deployment_history": {} + "deployment_history_shadow": {} } } diff --git a/acceptance/bundle/dms/existing-state/script b/acceptance/bundle/dms/existing-state/script index e64ca6febe6..f4557e9089f 100644 --- a/acceptance/bundle/dms/existing-state/script +++ b/acceptance/bundle/dms/existing-state/script @@ -16,7 +16,7 @@ musterr trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true $CLI bundle deploy title "A leftover local WAL with the wrong lineage is still refused" mkdir -p .databricks/bundle/default -printf '{"lineage":"stale","serial":99,"state_version":2,"features":{"deployment_history":{}}}\n' > .databricks/bundle/default/resources.json.wal +printf '{"lineage":"stale","serial":99,"state_version":2,"features":{"deployment_history_shadow":{}}}\n' > .databricks/bundle/default/resources.json.wal musterr trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true $CLI bundle deploy title "Destroy clears the deployment. With nothing recorded remotely the bundle is treated as new, so recording can be enabled once the stale WAL is gone" diff --git a/acceptance/bundle/dms/failed-delete/output.txt b/acceptance/bundle/dms/failed-delete/output.txt index e3241b3d66a..a8b5781edc8 100644 --- a/acceptance/bundle/dms/failed-delete/output.txt +++ b/acceptance/bundle/dms/failed-delete/output.txt @@ -67,7 +67,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "lineage": "[UUID]", "serial": 1, "features": { - "deployment_history": {} + "deployment_history_shadow": {} } } @@ -92,9 +92,28 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged "lineage": "[UUID]", "serial": 1, "features": { - "deployment_history": {} + "deployment_history_shadow": {} }, - "state": {} + "state": { + "resources.jobs.stuck": { + "__id__": "[STUCK_ID]", + "state": { + "deployment": { + "deployment_id": "[DEPLOYMENT_ID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-failed-delete-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "stuck", + "queue": { + "enabled": true + } + } + } + } } === A resource that fails to delete is recorded as a failed operation carrying the error, so the history says why rather than showing the resource gone @@ -176,22 +195,41 @@ API message: Fault injected by test. } ] -=== The local state file records failed version 2 +=== The local state file keeps the resource and its serial after the failed delete >>> print_state.py --no-dms { "state_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", - "serial": 2, + "serial": 1, "features": { - "deployment_history": {} + "deployment_history_shadow": {} }, - "state": {} + "state": { + "resources.jobs.stuck": { + "__id__": "[STUCK_ID]", + "state": { + "deployment": { + "deployment_id": "[DEPLOYMENT_ID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-failed-delete-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "stuck", + "queue": { + "enabled": true + } + } + } + } } === The remote checkpoint stays at serial 1 while DMS advances to version 2 >>> MSYS_NO_PATHCONV=1 [CLI] workspace export /Workspace/Users/[USERNAME]/.bundle/dms-failed-delete-[UNIQUE_NAME]/default/state/resources.json --format RAW {"serial":1} ->>> print_state.py -{"serial":2} +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments get-deployment deployments/[DEPLOYMENT_ID] -o json +{"last_version_id":"2"} diff --git a/acceptance/bundle/dms/failed-delete/script b/acceptance/bundle/dms/failed-delete/script index 2b733207f0a..5552fd18bad 100644 --- a/acceptance/bundle/dms/failed-delete/script +++ b/acceptance/bundle/dms/failed-delete/script @@ -29,12 +29,12 @@ trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${d title "The failed operation records the error message" trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-operations "deployments/${deployment_id}/versions/2" -o json | jq -S 'map({resource_key, action_type, status, error_message})' -title "The local state file records failed version 2" +title "The local state file keeps the resource and its serial after the failed delete" trace print_state.py --no-dms title "The remote checkpoint stays at serial 1 while DMS advances to version 2" trace MSYS_NO_PATHCONV=1 $CLI workspace export "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-failed-delete-${UNIQUE_NAME}/default/state/resources.json" --format RAW | jq -c '{serial}' -trace print_state.py | jq -c '{serial}' +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments get-deployment "deployments/${deployment_id}" -o json | jq -c '{last_version_id}' # Drain any accumulated requests (plan, state reads) so the harness has nothing to diff. rm -f "$OUT_REQUESTS" diff --git a/acceptance/bundle/dms/failed-recreate/output.txt b/acceptance/bundle/dms/failed-recreate/output.txt index 860460c3375..2e599afbde7 100644 --- a/acceptance/bundle/dms/failed-recreate/output.txt +++ b/acceptance/bundle/dms/failed-recreate/output.txt @@ -67,7 +67,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "lineage": "[UUID]", "serial": 1, "features": { - "deployment_history": {} + "deployment_history_shadow": {} } } @@ -92,9 +92,18 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged "lineage": "[UUID]", "serial": 1, "features": { - "deployment_history": {} + "deployment_history_shadow": {} }, - "state": {} + "state": { + "resources.schemas.foo": { + "__id__": "main.dms_failed_recreate_schema_[UNIQUE_NAME]", + "state": { + "catalog_name": "main", + "comment": "v1", + "name": "dms_failed_recreate_schema_[UNIQUE_NAME]" + } + } + } } === A recreate deletes and then fails to create. The delete went through, so the resource really is gone: the failed operation clears its state (self-contained, not assuming the delete's own write reached the service), rather than describing the resource as it was before the deploy @@ -179,9 +188,9 @@ Files: 2 uploaded, 0 deleted "lineage": "[UUID]", "serial": 2, "features": { - "deployment_history": {} + "deployment_history_shadow": {} }, "state": {} } -=== Planning from DMS state alone creates the schema, which is what has to happen: it no longer exists \ No newline at end of file +=== Planning after pulling remote file state creates the schema, which is what has to happen: it no longer exists \ No newline at end of file diff --git a/acceptance/bundle/dms/failed-recreate/script b/acceptance/bundle/dms/failed-recreate/script index 60bcc99df10..45d790615da 100644 --- a/acceptance/bundle/dms/failed-recreate/script +++ b/acceptance/bundle/dms/failed-recreate/script @@ -34,7 +34,7 @@ trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${d title "The raw state after failed recreate shows no resource" trace print_state.py --no-dms -title "Planning from DMS state alone creates the schema, which is what has to happen: it no longer exists" +title "Planning after pulling remote file state creates the schema, which is what has to happen: it no longer exists" rm -rf .databricks $CLI bundle plan &> LOG.plan_final # Ensure "$OUT_REQUESTS" is not left behind (RecordRequests creates it when running tests) diff --git a/acceptance/bundle/dms/failed-update-permissions/output.txt b/acceptance/bundle/dms/failed-update-permissions/output.txt index 47ef6cd6eac..e3b75a673de 100644 --- a/acceptance/bundle/dms/failed-update-permissions/output.txt +++ b/acceptance/bundle/dms/failed-update-permissions/output.txt @@ -15,7 +15,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "lineage": "[UUID]", "serial": 1, "features": { - "deployment_history": {} + "deployment_history_shadow": {} } } @@ -45,9 +45,50 @@ Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged "lineage": "[UUID]", "serial": 1, "features": { - "deployment_history": {} + "deployment_history_shadow": {} }, - "state": {} + "state": { + "resources.jobs.foo": { + "__id__": "[FOO_ID]", + "state": { + "deployment": { + "deployment_id": "[DEPLOYMENT_ID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-failed-update-permissions-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "foo", + "queue": { + "enabled": true + } + } + }, + "resources.jobs.foo.permissions": { + "__id__": "/jobs/[FOO_ID]", + "state": { + "object_id": "/jobs/[FOO_ID]", + "__embed__": [ + { + "level": "CAN_VIEW", + "user_name": "viewer@example.com" + }, + { + "level": "IS_OWNER", + "user_name": "[USERNAME]" + } + ] + }, + "depends_on": [ + { + "node": "resources.jobs.foo", + "label": "${resources.jobs.foo.id}" + } + ] + } + } } === A permissions update that fails records the failure with the permissions it still has. The resource ID here is a path rather than a number, and the state is the embedded permission list, so this covers a shape the resource tests do not @@ -98,11 +139,52 @@ Files: 1 uploaded, 0 deleted "state_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", - "serial": 2, + "serial": 1, "features": { - "deployment_history": {} + "deployment_history_shadow": {} }, - "state": {} + "state": { + "resources.jobs.foo": { + "__id__": "[FOO_ID]", + "state": { + "deployment": { + "deployment_id": "[DEPLOYMENT_ID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-failed-update-permissions-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "foo", + "queue": { + "enabled": true + } + } + }, + "resources.jobs.foo.permissions": { + "__id__": "/jobs/[FOO_ID]", + "state": { + "object_id": "/jobs/[FOO_ID]", + "__embed__": [ + { + "level": "CAN_VIEW", + "user_name": "viewer@example.com" + }, + { + "level": "IS_OWNER", + "user_name": "[USERNAME]" + } + ] + }, + "depends_on": [ + { + "node": "resources.jobs.foo", + "label": "${resources.jobs.foo.id}" + } + ] + } + } } >>> print_requests.py --dms //api/2.0/bundle --oneline diff --git a/acceptance/bundle/dms/mismatch-before-read/output.txt b/acceptance/bundle/dms/mismatch-before-read/output.txt index d81bb0875eb..1f51ec1804a 100644 --- a/acceptance/bundle/dms/mismatch-before-read/output.txt +++ b/acceptance/bundle/dms/mismatch-before-read/output.txt @@ -8,7 +8,15 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> fault.py GET /api/2.0/bundle/deployments/* 500 0 1 >>> env DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=false [CLI] bundle plan -Error: deployment history setting (false) does not match the existing state (true) +update jobs.foo -Update experimental.deployment_history to match the existing deployment, or run "databricks bundle destroy" to start over +Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged +>>> env DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=false [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-mismatch-before-read-[UNIQUE_NAME]/default/files... +Updated jobs.foo +Files: 1 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 0 unchanged + +>>> jq .features // {} .databricks/bundle/default/resources.json +{} diff --git a/acceptance/bundle/dms/mismatch-before-read/script b/acceptance/bundle/dms/mismatch-before-read/script index 8d82f0f9c95..a89228b2462 100644 --- a/acceptance/bundle/dms/mismatch-before-read/script +++ b/acceptance/bundle/dms/mismatch-before-read/script @@ -3,6 +3,9 @@ envsubst < databricks.yml.tmpl > databricks.yml trace $CLI bundle deploy update_file.py databricks.yml 'deployment_history: true' 'deployment_history: false' -# The configuration mismatch must be rejected before trying to read the deployment from DMS. +# Disabling file-backed history must not depend on DMS being available. +rm -f "$OUT_REQUESTS" trace fault.py "GET /api/2.0/bundle/deployments/*" 500 0 1 -musterr trace env DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=false $CLI bundle plan +trace env DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=false $CLI bundle plan +trace env DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=false $CLI bundle deploy +trace jq '.features // {}' .databricks/bundle/default/resources.json diff --git a/acceptance/bundle/dms/multiple-resources/output.txt b/acceptance/bundle/dms/multiple-resources/output.txt index 10c8a496c2b..215a2fb2534 100644 --- a/acceptance/bundle/dms/multiple-resources/output.txt +++ b/acceptance/bundle/dms/multiple-resources/output.txt @@ -25,7 +25,7 @@ Resources: 5 created, 0 changed, 0 deleted, 0 unchanged "lineage": "[UUID]", "serial": 1, "features": { - "deployment_history": {} + "deployment_history_shadow": {} } } @@ -70,9 +70,100 @@ Plan: 0 to add, 0 to change, 0 to delete, 5 unchanged "lineage": "[UUID]", "serial": 1, "features": { - "deployment_history": {} + "deployment_history_shadow": {} }, - "state": {} + "state": { + "resources.jobs.five": { + "__id__": "[FIVE_ID]", + "state": { + "deployment": { + "deployment_id": "[DEPLOYMENT_ID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "five", + "queue": { + "enabled": true + } + } + }, + "resources.jobs.four": { + "__id__": "[FOUR_ID]", + "state": { + "deployment": { + "deployment_id": "[DEPLOYMENT_ID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "four", + "queue": { + "enabled": true + } + } + }, + "resources.jobs.one": { + "__id__": "[ONE_ID]", + "state": { + "deployment": { + "deployment_id": "[DEPLOYMENT_ID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "one", + "queue": { + "enabled": true + } + } + }, + "resources.jobs.three": { + "__id__": "[THREE_ID]", + "state": { + "deployment": { + "deployment_id": "[DEPLOYMENT_ID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "three", + "queue": { + "enabled": true + } + } + }, + "resources.jobs.two": { + "__id__": "[TWO_ID]", + "state": { + "deployment": { + "deployment_id": "[DEPLOYMENT_ID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "two", + "queue": { + "enabled": true + } + } + } + } } === Redeploy with no changes: nothing is applied, so no version is created at all - the deployment's version tracks state changes, like serial @@ -91,7 +182,7 @@ Resources: 0 created, 0 changed, 0 deleted, 5 unchanged "lineage": "[UUID]", "serial": 1, "features": { - "deployment_history": {} + "deployment_history_shadow": {} } } @@ -136,7 +227,98 @@ Plan: 0 to add, 0 to change, 0 to delete, 5 unchanged "lineage": "[UUID]", "serial": 1, "features": { - "deployment_history": {} + "deployment_history_shadow": {} }, - "state": {} + "state": { + "resources.jobs.five": { + "__id__": "[FIVE_ID]", + "state": { + "deployment": { + "deployment_id": "[DEPLOYMENT_ID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "five", + "queue": { + "enabled": true + } + } + }, + "resources.jobs.four": { + "__id__": "[FOUR_ID]", + "state": { + "deployment": { + "deployment_id": "[DEPLOYMENT_ID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "four", + "queue": { + "enabled": true + } + } + }, + "resources.jobs.one": { + "__id__": "[ONE_ID]", + "state": { + "deployment": { + "deployment_id": "[DEPLOYMENT_ID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "one", + "queue": { + "enabled": true + } + } + }, + "resources.jobs.three": { + "__id__": "[THREE_ID]", + "state": { + "deployment": { + "deployment_id": "[DEPLOYMENT_ID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "three", + "queue": { + "enabled": true + } + } + }, + "resources.jobs.two": { + "__id__": "[TWO_ID]", + "state": { + "deployment": { + "deployment_id": "[DEPLOYMENT_ID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "two", + "queue": { + "enabled": true + } + } + } + } } diff --git a/acceptance/bundle/dms/no-drift/output.txt b/acceptance/bundle/dms/no-drift/output.txt index c2fe06a747e..14f650c01dd 100644 --- a/acceptance/bundle/dms/no-drift/output.txt +++ b/acceptance/bundle/dms/no-drift/output.txt @@ -35,7 +35,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "lineage": "[UUID]", "serial": 1, "features": { - "deployment_history": {} + "deployment_history_shadow": {} } } @@ -50,9 +50,52 @@ Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged "lineage": "[UUID]", "serial": 1, "features": { - "deployment_history": {} + "deployment_history_shadow": {} }, - "state": {} + "state": { + "resources.jobs.foo": { + "__id__": "[FOO_ID]", + "state": { + "deployment": { + "deployment_id": "[DEPLOYMENT_ID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-drift-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "foo", + "queue": { + "enabled": true + } + } + }, + "resources.pipelines.bar": { + "__id__": "[BAR_ID]", + "state": { + "catalog": "main", + "channel": "CURRENT", + "deployment": { + "deployment_id": "[DEPLOYMENT_ID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-drift-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" + }, + "edition": "ADVANCED", + "libraries": [ + { + "file": { + "path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-drift-[UNIQUE_NAME]/default/files/transform.py" + } + } + ], + "name": "bar-[UNIQUE_NAME]", + "schema": "default", + "serverless": true + } + } + } } === Plan without touching anything: the deployment stamp must not show as drift @@ -120,7 +163,7 @@ Resources: 0 created, 1 changed, 0 deleted, 1 unchanged } { "method": "PUT", - "path": "/api/2.0/pipelines/[UUID]", + "path": "/api/2.0/pipelines/[BAR_ID]", "body": { "catalog": "main", "channel": "CURRENT", @@ -181,7 +224,7 @@ Warning: The "development" property of pipelines is deprecated. Use "mode: devel "lineage": "[UUID]", "serial": 2, "features": { - "deployment_history": {} + "deployment_history_shadow": {} } } @@ -200,9 +243,53 @@ Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged "lineage": "[UUID]", "serial": 2, "features": { - "deployment_history": {} + "deployment_history_shadow": {} }, - "state": {} + "state": { + "resources.jobs.foo": { + "__id__": "[FOO_ID]", + "state": { + "deployment": { + "deployment_id": "[DEPLOYMENT_ID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-drift-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "foo", + "queue": { + "enabled": true + } + } + }, + "resources.pipelines.bar": { + "__id__": "[BAR_ID]", + "state": { + "catalog": "main", + "channel": "CURRENT", + "deployment": { + "deployment_id": "[DEPLOYMENT_ID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-drift-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "2" + }, + "development": true, + "edition": "ADVANCED", + "libraries": [ + { + "file": { + "path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-drift-[UNIQUE_NAME]/default/files/transform.py" + } + } + ], + "name": "bar-[UNIQUE_NAME]", + "schema": "default", + "serverless": true + } + } + } } === Plan again: no residual drift, and the version stamps are ignored diff --git a/acceptance/bundle/dms/no-drift/script b/acceptance/bundle/dms/no-drift/script index 66898d04cf9..503931ed646 100644 --- a/acceptance/bundle/dms/no-drift/script +++ b/acceptance/bundle/dms/no-drift/script @@ -2,6 +2,7 @@ envsubst < databricks.yml.tmpl > databricks.yml title "Deploy job foo and pipeline bar; both are recorded at version 1" trace $CLI bundle deploy +replace_ids.py deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-no-drift-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') add_repl "$deployment_id" DEPLOYMENT_ID diff --git a/acceptance/bundle/dms/no-resources/output.txt b/acceptance/bundle/dms/no-resources/output.txt index 884c22c6583..8380cc3885c 100644 --- a/acceptance/bundle/dms/no-resources/output.txt +++ b/acceptance/bundle/dms/no-resources/output.txt @@ -35,8 +35,9 @@ deployment_history_enabled true "plan_version": [PLAN_VERSION], "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", + "serial": 1, "features": { - "deployment_history": {} + "deployment_history_shadow": {} } } @@ -53,8 +54,9 @@ Plan: 0 to add, 0 to change, 0 to delete, 0 unchanged "state_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", + "serial": 1, "features": { - "deployment_history": {} + "deployment_history_shadow": {} }, "state": {} } @@ -70,10 +72,6 @@ Resources: 0 created, 0 changed, 0 deleted, 0 unchanged "method": "GET", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]" } -{ - "method": "GET", - "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/resources" -} { "method": "GET", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]" @@ -82,18 +80,10 @@ Resources: 0 created, 0 changed, 0 deleted, 0 unchanged "method": "GET", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/resources" } -{ - "method": "GET", - "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/resources" -} { "method": "GET", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]" } -{ - "method": "GET", - "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/resources" -} === The plan records the deployment_history feature >>> [CLI] bundle plan -o json @@ -101,8 +91,9 @@ Resources: 0 created, 0 changed, 0 deleted, 0 unchanged "plan_version": [PLAN_VERSION], "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", + "serial": 1, "features": { - "deployment_history": {} + "deployment_history_shadow": {} } } @@ -119,8 +110,9 @@ Plan: 0 to add, 0 to change, 0 to delete, 0 unchanged "state_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", + "serial": 1, "features": { - "deployment_history": {} + "deployment_history_shadow": {} }, "state": {} } diff --git a/acceptance/bundle/dms/operation-upload-fails-once/output.txt b/acceptance/bundle/dms/operation-upload-fails-once/output.txt index dc976dadead..d17e9ee26fb 100644 --- a/acceptance/bundle/dms/operation-upload-fails-once/output.txt +++ b/acceptance/bundle/dms/operation-upload-fails-once/output.txt @@ -14,7 +14,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "lineage": "[UUID]", "serial": 1, "features": { - "deployment_history": {} + "deployment_history_shadow": {} } } @@ -39,9 +39,18 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged "lineage": "[UUID]", "serial": 1, "features": { - "deployment_history": {} + "deployment_history_shadow": {} }, - "state": {} + "state": { + "resources.schemas.foo": { + "__id__": "main.dms_upload_fails_once_schema_[UNIQUE_NAME]", + "state": { + "catalog_name": "main", + "comment": "v1", + "name": "dms_upload_fails_once_schema_[UNIQUE_NAME]" + } + } + } } === A recreate records twice, and the first one fails: the second must still carry the sequence id staging left, because a rejected update earned no new one @@ -49,20 +58,15 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged >>> update_file.py databricks.yml catalog_name: main catalog_name: dms_other_[UNIQUE_NAME] ->>> musterr [CLI] bundle deploy --auto-approve +>>> [CLI] bundle deploy --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-upload-fails-once-[UNIQUE_NAME]/default/files... This action will result in the deletion or recreation of the following UC schemas. Any underlying data may be lost: recreate resources.schemas.foo Recreated schemas.foo -Error: recording operation for resources.schemas.foo: Fault injected by test. (500 INJECTED) - -Endpoint: PATCH [DATABRICKS_URL]/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/2/operations/schemas.foo?update_mask=state%2Cerror_message%2Cstatus -HTTP Status: 500 Internal Server Error -API error_code: INJECTED -API message: Fault injected by test. - +Warn: Failed to record deployment history: recording operation for resources.schemas.foo: Fault injected by test. Files: 2 uploaded, 0 deleted +Resources: 1 created, 0 changed, 1 deleted, 0 unchanged === The service lists the schema with the failure >>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[DEPLOYMENT_ID] -o json @@ -82,9 +86,18 @@ Files: 2 uploaded, 0 deleted "lineage": "[UUID]", "serial": 2, "features": { - "deployment_history": {} + "deployment_history_shadow": {} }, - "state": {} + "state": { + "resources.schemas.foo": { + "__id__": "dms_other_[UNIQUE_NAME].dms_upload_fails_once_schema_[UNIQUE_NAME]", + "state": { + "catalog_name": "dms_other_[UNIQUE_NAME]", + "comment": "v1", + "name": "dms_upload_fails_once_schema_[UNIQUE_NAME]" + } + } + } } >>> print_requests.py --dms //api/2.0/bundle @@ -139,3 +152,25 @@ Files: 2 uploaded, 0 deleted "completion_reason": "VERSION_COMPLETE_FAILURE" } } + +=== A completion failure does not block destroy or remote file cleanup +>>> fault.py POST /api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/3/complete 400 0 1 + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.schemas.foo + +This action will result in the deletion of the following UC schemas. Any underlying data may be lost: + delete resources.schemas.foo + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dms-upload-fails-once-[UNIQUE_NAME]/default + +Warn: Failed to complete deployment history version: Fault injected by test. +Destroy: 1 deleted + +>>> print_requests.py --dms //versions/3 --oneline +{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/3/operations/schemas.foo", "q": {"update_mask": "state,error_message,status"}, "body": {"error_message": "", "sequence_id": 0, "status": "OPERATION_STATUS_SUCCEEDED"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/3/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}} + +>>> musterr env MSYS_NO_PATHCONV=1 [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/dms-upload-fails-once-[UNIQUE_NAME]/default/state/resources.json +Error: Path (/Workspace/Users/[USERNAME]/.bundle/dms-upload-fails-once-[UNIQUE_NAME]/default/state/resources.json) doesn't exist. diff --git a/acceptance/bundle/dms/operation-upload-fails-once/script b/acceptance/bundle/dms/operation-upload-fails-once/script index 2b539134571..69dab957164 100644 --- a/acceptance/bundle/dms/operation-upload-fails-once/script +++ b/acceptance/bundle/dms/operation-upload-fails-once/script @@ -22,7 +22,7 @@ other_catalog="dms_other_${UNIQUE_NAME}" $CLI catalogs create "$other_catalog" &> LOG.catalog trap '$CLI catalogs delete "$other_catalog" --force >> LOG.catalog 2>&1' EXIT trace update_file.py databricks.yml "catalog_name: main" "catalog_name: ${other_catalog}" -trace musterr $CLI bundle deploy --auto-approve +trace $CLI bundle deploy --auto-approve title "The service lists the schema with the failure" deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-upload-fails-once-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') @@ -32,3 +32,10 @@ title "The raw recorded state after the failure" trace print_state.py --no-dms trace print_requests.py --dms //api/2.0/bundle + +title "A completion failure does not block destroy or remote file cleanup" +trace fault.py "POST /api/2.0/bundle/deployments/${deployment_id}/versions/3/complete" 400 0 1 +trace $CLI bundle destroy --auto-approve +trace print_requests.py --dms //versions/3 --oneline +trace musterr env MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-upload-fails-once-${UNIQUE_NAME}/default/state/resources.json" +rm -f "$OUT_REQUESTS" diff --git a/acceptance/bundle/dms/plan-without-dms/output.txt b/acceptance/bundle/dms/plan-without-dms/output.txt index ce1135f624d..db3e622639d 100644 --- a/acceptance/bundle/dms/plan-without-dms/output.txt +++ b/acceptance/bundle/dms/plan-without-dms/output.txt @@ -6,7 +6,7 @@ } === Enable DMS recording and try to deploy the non-DMS plan ->>> musterr [CLI] bundle deploy --plan tmp.plan.json +>>> musterr env DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true [CLI] bundle deploy --plan tmp.plan.json Error: this plan was created for a different set of state features than the target now has; run 'bundle plan' again diff --git a/acceptance/bundle/dms/plan-without-dms/script b/acceptance/bundle/dms/plan-without-dms/script index de35616b74e..8008168e6d9 100644 --- a/acceptance/bundle/dms/plan-without-dms/script +++ b/acceptance/bundle/dms/plan-without-dms/script @@ -5,11 +5,7 @@ $CLI bundle plan -o json > tmp.plan.json trace jq '{features}' < tmp.plan.json title "Enable DMS recording and try to deploy the non-DMS plan" -update_file.py databricks.yml 'bundle:' 'experimental: - deployment_history: true - -bundle:' -trace musterr $CLI bundle deploy --plan tmp.plan.json +trace musterr env DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true $CLI bundle deploy --plan tmp.plan.json trace print_requests.py //api/2.0/bundle --get rm -f tmp.plan.json "$OUT_REQUESTS" diff --git a/acceptance/bundle/dms/provenance/output.txt b/acceptance/bundle/dms/provenance/output.txt index 4decd10218b..3f00aae0d81 100644 --- a/acceptance/bundle/dms/provenance/output.txt +++ b/acceptance/bundle/dms/provenance/output.txt @@ -59,7 +59,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "lineage": "[UUID]", "serial": 1, "features": { - "deployment_history": {} + "deployment_history_shadow": {} } } @@ -100,7 +100,29 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged "lineage": "[UUID]", "serial": 1, "features": { - "deployment_history": {} + "deployment_history_shadow": {} }, - "state": {} + "state": { + "resources.jobs.foo": { + "__id__": "[FOO_ID]", + "state": { + "deployment": { + "deployment_id": "[DEPLOYMENT_ID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-provenance-[UNIQUE_NAME]/dev/state/metadata.json", + "version_id": "1" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 4, + "name": "[dev [USERNAME]] foo", + "queue": { + "enabled": true + }, + "tags": { + "dev": "[USERNAME]" + } + } + } + } } diff --git a/acceptance/bundle/dms/record-failure/output.txt b/acceptance/bundle/dms/record-failure/output.txt index 9f61f4a9274..4af2dd20f6d 100644 --- a/acceptance/bundle/dms/record-failure/output.txt +++ b/acceptance/bundle/dms/record-failure/output.txt @@ -18,7 +18,7 @@ Files: 2 uploaded, 0 deleted "lineage": "[UUID]", "serial": 1, "features": { - "deployment_history": {} + "deployment_history_shadow": {} } } @@ -92,12 +92,12 @@ Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged "lineage": "[UUID]", "serial": 1, "features": { - "deployment_history": {} + "deployment_history_shadow": {} }, "state": {} } -=== Redeploying from DMS state alone still creates the failed resource: it was never applied, so it must not be skipped as unchanged +=== Redeploying after pulling remote file state still creates the failed resource: it was never applied, so it must not be skipped as unchanged >>> [CLI] bundle plan create jobs.doomed diff --git a/acceptance/bundle/dms/record-failure/script b/acceptance/bundle/dms/record-failure/script index 806e7ab8b15..7e973b4b847 100644 --- a/acceptance/bundle/dms/record-failure/script +++ b/acceptance/bundle/dms/record-failure/script @@ -18,7 +18,7 @@ trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${d trace print_state.py --no-dms -title "Redeploying from DMS state alone still creates the failed resource: it was never applied, so it must not be skipped as unchanged" +title "Redeploying after pulling remote file state still creates the failed resource: it was never applied, so it must not be skipped as unchanged" rm -rf .databricks trace $CLI bundle plan # plan probes the state files concurrently, so the recorded order varies. Only the diff --git a/acceptance/bundle/dms/service-outage/databricks.yml.tmpl b/acceptance/bundle/dms/service-outage/databricks.yml.tmpl new file mode 100644 index 00000000000..1b91e6af7d2 --- /dev/null +++ b/acceptance/bundle/dms/service-outage/databricks.yml.tmpl @@ -0,0 +1,10 @@ +bundle: + name: dms-service-outage-$UNIQUE_NAME + +experimental: + deployment_history: true + +resources: + jobs: + foo: + name: original diff --git a/acceptance/bundle/dms/service-outage/out.test.toml b/acceptance/bundle/dms/service-outage/out.test.toml new file mode 100644 index 00000000000..88e6fbd847c --- /dev/null +++ b/acceptance/bundle/dms/service-outage/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/service-outage/output.txt b/acceptance/bundle/dms/service-outage/output.txt new file mode 100644 index 00000000000..54475eca787 --- /dev/null +++ b/acceptance/bundle/dms/service-outage/output.txt @@ -0,0 +1,297 @@ + +=== DMS outage: first-deploy +>>> fault.py GET /api/2.0/bundle/* 500 0 1000 + +>>> fault.py POST /api/2.0/bundle/* 500 0 1000 + +>>> fault.py PATCH /api/2.0/bundle/* 500 0 1000 + +>>> fault.py DELETE /api/2.0/bundle/* 500 0 1000 + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-service-outage-[UNIQUE_NAME]/default/files... +Warn: Could not enable deployment history for this deploy: failed to create deployment: Fault injected by test. +Created jobs.foo +Files: 2 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_state.py --no-dms +{ + "serial": 1, + "features": { + "deployment_history_shadow": {} + }, + "job": { + "id": "[FOO_ID]", + "name": "original" + } +} + +=== A fresh client updates the existing job using workspace file state +>>> update_file.py databricks.yml name: original name: after-outage + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-service-outage-[UNIQUE_NAME]/default/files... +Warn: Could not enable deployment history for this deploy: failed to create deployment: Fault injected by test. +Updated jobs.foo +Files: 2 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 0 unchanged + +>>> [CLI] jobs get [FOO_ID] -o json +{ + "job_id": [FOO_ID], + "name": "after-outage" +} + +>>> print_state.py --no-dms +{ + "serial": 2, + "features": { + "deployment_history_shadow": {} + }, + "job": { + "id": "[FOO_ID]", + "name": "after-outage" + } +} + +=== Planning still works with all DMS calls failing +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + +>>> print_requests.py --dms //api/2.0/bundle --get +{"method":"POST","path":"/api/2.0/bundle/deployments"} +{"method":"POST","path":"/api/2.0/bundle/deployments"} + +=== Destroy still deletes the job and workspace state +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.foo + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dms-service-outage-[UNIQUE_NAME]/default + +Warn: Could not record deployment history for this destroy: failed to create deployment: Fault injected by test. +Destroy: 1 deleted + +>>> musterr [CLI] jobs get [FOO_ID] +Error: Job [FOO_ID] does not exist. + +>>> musterr env MSYS_NO_PATHCONV=1 [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/dms-service-outage-[UNIQUE_NAME]/default/state/resources.json +Error: Path (/Workspace/Users/[USERNAME]/.bundle/dms-service-outage-[UNIQUE_NAME]/default/state/resources.json) doesn't exist. + +>>> print_requests.py --dms //api/2.0/bundle --get +{"method":"POST","path":"/api/2.0/bundle/deployments"} + +=== DMS outage: existing-deployment +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-service-outage-[UNIQUE_NAME]/default/files... +Created jobs.foo +Files: 2 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> update_file.py databricks.yml name: original name: before-outage + +>>> fault.py GET /api/2.0/bundle/* 500 0 1000 + +>>> fault.py POST /api/2.0/bundle/* 500 0 1000 + +>>> fault.py PATCH /api/2.0/bundle/* 500 0 1000 + +>>> fault.py DELETE /api/2.0/bundle/* 500 0 1000 + +>>> [CLI] bundle deploy +Warn: Could not read deployment history: Fault injected by test. +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-service-outage-[UNIQUE_NAME]/default/files... +Warn: Could not enable deployment history for this deploy: failed to create deployment: Fault injected by test. +Updated jobs.foo +Files: 1 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 0 unchanged + +>>> print_state.py --no-dms +{ + "serial": 2, + "features": { + "deployment_history_shadow": {} + }, + "job": { + "id": "[FOO_ID_2]", + "name": "before-outage" + } +} + +=== A fresh client updates the existing job using workspace file state +>>> update_file.py databricks.yml name: before-outage name: after-outage + +>>> [CLI] bundle deploy +Warn: Could not read deployment history: Fault injected by test. +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-service-outage-[UNIQUE_NAME]/default/files... +Warn: Could not enable deployment history for this deploy: failed to create deployment: Fault injected by test. +Updated jobs.foo +Files: 2 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 0 unchanged + +>>> [CLI] jobs get [FOO_ID_2] -o json +{ + "job_id": [FOO_ID_2], + "name": "after-outage" +} + +>>> print_state.py --no-dms +{ + "serial": 3, + "features": { + "deployment_history_shadow": {} + }, + "job": { + "id": "[FOO_ID_2]", + "name": "after-outage" + } +} + +=== Planning still works with all DMS calls failing +>>> [CLI] bundle plan +Warn: Could not read deployment history: Fault injected by test. +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + +>>> print_requests.py --dms //api/2.0/bundle --get +{"method":"GET","path":"/api/2.0/bundle/deployments/[DEPLOYMENT_ID]"} +{"method":"GET","path":"/api/2.0/bundle/deployments/[DEPLOYMENT_ID]"} +{"method":"GET","path":"/api/2.0/bundle/deployments/[DEPLOYMENT_ID]"} +{"method":"POST","path":"/api/2.0/bundle/deployments"} +{"method":"GET","path":"/api/2.0/bundle/deployments/[DEPLOYMENT_ID]"} +{"method":"GET","path":"/api/2.0/bundle/deployments/[DEPLOYMENT_ID]"} +{"method":"GET","path":"/api/2.0/bundle/deployments/[DEPLOYMENT_ID]"} +{"method":"POST","path":"/api/2.0/bundle/deployments"} +{"method":"GET","path":"/api/2.0/bundle/deployments/[DEPLOYMENT_ID]"} +{"method":"GET","path":"/api/2.0/bundle/deployments/[DEPLOYMENT_ID]"} +{"method":"GET","path":"/api/2.0/bundle/deployments/[DEPLOYMENT_ID]"} + +=== Destroy still deletes the job and workspace state +>>> [CLI] bundle destroy --auto-approve +Warn: Could not read deployment history: Fault injected by test. +The following resources will be deleted: + delete resources.jobs.foo + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dms-service-outage-[UNIQUE_NAME]/default + +Warn: Could not record deployment history for this destroy: failed to create deployment: Fault injected by test. +Destroy: 1 deleted + +>>> musterr [CLI] jobs get [FOO_ID_2] +Error: Job [FOO_ID_2] does not exist. + +>>> musterr env MSYS_NO_PATHCONV=1 [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/dms-service-outage-[UNIQUE_NAME]/default/state/resources.json +Error: Path (/Workspace/Users/[USERNAME]/.bundle/dms-service-outage-[UNIQUE_NAME]/default/state/resources.json) doesn't exist. + +>>> print_requests.py --dms //api/2.0/bundle --get +{"method":"GET","path":"/api/2.0/bundle/deployments/[DEPLOYMENT_ID]"} +{"method":"GET","path":"/api/2.0/bundle/deployments/[DEPLOYMENT_ID]"} +{"method":"GET","path":"/api/2.0/bundle/deployments/[DEPLOYMENT_ID]"} +{"method":"POST","path":"/api/2.0/bundle/deployments"} + +=== DMS outage: during-deploy +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-service-outage-[UNIQUE_NAME]/default/files... +Created jobs.foo +Files: 2 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> update_file.py databricks.yml name: original name: before-outage + +>>> fault.py GET /api/2.0/bundle/* 500 1 1000 + +>>> fault.py POST /api/2.0/bundle/* 500 1 1000 + +>>> fault.py PATCH /api/2.0/bundle/* 500 0 1000 + +>>> fault.py DELETE /api/2.0/bundle/* 500 0 1000 + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-service-outage-[UNIQUE_NAME]/default/files... +Updated jobs.foo +Warn: Failed to record deployment history: recording operation for resources.jobs.foo: Fault injected by test. +Files: 1 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 0 unchanged +Warn: Failed to complete deployment history version: Fault injected by test. + +>>> print_state.py --no-dms +{ + "serial": 2, + "features": { + "deployment_history_shadow": {} + }, + "job": { + "id": "[FOO_ID_3]", + "name": "before-outage" + } +} + +=== A fresh client updates the existing job using workspace file state +>>> update_file.py databricks.yml name: before-outage name: after-outage + +>>> [CLI] bundle deploy +Warn: Could not read deployment history: Fault injected by test. +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-service-outage-[UNIQUE_NAME]/default/files... +Warn: Could not enable deployment history for this deploy: failed to create deployment: Fault injected by test. +Updated jobs.foo +Files: 2 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 0 unchanged + +>>> [CLI] jobs get [FOO_ID_3] -o json +{ + "job_id": [FOO_ID_3], + "name": "after-outage" +} + +>>> print_state.py --no-dms +{ + "serial": 3, + "features": { + "deployment_history_shadow": {} + }, + "job": { + "id": "[FOO_ID_3]", + "name": "after-outage" + } +} + +=== Planning still works with all DMS calls failing +>>> [CLI] bundle plan +Warn: Could not read deployment history: Fault injected by test. +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + +>>> print_requests.py --dms //api/2.0/bundle --get +{"method":"GET","path":"/api/2.0/bundle/deployments/[DEPLOYMENT_ID]"} +{"method":"POST","path":"/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions"} +{"method":"PATCH","path":"/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/2/operations/jobs.foo"} +{"method":"POST","path":"/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/2/complete"} +{"method":"GET","path":"/api/2.0/bundle/deployments/[DEPLOYMENT_ID]"} +{"method":"GET","path":"/api/2.0/bundle/deployments/[DEPLOYMENT_ID]"} +{"method":"GET","path":"/api/2.0/bundle/deployments/[DEPLOYMENT_ID]"} +{"method":"POST","path":"/api/2.0/bundle/deployments"} +{"method":"GET","path":"/api/2.0/bundle/deployments/[DEPLOYMENT_ID]"} +{"method":"GET","path":"/api/2.0/bundle/deployments/[DEPLOYMENT_ID]"} +{"method":"GET","path":"/api/2.0/bundle/deployments/[DEPLOYMENT_ID]"} + +=== Destroy still deletes the job and workspace state +>>> [CLI] bundle destroy --auto-approve +Warn: Could not read deployment history: Fault injected by test. +The following resources will be deleted: + delete resources.jobs.foo + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dms-service-outage-[UNIQUE_NAME]/default + +Warn: Could not record deployment history for this destroy: failed to create deployment: Fault injected by test. +Destroy: 1 deleted + +>>> musterr [CLI] jobs get [FOO_ID_3] +Error: Job [FOO_ID_3] does not exist. + +>>> musterr env MSYS_NO_PATHCONV=1 [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/dms-service-outage-[UNIQUE_NAME]/default/state/resources.json +Error: Path (/Workspace/Users/[USERNAME]/.bundle/dms-service-outage-[UNIQUE_NAME]/default/state/resources.json) doesn't exist. + +>>> print_requests.py --dms //api/2.0/bundle --get +{"method":"GET","path":"/api/2.0/bundle/deployments/[DEPLOYMENT_ID]"} +{"method":"GET","path":"/api/2.0/bundle/deployments/[DEPLOYMENT_ID]"} +{"method":"GET","path":"/api/2.0/bundle/deployments/[DEPLOYMENT_ID]"} +{"method":"POST","path":"/api/2.0/bundle/deployments"} diff --git a/acceptance/bundle/dms/service-outage/script b/acceptance/bundle/dms/service-outage/script new file mode 100644 index 00000000000..e094bde6ea5 --- /dev/null +++ b/acceptance/bundle/dms/service-outage/script @@ -0,0 +1,47 @@ +for outage in first-deploy existing-deployment during-deploy; do + title "DMS outage: ${outage}" + envsubst < databricks.yml.tmpl > databricks.yml + job_name=original + + for method in GET POST PATCH DELETE; do + fault.py "${method} /api/2.0/bundle/*" 500 0 0 + done + + if [ "$outage" != first-deploy ]; then + trace $CLI bundle deploy + job_name=before-outage + trace update_file.py databricks.yml 'name: original' "name: ${job_name}" + fi + rm -f "$OUT_REQUESTS" + + # Let the current deployment and new version be read/created before DMS goes down. + offset=0 + if [ "$outage" = during-deploy ]; then + offset=1 + fi + trace fault.py "GET /api/2.0/bundle/*" 500 "$offset" 1000 + trace fault.py "POST /api/2.0/bundle/*" 500 "$offset" 1000 + trace fault.py "PATCH /api/2.0/bundle/*" 500 0 1000 + trace fault.py "DELETE /api/2.0/bundle/*" 500 0 1000 + trace $CLI bundle deploy + job_id=$(DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=false read_id.py foo) + trace print_state.py --no-dms | jq '{serial, features, job: (.state["resources.jobs.foo"] | {id: .__id__, name: .state.name})}' + + title "A fresh client updates the existing job using workspace file state" + trace update_file.py databricks.yml "name: ${job_name}" 'name: after-outage' + rm -rf .databricks + trace $CLI bundle deploy + trace $CLI jobs get "$job_id" -o json | jq '{job_id, name: .settings.name}' + trace print_state.py --no-dms | jq '{serial, features, job: (.state["resources.jobs.foo"] | {id: .__id__, name: .state.name})}' + + title "Planning still works with all DMS calls failing" + rm -rf .databricks + trace $CLI bundle plan + trace print_requests.py --dms //api/2.0/bundle --get | jq -c '{method, path}' + + title "Destroy still deletes the job and workspace state" + trace $CLI bundle destroy --auto-approve + trace musterr $CLI jobs get "$job_id" + trace musterr env MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-service-outage-${UNIQUE_NAME}/default/state/resources.json" + trace print_requests.py --dms //api/2.0/bundle --get | jq -c '{method, path}' +done diff --git a/acceptance/bundle/dms/service-outage/test.toml b/acceptance/bundle/dms/service-outage/test.toml new file mode 100644 index 00000000000..13abcee0b4b --- /dev/null +++ b/acceptance/bundle/dms/service-outage/test.toml @@ -0,0 +1,3 @@ +# Fault injection requires the local testserver. +Cloud = false +Env.DATABRICKS_BUNDLE_RETRY_INTERVAL_MS = "0" diff --git a/acceptance/bundle/dms/stale-local-state/databricks.yml.tmpl b/acceptance/bundle/dms/stale-local-state/databricks.yml.tmpl index a9b1eada1f9..564b0d03c87 100644 --- a/acceptance/bundle/dms/stale-local-state/databricks.yml.tmpl +++ b/acceptance/bundle/dms/stale-local-state/databricks.yml.tmpl @@ -1,9 +1,6 @@ bundle: name: dms-stale-local-state-$UNIQUE_NAME -experimental: - deployment_history: true - resources: jobs: foo: diff --git a/acceptance/bundle/dms/stale-local-state/output.txt b/acceptance/bundle/dms/stale-local-state/output.txt index 5c2b2225a84..b1f26591d10 100644 --- a/acceptance/bundle/dms/stale-local-state/output.txt +++ b/acceptance/bundle/dms/stale-local-state/output.txt @@ -4,35 +4,41 @@ {"serial":1,"features":null} >>> jq -c {serial, features} .databricks/resources.v2.json -{"serial":2,"features":{"deployment_history":{}}} +{"serial":2,"features":{"deployment_history_shadow":{}}} -=== Local recorded serial 3 ahead of DMS version 2 is rejected +=== File serial 3 ahead of DMS version 2 is allowed >>> update_file.py .databricks/bundle/default/resources.json "serial": 2 "serial": 3 ->>> musterr [CLI] bundle deploy -Error: inconsistent deployment state: resource state serial (3) is ahead of deployment history version (2) +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged - -=== 1. Config false and env false: newer remote state replaces the local copy and reads DMS +=== 1. Config false and env false: newer remote file state replaces the local copy >>> cp .databricks/resources.v1.json .databricks/bundle/default/resources.json >>> DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=false [CLI] bundle summary --force-pull -o json -Warn: Deployment history setting (false) does not match the existing state (true). Using the existing state. {"job_id":"[FOO_ID]"} >>> cmp .databricks/resources.v2.json .databricks/bundle/default/resources.json >>> print_requests.py --dms //resources --get --oneline -{"method": "GET", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/resources"} -=== 2. Config removed and env false: restore the same snapshot, then read remote DMS state +=== 2. Config removed and env false: restore the same snapshot, then read the remote file state >>> cp .databricks/resources.v1.json .databricks/bundle/default/resources.json >>> DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=false [CLI] bundle summary --force-pull -o json -Warn: Deployment history setting (false) does not match the existing state (true). Using the existing state. {"job_id":"[FOO_ID]"} >>> cmp .databricks/resources.v2.json .databricks/bundle/default/resources.json >>> print_requests.py --dms //resources --get --oneline -{"method": "GET", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/resources"} + +=== Destroy follows newer local state even when configuration enables history +>>> env DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.foo + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dms-stale-local-state-[UNIQUE_NAME]/default + +Destroy: 1 deleted + +>>> print_requests.py --dms //api/2.0/bundle --oneline diff --git a/acceptance/bundle/dms/stale-local-state/script b/acceptance/bundle/dms/stale-local-state/script index 38d0a3725b4..7f0cfbf9663 100644 --- a/acceptance/bundle/dms/stale-local-state/script +++ b/acceptance/bundle/dms/stale-local-state/script @@ -2,37 +2,45 @@ envsubst < databricks.yml.tmpl > databricks.yml title "Setup (LOG.setup): reconstruct full version 1 without a history marker, then deploy version 2" { - $CLI bundle deploy + DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true $CLI bundle deploy DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true print_state.py > .databricks/resources.v1.json DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true replace_ids.py update_file.py databricks.yml 'name: foo' 'name: updated' - $CLI bundle deploy + DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true $CLI bundle deploy cp .databricks/bundle/default/resources.json .databricks/resources.v2.json deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-stale-local-state-${UNIQUE_NAME}/default/state/resources.deployment.json" -o json | jq -r '.object_id') add_repl "$deployment_id" DEPLOYMENT_ID } &> LOG.setup +# The environment flag records best-effort history; YAML selects the same mode. +update_file.py databricks.yml 'bundle:' 'experimental: + deployment_history: true + +bundle:' trace jq -c '{serial, features}' .databricks/resources.v1.json trace jq -c '{serial, features}' .databricks/resources.v2.json -title "Local recorded serial 3 ahead of DMS version 2 is rejected" +title "File serial 3 ahead of DMS version 2 is allowed" trace update_file.py .databricks/bundle/default/resources.json '"serial": 2' '"serial": 3' -trace musterr $CLI bundle deploy +trace $CLI bundle plan rm -f "$OUT_REQUESTS" -title "1. Config false and env false: newer remote state replaces the local copy and reads DMS" +title "1. Config false and env false: newer remote file state replaces the local copy" update_file.py databricks.yml 'deployment_history: true' 'deployment_history: false' trace cp .databricks/resources.v1.json .databricks/bundle/default/resources.json trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=false $CLI bundle summary --force-pull -o json | jq -c '{job_id: .resources.jobs.foo.id}' trace cmp .databricks/resources.v2.json .databricks/bundle/default/resources.json trace print_requests.py --dms //resources --get --oneline -title "2. Config removed and env false: restore the same snapshot, then read remote DMS state" +title "2. Config removed and env false: restore the same snapshot, then read the remote file state" update_file.py databricks.yml $'experimental:\n deployment_history: false\n\n' '' trace cp .databricks/resources.v1.json .databricks/bundle/default/resources.json trace DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=false $CLI bundle summary --force-pull -o json | jq -c '{job_id: .resources.jobs.foo.id}' trace cmp .databricks/resources.v2.json .databricks/bundle/default/resources.json trace print_requests.py --dms //resources --get --oneline -DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true $CLI bundle destroy --auto-approve &> LOG.cleanup +title "Destroy follows newer local state even when configuration enables history" +jq 'del(.features) | .serial = 3' .databricks/resources.v2.json > .databricks/bundle/default/resources.json +trace env DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=true $CLI bundle destroy --auto-approve +trace print_requests.py --dms //api/2.0/bundle --oneline rm -f "$OUT_REQUESTS" diff --git a/acceptance/bundle/dms/stale-plan/output.txt b/acceptance/bundle/dms/stale-plan/output.txt index e9fbc70f832..ebc25578f19 100644 --- a/acceptance/bundle/dms/stale-plan/output.txt +++ b/acceptance/bundle/dms/stale-plan/output.txt @@ -5,7 +5,7 @@ "plan_version": [PLAN_VERSION], "cli_version": "[CLI_VERSION]", "features": { - "deployment_history": {} + "deployment_history_shadow": {} } } @@ -35,9 +35,28 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "lineage": "[UUID]", "serial": 1, "features": { - "deployment_history": {} + "deployment_history_shadow": {} }, - "state": {} + "state": { + "resources.jobs.foo": { + "__id__": "[FOO_ID]", + "state": { + "deployment": { + "deployment_id": "[DEPLOYMENT_ID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-stale-plan-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "foo", + "queue": { + "enabled": true + } + } + } + } } === A saved plan carries the version it was built against diff --git a/acceptance/bundle/dms/state-features/output.txt b/acceptance/bundle/dms/state-features/output.txt index 342fbfe6a44..576a9c9f6c1 100644 --- a/acceptance/bundle/dms/state-features/output.txt +++ b/acceptance/bundle/dms/state-features/output.txt @@ -1,5 +1,5 @@ -=== Deploy records version 3 and the deployment_history feature, with resources held in DMS +=== Deploy keeps resource state in the file and enforces best-effort history >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-state-features-[UNIQUE_NAME]/default/files... Created jobs.foo @@ -10,31 +10,58 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged { "state_version": 3, "features": { - "deployment_history": {} + "deployment_history_shadow": {} }, - "state": {} + "state": { + "resources.jobs.foo": { + "__id__": "[FOO_ID]", + "state": { + "deployment": { + "deployment_id": "[DEPLOYMENT_ID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-state-features-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "state-features", + "queue": { + "enabled": true + } + } + } + } } -=== Reopening version 3 reads resources from DMS and produces a no-op plan +=== Reopening reads resources from the file and produces a no-op plan >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged >>> print_requests.py --dms //resources --get --oneline -{"method": "GET", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/resources"} -=== Disabling deployment history is refused by plan and deploy, while destroy keeps using DMS from state +=== Disabling deployment history allows plan and deploy, and removes the file marker >>> env DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=false [CLI] bundle plan -Error: deployment history setting (false) does not match the existing state (true) - -Update experimental.deployment_history to match the existing deployment, or run "databricks bundle destroy" to start over +update jobs.foo +Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged >>> env DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=false [CLI] bundle deploy -Error: deployment history setting (false) does not match the existing state (true) +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-state-features-[UNIQUE_NAME]/default/files... +Updated jobs.foo +Files: 1 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 0 unchanged + +>>> jq .features // {} .databricks/bundle/default/resources.json +{} -Update experimental.deployment_history to match the existing deployment, or run "databricks bundle destroy" to start over +>>> print_requests.py --dms //api/2.0/bundle --oneline +=== Removing the YAML setting also leaves history disabled +>>> env DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=false [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged +=== Destroy also uses the disabled setting >>> env DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=false [CLI] bundle destroy --auto-approve The following resources will be deleted: delete resources.jobs.foo @@ -44,7 +71,3 @@ All files and directories at the following location will be deleted: /Workspace/ Destroy: 1 deleted >>> print_requests.py --dms //api/2.0/bundle --oneline -{"method": "POST", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions", "q": {"version_id": "2"}, "body": {"cli_version": "[CLI_VERSION]", "operations": [{"action_type": "OPERATION_ACTION_TYPE_DELETE", "resource_key": "jobs.foo"}], "previous_version_id": "1", "version_type": "VERSION_TYPE_DESTROY"}} -{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/2/operations/jobs.foo", "q": {"update_mask": "state,error_message,status"}, "body": {"error_message": "", "sequence_id": 0, "status": "OPERATION_STATUS_SUCCEEDED"}} -{"method": "POST", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/2/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}} -{"method": "DELETE", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]"} diff --git a/acceptance/bundle/dms/state-features/script b/acceptance/bundle/dms/state-features/script index 349845853cc..0f632d18fc0 100644 --- a/acceptance/bundle/dms/state-features/script +++ b/acceptance/bundle/dms/state-features/script @@ -1,21 +1,30 @@ envsubst < databricks.yml.tmpl > databricks.yml -title "Deploy records version 3 and the deployment_history feature, with resources held in DMS" +title "Deploy keeps resource state in the file and enforces best-effort history" trace $CLI bundle deploy +replace_ids.py trace jq '{state_version, features, state}' .databricks/bundle/default/resources.json deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-state-features-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') add_repl "$deployment_id" DEPLOYMENT_ID rm -f "$OUT_REQUESTS" -title "Reopening version 3 reads resources from DMS and produces a no-op plan" +title "Reopening reads resources from the file and produces a no-op plan" trace $CLI bundle plan trace print_requests.py --dms //resources --get --oneline -title "Disabling deployment history is refused by plan and deploy, while destroy keeps using DMS from state" +title "Disabling deployment history allows plan and deploy, and removes the file marker" update_file.py databricks.yml 'deployment_history: true' 'deployment_history: false' for command in plan deploy; do - musterr trace env DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=false $CLI bundle "$command" + trace env DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=false $CLI bundle "$command" done +trace jq '.features // {}' .databricks/bundle/default/resources.json +trace print_requests.py --dms //api/2.0/bundle --oneline + +title "Removing the YAML setting also leaves history disabled" +update_file.py databricks.yml $'experimental:\n deployment_history: false\n' '' +trace env DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=false $CLI bundle plan + +title "Destroy also uses the disabled setting" trace env DATABRICKS_BUNDLE_DEPLOYMENT_HISTORY=false $CLI bundle destroy --auto-approve trace print_requests.py --dms //api/2.0/bundle --oneline diff --git a/acceptance/bundle/dms/successful-recreate/output.txt b/acceptance/bundle/dms/successful-recreate/output.txt index f2ecc3f8cbc..41c9d29e8e2 100644 --- a/acceptance/bundle/dms/successful-recreate/output.txt +++ b/acceptance/bundle/dms/successful-recreate/output.txt @@ -67,7 +67,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "lineage": "[UUID]", "serial": 1, "features": { - "deployment_history": {} + "deployment_history_shadow": {} } } @@ -92,9 +92,18 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged "lineage": "[UUID]", "serial": 1, "features": { - "deployment_history": {} + "deployment_history_shadow": {} }, - "state": {} + "state": { + "resources.schemas.foo": { + "__id__": "main.dms_successful_recreate_schema_[UNIQUE_NAME]", + "state": { + "catalog_name": "main", + "comment": "v1", + "name": "dms_successful_recreate_schema_[UNIQUE_NAME]" + } + } + } } === Recreate writes state twice - the entry is dropped, then the new resource is saved @@ -170,7 +179,7 @@ Resources: 1 created, 0 changed, 1 deleted, 0 unchanged "lineage": "[UUID]", "serial": 2, "features": { - "deployment_history": {} + "deployment_history_shadow": {} } } @@ -195,9 +204,18 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged "lineage": "[UUID]", "serial": 2, "features": { - "deployment_history": {} + "deployment_history_shadow": {} }, - "state": {} + "state": { + "resources.schemas.foo": { + "__id__": "dms_other_[UNIQUE_NAME].dms_successful_recreate_schema_[UNIQUE_NAME]", + "state": { + "catalog_name": "dms_other_[UNIQUE_NAME]", + "comment": "v1", + "name": "dms_successful_recreate_schema_[UNIQUE_NAME]" + } + } + } } === Destroy: the delete is recorded with the id and no state diff --git a/acceptance/bundle/dms/version-never-created/output.txt b/acceptance/bundle/dms/version-never-created/output.txt index 989744df818..e4feb0d5398 100644 --- a/acceptance/bundle/dms/version-never-created/output.txt +++ b/acceptance/bundle/dms/version-never-created/output.txt @@ -1,29 +1,25 @@ -=== The first version fails, so no deployment record exists - only the node naming its ID ->>> musterr [CLI] bundle deploy +=== Version creation fails, but file-backed deployment still succeeds +>>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default/files... -Error: failed to create deployment version: Internal error (500 INTERNAL_ERROR) - -Endpoint: POST [DATABRICKS_URL]/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions?version_id=1 -HTTP Status: 500 Internal Server Error -API error_code: INTERNAL_ERROR -API message: Internal error - +Warn: Could not enable deployment history for this deploy: failed to create deployment version: Internal error +Created jobs.foo Files: 1 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] bundle plan -o json { "plan_version": [PLAN_VERSION], "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 1, "features": { - "deployment_history": {} + "deployment_history_shadow": {} } } >>> [CLI] bundle plan -create jobs.foo - -Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged >>> MSYS_NO_PATHCONV=1 [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default/state/resources.deployment.json { @@ -34,27 +30,22 @@ Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged {"method": "POST", "path": "/api/2.0/bundle/deployments", "body": {"display_name": "dms-version-never-created", "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default/state", "target_name": "default", "workspace_info": {"file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default/files", "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default"}}} {"method": "POST", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "operations": [{"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_key": "jobs.foo"}], "version_type": "VERSION_TYPE_DEPLOY"}} {"method": "GET", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]"} -{"method": "GET", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/resources"} {"method": "GET", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]"} -{"method": "GET", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/resources"} >>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[DEPLOYMENT_ID] -o json [] -=== The next deploy reuses the ID that node names and retries version 1, rather than creating a second deployment ->>> musterr [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default/files... -Error: failed to create deployment version: Internal error (500 INTERNAL_ERROR) +=== The next resource update also succeeds while history version creation fails +>>> update_file.py databricks.yml name: foo name: renamed -Endpoint: POST [DATABRICKS_URL]/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions?version_id=1 -HTTP Status: 500 Internal Server Error -API error_code: INTERNAL_ERROR -API message: Internal error - -Files: 0 uploaded, 0 deleted +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default/files... +Warn: Could not enable deployment history for this deploy: failed to create deployment version: Internal error +Updated jobs.foo +Files: 1 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 0 unchanged >>> print_requests.py --dms //api/2.0/bundle --get --oneline {"method": "GET", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/resources"} {"method": "GET", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]"} -{"method": "GET", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/resources"} -{"method": "POST", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "operations": [{"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_key": "jobs.foo"}], "version_type": "VERSION_TYPE_DEPLOY"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions", "q": {"version_id": "2"}, "body": {"cli_version": "[CLI_VERSION]", "operations": [{"action_type": "OPERATION_ACTION_TYPE_UPDATE", "resource_key": "jobs.foo"}], "version_type": "VERSION_TYPE_DEPLOY"}} diff --git a/acceptance/bundle/dms/version-never-created/script b/acceptance/bundle/dms/version-never-created/script index 9498fa05eff..5601cb2129b 100644 --- a/acceptance/bundle/dms/version-never-created/script +++ b/acceptance/bundle/dms/version-never-created/script @@ -1,5 +1,5 @@ -title "The first version fails, so no deployment record exists - only the node naming its ID" -trace musterr $CLI bundle deploy +title "Version creation fails, but file-backed deployment still succeeds" +trace $CLI bundle deploy dms_plan trace MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-version-never-created/default/state/resources.deployment.json" | jq '{object_type}' trace print_requests.py --dms //api/2.0/bundle --get --oneline @@ -8,6 +8,7 @@ deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/$ add_repl "$deployment_id" DEPLOYMENT_ID trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' -title "The next deploy reuses the ID that node names and retries version 1, rather than creating a second deployment" -trace musterr $CLI bundle deploy +title "The next resource update also succeeds while history version creation fails" +trace update_file.py databricks.yml 'name: foo' 'name: renamed' +trace $CLI bundle deploy trace print_requests.py --dms //api/2.0/bundle --get --oneline diff --git a/bundle/config/experimental.go b/bundle/config/experimental.go index 6fd5511b79b..cca59fc29c2 100644 --- a/bundle/config/experimental.go +++ b/bundle/config/experimental.go @@ -54,8 +54,8 @@ type Experimental struct { // service (DMS), which records deployment history and tracks what changed // across deployments. // - // Only for bundles with no deployed resources yet: DMS becomes the source of - // truth for their state. + // Resource state remains in resources.json. History uploads are best effort: + // DMS errors do not fail file-backed deployments. // An explicit value takes priority over the environment setting. DeploymentHistory *bool `json:"deployment_history,omitempty"` } diff --git a/bundle/direct/apply.go b/bundle/direct/apply.go index d4f4a2ffb41..9cdd5a4f505 100644 --- a/bundle/direct/apply.go +++ b/bundle/direct/apply.go @@ -265,7 +265,7 @@ func (d *DeploymentUnit) Delete(ctx context.Context, db *dstate.DeploymentState, log.Warnf(ctx, "Ignoring permission error when deleting %s id=%s: %s", d.ResourceKey, oldID, err) } else if d.deleteConfirmedGone(ctx, oldID) { log.Warnf(ctx, "Treating %s id=%s as already deleted despite delete error: %s", d.ResourceKey, oldID, err) - } else if db.IsDeploymentMetadataService() { + } else if db.RecordsHistory() { // When using the deployment metadata service, record the error. err = fmt.Errorf("deleting id=%s: %w", oldID, err) db.RecordFailure(d.ResourceKey, oldID, err) diff --git a/bundle/direct/dstate/mode.go b/bundle/direct/dstate/mode.go index c7d6440573c..f4938af0ef5 100644 --- a/bundle/direct/dstate/mode.go +++ b/bundle/direct/dstate/mode.go @@ -12,23 +12,19 @@ const ( // Finalize merges them into resources.json. DMS is unused. WritingFile - // WritingDMS reads and writes resource state through DMS without creating a WAL. + // WritingDeploymentHistoryOnly reads and writes resource state through DMS without creating a WAL. // DMS errors fail the deployment. - WritingDMS + WritingDeploymentHistoryOnly - // WritingFileWithHistory writes state through WAL to resources.json and records DMS history. + // WritingDeploymentHistoryShadow writes state through WAL to resources.json and records DMS history. // DMS errors and drain timeouts let the deployment continue. - WritingFileWithHistory - - // WritingFileWithHistoryEnforced writes state through WAL to resources.json and records DMS history. - // DMS errors and drain timeouts let the deployment continue. - // Persists deployment_history_best_effort to reject clients that cannot honor history recording. - WritingFileWithHistoryEnforced + // Persists deployment_history_shadow to reject clients that cannot honor history recording. + WritingDeploymentHistoryShadow ) func (m Mode) isWritable() bool { switch m { - case WritingFile, WritingDMS, WritingFileWithHistory, WritingFileWithHistoryEnforced: + case WritingFile, WritingDeploymentHistoryOnly, WritingDeploymentHistoryShadow: return true default: return false diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index 411eeb2af3d..384246913f4 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -47,9 +47,9 @@ const ( // deploying such a state without recording is refused. const FeatureDeploymentHistory = "deployment_history" -// FeatureDeploymentHistoryBestEffort requires writers to attempt DMS history recording. +// FeatureDeploymentHistoryShadow requires writers to attempt DMS history recording. // Resource state stays in resources.json; recording errors only warn. -const FeatureDeploymentHistoryBestEffort = "deployment_history_best_effort" +const FeatureDeploymentHistoryShadow = "deployment_history_shadow" // featuresDocURL explains the state-features mechanism in the error shown when this CLI refuses a // state that depends on a feature it does not recognize. @@ -59,8 +59,8 @@ const featuresDocURL = "https://docs.databricks.com/aws/en/dev-tools/bundles/sta // any feature not listed here is refused (see checkStateFeatures), so a newer CLI's feature is not // silently clobbered by this one. var recognizedFeatures = map[string]struct{}{ - FeatureDeploymentHistory: {}, - FeatureDeploymentHistoryBestEffort: {}, + FeatureDeploymentHistory: {}, + FeatureDeploymentHistoryShadow: {}, } // assertNoUnsupportedFeatures refuses a state that depends on a feature this CLI does not recognize. @@ -93,6 +93,9 @@ type DeploymentState struct { mu sync.Mutex walFile *os.File + // persistedFeatures tracks disk markers separately from configuration changes in Data. + persistedFeatures map[string]struct{} + // mode controls where resource state is read and written, whether DMS // history is recorded, and whether DMS errors fail the deployment. mode Mode @@ -181,7 +184,7 @@ func (db *DeploymentState) InitializeOperationBuffer(ctx context.Context, deploy db.mu.Lock() defer db.mu.Unlock() options := dms.BufferOptions{ - BestEffort: db.mode == WritingFileWithHistory || db.mode == WritingFileWithHistoryEnforced, + BestEffort: db.mode == WritingDeploymentHistoryShadow, OperationCount: operationCount, } db.operationBuffer = dms.StartOperationBuffer(ctx, deploymentID, versionID, options) @@ -202,7 +205,7 @@ func (db *DeploymentState) getOperationBuffer() *dms.OperationBuffer { // remains recoverable from the WAL. func (db *DeploymentState) RecordingError() error { if !db.IsDeploymentMetadataService() { - // TODO: Warn users when history recording fails in file-backed history modes. + // TODO: Warn users when history recording fails in deployment history shadow modes. return nil } buf := db.getOperationBuffer() @@ -230,7 +233,6 @@ func (db *DeploymentState) CompleteVersion(ctx context.Context, success bool) (b } db.versionCompleted = true deploymentID, versionID := db.DeploymentID, db.VersionID - bestEffort := db.mode == WritingFileWithHistory || db.mode == WritingFileWithHistoryEnforced db.reset() db.mu.Unlock() @@ -251,10 +253,6 @@ func (db *DeploymentState) CompleteVersion(ctx context.Context, success bool) (b CompletionReason: reason, }) if err != nil { - if bestEffort { - log.Warnf(ctx, "Failed to complete deployment history version: %s", err) - return false, nil - } return false, err } log.Infof(ctx, "Completed deployment version: deployment=%s version=%d reason=%s", deploymentID, versionID, reason) @@ -327,7 +325,7 @@ func (db *DeploymentState) SaveState(ctx context.Context, key, newID string, sta db.stateIDs[key] = newID } return err - case WritingDMS: + case WritingDeploymentHistoryOnly: db.Data.State[key] = entry db.stateIDs[key] = newID if buf := db.operationBuffer; buf != nil { @@ -339,7 +337,7 @@ func (db *DeploymentState) SaveState(ctx context.Context, key, newID string, sta } return nil - case WritingFileWithHistory, WritingFileWithHistoryEnforced: + case WritingDeploymentHistoryShadow: // File-backed modes record history only after the WAL append succeeds. if err := appendJSONLine(db.walFile, WALEntry{Key: key, Value: &entry}); err != nil { return err @@ -380,7 +378,7 @@ func (db *DeploymentState) DeleteState(ctx context.Context, key string, inProgre } delete(db.stateIDs, key) return nil - case WritingDMS: + case WritingDeploymentHistoryOnly: delete(db.Data.State, key) if buf := db.operationBuffer; buf != nil { buf.RecordOperation(ctx, key, inProgress, deletedID, nil) @@ -388,7 +386,7 @@ func (db *DeploymentState) DeleteState(ctx context.Context, key string, inProgre delete(db.stateIDs, key) return nil - case WritingFileWithHistory, WritingFileWithHistoryEnforced: + case WritingDeploymentHistoryShadow: if err := appendJSONLine(db.walFile, WALEntry{Key: key}); err != nil { return err } @@ -448,15 +446,15 @@ func (db *DeploymentState) StateFeatures() map[string]struct{} { } // GetSerial reports the counter used to validate saved plans. -// DMS modes use the service version; plain file modes use the file serial. +// WritingDeploymentHistoryOnly uses the service version; plain file modes use the file serial. // File modes with history use the larger counter because either can advance independently. func (db *DeploymentState) GetSerial() int { db.mu.Lock() defer db.mu.Unlock() switch db.mode { - case WritingDMS: + case WritingDeploymentHistoryOnly: return db.VersionID - case WritingFileWithHistory, WritingFileWithHistoryEnforced: + case ReadingFile, WritingDeploymentHistoryShadow: return max(db.VersionID, db.Data.Serial) default: return db.Data.Serial @@ -465,7 +463,7 @@ func (db *DeploymentState) GetSerial() int { // isDeploymentMetadataService is IsDeploymentMetadataService for callers already holding db.mu. func (db *DeploymentState) isDeploymentMetadataService() bool { - return db.mode == WritingDMS + return db.mode == WritingDeploymentHistoryOnly } // IsDeploymentMetadataService reports whether this deployment's resource state lives in the @@ -477,12 +475,15 @@ func (db *DeploymentState) IsDeploymentMetadataService() bool { return db.isDeploymentMetadataService() } -// RecordsHistory reports whether this mode records resource operations with DMS. +// RecordsHistory reports whether the loaded file state or selected write mode records DMS history. func (db *DeploymentState) RecordsHistory() bool { db.mu.Lock() defer db.mu.Unlock() switch db.mode { - case WritingDMS, WritingFileWithHistory, WritingFileWithHistoryEnforced: + case ReadingFile: + _, history := db.Data.Features[FeatureDeploymentHistoryShadow] + return history + case WritingDeploymentHistoryOnly, WritingDeploymentHistoryShadow: return true default: return false @@ -515,10 +516,10 @@ type WithRecovery bool // Open loads deployment state and enables reads or writes according to mode: // - ReadingFile: reads resources.json; state writes are disabled. // - WritingFile: reads resources.json and opens a WAL for state writes. -// - WritingDMS: reads and writes resources through DMS; no WAL is opened. -// - WritingFileWithHistory: behaves like WritingFile and also records DMS history. +// - WritingDeploymentHistoryOnly: reads and writes resources through DMS; no WAL is opened. +// - WritingDeploymentHistoryShadow: behaves like WritingFile and also records DMS history. // History errors do not fail the deployment. -// - WritingFileWithHistoryEnforced: also persists a feature marker that rejects +// - WritingDeploymentHistoryShadow: also persists a feature marker that rejects // older clients that do not support history recording. // // All modes read deployment metadata and feature markers from resources.json. @@ -557,6 +558,7 @@ func (db *DeploymentState) reset() { db.Path = "" db.Data = Database{} db.stateIDs = nil + db.persistedFeatures = nil // Telemetry and destroy still need the mode and IDs after version completion. // Clear the recording buffer once CompleteVersion has copied it. if db.versionCompleted { @@ -590,6 +592,7 @@ func (db *DeploymentState) unlockedOpen(ctx context.Context, path string, mode M return fmt.Errorf("migrating state %s: %w", path, err) } + db.persistedFeatures = maps.Clone(db.Data.Features) db.stateIDs = make(map[string]string) for key, entry := range db.Data.State { db.stateIDs[key] = entry.ID @@ -604,7 +607,7 @@ func (db *DeploymentState) unlockedOpen(ctx context.Context, path string, mode M return fmt.Errorf("failed to stat WAL file %s: %w", walPath, err) default: // WAL exists switch { - case mode == WritingDMS: + case mode == WritingDeploymentHistoryOnly: // A recorded deployment writes no WAL, so finding one means this state was written by a // deployment that did not record history. Refuse rather than discard it: the file is the // only record of that deploy's writes, and recording is refused below anyway. @@ -623,21 +626,21 @@ func (db *DeploymentState) unlockedOpen(ctx context.Context, path string, mode M // the config can neither start recording one that was not (its resources would be created a // second time) nor stop recording one that is (the service still holds it). _, recorded := db.Data.Features[FeatureDeploymentHistory] - if mode == WritingDMS && recorded && db.Data.Serial > dmsDeployment.LastVersionID { + if mode == WritingDeploymentHistoryOnly && recorded && db.Data.Serial > dmsDeployment.LastVersionID { return fmt.Errorf("inconsistent deployment state: resource state serial (%d) is ahead of deployment history version (%d)", db.Data.Serial, dmsDeployment.LastVersionID) } switch { - case mode == WritingDMS && !recorded && len(db.Data.State) == 0: + case mode == WritingDeploymentHistoryOnly && !recorded && len(db.Data.State) == 0: if db.Data.Features == nil { db.Data.Features = make(map[string]struct{}, 1) } db.Data.Features[FeatureDeploymentHistory] = struct{}{} recorded = true - case mode == WritingDMS && !recorded: + case mode == WritingDeploymentHistoryOnly && !recorded: return errors.New(`enabling experimental.deployment_history for an existing deployment is not supported Run "databricks bundle destroy" first, then deploy again with deployment history enabled`) - case mode != WritingDMS && recorded: + case mode != WritingDeploymentHistoryOnly && recorded: return ErrUnsettingRecording } @@ -678,9 +681,9 @@ Run "databricks bundle destroy" first, then deploy again with deployment history switch mode { case ReadingFile: return nil - case WritingDMS: + case WritingDeploymentHistoryOnly: return nil - case WritingFile, WritingFileWithHistory, WritingFileWithHistoryEnforced: + case WritingFile, WritingDeploymentHistoryShadow: return db.setupWal() } return nil @@ -809,9 +812,9 @@ func (db *DeploymentState) mergeWalIntoState(ctx context.Context) (bool, error) } hasEntries := lineNumber > 1 - // Enforced history can change the header's features without writing resource entries. - // Persist that marker too, before removing the WAL. - if lineNumber == 1 && !maps.Equal(db.Data.Features, newFeatures) { + // Persist marker changes even without resource writes. Compare against disk, + // because configuration may already have replaced the in-memory features. + if lineNumber == 1 && !maps.Equal(db.persistedFeatures, newFeatures) { hasEntries = true } @@ -855,7 +858,7 @@ func (db *DeploymentState) Finalize(ctx context.Context) (resourcestate.Exported } db.walFile = nil err = db.replayWAL(ctx) - } else if db.mode == WritingDMS { + } else if db.mode == WritingDeploymentHistoryOnly { // Stamp the header when persisting it: saved plans need a stable lineage, and // the last writer's CLI version must remain available for downgrade warnings. db.Data.Lineage = db.GetOrInitLineage() @@ -872,13 +875,13 @@ func (db *DeploymentState) Finalize(ctx context.Context) (resourcestate.Exported switch db.mode { case ReadingFile, WritingFile: // File-only modes have no history to drain. - case WritingDMS: + case WritingDeploymentHistoryOnly: if db.operationBuffer != nil { if drainErr := db.operationBuffer.Drain(ctx); drainErr != nil { err = errors.Join(err, drainErr) } } - case WritingFileWithHistory, WritingFileWithHistoryEnforced: + case WritingDeploymentHistoryShadow: if db.operationBuffer != nil { // File state is already persisted, so history errors only warn. // The buffer bounds the wait using the staged operation count. @@ -895,8 +898,8 @@ func (db *DeploymentState) Finalize(ctx context.Context) (resourcestate.Exported return state, err } -// UpgradeToWrite opens a WAL for file state without re-reading resources. -// DMS and file-history modes are already writable from Open, so upgrading them is a no-op. +// UpgradeToWrite enables writes using the history features already selected in Data. +// DMS-authoritative state writes to the service; file state opens a WAL. func (db *DeploymentState) UpgradeToWrite() error { db.mu.Lock() defer db.mu.Unlock() @@ -904,13 +907,22 @@ func (db *DeploymentState) UpgradeToWrite() error { if db.Path == "" { return errors.New("internal error: DeploymentState must be opened first") } + switch db.mode { - case WritingDMS, WritingFileWithHistory, WritingFileWithHistoryEnforced: + case WritingDeploymentHistoryOnly: + // Open already selects write mode for DMS-authoritative state. return nil - case WritingFile: + case WritingFile, WritingDeploymentHistoryShadow: return errors.New("internal error: DeploymentState is already open for write") case ReadingFile: + if _, only := db.Data.Features[FeatureDeploymentHistory]; only { + db.mode = WritingDeploymentHistoryOnly + return nil + } db.mode = WritingFile + if _, shadow := db.Data.Features[FeatureDeploymentHistoryShadow]; shadow { + db.mode = WritingDeploymentHistoryShadow + } if err := db.setupWal(); err != nil { db.mode = ReadingFile return err @@ -943,11 +955,16 @@ func (db *DeploymentState) setupWal() error { CLIVersion: build.GetInfo().Version, Features: maps.Clone(db.Data.Features), } - if db.mode == WritingFileWithHistoryEnforced { + switch db.mode { + case WritingDeploymentHistoryShadow: if walHead.Features == nil { walHead.Features = make(map[string]struct{}) } - walHead.Features[FeatureDeploymentHistoryBestEffort] = struct{}{} + walHead.Features[FeatureDeploymentHistoryShadow] = struct{}{} + case WritingFile: + walHead.Features = map[string]struct{}{} + default: + // Other modes preserve the existing markers. } if err := appendJSONLine(db.walFile, walHead); err != nil { // Remove the WAL created just above: O_EXCL means it is ours, and @@ -988,8 +1005,8 @@ func (db *DeploymentState) AssertOpenedForReadOrWrite() { func (db *DeploymentState) AssertOpenedForRead() { db.AssertOpenedForReadOrWrite() // DMS has no separate read-only mode. - // Planning reads DMS state while it is already in WritingDMS. - if db.mode.isWritable() && db.mode != WritingDMS { + // Planning reads DMS state while it is already in WritingDeploymentHistoryOnly. + if db.mode.isWritable() && db.mode != WritingDeploymentHistoryOnly { panic("internal error: DeploymentState must be opened in read mode") } } @@ -1062,7 +1079,8 @@ func (db *DeploymentState) ExportState(ctx context.Context) resourcestate.Export // the WAL. A torn write would therefore leave a state file that Open rejects // next to an intact WAL it never reads. func (db *DeploymentState) unlockedSave() error { - data, err := json.MarshalIndent(db.dataForFile(), "", " ") + fileData := db.dataForFile() + data, err := json.MarshalIndent(fileData, "", " ") if err != nil { return err } @@ -1071,6 +1089,7 @@ func (db *DeploymentState) unlockedSave() error { return fmt.Errorf("failed to save resources state to %#v: %w", db.Path, err) } + db.persistedFeatures = maps.Clone(fileData.Features) return nil } diff --git a/bundle/direct/dstate/state_test.go b/bundle/direct/dstate/state_test.go index a44afae8fab..c5bfea2783b 100644 --- a/bundle/direct/dstate/state_test.go +++ b/bundle/direct/dstate/state_test.go @@ -38,15 +38,55 @@ func TestOpenSaveFinalizeRoundTrip(t *testing.T) { mustFinalize(t, &db2) } -func TestFinalizeWithNoEntriesDoesNotWriteStateFile(t *testing.T) { - path := filepath.Join(t.TempDir(), "state.json") - - var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WritingFile, WithRecovery(true), OpenDmsArgs{})) - mustFinalize(t, &db) - - _, err := os.Stat(path) - assert.ErrorIs(t, err, os.ErrNotExist) +func TestFinalizeWithoutResourceWrites(t *testing.T) { + for _, mode := range []Mode{WritingFile, WritingDeploymentHistoryShadow} { + path := filepath.Join(t.TempDir(), "state.json") + var db DeploymentState + require.NoError(t, db.Open(t.Context(), path, ReadingFile, WithRecovery(true), OpenDmsArgs{})) + if mode == WritingDeploymentHistoryShadow { + // A first plan already knows history is enabled, before a state file exists. + db.Data.Features = map[string]struct{}{FeatureDeploymentHistoryShadow: {}} + } + require.NoError(t, db.UpgradeToWrite()) + require.EqualError(t, db.UpgradeToWrite(), "internal error: DeploymentState is already open for write") + mustFinalize(t, &db) + + if mode == WritingFile { + assert.NoFileExists(t, path) + continue + } + // Even an empty deployment must persist its history marker and stable lineage. + require.NoError(t, db.Open(t.Context(), path, ReadingFile, WithRecovery(false), OpenDmsArgs{})) + assert.Equal(t, 1, db.Data.Serial) + assert.NotEmpty(t, db.Data.Lineage) + assert.Contains(t, db.StateFeatures(), FeatureDeploymentHistoryShadow) + mustFinalize(t, &db) + + // Keeping history enabled without changes must not advance the file serial. + require.NoError(t, db.Open(t.Context(), path, WritingDeploymentHistoryShadow, WithRecovery(false), OpenDmsArgs{})) + mustFinalize(t, &db) + require.NoError(t, db.Open(t.Context(), path, ReadingFile, WithRecovery(false), OpenDmsArgs{})) + assert.Equal(t, 1, db.Data.Serial) + mustFinalize(t, &db) + + // Turning history off also persists its marker change without resource operations. + require.NoError(t, db.Open(t.Context(), path, WritingFile, WithRecovery(false), OpenDmsArgs{})) + mustFinalize(t, &db) + require.NoError(t, db.Open(t.Context(), path, ReadingFile, WithRecovery(false), OpenDmsArgs{})) + assert.Equal(t, 2, db.Data.Serial) + assert.NotContains(t, db.StateFeatures(), FeatureDeploymentHistoryShadow) + mustFinalize(t, &db) + + // Enabling history on existing state persists the marker, even without resource writes. + require.NoError(t, db.Open(t.Context(), path, ReadingFile, WithRecovery(false), OpenDmsArgs{})) + db.Data.Features = map[string]struct{}{FeatureDeploymentHistoryShadow: {}} + require.NoError(t, db.UpgradeToWrite()) + mustFinalize(t, &db) + require.NoError(t, db.Open(t.Context(), path, ReadingFile, WithRecovery(false), OpenDmsArgs{})) + assert.Equal(t, 3, db.Data.Serial) + assert.Contains(t, db.StateFeatures(), FeatureDeploymentHistoryShadow) + mustFinalize(t, &db) + } } func TestDataForFileSerial(t *testing.T) { @@ -68,7 +108,7 @@ func TestDataForFileSerial(t *testing.T) { VersionID: tt.versionID, } if tt.recording { - db.mode = WritingDMS + db.mode = WritingDeploymentHistoryOnly } db.Data.State["resources.jobs.foo"] = ResourceEntry{ID: "123", State: json.RawMessage(`{"name":"foo"}`)} @@ -183,7 +223,7 @@ func TestOpenUsesDMSVersionForSerial(t *testing.T) { ctx := cmdctx.SetWorkspaceClient(t.Context(), client) var db DeploymentState - err = db.Open(ctx, path, WritingDMS, WithRecovery(false), OpenDmsArgs{ + err = db.Open(ctx, path, WritingDeploymentHistoryOnly, WithRecovery(false), OpenDmsArgs{ DeploymentID: tt.deploymentID, LastVersionID: tt.lastVersionID, }) if tt.wantErr != "" { @@ -423,7 +463,7 @@ func TestOpenFailureLeavesStateClosed(t *testing.T) { } func TestGetSerialWithHistoryUsesLatestCounter(t *testing.T) { - for _, mode := range []Mode{WritingFileWithHistory, WritingFileWithHistoryEnforced} { + for _, mode := range []Mode{ReadingFile, WritingDeploymentHistoryShadow} { db := DeploymentState{mode: mode, Data: NewDatabase("", 4), VersionID: 7} assert.Equal(t, 7, db.GetSerial(), "mode=%d: service version is ahead", mode) db.Data.Serial = 9 diff --git a/bundle/phases/bind.go b/bundle/phases/bind.go index d8ff6aeaf3e..7646735f903 100644 --- a/bundle/phases/bind.go +++ b/bundle/phases/bind.go @@ -12,6 +12,7 @@ import ( "github.com/databricks/cli/bundle/config/engine" "github.com/databricks/cli/bundle/deploy/lock" "github.com/databricks/cli/bundle/deployplan" + bundleenv "github.com/databricks/cli/bundle/env" "github.com/databricks/cli/bundle/statemgmt" "github.com/databricks/cli/libs/agent" "github.com/databricks/cli/libs/cmdio" @@ -47,7 +48,11 @@ func Bind(ctx context.Context, b *bundle.Bundle, opts *BindOptions, stateDesc *s return } - if stateDesc.IsDMS() { + var configured *bool + if b.Config.Experimental != nil { + configured = b.Config.Experimental.DeploymentHistory + } + if stateDesc.IsDMS() || bundleenv.RecordsDeploymentHistory(ctx, configured) { logdiag.LogError(ctx, errors.New("bind is not supported for a bundle target that records deployment history")) return } diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index 13c5121f569..adf14d5ddf6 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -22,7 +22,6 @@ import ( "github.com/databricks/cli/bundle/statemgmt" "github.com/databricks/cli/libs/agent" "github.com/databricks/cli/libs/cmdio" - "github.com/databricks/cli/libs/dms" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/logdiag" "github.com/databricks/cli/libs/sync" @@ -170,9 +169,13 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand // The version is created only after approval; CompleteVersion is deferred before // lock.Release and no-ops until then. defer func() { - if b.DeploymentBundle.StateDB.IsDeploymentMetadataService() { + if b.DeploymentBundle.StateDB.RecordsHistory() { if _, err := b.DeploymentBundle.StateDB.CompleteVersion(ctx, !logdiag.HasError(ctx)); err != nil { - logdiag.LogError(ctx, err) + if b.DeploymentBundle.StateDB.IsDeploymentMetadataService() { + logdiag.LogError(ctx, err) + } else { + log.Warnf(ctx, "Failed to complete deployment history version: %s", err) + } } } bundle.ApplyContext(ctx, b, lock.Release(lock.GoalDeploy)) @@ -320,32 +323,13 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand // Create the deployment now that the plan is approved, so a declined deploy leaves none behind. // A first deploy's id did not exist at plan time - the version and any existing id were stamped // then - so stamp the one just created into the plan the apply reads. - if b.DeploymentBundle.StateDB.IsDeploymentMetadataService() { - firstDeploy := b.DeploymentBundle.StateDB.DeploymentID == "" - createOrUpdateDeployment(ctx, b, dmsDeployment) - if logdiag.HasError(ctx) { - return - } - if firstDeploy { - if err := b.DeploymentBundle.StampDeploymentIdForFirstVersion(b.DeploymentBundle.StateDB.DeploymentID); err != nil { - logdiag.LogError(ctx, err) - return - } - } - - // Only create a version when the plan has at least one operation to record, so it moves - // with serial. Done after the prompt, so a declined deploy never claims a number. - staged, err := stagedOperations(plan) - if err != nil { - logdiag.LogError(ctx, err) - return - } - if len(staged) > 0 { - if err := startVersion(ctx, b, dms.VersionTypeDeploy, staged); err != nil { + if b.DeploymentBundle.StateDB.RecordsHistory() { + if err := prepareDeploymentHistory(ctx, b, plan, dmsDeployment); err != nil { + if b.DeploymentBundle.StateDB.IsDeploymentMetadataService() { logdiag.LogError(ctx, err) return } - logDeploymentVersion(ctx, b) + log.Warnf(ctx, "Could not enable deployment history for this deploy: %s", err) } } diff --git a/bundle/phases/destroy.go b/bundle/phases/destroy.go index 07bbbbca385..cfac15e5930 100644 --- a/bundle/phases/destroy.go +++ b/bundle/phases/destroy.go @@ -16,7 +16,6 @@ import ( "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dms" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/logdiag" "github.com/databricks/databricks-sdk-go/apierr" @@ -156,20 +155,26 @@ func destroyCore(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan) { return } - if b.DeploymentBundle.StateDB.IsDeploymentMetadataService() { + if b.DeploymentBundle.StateDB.RecordsHistory() { // Complete version before deleting remote files; the deployment node is under statePath. completed, err := b.DeploymentBundle.StateDB.CompleteVersion(ctx, true) if err != nil { - logdiag.LogError(ctx, err) - return + if b.DeploymentBundle.StateDB.IsDeploymentMetadataService() { + logdiag.LogError(ctx, err) + return + } + log.Warnf(ctx, "Failed to complete deployment history version: %s", err) } // A completed destroy's resources are gone, so its deployment record is deleted too. if completed { deploymentID := b.DeploymentBundle.StateDB.DeploymentID w := b.WorkspaceClient(ctx) if err := deleteDeployment(ctx, w, deploymentID); err != nil { - logdiag.LogError(ctx, fmt.Errorf("failed to delete deployment: %w", err)) - return + if b.DeploymentBundle.StateDB.IsDeploymentMetadataService() { + logdiag.LogError(ctx, fmt.Errorf("failed to delete deployment: %w", err)) + return + } + log.Warnf(ctx, "Could not delete deployment history: %s", err) } } } @@ -293,15 +298,23 @@ func Destroy(ctx context.Context, b *bundle.Bundle) { // destroy records nothing. Deferred before lock.Release to hold the lock; a no-op once // destroyCore has completed the version. defer func() { - if b.DeploymentBundle.StateDB.IsDeploymentMetadataService() { + if b.DeploymentBundle.StateDB.RecordsHistory() { completed, err := b.DeploymentBundle.StateDB.CompleteVersion(ctx, !logdiag.HasError(ctx)) if err != nil { - logdiag.LogError(ctx, err) + if b.DeploymentBundle.StateDB.IsDeploymentMetadataService() { + logdiag.LogError(ctx, err) + } else { + log.Warnf(ctx, "Failed to complete deployment history version: %s", err) + } } else if completed { deploymentID := b.DeploymentBundle.StateDB.DeploymentID w := b.WorkspaceClient(ctx) if err := deleteDeployment(ctx, w, deploymentID); err != nil { - logdiag.LogError(ctx, fmt.Errorf("failed to delete deployment: %w", err)) + if b.DeploymentBundle.StateDB.IsDeploymentMetadataService() { + logdiag.LogError(ctx, fmt.Errorf("failed to delete deployment: %w", err)) + } else { + log.Warnf(ctx, "Could not delete deployment history: %s", err) + } } } } @@ -346,23 +359,20 @@ func Destroy(ctx context.Context, b *bundle.Bundle) { } } - // Upgrade from read (opened by process.go, or by Migrate) to write mode + // History follows the opened state, not configuration. if err := b.DeploymentBundle.StateDB.UpgradeToWrite(); err != nil { logdiag.LogError(ctx, err) return } - // Start the version for this destroy, now that it is approved. Everything else recording - // does follows from the buffer this opens. - if b.DeploymentBundle.StateDB.IsDeploymentMetadataService() { - staged, err := stagedOperations(plan) - if err != nil { - logdiag.LogError(ctx, err) - return - } - if err := startVersion(ctx, b, dms.VersionTypeDestroy, staged); err != nil { - logdiag.LogError(ctx, err) - return + // Record the approved destroy; deployment history shadow errors only warn. + if b.DeploymentBundle.StateDB.RecordsHistory() { + if err := prepareDestroyHistory(ctx, b, plan); err != nil { + if b.DeploymentBundle.StateDB.IsDeploymentMetadataService() { + logdiag.LogError(ctx, err) + return + } + log.Warnf(ctx, "Could not record deployment history for this destroy: %s", err) } } destroyCore(ctx, b, plan) diff --git a/bundle/phases/dms.go b/bundle/phases/dms.go index e2d7ea958c1..024aae7b0b6 100644 --- a/bundle/phases/dms.go +++ b/bundle/phases/dms.go @@ -16,7 +16,6 @@ import ( "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/dms" "github.com/databricks/cli/libs/log" - "github.com/databricks/cli/libs/logdiag" "github.com/databricks/cli/libs/retry" "github.com/databricks/cli/libs/workspaceurls" "github.com/databricks/databricks-sdk-go" @@ -73,13 +72,12 @@ func actionToSDK(a deployplan.ActionType) (bundledeployments.OperationActionType // run changed. current is the record the service holds (nil before the first recorded deploy), // diffed to mask the update down to what changed. Runs after approval, so a declined deploy leaves // no deployment behind; a first deploy's new id is then stamped into the plan (StampDeploymentID). -func createOrUpdateDeployment(ctx context.Context, b *bundle.Bundle, current *bundledeployments.Deployment) { +func createOrUpdateDeployment(ctx context.Context, b *bundle.Bundle, current *bundledeployments.Deployment) error { db := &b.DeploymentBundle w := b.WorkspaceClient(ctx) metadata, err := deploymentMetadata(b) if err != nil { - logdiag.LogError(ctx, fmt.Errorf("failed to compute deployment metadata: %w", err)) - return + return fmt.Errorf("failed to compute deployment metadata: %w", err) } deploymentID := db.StateDB.DeploymentID if deploymentID == "" { @@ -87,14 +85,12 @@ func createOrUpdateDeployment(ctx context.Context, b *bundle.Bundle, current *bu dep.InitialParentPath = b.Config.Workspace.StatePath created, err := w.BundleDeployments.CreateDeployment(ctx, bundledeployments.CreateDeploymentRequest{Deployment: dep}) if err != nil { - logdiag.LogError(ctx, fmt.Errorf("failed to create deployment: %w", err)) - return + return fmt.Errorf("failed to create deployment: %w", err) } // The id is the workspace node the server makes under the parent path. deploymentID, err = dms.DeploymentIDFromName(created.Name) if err != nil { - logdiag.LogError(ctx, fmt.Errorf("failed to create deployment: %w", err)) - return + return fmt.Errorf("failed to create deployment: %w", err) } db.StateDB.DeploymentID = deploymentID } else if mask := metadata.StaleFields(current); mask != "" { @@ -106,8 +102,7 @@ func createOrUpdateDeployment(ctx context.Context, b *bundle.Bundle, current *bu }) }) if err != nil { - logdiag.LogError(ctx, fmt.Errorf("failed to update deployment: %w", err)) - return + return fmt.Errorf("failed to update deployment: %w", err) } } @@ -115,6 +110,7 @@ func createOrUpdateDeployment(ctx context.Context, b *bundle.Bundle, current *bu bundle.ApplyFuncContext(ctx, b, func(_ context.Context, b *bundle.Bundle) { b.Config.Bundle.Deployment.DeploymentID = deploymentID }) + return nil } // deleteDeployment deletes the deployment record. A retry that finds it missing succeeds: the @@ -132,12 +128,54 @@ func deleteDeployment(ctx context.Context, w *databricks.WorkspaceClient, deploy return err } -// startVersion claims the version the run settled on and opens the buffer that records -// each state write under it. Called after approval, so a declined deploy never claims a number. +// prepareDeploymentHistory creates history metadata and a version after approval. +func prepareDeploymentHistory(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan, current *bundledeployments.Deployment) error { + firstDeploy := b.DeploymentBundle.StateDB.DeploymentID == "" + if err := createOrUpdateDeployment(ctx, b, current); err != nil { + return err + } + if firstDeploy { + if err := b.DeploymentBundle.StampDeploymentIdForFirstVersion(b.DeploymentBundle.StateDB.DeploymentID); err != nil { + return err + } + } + staged, err := stagedOperations(plan) + if err != nil { + return err + } + if len(staged) == 0 { + return nil + } + if err := startVersion(ctx, b, dms.VersionTypeDeploy, staged); err != nil { + return err + } + logDeploymentVersion(ctx, b) + return nil +} + +// prepareDestroyHistory records the approved destroy without changing its state source. +func prepareDestroyHistory(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan) error { + if !b.DeploymentBundle.StateDB.IsDeploymentMetadataService() && b.DeploymentBundle.StateDB.DeploymentID == "" { + if err := createOrUpdateDeployment(ctx, b, nil); err != nil { + return err + } + } + staged, err := stagedOperations(plan) + if err != nil { + return err + } + return startVersion(ctx, b, dms.VersionTypeDestroy, staged) +} + +// startVersion opens a history version after approval. File-backed history skips empty versions; +// DMS-only destroy retains its existing behavior. A declined deploy never claims a version. // A no-op when the bundle does not record deployment history. func startVersion(ctx context.Context, b *bundle.Bundle, versionType dms.VersionType, staged []bundledeployments.StagedOperation) error { db := &b.DeploymentBundle - if !db.StateDB.IsDeploymentMetadataService() { + if !db.StateDB.RecordsHistory() { + return nil + } + if len(staged) == 0 && !db.StateDB.IsDeploymentMetadataService() { return nil } w := b.WorkspaceClient(ctx) @@ -146,7 +184,7 @@ func startVersion(ctx context.Context, b *bundle.Bundle, versionType dms.Version // InitializeOperationBuffer moves the anchor below, so callers afterwards read the created // version straight off the state. previous := db.StateDB.VersionID - versionID := previous + 1 + versionID := max(previous, db.StateDB.Data.Serial) + 1 previousVersionID := "" if previous > 0 { previousVersionID = strconv.Itoa(previous) diff --git a/bundle/phases/telemetry.go b/bundle/phases/telemetry.go index 99464f3ef24..102a004aa1a 100644 --- a/bundle/phases/telemetry.go +++ b/bundle/phases/telemetry.go @@ -198,7 +198,7 @@ func aiRuntimeTaskMetrics(jobs map[string]*resources.Job) (present, scheduled, m // LogDeployTelemetry logs a telemetry event for a bundle deploy command. func LogDeployTelemetry(ctx context.Context, b *bundle.Bundle, errMsg string) { errMsg = telemetry.ScrubErrorMessage(errMsg) - b.Metrics.SetBoolValue(metrics.DeploymentHistoryEnabled, b.DeploymentBundle.StateDB.IsDeploymentMetadataService()) + b.Metrics.SetBoolValue(metrics.DeploymentHistoryEnabled, b.DeploymentBundle.StateDB.RecordsHistory()) resourcesCount := int64(0) _, err := dyn.MapByPattern(b.Config.Value(), dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey()), func(p dyn.Path, v dyn.Value) (dyn.Value, error) { diff --git a/bundle/statemgmt/state_pull.go b/bundle/statemgmt/state_pull.go index 616cd8af197..01d1fe9575d 100644 --- a/bundle/statemgmt/state_pull.go +++ b/bundle/statemgmt/state_pull.go @@ -77,6 +77,12 @@ func (s *StateDesc) IsDMS() bool { return ok } +// RecordsHistory includes both DMS-authoritative state and deployment history shadow. +func (s *StateDesc) RecordsHistory() bool { + _, bestEffort := s.Features["deployment_history_shadow"] + return bestEffort || s.IsDMS() +} + func localRead(ctx context.Context, fullPath string, engine engine.EngineType) *StateDesc { content, err := os.ReadFile(fullPath) if err != nil { diff --git a/cmd/bundle/utils/destroy.go b/cmd/bundle/utils/destroy.go index 7df0a78c328..61326ef0837 100644 --- a/cmd/bundle/utils/destroy.go +++ b/cmd/bundle/utils/destroy.go @@ -43,9 +43,8 @@ func CommandBundleDestroy(cmd *cobra.Command, args []string, autoApprove, forceD } }, // Skip context initialization if already initialized by parent command - SkipInitContext: skipInitContext, - AlwaysPull: true, - SkipEnforcingDeploymentHistorySetting: true, + SkipInitContext: skipInitContext, + AlwaysPull: true, PostStateFunc: func(ctx context.Context, b *bundle.Bundle, stateDesc *statemgmt.StateDesc) error { phases.Destroy(ctx, b) if logdiag.HasError(ctx) { diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index 8fa68f32f5e..c16f8280a08 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -96,9 +96,6 @@ type ProcessOptions struct { // (after state is opened and IDs loaded, before deferred Finalize). PostStateFunc func(ctx context.Context, b *bundle.Bundle, stateDesc *statemgmt.StateDesc) error - // If true, deployment history configuration is ignored after state is resolved. - SkipEnforcingDeploymentHistorySetting bool - // Indicate whether the bundle operation originates from the pipelines CLI IsPipelinesCLI bool } @@ -249,9 +246,6 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle } ctx = WithLineage(ctx, stateDesc) cmd.SetContext(ctx) - if stateDesc.Engine.IsDirect() { - resolveDeploymentHistory(ctx, b, stateDesc) - } // Record the engine the resolved state uses now, so deploy telemetry reports // it even when the deploy fails or is cancelled before deployCore runs. @@ -365,14 +359,46 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle } } - if stateDesc.Engine.IsDirect() && !opts.SkipEnforcingDeploymentHistorySetting { - if err := enforceDeploymentHistorySetting(ctx, b, stateDesc, opts.Deploy || opts.PreDeployChecks); err != nil { + if stateDesc.Engine.IsDirect() && (opts.Deploy || opts.PreDeployChecks) { + if err := enforceDeploymentHistorySetting(ctx, b, stateDesc, true); err != nil { logdiag.LogError(ctx, err) return b, stateDesc, root.ErrAlreadyPrinted } } - if needDirectState && stateDesc.IsDMS() { + stateMode := dstate.WritingFile + switch { + case stateDesc.IsDMS(): + // DMS-only state must keep its service-backed source. + stateMode = dstate.WritingDeploymentHistoryOnly + case opts.Deploy || opts.PreDeployChecks: + // Deploy and plan follow the current configuration. + if configuresDeploymentHistory(ctx, b) { + stateMode = dstate.WritingDeploymentHistoryShadow + if b.DeploymentBundle.StateDB.Data.Features == nil { + b.DeploymentBundle.StateDB.Data.Features = make(map[string]struct{}) + } + b.DeploymentBundle.StateDB.Data.Features[dstate.FeatureDeploymentHistoryShadow] = struct{}{} + } else { + delete(b.DeploymentBundle.StateDB.Data.Features, dstate.FeatureDeploymentHistoryShadow) + } + case b.DeploymentBundle.StateDB.RecordsHistory(): + // Other operations, including destroy, follow loaded state instead of configuration. + stateMode = dstate.WritingDeploymentHistoryShadow + } + + if needDirectState && stateMode == dstate.WritingDeploymentHistoryShadow { + if !cmdctx.HasWorkspaceClient(ctx) { + ctx = cmdctx.SetWorkspaceClient(ctx, b.WorkspaceClient(ctx)) + } + var err error + dmsDeployment, err = prepareDeploymentHistoryShadow(ctx, b) + if err != nil { + log.Warnf(ctx, "Could not read deployment history: %s", err) + } + } + + if needDirectState && stateMode == dstate.WritingDeploymentHistoryOnly { var err error dmsDeploymentID, dmsDeployment, err = fetchDeploymentFromStatePath(ctx, b.WorkspaceClient(ctx), b.Config.Workspace.StatePath) if err != nil { @@ -406,7 +432,7 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle if !cmdctx.HasWorkspaceClient(ctx) { ctx = cmdctx.SetWorkspaceClient(ctx, b.WorkspaceClient(ctx)) } - if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WritingDMS, dstate.WithRecovery(false), dstate.OpenDmsArgs{DeploymentID: dmsDeploymentID, LastVersionID: lastVersionID}); err != nil { + if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WritingDeploymentHistoryOnly, dstate.WithRecovery(false), dstate.OpenDmsArgs{DeploymentID: dmsDeploymentID, LastVersionID: lastVersionID}); err != nil { logdiag.LogError(ctx, err) return b, stateDesc, root.ErrAlreadyPrinted } @@ -638,11 +664,19 @@ func parseLastVersionID(dmsDeployment *bundledeployments.Deployment) (int, error // read from the deployment metadata service instead. func OpenDirectStateForRead(ctx context.Context, b *bundle.Bundle, stateDesc *statemgmt.StateDesc) error { _, localPath := b.StateFilenameDirect(ctx) - if !resolveDeploymentHistory(ctx, b, stateDesc) { + if !stateDesc.IsDMS() { if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.ReadingFile, dstate.WithRecovery(true), dstate.OpenDmsArgs{}); err != nil { return err } - return enforceDeploymentHistorySetting(ctx, b, stateDesc, false) + if err := enforceDeploymentHistorySetting(ctx, b, stateDesc, false); err != nil { + return err + } + if configuresDeploymentHistory(ctx, b) { + if _, err := prepareDeploymentHistoryShadow(ctx, b); err != nil { + log.Warnf(ctx, "Could not read deployment history: %s", err) + } + } + return nil } dmsDeploymentID, dmsDeployment, err := fetchDeploymentFromStatePath(ctx, b.WorkspaceClient(ctx), b.Config.Workspace.StatePath) @@ -657,31 +691,22 @@ func OpenDirectStateForRead(ctx context.Context, b *bundle.Bundle, stateDesc *st if !cmdctx.HasWorkspaceClient(ctx) { ctx = cmdctx.SetWorkspaceClient(ctx, b.WorkspaceClient(ctx)) } - if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WritingDMS, dstate.WithRecovery(false), dstate.OpenDmsArgs{DeploymentID: dmsDeploymentID, LastVersionID: lastVersionID}); err != nil { + if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WritingDeploymentHistoryOnly, dstate.WithRecovery(false), dstate.OpenDmsArgs{DeploymentID: dmsDeploymentID, LastVersionID: lastVersionID}); err != nil { return err } return enforceDeploymentHistorySetting(ctx, b, stateDesc, false) } -func resolveDeploymentHistory(ctx context.Context, b *bundle.Bundle, stateDesc *statemgmt.StateDesc) bool { - configured := configuresDeploymentHistory(ctx, b) - if stateDesc.SourcePath == "" { - if configured { - stateDesc.Features = map[string]struct{}{dstate.FeatureDeploymentHistory: {}} - } - return configured - } - - return stateDesc.IsDMS() -} - func enforceDeploymentHistorySetting(ctx context.Context, b *bundle.Bundle, stateDesc *statemgmt.StateDesc, requireMatch bool) error { if stateDesc.SourcePath == "" { return nil } configured := configuresDeploymentHistory(ctx, b) - recorded := stateDesc.IsDMS() - if configured == recorded { + recorded := stateDesc.RecordsHistory() + if b.DeploymentBundle.StateDB.IsOpen() { + recorded = b.DeploymentBundle.StateDB.RecordsHistory() + } + if configured == recorded || (!configured && !stateDesc.IsDMS()) { return nil } if requireMatch { @@ -698,6 +723,29 @@ Run "databricks bundle destroy" first, then deploy again with deployment history return nil } +// prepareDeploymentHistoryShadow reads version metadata, never resource state, from DMS. +func prepareDeploymentHistoryShadow(ctx context.Context, b *bundle.Bundle) (*bundledeployments.Deployment, error) { + id, deployment, err := fetchDeploymentFromStatePath(ctx, b.WorkspaceClient(ctx), b.Config.Workspace.StatePath) + if err != nil { + return nil, err + } + lastVersion, err := parseLastVersionID(deployment) + if err != nil { + return nil, err + } + db := &b.DeploymentBundle.StateDB + db.DeploymentID, db.VersionID = id, lastVersion + bundle.ApplyFuncContext(ctx, b, func(_ context.Context, b *bundle.Bundle) { + b.Config.Bundle.Deployment.DeploymentID = id + b.Config.Bundle.Deployment.LatestVersionID = lastVersion + }) + bundle.ApplyContext(ctx, b, metadata.AnnotateDeploymentVersion(max(db.Data.Serial, lastVersion)+1)) + if id != "" { + bundle.ApplyContext(ctx, b, metadata.AnnotateDeployment(id)) + } + return deployment, nil +} + func configuresDeploymentHistory(ctx context.Context, b *bundle.Bundle) bool { var configured *bool if b.Config.Experimental != nil { diff --git a/cmd/bundle/utils/process_test.go b/cmd/bundle/utils/process_test.go index b216f1102d2..21910039344 100644 --- a/cmd/bundle/utils/process_test.go +++ b/cmd/bundle/utils/process_test.go @@ -3,31 +3,11 @@ package utils import ( "testing" - "github.com/databricks/cli/bundle" - "github.com/databricks/cli/bundle/config" - "github.com/databricks/cli/bundle/direct/dstate" - "github.com/databricks/cli/bundle/statemgmt" "github.com/databricks/databricks-sdk-go/service/bundledeployments" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -func TestResolveDeploymentHistory(t *testing.T) { - b := &bundle.Bundle{Config: config.Root{ - Experimental: &config.Experimental{DeploymentHistory: new(true)}, - }} - - newState := &statemgmt.StateDesc{} - assert.True(t, resolveDeploymentHistory(t.Context(), b, newState)) - assert.True(t, newState.IsDMS()) - assert.False(t, resolveDeploymentHistory(t.Context(), b, &statemgmt.StateDesc{SourcePath: "resources.json"})) - b.Config.Experimental.DeploymentHistory = new(false) - assert.True(t, resolveDeploymentHistory(t.Context(), b, &statemgmt.StateDesc{ - SourcePath: "resources.json", - Features: map[string]struct{}{dstate.FeatureDeploymentHistory: {}}, - })) -} - func TestIsNewerVersion(t *testing.T) { tests := []struct { name string diff --git a/libs/dms/operation_buffer.go b/libs/dms/operation_buffer.go index 5d25eecc74b..7a77cdf83c8 100644 --- a/libs/dms/operation_buffer.go +++ b/libs/dms/operation_buffer.go @@ -17,7 +17,7 @@ const bufferedOperations = 10 const historyDrainTimeoutPerOperation = 5 * time.Second -// BufferOptions selects optional file-history recording behavior. +// BufferOptions selects optional deployment history shadow recording behavior. type BufferOptions struct { BestEffort bool // OperationCount counts staged resource keys. All recorded keys must be staged. @@ -207,11 +207,10 @@ func (b *OperationBuffer) write(ctx context.Context, key string, update Operatio } // Drain waits for buffered uploads before version completion. -// - WritingDMS: close the channel and wait for queued writes. -// - WritingFileWithHistory: wait up to five seconds per staged operation. -// - WritingFileWithHistoryEnforced: wait up to five seconds per staged operation. +// - WritingDeploymentHistoryOnly: close the channel and wait for queued writes. +// - WritingDeploymentHistoryShadow: wait up to five seconds per staged operation. // -// Both history modes cancel outstanding uploads when the timeout or ctx expires. +// Shadow history cancels outstanding uploads when the timeout or ctx expires. // It returns the first recording error and is safe to call more than once. func (b *OperationBuffer) Drain(ctx context.Context) error { b.stopQueue()