Skip to content

Commit ed3ba2e

Browse files
shreyas-goenkashreyas-goenka
committed
Default deployment history to enforced file-backed recording
1 parent 4d88efe commit ed3ba2e

55 files changed

Lines changed: 1197 additions & 324 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎acceptance/bin/print_state.py‎

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -174,23 +174,26 @@ def get_last_version_id(target):
174174

175175

176176
def print_recorded_state(filename, target):
177-
"""Print the state file with its resources filled in from the deployment metadata service.
177+
"""Print DMS resources using the state file's header.
178178
179-
While recording, the file itself carries only the header - the service holds the resources - so
180-
printing it raw would show an empty state and differ from the same test's non-recording run.
179+
This checks history uploads even though resources.json remains authoritative.
180+
Use --no-dms to inspect the file state instead.
181181
"""
182182
data = json.loads(open(filename).read())
183183
# Recording stamps each resource payload with the deployment and version; drop it here so the
184184
# printed state matches a non-recording run without every caller piping through nostamp.
185185
data["state"] = scrub(get_recorded_state(target))
186186

187-
# The file's serial is a copy of the DMS version to ensure outdated resource state is not read.
188-
# Rebuild in header order with the authoritative serial from the deployment.
187+
# Pair DMS resources with the DMS version. The file serial can advance independently.
189188
last_version_id = get_last_version_id(target)
190189
serial = int(last_version_id) if last_version_id else 0
191190

192191
# Recording itself is not what these tests assert, so drop the feature that marks it.
193-
features = {k: v for k, v in (data.get("features") or {}).items() if k != "deployment_history"}
192+
features = {
193+
k: v
194+
for k, v in (data.get("features") or {}).items()
195+
if k not in {"deployment_history", "deployment_history_best_effort"}
196+
}
194197

195198
rebuilt = {}
196199
for key in ("state_version", "cli_version", "lineage"):
@@ -213,7 +216,7 @@ def main():
213216
"--no-dms",
214217
action="store_true",
215218
help="Print resources.json as-is without fetching state from DMS. "
216-
"Use in bundle/dms tests to assert the tombstone content of the file.",
219+
"Use in bundle/dms tests to assert the file state and its feature markers.",
217220
)
218221
args = parser.parse_args()
219222

‎acceptance/bundle/deploy/readplan/lineage-mismatch/plan.dms.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,6 @@
55
"serial": 0,
66
"plan": {},
77
"features": {
8-
"deployment_history": {}
8+
"deployment_history_best_effort": {}
99
}
1010
}

‎acceptance/bundle/dms/bind-unbind-unsupported/databricks.yml.tmpl‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,5 @@
11
bundle:
22
name: dms-bind-unbind-$UNIQUE_NAME
3-
experimental:
4-
deployment_history: true
5-
63
resources:
74
jobs:
85
foo:

‎acceptance/bundle/dms/bind-unbind-unsupported/output.txt‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,13 +4,17 @@
44
Error: bind is not supported for a bundle target that records deployment history
55

66

7-
=== Deploy so the bundle records state, then unbind is refused too
7+
=== Unbind is allowed for file-backed history state
88
>>> [CLI] bundle deploy
99
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-bind-unbind-[UNIQUE_NAME]/default/files...
1010
Created jobs.foo
1111
Files: 2 uploaded, 0 deleted
1212
Resources: 1 created, 0 changed, 0 deleted, 0 unchanged
1313

14-
>>> musterr [CLI] bundle deployment unbind foo
15-
Error: unbind is not supported for a bundle target that records deployment history
14+
>>> [CLI] bundle deployment unbind foo
1615

16+
>>> jq {features, state} .databricks/bundle/default/resources.json
17+
{
18+
"features": null,
19+
"state": {}
20+
}

‎acceptance/bundle/dms/bind-unbind-unsupported/script‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,8 @@ job_id=$($CLI jobs create --json '{"name": "bind-target-'$UNIQUE_NAME'"}' | jq -
88
add_repl "$job_id" JOB_ID
99
trace musterr $CLI bundle deployment bind foo "$job_id"
1010

11-
# unbind refuses when it opens an existing recording state, so it needs a deployment first.
12-
title "Deploy so the bundle records state, then unbind is refused too"
11+
# File state remains writable even when it carries the history marker.
12+
title "Unbind is allowed for file-backed history state"
1313
trace $CLI bundle deploy
14-
trace musterr $CLI bundle deployment unbind foo
14+
trace $CLI bundle deployment unbind foo
15+
trace jq '{features, state}' .databricks/bundle/default/resources.json

‎acceptance/bundle/dms/declined-deploy/output.txt‎

Lines changed: 11 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged
6767
"lineage": "[UUID]",
6868
"serial": 1,
6969
"features": {
70-
"deployment_history": {}
70+
"deployment_history_best_effort": {}
7171
}
7272
}
7373

@@ -79,18 +79,10 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged
7979
"method": "GET",
8080
"path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]"
8181
}
82-
{
83-
"method": "GET",
84-
"path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/resources"
85-
}
8682
{
8783
"method": "GET",
8884
"path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]"
8985
}
90-
{
91-
"method": "GET",
92-
"path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/resources"
93-
}
9486

9587
>>> print_state.py --no-dms
9688
{
@@ -99,9 +91,17 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged
9991
"lineage": "[UUID]",
10092
"serial": 1,
10193
"features": {
102-
"deployment_history": {}
94+
"deployment_history_best_effort": {}
10395
},
104-
"state": {}
96+
"state": {
97+
"resources.schemas.foo": {
98+
"__id__": "main.dms_declined_deploy_schema_[UNIQUE_NAME]",
99+
"state": {
100+
"catalog_name": "main",
101+
"name": "dms_declined_deploy_schema_[UNIQUE_NAME]"
102+
}
103+
}
104+
}
105105
}
106106

107107
=== A destructive change without --auto-approve is declined: this console cannot prompt
@@ -124,10 +124,6 @@ Files: 2 uploaded, 0 deleted
124124
"method": "GET",
125125
"path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]"
126126
}
127-
{
128-
"method": "GET",
129-
"path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/resources"
130-
}
131127

132128
=== The recorded resources are unchanged by the decline: the schema is still at version 1
133129
>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[DEPLOYMENT_ID] -o json
@@ -163,10 +159,6 @@ false
163159
"method": "GET",
164160
"path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]"
165161
}
166-
{
167-
"method": "GET",
168-
"path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/resources"
169-
}
170162
{
171163
"method": "POST",
172164
"path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions",

‎acceptance/bundle/dms/depends-on/output.txt‎

Lines changed: 48 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
"plan_version": [PLAN_VERSION],
66
"cli_version": "[CLI_VERSION]",
77
"features": {
8-
"deployment_history": {}
8+
"deployment_history_best_effort": {}
99
}
1010
}
1111

@@ -61,7 +61,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged
6161
"lineage": "[UUID]",
6262
"serial": 1,
6363
"features": {
64-
"deployment_history": {}
64+
"deployment_history_best_effort": {}
6565
}
6666
}
6767

@@ -91,9 +91,53 @@ Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged
9191
"lineage": "[UUID]",
9292
"serial": 1,
9393
"features": {
94-
"deployment_history": {}
94+
"deployment_history_best_effort": {}
9595
},
96-
"state": {}
96+
"state": {
97+
"resources.jobs.child": {
98+
"__id__": "[CHILD_ID]",
99+
"state": {
100+
"deployment": {
101+
"deployment_id": "[DEPLOYMENT_ID]",
102+
"kind": "BUNDLE",
103+
"metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-depends-on-[UNIQUE_NAME]/default/state/metadata.json",
104+
"version_id": "1"
105+
},
106+
"description": "depends on [PARENT_ID]",
107+
"edit_mode": "UI_LOCKED",
108+
"format": "MULTI_TASK",
109+
"max_concurrent_runs": 1,
110+
"name": "child",
111+
"queue": {
112+
"enabled": true
113+
}
114+
},
115+
"depends_on": [
116+
{
117+
"node": "resources.jobs.parent",
118+
"label": "${resources.jobs.parent.id}"
119+
}
120+
]
121+
},
122+
"resources.jobs.parent": {
123+
"__id__": "[PARENT_ID]",
124+
"state": {
125+
"deployment": {
126+
"deployment_id": "[DEPLOYMENT_ID]",
127+
"kind": "BUNDLE",
128+
"metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-depends-on-[UNIQUE_NAME]/default/state/metadata.json",
129+
"version_id": "1"
130+
},
131+
"edit_mode": "UI_LOCKED",
132+
"format": "MULTI_TASK",
133+
"max_concurrent_runs": 1,
134+
"name": "parent",
135+
"queue": {
136+
"enabled": true
137+
}
138+
}
139+
}
140+
}
97141
}
98142

99143
>>> [CLI] bundle destroy --auto-approve

‎acceptance/bundle/dms/deployment-metadata-change/output.txt‎

Lines changed: 41 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged
6767
"lineage": "[UUID]",
6868
"serial": 1,
6969
"features": {
70-
"deployment_history": {}
70+
"deployment_history_best_effort": {}
7171
}
7272
}
7373

@@ -114,6 +114,25 @@ Resources: 0 created, 0 changed, 0 deleted, 1 unchanged
114114
}
115115
}
116116
}
117+
{
118+
"method": "POST",
119+
"path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions",
120+
"q": {
121+
"version_id": "2"
122+
},
123+
"body": {
124+
"cli_version": "[CLI_VERSION]",
125+
"previous_version_id": "1",
126+
"version_type": "VERSION_TYPE_DEPLOY"
127+
}
128+
}
129+
{
130+
"method": "POST",
131+
"path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/2/complete",
132+
"body": {
133+
"completion_reason": "VERSION_COMPLETE_SUCCESS"
134+
}
135+
}
117136

118137
>>> [CLI] bundle plan -t prod -o json
119138
Recommendation: target with 'mode: production' should set 'workspace.root_path' to make sure only one copy is deployed
@@ -126,9 +145,9 @@ A common practice is to use a username or principal name in this path, i.e. use
126145
"plan_version": [PLAN_VERSION],
127146
"cli_version": "[CLI_VERSION]",
128147
"lineage": "[UUID]",
129-
"serial": 1,
148+
"serial": 2,
130149
"features": {
131-
"deployment_history": {}
150+
"deployment_history_best_effort": {}
132151
}
133152
}
134153

@@ -163,6 +182,25 @@ Files: 0 uploaded, 0 deleted
163182
Resources: 0 created, 0 changed, 0 deleted, 1 unchanged
164183

165184
>>> print_requests.py --dms //api/2.0/bundle
185+
{
186+
"method": "POST",
187+
"path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions",
188+
"q": {
189+
"version_id": "3"
190+
},
191+
"body": {
192+
"cli_version": "[CLI_VERSION]",
193+
"previous_version_id": "2",
194+
"version_type": "VERSION_TYPE_DEPLOY"
195+
}
196+
}
197+
{
198+
"method": "POST",
199+
"path": "/api/2.0/bundle/deployments/[DEPLOYMENT_ID]/versions/3/complete",
200+
"body": {
201+
"completion_reason": "VERSION_COMPLETE_SUCCESS"
202+
}
203+
}
166204

167205
>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[DEPLOYMENT_ID] -o json
168206
[

‎acceptance/bundle/dms/empty-grants/output.txt‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -138,7 +138,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged
138138
"lineage": "[UUID]",
139139
"serial": 2,
140140
"features": {
141-
"deployment_history": {}
141+
"deployment_history_best_effort": {}
142142
}
143143
}
144144

@@ -163,9 +163,17 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged
163163
"lineage": "[UUID]",
164164
"serial": 2,
165165
"features": {
166-
"deployment_history": {}
166+
"deployment_history_best_effort": {}
167167
},
168-
"state": {}
168+
"state": {
169+
"resources.schemas.foo": {
170+
"__id__": "main.dms_empty_grants_[UNIQUE_NAME]",
171+
"state": {
172+
"catalog_name": "main",
173+
"name": "dms_empty_grants_[UNIQUE_NAME]"
174+
}
175+
}
176+
}
169177
}
170178

171179
>>> [CLI] bundle destroy --auto-approve

‎acceptance/bundle/dms/existing-state/output.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,6 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged
7373
{
7474
"state_version": 3,
7575
"features": {
76-
"deployment_history": {}
76+
"deployment_history_best_effort": {}
7777
}
7878
}

0 commit comments

Comments
 (0)