diff --git a/acceptance/bundle/override/cluster_id_clears_new_cluster_reference/databricks.yml b/acceptance/bundle/override/cluster_id_clears_new_cluster_reference/databricks.yml new file mode 100644 index 00000000000..d2e913f1abe --- /dev/null +++ b/acceptance/bundle/override/cluster_id_clears_new_cluster_reference/databricks.yml @@ -0,0 +1,49 @@ +bundle: + name: test-bundle + cluster_id: existing-cluster + +variables: + cluster: + type: complex + default: + spark_version: 15.4.x-scala2.12 + node_type_id: i3.xlarge + num_workers: 1 + +resources: + clusters: + source: + cluster_name: source + spark_version: 15.4.x-scala2.12 + node_type_id: i3.xlarge + num_workers: 2 + jobs: + job: + name: job + tasks: + - task_key: task + notebook_task: + notebook_path: /Workspace/notebook + # Replaced by bundle.cluster_id: the reference below new_cluster must go too. + new_cluster: + spark_version: 15.4.x-scala2.12 + node_type_id: i3.xlarge + num_workers: ${resources.clusters.source.num_workers} + - task_key: task_var + notebook_task: + notebook_path: /Workspace/notebook + # A reference at new_cluster must not bring it back either. + new_cluster: ${var.cluster} + - task_key: task_resource_ref + notebook_task: + notebook_path: /Workspace/notebook + # A reference to the new_cluster of another job is resolved after the override. + new_cluster: ${resources.jobs.other.job_clusters[0].new_cluster} + other: + name: other + job_clusters: + - job_cluster_key: shared + new_cluster: + spark_version: 15.4.x-scala2.12 + node_type_id: i3.xlarge + num_workers: 3 diff --git a/acceptance/bundle/override/cluster_id_clears_new_cluster_reference/out.test.toml b/acceptance/bundle/override/cluster_id_clears_new_cluster_reference/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/override/cluster_id_clears_new_cluster_reference/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/override/cluster_id_clears_new_cluster_reference/output.txt b/acceptance/bundle/override/cluster_id_clears_new_cluster_reference/output.txt new file mode 100644 index 00000000000..14e1e4bbefd --- /dev/null +++ b/acceptance/bundle/override/cluster_id_clears_new_cluster_reference/output.txt @@ -0,0 +1,27 @@ + +>>> [CLI] bundle validate -o json +[ + { + "task_key": "task", + "existing_cluster_id": "existing-cluster", + "new_cluster": null + }, + { + "task_key": "task_resource_ref", + "existing_cluster_id": "existing-cluster", + "new_cluster": null + }, + { + "task_key": "task_var", + "existing_cluster_id": "existing-cluster", + "new_cluster": null + } +] + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Created clusters.source +Created jobs.job +Created jobs.other +Files: 1 uploaded, 0 deleted +Resources: 3 created, 0 changed, 0 deleted, 0 unchanged diff --git a/acceptance/bundle/override/cluster_id_clears_new_cluster_reference/script b/acceptance/bundle/override/cluster_id_clears_new_cluster_reference/script new file mode 100644 index 00000000000..2016268365d --- /dev/null +++ b/acceptance/bundle/override/cluster_id_clears_new_cluster_reference/script @@ -0,0 +1,3 @@ +# bundle.cluster_id replaces new_cluster on every task, also when new_cluster holds a reference. +trace $CLI bundle validate -o json | jq '[.resources.jobs.job.tasks[] | {task_key, existing_cluster_id, new_cluster}]' +trace $CLI bundle deploy diff --git a/acceptance/bundle/override/preset_clears_forced_schedule_reference/databricks.yml b/acceptance/bundle/override/preset_clears_forced_schedule_reference/databricks.yml new file mode 100644 index 00000000000..25315e8078e --- /dev/null +++ b/acceptance/bundle/override/preset_clears_forced_schedule_reference/databricks.yml @@ -0,0 +1,39 @@ +bundle: + name: test-bundle + +resources: + quality_monitors: + src: + table_name: main.default.src + assets_dir: /Workspace/monitors/src + output_schema_name: main.default + inference_log: + granularities: [1 day] + model_id_col: model_id + prediction_col: prediction + problem_type: PROBLEM_TYPE_CLASSIFICATION + timestamp_col: timestamp + schedule: + quartz_cron_expression: "0 0 12 * * ?" + timezone_id: UTC + dst: + table_name: main.default.dst + assets_dir: /Workspace/monitors/dst + output_schema_name: main.default + inference_log: + granularities: [1 day] + model_id_col: model_id + prediction_col: prediction + problem_type: PROBLEM_TYPE_CLASSIFICATION + timestamp_col: timestamp + # Forced null: the target override replaces it with a reference. + schedule: null + +targets: + dev: + presets: + trigger_pause_status: PAUSED + resources: + quality_monitors: + dst: + schedule: ${resources.quality_monitors.src.schedule} diff --git a/acceptance/bundle/override/preset_clears_forced_schedule_reference/out.test.toml b/acceptance/bundle/override/preset_clears_forced_schedule_reference/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/override/preset_clears_forced_schedule_reference/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/override/preset_clears_forced_schedule_reference/output.txt b/acceptance/bundle/override/preset_clears_forced_schedule_reference/output.txt new file mode 100644 index 00000000000..eb07e4ec538 --- /dev/null +++ b/acceptance/bundle/override/preset_clears_forced_schedule_reference/output.txt @@ -0,0 +1,13 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/dev/files... +Created quality_monitors.dst +Created quality_monitors.src +Files: 1 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py //tables/main.default.dst/monitor +{ + "has_schedule": false, + "schedule": null +} diff --git a/acceptance/bundle/override/preset_clears_forced_schedule_reference/script b/acceptance/bundle/override/preset_clears_forced_schedule_reference/script new file mode 100644 index 00000000000..37d355db8c8 --- /dev/null +++ b/acceptance/bundle/override/preset_clears_forced_schedule_reference/script @@ -0,0 +1,4 @@ +# trigger_pause_status: PAUSED removes the schedules. The target override replaces the null schedule of dst +# with a reference to the schedule of src, which is removed too: the request carries no schedule. +trace $CLI bundle deploy +trace print_requests.py //tables/main.default.dst/monitor | jq '.body | {has_schedule: has("schedule"), schedule}' diff --git a/acceptance/bundle/override/preset_clears_forced_schedule_reference/test.toml b/acceptance/bundle/override/preset_clears_forced_schedule_reference/test.toml new file mode 100644 index 00000000000..159efe02696 --- /dev/null +++ b/acceptance/bundle/override/preset_clears_forced_schedule_reference/test.toml @@ -0,0 +1 @@ +RecordRequests = true diff --git a/acceptance/bundle/resource_deps/preset_tags_not_applied_to_whole_map_reference/databricks.yml b/acceptance/bundle/resource_deps/preset_tags_not_applied_to_whole_map_reference/databricks.yml new file mode 100644 index 00000000000..9b2e3df1ddf --- /dev/null +++ b/acceptance/bundle/resource_deps/preset_tags_not_applied_to_whole_map_reference/databricks.yml @@ -0,0 +1,18 @@ +bundle: + name: test-bundle + +presets: + tags: + preset_tag: ${resources.jobs.j.id} + +resources: + schemas: + s: + name: s + catalog_name: main + properties: {owner: fixed} + jobs: + j: + name: j + # tags is a reference to a whole map, and the preset tag above is not added to it. + tags: ${resources.schemas.s.properties} diff --git a/acceptance/bundle/resource_deps/preset_tags_not_applied_to_whole_map_reference/out.test.toml b/acceptance/bundle/resource_deps/preset_tags_not_applied_to_whole_map_reference/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/resource_deps/preset_tags_not_applied_to_whole_map_reference/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resource_deps/preset_tags_not_applied_to_whole_map_reference/output.txt b/acceptance/bundle/resource_deps/preset_tags_not_applied_to_whole_map_reference/output.txt new file mode 100644 index 00000000000..06316029ed1 --- /dev/null +++ b/acceptance/bundle/resource_deps/preset_tags_not_applied_to_whole_map_reference/output.txt @@ -0,0 +1,13 @@ + +>>> [CLI] bundle plan -o json +{ + "tags": { + "owner": "fixed" + }, + "depends_on": [ + { + "node": "resources.schemas.s", + "label": "${resources.schemas.s.properties}" + } + ] +} diff --git a/acceptance/bundle/resource_deps/preset_tags_not_applied_to_whole_map_reference/script b/acceptance/bundle/resource_deps/preset_tags_not_applied_to_whole_map_reference/script new file mode 100644 index 00000000000..a28b61a766a --- /dev/null +++ b/acceptance/bundle/resource_deps/preset_tags_not_applied_to_whole_map_reference/script @@ -0,0 +1,5 @@ +# The preset tag is not applied to a job whose tags is a reference to a whole map: the job gets the +# tags of the referenced schema only, and depends on the schema. +# The preset value references the job itself. If the preset tag were applied, this plan would fail +# with a cycle error. +trace $CLI bundle plan -o json | jq '.plan["resources.jobs.j"] | {tags: .new_state.value.tags, depends_on}' diff --git a/acceptance/bundle/resource_deps/preset_tags_not_applied_to_whole_map_reference/test.toml b/acceptance/bundle/resource_deps/preset_tags_not_applied_to_whole_map_reference/test.toml new file mode 100644 index 00000000000..e329f5af27f --- /dev/null +++ b/acceptance/bundle/resource_deps/preset_tags_not_applied_to_whole_map_reference/test.toml @@ -0,0 +1 @@ +Badness = "presets.tags are silently not applied to a job whose tags is a reference to a whole map." diff --git a/acceptance/bundle/validate/required_struct_reference/databricks.yml b/acceptance/bundle/validate/required_struct_reference/databricks.yml new file mode 100644 index 00000000000..60e02ad20cb --- /dev/null +++ b/acceptance/bundle/validate/required_struct_reference/databricks.yml @@ -0,0 +1,17 @@ +bundle: + name: test-bundle + +resources: + clusters: + source: + cluster_name: source + spark_version: 15.4.x-scala2.12 + is_single_node: true + target: + cluster_name: target + spark_version: 15.4.x-scala2.12 + # clients is a required field of workload_type. Both of its fields are references. + workload_type: + clients: + notebooks: ${resources.clusters.source.is_single_node} + jobs: ${resources.clusters.source.is_single_node} diff --git a/acceptance/bundle/validate/required_struct_reference/out.test.toml b/acceptance/bundle/validate/required_struct_reference/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/validate/required_struct_reference/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/validate/required_struct_reference/output.txt b/acceptance/bundle/validate/required_struct_reference/output.txt new file mode 100644 index 00000000000..2fc3e5bc32e --- /dev/null +++ b/acceptance/bundle/validate/required_struct_reference/output.txt @@ -0,0 +1,8 @@ + +>>> [CLI] bundle validate -o json +{ + "clients": { + "jobs": "${resources.clusters.source.is_single_node}", + "notebooks": "${resources.clusters.source.is_single_node}" + } +} diff --git a/acceptance/bundle/validate/required_struct_reference/script b/acceptance/bundle/validate/required_struct_reference/script new file mode 100644 index 00000000000..5d37f63d12b --- /dev/null +++ b/acceptance/bundle/validate/required_struct_reference/script @@ -0,0 +1,2 @@ +# workload_type.clients is required. A clients whose fields are all references is not reported as missing. +trace $CLI bundle validate -o json | jq .resources.clusters.target.workload_type