acc: record main's behaviour for reference, override and required-field edge cases

acceptance/bundle/override/cluster_id_clears_new_cluster_reference/databricks.yml

@@ -0,0 +1,49 @@
+bundle:
+  name: test-bundle
+  cluster_id: existing-cluster
+
+variables:
+  cluster:
+    type: complex
+    default:
+      spark_version: 15.4.x-scala2.12
+      node_type_id: i3.xlarge
+      num_workers: 1
+
+resources:
+  clusters:
+    source:
+      cluster_name: source
+      spark_version: 15.4.x-scala2.12
+      node_type_id: i3.xlarge
+      num_workers: 2
+  jobs:
+    job:
+      name: job
+      tasks:
+        - task_key: task
+          notebook_task:
+            notebook_path: /Workspace/notebook
+          # Replaced by bundle.cluster_id: the reference below new_cluster must go too.
+          new_cluster:
+            spark_version: 15.4.x-scala2.12
+            node_type_id: i3.xlarge
+            num_workers: ${resources.clusters.source.num_workers}
+        - task_key: task_var
+          notebook_task:
+            notebook_path: /Workspace/notebook
+          # A reference at new_cluster must not bring it back either.
+          new_cluster: ${var.cluster}
+        - task_key: task_resource_ref
+          notebook_task:
+            notebook_path: /Workspace/notebook
+          # A reference to the new_cluster of another job is resolved after the override.
+          new_cluster: ${resources.jobs.other.job_clusters[0].new_cluster}
+    other:
+      name: other
+      job_clusters:
+        - job_cluster_key: shared
+          new_cluster:
+            spark_version: 15.4.x-scala2.12
+            node_type_id: i3.xlarge
+            num_workers: 3

acceptance/bundle/override/cluster_id_clears_new_cluster_reference/out.test.toml

@@ -0,0 +1,2 @@
+Cloud = false
+EnvMatrix.DMS = ["", "true"]

acceptance/bundle/override/cluster_id_clears_new_cluster_reference/output.txt

@@ -0,0 +1,27 @@
+
+>>> [CLI] bundle validate -o json
+[
+  {
+    "task_key": "task",
+    "existing_cluster_id": "existing-cluster",
+    "new_cluster": null
+  },
+  {
+    "task_key": "task_resource_ref",
+    "existing_cluster_id": "existing-cluster",
+    "new_cluster": null
+  },
+  {
+    "task_key": "task_var",
+    "existing_cluster_id": "existing-cluster",
+    "new_cluster": null
+  }
+]
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
+Created clusters.source
+Created jobs.job
+Created jobs.other
+Files: 1 uploaded, 0 deleted
+Resources: 3 created, 0 changed, 0 deleted, 0 unchanged

acceptance/bundle/override/cluster_id_clears_new_cluster_reference/script

@@ -0,0 +1,3 @@
+# bundle.cluster_id replaces new_cluster on every task, also when new_cluster holds a reference.
+trace $CLI bundle validate -o json | jq '[.resources.jobs.job.tasks[] | {task_key, existing_cluster_id, new_cluster}]'
+trace $CLI bundle deploy

acceptance/bundle/override/preset_clears_forced_schedule_reference/databricks.yml

@@ -0,0 +1,39 @@
+bundle:
+  name: test-bundle
+
+resources:
+  quality_monitors:
+    src:
+      table_name: main.default.src
+      assets_dir: /Workspace/monitors/src
+      output_schema_name: main.default
+      inference_log:
+        granularities: [1 day]
+        model_id_col: model_id
+        prediction_col: prediction
+        problem_type: PROBLEM_TYPE_CLASSIFICATION
+        timestamp_col: timestamp
+      schedule:
+        quartz_cron_expression: "0 0 12 * * ?"
+        timezone_id: UTC
+    dst:
+      table_name: main.default.dst
+      assets_dir: /Workspace/monitors/dst
+      output_schema_name: main.default
+      inference_log:
+        granularities: [1 day]
+        model_id_col: model_id
+        prediction_col: prediction
+        problem_type: PROBLEM_TYPE_CLASSIFICATION
+        timestamp_col: timestamp
+      # Forced null: the target override replaces it with a reference.
+      schedule: null
+
+targets:
+  dev:
+    presets:
+      trigger_pause_status: PAUSED
+    resources:
+      quality_monitors:
+        dst:
+          schedule: ${resources.quality_monitors.src.schedule}

acceptance/bundle/override/preset_clears_forced_schedule_reference/out.test.toml

@@ -0,0 +1,2 @@
+Cloud = false
+EnvMatrix.DMS = ["", "true"]

acceptance/bundle/override/preset_clears_forced_schedule_reference/output.txt

@@ -0,0 +1,13 @@
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/dev/files...
+Created quality_monitors.dst
+Created quality_monitors.src
+Files: 1 uploaded, 0 deleted
+Resources: 2 created, 0 changed, 0 deleted, 0 unchanged
+
+>>> print_requests.py //tables/main.default.dst/monitor
+{
+  "has_schedule": false,
+  "schedule": null
+}

acceptance/bundle/override/preset_clears_forced_schedule_reference/script

@@ -0,0 +1,4 @@
+# trigger_pause_status: PAUSED removes the schedules. The target override replaces the null schedule of dst
+# with a reference to the schedule of src, which is removed too: the request carries no schedule.
+trace $CLI bundle deploy
+trace print_requests.py //tables/main.default.dst/monitor | jq '.body | {has_schedule: has("schedule"), schedule}'

acceptance/bundle/override/preset_clears_forced_schedule_reference/test.toml

@@ -0,0 +1 @@
+RecordRequests = true

acceptance/bundle/resource_deps/whole_map_reference_hides_entries/databricks.yml

@@ -0,0 +1,18 @@
+bundle:
+  name: test-bundle
+
+presets:
+  tags:
+    preset_tag: ${resources.jobs.j.id}
+
+resources:
+  schemas:
+    s:
+      name: s
+      catalog_name: main
+      properties: {owner: fixed}
+  jobs:
+    j:
+      name: j
+      # The whole tags map is a reference, so the preset tag above is not added to it.
+      tags: ${resources.schemas.s.properties}

acceptance/bundle/resource_deps/whole_map_reference_hides_entries/out.test.toml

@@ -0,0 +1,2 @@
+Cloud = false
+EnvMatrix.DMS = ["", "true"]

acceptance/bundle/resource_deps/whole_map_reference_hides_entries/output.txt

@@ -0,0 +1,13 @@
+
+>>> [CLI] bundle plan -o json
+{
+  "tags": {
+    "owner": "fixed"
+  },
+  "depends_on": [
+    {
+      "node": "resources.schemas.s",
+      "label": "${resources.schemas.s.properties}"
+    }
+  ]
+}

acceptance/bundle/resource_deps/whole_map_reference_hides_entries/script

@@ -0,0 +1,3 @@
+# When a whole map is a reference, the entries a preset adds to it are not applied: the job gets the
+# tags of the referenced schema only, and depends on the schema.
+trace $CLI bundle plan -o json | jq '.plan["resources.jobs.j"] | {tags: .new_state.value.tags, depends_on}'

acceptance/bundle/validate/nulls/target_adds_null_keeps_it/databricks.yml

@@ -0,0 +1,18 @@
+bundle:
+  name: test-bundle
+
+resources:
+  jobs:
+    j:
+      name: j
+      tasks:
+        - task_key: t
+          notebook_task:
+            notebook_path: ./nb.py
+
+targets:
+  dev:
+    resources:
+      jobs:
+        j:
+          queue: null

acceptance/bundle/validate/nulls/target_adds_null_keeps_it/nb.py

@@ -0,0 +1,2 @@
+# Databricks notebook source
+print(1)

acceptance/bundle/validate/nulls/target_adds_null_keeps_it/out.test.toml

@@ -0,0 +1,2 @@
+Cloud = false
+EnvMatrix.DMS = ["", "true"]

acceptance/bundle/validate/nulls/target_adds_null_keeps_it/output.txt

@@ -0,0 +1,7 @@
+
+>>> [CLI] bundle validate -t dev -o json
+Warning: required field "enabled" is not set
+  at resources.jobs.j.queue
+  in databricks.yml:18:18
+
+null

acceptance/bundle/validate/nulls/target_adds_null_keeps_it/script

@@ -0,0 +1,2 @@
+# A target can add an explicit null that is kept, like a null in the base file: the default queue is not applied.
+trace $CLI bundle validate -t dev -o json | jq '.resources.jobs.j.queue'

acceptance/bundle/validate/required_struct_reference/databricks.yml

@@ -0,0 +1,17 @@
+bundle:
+  name: test-bundle
+
+resources:
+  clusters:
+    source:
+      cluster_name: source
+      spark_version: 15.4.x-scala2.12
+      is_single_node: true
+    target:
+      cluster_name: target
+      spark_version: 15.4.x-scala2.12
+      # clients is a required field of workload_type. Both of its fields are references.
+      workload_type:
+        clients:
+          notebooks: ${resources.clusters.source.is_single_node}
+          jobs: ${resources.clusters.source.is_single_node}

acceptance/bundle/validate/required_struct_reference/out.test.toml

@@ -0,0 +1,2 @@
+Cloud = false
+EnvMatrix.DMS = ["", "true"]

acceptance/bundle/validate/required_struct_reference/output.txt

@@ -0,0 +1,8 @@
+
+>>> [CLI] bundle validate -o json
+{
+  "clients": {
+    "jobs": "${resources.clusters.source.is_single_node}",
+    "notebooks": "${resources.clusters.source.is_single_node}"
+  }
+}

acceptance/bundle/validate/required_struct_reference/script

@@ -0,0 +1,2 @@
+# workload_type.clients is required. A clients whose fields are all references is not reported as missing.
+trace $CLI bundle validate -o json | jq .resources.clusters.target.workload_type