Add lifecycle.depends_on to order resources without references

.nextchanges/bundles/depends-on.md

@@ -0,0 +1 @@
+* Added `lifecycle.depends_on` to order resources without referencing their fields. ([#7120](https://github.com/databricks/cli/pull/7120))

acceptance/bundle/lifecycle/depends-on-validation/databricks.yml

@@ -0,0 +1,16 @@
+bundle:
+  name: test-bundle
+
+resources:
+  jobs:
+    a_job:
+      name: a_job
+      lifecycle:
+        depends_on:
+          - resource: a_job
+          - resource: missing
+    b_job:
+      name: b_job
+      lifecycle:
+        depends_on:
+          - resource: a.b.c

acceptance/bundle/lifecycle/depends-on-validation/out.test.toml

@@ -0,0 +1,2 @@
+Cloud = false
+EnvMatrix.DMS = ["", "true"]

acceptance/bundle/lifecycle/depends-on-validation/output.txt

@@ -0,0 +1,19 @@
+Error: invalid lifecycle.depends_on: resource "a_job" cannot depend on itself
+  at resources.jobs.a_job.lifecycle.depends_on[0].resource
+  in databricks.yml:10:13
+
+Error: invalid lifecycle.depends_on: no such resource: missing
+  at resources.jobs.a_job.lifecycle.depends_on[1].resource
+  in databricks.yml:11:13
+
+Error: invalid lifecycle.depends_on: no such resource: a.b.c
+  at resources.jobs.b_job.lifecycle.depends_on[0].resource
+  in databricks.yml:16:13
+
+Name: test-bundle
+Target: default
+Workspace:
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default
+
+Found 3 errors

acceptance/bundle/lifecycle/depends-on-validation/script

@@ -0,0 +1 @@
+musterr $CLI bundle validate

acceptance/bundle/lifecycle/depends-on-validation/test.toml

@@ -0,0 +1,2 @@
+Cloud = false
+Ignore = [".databricks"]

acceptance/bundle/lifecycle/depends-on/databricks.yml

@@ -0,0 +1,17 @@
+bundle:
+  name: test-bundle
+
+resources:
+  jobs:
+    # No references between the jobs: the order comes from lifecycle.depends_on only.
+    a_job:
+      name: a_job
+      lifecycle:
+        depends_on:
+          - resource: z_job
+          - resource: pipelines.z_pipeline
+    z_job:
+      name: z_job
+  pipelines:
+    z_pipeline:
+      name: z_pipeline

acceptance/bundle/lifecycle/depends-on/out.test.toml

@@ -0,0 +1,2 @@
+Cloud = false
+EnvMatrix.DMS = ["", "true"]

acceptance/bundle/lifecycle/depends-on/output.txt

@@ -0,0 +1,45 @@
+
+=== Plan records the edges
+>>> [CLI] bundle plan -o json
+{
+  "resources.jobs.a_job": [
+    {
+      "node": "resources.jobs.z_job",
+      "label": "depends_on"
+    },
+    {
+      "node": "resources.pipelines.z_pipeline",
+      "label": "depends_on"
+    }
+  ],
+  "resources.jobs.z_job": null,
+  "resources.pipelines.z_pipeline": null
+}
+
+=== Deploy creates dependencies first
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
+Created jobs.a_job
+Created jobs.z_job
+Created pipelines.z_pipeline
+Files: 1 uploaded, 0 deleted
+Resources: 3 created, 0 changed, 0 deleted, 0 unchanged
+
+>>> print_requests.py //jobs //pipelines
+POST /api/2.2/jobs/create a_job
+
+=== Destroy deletes dependents first
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.jobs.a_job
+  delete resources.jobs.z_job
+  delete resources.pipelines.z_pipeline
+
+This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the
+Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion:
+  delete resources.pipelines.z_pipeline
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default
+
+Destroy: 3 deleted
+a_job deleted first

acceptance/bundle/lifecycle/depends-on/script

@@ -0,0 +1,15 @@
+title "Plan records the edges"
+trace $CLI bundle plan -o json | jq '.plan | map_values(.depends_on)'
+
+title "Deploy creates dependencies first"
+trace $CLI bundle deploy
+# z_job and z_pipeline are unordered relative to each other, so only check that a_job is last.
+trace print_requests.py //jobs //pipelines | jq -r '.method + " " + .path + " " + (.body.name // "")' | tail -n 1
+
+rm -f out.requests.txt
+
+title "Destroy deletes dependents first"
+a_job_id=$($CLI bundle summary -o json | jq -r .resources.jobs.a_job.id)
+trace $CLI bundle destroy --auto-approve
+first_deleted=$(print_requests.py //jobs/delete | jq -r .body.job_id | head -n 1)
+test "$first_deleted" = "$a_job_id" && echo "a_job deleted first"

acceptance/bundle/lifecycle/depends-on/test.toml

@@ -0,0 +1,3 @@
+Cloud = false
+RecordRequests = true
+Ignore = [".databricks"]

acceptance/bundle/migrate/auto/depends-on/databricks.yml

@@ -0,0 +1,12 @@
+bundle: {name: test-bundle}
+
+resources:
+  jobs:
+    test_job:
+      name: "Test Auto-Migrate Job"
+    # Added after the terraform deployment; terraform has no lifecycle.depends_on.
+    other_job:
+      name: "Other Job"
+      lifecycle:
+        depends_on:
+          - resource: test_job

acceptance/bundle/migrate/auto/depends-on/out.test.toml

@@ -0,0 +1,2 @@
+Cloud = false
+EnvMatrix.DMS = [""]

acceptance/bundle/migrate/auto/depends-on/output.txt

@@ -0,0 +1,27 @@
+
+=== Replay a prior terraform deployment
+>>> replay_tfstate.py terraform-requests.json terraform.tfstate
+
+=== Deploy migrates to direct and honors lifecycle.depends_on
+>>> [CLI] bundle deploy
+Notice: migrating your bundle to direct deployment engine (https://github.com/databricks/cli/issues/6765).
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
+Migrated 1 resource to direct deployment engine.
+Created jobs.other_job
+Files: 1 uploaded, 0 deleted
+Resources: 1 created, 0 changed, 0 deleted, 1 unchanged
+
+>>> print_migration_telemetry
+direct_migrated_via_default true
+
+=== The edge is recorded in the plan
+>>> [CLI] bundle plan -o json
+{
+  "resources.jobs.other_job": [
+    {
+      "node": "resources.jobs.test_job",
+      "label": "depends_on"
+    }
+  ],
+  "resources.jobs.test_job": null
+}

acceptance/bundle/migrate/auto/depends-on/script

@@ -0,0 +1,13 @@
+export DATABRICKS_BUNDLE_ENGINE=
+
+title "Replay a prior terraform deployment"
+trace replay_tfstate.py terraform-requests.json terraform.tfstate
+replace_ids.py
+rm -f "$OUT_REQUESTS"
+
+title "Deploy migrates to direct and honors lifecycle.depends_on"
+trace $CLI bundle deploy
+trace print_migration_telemetry
+
+title "The edge is recorded in the plan"
+trace $CLI bundle plan -o json | jq '.plan | map_values(.depends_on)'

acceptance/bundle/migrate/auto/depends-on/terraform-requests.json

@@ -0,0 +1,19 @@
+[
+  {
+    "method": "POST",
+    "path": "/api/2.2/jobs/create",
+    "body": {
+      "deployment": {
+        "kind": "BUNDLE",
+        "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/test-bundle/default/state/metadata.json"
+      },
+      "edit_mode": "UI_LOCKED",
+      "format": "MULTI_TASK",
+      "max_concurrent_runs": 1,
+      "name": "Test Auto-Migrate Job",
+      "queue": {
+        "enabled": true
+      }
+    }
+  }
+]

acceptance/bundle/migrate/auto/depends-on/terraform.tfstate

@@ -0,0 +1,110 @@
+{
+  "version": 4,
+  "terraform_version": "1.5.5",
+  "serial": 1,
+  "lineage": "c39488aa-8f36-06fd-a7cb-3f0c95b4c506",
+  "outputs": {},
+  "resources": [
+    {
+      "mode": "managed",
+      "type": "databricks_job",
+      "name": "test_job",
+      "provider": "provider[\"registry.terraform.io/databricks/databricks\"]",
+      "instances": [
+        {
+          "schema_version": 2,
+          "attributes": {
+            "always_running": false,
+            "budget_policy_id": null,
+            "continuous": [],
+            "control_run_state": false,
+            "dbt_task": [],
+            "deployment": [
+              {
+                "deployment_id": "",
+                "kind": "BUNDLE",
+                "metadata_file_path": "/Workspace/Users/tester@databricks.com/.bundle/test-bundle/default/state/metadata.json",
+                "version_id": ""
+              }
+            ],
+            "description": null,
+            "edit_mode": "UI_LOCKED",
+            "email_notifications": [
+              {
+                "no_alert_for_skipped_runs": false,
+                "on_duration_warning_threshold_exceeded": [],
+                "on_failure": [],
+                "on_start": [],
+                "on_streaming_backlog_exceeded": [],
+                "on_success": []
+              }
+            ],
+            "environment": [],
+            "existing_cluster_id": null,
+            "format": "MULTI_TASK",
+            "git_source": [],
+            "health": [],
+            "id": "8790612527183335000",
+            "job_cluster": [],
+            "library": [],
+            "max_concurrent_runs": 1,
+            "max_retries": 0,
+            "min_retry_interval_millis": 0,
+            "name": "Test Auto-Migrate Job",
+            "new_cluster": [],
+            "notebook_task": [],
+            "notification_settings": [],
+            "parameter": [],
+            "parent_path": null,
+            "performance_target": null,
+            "pipeline_task": [],
+            "provider_config": [
+              {
+                "workspace_id": "900800700600"
+              }
+            ],
+            "python_wheel_task": [],
+            "queue": [
+              {
+                "enabled": true
+              }
+            ],
+            "retry_on_timeout": false,
+            "run_as": [
+              {
+                "group_name": "",
+                "service_principal_name": "",
+                "user_name": "tester@databricks.com"
+              }
+            ],
+            "run_job_task": [],
+            "schedule": [],
+            "spark_jar_task": [],
+            "spark_python_task": [],
+            "spark_submit_task": [],
+            "tags": null,
+            "task": [],
+            "timeout_seconds": 0,
+            "timeouts": null,
+            "trigger": [],
+            "triggers": [],
+            "url": "[DATABRICKS_URL]/#job/8790612527183335000",
+            "usage_policy_id": null,
+            "webhook_notifications": [
+              {
+                "on_duration_warning_threshold_exceeded": [],
+                "on_failure": [],
+                "on_start": [],
+                "on_streaming_backlog_exceeded": [],
+                "on_success": []
+              }
+            ]
+          },
+          "sensitive_attributes": [],
+          "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxODAwMDAwMDAwMDAwLCJ1cGRhdGUiOjE4MDAwMDAwMDAwMDB9LCJzY2hlbWFfdmVyc2lvbiI6IjIifQ=="
+        }
+      ]
+    }
+  ],
+  "check_results": null
+}