add libraries field for Clusters

.nextchanges/bundles/cluster-libraries.md

@@ -0,0 +1 @@
+Add support for a `libraries` list on the `clusters` resource type in Declarative Automation Bundles. Libraries (whl, jar, pypi, maven, cran, egg, requirements) are installed on the all-purpose cluster via the Libraries API; local wheels/jars are built and uploaded automatically. Cluster libraries are only supported in direct deployment mode.

acceptance/bundle/invariant/configs/cluster_libraries.yml.tmpl

@@ -0,0 +1,14 @@
+bundle:
+  name: test-bundle-$UNIQUE_NAME
+
+resources:
+  clusters:
+    foo:
+      cluster_name: test-cluster-$UNIQUE_NAME
+      spark_version: 13.3.x-scala2.12
+      node_type_id: $NODE_TYPE_ID
+      instance_pool_id: $TEST_INSTANCE_POOL_ID
+      num_workers: 1
+      libraries:
+        - pypi:
+            package: requests

acceptance/bundle/invariant/migrate/test.toml

@@ -16,6 +16,8 @@ EnvMatrixExclude.no_genie_space = ["INPUT_CONFIG=genie_space.yml.tmpl"]
 EnvMatrixExclude.no_instance_pool = ["INPUT_CONFIG=instance_pool.yml.tmpl"]
 # Cluster policies are direct-only; the terraform deploy that seeds the migration fails for them.
 EnvMatrixExclude.no_cluster_policy = ["INPUT_CONFIG=cluster_policy.yml.tmpl"]
+# Cluster libraries are direct-only; the terraform deploy that seeds the migration fails for them.
+EnvMatrixExclude.no_cluster_libraries = ["INPUT_CONFIG=cluster_libraries.yml.tmpl"]
 
 # Cross-resource permission references (e.g. ${resources.jobs.job_b.permissions[0].level})
 # don't work in terraform mode: the terraform interpolator converts the path to

acceptance/bundle/invariant/test.toml

@@ -25,6 +25,7 @@ EnvMatrix.INPUT_CONFIG = [
     "catalog_optional_fields.yml.tmpl",
     "cluster.yml.tmpl",
     "cluster_apply_policy_default_values.yml.tmpl",
+    "cluster_libraries.yml.tmpl",
     "cluster_policy.yml.tmpl",
     "dashboard.yml.tmpl",
     "job_apply_policy_default_values_job_cluster.yml.tmpl",

acceptance/bundle/refschema/out.fields.txt

@@ -460,6 +460,23 @@ resources.clusters.*.jdbc_port	int	REMOTE
 resources.clusters.*.kind	compute.Kind	ALL
 resources.clusters.*.last_restarted_time	int64	REMOTE
 resources.clusters.*.last_state_loss_time	int64	REMOTE
+resources.clusters.*.libraries	[]compute.Library	INPUT
+resources.clusters.*.libraries[*]	compute.Library	INPUT
+resources.clusters.*.libraries[*].cran	*compute.RCranLibrary	INPUT
+resources.clusters.*.libraries[*].cran.package	string	INPUT
+resources.clusters.*.libraries[*].cran.repo	string	INPUT
+resources.clusters.*.libraries[*].egg	string	INPUT
+resources.clusters.*.libraries[*].jar	string	INPUT
+resources.clusters.*.libraries[*].maven	*compute.MavenLibrary	INPUT
+resources.clusters.*.libraries[*].maven.coordinates	string	INPUT
+resources.clusters.*.libraries[*].maven.exclusions	[]string	INPUT
+resources.clusters.*.libraries[*].maven.exclusions[*]	string	INPUT
+resources.clusters.*.libraries[*].maven.repo	string	INPUT
+resources.clusters.*.libraries[*].pypi	*compute.PythonPyPiLibrary	INPUT
+resources.clusters.*.libraries[*].pypi.package	string	INPUT
+resources.clusters.*.libraries[*].pypi.repo	string	INPUT
+resources.clusters.*.libraries[*].requirements	string	INPUT
+resources.clusters.*.libraries[*].whl	string	INPUT
 resources.clusters.*.lifecycle	*dresources.StateLifecycle	REMOTE	STATE
 resources.clusters.*.lifecycle	*resources.LifecycleWithStarted	INPUT
 resources.clusters.*.lifecycle	resources.Lifecycle	INPUT
@@ -610,6 +627,23 @@ resources.clusters.*.workload_type	*compute.WorkloadType	ALL
 resources.clusters.*.workload_type.clients	compute.ClientsTypes	ALL
 resources.clusters.*.workload_type.clients.jobs	bool	ALL
 resources.clusters.*.workload_type.clients.notebooks	bool	ALL
+resources.clusters.*.libraries.cluster_id	string	ALL
+resources.clusters.*.libraries[*]	compute.Library	ALL
+resources.clusters.*.libraries[*].cran	*compute.RCranLibrary	ALL
+resources.clusters.*.libraries[*].cran.package	string	ALL
+resources.clusters.*.libraries[*].cran.repo	string	ALL
+resources.clusters.*.libraries[*].egg	string	ALL
+resources.clusters.*.libraries[*].jar	string	ALL
+resources.clusters.*.libraries[*].maven	*compute.MavenLibrary	ALL
+resources.clusters.*.libraries[*].maven.coordinates	string	ALL
+resources.clusters.*.libraries[*].maven.exclusions	[]string	ALL
+resources.clusters.*.libraries[*].maven.exclusions[*]	string	ALL
+resources.clusters.*.libraries[*].maven.repo	string	ALL
+resources.clusters.*.libraries[*].pypi	*compute.PythonPyPiLibrary	ALL
+resources.clusters.*.libraries[*].pypi.package	string	ALL
+resources.clusters.*.libraries[*].pypi.repo	string	ALL
+resources.clusters.*.libraries[*].requirements	string	ALL
+resources.clusters.*.libraries[*].whl	string	ALL
 resources.clusters.*.permissions.object_id	string	ALL
 resources.clusters.*.permissions[*]	dresources.StatePermission	ALL
 resources.clusters.*.permissions[*].group_name	string	ALL

acceptance/bundle/resources/clusters/libraries-drift/databricks.yml.tmpl

@@ -0,0 +1,17 @@
+bundle:
+  name: cluster-libraries-drift-$UNIQUE_NAME
+
+workspace:
+  root_path: ~/.bundle/$UNIQUE_NAME
+
+resources:
+  clusters:
+    mycluster:
+      cluster_name: mycluster-$UNIQUE_NAME
+      spark_version: $DEFAULT_SPARK_VERSION
+      node_type_id: $NODE_TYPE_ID
+      instance_pool_id: $TEST_INSTANCE_POOL_ID
+      num_workers: 1
+      libraries:
+        - pypi:
+            package: requests

acceptance/bundle/resources/clusters/libraries-drift/output.txt

@@ -0,0 +1,12 @@
+
+=== Plan is a no-op immediately after deploy (no library drift)
+>>> [CLI] bundle plan
+Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged
+
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.clusters.mycluster
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]
+
+Destroy: 1 deleted

acceptance/bundle/resources/clusters/libraries-drift/script

@@ -0,0 +1,20 @@
+# A pypi library is used because its nested {package, repo} shape is the most
+# likely to drift (the status API echoing a repo we did not set); a workspace
+# wheel is covered by the local clusters/libraries test instead, since the
+# shared cloud test cluster rejects libraries from /Workspace paths.
+envsubst < databricks.yml.tmpl > databricks.yml
+
+cleanup() {
+    trace $CLI bundle destroy --auto-approve
+    rm -f out.requests.txt
+}
+trap cleanup EXIT
+
+# Cluster provisioning and library-install output is noisy and differs between
+# the fake and cloud, so route it to LOG and assert only the deterministic
+# drift signal below.
+$CLI bundle deploy &> LOG.deploy
+cat LOG.deploy | contains.py "Created clusters.mycluster.libraries" > /dev/null
+
+title "Plan is a no-op immediately after deploy (no library drift)"
+trace $CLI bundle plan

acceptance/bundle/resources/clusters/libraries-drift/test.toml

@@ -0,0 +1,6 @@
+Cloud = true
+RecordRequests = false
+
+EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"]
+
+Ignore = [".databricks", "databricks.yml"]

acceptance/bundle/resources/clusters/libraries-terraform-error/databricks.yml

@@ -0,0 +1,13 @@
+bundle:
+  name: cluster-libraries-terraform-error
+
+resources:
+  clusters:
+    mycluster:
+      cluster_name: mycluster
+      spark_version: 15.4.x-scala2.12
+      node_type_id: i3.xlarge
+      num_workers: 1
+      libraries:
+        - pypi:
+            package: requests

acceptance/bundle/resources/clusters/libraries-terraform-error/output.txt

@@ -0,0 +1,16 @@
+
+=== bundle plan fails with cluster libraries on terraform engine
+>>> errcode [CLI] bundle plan
+Error: cluster libraries are only supported in direct deployment mode
+  in databricks.yml:12:9
+
+
+Exit code: 1
+
+=== bundle deploy fails with cluster libraries on terraform engine
+>>> errcode [CLI] bundle deploy
+Error: cluster libraries are only supported in direct deployment mode
+  in databricks.yml:12:9
+
+
+Exit code: 1

acceptance/bundle/resources/clusters/libraries-terraform-error/script

@@ -0,0 +1,5 @@
+title "bundle plan fails with cluster libraries on terraform engine"
+trace errcode $CLI bundle plan
+
+title "bundle deploy fails with cluster libraries on terraform engine"
+trace errcode $CLI bundle deploy

acceptance/bundle/resources/clusters/libraries-terraform-error/test.toml

@@ -0,0 +1,4 @@
+Cloud = false
+RecordRequests = false
+
+EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform"]

acceptance/bundle/resources/clusters/libraries/databricks.yml

@@ -0,0 +1,14 @@
+bundle:
+  name: cluster-libraries
+
+resources:
+  clusters:
+    mycluster:
+      cluster_name: mycluster
+      spark_version: 15.4.x-scala2.12
+      node_type_id: i3.xlarge
+      num_workers: 1
+      libraries:
+        - pypi:
+            package: requests
+        - whl: ./dist/*.whl

acceptance/bundle/resources/clusters/libraries/dist/my_package-0.0.1-py3-none-any.whl

@@ -0,0 +1 @@
+dummy wheel contents

acceptance/bundle/resources/clusters/libraries/output.txt

@@ -0,0 +1,58 @@
+
+=== Deploy a cluster with a pypi and a local wheel library
+>>> [CLI] bundle deploy
+Uploading dist/my_package-0.0.1-py3-none-any.whl...
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/cluster-libraries/default/files...
+Created clusters.mycluster
+Created clusters.mycluster.libraries
+Files: 6 uploaded, 0 deleted
+Resources: 2 created, 0 changed, 0 deleted, 0 unchanged
+
+=== Libraries installed via the Libraries API (wheel rewritten to its uploaded path)
+>>> print_requests.py //libraries/install
+{
+  "method": "POST",
+  "path": "/api/2.0/libraries/install",
+  "body": {
+    "cluster_id": "[UUID]",
+    "libraries": [
+      {
+        "pypi": {
+          "package": "requests"
+        }
+      },
+      {
+        "whl": "/Workspace/Users/[USERNAME]/.bundle/cluster-libraries/default/artifacts/.internal/my_package-0.0.1-py3-none-any.whl"
+      }
+    ]
+  }
+}
+
+=== Removing the wheel and redeploying uninstalls it
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/cluster-libraries/default/files...
+Updated clusters.mycluster.libraries
+Files: 3 uploaded, 0 deleted
+Resources: 0 created, 1 changed, 0 deleted, 1 unchanged
+
+>>> print_requests.py //libraries/uninstall
+{
+  "method": "POST",
+  "path": "/api/2.0/libraries/uninstall",
+  "body": {
+    "cluster_id": "[UUID]",
+    "libraries": [
+      {
+        "whl": "/Workspace/Users/[USERNAME]/.bundle/cluster-libraries/default/artifacts/.internal/my_package-0.0.1-py3-none-any.whl"
+      }
+    ]
+  }
+}
+
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.clusters.mycluster
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/cluster-libraries/default
+
+Destroy: 1 deleted

acceptance/bundle/resources/clusters/libraries/script

@@ -0,0 +1,16 @@
+cleanup() {
+    trace $CLI bundle destroy --auto-approve
+    rm -f out.requests.txt
+}
+trap cleanup EXIT
+
+title "Deploy a cluster with a pypi and a local wheel library"
+trace $CLI bundle deploy
+
+title "Libraries installed via the Libraries API (wheel rewritten to its uploaded path)"
+trace print_requests.py //libraries/install
+
+title "Removing the wheel and redeploying uninstalls it"
+update_file.py databricks.yml "        - whl: ./dist/*.whl" ""
+trace $CLI bundle deploy
+trace print_requests.py //libraries/uninstall

acceptance/bundle/resources/clusters/libraries/test.toml

@@ -0,0 +1,10 @@
+Cloud = false
+RecordRequests = true
+
+EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"]
+
+Ignore = [".databricks"]
+
+[[Repls]]
+Old = "[0-9]{4}-[0-9]{6}-[0-9a-z]{8}"
+New = "[CLUSTER-ID]"

acceptance/invariant_test.go

@@ -19,8 +19,9 @@ const invariantConfigsDir = "bundle/invariant/configs"
 // LackingInvariantTest lists keys from config.ResourcesTypes that knowingly lack
 // a covering config in invariantConfigsDir. Keys match the ResourcesTypes
 // form: "<group>" for the resource itself, "<group>.permissions" / "<group>.grants"
-// for permissions/grants coverage. Add a config and remove the entry to close a gap;
-// the test fails if an entry here is actually covered, so the list only shrinks.
+// / "<group>.libraries" for sub-resource coverage. Add a config and remove the entry
+// to close a gap; the test fails if an entry here is actually covered, so the list
+// only shrinks.
 var LackingInvariantTest = map[string]bool{
 	"quality_monitors": true,
 }
@@ -30,10 +31,11 @@ var LackingInvariantTest = map[string]bool{
 // types supporting permissions or grants have at least one config exercising them.
 //
 // config.ResourcesTypes is the source of truth: it maps each resource group
-// (e.g. "jobs") to its Go type and, where the resource struct has a Permissions
-// or Grants field, adds derived keys "<group>.permissions" and "<group>.grants".
+// (e.g. "jobs") to its Go type and adds derived keys "<group>.permissions",
+// "<group>.grants", and "<group>.libraries" where the resource has the
+// corresponding sub-resource.
 func TestInvariantConfigsCoverage(t *testing.T) {
-	present, withPermissions, withGrants := scanInvariantConfigs(t)
+	present, withPermissions, withGrants, withLibraries := scanInvariantConfigs(t)
 
 	keys := make([]string, 0, len(config.ResourcesTypes))
 	for key := range config.ResourcesTypes {
@@ -53,6 +55,10 @@ func TestInvariantConfigsCoverage(t *testing.T) {
 			group := strings.TrimSuffix(key, ".grants")
 			covered = withGrants[group]
 			hint = "attaches grants to a " + group + " resource"
+		case strings.HasSuffix(key, ".libraries"):
+			group := strings.TrimSuffix(key, ".libraries")
+			covered = withLibraries[group]
+			hint = "attaches libraries to a " + group + " resource"
 		default:
 			covered = present[key]
 			hint = "defines a " + key + " resource"
@@ -69,12 +75,13 @@ func TestInvariantConfigsCoverage(t *testing.T) {
 }
 
 // scanInvariantConfigs parses every config in the invariant configs directory and
-// returns the set of resource groups present, the groups with at least one resource
-// carrying permissions, and the groups with at least one resource carrying grants.
-func scanInvariantConfigs(t *testing.T) (present, withPermissions, withGrants map[string]bool) {
+// returns the set of resource groups present, and the groups with at least one
+// resource carrying permissions, grants, or libraries.
+func scanInvariantConfigs(t *testing.T) (present, withPermissions, withGrants, withLibraries map[string]bool) {
 	present = map[string]bool{}
 	withPermissions = map[string]bool{}
 	withGrants = map[string]bool{}
+	withLibraries = map[string]bool{}
 
 	entries, err := os.ReadDir(invariantConfigsDir)
 	require.NoError(t, err)
@@ -114,9 +121,12 @@ func scanInvariantConfigs(t *testing.T) (present, withPermissions, withGrants ma
 				if cfg.Get("grants").Kind() != dyn.KindInvalid {
 					withGrants[groupName] = true
 				}
+				if cfg.Get("libraries").Kind() != dyn.KindInvalid {
+					withLibraries[groupName] = true
+				}
 			}
 		}
 	}
 
-	return present, withPermissions, withGrants
+	return present, withPermissions, withGrants, withLibraries
 }