diff --git a/acceptance/bundle/invariant/configs/materialization_set.yml.tmpl b/acceptance/bundle/invariant/configs/materialization_set.yml.tmpl new file mode 100644 index 00000000000..9966eadc384 --- /dev/null +++ b/acceptance/bundle/invariant/configs/materialization_set.yml.tmpl @@ -0,0 +1,34 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + features: + foo: + full_name: main.matset_inv_$UNIQUE_NAME.test_feature + source: + delta_table_source: + full_name: main.matset_inv_$UNIQUE_NAME.source_table + function: + aggregation_function: + count_function: + input: value + time_window: + sliding: + window_duration: 604800s + slide_duration: 86400s + entities: + - name: id + timeseries_column: + name: timestamp + + materialization_sets: + foo: + materialized_features: + - feature_name: ${resources.features.foo.full_name} + offline_store_config: + catalog_name: main + schema_name: matset_inv_$UNIQUE_NAME + table_name_prefix: offline + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" diff --git a/acceptance/bundle/invariant/configs/materialization_set.yml.tmpl-cleanup.sh b/acceptance/bundle/invariant/configs/materialization_set.yml.tmpl-cleanup.sh new file mode 100755 index 00000000000..80dcb68a26b --- /dev/null +++ b/acceptance/bundle/invariant/configs/materialization_set.yml.tmpl-cleanup.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +# The offline materialization leaves its own tables behind, so the schema is dropped with CASCADE. +echo "Cleaning up temporary source table" +$CLI tables delete main.matset_inv_$UNIQUE_NAME.source_table || true +$CLI schemas delete main.matset_inv_$UNIQUE_NAME --force || true diff --git a/acceptance/bundle/invariant/configs/materialization_set.yml.tmpl-init.sh b/acceptance/bundle/invariant/configs/materialization_set.yml.tmpl-init.sh new file mode 100755 index 00000000000..18cb15f25ae --- /dev/null +++ b/acceptance/bundle/invariant/configs/materialization_set.yml.tmpl-init.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +# The feature's delta_table_source must already exist, and the invariant harness provisions no +# tables of its own. +echo "Creating temporary source table: main.matset_inv_$UNIQUE_NAME.source_table" + +$CLI schemas create matset_inv_$UNIQUE_NAME main -o json | jq '{full_name}' + +# MSYS_NO_PATHCONV=1 prevents Git Bash on Windows from converting /api/... to a filesystem path. +MSYS_NO_PATHCONV=1 $CLI api post "/api/2.0/sql/statements/" --json "{ + \"warehouse_id\": \"$TEST_DEFAULT_WAREHOUSE_ID\", + \"statement\": \"CREATE TABLE main.matset_inv_$UNIQUE_NAME.source_table (id INT, value STRING, timestamp TIMESTAMP) USING DELTA\", + \"wait_timeout\": \"45s\" + }" > /dev/null diff --git a/acceptance/bundle/python/materialization_sets-support/databricks.yml b/acceptance/bundle/python/materialization_sets-support/databricks.yml new file mode 100644 index 00000000000..443e1853d07 --- /dev/null +++ b/acceptance/bundle/python/materialization_sets-support/databricks.yml @@ -0,0 +1,22 @@ +bundle: + name: my_project + +sync: {paths: []} # don't need to copy files + +python: + resources: + - "resources:load_resources" + mutators: + - "mutators:update_materialization_set" + +resources: + materialization_sets: + my_materialization_set_1: + materialized_features: + - feature_name: main.default.my_feature_1 + offline_store_config: + catalog_name: main + schema_name: default + table_name_prefix: my_feature_1_off + cron_schedule_trigger: + mode: DERIVED diff --git a/acceptance/bundle/python/materialization_sets-support/mutators.py b/acceptance/bundle/python/materialization_sets-support/mutators.py new file mode 100644 index 00000000000..ed93965ba56 --- /dev/null +++ b/acceptance/bundle/python/materialization_sets-support/mutators.py @@ -0,0 +1,23 @@ +from dataclasses import replace + +from databricks.bundles.core import materialization_set_mutator +from databricks.bundles.materialization_sets import MaterializationSet + + +@materialization_set_mutator +def update_materialization_set( + materialization_set: MaterializationSet, +) -> MaterializationSet: + # A set has no string field of its own, so the member's feature name is what a mutator can + # observably change. + members = list(materialization_set.materialized_features) + first = members[0] + assert isinstance(first.feature_name, str) + + return replace( + materialization_set, + materialized_features=[ + replace(first, feature_name=f"{first.feature_name}_updated"), + *members[1:], + ], + ) diff --git a/acceptance/bundle/python/materialization_sets-support/out.test.toml b/acceptance/bundle/python/materialization_sets-support/out.test.toml new file mode 100644 index 00000000000..aeddf3e619e --- /dev/null +++ b/acceptance/bundle/python/materialization_sets-support/out.test.toml @@ -0,0 +1,3 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] +EnvMatrix.PYDAB_VERSION = ["current"] diff --git a/acceptance/bundle/python/materialization_sets-support/output.txt b/acceptance/bundle/python/materialization_sets-support/output.txt new file mode 100644 index 00000000000..32a5aeb735d --- /dev/null +++ b/acceptance/bundle/python/materialization_sets-support/output.txt @@ -0,0 +1,48 @@ + +>>> uv run [UV_ARGS] -q [CLI] bundle validate --output json +{ + "experimental": { + "python": { + "mutators": [ + "mutators:update_materialization_set" + ], + "resources": [ + "resources:load_resources" + ] + } + }, + "resources": { + "materialization_sets": { + "my_materialization_set_1": { + "materialized_features": [ + { + "cron_schedule_trigger": { + "mode": "DERIVED" + }, + "feature_name": "main.default.my_feature_1_updated", + "offline_store_config": { + "catalog_name": "main", + "schema_name": "default", + "table_name_prefix": "my_feature_1_off" + } + } + ] + }, + "my_materialization_set_2": { + "materialized_features": [ + { + "cron_schedule_trigger": { + "mode": "DERIVED" + }, + "feature_name": "main.default.my_feature_2_updated", + "offline_store_config": { + "catalog_name": "main", + "schema_name": "default", + "table_name_prefix": "my_feature_2_off" + } + } + ] + } + } + } +} diff --git a/acceptance/bundle/python/materialization_sets-support/resources.py b/acceptance/bundle/python/materialization_sets-support/resources.py new file mode 100644 index 00000000000..44b87f3bc23 --- /dev/null +++ b/acceptance/bundle/python/materialization_sets-support/resources.py @@ -0,0 +1,24 @@ +from databricks.bundles.core import Resources + + +def load_resources() -> Resources: + resources = Resources() + + resources.add_materialization_set( + "my_materialization_set_2", + { + "materialized_features": [ + { + "feature_name": "main.default.my_feature_2", + "offline_store_config": { + "catalog_name": "main", + "schema_name": "default", + "table_name_prefix": "my_feature_2_off", + }, + "cron_schedule_trigger": {"mode": "DERIVED"}, + }, + ], + }, + ) + + return resources diff --git a/acceptance/bundle/python/materialization_sets-support/script b/acceptance/bundle/python/materialization_sets-support/script new file mode 100644 index 00000000000..e273fb45a53 --- /dev/null +++ b/acceptance/bundle/python/materialization_sets-support/script @@ -0,0 +1,5 @@ + +trace uv run $UV_ARGS -q $CLI bundle validate --output json | \ + jq "pick(.experimental.python, .resources)" + +rm -fr .databricks __pycache__ diff --git a/acceptance/bundle/python/materialization_sets-support/test.toml b/acceptance/bundle/python/materialization_sets-support/test.toml new file mode 100644 index 00000000000..1363c7008c8 --- /dev/null +++ b/acceptance/bundle/python/materialization_sets-support/test.toml @@ -0,0 +1,4 @@ +Cloud = false # tests don't interact with APIs + +# materialization sets are only supported in the current version of the wheel +EnvMatrix.PYDAB_VERSION = ["current"] diff --git a/acceptance/bundle/refschema/output/materialization_sets.txt b/acceptance/bundle/refschema/output/materialization_sets.txt new file mode 100644 index 00000000000..8fb61163502 --- /dev/null +++ b/acceptance/bundle/refschema/output/materialization_sets.txt @@ -0,0 +1,58 @@ +resources.materialization_sets.*.id string INPUT +resources.materialization_sets.*.lifecycle resources.Lifecycle INPUT +resources.materialization_sets.*.lifecycle.depends_on []resources.DependsOn INPUT +resources.materialization_sets.*.lifecycle.depends_on[*] resources.DependsOn INPUT +resources.materialization_sets.*.lifecycle.depends_on[*].resource string INPUT +resources.materialization_sets.*.lifecycle.prevent_destroy bool INPUT +resources.materialization_sets.*.materialization_set_id string ALL +resources.materialization_sets.*.materialized_features []ml.MaterializedFeature ALL +resources.materialization_sets.*.materialized_features[*] ml.MaterializedFeature ALL +resources.materialization_sets.*.materialized_features[*].budget_policy_id string ALL +resources.materialization_sets.*.materialized_features[*].cron_schedule string ALL +resources.materialization_sets.*.materialized_features[*].cron_schedule_trigger *ml.CronSchedule ALL +resources.materialization_sets.*.materialized_features[*].cron_schedule_trigger.cron_expression string ALL +resources.materialization_sets.*.materialized_features[*].cron_schedule_trigger.mode ml.CronScheduleMode ALL +resources.materialization_sets.*.materialized_features[*].cron_schedule_trigger.timezone_id string ALL +resources.materialization_sets.*.materialized_features[*].feature_name string ALL +resources.materialization_sets.*.materialized_features[*].is_online bool ALL +resources.materialization_sets.*.materialized_features[*].job_id int64 ALL +resources.materialization_sets.*.materialized_features[*].last_materialization_time string ALL +resources.materialization_sets.*.materialized_features[*].latest_backfill_operation string ALL +resources.materialization_sets.*.materialized_features[*].materialization_set_id string ALL +resources.materialization_sets.*.materialized_features[*].materialized_feature_id string ALL +resources.materialization_sets.*.materialized_features[*].notifications *ml.MaterializationNotifications ALL +resources.materialization_sets.*.materialized_features[*].notifications.on_failure *ml.MaterializationFailureNotification ALL +resources.materialization_sets.*.materialized_features[*].notifications.on_failure.destination_ids []string ALL +resources.materialization_sets.*.materialized_features[*].notifications.on_failure.destination_ids[*] string ALL +resources.materialization_sets.*.materialized_features[*].notifications.on_failure.email_addresses []string ALL +resources.materialization_sets.*.materialized_features[*].notifications.on_failure.email_addresses[*] string ALL +resources.materialization_sets.*.materialized_features[*].notifications.on_failure.final_attempt_only bool ALL +resources.materialization_sets.*.materialized_features[*].notifications.on_success *ml.MaterializationSuccessNotification ALL +resources.materialization_sets.*.materialized_features[*].notifications.on_success.destination_ids []string ALL +resources.materialization_sets.*.materialized_features[*].notifications.on_success.destination_ids[*] string ALL +resources.materialization_sets.*.materialized_features[*].notifications.on_success.email_addresses []string ALL +resources.materialization_sets.*.materialized_features[*].notifications.on_success.email_addresses[*] string ALL +resources.materialization_sets.*.materialized_features[*].offline_store_config *ml.OfflineStoreConfig ALL +resources.materialization_sets.*.materialized_features[*].offline_store_config.catalog_name string ALL +resources.materialization_sets.*.materialized_features[*].offline_store_config.schema_name string ALL +resources.materialization_sets.*.materialized_features[*].offline_store_config.table_name_prefix string ALL +resources.materialization_sets.*.materialized_features[*].online_store_config *ml.OnlineStoreConfig ALL +resources.materialization_sets.*.materialized_features[*].online_store_config.catalog_name string ALL +resources.materialization_sets.*.materialized_features[*].online_store_config.online_store_name string ALL +resources.materialization_sets.*.materialized_features[*].online_store_config.schema_name string ALL +resources.materialization_sets.*.materialized_features[*].online_store_config.table_name_prefix string ALL +resources.materialization_sets.*.materialized_features[*].pipeline_id string ALL +resources.materialization_sets.*.materialized_features[*].pipeline_schedule_state ml.MaterializedFeaturePipelineScheduleState ALL +resources.materialization_sets.*.materialized_features[*].streaming_mode *ml.StreamingMode ALL +resources.materialization_sets.*.materialized_features[*].streaming_mode.freshness_target string ALL +resources.materialization_sets.*.materialized_features[*].streaming_mode.mode ml.StreamingModeStreamingModeType ALL +resources.materialization_sets.*.materialized_features[*].streaming_mode.shuffle_partitions int ALL +resources.materialization_sets.*.materialized_features[*].table_name string ALL +resources.materialization_sets.*.materialized_features[*].table_trigger *ml.TableTrigger ALL +resources.materialization_sets.*.materialized_features[*].table_trigger.condition ml.TableTriggerTableTriggerCondition ALL +resources.materialization_sets.*.materialized_features[*].table_trigger.min_time_between_triggers *duration.Duration ALL +resources.materialization_sets.*.materialized_features[*].table_trigger.wait_after_last_change *duration.Duration ALL +resources.materialization_sets.*.materialized_features[*].tags map[string]string ALL +resources.materialization_sets.*.materialized_features[*].tags.* string ALL +resources.materialization_sets.*.modified_status string INPUT +resources.materialization_sets.*.url string INPUT diff --git a/acceptance/bundle/resources/materialization_sets/README.md b/acceptance/bundle/resources/materialization_sets/README.md new file mode 100644 index 00000000000..e430c49e196 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/README.md @@ -0,0 +1,285 @@ +# Materialization set acceptance tests + +These run locally against `libs/testserver/materialization_set.go`, which mirrors +`UpdateMaterializationSet`'s reconcile semantics rather than stubbing them. That is deliberate: one +`output.txt` serves both the local run and a `Cloud = true` run, so every rejection message here is +a claim about the real API, and the cloud run is what checks it. One message is not: the unknown +operation in `OperationsGet`, which the backend resolves inside the entity-store client rather than +from `FeatureStoreMessages`. It is marked as such in the fake and no case reaches it. + +## What the update endpoint can do + +The set update reconciles membership and nothing else -- `MutableMaterializedFeatureField.All` is +empty, so every member field is write-once. That shapes most of the cases below: a declared change +to any member field is caught by the planner as a `recreate`, and the only change that reaches the +server is a removal. + +## Create, update and delete are all long-running + +Each write verb returns a `databricks.longrunning.Operation` and the adapter waits on it, so the set +a create or update produces is read off the operation rather than the response body; a delete's +operation settles with no response at all. Execution is synchronous server-side, so the operation is +already terminal when the call returns and the wait settles on its first poll. + +Each verb also takes a `request_id`, and the server reserves an operation record against it before +acting: a repeat carrying a token already used gets the first operation back and changes nothing. +The adapter mints one per call, so the SDK's own retry of a lost response resolves to the first +operation instead of acting twice. The engine's `retryOnTransient` calls the adapter afresh and so +mints a new token; that is safe here because it never retries a create without the adapter opting +in, and re-running an update or a delete reconciles to the same state. No case replays a token, +because every adapter call mints its own. + +## A failure after the operation is reserved is not an HTTP error + +The backend raises only while it can still reject the request outright, and that boundary sits +earlier than "every validation". On the create path only three rules are reached before the record +is reserved: a member list that is empty, a supplied set id that already exists, and the schedule +states agreeing, which a wire-level validation hook checks before the handler runs at all. The +batch-and-streaming mixing rule, both destination rules and the sawtooth triple rule all live inside +the batch create the handler calls *after* reserving, so each of them settles the record and comes +back as HTTP 200. Measured against the live build, not read off the handler: `batch` and `sawtooth` +each failed on a golden still expecting a 400. Past that point a failure is the operation's outcome, +so all three verbs settle the record FAILED and return it with HTTP 200, carrying the error code and +message on the operation and the per-member outcome on the metadata's progress arm. A creation has +no progress arm: it has no per-member outcome to report. + +The generated wait flattens that into one string and reads no metadata, so the adapter re-reads the +settled operation to recover the code structurally and appends the members the change left alone. +It returns an `APIError` so the engine classifies and renders the failure as it would any other, +with the 200 the call actually returned as the status -- which maps to no sentinel, deliberately: a +member that could not be torn down must not reach the destroy step's permission-denied leniency, +whose purpose is to forgive a resource that is already gone. + +One gap this leaves is the engine's, not ours, and `settled_failure` pins it: `DeploymentUnit.Delete` +forgives a delete error matching none of the three cases it knows, so a partly deleted set exits +zero with its state dropped. The deployment-history leg is the one path that records and returns it. + +## What each case pins + +| Case | Claim | +|---|---| +| `basic` | create through the long-running operation, then a redeploy that plans nothing -- the server-assigned schedule state and the output-only member fields must not read as drift; the destroy's operation round trip | +| `bind` | unbinding leaves the set in the workspace and the plan reports `create`, which is the out-of-band shape; binding it back settles the first plan to `skip`, with the destination suppressed exactly as on a bundle-created set | +| `membership` | the delete cascade runs both ways -- removing either member of a feature's pair on its own is refused, while removing a whole pair plans as `update` and the other feature's pair keeps its ids | +| `removal_scope` | the cascade is scoped per feature: where every feature is materialized offline only, each contributes one member and can have it removed alone; the update's operation round trip | +| `schedule_state` | a member the config leaves unset adopts the state the server assigned; a declared change to it is a `recreate`, and that recreate still has to satisfy the create endpoint's rule that a feature's members agree | +| `immutable` | a write-once member field is caught by the planner as `recreate` and never reaches the server; adding a member routes to `update` so the server refuses it by name; destroy then redeploy is the route those refusals leave open, and the re-created set plans clean | +| `destination` | a destination field the server accepts on write and never returns on read is not drift, while a repoint the config actually asked for still recreates -- the two are told apart by whether the persisted value is recoverable | +| `batch` | a batch feature materialized offline creates and settles; removing its member empties the set, which the update endpoint refuses in favour of delete, whether the list is written empty or left unset; dropping the set resource deletes it; repointing the destination is a `recreate`; pausing the feature is a `recreate` that settles | +| `sawtooth` | the offline-and-online batch pair is refused at create, because a set is not expanded into the third materialization; all three declared creates and then plans nothing; the three members are one cascade group, so each of the three strict subsets of them is refused -- with the message depending on whether the removal names the member a delete is applied to -- and removing all three empties the set, which the update endpoint refuses | +| `settled_failure` | a failure the server hits after reserving an operation comes back as HTTP 200 with the operation FAILED: the adapter recovers the error code from the operation and the per-member outcome from its metadata, the update fails naming each member left behind and writes nothing, and the destroy reports success anyway because the engine forgives a delete error it cannot classify | +| `invalid_members` | two members reaching one member key are refused while the config is being read, with no request made. The message is the service's own wording for the same rejection, and both offenders are named | +| `mixed_triggers` | a set holding one feature that materializes in batch and one that materializes from a stream is refused at create, because the two need different triggers; the refusal settles an already-reserved operation record, so it arrives as HTTP 200 | +| `streaming` | a streaming online member needs no schedule and no cron; declaring a schedule state on it, or repointing its destination, plans as a `recreate` rather than an in-place update; with two features materialized to one destination and no offline member between them, each online member is its own cascade group, so one can be removed alone as an `update` | + +## Running against a workspace + +`sawtooth`, `streaming` and `mixed_triggers` reference an existing `Stream`, because a Stream +is not a bundle resource. **The `MATSET_STREAM` default names one developer's Stream and goes stale +when it is deleted**, which fails the run at the feature create with `Stream '...' not found`; list +what the workspace has with `GET /api/2.0/feature-engineering/streams?parent=.` and +override. Override `MATSET_CATALOG`, `MATSET_SCHEMA`, +`MATSET_ONLINE_STORE` and `MATSET_STREAM` to move them to another workspace, and set +`MATSET_PROFILE` to pin the bundle to a named profile. **`DATABRICKS_CONFIG_PROFILE` does not reach +the CLI subprocess**, and the harness builds its own workspace client from the ambient environment +before any script runs, so a local cloud run needs working ambient auth to the target workspace. +`batch` also needs `TEST_DEFAULT_WAREHOUSE_ID`, which the harness only supplies off cloud. + +The deployment-history leg (`DMS=true`) cannot run **against a workspace**: the bundle deployments +API rejects `features.` and `materialization_sets.` as resource-type prefixes, so every deploy of +either type fails there with `operations.resource_key must have a known resource type prefix`. +Select the `DMS=` leg for a cloud run until that allowlist includes them. The local `DMS=true` leg +does run, because the fake only requires the key to carry a prefix at all. + +### Running a cloud leg + +Three cases carry `Cloud = true`: `batch`, `immutable` and `removal_scope`. Each creates its own schema in +`main` and deletes it on exit, and materializes offline only, because the integration workspaces +have no online store. + +`sawtooth`, `streaming` and `mixed_triggers` are local only, because each needs a Stream source +and a Stream is neither a bundle resource nor a CLI command -- so the integration suite has no way +to provision one, and it would also need a Unity Catalog connection to a Kafka topic whose payload +the feature definitions name. Run those three against a workspace by hand, with the overrides below. +`basic`, `bind`, `destination`, `membership` and `schedule_state` are local only because each +declares an online member, so no integration run materializes online. `membership` runs by hand +against a workspace given a store in `MATSET_ONLINE_STORE`; the other four are written for the +local test server only. + +Override `MATSET_CATALOG` where `main` is absent, and `MATSET_SCHEMA`, `MATSET_ONLINE_STORE` and +`MATSET_STREAM` for the cases that read them. These reach the script because the harness forwards ambient +environment to the CLI subprocess, minus the SDK's own auth variables, which it replaces with the +ones it resolved. + +```bash +TOK=$(databricks auth token -p -o json | jq -r .access_token) +DATABRICKS_CONFIG_FILE=/dev/null DATABRICKS_HOST= DATABRICKS_TOKEN="$TOK" CLOUD_ENV=aws TEST_DEFAULT_WAREHOUSE_ID= DATABRICKS_LITESWAP_TARGET= go test . -run 'TestAccept/bundle/resources/materialization_sets//DMS=$' +``` + +Four parts of that are load-bearing: + +- **`DATABRICKS_CONFIG_FILE=/dev/null`.** With `RecordRequests = true` the harness proxies the run + and hands the subprocess a config carrying the *resolved config-file path*. The subprocess then + re-reads it and can fall back to `[__settings__] default_profile`, whose credentials have nothing + to do with the workspace under test. The symptom is a token refresh against + `/oidc/accounts//v1/token`. +- **A token rather than a profile.** The harness flattens its resolved auth into environment + variables, which overrides a `workspace: profile:` pinned in the bundle. A profile whose + `auth_type` is `databricks-cli` therefore gets re-resolved in the subprocess without the profile. +- **`/DMS=$`.** The `DMS=true` leg cannot run against a workspace: the bundle deployments API + rejects `features.` and `materialization_sets.` as resource-type prefixes. +- **The unit name only** in `DATABRICKS_LITESWAP_TARGET` -- `liteswap.Apply` prepends + `testenv://liteswap/`. Passing the full traffic id routes to base staging, where these endpoints + do not exist. + +The workspace needs a declarative feature preview enabled -- `validateAnyDeclarativePreviewEnabled` +rejects every create otherwise, with `declarativePreviewsNotEnabled`. + +### Two destination rules the fixtures have to satisfy + +Both are enforced on create by `BatchMaterializationUtils`, and both are now mirrored in the fake, +so an invalid fixture fails locally instead of only on cloud. + +1. **Every feature in a set must ask for an identical destination *pair*.** The compared value is + `(offlineStoreConfig, Option[onlineStoreConfig])` per feature, so matching catalog, schema and + prefix is not enough: a set cannot mix a feature materialized to both stores with one + materialized offline only. That is why `membership` gives every feature a pair and + `removal_scope` gives every feature an offline member alone -- the rule makes those two shapes + mutually exclusive within one set, which is why they cannot be one case. +2. **A feature's offline and online destinations must differ** in catalog, schema or prefix. The + fixtures suffix the offline prefix with `_off`. + +A third thing the fake does not check: that a member's feature exists. Declare the `features` +resources and reference them as `${resources.features..full_name}`, which also orders the +feature before the set -- a bare name leaves the create racing the feature, which is what made +`sawtooth` flaky. + +## Cloud runs 2026-10-20 + +Run against the `daniel-bai-fs2680` swap on workspace 2978662807315490. Two swap builds, and the +difference between them is the point. + +**First build, universe `afc23c250043d`.** `batch`, `sawtooth`, `streaming` and `immutable` passed; +`membership` and `removal_scope` failed, both refused with: + +``` +The fields 'pipeline_schedule_state' of materialized feature '' of feature '' cannot be +changed. Only removing members is supported; recreate the materialization set instead. +``` + +`comparableMaterializationSetMembers` copied the requested `pipelineScheduleState` raw onto the +comparison baseline while normalising the trigger beside it, so a bundle -- which never declares +the field, it being a backend default -- compared `None` against the stored `ACTIVE` and had every +membership change refused. Isolated live: the same removal succeeded when the surviving member +carried `pipeline_schedule_state: ACTIVE` and failed when it did not, nothing else different. + +**Second build, universe `8438b2287fb03`.** Fixed, and the validations moved into +`MaterializationSetUtils`. An omitted state now falls back to the stored one +(`.orElse(normalizedExistingMaterializedFeature.pipelineScheduleState)`), and the changed-field list +is derived from `MaterializedFeature.scalaDescriptor.fields` rather than a hand-maintained `Seq`, so +a member field added later is covered without touching this check. + +All six cases have passed against that build. The one golden this cost was `membership`: the cascade +refusal was reworded, and the fake carried the old text. + +``` +old: Changing one materialized feature of feature 'F' also changes '', which the request does + not change. Apply the same change to every materialized feature of the feature. +new: Changing materialized features '' of feature 'F' also changes materialized features + '', which the request does not change. Apply the same change to the unrequested + materialized features as well. +``` + +The new message names the removed members of the offending feature, so the fake groups removed +members by feature and sorts the ids the way `FeatureStoreMessages.quotedIds` renders them. +`materializedFeatureFieldCannotBeChanged` was reworded to match the server too, though no case +reaches it -- the planner classifies every member field as a recreate, so a declared change never +becomes an update request. + +**The workspace throws intermittent 500s that are not this resource.** Across five runs of the +`DMS=` leg, individual cases failed on `POST /api/2.0/feature-engineering/features` with `Unable to +create Feature '' in UC. If the issue persists, please retry or contact support.`, and once on +`Failed to create Lakebase table`. `immutable` lost three runs in a row this way, then passed twice in +a row when run on its own -- so the trigger is the parallel load of six bundles deploying at once, +not the fixture. Its two features are identical but for their names, and which one fails varies; a +direct two-feature create reproduced nothing, serial or concurrent. Run a case on its own before +reading a red run as a regression. + +**Third build, universe `275193b7facc8`.** The deletion cascade refusal became two messages, +picked by whether the removal names the member a delete is applied to: +`materializationSetMembersCannotBeRemovedAlone` when it does not, and +`deletionAffectsUnrequestedMaterializedFeatureMembers` when it does. The fake branches the same way +and `membership`'s two arms now show one each. A set mixing a batch-path feature with a +streaming-path one is also refused, which the fake had not modelled; `batch`'s last arm pins it, +with the message captured from this build. + +All six `Cloud = true` cases pass against this build, run one at a time, with no golden changed -- +so every message in the removal matrix is the server's own text and not a reading of its source. +That covers both cascade messages, the empty-set refusal that sends a caller to the delete endpoint, +the batch-versus-streaming mixing refusal, and the `streaming` case's claim that two stream features +materialized to one destination can share a set, which until then rested on the shared-destination +rule comparing a `(offline, Option[online])` pair that is identical for both. + +**Fourth build, universe `cd7a68312ca3`** -- the promotion tip, carrying the settled-failure +contract. All six `Cloud = true` cases pass against it, run one at a time. Three messages had been +reworded (`materializationSetMembersCannotBeRemovedAlone`, and both partial-change messages, which +now count what got through and point at the operation's progress), and two refusals moved from an +HTTP 400 to a settled FAILED operation. The fake and four goldens were corrected from what the live +run returned. + +**Fifth build, universe `92bd718e722a0`** -- the tip of `stack/fs2680-update-set-lro`, run through +the `daniel-bai-fs2680` swap on 2026-10-23. All six `Cloud = true` cases pass with no golden +changed, run one at a time, as does `features/lifecycle` on its `DMS=` leg. One of them had claims +nothing live had settled: `batch` carried two arms -- the null member-list spelling and the +drop-the-resource delete route -- that had only run against the fake. So the emptied-set refusal's "Delete the set instead" is the service's own +text, and the one-segment `materialized_features` path classifies as an update on the live API and +not just in the unit tests. + +The swap's own log is what proves the routing: during the 63-second run it served 5 +`CreateMaterializationSetRequest`, 7 `DeleteMaterializationSetRequest`, 6 +`GetMaterializationSetRequest`, 15 `ListMaterializationSetMembersRequest`, 3 +`CreateMaterializationSetMembersRequest` and 4 `DeleteMaterializedFeatureRequest`, interleaved with +this run's own fixture names. Zero `UpdateMaterializationSetRequest`, which is the case's one +coverage limit: every membership edit it makes is to an immutable field, so it plans a recreate and +never reaches the update endpoint. `batch`'s emptied-member arms are what exercise that. + +## Verified against a workspace, with no case of their own + +Run through the `daniel-bai-fs2680` swap on 2026-10-19, against a build carrying the write-once +schedule state. Every `Cloud = true` case passed on that run, so their goldens -- including +`batch`'s recreate carrying `PAUSED` on both members -- are confirmed against the real API. + +- **Creation across member kinds.** A batch feature with both destinations; two features sharing one + destination with a pair each; a sawtooth triple; a lone streaming RTM member. The sad paths too: + the sawtooth pair alone, members of one feature disagreeing on schedule state, two features with + different destinations, MBM on a non-Kinesis source, and an empty body. +- **Out-of-band creation is caught by the server, not by the planner.** The plan says `create` + because the bundle has no state for the resource; the deploy then fails with `ALREADY_EXISTS` on + the feature, and a set whose feature already has materializations fails with + `RESOURCE_ALREADY_EXISTS`. A set has no name to collide on, so that second check is the only thing + standing between an unbound set and a duplicate. The `bind` case pins the planner half of this + locally; the server half needs a workspace. +- **Get redacts the offline destination.** All four members of a two-feature set read back with + `offline_store_config` absent, which is what the `destination` suppression exists to absorb. + +Two behaviours previously recorded here are gone, not untested: a mixed update whose halves are both +legal, and the atomicity of a mixed update when one half is rejected. Both depended on a pause being +a legal update. With no mutable member field left, the only update is a removal, so there is no +mixed update to construct. + +## Not covered yet + +- **A set's materialized output tables outlive it.** `bundle destroy` removes the set and its + features, leaving each offline materialization's `*_partial_aggregates` table behind. A cloud case + that wants a clean schema has to drop those itself. + +- **A real member-teardown failure.** `settled_failure` produces the settled-FAILED shape + from a marker the fake looks for in a feature name, because a real teardown fails on + infrastructure no request can ask for. The adapter's reading of that shape is pinned; that + the server emits it for a genuine failure is not, and cannot be from here. + +- **A settled-but-failed operation.** Every refusal these cases reach is an HTTP error on the call + itself, raised before the operation record exists, so the wait is never what fails. The branch + that reports a failed operation -- naming the set from the operation metadata -- is covered only + by `TestMaterializationSetOperationTarget`, and the same injection the partial-failure case needs + would cover it end to end. diff --git a/acceptance/bundle/resources/materialization_sets/basic/databricks.yml b/acceptance/bundle/resources/materialization_sets/basic/databricks.yml new file mode 100644 index 00000000000..c347eae6cd1 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/basic/databricks.yml @@ -0,0 +1,45 @@ +bundle: + name: test-bundle + +resources: + features: + batchfeat: + full_name: main.myschema.batchfeat + source: + delta_table_source: + full_name: main.myschema.mysource + function: + aggregation_function: + count_function: + input: amount + time_window: + sliding: + window_duration: 604800s + slide_duration: 86400s + entities: + - name: account_id + timeseries_column: + name: event_ts + + materialization_sets: + set1: + materialized_features: + # One feature with both destinations: the offline member a delete cascades from, and the + # online member it cascades to. + - feature_name: main.myschema.batchfeat + offline_store_config: + catalog_name: main + schema_name: myschema + table_name_prefix: offline + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" + - feature_name: main.myschema.batchfeat + online_store_config: + online_store_name: danielb + catalog_name: main + schema_name: myschema + table_name_prefix: batchfeat + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" diff --git a/acceptance/bundle/resources/materialization_sets/basic/out.test.toml b/acceptance/bundle/resources/materialization_sets/basic/out.test.toml new file mode 100644 index 00000000000..19b5b15d11f --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/basic/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/resources/materialization_sets/basic/output.txt b/acceptance/bundle/resources/materialization_sets/basic/output.txt new file mode 100644 index 00000000000..ab357c236bf --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/basic/output.txt @@ -0,0 +1,115 @@ + +=== Summary before deploy +>>> [CLI] bundle summary -o json +{ + "modified_status": "created", + "materialized_features": [ + { + "feature_name": "[BATCHFEAT_ID]", + "is_online": null + }, + { + "feature_name": "[BATCHFEAT_ID]", + "is_online": null + } + ] +} + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Created features.batchfeat +Created materialization_sets.set1 +Files: 1 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +=== The create returns a long-running operation, and the created set is read back from it +>>> print_requests.py --keep //materialization-sets +{ + "method": "POST", + "path": "/api/2.0/feature-engineering/materialization-sets", + "q": { + "request_id": "[UUID]" + }, + "body": { + "materialized_features": [ + { + "cron_schedule_trigger": { + "cron_expression": "0 0 * * * ?", + "mode": "MANUAL" + }, + "feature_name": "[BATCHFEAT_ID]", + "offline_store_config": { + "catalog_name": "main", + "schema_name": "myschema", + "table_name_prefix": "offline" + } + }, + { + "cron_schedule_trigger": { + "cron_expression": "0 0 * * * ?", + "mode": "MANUAL" + }, + "feature_name": "[BATCHFEAT_ID]", + "online_store_config": { + "catalog_name": "main", + "online_store_name": "danielb", + "schema_name": "myschema", + "table_name_prefix": "batchfeat" + } + } + ] + } +} + +>>> print_requests.py --get //feature-engineering/operations +{ + "method": "GET", + "path": "/api/2.0/feature-engineering/operations/[UUID]" +} + +=== The server assigned a set id and a member id per member +>>> [CLI] bundle summary -o json +{ + "id": "[SET1_ID]" +} + +=== Redeploying with no config change must plan nothing +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Files: 0 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 2 unchanged + +>>> print_requests.py //materialization-sets + +=== The schedule state the server assigned must not read as drift +>>> [CLI] bundle plan -o json + +>>> jq .plan | with_entries(select(.key | startswith("resources.materialization_sets"))) | map_values(.action) out.plan.json +{ + "resources.materialization_sets.set1": "skip" +} + +=== The delete returns a long-running operation too, which the destroy waits on +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.features.batchfeat + delete resources.materialization_sets.set1 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default + +Destroy: 2 deleted + +>>> print_requests.py --keep //materialization-sets +{ + "method": "DELETE", + "path": "/api/2.0/feature-engineering/materialization-sets/[SET1_ID]", + "q": { + "request_id": "[UUID]" + } +} + +>>> print_requests.py --get //feature-engineering/operations +{ + "method": "GET", + "path": "/api/2.0/feature-engineering/operations/[UUID]" +} diff --git a/acceptance/bundle/resources/materialization_sets/basic/script b/acceptance/bundle/resources/materialization_sets/basic/script new file mode 100644 index 00000000000..4cc00d27b4b --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/basic/script @@ -0,0 +1,27 @@ +title "Summary before deploy" +trace $CLI bundle summary -o json | jq '.resources.materialization_sets.set1 | {modified_status, materialized_features: [.materialized_features[] | {feature_name, is_online}]}' + +trace $CLI bundle deploy +replace_ids.py + +title "The create returns a long-running operation, and the created set is read back from it" +trace print_requests.py --keep //materialization-sets +trace print_requests.py --get //feature-engineering/operations + +title "The server assigned a set id and a member id per member" +trace $CLI bundle summary -o json | jq '.resources.materialization_sets.set1 | {id}' + +title "Redeploying with no config change must plan nothing" +trace $CLI bundle deploy +trace print_requests.py //materialization-sets + +title "The schedule state the server assigned must not read as drift" +trace $CLI bundle plan -o json > out.plan.json +trace jq '.plan | with_entries(select(.key | startswith("resources.materialization_sets"))) | map_values(.action)' out.plan.json + +title "The delete returns a long-running operation too, which the destroy waits on" +trace $CLI bundle destroy --auto-approve +trace print_requests.py --keep //materialization-sets +trace print_requests.py --get //feature-engineering/operations + +rm -f out.requests.txt out.plan.json diff --git a/acceptance/bundle/resources/materialization_sets/batch/databricks.yml.tmpl b/acceptance/bundle/resources/materialization_sets/batch/databricks.yml.tmpl new file mode 100644 index 00000000000..f784b20188d --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/batch/databricks.yml.tmpl @@ -0,0 +1,34 @@ +bundle: + name: test-matset-batch-$UNIQUE_NAME + +resources: + features: + batchfeat: + full_name: $CATALOG_NAME.$SCHEMA_NAME.batch_$UNIQUE_NAME + source: + delta_table_source: + full_name: $TABLE_NAME + function: + aggregation_function: + count_function: + input: value + time_window: + sliding: + window_duration: 604800s + slide_duration: 86400s + entities: + - name: id + timeseries_column: + name: timestamp + + materialization_sets: + set1: + materialized_features: + - feature_name: ${resources.features.batchfeat.full_name} + offline_store_config: + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: batch_${UNIQUE_NAME}_off + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" diff --git a/acceptance/bundle/resources/materialization_sets/batch/drop-member-null.yml.tmpl b/acceptance/bundle/resources/materialization_sets/batch/drop-member-null.yml.tmpl new file mode 100644 index 00000000000..8c72a5c0c3c --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/batch/drop-member-null.yml.tmpl @@ -0,0 +1,26 @@ +bundle: + name: test-matset-batch-$UNIQUE_NAME + +resources: + features: + batchfeat: + full_name: $CATALOG_NAME.$SCHEMA_NAME.batch_$UNIQUE_NAME + source: + delta_table_source: + full_name: $TABLE_NAME + function: + aggregation_function: + count_function: + input: value + time_window: + sliding: + window_duration: 604800s + slide_duration: 86400s + entities: + - name: id + timeseries_column: + name: timestamp + + materialization_sets: + set1: + materialized_features: diff --git a/acceptance/bundle/resources/materialization_sets/batch/drop-member.yml.tmpl b/acceptance/bundle/resources/materialization_sets/batch/drop-member.yml.tmpl new file mode 100644 index 00000000000..234b08d05e6 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/batch/drop-member.yml.tmpl @@ -0,0 +1,26 @@ +bundle: + name: test-matset-batch-$UNIQUE_NAME + +resources: + features: + batchfeat: + full_name: $CATALOG_NAME.$SCHEMA_NAME.batch_$UNIQUE_NAME + source: + delta_table_source: + full_name: $TABLE_NAME + function: + aggregation_function: + count_function: + input: value + time_window: + sliding: + window_duration: 604800s + slide_duration: 86400s + entities: + - name: id + timeseries_column: + name: timestamp + + materialization_sets: + set1: + materialized_features: [] diff --git a/acceptance/bundle/resources/materialization_sets/batch/moved.yml.tmpl b/acceptance/bundle/resources/materialization_sets/batch/moved.yml.tmpl new file mode 100644 index 00000000000..d8c706964f4 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/batch/moved.yml.tmpl @@ -0,0 +1,34 @@ +bundle: + name: test-matset-batch-$UNIQUE_NAME + +resources: + features: + batchfeat: + full_name: $CATALOG_NAME.$SCHEMA_NAME.batch_$UNIQUE_NAME + source: + delta_table_source: + full_name: $TABLE_NAME + function: + aggregation_function: + count_function: + input: value + time_window: + sliding: + window_duration: 604800s + slide_duration: 86400s + entities: + - name: id + timeseries_column: + name: timestamp + + materialization_sets: + set1: + materialized_features: + - feature_name: ${resources.features.batchfeat.full_name} + offline_store_config: + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: moved_${UNIQUE_NAME}_off + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" diff --git a/acceptance/bundle/resources/materialization_sets/batch/no-set.yml.tmpl b/acceptance/bundle/resources/materialization_sets/batch/no-set.yml.tmpl new file mode 100644 index 00000000000..ed86dd96e53 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/batch/no-set.yml.tmpl @@ -0,0 +1,22 @@ +bundle: + name: test-matset-batch-$UNIQUE_NAME + +resources: + features: + batchfeat: + full_name: $CATALOG_NAME.$SCHEMA_NAME.batch_$UNIQUE_NAME + source: + delta_table_source: + full_name: $TABLE_NAME + function: + aggregation_function: + count_function: + input: value + time_window: + sliding: + window_duration: 604800s + slide_duration: 86400s + entities: + - name: id + timeseries_column: + name: timestamp diff --git a/acceptance/bundle/resources/materialization_sets/batch/out.test.toml b/acceptance/bundle/resources/materialization_sets/batch/out.test.toml new file mode 100644 index 00000000000..f68b4a89c36 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/batch/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/resources/materialization_sets/batch/output.txt b/acceptance/bundle/resources/materialization_sets/batch/output.txt new file mode 100644 index 00000000000..246055a3aa3 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/batch/output.txt @@ -0,0 +1,145 @@ + +>>> [CLI] schemas create matset_[UNIQUE_NAME] main -o json +{ + "full_name": "main.matset_[UNIQUE_NAME]" +} + +>>> create_table.py main.matset_[UNIQUE_NAME].src_[UNIQUE_NAME] +Created table main.matset_[UNIQUE_NAME].src_[UNIQUE_NAME] +Inserted sample data into main.matset_[UNIQUE_NAME].src_[UNIQUE_NAME] +Table main.matset_[UNIQUE_NAME].src_[UNIQUE_NAME] is now visible (catalog_name=main) + +=== A batch feature materialized offline creates, and a redeploy plans nothing +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-batch-[UNIQUE_NAME]/default/files... +Created features.batchfeat +Created materialization_sets.set1 +Files: 7 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +>>> [CLI] bundle plan -o json + +>>> set_actions +{ + "resources.materialization_sets.set1": "skip" +} + +=== Removing the member empties the set, which the update endpoint refuses in favour of the +=== delete endpoint. Restoring the member plans nothing, so no refusal wrote. +>>> [CLI] bundle plan -o json + +>>> set_actions +{ + "resources.materialization_sets.set1": "update" +} + +>>> musterr [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-batch-[UNIQUE_NAME]/default/files... +Error: cannot update resources.materialization_sets.set1: updating id=[UUID]: Updating materialization set '[UUID]' with no materialized features is not supported. Delete the set instead. (400 INVALID_PARAMETER_VALUE) + +Endpoint: PATCH [DATABRICKS_URL]/api/2.0/feature-engineering/materialization-sets/[UUID]?request_id=[UUID]&update_mask=%2A +HTTP Status: 400 Bad Request +API error_code: INVALID_PARAMETER_VALUE +API message: Updating materialization set '[UUID]' with no materialized features is not supported. Delete the set instead. + +Files: 2 uploaded, 0 deleted + +=== Writing no value instead of an empty list is the same declaration, and has to reach the same +=== endpoint: the member field emptied whole is one path rather than one per member, and the +=== immutable rules match it just as readily. +>>> [CLI] bundle plan -o json + +>>> set_actions +{ + "resources.materialization_sets.set1": "update" +} + +>>> musterr [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-batch-[UNIQUE_NAME]/default/files... +Error: cannot update resources.materialization_sets.set1: updating id=[UUID]: Updating materialization set '[UUID]' with no materialized features is not supported. Delete the set instead. (400 INVALID_PARAMETER_VALUE) + +Endpoint: PATCH [DATABRICKS_URL]/api/2.0/feature-engineering/materialization-sets/[UUID]?request_id=[UUID]&update_mask=%2A +HTTP Status: 400 Bad Request +API error_code: INVALID_PARAMETER_VALUE +API message: Updating materialization set '[UUID]' with no materialized features is not supported. Delete the set instead. + +Files: 2 uploaded, 0 deleted + +>>> [CLI] bundle plan -o json + +>>> set_actions +{ + "resources.materialization_sets.set1": "skip" +} + +=== Taking the refusal's advice: dropping the set resource rather than its members plans as a +=== delete and settles. This is the deploy path's delete, not bundle destroy's. +>>> [CLI] bundle plan -o json + +>>> set_actions +{ + "resources.materialization_sets.set1": "delete" +} + +>>> [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-batch-[UNIQUE_NAME]/default/files... + +This action will result in the deletion or recreation of the following materialization sets. +Every materialized feature in a recreated set is dropped and recomputed from scratch, including +the members whose configuration did not change: + delete resources.materialization_sets.set1 +Deleted materialization_sets.set1 +Files: 2 uploaded, 0 deleted +Resources: 0 created, 0 changed, 1 deleted, 1 unchanged + +>>> [CLI] bundle plan -o json + +>>> set_actions +{ + "resources.materialization_sets.set1": "create" +} + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-batch-[UNIQUE_NAME]/default/files... +Created materialization_sets.set1 +Files: 2 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 1 unchanged + +=== Repointing the destination is a recreate, not a silent no-op. The server cannot move a +=== materialization in place, so the set is rebuilt at the new location. +>>> [CLI] bundle plan -o json + +>>> set_actions +{ + "resources.materialization_sets.set1": "recreate" +} + +>>> jq -r .plan["resources.materialization_sets.set1"].changes | to_entries | map(select(.value.action == "recreate")) | .[] | "\(.value.reason) old=\(.value.old) new=\(.value.new)" out.plan.json +spec:immutable old=batch_[UNIQUE_NAME]_off new=moved_[UNIQUE_NAME]_off + +=== The schedule state is write-once, so pausing the feature is a recreate rather than the +=== in-place update the endpoint would refuse +>>> [CLI] bundle plan -o json + +>>> set_actions +{ + "resources.materialization_sets.set1": "recreate" +} + +>>> [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-batch-[UNIQUE_NAME]/default/files... + +This action will result in the deletion or recreation of the following materialization sets. +Every materialized feature in a recreated set is dropped and recomputed from scratch, including +the members whose configuration did not change: + recreate resources.materialization_sets.set1 +Recreated materialization_sets.set1 +Files: 2 uploaded, 0 deleted +Resources: 1 created, 0 changed, 1 deleted, 1 unchanged + +>>> [CLI] bundle plan -o json + +>>> set_actions +{ + "resources.materialization_sets.set1": "skip" +} diff --git a/acceptance/bundle/resources/materialization_sets/batch/paused.yml.tmpl b/acceptance/bundle/resources/materialization_sets/batch/paused.yml.tmpl new file mode 100644 index 00000000000..050b969662f --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/batch/paused.yml.tmpl @@ -0,0 +1,35 @@ +bundle: + name: test-matset-batch-$UNIQUE_NAME + +resources: + features: + batchfeat: + full_name: $CATALOG_NAME.$SCHEMA_NAME.batch_$UNIQUE_NAME + source: + delta_table_source: + full_name: $TABLE_NAME + function: + aggregation_function: + count_function: + input: value + time_window: + sliding: + window_duration: 604800s + slide_duration: 86400s + entities: + - name: id + timeseries_column: + name: timestamp + + materialization_sets: + set1: + materialized_features: + - feature_name: ${resources.features.batchfeat.full_name} + offline_store_config: + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: batch_${UNIQUE_NAME}_off + pipeline_schedule_state: PAUSED + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" diff --git a/acceptance/bundle/resources/materialization_sets/batch/script b/acceptance/bundle/resources/materialization_sets/batch/script new file mode 100644 index 00000000000..8d1dc2e7709 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/batch/script @@ -0,0 +1,79 @@ +export CATALOG_NAME="${MATSET_CATALOG:-main}" +export SCHEMA_NAME="matset_${UNIQUE_NAME}" +export TABLE_NAME="$CATALOG_NAME.$SCHEMA_NAME.src_$UNIQUE_NAME" + +# Each phase is its own template rather than a string edit, so the fixture a phase deploys is +# readable on its own. +render() { + envsubst < "$1" > databricks.yml + [ -n "${MATSET_PROFILE:-}" ] || return 0 + printf "\nworkspace:\n profile: %s\n" "$MATSET_PROFILE" >> databricks.yml +} + +set_actions() { + jq '.plan | with_entries(select(.key | startswith("resources.materialization_sets"))) | map_values(.action)' out.plan.json +} + +cleanup() { + $CLI bundle destroy --auto-approve > /dev/null 2>&1 || true + $CLI tables delete "$TABLE_NAME" > /dev/null 2>&1 || true + $CLI schemas delete "$CATALOG_NAME.$SCHEMA_NAME" --force > /dev/null 2>&1 || true +} + +trap cleanup EXIT + +trace $CLI schemas create "$SCHEMA_NAME" "$CATALOG_NAME" -o json | jq '{full_name}' +trace create_table.py "$TABLE_NAME" +render databricks.yml.tmpl + +title "A batch feature materialized offline creates, and a redeploy plans nothing" +trace $CLI bundle deploy +trace $CLI bundle plan -o json > out.plan.json +trace set_actions + +title "Removing the member empties the set, which the update endpoint refuses in favour of the" +title "delete endpoint. Restoring the member plans nothing, so no refusal wrote." +render drop-member.yml.tmpl +trace $CLI bundle plan -o json > out.plan.json +trace set_actions +trace musterr $CLI bundle deploy --auto-approve + +title "Writing no value instead of an empty list is the same declaration, and has to reach the same" +title "endpoint: the member field emptied whole is one path rather than one per member, and the" +title "immutable rules match it just as readily." +render drop-member-null.yml.tmpl +trace $CLI bundle plan -o json > out.plan.json +trace set_actions +trace musterr $CLI bundle deploy --auto-approve +render databricks.yml.tmpl +trace $CLI bundle plan -o json > out.plan.json +trace set_actions + +title "Taking the refusal's advice: dropping the set resource rather than its members plans as a" +title "delete and settles. This is the deploy path's delete, not bundle destroy's." +render no-set.yml.tmpl +trace $CLI bundle plan -o json > out.plan.json +trace set_actions +trace $CLI bundle deploy --auto-approve +render databricks.yml.tmpl +trace $CLI bundle plan -o json > out.plan.json +trace set_actions +trace $CLI bundle deploy + +title "Repointing the destination is a recreate, not a silent no-op. The server cannot move a" +title "materialization in place, so the set is rebuilt at the new location." +render moved.yml.tmpl +trace $CLI bundle plan -o json > out.plan.json +trace set_actions +trace jq -r '.plan["resources.materialization_sets.set1"].changes | to_entries | map(select(.value.action == "recreate")) | .[] | "\(.value.reason) old=\(.value.old) new=\(.value.new)"' out.plan.json + +title "The schedule state is write-once, so pausing the feature is a recreate rather than the" +title "in-place update the endpoint would refuse" +render paused.yml.tmpl +trace $CLI bundle plan -o json > out.plan.json +trace set_actions +trace $CLI bundle deploy --auto-approve +trace $CLI bundle plan -o json > out.plan.json +trace set_actions + +rm -f out.requests.txt out.plan.json diff --git a/acceptance/bundle/resources/materialization_sets/batch/test.toml b/acceptance/bundle/resources/materialization_sets/batch/test.toml new file mode 100644 index 00000000000..8423ea4a42f --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/batch/test.toml @@ -0,0 +1,31 @@ +# Creates its own schema and deletes it on exit. Override MATSET_CATALOG where `main` is absent. +# Offline members only, so the integration workspaces need no online store. The pair cascade and +# the members-disagree refusal are covered by membership and schedule_state, the mixed-trigger +# refusal by mixed_triggers. +Cloud = true + +Ignore = [ + ".databricks", + "databricks.yml", +] + +# A real materialization set or member id is 32 undashed hex characters; the harness's built-in +# replacement only matches the dashed form the fake produces. +[[Repls]] +Old = '\b[0-9a-f]{32}\b' +New = '[UUID]' + +# The workspace host, which only a cloud run puts in the output. +[[Repls]] +Old = 'https://[a-z0-9.-]+\.cloud\.databricks\.com' +New = '[DATABRICKS_URL]' + +# Fake SQL endpoint so create_table.py works against the local test server too. +[[Server]] +Pattern = "POST /api/2.0/sql/statements/" +Response.Body = '{"status": {"state": "SUCCEEDED"}, "manifest": {"schema": {"columns": []}}}' + +# The cleanup drops the source table; the fake has no delete handler for one. +[[Server]] +Pattern = "DELETE /api/2.1/unity-catalog/tables/{full_name}" +Response.Body = '{}' diff --git a/acceptance/bundle/resources/materialization_sets/bind/databricks.yml b/acceptance/bundle/resources/materialization_sets/bind/databricks.yml new file mode 100644 index 00000000000..0a956c87748 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/bind/databricks.yml @@ -0,0 +1,18 @@ +bundle: + name: test-bundle + +resources: + materialization_sets: + set1: + materialized_features: + - feature_name: main.myschema.bindfeat + offline_store_config: + catalog_name: main + schema_name: myschema + table_name_prefix: shared_off + - feature_name: main.myschema.bindfeat + online_store_config: + online_store_name: danielb + catalog_name: main + schema_name: myschema + table_name_prefix: shared diff --git a/acceptance/bundle/resources/materialization_sets/bind/out.test.toml b/acceptance/bundle/resources/materialization_sets/bind/out.test.toml new file mode 100644 index 00000000000..19b5b15d11f --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/bind/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/resources/materialization_sets/bind/output.txt b/acceptance/bundle/resources/materialization_sets/bind/output.txt new file mode 100644 index 00000000000..a5133235426 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/bind/output.txt @@ -0,0 +1,53 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Created materialization_sets.set1 +Files: 1 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +=== Unbinding leaves the set in the workspace and drops it from the bundle's state, which is +=== the same shape as a set created out of band: the plan reports create, because the bundle +=== has nothing recorded for it +>>> [CLI] bundle deployment unbind set1 + +>>> [CLI] bundle plan -o json + +>>> set_plan +{ + "resources.materialization_sets.set1": "create" +} + +=== Binding the existing set takes ownership of it again +>>> [CLI] bundle deployment bind set1 [UUID] --auto-approve +Successfully bound materialization_set with an id '[UUID]' +Run 'bundle deploy' to deploy changes to your workspace + +=== The first plan after a bind settles. Bind seeds state from the resolved config rather +=== than from a read, so the destination the read never returns is suppressed here exactly as +=== it is on a set the bundle created +>>> [CLI] bundle plan -o json + +>>> set_plan +{ + "resources.materialization_sets.set1": "skip" +} + +>>> destination_changes +materialized_features[member='feature_name=main.myschema.bindfeat|is_streaming_mode=false'].offline_store_config skip input_only_not_returned +materialized_features[member='feature_name=main.myschema.bindfeat|online_store_name=danielb|is_streaming_mode=false'].online_store_config.catalog_name skip input_only_not_returned +materialized_features[member='feature_name=main.myschema.bindfeat|online_store_name=danielb|is_streaming_mode=false'].online_store_config.schema_name skip input_only_not_returned +materialized_features[member='feature_name=main.myschema.bindfeat|online_store_name=danielb|is_streaming_mode=false'].online_store_config.table_name_prefix skip input_only_not_returned + +=== And the deploy is a no-op rather than a rebuild +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Files: 1 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 1 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.materialization_sets.set1 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default + +Destroy: 1 deleted diff --git a/acceptance/bundle/resources/materialization_sets/bind/script b/acceptance/bundle/resources/materialization_sets/bind/script new file mode 100644 index 00000000000..fa69819fc52 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/bind/script @@ -0,0 +1,36 @@ +set_plan() { + jq '.plan | with_entries(select(.key | startswith("resources.materialization_sets"))) | map_values(.action)' out.plan.json +} + +destination_changes() { + jq -r '.plan["resources.materialization_sets.set1"].changes // {} | to_entries + | map(select(.key | test("store_config"))) + | map("\(.key) \(.value.action) \(.value.reason)") | .[]' out.plan.json +} + +trace $CLI bundle deploy +set_id=$($CLI bundle summary -o json | jq -r '.resources.materialization_sets.set1.id') + +title "Unbinding leaves the set in the workspace and drops it from the bundle's state, which is" +title "the same shape as a set created out of band: the plan reports create, because the bundle" +title "has nothing recorded for it" +trace $CLI bundle deployment unbind set1 +trace $CLI bundle plan -o json > out.plan.json +trace set_plan + +title "Binding the existing set takes ownership of it again" +trace $CLI bundle deployment bind set1 "$set_id" --auto-approve + +title "The first plan after a bind settles. Bind seeds state from the resolved config rather" +title "than from a read, so the destination the read never returns is suppressed here exactly as" +title "it is on a set the bundle created" +trace $CLI bundle plan -o json > out.plan.json +trace set_plan +trace destination_changes + +title "And the deploy is a no-op rather than a rebuild" +trace $CLI bundle deploy + +trace $CLI bundle destroy --auto-approve + +rm -f out.plan.json diff --git a/acceptance/bundle/resources/materialization_sets/bind/test.toml b/acceptance/bundle/resources/materialization_sets/bind/test.toml new file mode 100644 index 00000000000..83e04113992 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/bind/test.toml @@ -0,0 +1,5 @@ +# bind and unbind refuse to run while deployment history recording is on. +EnvMatrix.DMS = [""] + +# The case asserts plan actions and skip reasons, not request shapes. +RecordRequests = false diff --git a/acceptance/bundle/resources/materialization_sets/destination/databricks.yml b/acceptance/bundle/resources/materialization_sets/destination/databricks.yml new file mode 100644 index 00000000000..b71b583e8c9 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/destination/databricks.yml @@ -0,0 +1,24 @@ +bundle: + name: test-bundle + +resources: + materialization_sets: + set1: + materialized_features: + - feature_name: main.myschema.destfeat + offline_store_config: + catalog_name: main + schema_name: myschema + table_name_prefix: destfeat_off + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" + - feature_name: main.myschema.destfeat + online_store_config: + online_store_name: danielb + catalog_name: main + schema_name: myschema + table_name_prefix: destfeat_on + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" diff --git a/acceptance/bundle/resources/materialization_sets/destination/out.test.toml b/acceptance/bundle/resources/materialization_sets/destination/out.test.toml new file mode 100644 index 00000000000..19b5b15d11f --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/destination/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/resources/materialization_sets/destination/output.txt b/acceptance/bundle/resources/materialization_sets/destination/output.txt new file mode 100644 index 00000000000..dae6c56dd9a --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/destination/output.txt @@ -0,0 +1,101 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Created materialization_sets.set1 +Files: 1 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +=== A destination the read never returns is not drift: the config holds a value the server +=== never echoes, and with no persisted value to compare it to the suppression lives in +=== OverrideChangeDesc +>>> [CLI] bundle plan -o json + +>>> destination_changes +{ + "action": "skip", + "store_config_changes": { + "materialized_features[member='feature_name=main.myschema.destfeat|is_streaming_mode=false'].offline_store_config": { + "action": "skip", + "reason": "input_only_not_returned" + }, + "materialized_features[member='feature_name=main.myschema.destfeat|online_store_name=danielb|is_streaming_mode=false'].online_store_config.catalog_name": { + "action": "skip", + "reason": "input_only_not_returned" + }, + "materialized_features[member='feature_name=main.myschema.destfeat|online_store_name=danielb|is_streaming_mode=false'].online_store_config.schema_name": { + "action": "skip", + "reason": "input_only_not_returned" + }, + "materialized_features[member='feature_name=main.myschema.destfeat|online_store_name=danielb|is_streaming_mode=false'].online_store_config.table_name_prefix": { + "action": "skip", + "reason": "input_only_not_returned" + } + } +} + +=== Repointing the online destination is a real change the config asked for, so it keeps its +=== persisted value and still recreates +>>> update_file.py databricks.yml catalog_name: main + schema_name: myschema + table_name_prefix: destfeat_on catalog_name: other + schema_name: myschema + table_name_prefix: destfeat_on + +>>> [CLI] bundle plan -o json + +>>> destination_changes +{ + "action": "recreate", + "store_config_changes": { + "materialized_features[member='feature_name=main.myschema.destfeat|is_streaming_mode=false'].offline_store_config": { + "action": "skip", + "reason": "input_only_not_returned" + }, + "materialized_features[member='feature_name=main.myschema.destfeat|online_store_name=danielb|is_streaming_mode=false'].online_store_config.catalog_name": { + "action": "recreate", + "reason": "spec:immutable" + }, + "materialized_features[member='feature_name=main.myschema.destfeat|online_store_name=danielb|is_streaming_mode=false'].online_store_config.schema_name": { + "action": "skip", + "reason": "input_only_not_returned" + }, + "materialized_features[member='feature_name=main.myschema.destfeat|online_store_name=danielb|is_streaming_mode=false'].online_store_config.table_name_prefix": { + "action": "skip", + "reason": "input_only_not_returned" + } + } +} + +>>> update_file.py databricks.yml catalog_name: other catalog_name: main + +=== The offline block is input-only whole, and repointing it recreates the same way +>>> update_file.py databricks.yml table_name_prefix: destfeat_off table_name_prefix: destfeat_moved + +>>> [CLI] bundle plan -o json + +>>> destination_changes +{ + "action": "recreate", + "store_config_changes": { + "materialized_features[member='feature_name=main.myschema.destfeat|is_streaming_mode=false'].offline_store_config": { + "action": "skip", + "reason": "input_only_not_returned" + }, + "materialized_features[member='feature_name=main.myschema.destfeat|is_streaming_mode=false'].offline_store_config.table_name_prefix": { + "action": "recreate", + "reason": "spec:immutable" + }, + "materialized_features[member='feature_name=main.myschema.destfeat|online_store_name=danielb|is_streaming_mode=false'].online_store_config.catalog_name": { + "action": "skip", + "reason": "input_only_not_returned" + }, + "materialized_features[member='feature_name=main.myschema.destfeat|online_store_name=danielb|is_streaming_mode=false'].online_store_config.schema_name": { + "action": "skip", + "reason": "input_only_not_returned" + }, + "materialized_features[member='feature_name=main.myschema.destfeat|online_store_name=danielb|is_streaming_mode=false'].online_store_config.table_name_prefix": { + "action": "skip", + "reason": "input_only_not_returned" + } + } +} diff --git a/acceptance/bundle/resources/materialization_sets/destination/script b/acceptance/bundle/resources/materialization_sets/destination/script new file mode 100644 index 00000000000..dde15565eb2 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/destination/script @@ -0,0 +1,32 @@ +trace $CLI bundle deploy + +destination_changes() { + jq '.plan["resources.materialization_sets.set1"] + | {action, store_config_changes: (.changes + | with_entries(select(.key | test("store_config"))) + | map_values({action, reason}))}' out.plan.json +} + +title "A destination the read never returns is not drift: the config holds a value the server" +title "never echoes, and with no persisted value to compare it to the suppression lives in" +title "OverrideChangeDesc" +trace $CLI bundle plan -o json > out.plan.json +trace destination_changes + +title "Repointing the online destination is a real change the config asked for, so it keeps its" +title "persisted value and still recreates" +trace update_file.py databricks.yml ' catalog_name: main + schema_name: myschema + table_name_prefix: destfeat_on' ' catalog_name: other + schema_name: myschema + table_name_prefix: destfeat_on' +trace $CLI bundle plan -o json > out.plan.json +trace destination_changes +trace update_file.py databricks.yml ' catalog_name: other' ' catalog_name: main' + +title "The offline block is input-only whole, and repointing it recreates the same way" +trace update_file.py databricks.yml 'table_name_prefix: destfeat_off' 'table_name_prefix: destfeat_moved' +trace $CLI bundle plan -o json > out.plan.json +trace destination_changes + +rm -f out.requests.txt out.plan.json diff --git a/acceptance/bundle/resources/materialization_sets/immutable/added-member.yml.tmpl b/acceptance/bundle/resources/materialization_sets/immutable/added-member.yml.tmpl new file mode 100644 index 00000000000..36ae6d6b313 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/immutable/added-member.yml.tmpl @@ -0,0 +1,59 @@ +bundle: + name: test-matset-immutable-$UNIQUE_NAME + +resources: + features: + feata: + full_name: $CATALOG_NAME.$SCHEMA_NAME.imma_$UNIQUE_NAME + source: + delta_table_source: + full_name: $TABLE_NAME + function: + aggregation_function: + count_function: + input: value + time_window: + sliding: + window_duration: 604800s + slide_duration: 86400s + entities: + - name: id + timeseries_column: + name: timestamp + featb: + full_name: $CATALOG_NAME.$SCHEMA_NAME.immb_$UNIQUE_NAME + source: + delta_table_source: + full_name: $TABLE_NAME + function: + aggregation_function: + count_function: + input: value + time_window: + sliding: + window_duration: 604800s + slide_duration: 86400s + entities: + - name: id + timeseries_column: + name: timestamp + + materialization_sets: + set1: + materialized_features: + - feature_name: ${resources.features.feata.full_name} + offline_store_config: + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: imm_${UNIQUE_NAME}_off + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" + - feature_name: ${resources.features.featb.full_name} + offline_store_config: + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: imm_${UNIQUE_NAME}_off + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" diff --git a/acceptance/bundle/resources/materialization_sets/immutable/databricks.yml.tmpl b/acceptance/bundle/resources/materialization_sets/immutable/databricks.yml.tmpl new file mode 100644 index 00000000000..9be8c5add3c --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/immutable/databricks.yml.tmpl @@ -0,0 +1,53 @@ +bundle: + name: test-matset-immutable-$UNIQUE_NAME + +resources: + features: + feata: + full_name: $CATALOG_NAME.$SCHEMA_NAME.imma_$UNIQUE_NAME + source: + delta_table_source: + full_name: $TABLE_NAME + function: + aggregation_function: + count_function: + input: value + time_window: + sliding: + window_duration: 604800s + slide_duration: 86400s + entities: + - name: id + timeseries_column: + name: timestamp + # Declared so the set can try to add a member for it. A member names a feature that must + # already exist, and the add is refused by the set update rather than by the feature lookup. + featb: + full_name: $CATALOG_NAME.$SCHEMA_NAME.immb_$UNIQUE_NAME + source: + delta_table_source: + full_name: $TABLE_NAME + function: + aggregation_function: + count_function: + input: value + time_window: + sliding: + window_duration: 604800s + slide_duration: 86400s + entities: + - name: id + timeseries_column: + name: timestamp + + materialization_sets: + set1: + materialized_features: + - feature_name: ${resources.features.feata.full_name} + offline_store_config: + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: imm_${UNIQUE_NAME}_off + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" diff --git a/acceptance/bundle/resources/materialization_sets/immutable/out.test.toml b/acceptance/bundle/resources/materialization_sets/immutable/out.test.toml new file mode 100644 index 00000000000..f68b4a89c36 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/immutable/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/resources/materialization_sets/immutable/output.txt b/acceptance/bundle/resources/materialization_sets/immutable/output.txt new file mode 100644 index 00000000000..1c9b76d9c5b --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/immutable/output.txt @@ -0,0 +1,82 @@ + +>>> [CLI] schemas create matset_[UNIQUE_NAME] main -o json +{ + "full_name": "main.matset_[UNIQUE_NAME]" +} + +>>> create_table.py main.matset_[UNIQUE_NAME].src_[UNIQUE_NAME] +Created table main.matset_[UNIQUE_NAME].src_[UNIQUE_NAME] +Inserted sample data into main.matset_[UNIQUE_NAME].src_[UNIQUE_NAME] +Table main.matset_[UNIQUE_NAME].src_[UNIQUE_NAME] is now visible (catalog_name=main) + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-immutable-[UNIQUE_NAME]/default/files... +Created features.feata +Created features.featb +Created materialization_sets.set1 +Files: 3 uploaded, 0 deleted +Resources: 3 created, 0 changed, 0 deleted, 0 unchanged + +=== A member field the endpoint cannot alter is caught by the planner as a recreate, so the +=== request never reaches the server +>>> [CLI] bundle plan -o json + +>>> set_action +{ + "resources.materialization_sets.set1": "recreate" +} + +=== Adding a member is routed to update, where the server refuses it by name rather than +=== recreating the set and destroying the members the config left alone +>>> [CLI] bundle plan -o json + +>>> set_action +{ + "resources.materialization_sets.set1": "update" +} + +>>> musterr [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-immutable-[UNIQUE_NAME]/default/files... +Error: cannot update resources.materialization_sets.set1: updating id=[UUID]: Materialized features cannot be added to an existing materialization set, and feature 'main.matset_[UNIQUE_NAME].immb_[UNIQUE_NAME]' does not match any member of this set. Recreate the set instead. (400 INVALID_PARAMETER_VALUE) + +Endpoint: PATCH [DATABRICKS_URL]/api/2.0/feature-engineering/materialization-sets/[UUID]?request_id=[UUID]&update_mask=%2A +HTTP Status: 400 Bad Request +API error_code: INVALID_PARAMETER_VALUE +API message: Materialized features cannot be added to an existing materialization set, and feature 'main.matset_[UNIQUE_NAME].immb_[UNIQUE_NAME]' does not match any member of this set. Recreate the set instead. + +Files: 2 uploaded, 0 deleted + +=== Every refusal above leaves destroy and redeploy as the only route, so that pair is what a +=== member-field change costs. The delete settles its own operation, and the re-created set +=== plans clean. +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.features.feata + delete resources.features.featb + delete resources.materialization_sets.set1 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-matset-immutable-[UNIQUE_NAME]/default + +Destroy: 3 deleted + +>>> [CLI] bundle plan -o json + +>>> set_action +{ + "resources.materialization_sets.set1": "create" +} + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-immutable-[UNIQUE_NAME]/default/files... +Created features.feata +Created features.featb +Created materialization_sets.set1 +Files: 4 uploaded, 0 deleted +Resources: 3 created, 0 changed, 0 deleted, 0 unchanged + +>>> [CLI] bundle plan -o json + +>>> set_action +{ + "resources.materialization_sets.set1": "skip" +} diff --git a/acceptance/bundle/resources/materialization_sets/immutable/script b/acceptance/bundle/resources/materialization_sets/immutable/script new file mode 100644 index 00000000000..f93032d6b55 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/immutable/script @@ -0,0 +1,54 @@ +export CATALOG_NAME="${MATSET_CATALOG:-main}" +export SCHEMA_NAME="matset_${UNIQUE_NAME}" +export TABLE_NAME="$CATALOG_NAME.$SCHEMA_NAME.src_$UNIQUE_NAME" + +# Each phase is its own template rather than a string edit, so the fixture a phase deploys is +# readable on its own. +render() { + envsubst < "$1" > databricks.yml + [ -n "${MATSET_PROFILE:-}" ] || return 0 + printf "\nworkspace:\n profile: %s\n" "$MATSET_PROFILE" >> databricks.yml +} + +set_action() { + jq '.plan | with_entries(select(.key | startswith("resources.materialization_sets"))) | map_values(.action)' out.plan.json +} + +cleanup() { + $CLI bundle destroy --auto-approve > /dev/null 2>&1 || true + $CLI tables delete "$TABLE_NAME" > /dev/null 2>&1 || true + $CLI schemas delete "$CATALOG_NAME.$SCHEMA_NAME" --force > /dev/null 2>&1 || true +} +trap cleanup EXIT + +trace $CLI schemas create "$SCHEMA_NAME" "$CATALOG_NAME" -o json | jq '{full_name}' +trace create_table.py "$TABLE_NAME" +render databricks.yml.tmpl + +trace $CLI bundle deploy + +title "A member field the endpoint cannot alter is caught by the planner as a recreate, so the" +title "request never reaches the server" +update_file.py databricks.yml '0 0 * * * ?' '0 30 * * * ?' +trace $CLI bundle plan -o json > out.plan.json +trace set_action + +title "Adding a member is routed to update, where the server refuses it by name rather than" +title "recreating the set and destroying the members the config left alone" +render added-member.yml.tmpl +trace $CLI bundle plan -o json > out.plan.json +trace set_action +trace musterr $CLI bundle deploy --auto-approve + +title "Every refusal above leaves destroy and redeploy as the only route, so that pair is what a" +title "member-field change costs. The delete settles its own operation, and the re-created set" +title "plans clean." +render databricks.yml.tmpl +trace $CLI bundle destroy --auto-approve +trace $CLI bundle plan -o json > out.plan.json +trace set_action +trace $CLI bundle deploy +trace $CLI bundle plan -o json > out.plan.json +trace set_action + +rm -f out.requests.txt out.plan.json diff --git a/acceptance/bundle/resources/materialization_sets/immutable/test.toml b/acceptance/bundle/resources/materialization_sets/immutable/test.toml new file mode 100644 index 00000000000..9f9dbff911d --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/immutable/test.toml @@ -0,0 +1,32 @@ +# Creates its own schema and deletes it on exit. Override MATSET_CATALOG where `main` is absent. +Cloud = true + +# Both legs run locally. A cloud run must select the DMS= leg: the bundle deployments API +# rejects features. and materialization_sets. as resource-type prefixes. +EnvMatrix.DMS = ["", "true"] + +Ignore = [ + ".databricks", + "databricks.yml", +] + +# A real materialization set or member id is 32 undashed hex characters; the harness's built-in +# replacement only matches the dashed form the fake produces. +[[Repls]] +Old = '\b[0-9a-f]{32}\b' +New = '[UUID]' + +# The workspace host, which only a cloud run puts in the output. +[[Repls]] +Old = 'https://[a-z0-9.-]+\.cloud\.databricks\.com' +New = '[DATABRICKS_URL]' + +# Fake SQL endpoint so create_table.py works against the local test server too. +[[Server]] +Pattern = "POST /api/2.0/sql/statements/" +Response.Body = '{"status": {"state": "SUCCEEDED"}, "manifest": {"schema": {"columns": []}}}' + +# The cleanup drops the source table; the fake has no delete handler for one. +[[Server]] +Pattern = "DELETE /api/2.1/unity-catalog/tables/{full_name}" +Response.Body = '{}' diff --git a/acceptance/bundle/resources/materialization_sets/invalid_members/duplicate.yml b/acceptance/bundle/resources/materialization_sets/invalid_members/duplicate.yml new file mode 100644 index 00000000000..937305da79a --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/invalid_members/duplicate.yml @@ -0,0 +1,23 @@ +bundle: + name: test-matset-invalid-members + +resources: + materialization_sets: + set1: + # The offline destination is outside the member key, because the read never returns it, so + # these two are the same member however their destinations are spelled. + materialized_features: + - feature_name: main.default.feat + offline_store_config: + catalog_name: main + schema_name: default + table_name_prefix: feat_off + cron_schedule_trigger: + mode: DERIVED + - feature_name: main.default.feat + offline_store_config: + catalog_name: main + schema_name: default + table_name_prefix: feat_elsewhere + cron_schedule_trigger: + mode: DERIVED diff --git a/acceptance/bundle/resources/materialization_sets/invalid_members/out.test.toml b/acceptance/bundle/resources/materialization_sets/invalid_members/out.test.toml new file mode 100644 index 00000000000..19b5b15d11f --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/invalid_members/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/resources/materialization_sets/invalid_members/output.txt b/acceptance/bundle/resources/materialization_sets/invalid_members/output.txt new file mode 100644 index 00000000000..0f15a98ed33 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/invalid_members/output.txt @@ -0,0 +1,16 @@ + +=== Two members differing only outside the member key are the same member declared twice +>>> musterr [CLI] bundle validate +Error: Features 'main.default.feat' in materialization set 'set1' each declares more than one materialized feature with the following same fields: feature name, online store name, trigger is streaming mode, which is not allowed. + at resources.materialization_sets.set1.materialized_features[0] + resources.materialization_sets.set1.materialized_features[1] + in databricks.yml:10:11 + databricks.yml:17:11 + +Name: test-matset-invalid-members +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/test-matset-invalid-members/default + +Found 1 error diff --git a/acceptance/bundle/resources/materialization_sets/invalid_members/script b/acceptance/bundle/resources/materialization_sets/invalid_members/script new file mode 100644 index 00000000000..c3abd09c99a --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/invalid_members/script @@ -0,0 +1,8 @@ +# Both rejections are read off the config, so neither reaches the server. +render() { + cp "$1" databricks.yml +} + +title "Two members differing only outside the member key are the same member declared twice" +render duplicate.yml +trace musterr $CLI bundle validate diff --git a/acceptance/bundle/resources/materialization_sets/membership/databricks.yml.tmpl b/acceptance/bundle/resources/materialization_sets/membership/databricks.yml.tmpl new file mode 100644 index 00000000000..340901165da --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/membership/databricks.yml.tmpl @@ -0,0 +1,78 @@ +bundle: + name: test-matset-membership-$UNIQUE_NAME + +resources: + features: + feata: + full_name: $CATALOG_NAME.$SCHEMA_NAME.feata_$UNIQUE_NAME + source: + delta_table_source: + full_name: $TABLE_NAME + function: + aggregation_function: + count_function: + input: value + time_window: + sliding: + window_duration: 604800s + slide_duration: 86400s + entities: + - name: id + timeseries_column: + name: timestamp + featb: + full_name: $CATALOG_NAME.$SCHEMA_NAME.featb_$UNIQUE_NAME + source: + delta_table_source: + full_name: $TABLE_NAME + function: + aggregation_function: + count_function: + input: value + time_window: + sliding: + window_duration: 604800s + slide_duration: 86400s + entities: + - name: id + timeseries_column: + name: timestamp + + materialization_sets: + set1: + # Both features are materialized to both stores, so either member of a pair cascades to the other. + materialized_features: + - feature_name: ${resources.features.feata.full_name} + offline_store_config: + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: ${PREFIX}_off + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" + - feature_name: ${resources.features.feata.full_name} + online_store_config: + online_store_name: $ONLINE_STORE_NAME + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: $PREFIX + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" + - feature_name: ${resources.features.featb.full_name} + offline_store_config: + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: ${PREFIX}_off + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" + - feature_name: ${resources.features.featb.full_name} + online_store_config: + online_store_name: $ONLINE_STORE_NAME + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: $PREFIX + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" diff --git a/acceptance/bundle/resources/materialization_sets/membership/no-feata.yml.tmpl b/acceptance/bundle/resources/materialization_sets/membership/no-feata.yml.tmpl new file mode 100644 index 00000000000..d8d5e2f7c27 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/membership/no-feata.yml.tmpl @@ -0,0 +1,61 @@ +bundle: + name: test-matset-membership-$UNIQUE_NAME + +resources: + features: + feata: + full_name: $CATALOG_NAME.$SCHEMA_NAME.feata_$UNIQUE_NAME + source: + delta_table_source: + full_name: $TABLE_NAME + function: + aggregation_function: + count_function: + input: value + time_window: + sliding: + window_duration: 604800s + slide_duration: 86400s + entities: + - name: id + timeseries_column: + name: timestamp + featb: + full_name: $CATALOG_NAME.$SCHEMA_NAME.featb_$UNIQUE_NAME + source: + delta_table_source: + full_name: $TABLE_NAME + function: + aggregation_function: + count_function: + input: value + time_window: + sliding: + window_duration: 604800s + slide_duration: 86400s + entities: + - name: id + timeseries_column: + name: timestamp + + materialization_sets: + set1: + # feata's whole pair dropped; featb's pair survives. + materialized_features: + - feature_name: ${resources.features.featb.full_name} + offline_store_config: + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: ${PREFIX}_off + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" + - feature_name: ${resources.features.featb.full_name} + online_store_config: + online_store_name: $ONLINE_STORE_NAME + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: $PREFIX + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" diff --git a/acceptance/bundle/resources/materialization_sets/membership/no-offline.yml.tmpl b/acceptance/bundle/resources/materialization_sets/membership/no-offline.yml.tmpl new file mode 100644 index 00000000000..4bee71b4545 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/membership/no-offline.yml.tmpl @@ -0,0 +1,70 @@ +bundle: + name: test-matset-membership-$UNIQUE_NAME + +resources: + features: + feata: + full_name: $CATALOG_NAME.$SCHEMA_NAME.feata_$UNIQUE_NAME + source: + delta_table_source: + full_name: $TABLE_NAME + function: + aggregation_function: + count_function: + input: value + time_window: + sliding: + window_duration: 604800s + slide_duration: 86400s + entities: + - name: id + timeseries_column: + name: timestamp + featb: + full_name: $CATALOG_NAME.$SCHEMA_NAME.featb_$UNIQUE_NAME + source: + delta_table_source: + full_name: $TABLE_NAME + function: + aggregation_function: + count_function: + input: value + time_window: + sliding: + window_duration: 604800s + slide_duration: 86400s + entities: + - name: id + timeseries_column: + name: timestamp + + materialization_sets: + set1: + # feata's offline member dropped on its own. + materialized_features: + - feature_name: ${resources.features.feata.full_name} + online_store_config: + online_store_name: $ONLINE_STORE_NAME + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: $PREFIX + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" + - feature_name: ${resources.features.featb.full_name} + offline_store_config: + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: ${PREFIX}_off + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" + - feature_name: ${resources.features.featb.full_name} + online_store_config: + online_store_name: $ONLINE_STORE_NAME + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: $PREFIX + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" diff --git a/acceptance/bundle/resources/materialization_sets/membership/no-online.yml.tmpl b/acceptance/bundle/resources/materialization_sets/membership/no-online.yml.tmpl new file mode 100644 index 00000000000..5895db5f4cf --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/membership/no-online.yml.tmpl @@ -0,0 +1,69 @@ +bundle: + name: test-matset-membership-$UNIQUE_NAME + +resources: + features: + feata: + full_name: $CATALOG_NAME.$SCHEMA_NAME.feata_$UNIQUE_NAME + source: + delta_table_source: + full_name: $TABLE_NAME + function: + aggregation_function: + count_function: + input: value + time_window: + sliding: + window_duration: 604800s + slide_duration: 86400s + entities: + - name: id + timeseries_column: + name: timestamp + featb: + full_name: $CATALOG_NAME.$SCHEMA_NAME.featb_$UNIQUE_NAME + source: + delta_table_source: + full_name: $TABLE_NAME + function: + aggregation_function: + count_function: + input: value + time_window: + sliding: + window_duration: 604800s + slide_duration: 86400s + entities: + - name: id + timeseries_column: + name: timestamp + + materialization_sets: + set1: + # feata's online member dropped on its own. + materialized_features: + - feature_name: ${resources.features.feata.full_name} + offline_store_config: + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: ${PREFIX}_off + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" + - feature_name: ${resources.features.featb.full_name} + offline_store_config: + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: ${PREFIX}_off + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" + - feature_name: ${resources.features.featb.full_name} + online_store_config: + online_store_name: $ONLINE_STORE_NAME + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: $PREFIX + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" diff --git a/acceptance/bundle/resources/materialization_sets/membership/out.test.toml b/acceptance/bundle/resources/materialization_sets/membership/out.test.toml new file mode 100644 index 00000000000..19b5b15d11f --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/membership/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/resources/materialization_sets/membership/output.txt b/acceptance/bundle/resources/materialization_sets/membership/output.txt new file mode 100644 index 00000000000..b294d388c2e --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/membership/output.txt @@ -0,0 +1,71 @@ + +>>> [CLI] schemas create matset_[UNIQUE_NAME] main -o json +{ + "full_name": "main.matset_[UNIQUE_NAME]" +} + +>>> create_table.py main.matset_[UNIQUE_NAME].src_[UNIQUE_NAME] +Created table main.matset_[UNIQUE_NAME].src_[UNIQUE_NAME] +Inserted sample data into main.matset_[UNIQUE_NAME].src_[UNIQUE_NAME] +Table main.matset_[UNIQUE_NAME].src_[UNIQUE_NAME] is now visible (catalog_name=main) + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-membership-[UNIQUE_NAME]/default/files... +Created features.feata +Created features.featb +Created materialization_sets.set1 +Files: 5 uploaded, 0 deleted +Resources: 3 created, 0 changed, 0 deleted, 0 unchanged + +=== The cascade is per feature and runs both ways. Removing feata's online member alone is +=== refused, because the delete also reaches the offline member the config still declares +>>> musterr [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-membership-[UNIQUE_NAME]/default/files... +Error: cannot update resources.materialization_sets.set1: updating id=[UUID]: Materialized features '[UUID]' of feature 'main.matset_[UNIQUE_NAME].feata_[UNIQUE_NAME]' are deleted only by deleting materialized features '[UUID]', which the request does not supply. Remove those as well. (400 INVALID_PARAMETER_VALUE) + +Endpoint: PATCH [DATABRICKS_URL]/api/2.0/feature-engineering/materialization-sets/[UUID]?request_id=[UUID]&update_mask=%2A +HTTP Status: 400 Bad Request +API error_code: INVALID_PARAMETER_VALUE +API message: Materialized features '[UUID]' of feature 'main.matset_[UNIQUE_NAME].feata_[UNIQUE_NAME]' are deleted only by deleting materialized features '[UUID]', which the request does not supply. Remove those as well. + +Files: 1 uploaded, 0 deleted + +=== And removing feata's offline member alone is refused for the same reason, from the other +=== direction +>>> musterr [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-membership-[UNIQUE_NAME]/default/files... +Error: cannot update resources.materialization_sets.set1: updating id=[UUID]: Deleting materialized features '[UUID]' of feature 'main.matset_[UNIQUE_NAME].feata_[UNIQUE_NAME]' also deletes materialized features '[UUID]', which the request does not delete. Delete the unrequested materialized features as well. (400 INVALID_PARAMETER_VALUE) + +Endpoint: PATCH [DATABRICKS_URL]/api/2.0/feature-engineering/materialization-sets/[UUID]?request_id=[UUID]&update_mask=%2A +HTTP Status: 400 Bad Request +API error_code: INVALID_PARAMETER_VALUE +API message: Deleting materialized features '[UUID]' of feature 'main.matset_[UNIQUE_NAME].feata_[UNIQUE_NAME]' also deletes materialized features '[UUID]', which the request does not delete. Delete the unrequested materialized features as well. + +Files: 1 uploaded, 0 deleted + +=== Removing a whole feature's pair plans as update, not recreate, and the other feature's +=== members keep their ids, so the set was reconciled rather than rebuilt +>>> [CLI] bundle plan -o json + +>>> set_action +{ + "resources.materialization_sets.set1": "update" +} + +>>> [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-membership-[UNIQUE_NAME]/default/files... +Updated materialization_sets.set1 +Files: 2 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 2 unchanged + +>>> set_members +[ + { + "feature_name": "${resources.features.featb.full_name}", + "is_online": null + }, + { + "feature_name": "${resources.features.featb.full_name}", + "is_online": null + } +] diff --git a/acceptance/bundle/resources/materialization_sets/membership/script b/acceptance/bundle/resources/materialization_sets/membership/script new file mode 100644 index 00000000000..9d112f6e31f --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/membership/script @@ -0,0 +1,55 @@ +export CATALOG_NAME="${MATSET_CATALOG:-main}" +export SCHEMA_NAME="matset_${UNIQUE_NAME}" +export ONLINE_STORE_NAME="${MATSET_ONLINE_STORE:-danielb}" +export TABLE_NAME="$CATALOG_NAME.$SCHEMA_NAME.src_$UNIQUE_NAME" +export PREFIX="mat_$UNIQUE_NAME" + +# Each phase is its own template rather than a string edit, so the membership a phase declares is +# readable on its own. +render() { + envsubst < "$1" > databricks.yml + [ -n "${MATSET_PROFILE:-}" ] || return 0 + printf "\nworkspace:\n profile: %s\n" "$MATSET_PROFILE" >> databricks.yml +} + +set_action() { + jq '.plan | with_entries(select(.key | startswith("resources.materialization_sets"))) | map_values(.action)' out.plan.json +} + +set_members() { + $CLI bundle summary -o json | jq '.resources.materialization_sets.set1.materialized_features | map({feature_name, is_online})' +} + +cleanup() { + $CLI bundle destroy --auto-approve > /dev/null 2>&1 || true + $CLI tables delete "$TABLE_NAME" > /dev/null 2>&1 || true + $CLI schemas delete "$CATALOG_NAME.$SCHEMA_NAME" --force > /dev/null 2>&1 || true +} + +trap cleanup EXIT + +trace $CLI schemas create "$SCHEMA_NAME" "$CATALOG_NAME" -o json | jq '{full_name}' +trace create_table.py "$TABLE_NAME" +render databricks.yml.tmpl + +trace $CLI bundle deploy + +title "The cascade is per feature and runs both ways. Removing feata's online member alone is" +title "refused, because the delete also reaches the offline member the config still declares" +render no-online.yml.tmpl +trace musterr $CLI bundle deploy --auto-approve + +title "And removing feata's offline member alone is refused for the same reason, from the other" +title "direction" +render no-offline.yml.tmpl +trace musterr $CLI bundle deploy --auto-approve + +title "Removing a whole feature's pair plans as update, not recreate, and the other feature's" +title "members keep their ids, so the set was reconciled rather than rebuilt" +render no-feata.yml.tmpl +trace $CLI bundle plan -o json > out.plan.json +trace set_action +trace $CLI bundle deploy --auto-approve +trace set_members + +rm -f out.plan.json diff --git a/acceptance/bundle/resources/materialization_sets/membership/test.toml b/acceptance/bundle/resources/materialization_sets/membership/test.toml new file mode 100644 index 00000000000..df19e58423b --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/membership/test.toml @@ -0,0 +1,34 @@ +# Local only. Every arm needs an online member, and the integration workspaces have no online store. +# Run it against a workspace by hand, naming one with MATSET_ONLINE_STORE -- see README.md. Creates +# its own schema and deletes it on exit. +Cloud = false + +# Both legs run locally. A cloud run must select the DMS= leg: the bundle deployments API +# rejects features. and materialization_sets. as resource-type prefixes. +EnvMatrix.DMS = ["", "true"] + +Ignore = [ + ".databricks", + "databricks.yml", +] + +# A real materialization set or member id is 32 undashed hex characters; the harness's built-in +# replacement only matches the dashed form the fake produces. +[[Repls]] +Old = '\b[0-9a-f]{32}\b' +New = '[UUID]' + +# The workspace host, which only a cloud run puts in the output. +[[Repls]] +Old = 'https://[a-z0-9.-]+\.cloud\.databricks\.com' +New = '[DATABRICKS_URL]' + +# Fake SQL endpoint so create_table.py works against the local test server too. +[[Server]] +Pattern = "POST /api/2.0/sql/statements/" +Response.Body = '{"status": {"state": "SUCCEEDED"}, "manifest": {"schema": {"columns": []}}}' + +# The cleanup drops the source table; the fake has no delete handler for one. +[[Server]] +Pattern = "DELETE /api/2.1/unity-catalog/tables/{full_name}" +Response.Body = '{}' diff --git a/acceptance/bundle/resources/materialization_sets/mixed_triggers/databricks.yml.tmpl b/acceptance/bundle/resources/materialization_sets/mixed_triggers/databricks.yml.tmpl new file mode 100644 index 00000000000..bb2a2718269 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/mixed_triggers/databricks.yml.tmpl @@ -0,0 +1,60 @@ +bundle: + name: test-matset-mixed-triggers-$UNIQUE_NAME + +resources: + features: + batchfeat: + full_name: $CATALOG_NAME.$SCHEMA_NAME.batch_$UNIQUE_NAME + source: + delta_table_source: + full_name: $TABLE_NAME + function: + aggregation_function: + count_function: + input: value + time_window: + sliding: + window_duration: 604800s + slide_duration: 86400s + entities: + - name: id + timeseries_column: + name: timestamp + + # Stream-sourced, so it materializes from a stream while batchfeat materializes in batch. + streamfeat: + full_name: $CATALOG_NAME.$SCHEMA_NAME.batchstream_$UNIQUE_NAME + source: + stream_source: + full_name: $STREAM_FULL_NAME + function: + aggregation_function: + count_function: + input: value.passenger_count_1 + time_window: + rolling: + window_duration: 3600s + entities: + - name: value.pickup_zip + timeseries_column: + name: value.event_timestamp + + materialization_sets: + set1: + materialized_features: + - feature_name: ${resources.features.batchfeat.full_name} + offline_store_config: + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: batch_${UNIQUE_NAME}_off + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" + - feature_name: ${resources.features.streamfeat.full_name} + online_store_config: + online_store_name: $ONLINE_STORE_NAME + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: batchstream_$UNIQUE_NAME + streaming_mode: + mode: STREAMING_MODE_TYPE_RTM diff --git a/acceptance/bundle/resources/materialization_sets/mixed_triggers/out.test.toml b/acceptance/bundle/resources/materialization_sets/mixed_triggers/out.test.toml new file mode 100644 index 00000000000..19b5b15d11f --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/mixed_triggers/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/resources/materialization_sets/mixed_triggers/output.txt b/acceptance/bundle/resources/materialization_sets/mixed_triggers/output.txt new file mode 100644 index 00000000000..4f281b89c9d --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/mixed_triggers/output.txt @@ -0,0 +1,22 @@ + +>>> create_table.py danielb.fs2680.src_[UNIQUE_NAME] +Created table danielb.fs2680.src_[UNIQUE_NAME] +Inserted sample data into danielb.fs2680.src_[UNIQUE_NAME] +Table danielb.fs2680.src_[UNIQUE_NAME] is now visible (catalog_name=danielb) + +=== A set cannot hold one feature that materializes in batch and one that materializes from a +=== stream, because the two need different triggers. The create endpoint reserves its operation +=== record before it reaches this rule, so the refusal settles that record and arrives as +=== HTTP 200 rather than as an error on the call. +>>> musterr [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-mixed-triggers-[UNIQUE_NAME]/default/files... +Created features.batchfeat +Created features.streamfeat +Error: cannot create resources.materialization_sets.set1: materialization set [UUID] operation failed: Batch and streaming features cannot be mixed in a single materialize_features call because they require different trigger types. Issue two separate calls: one for batch features and one for streaming features. Batch features: danielb.fs2680.batch_[UNIQUE_NAME]. Streaming features: danielb.fs2680.batchstream_[UNIQUE_NAME] (200 INVALID_PARAMETER_VALUE) + +Endpoint: n/a +HTTP Status: 200 +API error_code: INVALID_PARAMETER_VALUE +API message: materialization set [UUID] operation failed: Batch and streaming features cannot be mixed in a single materialize_features call because they require different trigger types. Issue two separate calls: one for batch features and one for streaming features. Batch features: danielb.fs2680.batch_[UNIQUE_NAME]. Streaming features: danielb.fs2680.batchstream_[UNIQUE_NAME] + +Files: 2 uploaded, 0 deleted diff --git a/acceptance/bundle/resources/materialization_sets/mixed_triggers/script b/acceptance/bundle/resources/materialization_sets/mixed_triggers/script new file mode 100644 index 00000000000..0e8545e78d0 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/mixed_triggers/script @@ -0,0 +1,28 @@ +export CATALOG_NAME="${MATSET_CATALOG:-danielb}" +export SCHEMA_NAME="${MATSET_SCHEMA:-fs2680}" +export ONLINE_STORE_NAME="${MATSET_ONLINE_STORE:-danielb}" +export STREAM_FULL_NAME="${MATSET_STREAM:-danielb.fs2680.fs2680_dch48o_stream}" +export TABLE_NAME="$CATALOG_NAME.$SCHEMA_NAME.src_$UNIQUE_NAME" + +render() { + envsubst < "$1" > databricks.yml + [ -n "${MATSET_PROFILE:-}" ] || return 0 + printf "\nworkspace:\n profile: %s\n" "$MATSET_PROFILE" >> databricks.yml +} + +cleanup() { + $CLI bundle destroy --auto-approve > /dev/null 2>&1 || true + $CLI tables delete "$TABLE_NAME" > /dev/null 2>&1 || true +} +trap cleanup EXIT + +trace create_table.py "$TABLE_NAME" +render databricks.yml.tmpl + +title "A set cannot hold one feature that materializes in batch and one that materializes from a" +title "stream, because the two need different triggers. The create endpoint reserves its operation" +title "record before it reaches this rule, so the refusal settles that record and arrives as" +title "HTTP 200 rather than as an error on the call." +trace musterr $CLI bundle deploy --auto-approve + +rm -f out.requests.txt diff --git a/acceptance/bundle/resources/materialization_sets/mixed_triggers/test.toml b/acceptance/bundle/resources/materialization_sets/mixed_triggers/test.toml new file mode 100644 index 00000000000..4f780663f61 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/mixed_triggers/test.toml @@ -0,0 +1,32 @@ +# Local only. This case needs a Stream source, and a Stream is neither a bundle resource nor a CLI +# command, so the integration suite has no way to provision one; it also needs a Unity Catalog +# connection to a Kafka topic whose payload the feature definitions name. Run it against a +# workspace by hand, overriding MATSET_CATALOG / MATSET_SCHEMA / MATSET_ONLINE_STORE / +# MATSET_STREAM -- see README.md. +Cloud = false + +Ignore = [ + ".databricks", + "databricks.yml", +] + +# A real materialization set or member id is 32 undashed hex characters; the harness's built-in +# replacement only matches the dashed form the fake produces. +[[Repls]] +Old = '\b[0-9a-f]{32}\b' +New = '[UUID]' + +# The workspace host, which only a cloud run puts in the output. +[[Repls]] +Old = 'https://[a-z0-9.-]+\.cloud\.databricks\.com' +New = '[DATABRICKS_URL]' + +# Fake SQL endpoint so create_table.py works against the local test server too. +[[Server]] +Pattern = "POST /api/2.0/sql/statements/" +Response.Body = '{"status": {"state": "SUCCEEDED"}, "manifest": {"schema": {"columns": []}}}' + +# The cleanup drops the source table; the fake has no delete handler for one. +[[Server]] +Pattern = "DELETE /api/2.1/unity-catalog/tables/{full_name}" +Response.Body = '{}' diff --git a/acceptance/bundle/resources/materialization_sets/removal_scope/databricks.yml.tmpl b/acceptance/bundle/resources/materialization_sets/removal_scope/databricks.yml.tmpl new file mode 100644 index 00000000000..6d2acd4fe7f --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/removal_scope/databricks.yml.tmpl @@ -0,0 +1,60 @@ +bundle: + name: test-matset-removal-scope-$UNIQUE_NAME + +resources: + features: + featf1: + full_name: $CATALOG_NAME.$SCHEMA_NAME.featf1_$UNIQUE_NAME + source: + delta_table_source: + full_name: $TABLE_NAME + function: + aggregation_function: + count_function: + input: value + time_window: + sliding: + window_duration: 604800s + slide_duration: 86400s + entities: + - name: id + timeseries_column: + name: timestamp + featf2: + full_name: $CATALOG_NAME.$SCHEMA_NAME.featf2_$UNIQUE_NAME + source: + delta_table_source: + full_name: $TABLE_NAME + function: + aggregation_function: + count_function: + input: value + time_window: + sliding: + window_duration: 604800s + slide_duration: 86400s + entities: + - name: id + timeseries_column: + name: timestamp + + materialization_sets: + set1: + # Each feature is materialized offline only, so each contributes exactly one member. + materialized_features: + - feature_name: ${resources.features.featf1.full_name} + offline_store_config: + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: $PREFIX + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" + - feature_name: ${resources.features.featf2.full_name} + offline_store_config: + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: $PREFIX + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" diff --git a/acceptance/bundle/resources/materialization_sets/removal_scope/no-f1.yml.tmpl b/acceptance/bundle/resources/materialization_sets/removal_scope/no-f1.yml.tmpl new file mode 100644 index 00000000000..ac6257da73e --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/removal_scope/no-f1.yml.tmpl @@ -0,0 +1,52 @@ +bundle: + name: test-matset-removal-scope-$UNIQUE_NAME + +resources: + features: + featf1: + full_name: $CATALOG_NAME.$SCHEMA_NAME.featf1_$UNIQUE_NAME + source: + delta_table_source: + full_name: $TABLE_NAME + function: + aggregation_function: + count_function: + input: value + time_window: + sliding: + window_duration: 604800s + slide_duration: 86400s + entities: + - name: id + timeseries_column: + name: timestamp + featf2: + full_name: $CATALOG_NAME.$SCHEMA_NAME.featf2_$UNIQUE_NAME + source: + delta_table_source: + full_name: $TABLE_NAME + function: + aggregation_function: + count_function: + input: value + time_window: + sliding: + window_duration: 604800s + slide_duration: 86400s + entities: + - name: id + timeseries_column: + name: timestamp + + materialization_sets: + set1: + # featf1's only member dropped; featf2 is untouched. + materialized_features: + - feature_name: ${resources.features.featf2.full_name} + offline_store_config: + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: $PREFIX + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" diff --git a/acceptance/bundle/resources/materialization_sets/removal_scope/out.test.toml b/acceptance/bundle/resources/materialization_sets/removal_scope/out.test.toml new file mode 100644 index 00000000000..f68b4a89c36 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/removal_scope/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/resources/materialization_sets/removal_scope/output.txt b/acceptance/bundle/resources/materialization_sets/removal_scope/output.txt new file mode 100644 index 00000000000..2942f8ed3b6 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/removal_scope/output.txt @@ -0,0 +1,75 @@ + +>>> [CLI] schemas create matset_[UNIQUE_NAME] main -o json +{ + "full_name": "main.matset_[UNIQUE_NAME]" +} + +>>> create_table.py main.matset_[UNIQUE_NAME].src_[UNIQUE_NAME] +Created table main.matset_[UNIQUE_NAME].src_[UNIQUE_NAME] +Inserted sample data into main.matset_[UNIQUE_NAME].src_[UNIQUE_NAME] +Table main.matset_[UNIQUE_NAME].src_[UNIQUE_NAME] is now visible (catalog_name=main) + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-removal-scope-[UNIQUE_NAME]/default/files... +Created features.featf1 +Created features.featf2 +Created materialization_sets.set1 +Files: 3 uploaded, 0 deleted +Resources: 3 created, 0 changed, 0 deleted, 0 unchanged + +=== A feature contributing one member can have it removed alone: the cascade set is that one +=== member, and the feature the request says nothing about is untouched +>>> [CLI] bundle plan -o json + +>>> set_action +{ + "resources.materialization_sets.set1": "update" +} + +>>> [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-removal-scope-[UNIQUE_NAME]/default/files... +Updated materialization_sets.set1 +Files: 2 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 2 unchanged + +>>> set_members +[ + { + "feature_name": "${resources.features.featf2.full_name}", + "is_online": null + } +] + +=== The update returns a long-running operation, and the reconciled set is read back from it +>>> print_requests.py --keep //materialization-sets +{ + "method": "PATCH", + "path": "/api/2.0/feature-engineering/materialization-sets/[UUID]", + "q": { + "request_id": "[UUID]", + "update_mask": "*" + }, + "body": { + "materialization_set_id": "[UUID]", + "materialized_features": [ + { + "cron_schedule_trigger": { + "cron_expression": "0 0 * * * ?", + "mode": "MANUAL" + }, + "feature_name": "main.matset_[UNIQUE_NAME].featf2_[UNIQUE_NAME]", + "offline_store_config": { + "catalog_name": "main", + "schema_name": "matset_[UNIQUE_NAME]", + "table_name_prefix": "mat_[UNIQUE_NAME]" + } + } + ] + } +} + +>>> print_requests.py --get //feature-engineering/operations +{ + "method": "GET", + "path": "/api/2.0/feature-engineering/operations/[UUID]" +} diff --git a/acceptance/bundle/resources/materialization_sets/removal_scope/script b/acceptance/bundle/resources/materialization_sets/removal_scope/script new file mode 100644 index 00000000000..88f0eab5039 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/removal_scope/script @@ -0,0 +1,50 @@ +export CATALOG_NAME="${MATSET_CATALOG:-main}" +export SCHEMA_NAME="matset_${UNIQUE_NAME}" +export TABLE_NAME="$CATALOG_NAME.$SCHEMA_NAME.src_$UNIQUE_NAME" +export PREFIX="mat_$UNIQUE_NAME" + +# Each phase is its own template rather than a string edit, so the membership a phase declares is +# readable on its own. +render() { + envsubst < "$1" > databricks.yml + [ -n "${MATSET_PROFILE:-}" ] || return 0 + printf "\nworkspace:\n profile: %s\n" "$MATSET_PROFILE" >> databricks.yml +} + +set_action() { + jq '.plan | with_entries(select(.key | startswith("resources.materialization_sets"))) | map_values(.action)' out.plan.json +} + +set_members() { + $CLI bundle summary -o json | jq '.resources.materialization_sets.set1.materialized_features | map({feature_name, is_online})' +} + +cleanup() { + $CLI bundle destroy --auto-approve > /dev/null 2>&1 || true + $CLI tables delete "$TABLE_NAME" > /dev/null 2>&1 || true + $CLI schemas delete "$CATALOG_NAME.$SCHEMA_NAME" --force > /dev/null 2>&1 || true +} +trap cleanup EXIT + +trace $CLI schemas create "$SCHEMA_NAME" "$CATALOG_NAME" -o json | jq '{full_name}' +trace create_table.py "$TABLE_NAME" +render databricks.yml.tmpl + +trace $CLI bundle deploy + +# The create's requests are pinned by the basic case, so drop them and let the update's stand alone. +rm -f "$OUT_REQUESTS" + +title "A feature contributing one member can have it removed alone: the cascade set is that one" +title "member, and the feature the request says nothing about is untouched" +render no-f1.yml.tmpl +trace $CLI bundle plan -o json > out.plan.json +trace set_action +trace $CLI bundle deploy --auto-approve +trace set_members + +title "The update returns a long-running operation, and the reconciled set is read back from it" +trace print_requests.py --keep //materialization-sets +trace print_requests.py --get //feature-engineering/operations + +rm -f out.plan.json diff --git a/acceptance/bundle/resources/materialization_sets/removal_scope/test.toml b/acceptance/bundle/resources/materialization_sets/removal_scope/test.toml new file mode 100644 index 00000000000..9f9dbff911d --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/removal_scope/test.toml @@ -0,0 +1,32 @@ +# Creates its own schema and deletes it on exit. Override MATSET_CATALOG where `main` is absent. +Cloud = true + +# Both legs run locally. A cloud run must select the DMS= leg: the bundle deployments API +# rejects features. and materialization_sets. as resource-type prefixes. +EnvMatrix.DMS = ["", "true"] + +Ignore = [ + ".databricks", + "databricks.yml", +] + +# A real materialization set or member id is 32 undashed hex characters; the harness's built-in +# replacement only matches the dashed form the fake produces. +[[Repls]] +Old = '\b[0-9a-f]{32}\b' +New = '[UUID]' + +# The workspace host, which only a cloud run puts in the output. +[[Repls]] +Old = 'https://[a-z0-9.-]+\.cloud\.databricks\.com' +New = '[DATABRICKS_URL]' + +# Fake SQL endpoint so create_table.py works against the local test server too. +[[Server]] +Pattern = "POST /api/2.0/sql/statements/" +Response.Body = '{"status": {"state": "SUCCEEDED"}, "manifest": {"schema": {"columns": []}}}' + +# The cleanup drops the source table; the fake has no delete handler for one. +[[Server]] +Pattern = "DELETE /api/2.1/unity-catalog/tables/{full_name}" +Response.Body = '{}' diff --git a/acceptance/bundle/resources/materialization_sets/sawtooth/databricks.yml.tmpl b/acceptance/bundle/resources/materialization_sets/sawtooth/databricks.yml.tmpl new file mode 100644 index 00000000000..8f76bfee764 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/sawtooth/databricks.yml.tmpl @@ -0,0 +1,51 @@ +bundle: + name: test-matset-sawtooth-$UNIQUE_NAME + +resources: + features: + sawfeat: + full_name: $CATALOG_NAME.$SCHEMA_NAME.saw_$UNIQUE_NAME + source: + stream_source: + full_name: $STREAM_FULL_NAME + function: + aggregation_function: + sum: + input: value.passenger_count_1 + time_window: + sawtooth: + window_duration: 259200s + entities: + - name: value.pickup_zip + timeseries_column: + name: value.event_timestamp + + materialization_sets: + set1: + materialized_features: + # Offline batch, on the schedule the service derives for a sawtooth window. + - feature_name: ${resources.features.sawfeat.full_name} + offline_store_config: + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: saw_${UNIQUE_NAME}_off + cron_schedule_trigger: + mode: DERIVED + # Online batch, sharing its destination with the streaming member below. + - feature_name: ${resources.features.sawfeat.full_name} + online_store_config: + online_store_name: $ONLINE_STORE_NAME + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: saw_$UNIQUE_NAME + cron_schedule_trigger: + mode: DERIVED + # Online streaming. RTM is the only streaming mode a non-Kinesis stream source accepts. + - feature_name: ${resources.features.sawfeat.full_name} + online_store_config: + online_store_name: $ONLINE_STORE_NAME + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: saw_$UNIQUE_NAME + streaming_mode: + mode: STREAMING_MODE_TYPE_RTM diff --git a/acceptance/bundle/resources/materialization_sets/sawtooth/drop-all.yml.tmpl b/acceptance/bundle/resources/materialization_sets/sawtooth/drop-all.yml.tmpl new file mode 100644 index 00000000000..34db72c23f8 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/sawtooth/drop-all.yml.tmpl @@ -0,0 +1,25 @@ +bundle: + name: test-matset-sawtooth-$UNIQUE_NAME + +resources: + features: + sawfeat: + full_name: $CATALOG_NAME.$SCHEMA_NAME.saw_$UNIQUE_NAME + source: + stream_source: + full_name: $STREAM_FULL_NAME + function: + aggregation_function: + sum: + input: value.passenger_count_1 + time_window: + sawtooth: + window_duration: 259200s + entities: + - name: value.pickup_zip + timeseries_column: + name: value.event_timestamp + + materialization_sets: + set1: + materialized_features: [] diff --git a/acceptance/bundle/resources/materialization_sets/sawtooth/drop-batch-pair.yml.tmpl b/acceptance/bundle/resources/materialization_sets/sawtooth/drop-batch-pair.yml.tmpl new file mode 100644 index 00000000000..cfae25e8a3b --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/sawtooth/drop-batch-pair.yml.tmpl @@ -0,0 +1,34 @@ +bundle: + name: test-matset-sawtooth-$UNIQUE_NAME + +resources: + features: + sawfeat: + full_name: $CATALOG_NAME.$SCHEMA_NAME.saw_$UNIQUE_NAME + source: + stream_source: + full_name: $STREAM_FULL_NAME + function: + aggregation_function: + sum: + input: value.passenger_count_1 + time_window: + sawtooth: + window_duration: 259200s + entities: + - name: value.pickup_zip + timeseries_column: + name: value.event_timestamp + + materialization_sets: + set1: + materialized_features: + # Online streaming. RTM is the only streaming mode a non-Kinesis stream source accepts. + - feature_name: ${resources.features.sawfeat.full_name} + online_store_config: + online_store_name: $ONLINE_STORE_NAME + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: saw_$UNIQUE_NAME + streaming_mode: + mode: STREAMING_MODE_TYPE_RTM diff --git a/acceptance/bundle/resources/materialization_sets/sawtooth/drop-offline.yml.tmpl b/acceptance/bundle/resources/materialization_sets/sawtooth/drop-offline.yml.tmpl new file mode 100644 index 00000000000..9f86ac53015 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/sawtooth/drop-offline.yml.tmpl @@ -0,0 +1,43 @@ +bundle: + name: test-matset-sawtooth-$UNIQUE_NAME + +resources: + features: + sawfeat: + full_name: $CATALOG_NAME.$SCHEMA_NAME.saw_$UNIQUE_NAME + source: + stream_source: + full_name: $STREAM_FULL_NAME + function: + aggregation_function: + sum: + input: value.passenger_count_1 + time_window: + sawtooth: + window_duration: 259200s + entities: + - name: value.pickup_zip + timeseries_column: + name: value.event_timestamp + + materialization_sets: + set1: + materialized_features: + # Online batch, sharing its destination with the streaming member below. + - feature_name: ${resources.features.sawfeat.full_name} + online_store_config: + online_store_name: $ONLINE_STORE_NAME + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: saw_$UNIQUE_NAME + cron_schedule_trigger: + mode: DERIVED + # Online streaming. RTM is the only streaming mode a non-Kinesis stream source accepts. + - feature_name: ${resources.features.sawfeat.full_name} + online_store_config: + online_store_name: $ONLINE_STORE_NAME + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: saw_$UNIQUE_NAME + streaming_mode: + mode: STREAMING_MODE_TYPE_RTM diff --git a/acceptance/bundle/resources/materialization_sets/sawtooth/drop-online-batch.yml.tmpl b/acceptance/bundle/resources/materialization_sets/sawtooth/drop-online-batch.yml.tmpl new file mode 100644 index 00000000000..3969ab6e672 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/sawtooth/drop-online-batch.yml.tmpl @@ -0,0 +1,42 @@ +bundle: + name: test-matset-sawtooth-$UNIQUE_NAME + +resources: + features: + sawfeat: + full_name: $CATALOG_NAME.$SCHEMA_NAME.saw_$UNIQUE_NAME + source: + stream_source: + full_name: $STREAM_FULL_NAME + function: + aggregation_function: + sum: + input: value.passenger_count_1 + time_window: + sawtooth: + window_duration: 259200s + entities: + - name: value.pickup_zip + timeseries_column: + name: value.event_timestamp + + materialization_sets: + set1: + materialized_features: + # Offline batch, on the schedule the service derives for a sawtooth window. + - feature_name: ${resources.features.sawfeat.full_name} + offline_store_config: + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: saw_${UNIQUE_NAME}_off + cron_schedule_trigger: + mode: DERIVED + # Online streaming. RTM is the only streaming mode a non-Kinesis stream source accepts. + - feature_name: ${resources.features.sawfeat.full_name} + online_store_config: + online_store_name: $ONLINE_STORE_NAME + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: saw_$UNIQUE_NAME + streaming_mode: + mode: STREAMING_MODE_TYPE_RTM diff --git a/acceptance/bundle/resources/materialization_sets/sawtooth/out.test.toml b/acceptance/bundle/resources/materialization_sets/sawtooth/out.test.toml new file mode 100644 index 00000000000..19b5b15d11f --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/sawtooth/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/resources/materialization_sets/sawtooth/output.txt b/acceptance/bundle/resources/materialization_sets/sawtooth/output.txt new file mode 100644 index 00000000000..1b4abc7f4d0 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/sawtooth/output.txt @@ -0,0 +1,117 @@ + +=== A sawtooth feature declaring only the offline and online batch pair is refused at create. +=== The pair is not expanded into the third materialization: a set stamps its id on every +=== member before the backend reaches that normalization, and the normalization is skipped +=== whenever the id is set, so one member stays one materialized feature. +>>> musterr [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-sawtooth-[UNIQUE_NAME]/default/files... +Created features.sawfeat +Error: cannot create resources.materialization_sets.set1: materialization set [UUID] operation failed: Feature 'danielb.fs2680.saw_[UNIQUE_NAME]' uses a sawtooth window, which requires an offline batch materialization, an online batch materialization and an online streaming materialization. Declare one materialize feature request for each. (200 BAD_REQUEST) + +Endpoint: n/a +HTTP Status: 200 +API error_code: BAD_REQUEST +API message: materialization set [UUID] operation failed: Feature 'danielb.fs2680.saw_[UNIQUE_NAME]' uses a sawtooth window, which requires an offline batch materialization, an online batch materialization and an online streaming materialization. Declare one materialize feature request for each. + +Files: 7 uploaded, 0 deleted + +=== Declaring all three creates the set +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-sawtooth-[UNIQUE_NAME]/default/files... +Created materialization_sets.set1 +Files: 1 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 1 unchanged + +=== A redeploy plans nothing. Both online members carry a destination the read never returns, +=== so this is also what proves the input-only suppression holds for a three-member set. +>>> [CLI] bundle plan -o json + +>>> jq .plan | with_entries(select(.key | startswith("resources.materialization_sets"))) | map_values(.action) out.plan.json +{ + "resources.materialization_sets.set1": "skip" +} + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-sawtooth-[UNIQUE_NAME]/default/files... +Files: 1 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 2 unchanged + +=== Every suppressed destination change, by reason +>>> jq -r .plan["resources.materialization_sets.set1"].changes | to_entries | map(select(.value.reason == "input_only_not_returned")) | length out.plan.json +7 + +=== A sawtooth feature's three members are one cascade group, so every strict subset of them +=== is refused. Removing the offline member reaches both online ones, +>>> [CLI] bundle plan -o json + +>>> set_action +update + +>>> musterr [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-sawtooth-[UNIQUE_NAME]/default/files... +Error: cannot update resources.materialization_sets.set1: updating id=[UUID]: Deleting materialized features '[UUID]' of feature 'danielb.fs2680.saw_[UNIQUE_NAME]' also deletes materialized features '[UUID]', '[UUID]', which the request does not delete. Delete the unrequested materialized features as well. (400 INVALID_PARAMETER_VALUE) + +Endpoint: PATCH [DATABRICKS_URL]/api/2.0/feature-engineering/materialization-sets/[UUID]?request_id=[UUID]&update_mask=%2A +HTTP Status: 400 Bad Request +API error_code: INVALID_PARAMETER_VALUE +API message: Deleting materialized features '[UUID]' of feature 'danielb.fs2680.saw_[UNIQUE_NAME]' also deletes materialized features '[UUID]', '[UUID]', which the request does not delete. Delete the unrequested materialized features as well. + +Files: 2 uploaded, 0 deleted + +=== removing the online batch member names no member a delete can be applied to, +>>> [CLI] bundle plan -o json + +>>> set_action +update + +>>> musterr [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-sawtooth-[UNIQUE_NAME]/default/files... +Error: cannot update resources.materialization_sets.set1: updating id=[UUID]: Materialized features '[UUID]' of feature 'danielb.fs2680.saw_[UNIQUE_NAME]' are deleted only by deleting materialized features '[UUID]', '[UUID]', which the request does not supply. Remove those as well. (400 INVALID_PARAMETER_VALUE) + +Endpoint: PATCH [DATABRICKS_URL]/api/2.0/feature-engineering/materialization-sets/[UUID]?request_id=[UUID]&update_mask=%2A +HTTP Status: 400 Bad Request +API error_code: INVALID_PARAMETER_VALUE +API message: Materialized features '[UUID]' of feature 'danielb.fs2680.saw_[UNIQUE_NAME]' are deleted only by deleting materialized features '[UUID]', '[UUID]', which the request does not supply. Remove those as well. + +Files: 2 uploaded, 0 deleted + +=== and removing the batch pair still leaves the streaming member behind +>>> [CLI] bundle plan -o json + +>>> set_action +update + +>>> musterr [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-sawtooth-[UNIQUE_NAME]/default/files... +Error: cannot update resources.materialization_sets.set1: updating id=[UUID]: Deleting materialized features '[UUID]', '[UUID]' of feature 'danielb.fs2680.saw_[UNIQUE_NAME]' also deletes materialized features '[UUID]', which the request does not delete. Delete the unrequested materialized features as well. (400 INVALID_PARAMETER_VALUE) + +Endpoint: PATCH [DATABRICKS_URL]/api/2.0/feature-engineering/materialization-sets/[UUID]?request_id=[UUID]&update_mask=%2A +HTTP Status: 400 Bad Request +API error_code: INVALID_PARAMETER_VALUE +API message: Deleting materialized features '[UUID]', '[UUID]' of feature 'danielb.fs2680.saw_[UNIQUE_NAME]' also deletes materialized features '[UUID]', which the request does not delete. Delete the unrequested materialized features as well. + +Files: 2 uploaded, 0 deleted + +=== Removing all three satisfies the cascade but empties the set, which the update endpoint +=== refuses in favour of the delete endpoint. None of the four refusals wrote anything, so +=== restoring the config plans nothing. +>>> [CLI] bundle plan -o json + +>>> set_action +update + +>>> musterr [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-sawtooth-[UNIQUE_NAME]/default/files... +Error: cannot update resources.materialization_sets.set1: updating id=[UUID]: Updating materialization set '[UUID]' with no materialized features is not supported. Delete the set instead. (400 INVALID_PARAMETER_VALUE) + +Endpoint: PATCH [DATABRICKS_URL]/api/2.0/feature-engineering/materialization-sets/[UUID]?request_id=[UUID]&update_mask=%2A +HTTP Status: 400 Bad Request +API error_code: INVALID_PARAMETER_VALUE +API message: Updating materialization set '[UUID]' with no materialized features is not supported. Delete the set instead. + +Files: 2 uploaded, 0 deleted + +>>> [CLI] bundle plan -o json + +>>> set_action +skip diff --git a/acceptance/bundle/resources/materialization_sets/sawtooth/script b/acceptance/bundle/resources/materialization_sets/sawtooth/script new file mode 100644 index 00000000000..0184d2f2f84 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/sawtooth/script @@ -0,0 +1,82 @@ +# A sawtooth feature needs a Stream source, and a Stream is not a bundle resource, so this case +# references one that already exists rather than creating it. Everything else is per-run. +export CATALOG_NAME="${MATSET_CATALOG:-danielb}" +export SCHEMA_NAME="${MATSET_SCHEMA:-fs2680}" +export ONLINE_STORE_NAME="${MATSET_ONLINE_STORE:-danielb}" +export STREAM_FULL_NAME="${MATSET_STREAM:-danielb.fs2680.fs2680_dch48o_stream}" + +# The harness resolves auth from the ambient environment, which on a cloud run is the workspace +# under test. Pointing a local run at a named profile needs it in the bundle, because +# DATABRICKS_CONFIG_PROFILE does not survive into the CLI subprocess. +pin_profile() { + [ -n "${MATSET_PROFILE:-}" ] || return 0 + printf "\nworkspace:\n profile: %s\n" "$MATSET_PROFILE" >> databricks.yml +} + +render() { + envsubst < "$1" > databricks.yml + pin_profile +} + +set_action() { + jq -r '.plan["resources.materialization_sets.set1"].action' out.plan.json +} + +cleanup() { + $CLI bundle destroy --auto-approve > /dev/null 2>&1 || true +} +trap cleanup EXIT + +title "A sawtooth feature declaring only the offline and online batch pair is refused at create." +title "The pair is not expanded into the third materialization: a set stamps its id on every" +title "member before the backend reaches that normalization, and the normalization is skipped" +title "whenever the id is set, so one member stays one materialized feature." +envsubst < twoleg.yml.tmpl > databricks.yml +pin_profile +trace musterr $CLI bundle deploy + +title "Declaring all three creates the set" +envsubst < databricks.yml.tmpl > databricks.yml +pin_profile +trace $CLI bundle deploy + +title "A redeploy plans nothing. Both online members carry a destination the read never returns," +title "so this is also what proves the input-only suppression holds for a three-member set." +trace $CLI bundle plan -o json > out.plan.json +trace jq '.plan | with_entries(select(.key | startswith("resources.materialization_sets"))) | map_values(.action)' out.plan.json +trace $CLI bundle deploy + +title "Every suppressed destination change, by reason" +trace jq -r '.plan["resources.materialization_sets.set1"].changes | to_entries | map(select(.value.reason == "input_only_not_returned")) | length' out.plan.json + +title "A sawtooth feature's three members are one cascade group, so every strict subset of them" +title "is refused. Removing the offline member reaches both online ones," +render drop-offline.yml.tmpl +trace $CLI bundle plan -o json > out.plan.json +trace set_action +trace musterr $CLI bundle deploy --auto-approve + +title "removing the online batch member names no member a delete can be applied to," +render drop-online-batch.yml.tmpl +trace $CLI bundle plan -o json > out.plan.json +trace set_action +trace musterr $CLI bundle deploy --auto-approve + +title "and removing the batch pair still leaves the streaming member behind" +render drop-batch-pair.yml.tmpl +trace $CLI bundle plan -o json > out.plan.json +trace set_action +trace musterr $CLI bundle deploy --auto-approve + +title "Removing all three satisfies the cascade but empties the set, which the update endpoint" +title "refuses in favour of the delete endpoint. None of the four refusals wrote anything, so" +title "restoring the config plans nothing." +render drop-all.yml.tmpl +trace $CLI bundle plan -o json > out.plan.json +trace set_action +trace musterr $CLI bundle deploy --auto-approve +render databricks.yml.tmpl +trace $CLI bundle plan -o json > out.plan.json +trace set_action + +rm -f out.requests.txt out.plan.json diff --git a/acceptance/bundle/resources/materialization_sets/sawtooth/test.toml b/acceptance/bundle/resources/materialization_sets/sawtooth/test.toml new file mode 100644 index 00000000000..f68e59a5af5 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/sawtooth/test.toml @@ -0,0 +1,22 @@ +# Local only. These cases need a Stream source, and a Stream is neither a bundle resource nor a CLI +# command, so the integration suite has no way to provision one; it also needs a Unity Catalog +# connection to a Kafka topic whose payload the feature definitions name. Run them against a +# workspace by hand, overriding MATSET_CATALOG / MATSET_SCHEMA / MATSET_ONLINE_STORE / +# MATSET_STREAM -- see README.md. +Cloud = false + +Ignore = [ + ".databricks", + "databricks.yml", +] + +# A real materialization set or member id is 32 undashed hex characters; the harness's built-in +# replacement only matches the dashed form the fake produces. +[[Repls]] +Old = '\b[0-9a-f]{32}\b' +New = '[UUID]' + +# The workspace host, which only a cloud run puts in the output. +[[Repls]] +Old = 'https://[a-z0-9.-]+\.cloud\.databricks\.com' +New = '[DATABRICKS_URL]' diff --git a/acceptance/bundle/resources/materialization_sets/sawtooth/twoleg.yml.tmpl b/acceptance/bundle/resources/materialization_sets/sawtooth/twoleg.yml.tmpl new file mode 100644 index 00000000000..54b4be4fb96 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/sawtooth/twoleg.yml.tmpl @@ -0,0 +1,42 @@ +bundle: + name: test-matset-sawtooth-$UNIQUE_NAME + +resources: + features: + sawfeat: + full_name: $CATALOG_NAME.$SCHEMA_NAME.saw_$UNIQUE_NAME + source: + stream_source: + full_name: $STREAM_FULL_NAME + function: + aggregation_function: + sum: + input: value.passenger_count_1 + time_window: + sawtooth: + window_duration: 259200s + entities: + - name: value.pickup_zip + timeseries_column: + name: value.event_timestamp + + materialization_sets: + set1: + materialized_features: + # Offline batch, on the schedule the service derives for a sawtooth window. + - feature_name: ${resources.features.sawfeat.full_name} + offline_store_config: + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: saw_${UNIQUE_NAME}_off + cron_schedule_trigger: + mode: DERIVED + # Online batch, sharing its destination with the streaming member below. + - feature_name: ${resources.features.sawfeat.full_name} + online_store_config: + online_store_name: $ONLINE_STORE_NAME + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: saw_$UNIQUE_NAME + cron_schedule_trigger: + mode: DERIVED diff --git a/acceptance/bundle/resources/materialization_sets/schedule_state/databricks.yml b/acceptance/bundle/resources/materialization_sets/schedule_state/databricks.yml new file mode 100644 index 00000000000..d4064e0b26a --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/schedule_state/databricks.yml @@ -0,0 +1,19 @@ +bundle: + name: test-bundle + +resources: + materialization_sets: + set1: + materialized_features: + - feature_name: main.myschema.featA + offline_store_config: + catalog_name: main + schema_name: myschema + table_name_prefix: offline + pipeline_schedule_state: ACTIVE + - feature_name: main.myschema.featA + online_store_config: + online_store_name: danielb + catalog_name: main + schema_name: myschema + table_name_prefix: feata diff --git a/acceptance/bundle/resources/materialization_sets/schedule_state/out.test.toml b/acceptance/bundle/resources/materialization_sets/schedule_state/out.test.toml new file mode 100644 index 00000000000..19b5b15d11f --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/schedule_state/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/resources/materialization_sets/schedule_state/output.txt b/acceptance/bundle/resources/materialization_sets/schedule_state/output.txt new file mode 100644 index 00000000000..cade39560de --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/schedule_state/output.txt @@ -0,0 +1,86 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Created materialization_sets.set1 +Files: 1 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +=== A member the config leaves unset adopts the state the server set on create, so the next +=== plan is empty rather than reporting drift the bundle cannot write +>>> [CLI] bundle plan -o json + +>>> jq .plan | with_entries(select(.key | startswith("resources.materialization_sets"))) | map_values(.action) out.plan.json +{ + "resources.materialization_sets.set1": "skip" +} + +>>> [CLI] bundle summary -o json +[ + { + "is_online": null, + "pipeline_schedule_state": "ACTIVE" + }, + { + "is_online": null, + "pipeline_schedule_state": null + } +] + +=== The state is write-once, so pausing the feature is a recreate rather than the in-place +=== update the server would refuse. Every member of the feature has to ask for the same state +>>> update_file.py databricks.yml pipeline_schedule_state: ACTIVE pipeline_schedule_state: PAUSED + +>>> update_file.py databricks.yml table_name_prefix: feata table_name_prefix: feata + pipeline_schedule_state: PAUSED + +>>> [CLI] bundle plan -o json + +>>> jq .plan | with_entries(select(.key | startswith("resources.materialization_sets"))) | map_values(.action) out.plan.json +{ + "resources.materialization_sets.set1": "recreate" +} + +>>> [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... + +This action will result in the deletion or recreation of the following materialization sets. +Every materialized feature in a recreated set is dropped and recomputed from scratch, including +the members whose configuration did not change: + recreate resources.materialization_sets.set1 +Recreated materialization_sets.set1 +Files: 2 uploaded, 0 deleted +Resources: 1 created, 0 changed, 1 deleted, 0 unchanged + +>>> [CLI] bundle summary -o json +[ + { + "is_online": null, + "pipeline_schedule_state": "PAUSED" + }, + { + "is_online": null, + "pipeline_schedule_state": "PAUSED" + } +] + +=== Resuming one member alone plans as a recreate too, and the create endpoint refuses the +=== members of one feature disagreeing +>>> update_file.py databricks.yml pipeline_schedule_state: PAUSED + - feature_name pipeline_schedule_state: ACTIVE + - feature_name + +>>> musterr [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... + +This action will result in the deletion or recreation of the following materialization sets. +Every materialized feature in a recreated set is dropped and recomputed from scratch, including +the members whose configuration did not change: + recreate resources.materialization_sets.set1 +Error: cannot recreate resources.materialization_sets.set1: All materialized features in a BatchCreateMaterializedFeaturesRequest call must have the same pipeline schedule state. (400 INVALID_PARAMETER_VALUE) + +Endpoint: POST [DATABRICKS_URL]/api/2.0/feature-engineering/materialization-sets?request_id=[UUID] +HTTP Status: 400 Bad Request +API error_code: INVALID_PARAMETER_VALUE +API message: All materialized features in a BatchCreateMaterializedFeaturesRequest call must have the same pipeline schedule state. + +Files: 1 uploaded, 0 deleted diff --git a/acceptance/bundle/resources/materialization_sets/schedule_state/script b/acceptance/bundle/resources/materialization_sets/schedule_state/script new file mode 100644 index 00000000000..6c65ff96bbf --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/schedule_state/script @@ -0,0 +1,26 @@ +trace $CLI bundle deploy + +title "A member the config leaves unset adopts the state the server set on create, so the next" +title "plan is empty rather than reporting drift the bundle cannot write" +trace $CLI bundle plan -o json > out.plan.json +trace jq '.plan | with_entries(select(.key | startswith("resources.materialization_sets"))) | map_values(.action)' out.plan.json +trace $CLI bundle summary -o json | jq '.resources.materialization_sets.set1.materialized_features | map({is_online, pipeline_schedule_state})' + +title "The state is write-once, so pausing the feature is a recreate rather than the in-place" +title "update the server would refuse. Every member of the feature has to ask for the same state" +trace update_file.py databricks.yml 'pipeline_schedule_state: ACTIVE' 'pipeline_schedule_state: PAUSED' +trace update_file.py databricks.yml ' table_name_prefix: feata' ' table_name_prefix: feata + pipeline_schedule_state: PAUSED' +trace $CLI bundle plan -o json > out.plan.json +trace jq '.plan | with_entries(select(.key | startswith("resources.materialization_sets"))) | map_values(.action)' out.plan.json +trace $CLI bundle deploy --auto-approve +trace $CLI bundle summary -o json | jq '.resources.materialization_sets.set1.materialized_features | map({is_online, pipeline_schedule_state})' + +title "Resuming one member alone plans as a recreate too, and the create endpoint refuses the" +title "members of one feature disagreeing" +trace update_file.py databricks.yml ' pipeline_schedule_state: PAUSED + - feature_name' ' pipeline_schedule_state: ACTIVE + - feature_name' +trace musterr $CLI bundle deploy --auto-approve + +rm -f out.requests.txt out.plan.json diff --git a/acceptance/bundle/resources/materialization_sets/settled_failure/databricks.yml.tmpl b/acceptance/bundle/resources/materialization_sets/settled_failure/databricks.yml.tmpl new file mode 100644 index 00000000000..217618ecf63 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/settled_failure/databricks.yml.tmpl @@ -0,0 +1,79 @@ +bundle: + name: test-matset-settled-$UNIQUE_NAME + +resources: + features: + okfeat: + full_name: $CATALOG_NAME.$SCHEMA_NAME.ok_$UNIQUE_NAME + source: + delta_table_source: + full_name: $TABLE_NAME + function: + aggregation_function: + count_function: + input: value + time_window: + sliding: + window_duration: 604800s + slide_duration: 86400s + entities: + - name: id + timeseries_column: + name: timestamp + # The fake refuses to tear down any member of a feature whose name carries this marker, which + # is how a fixture asks for a failure that real infrastructure would produce. + failfeat: + full_name: $CATALOG_NAME.$SCHEMA_NAME.teardown_fails_$UNIQUE_NAME + source: + delta_table_source: + full_name: $TABLE_NAME + function: + aggregation_function: + count_function: + input: id + time_window: + sliding: + window_duration: 604800s + slide_duration: 86400s + entities: + - name: id + timeseries_column: + name: timestamp + + materialization_sets: + set1: + materialized_features: + - feature_name: ${resources.features.okfeat.full_name} + offline_store_config: + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: sf_${UNIQUE_NAME}_off + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" + - feature_name: ${resources.features.okfeat.full_name} + online_store_config: + online_store_name: $ONLINE_STORE_NAME + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: sf_$UNIQUE_NAME + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" + - feature_name: ${resources.features.failfeat.full_name} + offline_store_config: + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: sf_${UNIQUE_NAME}_off + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" + - feature_name: ${resources.features.failfeat.full_name} + online_store_config: + online_store_name: $ONLINE_STORE_NAME + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: sf_$UNIQUE_NAME + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" diff --git a/acceptance/bundle/resources/materialization_sets/settled_failure/drop-failing.yml.tmpl b/acceptance/bundle/resources/materialization_sets/settled_failure/drop-failing.yml.tmpl new file mode 100644 index 00000000000..ee6c39a3883 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/settled_failure/drop-failing.yml.tmpl @@ -0,0 +1,62 @@ +bundle: + name: test-matset-settled-$UNIQUE_NAME + +resources: + features: + okfeat: + full_name: $CATALOG_NAME.$SCHEMA_NAME.ok_$UNIQUE_NAME + source: + delta_table_source: + full_name: $TABLE_NAME + function: + aggregation_function: + count_function: + input: value + time_window: + sliding: + window_duration: 604800s + slide_duration: 86400s + entities: + - name: id + timeseries_column: + name: timestamp + # The fake refuses to tear down any member of a feature whose name carries this marker, which + # is how a fixture asks for a failure that real infrastructure would produce. + failfeat: + full_name: $CATALOG_NAME.$SCHEMA_NAME.teardown_fails_$UNIQUE_NAME + source: + delta_table_source: + full_name: $TABLE_NAME + function: + aggregation_function: + count_function: + input: id + time_window: + sliding: + window_duration: 604800s + slide_duration: 86400s + entities: + - name: id + timeseries_column: + name: timestamp + + materialization_sets: + set1: + materialized_features: + - feature_name: ${resources.features.okfeat.full_name} + offline_store_config: + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: sf_${UNIQUE_NAME}_off + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" + - feature_name: ${resources.features.okfeat.full_name} + online_store_config: + online_store_name: $ONLINE_STORE_NAME + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: sf_$UNIQUE_NAME + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" diff --git a/acceptance/bundle/resources/materialization_sets/settled_failure/out.test.toml b/acceptance/bundle/resources/materialization_sets/settled_failure/out.test.toml new file mode 100644 index 00000000000..19b5b15d11f --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/settled_failure/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/resources/materialization_sets/settled_failure/output.txt b/acceptance/bundle/resources/materialization_sets/settled_failure/output.txt new file mode 100644 index 00000000000..38912722d51 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/settled_failure/output.txt @@ -0,0 +1,48 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-settled-[UNIQUE_NAME]/default/files... +Created features.failfeat +Created features.okfeat +Created materialization_sets.set1 +Files: 3 uploaded, 0 deleted +Resources: 3 created, 0 changed, 0 deleted, 0 unchanged + +=== A member the server cannot tear down settles the update's operation as FAILED and returns +=== it with HTTP 200. The adapter reads the code off the operation and the per-member outcome +=== off its metadata, so the deploy fails naming each member left behind. +>>> [CLI] bundle plan -o json + +>>> set_action +update + +>>> musterr [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-settled-[UNIQUE_NAME]/default/files... +Error: cannot update resources.materialization_sets.set1: updating id=[UUID]: materialization set [UUID] operation failed: Changed 0 of 2 materialized features in materialization set '[UUID]'. The operation's progress lists which update succeeded, which failed and why.; members left unchanged: [UUID] (INTERNAL_ERROR): Failed to delete materialized feature '[UUID]'. If the issue persists, please retry or contact support.; [UUID] (INTERNAL_ERROR): Failed to delete materialized feature '[UUID]'. If the issue persists, please retry or contact support. (200 INTERNAL_ERROR) + +Endpoint: n/a +HTTP Status: 200 +API error_code: INTERNAL_ERROR +API message: materialization set [UUID] operation failed: Changed 0 of 2 materialized features in materialization set '[UUID]'. The operation's progress lists which update succeeded, which failed and why.; members left unchanged: [UUID] (INTERNAL_ERROR): Failed to delete materialized feature '[UUID]'. If the issue persists, please retry or contact support.; [UUID] (INTERNAL_ERROR): Failed to delete materialized feature '[UUID]'. If the issue persists, please retry or contact support. + +Files: 2 uploaded, 0 deleted + +=== Nothing was written, so the removal is still pending +>>> [CLI] bundle plan -o json + +>>> set_action +update + +=== The delete settles FAILED the same way, and here the engine decides what the user sees: +=== its destroy step forgives a delete error matching none of the cases it knows, so the +=== destroy reports success and drops the state while the set stays in the workspace. The leg +=== with deployment-history recording takes the one branch that records and returns the +=== failure, which is why this case runs on the other leg only. +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.features.failfeat + delete resources.features.okfeat + delete resources.materialization_sets.set1 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-matset-settled-[UNIQUE_NAME]/default + +Destroy: 3 deleted diff --git a/acceptance/bundle/resources/materialization_sets/settled_failure/script b/acceptance/bundle/resources/materialization_sets/settled_failure/script new file mode 100644 index 00000000000..b0882e8c2ed --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/settled_failure/script @@ -0,0 +1,36 @@ +export CATALOG_NAME="${MATSET_CATALOG:-main}" +export SCHEMA_NAME="${MATSET_SCHEMA:-myschema}" +export ONLINE_STORE_NAME="${MATSET_ONLINE_STORE:-danielb}" +export TABLE_NAME="$CATALOG_NAME.$SCHEMA_NAME.src_$UNIQUE_NAME" + +render() { + envsubst < "$1" > databricks.yml +} + +set_action() { + jq -r '.plan["resources.materialization_sets.set1"].action' out.plan.json +} + +render databricks.yml.tmpl +trace $CLI bundle deploy + +title "A member the server cannot tear down settles the update's operation as FAILED and returns" +title "it with HTTP 200. The adapter reads the code off the operation and the per-member outcome" +title "off its metadata, so the deploy fails naming each member left behind." +render drop-failing.yml.tmpl +trace $CLI bundle plan -o json > out.plan.json +trace set_action +trace musterr $CLI bundle deploy --auto-approve + +title "Nothing was written, so the removal is still pending" +trace $CLI bundle plan -o json > out.plan.json +trace set_action + +title "The delete settles FAILED the same way, and here the engine decides what the user sees:" +title "its destroy step forgives a delete error matching none of the cases it knows, so the" +title "destroy reports success and drops the state while the set stays in the workspace. The leg" +title "with deployment-history recording takes the one branch that records and returns the" +title "failure, which is why this case runs on the other leg only." +trace $CLI bundle destroy --auto-approve + +rm -f out.requests.txt out.plan.json diff --git a/acceptance/bundle/resources/materialization_sets/settled_failure/test.toml b/acceptance/bundle/resources/materialization_sets/settled_failure/test.toml new file mode 100644 index 00000000000..61fdc6235a0 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/settled_failure/test.toml @@ -0,0 +1,12 @@ +# Pinned to the leg without deployment-history recording, where the engine's destroy step forgives +# a delete error it cannot classify. With recording on it takes the branch that records the failure +# and returns it, so the destroy fails instead and the two legs cannot share one golden. +EnvMatrix.DMS = [""] + +# Local only, and not because the claims are redundant: the failure is produced by a marker the +# fake looks for in a feature name, since a real teardown fails on infrastructure no request can +# ask for. Against a workspace this case would simply succeed. +Ignore = [ + ".databricks", + "databricks.yml", +] diff --git a/acceptance/bundle/resources/materialization_sets/streaming/databricks.yml.tmpl b/acceptance/bundle/resources/materialization_sets/streaming/databricks.yml.tmpl new file mode 100644 index 00000000000..5e09efb43e1 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/streaming/databricks.yml.tmpl @@ -0,0 +1,64 @@ +bundle: + name: test-matset-streaming-$UNIQUE_NAME + +resources: + features: + # A second feature materialized to the same destination, so removing one feature's member + # leaves a set behind. A set must be emptied through the delete endpoint, not an update. + streamfeat2: + full_name: $CATALOG_NAME.$SCHEMA_NAME.stream2_$UNIQUE_NAME + source: + stream_source: + full_name: $STREAM_FULL_NAME + function: + aggregation_function: + count_function: + input: value.passenger_count_1 + time_window: + rolling: + window_duration: 3600s + entities: + - name: value.pickup_zip + timeseries_column: + name: value.event_timestamp + + streamfeat: + full_name: $CATALOG_NAME.$SCHEMA_NAME.stream_$UNIQUE_NAME + source: + stream_source: + full_name: $STREAM_FULL_NAME + function: + aggregation_function: + count_function: + input: value.passenger_count_1 + # A Stream source accepts a rolling or a sawtooth window. Rolling keeps this feature on + # the plain streaming path, where one member is one materialized feature. + time_window: + rolling: + window_duration: 3600s + entities: + - name: value.pickup_zip + timeseries_column: + name: value.event_timestamp + + materialization_sets: + set1: + materialized_features: + # Online streaming only: no cron schedule, and no pipeline schedule state to agree on. + - feature_name: ${resources.features.streamfeat.full_name} + online_store_config: + online_store_name: $ONLINE_STORE_NAME + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: stream_$UNIQUE_NAME + streaming_mode: + mode: STREAMING_MODE_TYPE_RTM + # Online streaming only: no cron schedule, and no pipeline schedule state to agree on. + - feature_name: ${resources.features.streamfeat2.full_name} + online_store_config: + online_store_name: $ONLINE_STORE_NAME + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: stream_$UNIQUE_NAME + streaming_mode: + mode: STREAMING_MODE_TYPE_RTM diff --git a/acceptance/bundle/resources/materialization_sets/streaming/drop-one.yml.tmpl b/acceptance/bundle/resources/materialization_sets/streaming/drop-one.yml.tmpl new file mode 100644 index 00000000000..a425fee6857 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/streaming/drop-one.yml.tmpl @@ -0,0 +1,55 @@ +bundle: + name: test-matset-streaming-$UNIQUE_NAME + +resources: + features: + # A second feature materialized to the same destination, so removing one feature's member + # leaves a set behind. A set must be emptied through the delete endpoint, not an update. + streamfeat2: + full_name: $CATALOG_NAME.$SCHEMA_NAME.stream2_$UNIQUE_NAME + source: + stream_source: + full_name: $STREAM_FULL_NAME + function: + aggregation_function: + count_function: + input: value.passenger_count_1 + time_window: + rolling: + window_duration: 3600s + entities: + - name: value.pickup_zip + timeseries_column: + name: value.event_timestamp + + streamfeat: + full_name: $CATALOG_NAME.$SCHEMA_NAME.stream_$UNIQUE_NAME + source: + stream_source: + full_name: $STREAM_FULL_NAME + function: + aggregation_function: + count_function: + input: value.passenger_count_1 + # A Stream source accepts a rolling or a sawtooth window. Rolling keeps this feature on + # the plain streaming path, where one member is one materialized feature. + time_window: + rolling: + window_duration: 3600s + entities: + - name: value.pickup_zip + timeseries_column: + name: value.event_timestamp + + materialization_sets: + set1: + materialized_features: + # Online streaming only: no cron schedule, and no pipeline schedule state to agree on. + - feature_name: ${resources.features.streamfeat.full_name} + online_store_config: + online_store_name: $ONLINE_STORE_NAME + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: stream_$UNIQUE_NAME + streaming_mode: + mode: STREAMING_MODE_TYPE_RTM diff --git a/acceptance/bundle/resources/materialization_sets/streaming/moved.yml.tmpl b/acceptance/bundle/resources/materialization_sets/streaming/moved.yml.tmpl new file mode 100644 index 00000000000..005b9a122ed --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/streaming/moved.yml.tmpl @@ -0,0 +1,64 @@ +bundle: + name: test-matset-streaming-$UNIQUE_NAME + +resources: + features: + # A second feature materialized to the same destination, so removing one feature's member + # leaves a set behind. A set must be emptied through the delete endpoint, not an update. + streamfeat2: + full_name: $CATALOG_NAME.$SCHEMA_NAME.stream2_$UNIQUE_NAME + source: + stream_source: + full_name: $STREAM_FULL_NAME + function: + aggregation_function: + count_function: + input: value.passenger_count_1 + time_window: + rolling: + window_duration: 3600s + entities: + - name: value.pickup_zip + timeseries_column: + name: value.event_timestamp + + streamfeat: + full_name: $CATALOG_NAME.$SCHEMA_NAME.stream_$UNIQUE_NAME + source: + stream_source: + full_name: $STREAM_FULL_NAME + function: + aggregation_function: + count_function: + input: value.passenger_count_1 + # A Stream source accepts a rolling or a sawtooth window. Rolling keeps this feature on + # the plain streaming path, where one member is one materialized feature. + time_window: + rolling: + window_duration: 3600s + entities: + - name: value.pickup_zip + timeseries_column: + name: value.event_timestamp + + materialization_sets: + set1: + materialized_features: + # Online streaming only: no cron schedule, and no pipeline schedule state to agree on. + - feature_name: ${resources.features.streamfeat.full_name} + online_store_config: + online_store_name: $ONLINE_STORE_NAME + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: stream_${UNIQUE_NAME}_moved + streaming_mode: + mode: STREAMING_MODE_TYPE_RTM + # Online streaming only: no cron schedule, and no pipeline schedule state to agree on. + - feature_name: ${resources.features.streamfeat2.full_name} + online_store_config: + online_store_name: $ONLINE_STORE_NAME + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: stream_$UNIQUE_NAME + streaming_mode: + mode: STREAMING_MODE_TYPE_RTM diff --git a/acceptance/bundle/resources/materialization_sets/streaming/out.test.toml b/acceptance/bundle/resources/materialization_sets/streaming/out.test.toml new file mode 100644 index 00000000000..19b5b15d11f --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/streaming/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/resources/materialization_sets/streaming/output.txt b/acceptance/bundle/resources/materialization_sets/streaming/output.txt new file mode 100644 index 00000000000..b9ee84b6ae9 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/streaming/output.txt @@ -0,0 +1,61 @@ + +=== A streaming member carries a trigger and no schedule, so none of the batch rules apply +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-streaming-[UNIQUE_NAME]/default/files... +Created features.streamfeat +Created features.streamfeat2 +Created materialization_sets.set1 +Files: 4 uploaded, 0 deleted +Resources: 3 created, 0 changed, 0 deleted, 0 unchanged + +=== A redeploy plans nothing. The schedule state the server assigns to a lone streaming member +=== must not read as drift, and neither must its online destination. +>>> [CLI] bundle plan -o json + +>>> jq .plan | with_entries(select(.key | startswith("resources.materialization_sets"))) | map_values(.action) out.plan.json +{ + "resources.materialization_sets.set1": "skip" +} + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-streaming-[UNIQUE_NAME]/default/files... +Files: 1 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 3 unchanged + +=== The schedule state is write-once, so declaring one on a streaming member that had none +=== plans as a recreate rather than an in-place update +>>> update_file.py databricks.yml streaming_mode: pipeline_schedule_state: PAUSED + streaming_mode: + +>>> [CLI] bundle plan -o json + +>>> jq .plan | with_entries(select(.key | startswith("resources.materialization_sets"))) | map_values(.action) out.plan.json +{ + "resources.materialization_sets.set1": "recreate" +} + +=== Repointing a member's online destination is a recreate: the server cannot move a +=== materialization in place, so the whole set is rebuilt at the new location +>>> [CLI] bundle plan -o json + +>>> set_action +recreate + +=== Removing one feature's streaming member is an update the endpoint accepts. With no offline +=== member to cascade from, each online member is its own cascade group, so the other feature's +=== member is untouched and the set is left non-empty. +>>> [CLI] bundle plan -o json + +>>> set_action +update + +>>> [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-streaming-[UNIQUE_NAME]/default/files... +Updated materialization_sets.set1 +Files: 2 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 2 unchanged + +>>> [CLI] bundle plan -o json + +>>> set_action +skip diff --git a/acceptance/bundle/resources/materialization_sets/streaming/script b/acceptance/bundle/resources/materialization_sets/streaming/script new file mode 100644 index 00000000000..16a98ab9394 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/streaming/script @@ -0,0 +1,64 @@ +# Like the sawtooth case, this references an existing Stream rather than creating one. +export CATALOG_NAME="${MATSET_CATALOG:-danielb}" +export SCHEMA_NAME="${MATSET_SCHEMA:-fs2680}" +export ONLINE_STORE_NAME="${MATSET_ONLINE_STORE:-danielb}" +export STREAM_FULL_NAME="${MATSET_STREAM:-danielb.fs2680.fs2680_dch48o_stream}" + +# The harness resolves auth from the ambient environment, which on a cloud run is the workspace +# under test. Pointing a local run at a named profile needs it in the bundle, because +# DATABRICKS_CONFIG_PROFILE does not survive into the CLI subprocess. +pin_profile() { + [ -n "${MATSET_PROFILE:-}" ] || return 0 + printf "\nworkspace:\n profile: %s\n" "$MATSET_PROFILE" >> databricks.yml +} + +render() { + envsubst < "$1" > databricks.yml + pin_profile +} + +set_action() { + jq -r '.plan["resources.materialization_sets.set1"].action' out.plan.json +} + +cleanup() { + $CLI bundle destroy --auto-approve > /dev/null 2>&1 || true +} +trap cleanup EXIT + +envsubst < databricks.yml.tmpl > databricks.yml +pin_profile + +title "A streaming member carries a trigger and no schedule, so none of the batch rules apply" +trace $CLI bundle deploy + +title "A redeploy plans nothing. The schedule state the server assigns to a lone streaming member" +title "must not read as drift, and neither must its online destination." +trace $CLI bundle plan -o json > out.plan.json +trace jq '.plan | with_entries(select(.key | startswith("resources.materialization_sets"))) | map_values(.action)' out.plan.json +trace $CLI bundle deploy + +title "The schedule state is write-once, so declaring one on a streaming member that had none" +title "plans as a recreate rather than an in-place update" +trace update_file.py databricks.yml ' streaming_mode:' ' pipeline_schedule_state: PAUSED + streaming_mode:' +trace $CLI bundle plan -o json > out.plan.json +trace jq '.plan | with_entries(select(.key | startswith("resources.materialization_sets"))) | map_values(.action)' out.plan.json + +title "Repointing a member's online destination is a recreate: the server cannot move a" +title "materialization in place, so the whole set is rebuilt at the new location" +render moved.yml.tmpl +trace $CLI bundle plan -o json > out.plan.json +trace set_action + +title "Removing one feature's streaming member is an update the endpoint accepts. With no offline" +title "member to cascade from, each online member is its own cascade group, so the other feature's" +title "member is untouched and the set is left non-empty." +render drop-one.yml.tmpl +trace $CLI bundle plan -o json > out.plan.json +trace set_action +trace $CLI bundle deploy --auto-approve +trace $CLI bundle plan -o json > out.plan.json +trace set_action + +rm -f out.requests.txt out.plan.json diff --git a/acceptance/bundle/resources/materialization_sets/streaming/test.toml b/acceptance/bundle/resources/materialization_sets/streaming/test.toml new file mode 100644 index 00000000000..f68e59a5af5 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/streaming/test.toml @@ -0,0 +1,22 @@ +# Local only. These cases need a Stream source, and a Stream is neither a bundle resource nor a CLI +# command, so the integration suite has no way to provision one; it also needs a Unity Catalog +# connection to a Kafka topic whose payload the feature definitions name. Run them against a +# workspace by hand, overriding MATSET_CATALOG / MATSET_SCHEMA / MATSET_ONLINE_STORE / +# MATSET_STREAM -- see README.md. +Cloud = false + +Ignore = [ + ".databricks", + "databricks.yml", +] + +# A real materialization set or member id is 32 undashed hex characters; the harness's built-in +# replacement only matches the dashed form the fake produces. +[[Repls]] +Old = '\b[0-9a-f]{32}\b' +New = '[UUID]' + +# The workspace host, which only a cloud run puts in the output. +[[Repls]] +Old = 'https://[a-z0-9.-]+\.cloud\.databricks\.com' +New = '[DATABRICKS_URL]' diff --git a/acceptance/bundle/resources/materialization_sets/test.toml b/acceptance/bundle/resources/materialization_sets/test.toml new file mode 100644 index 00000000000..80ac671d526 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/test.toml @@ -0,0 +1,12 @@ +# Local-only for now. The fake in libs/testserver/materialization_set.go mirrors +# UpdateMaterializationSet's reconcile semantics, so these goldens are a claim about the real API; +# turning on Cloud = true checks that claim against a workspace. See README.md for the delta. +Ignore = [ + "databricks.yml", + ".databricks", +] + +# Deployment history has no "materialization_sets." or "features." resource-key prefix, so the +# deploy fails before anything is created. See dmsResourceTypes in +# libs/testserver/bundledeployments.go, which mirrors Universe. +EnvMatrixExclude.no_matset_dms = ["DMS=true"] diff --git a/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go b/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go index 4ead9785e06..3b3eef2fe64 100644 --- a/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go +++ b/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go @@ -24,6 +24,7 @@ var unsupportedResources = []string{ "volumes", "schemas", "features", + "materialization_sets", "quality_monitors", "registered_models", "model_services", diff --git a/bundle/config/mutator/resourcemutator/apply_target_mode_test.go b/bundle/config/mutator/resourcemutator/apply_target_mode_test.go index e9ea418d1d6..fd5f61bc111 100644 --- a/bundle/config/mutator/resourcemutator/apply_target_mode_test.go +++ b/bundle/config/mutator/resourcemutator/apply_target_mode_test.go @@ -159,6 +159,13 @@ func mockBundle(mode config.Mode) *bundle.Bundle { Features: map[string]*resources.Feature{ "feature1": {Feature: ml.Feature{FullName: "catalog1.schema1.feature1"}}, }, + MaterializationSets: map[string]*resources.MaterializationSet{ + "materialization_set1": {MaterializationSet: ml.MaterializationSet{ + MaterializedFeatures: []ml.MaterializedFeature{ + {FeatureName: "catalog1.schema1.feature1"}, + }, + }}, + }, Clusters: map[string]*resources.Cluster{ "cluster1": {ClusterSpec: compute.ClusterSpec{ClusterName: "cluster1", SparkVersion: "13.2.x", NumWorkers: 1}}, }, diff --git a/bundle/config/mutator/resourcemutator/capture_uc_dependencies.go b/bundle/config/mutator/resourcemutator/capture_uc_dependencies.go index a0e8d05fbde..f2893698bcb 100644 --- a/bundle/config/mutator/resourcemutator/capture_uc_dependencies.go +++ b/bundle/config/mutator/resourcemutator/capture_uc_dependencies.go @@ -14,13 +14,15 @@ import ( type captureUCDependencies struct{} // If a user defines a UC schema in the bundle, they can refer to it in SDP pipelines, -// UC Volumes, Registered Models, Quality Monitors, Model Serving Endpoints, or Vector Search -// Indexes using the `${resources.schemas..name}` syntax. Using this syntax allows +// UC Volumes, Registered Models, Quality Monitors, Model Serving Endpoints, Vector Search +// Indexes, Features, or Materialization Sets using the `${resources.schemas..name}` +// syntax. Using this syntax allows // TF to capture the deploy time dependency this resource has on the schema and deploy changes // to the schema before deploying the dependent resource. // // Similarly, if a user defines a UC catalog in the bundle, they can refer to it in UC schemas, -// UC Volumes, Registered Models, Model Serving Endpoints, or Vector Search Indexes using the +// UC Volumes, Registered Models, Model Serving Endpoints, Vector Search Indexes, Features, or +// Materialization Sets using the // `${resources.catalogs..name}` syntax. This captures the deploy time // dependency the resource has on the catalog. // @@ -204,6 +206,22 @@ func (m *captureUCDependencies) Apply(ctx context.Context, b *bundle.Bundle) dia catalogName, schemaName := parts[0], parts[1] f.FullName = resolveCatalog(b, catalogName) + "." + resolveSchema(b, catalogName, schemaName) + "." + parts[2] } + for _, set := range b.Config.Resources.MaterializationSets { + if set == nil { + continue + } + for i := range set.MaterializedFeatures { + member := &set.MaterializedFeatures[i] + if offline := member.OfflineStoreConfig; offline != nil { + offline.SchemaName = resolveSchema(b, offline.CatalogName, offline.SchemaName) + offline.CatalogName = resolveCatalog(b, offline.CatalogName) + } + if online := member.OnlineStoreConfig; online != nil { + online.SchemaName = resolveSchema(b, online.CatalogName, online.SchemaName) + online.CatalogName = resolveCatalog(b, online.CatalogName) + } + } + } for _, mse := range b.Config.Resources.ModelServingEndpoints { if mse == nil { continue diff --git a/bundle/config/mutator/resourcemutator/capture_uc_dependencies_test.go b/bundle/config/mutator/resourcemutator/capture_uc_dependencies_test.go index 347f0b72caa..15094079ee2 100644 --- a/bundle/config/mutator/resourcemutator/capture_uc_dependencies_test.go +++ b/bundle/config/mutator/resourcemutator/capture_uc_dependencies_test.go @@ -365,6 +365,7 @@ func TestCaptureUCDependenciesNilResources(t *testing.T) { QualityMonitors: map[string]*resources.QualityMonitor{"nil": nil, "empty": {}}, ModelServingEndpoints: map[string]*resources.ModelServingEndpoint{"nil": nil, "empty": {}}, VectorSearchIndexes: map[string]*resources.VectorSearchIndex{"nil": nil, "empty": {}}, + MaterializationSets: map[string]*resources.MaterializationSet{"nil": nil, "empty": {}}, }, }, } @@ -373,6 +374,72 @@ func TestCaptureUCDependenciesNilResources(t *testing.T) { require.Nil(t, d) } +func TestCaptureUCDependenciesMaterializationSet(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + Catalogs: map[string]*resources.Catalog{ + "my_catalog": {CreateCatalog: catalog.CreateCatalog{Name: "mycatalog"}}, + }, + Schemas: map[string]*resources.Schema{ + "my_schema": {CreateSchema: catalog.CreateSchema{CatalogName: "mycatalog", Name: "myschema"}}, + }, + MaterializationSets: map[string]*resources.MaterializationSet{ + "set1": {MaterializationSet: ml.MaterializationSet{ + MaterializedFeatures: []ml.MaterializedFeature{ + { + FeatureName: "mycatalog.myschema.feat", + OfflineStoreConfig: &ml.OfflineStoreConfig{ + CatalogName: "mycatalog", SchemaName: "myschema", TableNamePrefix: "feat_off", + }, + }, + { + FeatureName: "mycatalog.myschema.feat", + OnlineStoreConfig: &ml.OnlineStoreConfig{ + OnlineStoreName: "store", CatalogName: "mycatalog", + SchemaName: "myschema", TableNamePrefix: "feat", + }, + }, + // Neither destination declared in the bundle, so both pass through. + { + FeatureName: "other.other.feat", + OfflineStoreConfig: &ml.OfflineStoreConfig{ + CatalogName: "other", SchemaName: "other", TableNamePrefix: "feat_off", + }, + }, + // A member with no destination at all. + {FeatureName: "mycatalog.myschema.feat"}, + }, + }}, + "nil_set": nil, + }, + }, + }, + } + + d := bundle.Apply(t.Context(), b, CaptureUCDependencies()) + require.Nil(t, d) + + schemaRef := "${resources.schemas.my_schema.name}" + catalogRef := "${resources.catalogs.my_catalog.name}" + members := b.Config.Resources.MaterializationSets["set1"].MaterializedFeatures + + assert.Equal(t, catalogRef, members[0].OfflineStoreConfig.CatalogName) + assert.Equal(t, schemaRef, members[0].OfflineStoreConfig.SchemaName) + assert.Equal(t, "feat_off", members[0].OfflineStoreConfig.TableNamePrefix) + + assert.Equal(t, catalogRef, members[1].OnlineStoreConfig.CatalogName) + assert.Equal(t, schemaRef, members[1].OnlineStoreConfig.SchemaName) + assert.Equal(t, "store", members[1].OnlineStoreConfig.OnlineStoreName) + + assert.Equal(t, "other", members[2].OfflineStoreConfig.CatalogName) + assert.Equal(t, "other", members[2].OfflineStoreConfig.SchemaName) + + assert.Nil(t, members[3].OfflineStoreConfig) + assert.Nil(t, members[3].OnlineStoreConfig) + assert.Nil(t, b.Config.Resources.MaterializationSets["nil_set"]) +} + func TestSplitUCName(t *testing.T) { tests := []struct { name string diff --git a/bundle/config/mutator/resourcemutator/normalize_materialization_set_materialized_feature_members.go b/bundle/config/mutator/resourcemutator/normalize_materialization_set_materialized_feature_members.go new file mode 100644 index 00000000000..14b91bc9797 --- /dev/null +++ b/bundle/config/mutator/resourcemutator/normalize_materialization_set_materialized_feature_members.go @@ -0,0 +1,108 @@ +package resourcemutator + +import ( + "context" + "fmt" + "maps" + "slices" + "strings" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/databricks-sdk-go/service/ml" +) + +type normalizeMaterializationSetMaterializedFeatureMembers struct{} + +func NormalizeMaterializationSetMaterializedFeatureMembers() bundle.Mutator { + return &normalizeMaterializationSetMaterializedFeatureMembers{} +} + +func (m *normalizeMaterializationSetMaterializedFeatureMembers) Name() string { + return "NormalizeMaterializationSetMaterializedFeatureMembers" +} + +// quotedFeatureNames renders a feature name list the way FeatureStoreMessages.quotedIds does. +func quotedFeatureNames(names []string) string { + quoted := make([]string, len(names)) + for i, name := range names { + quoted[i] = "'" + name + "'" + } + return strings.Join(quoted, ", ") +} + +// duplicateMemberKeyDiagnostic rejects a set listing two members with the same member key, which is +// the identity resources.MaterializedFeatureMemberKey builds and the server's own +// MaterializationSetMemberKey shares. +// +// It mirrors MaterializationSetUtils.validateMaterializationSetMemberKeysAreUnique: group the +// members by key, then name every feature holding a key more than one member shares, once and +// sorted. +func duplicateMemberKeyDiagnostic( + b *bundle.Bundle, + setKey string, + members []ml.MaterializedFeature, +) *diag.Diagnostic { + indicesByKey := map[string][]int{} + for i, member := range members { + _, memberKey := resources.MaterializedFeatureMemberKey(member) + indicesByKey[memberKey] = append(indicesByKey[memberKey], i) + } + + featureNames := map[string]bool{} + var indices []int + for _, group := range indicesByKey { + if len(group) < 2 { + continue + } + for _, i := range group { + featureNames[members[i].FeatureName] = true + indices = append(indices, i) + } + } + if len(featureNames) == 0 { + return nil + } + + setPath := structpath.NewPath(nil, "resources", "materialization_sets", setKey) + slices.Sort(indices) + + var paths []*structpath.PathNode + var locations []diag.Location + for _, i := range indices { + path := structpath.NewPath(setPath, "materialized_features", i) + paths = append(paths, path) + locations = append(locations, b.Config.GetLocationsOf(path)...) + } + + return &diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf( + "Features %s in materialization set '%s' each declares more than one materialized "+ + "feature with the following same fields: %s, which is not allowed.", + quotedFeatureNames(slices.Sorted(maps.Keys(featureNames))), setKey, + resources.MaterializedFeatureMemberKeyFieldDescription), + Paths: paths, + Locations: locations, + } +} + +func (m *normalizeMaterializationSetMaterializedFeatureMembers) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { + var diags diag.Diagnostics + + // Map iteration order is randomized; sort by name for stable output. + for _, setKey := range slices.Sorted(maps.Keys(b.Config.Resources.MaterializationSets)) { + set := b.Config.Resources.MaterializationSets[setKey] + if set == nil { + continue + } + + if duplicate := duplicateMemberKeyDiagnostic(b, setKey, set.MaterializedFeatures); duplicate != nil { + diags = diags.Append(*duplicate) + } + } + + return diags +} diff --git a/bundle/config/mutator/resourcemutator/normalize_materialization_set_materialized_feature_members_test.go b/bundle/config/mutator/resourcemutator/normalize_materialization_set_materialized_feature_members_test.go new file mode 100644 index 00000000000..1d481c9bb1e --- /dev/null +++ b/bundle/config/mutator/resourcemutator/normalize_materialization_set_materialized_feature_members_test.go @@ -0,0 +1,181 @@ +package resourcemutator_test + +import ( + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/mutator/resourcemutator" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/databricks-sdk-go/service/ml" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const memberFeatureName = "main.myschema.feat" + +func offlineStore() *ml.OfflineStoreConfig { + return &ml.OfflineStoreConfig{ + CatalogName: "main", + SchemaName: "myschema", + TableNamePrefix: "feat_off", + } +} + +func onlineStore() *ml.OnlineStoreConfig { + return &ml.OnlineStoreConfig{ + OnlineStoreName: "store", + CatalogName: "main", + SchemaName: "myschema", + TableNamePrefix: "feat", + } +} + +func offlineMember(featureName string) ml.MaterializedFeature { + return ml.MaterializedFeature{ + FeatureName: featureName, + OfflineStoreConfig: offlineStore(), + CronScheduleTrigger: &ml.CronSchedule{Mode: ml.CronScheduleModeDerived}, + } +} + +func onlineMember(featureName, storeName string) ml.MaterializedFeature { + store := onlineStore() + store.OnlineStoreName = storeName + + return ml.MaterializedFeature{ + FeatureName: featureName, + OnlineStoreConfig: store, + CronScheduleTrigger: &ml.CronSchedule{Mode: ml.CronScheduleModeDerived}, + } +} + +func normalizeSetDiags( + t *testing.T, + set *resources.MaterializationSet, +) (*resources.MaterializationSet, diag.Diagnostics) { + t.Helper() + + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + MaterializationSets: map[string]*resources.MaterializationSet{"set1": set}, + }, + }, + } + diags := bundle.Apply(t.Context(), b, resourcemutator.NormalizeMaterializationSetMaterializedFeatureMembers()) + + return b.Config.Resources.MaterializationSets["set1"], diags +} + +func normalizeSet(t *testing.T, set *resources.MaterializationSet) *resources.MaterializationSet { + t.Helper() + + normalized, diags := normalizeSetDiags(t, set) + require.NoError(t, diags.Error()) + + return normalized +} + +func pathStrings(paths []*structpath.PathNode) []string { + out := make([]string, len(paths)) + for i, path := range paths { + out[i] = path.String() + } + return out +} + +// duplicateMemberKeyError is the wording the service uses for the same rejection, so the two read +// alike wherever a caller meets it. See FeatureStoreMessages.materializationSetMemberKeysNotUnique. +func duplicateMemberKeyError(featureNames string) string { + return "Features " + featureNames + " in materialization set 'set1' each declares more than " + + "one materialized feature with the following same fields: feature name, online store " + + "name, trigger is streaming mode, which is not allowed." +} + +func TestNormalizeMaterializationSetMaterializedFeatureMembersAcceptsDistinctKeys(t *testing.T) { + set := normalizeSet(t, &resources.MaterializationSet{ + MaterializedFeatures: []ml.MaterializedFeature{ + offlineMember(memberFeatureName), + onlineMember(memberFeatureName, "store"), + onlineMember(memberFeatureName, "other-store"), + offlineMember("main.myschema.other"), + }, + }) + + assert.Len(t, set.MaterializedFeatures, 4) +} + +func TestNormalizeMaterializationSetMaterializedFeatureMembersAcceptsAnEmptySet(t *testing.T) { + set := normalizeSet(t, &resources.MaterializationSet{}) + + assert.Nil(t, set.MaterializedFeatures) +} + +func TestNormalizeMaterializationSetMaterializedFeatureMembersRejectsTheSameMemberTwice(t *testing.T) { + _, diags := normalizeSetDiags(t, &resources.MaterializationSet{ + MaterializedFeatures: []ml.MaterializedFeature{ + offlineMember(memberFeatureName), + onlineMember(memberFeatureName, "store"), + offlineMember(memberFeatureName), + }, + }) + + require.Len(t, diags, 1) + assert.Equal(t, diag.Error, diags[0].Severity) + assert.Equal(t, duplicateMemberKeyError("'"+memberFeatureName+"'"), diags[0].Summary) + assert.Equal(t, + []string{ + "resources.materialization_sets.set1.materialized_features[0]", + "resources.materialization_sets.set1.materialized_features[2]", + }, + pathStrings(diags[0].Paths)) +} + +// The offline destination is outside the member key, because the read never returns it, so two +// members differing only there are the same member declared twice. +func TestNormalizeMaterializationSetMaterializedFeatureMembersRejectsMembersDifferingOutsideTheKey(t *testing.T) { + elsewhere := offlineMember(memberFeatureName) + elsewhere.OfflineStoreConfig = &ml.OfflineStoreConfig{ + CatalogName: "main", SchemaName: "myschema", TableNamePrefix: "feat_elsewhere", + } + + _, diags := normalizeSetDiags(t, &resources.MaterializationSet{ + MaterializedFeatures: []ml.MaterializedFeature{offlineMember(memberFeatureName), elsewhere}, + }) + + require.Len(t, diags, 1) + assert.Equal(t, duplicateMemberKeyError("'"+memberFeatureName+"'"), diags[0].Summary) +} + +func TestNormalizeMaterializationSetMaterializedFeatureMembersNamesEveryCollidingFeatureSorted(t *testing.T) { + _, diags := normalizeSetDiags(t, &resources.MaterializationSet{ + MaterializedFeatures: []ml.MaterializedFeature{ + onlineMember("main.myschema.other", "store"), + onlineMember("main.myschema.other", "store"), + offlineMember(memberFeatureName), + offlineMember(memberFeatureName), + }, + }) + + require.Len(t, diags, 1) + assert.Equal(t, + duplicateMemberKeyError("'"+memberFeatureName+"', 'main.myschema.other'"), + diags[0].Summary) +} + +func TestNormalizeMaterializationSetMaterializedFeatureMembersNamesAFeatureOnceForTwoCollisions(t *testing.T) { + _, diags := normalizeSetDiags(t, &resources.MaterializationSet{ + MaterializedFeatures: []ml.MaterializedFeature{ + offlineMember(memberFeatureName), + offlineMember(memberFeatureName), + onlineMember(memberFeatureName, "store"), + onlineMember(memberFeatureName, "store"), + }, + }) + + require.Len(t, diags, 1) + assert.Equal(t, duplicateMemberKeyError("'"+memberFeatureName+"'"), diags[0].Summary) +} diff --git a/bundle/config/mutator/resourcemutator/resource_mutator.go b/bundle/config/mutator/resourcemutator/resource_mutator.go index 2262ad6c0a5..e00444cc773 100644 --- a/bundle/config/mutator/resourcemutator/resource_mutator.go +++ b/bundle/config/mutator/resourcemutator/resource_mutator.go @@ -149,6 +149,10 @@ func applyNormalizeMutators(ctx context.Context, b *bundle.Bundle) { // Expands glob patterns in pipeline library paths to include all matching files ExpandPipelineGlobPaths(), + // Reads (typed): resources.materialization_sets.*.materialized_features + // Rejects a set listing two materialized features with the same member key + NormalizeMaterializationSetMaterializedFeatureMembers(), + // Reads (dynamic): resources.jobs.*.job_clusters (reads job clusters to merge) // Updates (dynamic): resources.jobs.*.job_clusters (merges job clusters with the same job_cluster_key) MergeJobClusters(), diff --git a/bundle/config/mutator/resourcemutator/run_as_test.go b/bundle/config/mutator/resourcemutator/run_as_test.go index 200939f2dd0..15b303bd7c7 100644 --- a/bundle/config/mutator/resourcemutator/run_as_test.go +++ b/bundle/config/mutator/resourcemutator/run_as_test.go @@ -54,6 +54,7 @@ func allResourceTypes(t *testing.T) []string { "internal_immutable_snapshots", "job_runs", "jobs", + "materialization_sets", "mcp_services", "model_provider_services", "model_services", @@ -188,6 +189,7 @@ var allowList = []string{ "external_locations", "synced_database_tables", "jobs", + "materialization_sets", "pipelines", "models", "model_services", diff --git a/bundle/config/resources.go b/bundle/config/resources.go index 75fef099133..77b56bb6144 100644 --- a/bundle/config/resources.go +++ b/bundle/config/resources.go @@ -17,6 +17,7 @@ type Resources struct { Pipelines map[string]*resources.Pipeline `json:"pipelines,omitempty"` Features map[string]*resources.Feature `json:"features,omitempty"` + MaterializationSets map[string]*resources.MaterializationSet `json:"materialization_sets,omitempty"` Models map[string]*resources.MlflowModel `json:"models,omitempty"` Experiments map[string]*resources.MlflowExperiment `json:"experiments,omitempty"` ModelServingEndpoints map[string]*resources.ModelServingEndpoint `json:"model_serving_endpoints,omitempty"` @@ -116,6 +117,7 @@ func (r *Resources) AllResources() []ResourceGroup { collectResourceMap(descriptions["models"], r.Models), collectResourceMap(descriptions["experiments"], r.Experiments), collectResourceMap(descriptions["features"], r.Features), + collectResourceMap(descriptions["materialization_sets"], r.MaterializationSets), collectResourceMap(descriptions["model_serving_endpoints"], r.ModelServingEndpoints), collectResourceMap(descriptions["model_services"], r.ModelServices), collectResourceMap(descriptions["mcp_services"], r.McpServices), @@ -203,6 +205,7 @@ func SupportedResources() map[string]resources.ResourceDescription { "models": (&resources.MlflowModel{}).ResourceDescription(), "experiments": (&resources.MlflowExperiment{}).ResourceDescription(), "features": (&resources.Feature{}).ResourceDescription(), + "materialization_sets": (&resources.MaterializationSet{}).ResourceDescription(), "instance_pools": (&resources.InstancePool{}).ResourceDescription(), "model_serving_endpoints": (&resources.ModelServingEndpoint{}).ResourceDescription(), "model_services": (&resources.ModelService{}).ResourceDescription(), diff --git a/bundle/config/resources/materialization_set.go b/bundle/config/resources/materialization_set.go new file mode 100644 index 00000000000..4104d6fb012 --- /dev/null +++ b/bundle/config/resources/materialization_set.go @@ -0,0 +1,107 @@ +package resources + +import ( + "context" + "net/url" + "strconv" + "strings" + + "github.com/databricks/cli/libs/log" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/apierr" + "github.com/databricks/databricks-sdk-go/marshal" + "github.com/databricks/databricks-sdk-go/service/ml" +) + +// MaterializationSet is the bundle config for a group of materialized features created and +// managed as one unit. +type MaterializationSet struct { + BaseResource + ID string `json:"id,omitempty" bundle:"readonly"` + ml.MaterializationSet +} + +func (s *MaterializationSet) UnmarshalJSON(b []byte) error { + return marshal.Unmarshal(b, s) +} + +func (s MaterializationSet) MarshalJSON() ([]byte, error) { + return marshal.Marshal(s) +} + +// escapeMemberKeyPart percent-escapes the separator so the join in MaterializedFeatureMemberKey is +// injective. A feature name is validated only for length and non-emptiness, so it may contain the +// separator; two members sharing a key would be diffed as one, and the request would then omit a +// member the server deletes because it cannot see it. +func escapeMemberKeyPart(part string) string { + part = strings.ReplaceAll(part, "%", "%25") + return strings.ReplaceAll(part, "|", "%7C") +} + +// MaterializedFeatureMemberKey identifies a materialized feature member by content rather than +// position, as labelled components joined by the separator escapeMemberKeyPart escapes. +// +// online_store_name is omitted rather than left empty when the member has no online destination, +// which is what keeps an offline member distinct from an online one whose store name is unset. +// +// The server pairs members by the same components in MaterializationSetMemberKey -- see +// feature-store/src/main/scala/com/databricks/feature-store/utils/materialized-features/MaterializationSetUtils.scala +// -- holding them in a case class rather than a joined string, so this encoding is the CLI's own +// and only the component list has to agree. Adding, dropping or redefining one takes the same +// change there. +func MaterializedFeatureMemberKey(member ml.MaterializedFeature) (string, string) { + parts := []string{"feature_name=" + escapeMemberKeyPart(member.FeatureName)} + if member.OnlineStoreConfig != nil { + parts = append(parts, + "online_store_name="+escapeMemberKeyPart(member.OnlineStoreConfig.OnlineStoreName)) + } + parts = append(parts, "is_streaming_mode="+strconv.FormatBool(member.StreamingMode != nil)) + + return "member", strings.Join(parts, "|") +} + +// MaterializedFeatureMemberKeyFieldDescription names the key's components in plain words. It +// mirrors MaterializationSetMemberKey.fieldDescription in MaterializationSetUtils.scala, so the +// CLI's rejection reads like the service's own. +const MaterializedFeatureMemberKeyFieldDescription = "feature name, online store name, trigger is streaming mode" + +func (s *MaterializationSet) Exists(ctx context.Context, w *databricks.WorkspaceClient, id string) (bool, error) { + _, err := w.FeatureEngineering.GetMaterializationSet(ctx, ml.GetMaterializationSetRequest{ + MaterializationSetId: id, + }) + if err != nil { + log.Debugf(ctx, "materialization set %s does not exist: %v", id, err) + + if apierr.IsMissing(err) { + return false, nil + } + + return false, err + } + + return true, nil +} + +func (*MaterializationSet) ResourceDescription() ResourceDescription { + return ResourceDescription{ + SingularName: "materialization_set", + PluralName: "materialization_sets", + SingularTitle: "Materialization Set", + PluralTitle: "Materialization Sets", + } +} + +// GetName returns the in-product name, which is empty: a set is identified only by its +// server-assigned id. +func (s *MaterializationSet) GetName() string { + return "" +} + +// GetURL reports that this type has no workspace URL: a set has no standalone page, and its +// members surface under their own features. +func (s *MaterializationSet) GetURL() (string, bool) { + return "", false +} + +// InitializeURL leaves URL unset. See GetURL. +func (s *MaterializationSet) InitializeURL(_ url.URL) {} diff --git a/bundle/config/resources/materialization_set_test.go b/bundle/config/resources/materialization_set_test.go new file mode 100644 index 00000000000..aae15b9300e --- /dev/null +++ b/bundle/config/resources/materialization_set_test.go @@ -0,0 +1,101 @@ +package resources + +import ( + "net/url" + "testing" + + "github.com/databricks/databricks-sdk-go/service/ml" + + "github.com/stretchr/testify/assert" +) + +// TestMaterializationSetReportsNoWorkspaceURL pins the signal that keeps `bundle summary` from +// printing a URL line for a set. BaseResource.GetURL returns ("", true), which the summary reads as +// a supported type awaiting deployment and renders as "(not deployed)". +func TestMaterializationSetReportsNoWorkspaceURL(t *testing.T) { + set := &MaterializationSet{ID: "0123456789abcdef0123456789abcdef"} + set.URL = "https://example.cloud.databricks.com/explore/data/features" + + rawURL, supportsURL := set.GetURL() + assert.False(t, supportsURL) + assert.Empty(t, rawURL) + + set.InitializeURL(url.URL{Scheme: "https", Host: "example.cloud.databricks.com"}) + rawURL, supportsURL = set.GetURL() + assert.False(t, supportsURL, "InitializeURL must not make the type look URL-bearing") + assert.Empty(t, rawURL) +} + +func TestMaterializationSetMemberKey(t *testing.T) { + offline := ml.MaterializedFeature{ + FeatureName: "main.default.feat", + OfflineStoreConfig: &ml.OfflineStoreConfig{CatalogName: "main", SchemaName: "default"}, + } + online := ml.MaterializedFeature{ + FeatureName: "main.default.feat", + OnlineStoreConfig: &ml.OnlineStoreConfig{OnlineStoreName: "store_one"}, + } + streaming := ml.MaterializedFeature{ + FeatureName: "main.default.feat", + OfflineStoreConfig: &ml.OfflineStoreConfig{CatalogName: "main", SchemaName: "default"}, + StreamingMode: &ml.StreamingMode{}, + } + + field, value := MaterializedFeatureMemberKey(offline) + assert.Equal(t, "member", field) + assert.Equal(t, "feature_name=main.default.feat|is_streaming_mode=false", value) + + _, onlineValue := MaterializedFeatureMemberKey(online) + assert.Equal(t, "feature_name=main.default.feat|online_store_name=store_one|is_streaming_mode=false", onlineValue) + + _, streamingValue := MaterializedFeatureMemberKey(streaming) + assert.Equal(t, "feature_name=main.default.feat|is_streaming_mode=true", streamingValue) + + // The three members of one feature must not collide with each other. + assert.NotEqual(t, value, onlineValue) + assert.NotEqual(t, value, streamingValue) +} + +// A feature name may contain the separator, so without escaping these two distinct members would +// share a key: the plan would diff them as one and the update request would omit the member the +// server then deletes. +func TestMaterializationSetMemberKeyEscapesSeparator(t *testing.T) { + nameHoldsSeparator := ml.MaterializedFeature{ + FeatureName: "main.default.a|b", + OnlineStoreConfig: &ml.OnlineStoreConfig{OnlineStoreName: "c"}, + } + storeHoldsSeparator := ml.MaterializedFeature{ + FeatureName: "main.default.a", + OnlineStoreConfig: &ml.OnlineStoreConfig{OnlineStoreName: "b|c"}, + } + + _, first := MaterializedFeatureMemberKey(nameHoldsSeparator) + _, second := MaterializedFeatureMemberKey(storeHoldsSeparator) + assert.NotEqual(t, first, second) + + // The escape character itself is escaped first, so a name cannot forge an escaped separator. + _, forged := MaterializedFeatureMemberKey(ml.MaterializedFeature{ + FeatureName: "main.default.a%7Cb", + OnlineStoreConfig: &ml.OnlineStoreConfig{OnlineStoreName: "c"}, + }) + assert.NotEqual(t, first, forged) +} + +// An absent online destination and an online destination carrying no store name are distinct +// members, which is why the component is omitted rather than left empty. +func TestMaterializationSetMemberKeySeparatesAbsentOnlineStoreFromEmptyName(t *testing.T) { + _, offline := MaterializedFeatureMemberKey(ml.MaterializedFeature{ + FeatureName: "main.default.feat", + OfflineStoreConfig: &ml.OfflineStoreConfig{CatalogName: "main", SchemaName: "default"}, + }) + _, onlineWithoutName := MaterializedFeatureMemberKey(ml.MaterializedFeature{ + FeatureName: "main.default.feat", + OnlineStoreConfig: &ml.OnlineStoreConfig{CatalogName: "main", SchemaName: "default"}, + }) + + assert.Equal(t, "feature_name=main.default.feat|is_streaming_mode=false", offline) + assert.Equal(t, + "feature_name=main.default.feat|online_store_name=|is_streaming_mode=false", + onlineWithoutName) + assert.NotEqual(t, offline, onlineWithoutName) +} diff --git a/bundle/config/resources_test.go b/bundle/config/resources_test.go index 963e7efd4ff..4cdafc52747 100644 --- a/bundle/config/resources_test.go +++ b/bundle/config/resources_test.go @@ -128,6 +128,9 @@ func TestBundleResourcePluralNamesResolveInWorkspaceURLs(t *testing.T) { // Resources that intentionally have no workspace URL. noURL := map[string]bool{ "external_locations": true, + // A materialization set has no standalone workspace page; its members + // surface under their own features. + "materialization_sets": true, // A job run does have a workspace URL, but it's addressed by two IDs // (job + run) so it can't be expressed as a single-ID pattern here; it's // built in JobRun.InitializeURL via workspaceurls.JobRunURL instead. @@ -368,6 +371,15 @@ func TestResourcesBindSupport(t *testing.T) { }, }, }, + MaterializationSets: map[string]*resources.MaterializationSet{ + "my_materialization_set": { + MaterializationSet: ml.MaterializationSet{ + MaterializedFeatures: []ml.MaterializedFeature{ + {FeatureName: "main.myschema.my_feature"}, + }, + }, + }, + }, VectorSearchEndpoints: map[string]*resources.VectorSearchEndpoint{ "my_vector_search_endpoint": { CreateEndpoint: vectorsearch.CreateEndpoint{ @@ -399,6 +411,7 @@ func TestResourcesBindSupport(t *testing.T) { m.GetMockPipelinesAPI().EXPECT().Get(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockExperimentsAPI().EXPECT().GetExperiment(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockFeatureEngineeringAPI().EXPECT().GetFeature(mock.Anything, mock.Anything).Return(nil, nil) + m.GetMockFeatureEngineeringAPI().EXPECT().GetMaterializationSet(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockRegisteredModelsAPI().EXPECT().Get(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockCatalogsAPI().EXPECT().GetByName(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockExternalLocationsAPI().EXPECT().GetByName(mock.Anything, mock.Anything).Return(nil, nil) diff --git a/bundle/direct/dresources/all.go b/bundle/direct/dresources/all.go index 88f08048235..b13465550ca 100644 --- a/bundle/direct/dresources/all.go +++ b/bundle/direct/dresources/all.go @@ -12,6 +12,7 @@ var SupportedResources = map[string]any{ "pipelines": (*ResourcePipeline)(nil), "experiments": (*ResourceExperiment)(nil), "features": (*ResourceFeature)(nil), + "materialization_sets": (*ResourceMaterializationSet)(nil), "catalogs": (*ResourceCatalog)(nil), "schemas": (*ResourceSchema)(nil), "external_locations": (*ResourceExternalLocation)(nil), diff --git a/bundle/direct/dresources/all_test.go b/bundle/direct/dresources/all_test.go index 140f3acf1e5..451ca5fefce 100644 --- a/bundle/direct/dresources/all_test.go +++ b/bundle/direct/dresources/all_test.go @@ -7,6 +7,7 @@ import ( "os" "path/filepath" "reflect" + "slices" "strconv" "strings" "testing" @@ -123,6 +124,31 @@ var testConfig map[string]any = map[string]any{ TimeseriesColumn: &ml.TimeseriesColumn{Name: "ts"}, }, + // A feature materialized to both an offline table and one online store: the two members + // share a feature name, so they exercise the member key's online-store discriminator. + "materialization_sets": &resources.MaterializationSet{ + MaterializedFeatures: []ml.MaterializedFeature{ + { + FeatureName: "main.default.my_feature", + CronScheduleTrigger: &ml.CronSchedule{CronExpression: "0 0 * * * ?"}, + OfflineStoreConfig: &ml.OfflineStoreConfig{ + CatalogName: "main", + SchemaName: "default", + TableNamePrefix: "offline", + }, + }, + { + FeatureName: "main.default.my_feature", + CronScheduleTrigger: &ml.CronSchedule{CronExpression: "0 0 * * * ?"}, + OnlineStoreConfig: &ml.OnlineStoreConfig{ + CatalogName: "main", + SchemaName: "default", + OnlineStoreName: "my_store", + }, + }, + }, + }, + "experiments": &resources.MlflowExperiment{ Name: "my-experiment", Tags: []ml.ExperimentTag{ @@ -943,22 +969,27 @@ func TestAll(t *testing.T) { // testIgnoreFilter encapsulates the logic for filtering fields based on ignore_remote_changes config. type testIgnoreFilter struct { - ignoreFields map[string]bool - adapter *Adapter + ignoreFields map[string]bool + ignorePatterns []*structpath.PatternNode + adapter *Adapter } // newTestIgnoreFilter creates a filter from the adapter's resource configs. // It also ignores fields that exist in StateType but not in RemoteType, because // those are automatically suppressed by the planner (reason: missing_in_remote) // and RemapState cannot populate them from remote state. -func newTestIgnoreFilter(adapter *Adapter) *testIgnoreFilter { +func newTestIgnoreFilter(adapter *Adapter, group string) *testIgnoreFilter { ignoreFields := make(map[string]bool) + var ignorePatterns []*structpath.PatternNode for _, cfg := range []*ResourceLifecycleConfig{adapter.ResourceConfig(), adapter.GeneratedResourceConfig()} { if cfg == nil { continue } for _, p := range cfg.IgnoreRemoteChanges { ignoreFields[p.Field.String()] = true + if group == "materialization_sets" { + ignorePatterns = append(ignorePatterns, p.Field) + } } } // Auto-include fields present in StateType but absent from RemoteType. @@ -972,7 +1003,7 @@ func newTestIgnoreFilter(adapter *Adapter) *testIgnoreFilter { } return true }) - return &testIgnoreFilter{ignoreFields: ignoreFields, adapter: adapter} + return &testIgnoreFilter{ignoreFields: ignoreFields, ignorePatterns: ignorePatterns, adapter: adapter} } // shouldIgnore returns true if the field at the given path should be ignored. @@ -991,7 +1022,19 @@ func (f *testIgnoreFilter) shouldIgnore(path string) bool { topLevelField = parts[0] } - return f.ignoreFields[topLevelField] + if f.ignoreFields[topLevelField] { + return true + } + + // Pattern rules are collected for materialization_sets alone, so this loop is empty for every + // other resource. A rule inside a slice (foo[*].bar) never matches the indexed path a walk + // produces (foo[0].bar); jobs, apps and database_instances carry such rules too, so widening + // the comparison to them is its own change. + parsed, err := structpath.ParsePath(path) + if err != nil { + return false + } + return slices.ContainsFunc(f.ignorePatterns, parsed.HasPatternPrefix) } // filterChanges returns only the changes that should not be ignored. @@ -1082,7 +1125,7 @@ func testCRUD(t *testing.T, group string, adapter *Adapter, client *databricks.W require.NotNil(t, remappedState) // Create filter for fields that should be ignored based on ignore_remote_changes config. - ignoreFilter := newTestIgnoreFilter(adapter) + ignoreFilter := newTestIgnoreFilter(adapter, group) if remoteStateFromCreate != nil { remappedRemoteStateFromCreate, err := adapter.RemapState(remoteStateFromCreate) diff --git a/bundle/direct/dresources/apitypes.generated.yml b/bundle/direct/dresources/apitypes.generated.yml index 317a04442c4..82540c55ff2 100644 --- a/bundle/direct/dresources/apitypes.generated.yml +++ b/bundle/direct/dresources/apitypes.generated.yml @@ -30,6 +30,8 @@ job_runs: jobs.RunNow jobs: jobs.JobSettings +materialization_sets: ml.MaterializationSet + mcp_services: catalog.CreateMcpServiceRequest model_provider_services: catalog.CreateModelProviderServiceRequest diff --git a/bundle/direct/dresources/configs/materialization_sets.generated.yml b/bundle/direct/dresources/configs/materialization_sets.generated.yml new file mode 100644 index 00000000000..a4e314a2921 --- /dev/null +++ b/bundle/direct/dresources/configs/materialization_sets.generated.yml @@ -0,0 +1,36 @@ +# Generated, do not edit. + +recreate_on_changes: + - field: materialization_set_id + reason: spec:immutable + - field: materialized_features[*].materialized_feature_id + reason: spec:immutable + - field: materialized_features[*].offline_store_config + reason: spec:immutable + - field: materialized_features[*].online_store_config + reason: spec:immutable + +ignore_remote_changes: + - field: materialized_features[*].offline_store_config + reason: spec:input_only + - field: materialized_features[*].online_store_config.catalog_name + reason: spec:input_only + - field: materialized_features[*].online_store_config.schema_name + reason: spec:input_only + - field: materialized_features[*].online_store_config.table_name_prefix + reason: spec:input_only + + - field: materialized_features[*].is_online + reason: spec:output_only + - field: materialized_features[*].job_id + reason: spec:output_only + - field: materialized_features[*].last_materialization_time + reason: spec:output_only + - field: materialized_features[*].latest_backfill_operation + reason: spec:output_only + - field: materialized_features[*].materialization_set_id + reason: spec:output_only + - field: materialized_features[*].pipeline_id + reason: spec:output_only + - field: materialized_features[*].table_name + reason: spec:output_only diff --git a/bundle/direct/dresources/configs/materialization_sets.yml b/bundle/direct/dresources/configs/materialization_sets.yml new file mode 100644 index 00000000000..992130fec37 --- /dev/null +++ b/bundle/direct/dresources/configs/materialization_sets.yml @@ -0,0 +1,24 @@ +recreate_on_changes: + - field: materialized_features[*].feature_name + reason: immutable + - field: materialized_features[*].cron_schedule + reason: immutable + - field: materialized_features[*].cron_schedule_trigger + reason: immutable + - field: materialized_features[*].table_trigger + reason: immutable + - field: materialized_features[*].streaming_mode + reason: immutable + - field: materialized_features[*].pipeline_schedule_state + reason: immutable + - field: materialized_features[*].tags + reason: immutable + - field: materialized_features[*].budget_policy_id + reason: immutable + - field: materialized_features[*].notifications + reason: immutable + +backend_defaults: + # A member the config leaves unset still reports the state the server set on create, which the + # bundle never wrote, so the undeclared case diffs as old=nil, new=nil, remote=ACTIVE. + - field: materialized_features[*].pipeline_schedule_state diff --git a/bundle/direct/dresources/materialization_set.go b/bundle/direct/dresources/materialization_set.go new file mode 100644 index 00000000000..05eeb3d07bc --- /dev/null +++ b/bundle/direct/dresources/materialization_set.go @@ -0,0 +1,257 @@ +package dresources + +import ( + "context" + "fmt" + "net/http" + "slices" + "strings" + "uuid" + + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/apierr" + "github.com/databricks/databricks-sdk-go/common/types/fieldmask" + "github.com/databricks/databricks-sdk-go/service/ml" +) + +// https://docs.databricks.com/api/workspace/featureengineering +type ResourceMaterializationSet struct { + client *databricks.WorkspaceClient +} + +func (*ResourceMaterializationSet) New(client *databricks.WorkspaceClient) *ResourceMaterializationSet { + return &ResourceMaterializationSet{client: client} +} + +func (*ResourceMaterializationSet) PrepareState(input *resources.MaterializationSet) *ml.MaterializationSet { + return &input.MaterializationSet +} + +func (*ResourceMaterializationSet) KeyedSlices() map[string]any { + return map[string]any{ + "materialized_features": resources.MaterializedFeatureMemberKey, + } +} + +var ( + materializedFeaturesPattern = structpath.NewPatternDotString(nil, "materialized_features") + materializedFeatureMember = structpath.NewPatternBracketStar(materializedFeaturesPattern) + + // The destination fields the server accepts on write and never returns on read. The whole + // offline block is input-only; online_store_name is returned, so it is deliberately absent. + redactedDestinationPatterns = []*structpath.PatternNode{ + structpath.NewPatternDotString(materializedFeatureMember, "offline_store_config"), + structpath.NewPatternDotString( + structpath.NewPatternDotString(materializedFeatureMember, "online_store_config"), + "catalog_name", + ), + structpath.NewPatternDotString( + structpath.NewPatternDotString(materializedFeatureMember, "online_store_config"), + "schema_name", + ), + structpath.NewPatternDotString( + structpath.NewPatternDotString(materializedFeatureMember, "online_store_config"), + "table_name_prefix", + ), + } +) + +// isRedactedDestinationField reports whether path names a destination field the read never returns. +func isRedactedDestinationField(path *structpath.PathNode) bool { + return slices.ContainsFunc(redactedDestinationPatterns, path.HasPatternPrefix) +} + +// isAbsentRemotely reports whether a read returned nothing for a field, which for a redacted field +// is indistinguishable from the server having no value. +func isAbsentRemotely(remote any) bool { + if remote == nil { + return true + } + text, isString := remote.(string) + return isString && text == "" +} + +// OverrideChangeDesc does two things the generic classification gets wrong here: it marks a whole +// materialized feature member added to or removed from the set as an update rather than a +// destroy-and-recreate, and it suppresses the drift left behind by an ignore_remote_changes rule +// that cannot fire. +func (*ResourceMaterializationSet) OverrideChangeDesc(_ context.Context, path *structpath.PathNode, change *ChangeDesc, _ *ml.MaterializationSet) error { + // A member added or removed whole is a two-segment path, materialized_features[member=...] + // carrying the key materializationSetMemberKey builds, and the member list emptied whole is the + // one-segment materialized_features. An immutable field rule + // under materialized_features[*] matches either, because the matcher truncates a pattern longer + // than the path. Both count as an update rather than a destroy-and-recreate, because + // reconciling membership is what the update endpoint does. + if path.Len() <= 2 && path.HasPatternPrefix(materializedFeaturesPattern) { + change.Action = deployplan.Update + return nil + } + + // The ignore_remote_changes suppression on a destination field cannot fire for this resource, + // which keys its member slice on a composite key defined by materializationSetMemberKey. The + // suppression fires only when the persisted value equals the configured + // one, and the persisted-value lookup resolves that composite against the element's field names, + // which never match, so it always returns nothing. An unset change.Old is the surrogate for the + // persisted value being unreachable; a repoint the config asked for carries that value in from + // the local diff instead, and so still recreates. + if isRedactedDestinationField(path) && change.Old == nil && change.New != nil && + isAbsentRemotely(change.Remote) { + change.Action = deployplan.Skip + change.Reason = "input_only_not_returned" + } + return nil +} + +func (r *ResourceMaterializationSet) DoRead(ctx context.Context, id string) (*ml.MaterializationSet, error) { + return r.client.FeatureEngineering.GetMaterializationSet(ctx, ml.GetMaterializationSetRequest{ + MaterializationSetId: id, + }) +} + +// DoCreate submits the set and waits for the long-running operation the create returns. The created +// set arrives as that operation's response, which Wait reaches by polling GetOperation. +func (r *ResourceMaterializationSet) DoCreate(ctx context.Context, config *ml.MaterializationSet) (string, *ml.MaterializationSet, error) { + // The endpoint takes an idempotency token, and a repeated request carrying one already used + // returns the operation the first request with the same idempotency token created. Minted here + // so the token the server sees is the one this call chose. + operation, err := r.client.FeatureEngineering.CreateMaterializationSet(ctx, ml.CreateMaterializationSetRequest{ + MaterializationSet: *config, + RequestId: uuid.New().String(), + ForceSendFields: nil, + }) + if err != nil { + return "", nil, err + } + + set, err := operation.Wait(ctx) + if err != nil { + return "", nil, r.materializationSetOperationFailure(ctx, operation, err) + } + // The id is what the engine records the resource under, so an operation that settles without + // one is a failure even though the wait succeeded. + if set == nil || set.MaterializationSetId == "" { + return "", nil, fmt.Errorf("%s: creation finished without returning the set", + materializationSetOperationTarget(operation)) + } + return set.MaterializationSetId, set, nil +} + +// materializationSetChangeFailures names each materialized feature member a partly applied change +// in the update or delete action could not reach, from whichever progress arm the operation's +// metadata carries. Empty for a creation action, and for a failure (e.g. a validation error) raised +// before the server performed the action. +func materializationSetChangeFailures(metadata *ml.MaterializationSetOperationMetadata) []string { + if metadata == nil { + return nil + } + var members []ml.MaterializedFeatureChangeFailure + switch { + case metadata.UpdateProgress != nil: + members = metadata.UpdateProgress.FailedMaterializedFeatures + case metadata.DeletionProgress != nil: + members = metadata.DeletionProgress.FailedMaterializedFeatures + } + + failures := make([]string, 0, len(members)) + for _, member := range members { + failures = append(failures, fmt.Sprintf("%s (%s): %s", + member.MaterializedFeatureId, member.ErrorCode, member.Message)) + } + return failures +} + +// materializationSetOperationFailure restates a response operation with FAILED status as an API +// error. +// +// StatusCode is set to 200 because the create, update and delete endpoints always return an +// operation with FAILED status when an error is raised, except for the errors raised before the +// action is executed, such as a validation error on the request. +func (r *ResourceMaterializationSet) materializationSetOperationFailure( + ctx context.Context, + operation materializationSetOperation, + waitErr error, +) error { + target := materializationSetOperationTarget(operation) + settled, err := r.client.FeatureEngineering.GetOperation(ctx, ml.GetOperationRequest{ + Name: operation.Name(), + }) + if err != nil || settled.Error == nil { + return fmt.Errorf("%s: %w", target, waitErr) + } + + message := settled.Error.Message + if metadata, err := operation.Metadata(); err == nil { + if failures := materializationSetChangeFailures(metadata); len(failures) > 0 { + message += "; members left unchanged: " + strings.Join(failures, "; ") + } + } + return &apierr.APIError{ + ErrorCode: string(settled.Error.ErrorCode), + Message: target + " operation failed: " + message, + StatusCode: http.StatusOK, + ResponseWrapper: nil, + Details: nil, //nolint:staticcheck // SA1019: deprecated, but exhaustruct requires every field + } +} + +// materializationSetOperation is what a create, update and delete operation have in common: the +// three generated interfaces share these two methods and differ only in what Wait returns. +type materializationSetOperation interface { + Name() string + Metadata() (*ml.MaterializationSetOperationMetadata, error) +} + +// materializationSetOperationTarget names the set an operation was changing, taking the set id from +// the operation metadata when the server supplied it and falling back to the operation name. +func materializationSetOperationTarget(operation materializationSetOperation) string { + metadata, err := operation.Metadata() + if err == nil && metadata != nil && metadata.MaterializationSetId != "" { + return "materialization set " + metadata.MaterializationSetId + } + return "operation " + operation.Name() +} + +// DoUpdate reconciles the set's membership and waits for the long-running operation the update +// returns. The reconciled set arrives as that operation's response. +func (r *ResourceMaterializationSet) DoUpdate(ctx context.Context, id string, config *ml.MaterializationSet, _ *PlanEntry) (*ml.MaterializationSet, error) { + update := *config + update.MaterializationSetId = id + + operation, err := r.client.FeatureEngineering.UpdateMaterializationSet(ctx, ml.UpdateMaterializationSetRequest{ + MaterializationSetId: id, + MaterializationSet: update, + UpdateMask: *fieldmask.New([]string{"*"}), + RequestId: uuid.New().String(), + ForceSendFields: nil, + }) + if err != nil { + return nil, err + } + + set, err := operation.Wait(ctx) + if err != nil { + return nil, r.materializationSetOperationFailure(ctx, operation, err) + } + return set, nil +} + +// DoDelete deletes the set and waits for the long-running operation the delete returns, which +// settles with no response body. +func (r *ResourceMaterializationSet) DoDelete(ctx context.Context, id string, _ *ml.MaterializationSet) error { + operation, err := r.client.FeatureEngineering.DeleteMaterializationSet(ctx, ml.DeleteMaterializationSetRequest{ + MaterializationSetId: id, + RequestId: uuid.New().String(), + AllowMissing: false, + ForceSendFields: nil, + }) + if err != nil { + return err + } + if err := operation.Wait(ctx); err != nil { + return r.materializationSetOperationFailure(ctx, operation, err) + } + return nil +} diff --git a/bundle/direct/dresources/materialization_set_coverage_test.go b/bundle/direct/dresources/materialization_set_coverage_test.go new file mode 100644 index 00000000000..293c4d7fa51 --- /dev/null +++ b/bundle/direct/dresources/materialization_set_coverage_test.go @@ -0,0 +1,136 @@ +package dresources + +import ( + "reflect" + "strings" + "testing" + + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/databricks-sdk-go/service/ml" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// setFieldsWithoutRules are the set's own fields that no rule covers: the member slice, whose +// elements are diffed by key, and the id, which the generated config already classifies. +var setFieldsWithoutRules = map[string]bool{ + "materialized_features": true, +} + +// classifiedFields collects the field patterns either lifecycle config mentions, from both the +// hand-written and generated halves, since the planner consults them in sequence. Backend defaults +// are returned apart: one absorbs a field the config leaves unset and says nothing about a value +// the config does declare. +func classifiedFields(t *testing.T) (onDeclaredValue, onlyWhenUndeclared map[string]bool) { + t.Helper() + + handWritten := GetResourceConfig("materialization_sets") + require.NotNil(t, handWritten) + generated := GetGeneratedResourceConfig("materialization_sets") + require.NotNil(t, generated) + + onDeclaredValue = map[string]bool{} + onlyWhenUndeclared = map[string]bool{} + for _, cfg := range []*ResourceLifecycleConfig{handWritten, generated} { + for _, rules := range [][]FieldRule{ + cfg.RecreateOnChanges, + cfg.IgnoreRemoteChanges, + cfg.IgnoreLocalChanges, + cfg.ProvidedIDFields, + cfg.UpdatableIDFields, + cfg.NormalizeSlash, + cfg.StableOutputFields, + } { + for _, rule := range rules { + onDeclaredValue[rule.Field.String()] = true + } + } + for _, rule := range cfg.BackendDefaults { + onlyWhenUndeclared[rule.Field.String()] = true + } + } + return onDeclaredValue, onlyWhenUndeclared +} + +// memberFieldPath builds the path the planner hands to OverrideChangeDesc for one member field. +func memberFieldPath(t *testing.T, memberKey, field string) *structpath.PathNode { + t.Helper() + path, err := structpath.ParsePath( + "materialized_features[member='" + memberKey + "']." + field) + require.NoError(t, err) + return path +} + +// TestMaterializationSetAllMemberFieldsAreClassified keeps the hand-written recreate list +// exhaustive. TestNoUpdateResourcesCoverAllFields stops at HasDoUpdate, and this resource has one, +// so nothing else holds the line: a member field added to the proto later would be classified as a +// plain update, and the endpoint would accept the request and write nothing. +func TestMaterializationSetAllMemberFieldsAreClassified(t *testing.T) { + onDeclaredValue, _ := classifiedFields(t) + + for field := range reflect.TypeFor[ml.MaterializedFeature]().Fields() { + name, ok := jsonFieldName(field) + if !ok { + continue + } + assert.Truef(t, onDeclaredValue["materialized_features[*]."+name], + "member field %q has no rule covering a value the config declares; the set update "+ + "reconciles membership only, so add it to recreate_on_changes. A backend_defaults "+ + "entry does not count: it fires only while the config leaves the field unset", + name, + ) + } +} + +// TestMaterializationSetAllSetFieldsAreClassified does the same for the set's own fields. +func TestMaterializationSetAllSetFieldsAreClassified(t *testing.T) { + onDeclaredValue, onlyWhenUndeclared := classifiedFields(t) + + for field := range reflect.TypeFor[ml.MaterializationSet]().Fields() { + name, ok := jsonFieldName(field) + if !ok || setFieldsWithoutRules[name] { + continue + } + assert.Truef(t, onDeclaredValue[name] || onlyWhenUndeclared[name], + "set field %q is declared in neither materialization_sets.yml nor its generated half", + name, + ) + } +} + +// TestMaterializationSetDestinationFieldsAreSuppressed pins the fields OverrideChangeDesc skips. +// They are input-only, so the read never returns them and the generated rule that would absorb +// the difference cannot fire; a field dropped from this list silently re-plans as a recreate. +func TestMaterializationSetDestinationFieldsAreSuppressed(t *testing.T) { + offline := &ml.MaterializedFeature{ + OfflineStoreConfig: &ml.OfflineStoreConfig{CatalogName: "c", SchemaName: "s", TableNamePrefix: "p"}, + } + online := &ml.MaterializedFeature{ + OnlineStoreConfig: &ml.OnlineStoreConfig{OnlineStoreName: "store", CatalogName: "c"}, + } + + for _, field := range []string{"catalog_name", "schema_name", "table_name_prefix"} { + _, offlineKey := resources.MaterializedFeatureMemberKey(*offline) + _, onlineKey := resources.MaterializedFeatureMemberKey(*online) + assert.Truef(t, + isRedactedDestinationField(memberFieldPath(t, offlineKey, "offline_store_config."+field)), + "offline_store_config.%s must be suppressed", field) + assert.Truef(t, + isRedactedDestinationField(memberFieldPath(t, onlineKey, "online_store_config."+field)), + "online_store_config.%s must be suppressed", field) + } + + // online_store_name is returned by the read, so a repoint of it must stay visible. + _, onlineKey := resources.MaterializedFeatureMemberKey(*online) + assert.False(t, + isRedactedDestinationField(memberFieldPath(t, onlineKey, "online_store_config.online_store_name"))) +} + +func jsonFieldName(field reflect.StructField) (string, bool) { + tag := field.Tag.Get("json") + if tag == "" || tag == "-" { + return "", false + } + return strings.TrimSuffix(tag, ",omitempty"), true +} diff --git a/bundle/direct/dresources/materialization_set_test.go b/bundle/direct/dresources/materialization_set_test.go new file mode 100644 index 00000000000..83646da3032 --- /dev/null +++ b/bundle/direct/dresources/materialization_set_test.go @@ -0,0 +1,142 @@ +package dresources + +import ( + "errors" + "testing" + + "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/databricks-sdk-go/service/ml" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// operationStub stands in for a generated operation interface. The three verbs' interfaces differ +// only in what Wait returns, which materializationSetOperationTarget never calls. +type operationStub struct { + name string + metadata *ml.MaterializationSetOperationMetadata + err error +} + +func (s operationStub) Name() string { return s.name } + +func (s operationStub) Metadata() (*ml.MaterializationSetOperationMetadata, error) { + return s.metadata, s.err +} + +// The set id names what a failed create, update or delete was changing. The server assigns it +// before reserving the operation record, so the fallback to the operation name covers the cases +// where the metadata is unreadable rather than a set that has no id. +func TestMaterializationSetOperationTarget(t *testing.T) { + assert.Equal(t, "materialization set set-1", materializationSetOperationTarget(operationStub{ + name: "operations/abc", + metadata: &ml.MaterializationSetOperationMetadata{MaterializationSetId: "set-1"}, + })) + + assert.Equal(t, "operation operations/abc", materializationSetOperationTarget(operationStub{ + name: "operations/abc", + metadata: &ml.MaterializationSetOperationMetadata{}, + })) + + assert.Equal(t, "operation operations/abc", materializationSetOperationTarget(operationStub{ + name: "operations/abc", + })) + + assert.Equal(t, "operation operations/abc", materializationSetOperationTarget(operationStub{ + name: "operations/abc", + err: errors.New("metadata is not a MaterializationSetOperationMetadata"), + })) +} + +func TestMaterializationSetChangeFailuresFromUpdateProgress(t *testing.T) { + failures := materializationSetChangeFailures(&ml.MaterializationSetOperationMetadata{ + UpdateProgress: &ml.MaterializationSetUpdateProgress{ + DeletedMaterializedFeatureIds: []string{"mf-gone"}, + FailedMaterializedFeatures: []ml.MaterializedFeatureChangeFailure{ + { + MaterializedFeatureId: "mf-kept", + ErrorCode: "PERMISSION_DENIED", + Message: "no permission on the online table", + }, + }, + }, + }) + + assert.Equal(t, []string{"mf-kept (PERMISSION_DENIED): no permission on the online table"}, failures) +} + +func TestMaterializationSetChangeFailuresFromDeletionProgress(t *testing.T) { + failures := materializationSetChangeFailures(&ml.MaterializationSetOperationMetadata{ + DeletionProgress: &ml.MaterializationSetDeletionProgress{ + FailedMaterializedFeatures: []ml.MaterializedFeatureChangeFailure{ + {MaterializedFeatureId: "mf-1", ErrorCode: "INTERNAL_ERROR", Message: "pipeline stuck"}, + {MaterializedFeatureId: "mf-2", ErrorCode: "RESOURCE_CONFLICT", Message: "in use"}, + }, + }, + }) + + assert.Equal(t, []string{ + "mf-1 (INTERNAL_ERROR): pipeline stuck", + "mf-2 (RESOURCE_CONFLICT): in use", + }, failures) +} + +func TestMaterializationSetChangeFailuresWithoutFailedMembers(t *testing.T) { + // A creation carries no progress arm at all, and a change that fully applied carries one + // naming no failure; neither has a member to report. + assert.Empty(t, materializationSetChangeFailures(&ml.MaterializationSetOperationMetadata{})) + assert.Empty(t, materializationSetChangeFailures(&ml.MaterializationSetOperationMetadata{ + DeletionProgress: &ml.MaterializationSetDeletionProgress{ + DeletedMaterializedFeatureIds: []string{"mf-1"}, + }, + })) + assert.Empty(t, materializationSetChangeFailures(nil)) +} + +// TestMaterializationSetMembershipChangeIsAnUpdate enumerates every path shape the planner hands +// this resource, so the membership carve-out's reach is visible rather than inferred. The two +// shapes it covers are a member added or removed whole and the member list emptied whole; every +// other path keeps the action the rules gave it. +func TestMaterializationSetMembershipChangeIsAnUpdate(t *testing.T) { + const member = "materialized_features[member='feature_name=main.ns.feat|online_store_name=store|is_streaming_mode=false']" + + for _, test := range []struct { + path string + action deployplan.ActionType + }{ + {"materialized_features", deployplan.Update}, + {member, deployplan.Update}, + {"materialized_features[member='feature_name=main.ns.feat|is_streaming_mode=false']", deployplan.Update}, + {member + ".feature_name", deployplan.Recreate}, + {member + ".cron_schedule_trigger", deployplan.Recreate}, + {member + ".online_store_config.table_name_prefix", deployplan.Recreate}, + {"materialization_set_id", deployplan.Recreate}, + } { + t.Run(test.path, func(t *testing.T) { + path, err := structpath.ParsePath(test.path) + require.NoError(t, err) + + change := &ChangeDesc{Action: deployplan.Recreate, Reason: "immutable"} + require.NoError(t, (&ResourceMaterializationSet{}).OverrideChangeDesc( + t.Context(), path, change, nil)) + assert.Equal(t, test.action, change.Action) + }) + } +} + +// TestMaterializationSetDestinationSuppressionStillReachesMemberFields keeps the two blocks of +// OverrideChangeDesc disjoint: widening the membership carve-out to one segment must not shadow the +// input-only suppression, which only ever sees a member's destination three segments down. +func TestMaterializationSetDestinationSuppressionStillReachesMemberFields(t *testing.T) { + path, err := structpath.ParsePath( + "materialized_features[member='feature_name=main.ns.feat|is_streaming_mode=false'].offline_store_config") + require.NoError(t, err) + + change := &ChangeDesc{Action: deployplan.Recreate, Reason: "spec:immutable", New: "declared"} + require.NoError(t, (&ResourceMaterializationSet{}).OverrideChangeDesc( + t.Context(), path, change, nil)) + + assert.Equal(t, deployplan.Skip, change.Action) + assert.Equal(t, "input_only_not_returned", change.Reason) +} diff --git a/bundle/direct/keyed_member_pattern_match_test.go b/bundle/direct/keyed_member_pattern_match_test.go new file mode 100644 index 00000000000..ac5e15f1a92 --- /dev/null +++ b/bundle/direct/keyed_member_pattern_match_test.go @@ -0,0 +1,29 @@ +package direct + +import ( + "testing" + + "github.com/databricks/cli/libs/structs/structpath" + "github.com/stretchr/testify/assert" +) + +// A whole-member change on a keyed slice is a two-segment path, so it matches any deeper +// per-field pattern through the short-path branch of matchesFieldRuleBidirectional. A +// per-field recreate rule therefore recreates the parent when a member is merely added or +// removed, which is why a keyed-slice resource needs a carve-out before classification. +func TestKeyedMemberChangeMatchesPerFieldPattern(t *testing.T) { + // configs/.yml: materialized_features[*].feature_name + pattern := structpath.NewPatternDotString(nil, "materialized_features") + pattern = structpath.NewPatternBracketStar(pattern) + perField := structpath.NewPatternDotString(pattern, "feature_name") + + // structdiff emits this for a member present on one side only. + member := structpath.NewDotString(nil, "materialized_features") + member = structpath.NewKeyValue(member, "feature_name", "main.ns.feat") + + assert.Equal(t, 3, perField.Len()) + assert.Equal(t, 2, member.Len()) + assert.False(t, member.HasPatternPrefix(perField), "member is shorter, so not a direct prefix match") + assert.True(t, matchesFieldRuleBidirectional(member, perField), + "whole-member change still matches the per-field rule via the truncated pattern") +} diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index d4885599805..1301e8c08b8 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -1265,6 +1265,18 @@ resources: "table_update": "description": |- PLACEHOLDER + "materialization_sets": + "description": |- + PLACEHOLDER + "$fields": + "lifecycle": + "description": |- + PLACEHOLDER + "materialized_features": + "$fields": + "cron_schedule": + "description": |- + PLACEHOLDER "mcp_services": "description": |- PLACEHOLDER diff --git a/bundle/internal/validation/generated/enum_fields.go b/bundle/internal/validation/generated/enum_fields.go index bffa6eb4df6..9661b12e0cd 100644 --- a/bundle/internal/validation/generated/enum_fields.go +++ b/bundle/internal/validation/generated/enum_fields.go @@ -169,6 +169,11 @@ var EnumFields = map[string][]string{ "resources.jobs.*.triggers[*].sql_condition.trigger_mode": {"QUERY_RETURNS_ROWS", "RESULT_VALUE_CHANGES"}, "resources.jobs.*.triggers[*].table_update.condition": {"ALL_UPDATED", "ANY_UPDATED"}, + "resources.materialization_sets.*.materialized_features[*].cron_schedule_trigger.mode": {"DERIVED", "MANUAL"}, + "resources.materialization_sets.*.materialized_features[*].pipeline_schedule_state": {"ACTIVE", "PAUSED", "SNAPSHOT"}, + "resources.materialization_sets.*.materialized_features[*].streaming_mode.mode": {"STREAMING_MODE_TYPE_MBM", "STREAMING_MODE_TYPE_RTM"}, + "resources.materialization_sets.*.materialized_features[*].table_trigger.condition": {"TABLE_TRIGGER_CONDITION_ALL_UPDATED", "TABLE_TRIGGER_CONDITION_ANY_UPDATED"}, + "resources.mcp_services.*.config.rate_limits[*].key": {"RATE_LIMIT_KEY_SERVICE", "RATE_LIMIT_KEY_SERVICE_PRINCIPAL", "RATE_LIMIT_KEY_USER", "RATE_LIMIT_KEY_USER_DEFAULT", "RATE_LIMIT_KEY_USER_GROUP"}, "resources.mcp_services.*.config.rate_limits[*].renewal_period": {"RATE_LIMIT_RENEWAL_PERIOD_HOUR", "RATE_LIMIT_RENEWAL_PERIOD_MINUTE"}, "resources.mcp_services.*.grants[*].privileges[*]": {"ACCESS", "ALL_PRIVILEGES", "APPLY_TAG", "BROWSE", "CREATE", "CREATE_CATALOG", "CREATE_CLEAN_ROOM", "CREATE_CONNECTION", "CREATE_EXTERNAL_LOCATION", "CREATE_EXTERNAL_TABLE", "CREATE_EXTERNAL_VOLUME", "CREATE_FOREIGN_CATALOG", "CREATE_FOREIGN_SECURABLE", "CREATE_FUNCTION", "CREATE_MANAGED_STORAGE", "CREATE_MATERIALIZED_VIEW", "CREATE_MODEL", "CREATE_PROVIDER", "CREATE_RECIPIENT", "CREATE_SCHEMA", "CREATE_SERVICE_CREDENTIAL", "CREATE_SHARE", "CREATE_STORAGE_CREDENTIAL", "CREATE_TABLE", "CREATE_VIEW", "CREATE_VOLUME", "EXECUTE", "EXECUTE_CLEAN_ROOM_TASK", "EXTERNAL_USE_LOCATION", "EXTERNAL_USE_SCHEMA", "MANAGE", "MANAGE_ALLOWLIST", "MODIFY", "MODIFY_CLEAN_ROOM", "READ_FILES", "READ_METADATA", "READ_PRIVATE_FILES", "READ_VOLUME", "REFRESH", "SELECT", "SET_SHARE_PERMISSION", "USAGE", "USE_CATALOG", "USE_CONNECTION", "USE_MARKETPLACE_ASSETS", "USE_PROVIDER", "USE_RECIPIENT", "USE_SCHEMA", "USE_SHARE", "WRITE_FILES", "WRITE_PRIVATE_FILES", "WRITE_VOLUME"}, diff --git a/bundle/internal/validation/generated/required_fields.go b/bundle/internal/validation/generated/required_fields.go index 5e139f34798..6fa10add589 100644 --- a/bundle/internal/validation/generated/required_fields.go +++ b/bundle/internal/validation/generated/required_fields.go @@ -271,6 +271,11 @@ var RequiredFields = map[string][]string{ "resources.jobs.*.webhook_notifications.on_streaming_backlog_exceeded[*]": {"id"}, "resources.jobs.*.webhook_notifications.on_success[*]": {"id"}, + "resources.materialization_sets.*.lifecycle.depends_on[*]": {"resource"}, + "resources.materialization_sets.*.materialized_features[*]": {"feature_name"}, + "resources.materialization_sets.*.materialized_features[*].offline_store_config": {"catalog_name", "schema_name", "table_name_prefix"}, + "resources.materialization_sets.*.materialized_features[*].online_store_config": {"catalog_name", "online_store_name", "schema_name", "table_name_prefix"}, + "resources.mcp_services.*": {"parent", "mcp_service_id"}, "resources.mcp_services.*.config.rate_limits[*]": {"key", "renewal_period"}, "resources.mcp_services.*.config.source_connection": {"name"}, diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index 4dbf0f2f839..8f7d42e2979 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -40,6 +40,7 @@ var deployApprovalGroups = []approvalGroup{ {group: "postgres_databases", message: deleteOrRecreatePostgresDatabaseMessage}, {group: "vector_search_indexes", message: deleteOrRecreateVectorSearchIndexMessage}, {group: "genie_spaces", message: deleteOrRecreateGenieSpaceMessage}, + {group: "materialization_sets", message: deleteOrRecreateMaterializationSetMessage}, } func approvalForDeploy(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan) (bool, error) { diff --git a/bundle/phases/messages.go b/bundle/phases/messages.go index 03eba35d8cf..bca18c53ab7 100644 --- a/bundle/phases/messages.go +++ b/bundle/phases/messages.go @@ -49,6 +49,11 @@ index drops all upserted vectors. Both can be expensive to rebuild:` deleteOrRecreateGenieSpaceMessage = ` This action will result in the deletion or recreation of the following Genie spaces. The conversation history attached to a deleted space is permanently lost:` + + deleteOrRecreateMaterializationSetMessage = ` +This action will result in the deletion or recreation of the following materialization sets. +Every materialized feature in a recreated set is dropped and recomputed from scratch, including +the members whose configuration did not change:` ) // DataLossWarning is the warning shown when a non-interactive command is about diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 959273d2586..fb51a13ede7 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -1723,6 +1723,36 @@ } ] }, + "resources.MaterializationSet": { + "oneOf": [ + { + "type": "object", + "description": "A group of materialized features created and managed as a single unit.", + "properties": { + "lifecycle": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.Lifecycle" + }, + "materialization_set_id": { + "description": "[Private Preview] Server-assigned unique identifier for the materialization set.", + "$ref": "#/$defs/string", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + }, + "materialized_features": { + "description": "[Private Preview] The materialized features in this set. Order is not significant.\nUpdating the set with a materialized feature removed from this set will cause it to be deleted.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/ml.MaterializedFeature", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\._*\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "resources.McpService": { "oneOf": [ { @@ -4203,6 +4233,9 @@ "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.Job", "markdownDescription": "The job definitions for the bundle, where each key is the name of the job. See [jobs](https://docs.databricks.com/dev-tools/bundles/resources.html#jobs)." }, + "materialization_sets": { + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.MaterializationSet" + }, "mcp_services": { "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.McpService" }, @@ -12835,6 +12868,58 @@ } ] }, + "ml.CronSchedule": { + "oneOf": [ + { + "type": "object", + "description": "A cron-based schedule trigger for the materialization pipeline.", + "properties": { + "cron_expression": { + "description": "The cron expression defining the schedule (e.g., \"0 0 * * *\" for daily at midnight). The\nschedule is interpreted in timezone_id (defaults to UTC). Required when mode is MANUAL (or\nunset). Left empty when mode is DERIVED, where the service computes it (aligned to UTC) from\nthe features' window timing and fills it in on the response.", + "$ref": "#/$defs/string", + "x-databricks-launch-stage": "GA" + }, + "mode": { + "description": "[Private Preview] How the schedule is determined. Defaults to MANUAL when unset.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/ml.CronScheduleMode", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + }, + "timezone_id": { + "description": "[Private Preview] A Java timezone ID. The schedule is resolved with respect to this timezone. Defaults to UTC\nwhen omitted. Can only be configured for MANUAL schedules; DERIVED schedules are always aligned\nto UTC.", + "$ref": "#/$defs/string", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\._*\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "ml.CronScheduleMode": { + "oneOf": [ + { + "type": "string", + "description": "The way a materialization schedule is arrived at.", + "enum": [ + "MANUAL", + "DERIVED" + ], + "enumDescriptions": [ + "[Private Preview]", + "[Private Preview]" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\._*\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "ml.CustomUdf": { "oneOf": [ { @@ -13569,6 +13654,199 @@ } ] }, + "ml.MaterializationFailureNotification": { + "oneOf": [ + { + "type": "object", + "description": "Recipients to notify when a materialization run fails.", + "properties": { + "destination_ids": { + "description": "[Private Preview] IDs of the notification destinations (for example Slack, Microsoft Teams, PagerDuty, or a\ngeneric webhook) to notify. Not supported for streaming materialized features.", + "$ref": "#/$defs/slice/string", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + }, + "email_addresses": { + "description": "[Private Preview] Email addresses to notify.", + "$ref": "#/$defs/slice/string", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + }, + "final_attempt_only": { + "description": "[Private Preview] If true, notify only when the final attempt of a run fails. If false or unset, notify on\nevery failed attempt, including attempts that will be retried.\n\nBatch materialization does not retry failures that need a fix on your side, such as missing\npermissions or invalid configuration, so the first attempt is the final one. Other batch\nfailures are retried up to twice.\n\nStreaming materialization restarts a failed update indefinitely unless the error cannot be\nretried. With this set, a streaming materialized feature notifies only on errors that cannot\nbe retried, and never on failures that are restarted.", + "$ref": "#/$defs/bool", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\._*\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "ml.MaterializationNotifications": { + "oneOf": [ + { + "type": "object", + "description": "Notifications for the jobs and pipelines that materialize a feature, one field per trigger.", + "properties": { + "on_failure": { + "description": "[Private Preview] Who to notify when a run fails, and on which attempts.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/ml.MaterializationFailureNotification", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + }, + "on_success": { + "description": "[Private Preview] Who to notify when a run succeeds.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/ml.MaterializationSuccessNotification", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\._*\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "ml.MaterializationSuccessNotification": { + "oneOf": [ + { + "type": "object", + "description": "Recipients to notify when a materialization run succeeds.", + "properties": { + "destination_ids": { + "description": "[Private Preview] IDs of the notification destinations (for example Slack, Microsoft Teams, PagerDuty, or a\ngeneric webhook) to notify. Not supported for streaming materialized features.", + "$ref": "#/$defs/slice/string", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + }, + "email_addresses": { + "description": "[Private Preview] Email addresses to notify.", + "$ref": "#/$defs/slice/string", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\._*\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "ml.MaterializedFeature": { + "oneOf": [ + { + "type": "object", + "description": "A materialized feature represents a feature that is continuously computed and stored.", + "properties": { + "budget_policy_id": { + "description": "[Private Preview] The ID of the budget policy used to attribute the serverless compute cost of this\nmaterialization. If not specified, a default budget policy may be applied.", + "$ref": "#/$defs/string", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + }, + "cron_schedule": { + "description": "[Private Preview]", + "$ref": "#/$defs/string", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "deprecationMessage": "This field is deprecated", + "doNotSuggest": true, + "deprecated": true + }, + "cron_schedule_trigger": { + "description": "A cron-based schedule trigger for the materialization pipeline.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/ml.CronSchedule", + "x-databricks-launch-stage": "GA" + }, + "feature_name": { + "description": "The full name of the feature in Unity Catalog.", + "$ref": "#/$defs/string", + "x-databricks-launch-stage": "GA" + }, + "materialized_feature_id": { + "description": "Server-assigned unique identifier for the materialized feature.", + "$ref": "#/$defs/string", + "x-databricks-launch-stage": "GA" + }, + "notifications": { + "description": "[Private Preview] Notification configuration around the materialization job or pipeline lifecycle. They are\napplied to every job and pipeline that materializes this feature. Features which are\nmaterialized in the same pipeline will share the same notification.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/ml.MaterializationNotifications", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + }, + "offline_store_config": { + "description": "Destination for writing feature values to an offline Delta table. The resulting table is\nreturned as `table_name`.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/ml.OfflineStoreConfig", + "x-databricks-launch-stage": "GA" + }, + "online_store_config": { + "description": "Destination for writing feature values to an online Lakebase table. The resulting table is\nreturned as `table_name`.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/ml.OnlineStoreConfig", + "x-databricks-launch-stage": "GA" + }, + "pipeline_schedule_state": { + "description": "[Private Preview] The schedule state of the materialization pipeline.\nHidden from GraphQL: being deprecated, so not exposed to Catalog Explorer.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/ml.MaterializedFeaturePipelineScheduleState", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + }, + "streaming_mode": { + "description": "The Structured Streaming trigger mode used for materialization. Real-time mode (RTM) targets\nsub-second latency for operational workloads; micro-batch mode (MBM) favors cost efficiency\nfor ETL and analytics workloads.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/ml.StreamingMode", + "x-databricks-launch-stage": "GA" + }, + "table_trigger": { + "description": "A trigger that fires when the upstream source table changes.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/ml.TableTrigger", + "x-databricks-launch-stage": "GA" + }, + "tags": { + "description": "[Private Preview] Custom tags to associate with this materialization. They are applied to the materialization\njob (for batch features) or pipeline (for streaming features) and forwarded to the underlying\ncompute as cluster tags, so materialization cost can be attributed in the billing system\ntables. These tags apply only to the materialization compute; they are not applied to the\nUnity Catalog Feature resource itself, whose tags are managed separately through the Unity\nCatalog tagging API. A maximum of 25 tags is supported; keys and values are subject to the\nsame limitations as cluster tags.", + "$ref": "#/$defs/map/string", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + } + }, + "additionalProperties": false, + "required": [ + "feature_name" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\._*\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "ml.MaterializedFeaturePipelineScheduleState": { + "oneOf": [ + { + "type": "string", + "enum": [ + "SNAPSHOT", + "ACTIVE", + "PAUSED" + ], + "enumDescriptions": [ + "[Private Preview]", + "[Private Preview]", + "[Private Preview]" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\._*\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "ml.MaxFunction": { "oneOf": [ { @@ -13640,6 +13918,82 @@ } ] }, + "ml.OfflineStoreConfig": { + "oneOf": [ + { + "type": "object", + "description": "Configuration for offline store destination.", + "properties": { + "catalog_name": { + "description": "The Unity Catalog catalog name.", + "$ref": "#/$defs/string", + "x-databricks-launch-stage": "GA" + }, + "schema_name": { + "description": "The Unity Catalog schema name.", + "$ref": "#/$defs/string", + "x-databricks-launch-stage": "GA" + }, + "table_name_prefix": { + "description": "Prefix for Unity Catalog table name.\nThe materialized feature will be stored in a table with this prefix and a generated postfix.", + "$ref": "#/$defs/string", + "x-databricks-launch-stage": "GA" + } + }, + "additionalProperties": false, + "required": [ + "catalog_name", + "schema_name", + "table_name_prefix" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\._*\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "ml.OnlineStoreConfig": { + "oneOf": [ + { + "type": "object", + "description": "Configuration for online store destination.", + "properties": { + "catalog_name": { + "description": "The Unity Catalog catalog name. This name is also used as the Lakebase logical database name.\nQuoting is handled by the backend where needed, do not pre-quote it.", + "$ref": "#/$defs/string", + "x-databricks-launch-stage": "GA" + }, + "online_store_name": { + "description": "The name of the target online store.", + "$ref": "#/$defs/string", + "x-databricks-launch-stage": "GA" + }, + "schema_name": { + "description": "The Unity Catalog schema name. This name is also used as the Lakebase schema name under the database.\nQuoting is handled by the backend where needed, do not pre-quote it.", + "$ref": "#/$defs/string", + "x-databricks-launch-stage": "GA" + }, + "table_name_prefix": { + "description": "Prefix for Unity Catalog table name.\nThe materialized feature will be stored in a Lakebase table with this prefix and a generated postfix.", + "$ref": "#/$defs/string", + "x-databricks-launch-stage": "GA" + } + }, + "additionalProperties": false, + "required": [ + "catalog_name", + "online_store_name", + "schema_name", + "table_name_prefix" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\._*\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "ml.RegisteredModelPermissionLevel": { "oneOf": [ { @@ -13981,6 +14335,53 @@ } ] }, + "ml.StreamingMode": { + "oneOf": [ + { + "type": "object", + "description": "The streaming mode configuration for a streaming materialization pipeline.", + "properties": { + "freshness_target": { + "description": "[Private Preview] The desired data freshness for feature materialization, expressed as a\nduration string (e.g. \"1 minute\").", + "$ref": "#/$defs/string", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + }, + "mode": { + "description": "The type of streaming mode used by the materialization pipeline.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/ml.StreamingModeStreamingModeType", + "x-databricks-launch-stage": "GA" + }, + "shuffle_partitions": { + "description": "[Private Preview] Number of shuffle partitions for streaming materialization of this feature.\nHigher values process high-throughput features with more parallelism at higher compute cost.\nMaterialized features which are computed together will use the largest value set among them.", + "$ref": "#/$defs/int", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\._*\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "ml.StreamingModeStreamingModeType": { + "oneOf": [ + { + "type": "string", + "enum": [ + "STREAMING_MODE_TYPE_RTM", + "STREAMING_MODE_TYPE_MBM" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\._*\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "ml.SumFunction": { "oneOf": [ { @@ -14004,6 +14405,59 @@ } ] }, + "ml.TableTrigger": { + "oneOf": [ + { + "type": "object", + "description": "A trigger that fires when the feature's source table changes: a DeltaTableSource's table, or an\nSdpSource's input_tables.", + "properties": { + "condition": { + "description": "[Private Preview] Which source table updates trigger a materialization run. If not set, any source table update\ntriggers a run.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/ml.TableTriggerTableTriggerCondition", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + }, + "min_time_between_triggers": { + "description": "[Private Preview] The minimum time between two triggered runs, so a frequently updated source table does not\ntrigger runs too often. If not set, a feature with a sliding window uses half its\nslide_duration and a feature with a tumbling window uses half its window_duration, clamped to\nbetween 5 minutes and 1 hour. Other features have no minimum.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/common/types/duration.Duration", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + }, + "wait_after_last_change": { + "description": "[Private Preview] How long the source table must go without updates before a run starts, so a series of\nupdates settles before the run. If not set, a run starts as soon as the source table updates.", + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/common/types/duration.Duration", + "x-databricks-launch-stage": "PRIVATE_PREVIEW", + "doNotSuggest": true + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\._*\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "ml.TableTriggerTableTriggerCondition": { + "oneOf": [ + { + "type": "string", + "description": "Which source table updates trigger a materialization run.", + "enum": [ + "TABLE_TRIGGER_CONDITION_ANY_UPDATED", + "TABLE_TRIGGER_CONDITION_ALL_UPDATED" + ], + "enumDescriptions": [ + "[Private Preview]", + "[Private Preview]" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\._*\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "ml.TimeWindow": { "oneOf": [ { @@ -20233,6 +20687,20 @@ } ] }, + "resources.MaterializationSet": { + "oneOf": [ + { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.MaterializationSet" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\._*\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "resources.McpService": { "oneOf": [ { @@ -21482,6 +21950,20 @@ } ] }, + "ml.MaterializedFeature": { + "oneOf": [ + { + "type": "array", + "items": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/ml.MaterializedFeature" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\._*\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "ml.ModelTag": { "oneOf": [ { diff --git a/bundle/statemgmt/state_load_test.go b/bundle/statemgmt/state_load_test.go index 4ec77578d2b..d55633c14ff 100644 --- a/bundle/statemgmt/state_load_test.go +++ b/bundle/statemgmt/state_load_test.go @@ -32,6 +32,7 @@ func TestStateToBundleEmptyLocalResources(t *testing.T) { "resources.models.test_mlflow_model": {ID: "1"}, "resources.experiments.test_mlflow_experiment": {ID: "1"}, "resources.features.test_feature": {ID: "main.default.test_feature"}, + "resources.materialization_sets.test_materialization_set": {ID: "set1"}, "resources.model_serving_endpoints.test_model_serving": {ID: "1"}, "resources.model_services.test_model_service": {ID: "main.default.test_model_service"}, "resources.mcp_services.test_mcp_service": {ID: "main.default.test_mcp_service"}, @@ -225,6 +226,15 @@ func TestStateToBundleEmptyRemoteResources(t *testing.T) { }, }, }, + MaterializationSets: map[string]*resources.MaterializationSet{ + "test_materialization_set": { + MaterializationSet: ml.MaterializationSet{ + MaterializedFeatures: []ml.MaterializedFeature{ + {FeatureName: "main.default.test_feature"}, + }, + }, + }, + }, ModelServingEndpoints: map[string]*resources.ModelServingEndpoint{ "test_model_serving": { CreateServingEndpoint: serving.CreateServingEndpoint{ @@ -664,6 +674,15 @@ func TestStateToBundleModifiedResources(t *testing.T) { }, }, }, + MaterializationSets: map[string]*resources.MaterializationSet{ + "test_materialization_set": { + MaterializationSet: ml.MaterializationSet{ + MaterializedFeatures: []ml.MaterializedFeature{ + {FeatureName: "main.default.test_feature"}, + }, + }, + }, + }, ModelServingEndpoints: map[string]*resources.ModelServingEndpoint{ "test_model_serving": { CreateServingEndpoint: serving.CreateServingEndpoint{ diff --git a/libs/testserver/fake_workspace.go b/libs/testserver/fake_workspace.go index 647bd4c38cd..6311eceaeb3 100644 --- a/libs/testserver/fake_workspace.go +++ b/libs/testserver/fake_workspace.go @@ -243,6 +243,8 @@ type FakeWorkspace struct { ModelProviderServices map[string]catalog.ModelProviderService Features map[string]ml.Feature + MaterializationSets map[string]ml.MaterializationSet + Operations map[string]ml.Operation ServingEndpoints map[string]serving.ServingEndpointDetailed VectorSearchEndpoints map[string]vectorsearch.EndpointInfo VectorSearchIndexes map[string]fakeVectorSearchIndex @@ -528,6 +530,8 @@ func NewFakeWorkspace(url, token string) *FakeWorkspace { McpServices: map[string]catalog.McpService{}, ModelProviderServices: map[string]catalog.ModelProviderService{}, Features: map[string]ml.Feature{}, + MaterializationSets: map[string]ml.MaterializationSet{}, + Operations: map[string]ml.Operation{}, Volumes: map[string]catalog.VolumeInfo{}, Dashboards: NewEventualMap[string, *fakeDashboard](eventualConsistency), PublishedDashboards: map[string]dashboards.PublishedDashboard{}, diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go index 73839619c3a..4ed1f05761c 100644 --- a/libs/testserver/handlers.go +++ b/libs/testserver/handlers.go @@ -883,6 +883,23 @@ func AddDefaultHandlers(server *Server) { return MapDelete(req.Workspace, req.Workspace.Features, req.Vars["full_name"]) }) + // Feature Engineering materialization sets: + server.Handle("POST", "/api/2.0/feature-engineering/materialization-sets", func(req Request) any { + return req.Workspace.MaterializationSetsCreate(req) + }) + server.Handle("GET", "/api/2.0/feature-engineering/operations/{operation_id}", func(req Request) any { + return req.Workspace.OperationsGet(req.Vars["operation_id"]) + }) + server.Handle("GET", "/api/2.0/feature-engineering/materialization-sets/{materialization_set_id}", func(req Request) any { + return MapGet(req.Workspace, req.Workspace.MaterializationSets, req.Vars["materialization_set_id"]) + }) + server.Handle("PATCH", "/api/2.0/feature-engineering/materialization-sets/{materialization_set_id}", func(req Request) any { + return req.Workspace.MaterializationSetsUpdate(req, req.Vars["materialization_set_id"]) + }) + server.Handle("DELETE", "/api/2.0/feature-engineering/materialization-sets/{materialization_set_id}", func(req Request) any { + return req.Workspace.MaterializationSetsDelete(req.Vars["materialization_set_id"]) + }) + // Model Provider Services (AI Gateway): server.Handle("POST", "/api/2.1/unity-catalog/model-provider-services", func(req Request) any { diff --git a/libs/testserver/materialization_set.go b/libs/testserver/materialization_set.go new file mode 100644 index 00000000000..5e71ed1da20 --- /dev/null +++ b/libs/testserver/materialization_set.go @@ -0,0 +1,765 @@ +package testserver + +import ( + "encoding/json" + "fmt" + "net/http" + "slices" + "strings" + + "github.com/databricks/databricks-sdk-go/service/ml" +) + +// This fake mirrors UpdateMaterializationSet closely enough that one set of acceptance goldens +// serves both the local run and a run against a real workspace. Where it rejects, it returns the +// backend's own message text, so a golden recorded locally is a claim about the real API that the +// cloud run then checks. + +// materializationSetRefusal is a rule's verdict, separated from how it is reported: a rule the +// backend reaches before reserving an operation record becomes an HTTP error, and one it reaches +// after becomes a settled FAILED operation. +type materializationSetRefusal struct { + errorCode string + message string +} + +func materializationSetError(statusCode int, errorCode, message string) Response { + return Response{ + StatusCode: statusCode, + Body: map[string]string{ + "error_code": errorCode, + "message": message, + }, + } +} + +// quotedMaterializedFeatureIDs renders an id list the way FeatureStoreMessages.quotedIds does. +func quotedMaterializedFeatureIDs(ids []string) string { + quoted := make([]string, len(ids)) + for i, id := range ids { + quoted[i] = "'" + id + "'" + } + return strings.Join(quoted, ", ") +} + +// materializedFeatureFieldCannotBeChanged refuses a field the set update cannot write. No +// acceptance case reaches it: the planner classifies every member field as a recreate, so a +// declared change never becomes an update request. +func materializedFeatureFieldCannotBeChanged( + featureName, materializedFeatureID, fieldPath string, +) Response { + return materializationSetError(http.StatusBadRequest, "INVALID_PARAMETER_VALUE", + fmt.Sprintf("The fields '%s' of materialized feature '%s' of feature '%s' cannot be "+ + "changed. Only removing members is supported; recreate the materialization set "+ + "instead.", fieldPath, materializedFeatureID, featureName)) +} + +// materializationSetMemberKey matches a requested member to a stored one the way the backend does: +// by feature name, online store and whether the trigger is streaming. Feature name alone is not +// unique because one feature has an offline member plus one member per online store. +func materializationSetMemberKey(member ml.MaterializedFeature) string { + onlineStoreName := "" + if member.OnlineStoreConfig != nil { + onlineStoreName = member.OnlineStoreConfig.OnlineStoreName + } + return fmt.Sprintf("%s|%s|%t", member.FeatureName, onlineStoreName, member.StreamingMode != nil) +} + +// effectivePipelineScheduleState resolves the create-path default, so a request mixing unset and +// ACTIVE members is not read as a disagreement. +func effectivePipelineScheduleState( + state ml.MaterializedFeaturePipelineScheduleState, +) ml.MaterializedFeaturePipelineScheduleState { + if state == "" { + return ml.MaterializedFeaturePipelineScheduleStateActive + } + return state +} + +// populateMaterializationSetMember fills the fields the backend assigns on create, then redacts +// the INPUT_ONLY destination fields the way the response path does. Storing them redacted is what +// makes a read here match a read from a real workspace, which an echoing fake hides. +func populateMaterializationSetMember(member ml.MaterializedFeature, setID string) ml.MaterializedFeature { + member.MaterializedFeatureId = nextUUID() + member.MaterializationSetId = setID + member.IsOnline = member.OnlineStoreConfig != nil + member.PipelineScheduleState = effectivePipelineScheduleState(member.PipelineScheduleState) + member.OfflineStoreConfig = nil + if member.OnlineStoreConfig != nil { + member.OnlineStoreConfig = &ml.OnlineStoreConfig{ + OnlineStoreName: member.OnlineStoreConfig.OnlineStoreName, + } + } + return member +} + +// validateBatchPipelineScheduleStates mirrors the create-path rule that every batch member agrees +// on one effective schedule state. Streaming members never carry one, so they are excluded. +func validateBatchPipelineScheduleStates(members []ml.MaterializedFeature) *Response { + states := map[ml.MaterializedFeaturePipelineScheduleState]bool{} + for _, member := range members { + if member.StreamingMode != nil { + continue + } + states[effectivePipelineScheduleState(member.PipelineScheduleState)] = true + } + if len(states) > 1 { + response := materializationSetError(http.StatusBadRequest, "INVALID_PARAMETER_VALUE", + "All materialized features in a BatchCreateMaterializedFeaturesRequest call must "+ + "have the same pipeline schedule state.") + return &response + } + return nil +} + +// validateSingleCronScheduleMode mirrors the create-path rule that one request carries one cron +// schedule mode. A sawtooth feature must be DERIVED and a batch-only feature MANUAL, so this is +// what a set holding both runs into first. +func validateSingleCronScheduleMode(members []ml.MaterializedFeature) *Response { + modes := map[ml.CronScheduleMode]bool{} + for _, member := range members { + if member.CronScheduleTrigger == nil { + continue + } + modes[member.CronScheduleTrigger.Mode] = true + } + if len(modes) > 1 { + response := materializationSetError(http.StatusBadRequest, "INVALID_PARAMETER_VALUE", + "All cron-scheduled features in a single request must use the same cron schedule "+ + "mode; a request cannot mix DERIVED (service-derived) and MANUAL "+ + "(caller-supplied) schedules.") + return &response + } + return nil +} + +// featureMaterializationDestination renders the destination pair the create compares across +// features: a feature's offline config plus the online config it is also materialized to, if any. +func featureMaterializationDestination(members []ml.MaterializedFeature) string { + offline, online := "", "" + for _, member := range members { + if cfg := member.OfflineStoreConfig; cfg != nil { + offline = cfg.CatalogName + "." + cfg.SchemaName + "." + cfg.TableNamePrefix + } + if cfg := member.OnlineStoreConfig; cfg != nil { + online = cfg.OnlineStoreName + ":" + cfg.CatalogName + "." + cfg.SchemaName + "." + + cfg.TableNamePrefix + } + } + return offline + "|" + online +} + +// validateSharedMaterializationDestination mirrors the create-path rule that every feature in a +// set asks for an identical destination pair. Because the pair carries the online config, a set +// cannot mix a feature materialized to both stores with one materialized offline only. +func validateSharedMaterializationDestination(members []ml.MaterializedFeature) *materializationSetRefusal { + membersByFeature := map[string][]ml.MaterializedFeature{} + var featureOrder []string + for _, member := range members { + if _, seen := membersByFeature[member.FeatureName]; !seen { + featureOrder = append(featureOrder, member.FeatureName) + } + membersByFeature[member.FeatureName] = append(membersByFeature[member.FeatureName], member) + } + + destinations := map[string]bool{} + for _, featureName := range featureOrder { + destinations[featureMaterializationDestination(membersByFeature[featureName])] = true + } + if len(destinations) > 1 { + return &materializationSetRefusal{ + errorCode: "BAD_REQUEST", + message: "All features must share the same materialization destination.", + } + } + return validateOfflineAndOnlineDestinationsDiffer(members) +} + +// validateOfflineAndOnlineDestinationsDiffer mirrors the create-path rule that a feature +// materialized to both stores must separate them by catalog, schema or prefix. +func validateOfflineAndOnlineDestinationsDiffer(members []ml.MaterializedFeature) *materializationSetRefusal { + var offline *ml.OfflineStoreConfig + var online *ml.OnlineStoreConfig + for _, member := range members { + if member.OfflineStoreConfig != nil { + offline = member.OfflineStoreConfig + } + if member.OnlineStoreConfig != nil { + online = member.OnlineStoreConfig + } + } + if offline == nil || online == nil { + return nil + } + if online.CatalogName != offline.CatalogName || online.SchemaName != offline.SchemaName || + online.TableNamePrefix != offline.TableNamePrefix { + return nil + } + return &materializationSetRefusal{ + errorCode: "BAD_REQUEST", + message: fmt.Sprintf("Offline and online materialization destinations must differ. Found "+ + "identical catalog names '%s', schema names '%s', and table name prefixes '%s'.", + offline.CatalogName, offline.SchemaName, offline.TableNamePrefix), + } +} + +// sawtoothMaterializedFeatureCount is how many materialized features a sawtooth window produces: +// an offline batch one, an online batch one and an online streaming one. +const sawtoothMaterializedFeatureCount = 3 + +// isSawtoothFeature reports whether a stored feature aggregates over a sawtooth window. An unknown +// feature is not treated as sawtooth: the real create resolves the feature first and fails on a +// missing one, which is a separate rejection this fake does not model. +func (s *FakeWorkspace) isSawtoothFeature(featureName string) bool { + feature, stored := s.Features[featureName] + if !stored { + return false + } + aggregation := feature.Function.AggregationFunction + return aggregation != nil && aggregation.TimeWindow != nil && aggregation.TimeWindow.Sawtooth != nil +} + +// validateBatchAndStreamingFeaturesNotMixed mirrors the create-path rule that one call materializes +// either batch features or streaming features. A Stream-sourced feature always materializes on the +// streaming path; a Delta-sourced one joins it only when a member of it carries a streaming +// trigger. An unknown feature falls in neither group, for the reason isSawtoothFeature gives. +func (s *FakeWorkspace) validateBatchAndStreamingFeaturesNotMixed( + members []ml.MaterializedFeature, +) *materializationSetRefusal { + var batchFeatureNames, streamingFeatureNames []string + for _, featureName := range materializationSetFeatureNames(members) { + feature, stored := s.Features[featureName] + if !stored { + continue + } + hasStreamingMember := false + for _, member := range members { + if member.FeatureName == featureName && member.StreamingMode != nil { + hasStreamingMember = true + } + } + switch { + case feature.Source.StreamSource != nil || hasStreamingMember: + streamingFeatureNames = append(streamingFeatureNames, featureName) + case feature.Source.DeltaTableSource != nil: + batchFeatureNames = append(batchFeatureNames, featureName) + } + } + if len(batchFeatureNames) == 0 || len(streamingFeatureNames) == 0 { + return nil + } + + return &materializationSetRefusal{ + errorCode: "INVALID_PARAMETER_VALUE", + message: fmt.Sprintf("Batch and streaming features cannot be mixed in a single "+ + "materialize_features call because they require different trigger types. Issue two "+ + "separate calls: one for batch features and one for streaming features. Batch "+ + "features: %s. Streaming features: %s", + strings.Join(batchFeatureNames, ", "), strings.Join(streamingFeatureNames, ", ")), + } +} + +// isDerivedCronSchedule mirrors MaterializedFeatureUtils.isDerivedCronSchedule: only a cron +// schedule trigger carrying DERIVED opts out of a pinned schedule, and an unset mode reads as +// MANUAL. +func isDerivedCronSchedule(member ml.MaterializedFeature) bool { + return member.CronScheduleTrigger != nil && + member.CronScheduleTrigger.Mode == ml.CronScheduleModeDerived +} + +// validateSawtoothMaterializedFeatures mirrors the rule that a sawtooth feature declares all three +// of its materialized features itself. The request is not expanded for it: a materialization set +// stamps a set id on every member before the backend reaches its normalization, and that +// normalization is skipped whenever the id is set, so a member maps to exactly one materialized +// feature and the set stays diffable. +func (s *FakeWorkspace) validateSawtoothMaterializedFeatures( + members []ml.MaterializedFeature, +) *materializationSetRefusal { + byFeature := map[string][]ml.MaterializedFeature{} + var featureOrder []string + for _, member := range members { + if !s.isSawtoothFeature(member.FeatureName) { + continue + } + if _, seen := byFeature[member.FeatureName]; !seen { + featureOrder = append(featureOrder, member.FeatureName) + } + byFeature[member.FeatureName] = append(byFeature[member.FeatureName], member) + } + + for _, featureName := range featureOrder { + var offlineBatch, onlineBatch, onlineStreaming int + for _, member := range byFeature[featureName] { + switch { + case member.OnlineStoreConfig != nil && member.StreamingMode != nil: + onlineStreaming++ + case member.OnlineStoreConfig != nil: + onlineBatch++ + case member.OfflineStoreConfig != nil && member.StreamingMode == nil: + offlineBatch++ + } + } + if len(byFeature[featureName]) != sawtoothMaterializedFeatureCount || + offlineBatch != 1 || onlineBatch != 1 || onlineStreaming != 1 { + return &materializationSetRefusal{ + errorCode: "BAD_REQUEST", + message: fmt.Sprintf( + "Feature '%s' uses a sawtooth window, which requires an offline batch "+ + "materialization, an online batch materialization and an online streaming "+ + "materialization. Declare one materialize feature request for each.", + featureName), + } + } + + for _, member := range byFeature[featureName] { + if member.StreamingMode != nil || isDerivedCronSchedule(member) { + continue + } + return &materializationSetRefusal{ + errorCode: "BAD_REQUEST", + message: fmt.Sprintf( + "Feature '%s' must declare a DERIVED cron schedule on its batch materialize "+ + "feature requests. The sawtooth batch materializations refresh on a daily "+ + "schedule the service computes, so a caller neither pins nor omits it.", + featureName), + } + } + } + return nil +} + +// MaterializationSetsCreate fakes POST /api/2.0/feature-engineering/materialization-sets. +func (s *FakeWorkspace) MaterializationSetsCreate(req Request) Response { + defer s.LockUnlock()() + + var set ml.MaterializationSet + if err := json.Unmarshal(req.Body, &set); err != nil { + return materializationSetInternalError(err) + } + + if len(set.MaterializedFeatures) == 0 { + return materializationSetError(http.StatusBadRequest, "INVALID_PARAMETER_VALUE", + "A materialization set requires at least one materialized feature.") + } + if bad := validateBatchPipelineScheduleStates(set.MaterializedFeatures); bad != nil { + return *bad + } + if bad := validateSingleCronScheduleMode(set.MaterializedFeatures); bad != nil { + return *bad + } + set.MaterializationSetId = nextUUID() + + // The operation record is reserved here, so what follows is the operation's outcome rather + // than a rejected request: the backend reaches these rules inside the batch create it calls + // after reserving, and reports them by settling the record. + for _, bad := range []*materializationSetRefusal{ + s.validateBatchAndStreamingFeaturesNotMixed(set.MaterializedFeatures), + validateSharedMaterializationDestination(set.MaterializedFeatures), + s.validateSawtoothMaterializedFeatures(set.MaterializedFeatures), + } { + if bad == nil { + continue + } + return s.materializationSetFailedOperationResponse(set.MaterializationSetId, + set.MaterializedFeatures, bad.errorCode, bad.message, nil) + } + for i, member := range set.MaterializedFeatures { + set.MaterializedFeatures[i] = populateMaterializationSetMember(member, set.MaterializationSetId) + } + + s.MaterializationSets[set.MaterializationSetId] = set + + return s.materializationSetOperationResponse(set.MaterializationSetId, set.MaterializedFeatures, set) +} + +// materializationSetOperationResponse records the terminal operation a set change returns and +// renders it as the endpoint's response. The work is finished by the time the response is written, +// so the operation is recorded already terminal; a caller still has to poll it, because a create's +// or update's resulting set is reachable only from there. +func (s *FakeWorkspace) materializationSetOperationResponse( + setID string, + members []ml.MaterializedFeature, + result any, +) Response { + metadata, err := json.Marshal(ml.MaterializationSetOperationMetadata{ + FeatureFullNames: materializationSetFeatureNames(members), + MaterializationSetId: setID, + State: ml.MaterializationSetOperationMetadataStateSucceeded, + }) + if err != nil { + return materializationSetInternalError(err) + } + response, err := json.Marshal(result) + if err != nil { + return materializationSetInternalError(err) + } + + operation := ml.Operation{ + Name: "operations/" + nextUUID(), + Done: true, + Metadata: metadata, + Response: response, + } + s.Operations[operation.Name] = operation + return Response{Body: operation} +} + +// materializedFeatureTeardownFailsMarker is how a fixture asks the fake for a member the backend +// cannot tear down. A real teardown fails on infrastructure -- a job, a pipeline or a table the +// delete cannot reach -- which no request can ask for, so the trigger has to be in the fixture. +const materializedFeatureTeardownFailsMarker = "teardown_fails" + +// materializationSetTeardownFailures reports the members of `targets` the fake refuses to tear +// down, one failure each, in target order. +func materializationSetTeardownFailures( + targets [][]ml.MaterializedFeature, + removedIDs map[string]bool, +) []ml.MaterializedFeatureChangeFailure { + var failures []ml.MaterializedFeatureChangeFailure + for _, target := range targets { + if !strings.Contains(target[0].FeatureName, materializedFeatureTeardownFailsMarker) { + continue + } + for _, member := range target { + if !removedIDs[member.MaterializedFeatureId] { + continue + } + failures = append(failures, ml.MaterializedFeatureChangeFailure{ + MaterializedFeatureId: member.MaterializedFeatureId, + ErrorCode: "INTERNAL_ERROR", + Message: fmt.Sprintf( + "Failed to delete materialized feature '%s'. If the issue persists, please "+ + "retry or contact support.", member.MaterializedFeatureId), + }) + } + } + return failures +} + +// partialMaterializationSetChangeErrorCode collapses the member failures to one code the way the +// backend does: a single code is reported as itself, a permission failure wins over any other, and +// a mix of client errors becomes BAD_REQUEST. +func partialMaterializationSetChangeErrorCode( + failures []ml.MaterializedFeatureChangeFailure, +) string { + codes := map[string]bool{} + for _, failure := range failures { + codes[failure.ErrorCode] = true + } + switch { + case len(codes) == 1: + for code := range codes { + return code + } + return "INTERNAL_ERROR" + case codes["PERMISSION_DENIED"]: + return "PERMISSION_DENIED" + default: + return "INTERNAL_ERROR" + } +} + +// materializationSetFailedOperationResponse settles an operation as FAILED and returns it with +// HTTP 200, which is how the backend reports a change it reserved an operation for and then could +// not finish. The per-member outcome rides on the metadata's progress arm. +func (s *FakeWorkspace) materializationSetFailedOperationResponse( + setID string, + members []ml.MaterializedFeature, + errorCode string, + message string, + progress func(*ml.MaterializationSetOperationMetadata), +) Response { + settled := ml.MaterializationSetOperationMetadata{ + FeatureFullNames: materializationSetFeatureNames(members), + MaterializationSetId: setID, + State: ml.MaterializationSetOperationMetadataStateFailed, + } + if progress != nil { + progress(&settled) + } + + metadata, err := json.Marshal(settled) + if err != nil { + return materializationSetInternalError(err) + } + + operation := ml.Operation{ + Name: "operations/" + nextUUID(), + Done: true, + Metadata: metadata, + Error: &ml.DatabricksServiceExceptionWithDetailsProto{ + ErrorCode: ml.ErrorCode(errorCode), + Message: message, + }, + } + s.Operations[operation.Name] = operation + return Response{Body: operation} +} + +// materializationSetFeatureNames lists each member's feature once in member order, which is what +// the operation metadata reports. +func materializationSetFeatureNames(members []ml.MaterializedFeature) []string { + seen := map[string]bool{} + var names []string + for _, member := range members { + if !seen[member.FeatureName] { + seen[member.FeatureName] = true + names = append(names, member.FeatureName) + } + } + return names +} + +func materializationSetInternalError(err error) Response { + return Response{ + Body: fmt.Sprintf("internal error: %s", err), + StatusCode: http.StatusInternalServerError, + } +} + +// materializationSetNotFound refuses a set id no set holds. +func materializationSetNotFound(setID string) Response { + return materializationSetError(http.StatusNotFound, "NOT_FOUND", + fmt.Sprintf("Materialization set with ID '%s' does not exist.", setID)) +} + +// MaterializationSetsDelete fakes DELETE +// /api/2.0/feature-engineering/materialization-sets/{materialization_set_id}. The delete returns a +// long-running operation too, settling with an empty response. +func (s *FakeWorkspace) MaterializationSetsDelete(setID string) Response { + defer s.LockUnlock()() + + existing, ok := s.MaterializationSets[setID] + if !ok { + return materializationSetNotFound(setID) + } + + allIDs := map[string]bool{} + for _, member := range existing.MaterializedFeatures { + allIDs[member.MaterializedFeatureId] = true + } + targets := materializationSetDeletionTargets(existing.MaterializedFeatures) + if failures := materializationSetTeardownFailures(targets, allIDs); len(failures) > 0 { + // The set is kept so the members that still need deleting can be retried. + return s.materializationSetFailedOperationResponse(setID, existing.MaterializedFeatures, + partialMaterializationSetChangeErrorCode(failures), + fmt.Sprintf("Deleted %d of %d materialized features in materialization set '%s'. "+ + "The operation's progress lists which deletion succeeded, which failed and why.", + 0, len(failures), setID), + func(metadata *ml.MaterializationSetOperationMetadata) { + metadata.DeletionProgress = &ml.MaterializationSetDeletionProgress{ + FailedMaterializedFeatures: failures, + } + }) + } + + delete(s.MaterializationSets, setID) + + return s.materializationSetOperationResponse(setID, existing.MaterializedFeatures, struct{}{}) +} + +// OperationsGet fakes GET /api/2.0/feature-engineering/operations/{operation_id}. +func (s *FakeWorkspace) OperationsGet(operationID string) Response { + defer s.LockUnlock()() + + operation, ok := s.Operations["operations/"+operationID] + if !ok { + // Unlike every other message here, this wording is not taken from the backend: an unknown + // operation is resolved inside the entity-store client rather than from + // FeatureStoreMessages. No case reaches this branch, so nothing pins it either way. + return materializationSetError(http.StatusNotFound, "NOT_FOUND", + fmt.Sprintf("Operation with name 'operations/%s' does not exist.", operationID)) + } + return Response{Body: operation} +} + +// effectiveCronScheduleMode treats an unset mode as MANUAL, so spelling MANUAL out is no change. +func effectiveCronScheduleMode(mode ml.CronScheduleMode) ml.CronScheduleMode { + if mode == "" { + return ml.CronScheduleModeManual + } + return mode +} + +// isCronScheduleChangeRequested mirrors the backend's comparison. The mode is always compared, so +// dropping DERIVED reads as asking to stop deriving; a DERIVED schedule's expression and timezone +// belong to the service, so leaving them out asks for nothing. That is what keeps a sawtooth +// feature updatable, where comparing the field as a whole would reject every update after create. +func isCronScheduleChangeRequested(requested, existing *ml.CronSchedule) bool { + if requested == nil || existing == nil { + return false + } + requestedMode := effectiveCronScheduleMode(requested.Mode) + if requestedMode != effectiveCronScheduleMode(existing.Mode) { + return true + } + if requestedMode == ml.CronScheduleModeDerived { + if requested.CronExpression != "" && requested.CronExpression != existing.CronExpression { + return true + } + } else if requested.CronExpression != existing.CronExpression { + return true + } + return requested.TimezoneId != "" && requested.TimezoneId != existing.TimezoneId +} + +// materializationSetDeletionTargets groups each feature's members into the one a delete is applied +// to and the ones it cascades to. Deleting a feature's offline member deletes its online ones; a +// feature with no offline member is deleted through each online one. +func materializationSetDeletionTargets(members []ml.MaterializedFeature) [][]ml.MaterializedFeature { + byFeature := map[string][]ml.MaterializedFeature{} + var featureOrder []string + for _, member := range members { + if _, seen := byFeature[member.FeatureName]; !seen { + featureOrder = append(featureOrder, member.FeatureName) + } + byFeature[member.FeatureName] = append(byFeature[member.FeatureName], member) + } + + var targets [][]ml.MaterializedFeature + for _, featureName := range featureOrder { + var offline, online []ml.MaterializedFeature + for _, member := range byFeature[featureName] { + if member.IsOnline { + online = append(online, member) + } else { + offline = append(offline, member) + } + } + if len(offline) == 0 { + for _, member := range online { + targets = append(targets, []ml.MaterializedFeature{member}) + } + continue + } + // Assumes one offline destination per feature, as the backend does. + targets = append(targets, append([]ml.MaterializedFeature{offline[0]}, online...)) + } + return targets +} + +// MaterializationSetsUpdate fakes PATCH +// /api/2.0/feature-engineering/materialization-sets/{materialization_set_id}. Field updates are +// validated before any deletion, so the destructive half is never attempted on a set that could +// not be fully updated. +func (s *FakeWorkspace) MaterializationSetsUpdate(req Request, setID string) Response { + defer s.LockUnlock()() + + existing, ok := s.MaterializationSets[setID] + if !ok { + return materializationSetNotFound(setID) + } + + var requested ml.MaterializationSet + if err := json.Unmarshal(req.Body, &requested); err != nil { + return materializationSetInternalError(err) + } + + if len(requested.MaterializedFeatures) == 0 { + return materializationSetError(http.StatusBadRequest, "INVALID_PARAMETER_VALUE", + fmt.Sprintf("Updating materialization set '%s' with no materialized features is not "+ + "supported. Delete the set instead.", setID)) + } + + storedByKey := map[string]ml.MaterializedFeature{} + for _, member := range existing.MaterializedFeatures { + storedByKey[materializationSetMemberKey(member)] = member + } + + // Match each requested member to a stored one. A requested member matching nothing is an + // addition, which this endpoint does not support. + matched := make([]ml.MaterializedFeature, 0, len(requested.MaterializedFeatures)) + requestedKeys := map[string]bool{} + for _, member := range requested.MaterializedFeatures { + stored, found := storedByKey[materializationSetMemberKey(member)] + if !found { + return materializationSetError(http.StatusBadRequest, "INVALID_PARAMETER_VALUE", + fmt.Sprintf("Materialized features cannot be added to an existing materialization "+ + "set, and feature '%s' does not match any member of this set. Recreate the "+ + "set instead.", member.FeatureName)) + } + if isCronScheduleChangeRequested(member.CronScheduleTrigger, stored.CronScheduleTrigger) { + return materializedFeatureFieldCannotBeChanged(member.FeatureName, + stored.MaterializedFeatureId, "trigger") + } + // An unset state asks for no change: the server falls back to the stored one before + // comparing, so a member the config never declares is not read as a change. + if member.PipelineScheduleState != "" && + member.PipelineScheduleState != stored.PipelineScheduleState { + return materializedFeatureFieldCannotBeChanged(member.FeatureName, + stored.MaterializedFeatureId, "pipeline_schedule_state") + } + requestedKeys[materializationSetMemberKey(member)] = true + matched = append(matched, stored) + } + + // Delete the members the request omits. Because a delete cascades from a feature's offline + // member to its online ones, a removal that would also reach a member the request still lists + // cannot be expressed and is refused before anything is written. The server groups the removed + // members by feature and names that feature's removed ids, so the message is built the same way. + removedIDs := map[string]bool{} + featureNameToRemovedIDs := map[string][]string{} + for _, member := range existing.MaterializedFeatures { + if !requestedKeys[materializationSetMemberKey(member)] { + removedIDs[member.MaterializedFeatureId] = true + featureNameToRemovedIDs[member.FeatureName] = append( + featureNameToRemovedIDs[member.FeatureName], member.MaterializedFeatureId) + } + } + for _, target := range materializationSetDeletionTargets(existing.MaterializedFeatures) { + var unrequested []string + requestedForChange := false + for _, member := range target { + if removedIDs[member.MaterializedFeatureId] { + requestedForChange = true + } else { + unrequested = append(unrequested, member.MaterializedFeatureId) + } + } + // A target the caller asked nothing about is not in play. + if !requestedForChange || len(unrequested) == 0 { + continue + } + featureName := target[0].FeatureName + removedForFeature := append([]string(nil), featureNameToRemovedIDs[featureName]...) + slices.Sort(removedForFeature) + // The server has a message for each direction. target[0] is the member a delete is applied + // to, so naming it means the delete reaches past what the request asked for, and omitting + // it means the named members have no delete of their own. + if removedIDs[target[0].MaterializedFeatureId] { + return materializationSetError(http.StatusBadRequest, "INVALID_PARAMETER_VALUE", + fmt.Sprintf("Deleting materialized features %s of feature '%s' also deletes "+ + "materialized features %s, which the request does not delete. Delete the "+ + "unrequested materialized features as well.", + quotedMaterializedFeatureIDs(removedForFeature), featureName, + quotedMaterializedFeatureIDs(unrequested))) + } + return materializationSetError(http.StatusBadRequest, "INVALID_PARAMETER_VALUE", + fmt.Sprintf("Materialized features %s of feature '%s' are deleted only by deleting "+ + "materialized features %s, which the request does not supply. Remove those as "+ + "well.", + quotedMaterializedFeatureIDs(removedForFeature), featureName, + quotedMaterializedFeatureIDs(unrequested))) + } + + targets := materializationSetDeletionTargets(existing.MaterializedFeatures) + if failures := materializationSetTeardownFailures(targets, removedIDs); len(failures) > 0 { + // Nothing is written: the one target this fake can fail is the only target, so no member + // of the change got through. + return s.materializationSetFailedOperationResponse(setID, existing.MaterializedFeatures, + partialMaterializationSetChangeErrorCode(failures), + fmt.Sprintf("Changed %d of %d materialized features in materialization set '%s'. "+ + "The operation's progress lists which update succeeded, which failed and why.", + 0, len(failures), setID), + func(metadata *ml.MaterializationSetOperationMetadata) { + metadata.UpdateProgress = &ml.MaterializationSetUpdateProgress{ + FailedMaterializedFeatures: failures, + } + }) + } + + existing.MaterializedFeatures = matched + s.MaterializationSets[setID] = existing + + return s.materializationSetOperationResponse(setID, existing.MaterializedFeatures, existing) +} diff --git a/python/databricks/bundles/core/__init__.py b/python/databricks/bundles/core/__init__.py index b40cb9f9684..817f890ed51 100644 --- a/python/databricks/bundles/core/__init__.py +++ b/python/databricks/bundles/core/__init__.py @@ -32,6 +32,7 @@ "load_resources_from_module", "load_resources_from_modules", "load_resources_from_package_module", + "materialization_set_mutator", "mcp_service_mutator", "mlflow_experiment_mutator", "mlflow_model_mutator", @@ -73,6 +74,7 @@ instance_pool_mutator, job_mutator, job_run_mutator, + materialization_set_mutator, mcp_service_mutator, mlflow_experiment_mutator, mlflow_model_mutator, diff --git a/python/databricks/bundles/core/_generated/__init__.py b/python/databricks/bundles/core/_generated/__init__.py index 6a56d2eb856..8c291d0a244 100644 --- a/python/databricks/bundles/core/_generated/__init__.py +++ b/python/databricks/bundles/core/_generated/__init__.py @@ -53,6 +53,10 @@ job_run_mutator, ) from databricks.bundles.core._generated.jobs import _JobResources, job_mutator +from databricks.bundles.core._generated.materialization_sets import ( + _MaterializationSetResources, + materialization_set_mutator, +) from databricks.bundles.core._generated.mcp_services import ( _McpServiceResources, mcp_service_mutator, @@ -129,6 +133,7 @@ "instance_pool_mutator", "job_mutator", "job_run_mutator", + "materialization_set_mutator", "mcp_service_mutator", "mlflow_experiment_mutator", "mlflow_model_mutator", @@ -165,6 +170,7 @@ class _GeneratedResources( _InstancePoolResources, _JobRunResources, _JobResources, + _MaterializationSetResources, _McpServiceResources, _ModelProviderServiceResources, _ModelServiceResources, @@ -202,6 +208,7 @@ def _all_resource_types() -> "tuple[_ResourceType, ...]": instance_pools, job_runs, jobs, + materialization_sets, mcp_services, model_provider_services, model_services, @@ -236,6 +243,7 @@ def _all_resource_types() -> "tuple[_ResourceType, ...]": instance_pools._resource_type(), job_runs._resource_type(), jobs._resource_type(), + materialization_sets._resource_type(), mcp_services._resource_type(), model_provider_services._resource_type(), model_services._resource_type(), diff --git a/python/databricks/bundles/core/_generated/materialization_sets.py b/python/databricks/bundles/core/_generated/materialization_sets.py new file mode 100644 index 00000000000..281ffe5c8e9 --- /dev/null +++ b/python/databricks/bundles/core/_generated/materialization_sets.py @@ -0,0 +1,126 @@ +# Code generated by pydabs-codegen. DO NOT EDIT. + +from collections.abc import Callable +from typing import TYPE_CHECKING, Optional, overload + +from databricks.bundles.core._bundle import Bundle +from databricks.bundles.core._location import Location +from databricks.bundles.core._resource_mutator import ResourceMutator +from databricks.bundles.core._transform import _transform + +if TYPE_CHECKING: + from databricks.bundles.core._resource_type import _ResourceType + from databricks.bundles.materialization_sets._models.materialization_set import ( + MaterializationSet, + MaterializationSetParam, + ) + + +def _resource_type() -> "_ResourceType": + from databricks.bundles.core._resource_type import _ResourceType + from databricks.bundles.materialization_sets._models.materialization_set import ( + MaterializationSet, + ) + + return _ResourceType( + resource_type=MaterializationSet, + singular_name="materialization_set", + plural_name="materialization_sets", + ) + + +class _MaterializationSetResources: + """ + Generated materialization_set accessors, mixed into Resources. + """ + + # Provided by the Resources subclass; declared here so the generated methods + # below type-check. + _resources: dict[str, dict] + + if TYPE_CHECKING: + + def add_location(self, path: tuple[str, ...], location: Location) -> None: ... + + def add_diagnostic_error( + self, + msg: str, + *, + detail: Optional[str] = None, + path: Optional[tuple[str, ...]] = None, + location: Optional[Location] = None, + ) -> None: ... + + @property + def materialization_sets(self) -> dict[str, "MaterializationSet"]: + return self._resources["materialization_sets"] + + def add_materialization_set( + self, + resource_name: str, + materialization_set: "MaterializationSetParam", + *, + location: Optional[Location] = None, + ) -> None: + """ + Adds the resource materialization_set to the collection of resources. Resource name must be unique across all materialization_sets. + + :param resource_name: unique identifier for the materialization_set + :param materialization_set: the materialization_set to add, can be MaterializationSet or dict + :param location: optional location of the materialization_set in the source code + """ + from databricks.bundles.materialization_sets._models.materialization_set import ( + MaterializationSet, + ) + + materialization_set = _transform(MaterializationSet, materialization_set) + path = ("resources", "materialization_sets", resource_name) + location = location or Location.from_stack_frame(depth=1) + + if self._resources["materialization_sets"].get(resource_name): + self.add_diagnostic_error( + msg=f"Duplicate resource name '{resource_name}' for resource 'materialization_set'. Resource names must be unique.", + location=location, + path=path, + ) + else: + if location: + self.add_location(path, location) + + self._resources["materialization_sets"][resource_name] = materialization_set + + +@overload +def materialization_set_mutator( + function: Callable[[Bundle, "MaterializationSet"], "MaterializationSet"], +) -> ResourceMutator["MaterializationSet"]: ... + + +@overload +def materialization_set_mutator( + function: Callable[["MaterializationSet"], "MaterializationSet"], +) -> ResourceMutator["MaterializationSet"]: ... + + +def materialization_set_mutator( + function: Callable, +) -> ResourceMutator["MaterializationSet"]: + """ + Decorator for defining mutator for materialization_sets. Function should return a new instance of the materialization_set + with the desired changes, instead of mutating the input materialization_set. + + Example: + + .. code-block:: python + + @materialization_set_mutator + def my_materialization_set_mutator(bundle: Bundle, materialization_set: MaterializationSet) -> MaterializationSet: + return replace(materialization_set, ...) + + :param function: Function that mutates materialization_sets. + """ + from databricks.bundles.materialization_sets._models.materialization_set import ( + MaterializationSet, + ) + + return ResourceMutator(resource_type=MaterializationSet, function=function) diff --git a/python/databricks/bundles/materialization_sets/__init__.py b/python/databricks/bundles/materialization_sets/__init__.py new file mode 100644 index 00000000000..06bc0572543 --- /dev/null +++ b/python/databricks/bundles/materialization_sets/__init__.py @@ -0,0 +1,126 @@ +# Code generated by pydabs-codegen. DO NOT EDIT. + +__all__ = [ + "CronSchedule", + "CronScheduleDict", + "CronScheduleMode", + "CronScheduleModeParam", + "CronScheduleParam", + "DependsOn", + "DependsOnDict", + "DependsOnParam", + "Lifecycle", + "LifecycleDict", + "LifecycleParam", + "MaterializationFailureNotification", + "MaterializationFailureNotificationDict", + "MaterializationFailureNotificationParam", + "MaterializationNotifications", + "MaterializationNotificationsDict", + "MaterializationNotificationsParam", + "MaterializationSet", + "MaterializationSetDict", + "MaterializationSetParam", + "MaterializationSuccessNotification", + "MaterializationSuccessNotificationDict", + "MaterializationSuccessNotificationParam", + "MaterializedFeature", + "MaterializedFeatureDict", + "MaterializedFeatureParam", + "MaterializedFeaturePipelineScheduleState", + "MaterializedFeaturePipelineScheduleStateParam", + "OfflineStoreConfig", + "OfflineStoreConfigDict", + "OfflineStoreConfigParam", + "OnlineStoreConfig", + "OnlineStoreConfigDict", + "OnlineStoreConfigParam", + "StreamingMode", + "StreamingModeDict", + "StreamingModeParam", + "StreamingModeStreamingModeType", + "StreamingModeStreamingModeTypeParam", + "TableTrigger", + "TableTriggerDict", + "TableTriggerParam", + "TableTriggerTableTriggerCondition", + "TableTriggerTableTriggerConditionParam", +] + + +from databricks.bundles.materialization_sets._models.cron_schedule import ( + CronSchedule, + CronScheduleDict, + CronScheduleParam, +) +from databricks.bundles.materialization_sets._models.cron_schedule_mode import ( + CronScheduleMode, + CronScheduleModeParam, +) +from databricks.bundles.materialization_sets._models.depends_on import ( + DependsOn, + DependsOnDict, + DependsOnParam, +) +from databricks.bundles.materialization_sets._models.lifecycle import ( + Lifecycle, + LifecycleDict, + LifecycleParam, +) +from databricks.bundles.materialization_sets._models.materialization_failure_notification import ( + MaterializationFailureNotification, + MaterializationFailureNotificationDict, + MaterializationFailureNotificationParam, +) +from databricks.bundles.materialization_sets._models.materialization_notifications import ( + MaterializationNotifications, + MaterializationNotificationsDict, + MaterializationNotificationsParam, +) +from databricks.bundles.materialization_sets._models.materialization_set import ( + MaterializationSet, + MaterializationSetDict, + MaterializationSetParam, +) +from databricks.bundles.materialization_sets._models.materialization_success_notification import ( + MaterializationSuccessNotification, + MaterializationSuccessNotificationDict, + MaterializationSuccessNotificationParam, +) +from databricks.bundles.materialization_sets._models.materialized_feature import ( + MaterializedFeature, + MaterializedFeatureDict, + MaterializedFeatureParam, +) +from databricks.bundles.materialization_sets._models.materialized_feature_pipeline_schedule_state import ( + MaterializedFeaturePipelineScheduleState, + MaterializedFeaturePipelineScheduleStateParam, +) +from databricks.bundles.materialization_sets._models.offline_store_config import ( + OfflineStoreConfig, + OfflineStoreConfigDict, + OfflineStoreConfigParam, +) +from databricks.bundles.materialization_sets._models.online_store_config import ( + OnlineStoreConfig, + OnlineStoreConfigDict, + OnlineStoreConfigParam, +) +from databricks.bundles.materialization_sets._models.streaming_mode import ( + StreamingMode, + StreamingModeDict, + StreamingModeParam, +) +from databricks.bundles.materialization_sets._models.streaming_mode_streaming_mode_type import ( + StreamingModeStreamingModeType, + StreamingModeStreamingModeTypeParam, +) +from databricks.bundles.materialization_sets._models.table_trigger import ( + TableTrigger, + TableTriggerDict, + TableTriggerParam, +) +from databricks.bundles.materialization_sets._models.table_trigger_table_trigger_condition import ( + TableTriggerTableTriggerCondition, + TableTriggerTableTriggerConditionParam, +) diff --git a/python/databricks/bundles/materialization_sets/_models/cron_schedule.py b/python/databricks/bundles/materialization_sets/_models/cron_schedule.py new file mode 100644 index 00000000000..891502e0082 --- /dev/null +++ b/python/databricks/bundles/materialization_sets/_models/cron_schedule.py @@ -0,0 +1,86 @@ +# Code generated by pydabs-codegen. DO NOT EDIT. + +from dataclasses import dataclass +from typing import TYPE_CHECKING, TypedDict + +from databricks.bundles.core._transform import _transform +from databricks.bundles.core._transform_to_json import _transform_to_json_value +from databricks.bundles.core._variable import VariableOrOptional +from databricks.bundles.materialization_sets._models.cron_schedule_mode import ( + CronScheduleMode, + CronScheduleModeParam, +) + +if TYPE_CHECKING: + from typing_extensions import Self + + +@dataclass(kw_only=True) +class CronSchedule: + """ + :meta private: [EXPERIMENTAL] + + A cron-based schedule trigger for the materialization pipeline. + """ + + cron_expression: VariableOrOptional[str] = None + """ + The cron expression defining the schedule (e.g., "0 0 * * *" for daily at midnight). The + schedule is interpreted in timezone_id (defaults to UTC). Required when mode is MANUAL (or + unset). Left empty when mode is DERIVED, where the service computes it (aligned to UTC) from + the features' window timing and fills it in on the response. + """ + + mode: VariableOrOptional[CronScheduleMode] = None + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] How the schedule is determined. Defaults to MANUAL when unset. + """ + + timezone_id: VariableOrOptional[str] = None + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] A Java timezone ID. The schedule is resolved with respect to this timezone. Defaults to UTC + when omitted. Can only be configured for MANUAL schedules; DERIVED schedules are always aligned + to UTC. + """ + + @classmethod + def from_dict(cls, value: "CronScheduleDict") -> "Self": + return _transform(cls, value) + + def as_dict(self) -> "CronScheduleDict": + return _transform_to_json_value(self) # type:ignore + + +class CronScheduleDict(TypedDict, total=False): + """""" + + cron_expression: VariableOrOptional[str] + """ + The cron expression defining the schedule (e.g., "0 0 * * *" for daily at midnight). The + schedule is interpreted in timezone_id (defaults to UTC). Required when mode is MANUAL (or + unset). Left empty when mode is DERIVED, where the service computes it (aligned to UTC) from + the features' window timing and fills it in on the response. + """ + + mode: VariableOrOptional[CronScheduleModeParam] + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] How the schedule is determined. Defaults to MANUAL when unset. + """ + + timezone_id: VariableOrOptional[str] + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] A Java timezone ID. The schedule is resolved with respect to this timezone. Defaults to UTC + when omitted. Can only be configured for MANUAL schedules; DERIVED schedules are always aligned + to UTC. + """ + + +CronScheduleParam = CronScheduleDict | CronSchedule diff --git a/python/databricks/bundles/materialization_sets/_models/cron_schedule_mode.py b/python/databricks/bundles/materialization_sets/_models/cron_schedule_mode.py new file mode 100644 index 00000000000..163f0116833 --- /dev/null +++ b/python/databricks/bundles/materialization_sets/_models/cron_schedule_mode.py @@ -0,0 +1,18 @@ +# Code generated by pydabs-codegen. DO NOT EDIT. + +from enum import Enum +from typing import Literal + + +class CronScheduleMode(Enum): + """ + :meta private: [EXPERIMENTAL] + + The way a materialization schedule is arrived at. + """ + + MANUAL = "MANUAL" + DERIVED = "DERIVED" + + +CronScheduleModeParam = Literal["MANUAL", "DERIVED"] | CronScheduleMode diff --git a/python/databricks/bundles/materialization_sets/_models/depends_on.py b/python/databricks/bundles/materialization_sets/_models/depends_on.py new file mode 100644 index 00000000000..92fe3d5ea1e --- /dev/null +++ b/python/databricks/bundles/materialization_sets/_models/depends_on.py @@ -0,0 +1,40 @@ +# Code generated by pydabs-codegen. DO NOT EDIT. + +from dataclasses import dataclass +from typing import TYPE_CHECKING, TypedDict + +from databricks.bundles.core._transform import _transform +from databricks.bundles.core._transform_to_json import _transform_to_json_value +from databricks.bundles.core._variable import VariableOr + +if TYPE_CHECKING: + from typing_extensions import Self + + +@dataclass(kw_only=True) +class DependsOn: + """""" + + resource: VariableOr[str] + """ + Key of the resource to depend on, as `foo` or `jobs.foo`. + """ + + @classmethod + def from_dict(cls, value: "DependsOnDict") -> "Self": + return _transform(cls, value) + + def as_dict(self) -> "DependsOnDict": + return _transform_to_json_value(self) # type:ignore + + +class DependsOnDict(TypedDict, total=False): + """""" + + resource: VariableOr[str] + """ + Key of the resource to depend on, as `foo` or `jobs.foo`. + """ + + +DependsOnParam = DependsOnDict | DependsOn diff --git a/python/databricks/bundles/materialization_sets/_models/lifecycle.py b/python/databricks/bundles/materialization_sets/_models/lifecycle.py new file mode 100644 index 00000000000..e2721031364 --- /dev/null +++ b/python/databricks/bundles/materialization_sets/_models/lifecycle.py @@ -0,0 +1,54 @@ +# Code generated by pydabs-codegen. DO NOT EDIT. + +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, TypedDict + +from databricks.bundles.core._transform import _transform +from databricks.bundles.core._transform_to_json import _transform_to_json_value +from databricks.bundles.core._variable import VariableOrList, VariableOrOptional +from databricks.bundles.materialization_sets._models.depends_on import ( + DependsOn, + DependsOnParam, +) + +if TYPE_CHECKING: + from typing_extensions import Self + + +@dataclass(kw_only=True) +class Lifecycle: + """""" + + depends_on: VariableOrList[DependsOn] = field(default_factory=list) + """ + Resources that must be deployed before this resource and deleted after it. Unlike variable references, this only affects ordering. + """ + + prevent_destroy: VariableOrOptional[bool] = None + """ + Lifecycle setting to prevent the resource from being destroyed. + """ + + @classmethod + def from_dict(cls, value: "LifecycleDict") -> "Self": + return _transform(cls, value) + + def as_dict(self) -> "LifecycleDict": + return _transform_to_json_value(self) # type:ignore + + +class LifecycleDict(TypedDict, total=False): + """""" + + depends_on: VariableOrList[DependsOnParam] + """ + Resources that must be deployed before this resource and deleted after it. Unlike variable references, this only affects ordering. + """ + + prevent_destroy: VariableOrOptional[bool] + """ + Lifecycle setting to prevent the resource from being destroyed. + """ + + +LifecycleParam = LifecycleDict | Lifecycle diff --git a/python/databricks/bundles/materialization_sets/_models/materialization_failure_notification.py b/python/databricks/bundles/materialization_sets/_models/materialization_failure_notification.py new file mode 100644 index 00000000000..028550eff6b --- /dev/null +++ b/python/databricks/bundles/materialization_sets/_models/materialization_failure_notification.py @@ -0,0 +1,98 @@ +# Code generated by pydabs-codegen. DO NOT EDIT. + +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, TypedDict + +from databricks.bundles.core._transform import _transform +from databricks.bundles.core._transform_to_json import _transform_to_json_value +from databricks.bundles.core._variable import VariableOrList, VariableOrOptional + +if TYPE_CHECKING: + from typing_extensions import Self + + +@dataclass(kw_only=True) +class MaterializationFailureNotification: + """ + :meta private: [EXPERIMENTAL] + + Recipients to notify when a materialization run fails. + """ + + destination_ids: VariableOrList[str] = field(default_factory=list) + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] IDs of the notification destinations (for example Slack, Microsoft Teams, PagerDuty, or a + generic webhook) to notify. Not supported for streaming materialized features. + """ + + email_addresses: VariableOrList[str] = field(default_factory=list) + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] Email addresses to notify. + """ + + final_attempt_only: VariableOrOptional[bool] = None + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] If true, notify only when the final attempt of a run fails. If false or unset, notify on + every failed attempt, including attempts that will be retried. + + Batch materialization does not retry failures that need a fix on your side, such as missing + permissions or invalid configuration, so the first attempt is the final one. Other batch + failures are retried up to twice. + + Streaming materialization restarts a failed update indefinitely unless the error cannot be + retried. With this set, a streaming materialized feature notifies only on errors that cannot + be retried, and never on failures that are restarted. + """ + + @classmethod + def from_dict(cls, value: "MaterializationFailureNotificationDict") -> "Self": + return _transform(cls, value) + + def as_dict(self) -> "MaterializationFailureNotificationDict": + return _transform_to_json_value(self) # type:ignore + + +class MaterializationFailureNotificationDict(TypedDict, total=False): + """""" + + destination_ids: VariableOrList[str] + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] IDs of the notification destinations (for example Slack, Microsoft Teams, PagerDuty, or a + generic webhook) to notify. Not supported for streaming materialized features. + """ + + email_addresses: VariableOrList[str] + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] Email addresses to notify. + """ + + final_attempt_only: VariableOrOptional[bool] + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] If true, notify only when the final attempt of a run fails. If false or unset, notify on + every failed attempt, including attempts that will be retried. + + Batch materialization does not retry failures that need a fix on your side, such as missing + permissions or invalid configuration, so the first attempt is the final one. Other batch + failures are retried up to twice. + + Streaming materialization restarts a failed update indefinitely unless the error cannot be + retried. With this set, a streaming materialized feature notifies only on errors that cannot + be retried, and never on failures that are restarted. + """ + + +MaterializationFailureNotificationParam = ( + MaterializationFailureNotificationDict | MaterializationFailureNotification +) diff --git a/python/databricks/bundles/materialization_sets/_models/materialization_notifications.py b/python/databricks/bundles/materialization_sets/_models/materialization_notifications.py new file mode 100644 index 00000000000..6ee7e70e2d0 --- /dev/null +++ b/python/databricks/bundles/materialization_sets/_models/materialization_notifications.py @@ -0,0 +1,72 @@ +# Code generated by pydabs-codegen. DO NOT EDIT. + +from dataclasses import dataclass +from typing import TYPE_CHECKING, TypedDict + +from databricks.bundles.core._transform import _transform +from databricks.bundles.core._transform_to_json import _transform_to_json_value +from databricks.bundles.core._variable import VariableOrOptional +from databricks.bundles.materialization_sets._models.materialization_failure_notification import ( + MaterializationFailureNotification, + MaterializationFailureNotificationParam, +) +from databricks.bundles.materialization_sets._models.materialization_success_notification import ( + MaterializationSuccessNotification, + MaterializationSuccessNotificationParam, +) + +if TYPE_CHECKING: + from typing_extensions import Self + + +@dataclass(kw_only=True) +class MaterializationNotifications: + """ + :meta private: [EXPERIMENTAL] + + Notifications for the jobs and pipelines that materialize a feature, one field per trigger. + """ + + on_failure: VariableOrOptional[MaterializationFailureNotification] = None + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] Who to notify when a run fails, and on which attempts. + """ + + on_success: VariableOrOptional[MaterializationSuccessNotification] = None + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] Who to notify when a run succeeds. + """ + + @classmethod + def from_dict(cls, value: "MaterializationNotificationsDict") -> "Self": + return _transform(cls, value) + + def as_dict(self) -> "MaterializationNotificationsDict": + return _transform_to_json_value(self) # type:ignore + + +class MaterializationNotificationsDict(TypedDict, total=False): + """""" + + on_failure: VariableOrOptional[MaterializationFailureNotificationParam] + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] Who to notify when a run fails, and on which attempts. + """ + + on_success: VariableOrOptional[MaterializationSuccessNotificationParam] + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] Who to notify when a run succeeds. + """ + + +MaterializationNotificationsParam = ( + MaterializationNotificationsDict | MaterializationNotifications +) diff --git a/python/databricks/bundles/materialization_sets/_models/materialization_set.py b/python/databricks/bundles/materialization_sets/_models/materialization_set.py new file mode 100644 index 00000000000..7341bec7cbd --- /dev/null +++ b/python/databricks/bundles/materialization_sets/_models/materialization_set.py @@ -0,0 +1,77 @@ +# Code generated by pydabs-codegen. DO NOT EDIT. + +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, TypedDict + +from databricks.bundles.core._resource import Resource +from databricks.bundles.core._transform import _transform +from databricks.bundles.core._transform_to_json import _transform_to_json_value +from databricks.bundles.core._variable import VariableOrList, VariableOrOptional +from databricks.bundles.materialization_sets._models.lifecycle import ( + Lifecycle, + LifecycleParam, +) +from databricks.bundles.materialization_sets._models.materialized_feature import ( + MaterializedFeature, + MaterializedFeatureParam, +) + +if TYPE_CHECKING: + from typing_extensions import Self + + +@dataclass(kw_only=True) +class MaterializationSet(Resource): + """ + A group of materialized features created and managed as a single unit. + """ + + lifecycle: VariableOrOptional[Lifecycle] = None + + materialization_set_id: VariableOrOptional[str] = None + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] Server-assigned unique identifier for the materialization set. + """ + + materialized_features: VariableOrList[MaterializedFeature] = field( + default_factory=list + ) + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] The materialized features in this set. Order is not significant. + Updating the set with a materialized feature removed from this set will cause it to be deleted. + """ + + @classmethod + def from_dict(cls, value: "MaterializationSetDict") -> "Self": + return _transform(cls, value) + + def as_dict(self) -> "MaterializationSetDict": + return _transform_to_json_value(self) # type:ignore + + +class MaterializationSetDict(TypedDict, total=False): + """""" + + lifecycle: VariableOrOptional[LifecycleParam] + + materialization_set_id: VariableOrOptional[str] + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] Server-assigned unique identifier for the materialization set. + """ + + materialized_features: VariableOrList[MaterializedFeatureParam] + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] The materialized features in this set. Order is not significant. + Updating the set with a materialized feature removed from this set will cause it to be deleted. + """ + + +MaterializationSetParam = MaterializationSetDict | MaterializationSet diff --git a/python/databricks/bundles/materialization_sets/_models/materialization_success_notification.py b/python/databricks/bundles/materialization_sets/_models/materialization_success_notification.py new file mode 100644 index 00000000000..830db577c7d --- /dev/null +++ b/python/databricks/bundles/materialization_sets/_models/materialization_success_notification.py @@ -0,0 +1,66 @@ +# Code generated by pydabs-codegen. DO NOT EDIT. + +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, TypedDict + +from databricks.bundles.core._transform import _transform +from databricks.bundles.core._transform_to_json import _transform_to_json_value +from databricks.bundles.core._variable import VariableOrList + +if TYPE_CHECKING: + from typing_extensions import Self + + +@dataclass(kw_only=True) +class MaterializationSuccessNotification: + """ + :meta private: [EXPERIMENTAL] + + Recipients to notify when a materialization run succeeds. + """ + + destination_ids: VariableOrList[str] = field(default_factory=list) + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] IDs of the notification destinations (for example Slack, Microsoft Teams, PagerDuty, or a + generic webhook) to notify. Not supported for streaming materialized features. + """ + + email_addresses: VariableOrList[str] = field(default_factory=list) + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] Email addresses to notify. + """ + + @classmethod + def from_dict(cls, value: "MaterializationSuccessNotificationDict") -> "Self": + return _transform(cls, value) + + def as_dict(self) -> "MaterializationSuccessNotificationDict": + return _transform_to_json_value(self) # type:ignore + + +class MaterializationSuccessNotificationDict(TypedDict, total=False): + """""" + + destination_ids: VariableOrList[str] + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] IDs of the notification destinations (for example Slack, Microsoft Teams, PagerDuty, or a + generic webhook) to notify. Not supported for streaming materialized features. + """ + + email_addresses: VariableOrList[str] + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] Email addresses to notify. + """ + + +MaterializationSuccessNotificationParam = ( + MaterializationSuccessNotificationDict | MaterializationSuccessNotification +) diff --git a/python/databricks/bundles/materialization_sets/_models/materialized_feature.py b/python/databricks/bundles/materialization_sets/_models/materialized_feature.py new file mode 100644 index 00000000000..0b475cffbe6 --- /dev/null +++ b/python/databricks/bundles/materialization_sets/_models/materialized_feature.py @@ -0,0 +1,224 @@ +# Code generated by pydabs-codegen. DO NOT EDIT. + +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, TypedDict + +from databricks.bundles.core._transform import _transform +from databricks.bundles.core._transform_to_json import _transform_to_json_value +from databricks.bundles.core._variable import ( + VariableOr, + VariableOrDict, + VariableOrOptional, +) +from databricks.bundles.materialization_sets._models.cron_schedule import ( + CronSchedule, + CronScheduleParam, +) +from databricks.bundles.materialization_sets._models.materialization_notifications import ( + MaterializationNotifications, + MaterializationNotificationsParam, +) +from databricks.bundles.materialization_sets._models.materialized_feature_pipeline_schedule_state import ( + MaterializedFeaturePipelineScheduleState, + MaterializedFeaturePipelineScheduleStateParam, +) +from databricks.bundles.materialization_sets._models.offline_store_config import ( + OfflineStoreConfig, + OfflineStoreConfigParam, +) +from databricks.bundles.materialization_sets._models.online_store_config import ( + OnlineStoreConfig, + OnlineStoreConfigParam, +) +from databricks.bundles.materialization_sets._models.streaming_mode import ( + StreamingMode, + StreamingModeParam, +) +from databricks.bundles.materialization_sets._models.table_trigger import ( + TableTrigger, + TableTriggerParam, +) + +if TYPE_CHECKING: + from typing_extensions import Self + + +@dataclass(kw_only=True) +class MaterializedFeature: + """ + :meta private: [EXPERIMENTAL] + + A materialized feature represents a feature that is continuously computed and stored. + """ + + feature_name: VariableOr[str] + """ + The full name of the feature in Unity Catalog. + """ + + budget_policy_id: VariableOrOptional[str] = None + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] The ID of the budget policy used to attribute the serverless compute cost of this + materialization. If not specified, a default budget policy may be applied. + """ + + cron_schedule_trigger: VariableOrOptional[CronSchedule] = None + """ + A cron-based schedule trigger for the materialization pipeline. + """ + + materialized_feature_id: VariableOrOptional[str] = None + """ + Server-assigned unique identifier for the materialized feature. + """ + + notifications: VariableOrOptional[MaterializationNotifications] = None + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] Notification configuration around the materialization job or pipeline lifecycle. They are + applied to every job and pipeline that materializes this feature. Features which are + materialized in the same pipeline will share the same notification. + """ + + offline_store_config: VariableOrOptional[OfflineStoreConfig] = None + """ + Destination for writing feature values to an offline Delta table. The resulting table is + returned as `table_name`. + """ + + online_store_config: VariableOrOptional[OnlineStoreConfig] = None + """ + Destination for writing feature values to an online Lakebase table. The resulting table is + returned as `table_name`. + """ + + pipeline_schedule_state: VariableOrOptional[ + MaterializedFeaturePipelineScheduleState + ] = None + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] The schedule state of the materialization pipeline. + Hidden from GraphQL: being deprecated, so not exposed to Catalog Explorer. + """ + + streaming_mode: VariableOrOptional[StreamingMode] = None + """ + The Structured Streaming trigger mode used for materialization. Real-time mode (RTM) targets + sub-second latency for operational workloads; micro-batch mode (MBM) favors cost efficiency + for ETL and analytics workloads. + """ + + table_trigger: VariableOrOptional[TableTrigger] = None + """ + A trigger that fires when the upstream source table changes. + """ + + tags: VariableOrDict[str] = field(default_factory=dict) + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] Custom tags to associate with this materialization. They are applied to the materialization + job (for batch features) or pipeline (for streaming features) and forwarded to the underlying + compute as cluster tags, so materialization cost can be attributed in the billing system + tables. These tags apply only to the materialization compute; they are not applied to the + Unity Catalog Feature resource itself, whose tags are managed separately through the Unity + Catalog tagging API. A maximum of 25 tags is supported; keys and values are subject to the + same limitations as cluster tags. + """ + + @classmethod + def from_dict(cls, value: "MaterializedFeatureDict") -> "Self": + return _transform(cls, value) + + def as_dict(self) -> "MaterializedFeatureDict": + return _transform_to_json_value(self) # type:ignore + + +class MaterializedFeatureDict(TypedDict, total=False): + """""" + + feature_name: VariableOr[str] + """ + The full name of the feature in Unity Catalog. + """ + + budget_policy_id: VariableOrOptional[str] + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] The ID of the budget policy used to attribute the serverless compute cost of this + materialization. If not specified, a default budget policy may be applied. + """ + + cron_schedule_trigger: VariableOrOptional[CronScheduleParam] + """ + A cron-based schedule trigger for the materialization pipeline. + """ + + materialized_feature_id: VariableOrOptional[str] + """ + Server-assigned unique identifier for the materialized feature. + """ + + notifications: VariableOrOptional[MaterializationNotificationsParam] + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] Notification configuration around the materialization job or pipeline lifecycle. They are + applied to every job and pipeline that materializes this feature. Features which are + materialized in the same pipeline will share the same notification. + """ + + offline_store_config: VariableOrOptional[OfflineStoreConfigParam] + """ + Destination for writing feature values to an offline Delta table. The resulting table is + returned as `table_name`. + """ + + online_store_config: VariableOrOptional[OnlineStoreConfigParam] + """ + Destination for writing feature values to an online Lakebase table. The resulting table is + returned as `table_name`. + """ + + pipeline_schedule_state: VariableOrOptional[ + MaterializedFeaturePipelineScheduleStateParam + ] + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] The schedule state of the materialization pipeline. + Hidden from GraphQL: being deprecated, so not exposed to Catalog Explorer. + """ + + streaming_mode: VariableOrOptional[StreamingModeParam] + """ + The Structured Streaming trigger mode used for materialization. Real-time mode (RTM) targets + sub-second latency for operational workloads; micro-batch mode (MBM) favors cost efficiency + for ETL and analytics workloads. + """ + + table_trigger: VariableOrOptional[TableTriggerParam] + """ + A trigger that fires when the upstream source table changes. + """ + + tags: VariableOrDict[str] + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] Custom tags to associate with this materialization. They are applied to the materialization + job (for batch features) or pipeline (for streaming features) and forwarded to the underlying + compute as cluster tags, so materialization cost can be attributed in the billing system + tables. These tags apply only to the materialization compute; they are not applied to the + Unity Catalog Feature resource itself, whose tags are managed separately through the Unity + Catalog tagging API. A maximum of 25 tags is supported; keys and values are subject to the + same limitations as cluster tags. + """ + + +MaterializedFeatureParam = MaterializedFeatureDict | MaterializedFeature diff --git a/python/databricks/bundles/materialization_sets/_models/materialized_feature_pipeline_schedule_state.py b/python/databricks/bundles/materialization_sets/_models/materialized_feature_pipeline_schedule_state.py new file mode 100644 index 00000000000..204205b9782 --- /dev/null +++ b/python/databricks/bundles/materialization_sets/_models/materialized_feature_pipeline_schedule_state.py @@ -0,0 +1,19 @@ +# Code generated by pydabs-codegen. DO NOT EDIT. + +from enum import Enum +from typing import Literal + + +class MaterializedFeaturePipelineScheduleState(Enum): + """ + :meta private: [EXPERIMENTAL] + """ + + SNAPSHOT = "SNAPSHOT" + ACTIVE = "ACTIVE" + PAUSED = "PAUSED" + + +MaterializedFeaturePipelineScheduleStateParam = ( + Literal["SNAPSHOT", "ACTIVE", "PAUSED"] | MaterializedFeaturePipelineScheduleState +) diff --git a/python/databricks/bundles/materialization_sets/_models/offline_store_config.py b/python/databricks/bundles/materialization_sets/_models/offline_store_config.py new file mode 100644 index 00000000000..f2448ca1e5a --- /dev/null +++ b/python/databricks/bundles/materialization_sets/_models/offline_store_config.py @@ -0,0 +1,66 @@ +# Code generated by pydabs-codegen. DO NOT EDIT. + +from dataclasses import dataclass +from typing import TYPE_CHECKING, TypedDict + +from databricks.bundles.core._transform import _transform +from databricks.bundles.core._transform_to_json import _transform_to_json_value +from databricks.bundles.core._variable import VariableOr + +if TYPE_CHECKING: + from typing_extensions import Self + + +@dataclass(kw_only=True) +class OfflineStoreConfig: + """ + :meta private: [EXPERIMENTAL] + + Configuration for offline store destination. + """ + + catalog_name: VariableOr[str] + """ + The Unity Catalog catalog name. + """ + + schema_name: VariableOr[str] + """ + The Unity Catalog schema name. + """ + + table_name_prefix: VariableOr[str] + """ + Prefix for Unity Catalog table name. + The materialized feature will be stored in a table with this prefix and a generated postfix. + """ + + @classmethod + def from_dict(cls, value: "OfflineStoreConfigDict") -> "Self": + return _transform(cls, value) + + def as_dict(self) -> "OfflineStoreConfigDict": + return _transform_to_json_value(self) # type:ignore + + +class OfflineStoreConfigDict(TypedDict, total=False): + """""" + + catalog_name: VariableOr[str] + """ + The Unity Catalog catalog name. + """ + + schema_name: VariableOr[str] + """ + The Unity Catalog schema name. + """ + + table_name_prefix: VariableOr[str] + """ + Prefix for Unity Catalog table name. + The materialized feature will be stored in a table with this prefix and a generated postfix. + """ + + +OfflineStoreConfigParam = OfflineStoreConfigDict | OfflineStoreConfig diff --git a/python/databricks/bundles/materialization_sets/_models/online_store_config.py b/python/databricks/bundles/materialization_sets/_models/online_store_config.py new file mode 100644 index 00000000000..fd6ba9bcc6e --- /dev/null +++ b/python/databricks/bundles/materialization_sets/_models/online_store_config.py @@ -0,0 +1,80 @@ +# Code generated by pydabs-codegen. DO NOT EDIT. + +from dataclasses import dataclass +from typing import TYPE_CHECKING, TypedDict + +from databricks.bundles.core._transform import _transform +from databricks.bundles.core._transform_to_json import _transform_to_json_value +from databricks.bundles.core._variable import VariableOr + +if TYPE_CHECKING: + from typing_extensions import Self + + +@dataclass(kw_only=True) +class OnlineStoreConfig: + """ + :meta private: [EXPERIMENTAL] + + Configuration for online store destination. + """ + + catalog_name: VariableOr[str] + """ + The Unity Catalog catalog name. This name is also used as the Lakebase logical database name. + Quoting is handled by the backend where needed, do not pre-quote it. + """ + + online_store_name: VariableOr[str] + """ + The name of the target online store. + """ + + schema_name: VariableOr[str] + """ + The Unity Catalog schema name. This name is also used as the Lakebase schema name under the database. + Quoting is handled by the backend where needed, do not pre-quote it. + """ + + table_name_prefix: VariableOr[str] + """ + Prefix for Unity Catalog table name. + The materialized feature will be stored in a Lakebase table with this prefix and a generated postfix. + """ + + @classmethod + def from_dict(cls, value: "OnlineStoreConfigDict") -> "Self": + return _transform(cls, value) + + def as_dict(self) -> "OnlineStoreConfigDict": + return _transform_to_json_value(self) # type:ignore + + +class OnlineStoreConfigDict(TypedDict, total=False): + """""" + + catalog_name: VariableOr[str] + """ + The Unity Catalog catalog name. This name is also used as the Lakebase logical database name. + Quoting is handled by the backend where needed, do not pre-quote it. + """ + + online_store_name: VariableOr[str] + """ + The name of the target online store. + """ + + schema_name: VariableOr[str] + """ + The Unity Catalog schema name. This name is also used as the Lakebase schema name under the database. + Quoting is handled by the backend where needed, do not pre-quote it. + """ + + table_name_prefix: VariableOr[str] + """ + Prefix for Unity Catalog table name. + The materialized feature will be stored in a Lakebase table with this prefix and a generated postfix. + """ + + +OnlineStoreConfigParam = OnlineStoreConfigDict | OnlineStoreConfig diff --git a/python/databricks/bundles/materialization_sets/_models/streaming_mode.py b/python/databricks/bundles/materialization_sets/_models/streaming_mode.py new file mode 100644 index 00000000000..f5ec9f95022 --- /dev/null +++ b/python/databricks/bundles/materialization_sets/_models/streaming_mode.py @@ -0,0 +1,82 @@ +# Code generated by pydabs-codegen. DO NOT EDIT. + +from dataclasses import dataclass +from typing import TYPE_CHECKING, TypedDict + +from databricks.bundles.core._transform import _transform +from databricks.bundles.core._transform_to_json import _transform_to_json_value +from databricks.bundles.core._variable import VariableOrOptional +from databricks.bundles.materialization_sets._models.streaming_mode_streaming_mode_type import ( + StreamingModeStreamingModeType, + StreamingModeStreamingModeTypeParam, +) + +if TYPE_CHECKING: + from typing_extensions import Self + + +@dataclass(kw_only=True) +class StreamingMode: + """ + :meta private: [EXPERIMENTAL] + + The streaming mode configuration for a streaming materialization pipeline. + """ + + freshness_target: VariableOrOptional[str] = None + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] The desired data freshness for feature materialization, expressed as a + duration string (e.g. "1 minute"). + """ + + mode: VariableOrOptional[StreamingModeStreamingModeType] = None + """ + The type of streaming mode used by the materialization pipeline. + """ + + shuffle_partitions: VariableOrOptional[int] = None + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] Number of shuffle partitions for streaming materialization of this feature. + Higher values process high-throughput features with more parallelism at higher compute cost. + Materialized features which are computed together will use the largest value set among them. + """ + + @classmethod + def from_dict(cls, value: "StreamingModeDict") -> "Self": + return _transform(cls, value) + + def as_dict(self) -> "StreamingModeDict": + return _transform_to_json_value(self) # type:ignore + + +class StreamingModeDict(TypedDict, total=False): + """""" + + freshness_target: VariableOrOptional[str] + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] The desired data freshness for feature materialization, expressed as a + duration string (e.g. "1 minute"). + """ + + mode: VariableOrOptional[StreamingModeStreamingModeTypeParam] + """ + The type of streaming mode used by the materialization pipeline. + """ + + shuffle_partitions: VariableOrOptional[int] + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] Number of shuffle partitions for streaming materialization of this feature. + Higher values process high-throughput features with more parallelism at higher compute cost. + Materialized features which are computed together will use the largest value set among them. + """ + + +StreamingModeParam = StreamingModeDict | StreamingMode diff --git a/python/databricks/bundles/materialization_sets/_models/streaming_mode_streaming_mode_type.py b/python/databricks/bundles/materialization_sets/_models/streaming_mode_streaming_mode_type.py new file mode 100644 index 00000000000..35feb30775d --- /dev/null +++ b/python/databricks/bundles/materialization_sets/_models/streaming_mode_streaming_mode_type.py @@ -0,0 +1,19 @@ +# Code generated by pydabs-codegen. DO NOT EDIT. + +from enum import Enum +from typing import Literal + + +class StreamingModeStreamingModeType(Enum): + """ + :meta private: [EXPERIMENTAL] + """ + + STREAMING_MODE_TYPE_RTM = "STREAMING_MODE_TYPE_RTM" + STREAMING_MODE_TYPE_MBM = "STREAMING_MODE_TYPE_MBM" + + +StreamingModeStreamingModeTypeParam = ( + Literal["STREAMING_MODE_TYPE_RTM", "STREAMING_MODE_TYPE_MBM"] + | StreamingModeStreamingModeType +) diff --git a/python/databricks/bundles/materialization_sets/_models/table_trigger.py b/python/databricks/bundles/materialization_sets/_models/table_trigger.py new file mode 100644 index 00000000000..bee81ccf16e --- /dev/null +++ b/python/databricks/bundles/materialization_sets/_models/table_trigger.py @@ -0,0 +1,91 @@ +# Code generated by pydabs-codegen. DO NOT EDIT. + +from dataclasses import dataclass +from typing import TYPE_CHECKING, TypedDict + +from databricks.bundles.core._transform import _transform +from databricks.bundles.core._transform_to_json import _transform_to_json_value +from databricks.bundles.core._variable import VariableOrOptional +from databricks.bundles.materialization_sets._models.table_trigger_table_trigger_condition import ( + TableTriggerTableTriggerCondition, + TableTriggerTableTriggerConditionParam, +) + +if TYPE_CHECKING: + from typing_extensions import Self + + +@dataclass(kw_only=True) +class TableTrigger: + """ + :meta private: [EXPERIMENTAL] + + A trigger that fires when the feature's source table changes: a DeltaTableSource's table, or an + SdpSource's input_tables. + """ + + condition: VariableOrOptional[TableTriggerTableTriggerCondition] = None + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] Which source table updates trigger a materialization run. If not set, any source table update + triggers a run. + """ + + min_time_between_triggers: VariableOrOptional[str] = None + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] The minimum time between two triggered runs, so a frequently updated source table does not + trigger runs too often. If not set, a feature with a sliding window uses half its + slide_duration and a feature with a tumbling window uses half its window_duration, clamped to + between 5 minutes and 1 hour. Other features have no minimum. + """ + + wait_after_last_change: VariableOrOptional[str] = None + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] How long the source table must go without updates before a run starts, so a series of + updates settles before the run. If not set, a run starts as soon as the source table updates. + """ + + @classmethod + def from_dict(cls, value: "TableTriggerDict") -> "Self": + return _transform(cls, value) + + def as_dict(self) -> "TableTriggerDict": + return _transform_to_json_value(self) # type:ignore + + +class TableTriggerDict(TypedDict, total=False): + """""" + + condition: VariableOrOptional[TableTriggerTableTriggerConditionParam] + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] Which source table updates trigger a materialization run. If not set, any source table update + triggers a run. + """ + + min_time_between_triggers: VariableOrOptional[str] + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] The minimum time between two triggered runs, so a frequently updated source table does not + trigger runs too often. If not set, a feature with a sliding window uses half its + slide_duration and a feature with a tumbling window uses half its window_duration, clamped to + between 5 minutes and 1 hour. Other features have no minimum. + """ + + wait_after_last_change: VariableOrOptional[str] + """ + :meta private: [EXPERIMENTAL] + + [Private Preview] How long the source table must go without updates before a run starts, so a series of + updates settles before the run. If not set, a run starts as soon as the source table updates. + """ + + +TableTriggerParam = TableTriggerDict | TableTrigger diff --git a/python/databricks/bundles/materialization_sets/_models/table_trigger_table_trigger_condition.py b/python/databricks/bundles/materialization_sets/_models/table_trigger_table_trigger_condition.py new file mode 100644 index 00000000000..705769ba499 --- /dev/null +++ b/python/databricks/bundles/materialization_sets/_models/table_trigger_table_trigger_condition.py @@ -0,0 +1,23 @@ +# Code generated by pydabs-codegen. DO NOT EDIT. + +from enum import Enum +from typing import Literal + + +class TableTriggerTableTriggerCondition(Enum): + """ + :meta private: [EXPERIMENTAL] + + Which source table updates trigger a materialization run. + """ + + TABLE_TRIGGER_CONDITION_ANY_UPDATED = "TABLE_TRIGGER_CONDITION_ANY_UPDATED" + TABLE_TRIGGER_CONDITION_ALL_UPDATED = "TABLE_TRIGGER_CONDITION_ALL_UPDATED" + + +TableTriggerTableTriggerConditionParam = ( + Literal[ + "TABLE_TRIGGER_CONDITION_ANY_UPDATED", "TABLE_TRIGGER_CONDITION_ALL_UPDATED" + ] + | TableTriggerTableTriggerCondition +) diff --git a/python/databricks_tests/core/_generated/__init__.py b/python/databricks_tests/core/_generated/__init__.py index e1e07830307..52f25289e52 100644 --- a/python/databricks_tests/core/_generated/__init__.py +++ b/python/databricks_tests/core/_generated/__init__.py @@ -16,6 +16,7 @@ instance_pools, job_runs, jobs, + materialization_sets, mcp_services, model_provider_services, model_services, @@ -52,6 +53,7 @@ instance_pools._test_case(), job_runs._test_case(), jobs._test_case(), + materialization_sets._test_case(), mcp_services._test_case(), model_provider_services._test_case(), model_services._test_case(), diff --git a/python/databricks_tests/core/_generated/materialization_sets.py b/python/databricks_tests/core/_generated/materialization_sets.py new file mode 100644 index 00000000000..623eabd547b --- /dev/null +++ b/python/databricks_tests/core/_generated/materialization_sets.py @@ -0,0 +1,21 @@ +# Code generated by pydabs-codegen. DO NOT EDIT. + +from databricks.bundles.core import Resources, materialization_set_mutator +from databricks.bundles.core._generated.materialization_sets import _resource_type +from databricks.bundles.materialization_sets._models.lifecycle import Lifecycle +from databricks.bundles.materialization_sets._models.materialization_set import ( + MaterializationSet, +) +from databricks_tests.core._resource_test_case import ResourceTestCase + + +def _test_case(): + return ( + ResourceTestCase( + add_resource=Resources.add_materialization_set, + dict_example={"lifecycle": {}}, + dataclass_example=MaterializationSet(lifecycle=Lifecycle()), + mutator=materialization_set_mutator, + ), + _resource_type(), + ) diff --git a/python/databricks_tests/core/public_api.txt b/python/databricks_tests/core/public_api.txt index 3c34aac5aa2..6c5d6f1e096 100644 --- a/python/databricks_tests/core/public_api.txt +++ b/python/databricks_tests/core/public_api.txt @@ -31,6 +31,7 @@ __all__ = [ load_resources_from_module, load_resources_from_modules, load_resources_from_package_module, + materialization_set_mutator, mcp_service_mutator, mlflow_experiment_mutator, mlflow_model_mutator, @@ -108,6 +109,7 @@ class Resources: def add_job(self, resource_name: str, job: JobParam, *, location: Union[Location, None] = None) -> None def add_job_run(self, resource_name: str, job_run: JobRunParam, *, location: Union[Location, None] = None) -> None def add_location(self, path: tuple[str, ...], location: Location) -> None + def add_materialization_set(self, resource_name: str, materialization_set: MaterializationSetParam, *, location: Union[Location, None] = None) -> None def add_mcp_service(self, resource_name: str, mcp_service: McpServiceParam, *, location: Union[Location, None] = None) -> None def add_mlflow_experiment(self, resource_name: str, mlflow_experiment: MlflowExperimentParam, *, location: Union[Location, None] = None) -> None def add_mlflow_model(self, resource_name: str, mlflow_model: MlflowModelParam, *, location: Union[Location, None] = None) -> None @@ -143,6 +145,7 @@ class Resources: @property instance_pools -> dict[str, InstancePool] @property job_runs -> dict[str, JobRun] @property jobs -> dict[str, Job] + @property materialization_sets -> dict[str, MaterializationSet] @property mcp_services -> dict[str, McpService] @property model_provider_services -> dict[str, ModelProviderService] @property model_services -> dict[str, ModelService] @@ -241,6 +244,10 @@ def load_resources_from_modules(modules: Iterable[module]) -> Resources def load_resources_from_package_module(package_module: module) -> Resources +@overload def materialization_set_mutator(function: Callable[[Bundle, MaterializationSet], MaterializationSet]) -> ResourceMutator[MaterializationSet] +@overload def materialization_set_mutator(function: Callable[[MaterializationSet], MaterializationSet]) -> ResourceMutator[MaterializationSet] +def materialization_set_mutator(function: Callable) -> ResourceMutator[MaterializationSet] + @overload def mcp_service_mutator(function: Callable[[Bundle, McpService], McpService]) -> ResourceMutator[McpService] @overload def mcp_service_mutator(function: Callable[[McpService], McpService]) -> ResourceMutator[McpService] def mcp_service_mutator(function: Callable) -> ResourceMutator[McpService] @@ -326,6 +333,7 @@ singular_name=genie_space plural_name=genie_spaces resource_type=GenieSpace singular_name=instance_pool plural_name=instance_pools resource_type=InstancePool singular_name=job plural_name=jobs resource_type=Job singular_name=job_run plural_name=job_runs resource_type=JobRun +singular_name=materialization_set plural_name=materialization_sets resource_type=MaterializationSet singular_name=mcp_service plural_name=mcp_services resource_type=McpService singular_name=mlflow_experiment plural_name=experiments resource_type=MlflowExperiment singular_name=mlflow_model plural_name=models resource_type=MlflowModel diff --git a/python/docs/databricks.bundles.materialization_sets.rst b/python/docs/databricks.bundles.materialization_sets.rst new file mode 100644 index 00000000000..2ea2e0c1feb --- /dev/null +++ b/python/docs/databricks.bundles.materialization_sets.rst @@ -0,0 +1,11 @@ +Materialization Sets +=============================== + +.. currentmodule:: databricks.bundles.materialization_sets + +**Package:** ``databricks.bundles.materialization_sets`` + +Classes +--------------- + +.. automodule:: databricks.bundles.materialization_sets diff --git a/python/docs/index.rst b/python/docs/index.rst index 5ca4bda8ede..1541e0eefba 100644 --- a/python/docs/index.rst +++ b/python/docs/index.rst @@ -25,6 +25,7 @@ See `Bundle configuration in Python