diff --git a/.nextchanges/bundles/deploy-progressive-resource-output.md b/.nextchanges/bundles/deploy-progressive-resource-output.md new file mode 100644 index 00000000000..60687d5787c --- /dev/null +++ b/.nextchanges/bundles/deploy-progressive-resource-output.md @@ -0,0 +1 @@ +`bundle deploy` on the direct engine now reports each resource as soon as it is deployed, instead of listing them all after the deployment finishes. A deploy that fails part way through now reports the resources it did apply. diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index 9bb65e7ed41..f084925e4c6 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -994,6 +994,7 @@ func runTest(t *testing.T, printedRepls := false pathFilter := preparePathFilter(config, customEnv) + sortLines := compileSortLines(t, config) // Compare expected outputs for relPath := range outputs { @@ -1001,7 +1002,7 @@ func runTest(t *testing.T, continue } - doComparison(t, repls, dir, tmpDir, relPath, &printedRepls) + doComparison(t, repls, sortLines, dir, tmpDir, relPath, &printedRepls) } // Make sure there are not unaccounted for new files @@ -1032,7 +1033,7 @@ func runTest(t *testing.T, if strings.HasPrefix(relPath, "out") { // We have a new file starting with "out" // Show the contents & support overwrite mode for it: - doComparison(t, repls, dir, tmpDir, relPath, &printedRepls) + doComparison(t, repls, sortLines, dir, tmpDir, relPath, &printedRepls) } } @@ -1105,7 +1106,24 @@ func addEnvVar(t *testing.T, env []string, repls *testdiff.ReplacementsContext, return append(env, key+"="+newValue) } -func doComparison(t *testing.T, repls testdiff.ReplacementsContext, dirRef, dirNew, relPath string, printedRepls *bool) { +// compileSortLines compiles the enabled SortLines patterns from the test config. +// Patterns are returned in name order so the result does not depend on map iteration. +func compileSortLines(t *testing.T, config internal.TestConfig) []*regexp.Regexp { + result := make([]*regexp.Regexp, 0, len(config.SortLines)) + for _, name := range slices.Sorted(maps.Keys(config.SortLines)) { + if on, ok := config.SortLinesOn[name]; ok && !on { + continue + } + re, err := regexp.Compile(config.SortLines[name]) + if err != nil { + t.Fatalf("Invalid SortLines pattern %s = %#v: %s", name, config.SortLines[name], err) + } + result = append(result, re) + } + return result +} + +func doComparison(t *testing.T, repls testdiff.ReplacementsContext, sortLines []*regexp.Regexp, dirRef, dirNew, relPath string, printedRepls *bool) { pathRef := filepath.Join(dirRef, relPath) pathNew := filepath.Join(dirNew, relPath) bufRef, okRef := tryReading(t, pathRef) @@ -1124,6 +1142,14 @@ func doComparison(t *testing.T, repls testdiff.ReplacementsContext, dirRef, dirN valueNew = repls.Replace(valueNew) } + // Canonicalize line runs whose order the command does not guarantee. Applied to + // the reference too so a hand-edited golden compares the same way; sorting is + // idempotent, so a stored reference is unaffected. + for _, re := range sortLines { + valueRef = testdiff.SortLineRuns(valueRef, re) + valueNew = testdiff.SortLineRuns(valueNew, re) + } + // In update mode, regenerating the reference files is the goal: each branch below // writes or removes the reference and returns without failing the test. Genuine // problems still fail — read errors (via tryReading above), write errors (via diff --git a/acceptance/bundle/deploy/partial-summary-on-push-fail/output.txt b/acceptance/bundle/deploy/partial-summary-on-push-fail/output.txt index 36d57ffdbfa..fd83af4f0c5 100644 --- a/acceptance/bundle/deploy/partial-summary-on-push-fail/output.txt +++ b/acceptance/bundle/deploy/partial-summary-on-push-fail/output.txt @@ -1,6 +1,7 @@ >>> errcode [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Created jobs.my_job Error: access denied: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/resources.json (403 INJECTED) Endpoint: POST [DATABRICKS_URL]/api/2.0/workspace-files/import-file/Workspace%2FUsers%2F[USERNAME]%2F.bundle%2Ftest-bundle%2Fdefault%2Fstate%2Fresources.json?overwrite=true diff --git a/acceptance/bundle/deploy/wal/chain-3-jobs/output.txt b/acceptance/bundle/deploy/wal/chain-3-jobs/output.txt index ddba262ca36..d9ae467f59b 100644 --- a/acceptance/bundle/deploy/wal/chain-3-jobs/output.txt +++ b/acceptance/bundle/deploy/wal/chain-3-jobs/output.txt @@ -2,6 +2,8 @@ >>> errcode [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/wal-chain-test/default/files... +Created jobs.job_01 +Created jobs.job_02 [PROCESS_KILLED] Exit code: [KILLED] diff --git a/acceptance/bundle/deploy/wal/crash-after-create/output.txt b/acceptance/bundle/deploy/wal/crash-after-create/output.txt index 09f5d04a69e..e0e0bf1a627 100644 --- a/acceptance/bundle/deploy/wal/crash-after-create/output.txt +++ b/acceptance/bundle/deploy/wal/crash-after-create/output.txt @@ -2,6 +2,7 @@ >>> errcode [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/wal-crash-test/default/files... +Created jobs.job_a [PROCESS_KILLED] Exit code: [KILLED] diff --git a/acceptance/bundle/migrate/auto-migrate-postdeploy-script/databricks.yml b/acceptance/bundle/migrate/auto-migrate-postdeploy-script/databricks.yml new file mode 100644 index 00000000000..747555aaab9 --- /dev/null +++ b/acceptance/bundle/migrate/auto-migrate-postdeploy-script/databricks.yml @@ -0,0 +1,7 @@ +bundle: + name: test-bundle + engine: direct + +experimental: + scripts: + postdeploy: "python3 ./myscript.py $POSTDEPLOY_EXITCODE postdeploy" diff --git a/acceptance/bundle/migrate/auto-migrate-postdeploy-script/myscript.py b/acceptance/bundle/migrate/auto-migrate-postdeploy-script/myscript.py new file mode 100644 index 00000000000..c039766d65f --- /dev/null +++ b/acceptance/bundle/migrate/auto-migrate-postdeploy-script/myscript.py @@ -0,0 +1,5 @@ +import sys + +exitcode, name = sys.argv[1], sys.argv[2] +print(f"from {name}: hello") +sys.exit(int(exitcode)) diff --git a/acceptance/bundle/migrate/auto-migrate-postdeploy-script/out.test.toml b/acceptance/bundle/migrate/auto-migrate-postdeploy-script/out.test.toml new file mode 100644 index 00000000000..0938e678987 --- /dev/null +++ b/acceptance/bundle/migrate/auto-migrate-postdeploy-script/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/migrate/auto-migrate-postdeploy-script/output.txt b/acceptance/bundle/migrate/auto-migrate-postdeploy-script/output.txt new file mode 100644 index 00000000000..64a7fce0500 --- /dev/null +++ b/acceptance/bundle/migrate/auto-migrate-postdeploy-script/output.txt @@ -0,0 +1,24 @@ + +=== postdeploy script output comes before the migration notice +>>> [CLI] bundle deploy +Warn: Direct engine requested in bundle.engine setting at [TEST_TMP_DIR]/databricks.yml:3:11 but the existing state uses "terraform". Deploying on "terraform"; will attempt to migrate the state to the direct engine after this deploy. +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Files: 6 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 0 unchanged +Executing 'postdeploy' script +from postdeploy: hello +Removing empty terraform state; direct engine will be used on the next deploy (opted in via bundle.engine setting at [TEST_TMP_DIR]/databricks.yml:3:11)... + +=== Same when the postdeploy script fails: the migration still runs +>>> errcode [CLI] bundle deploy +Warn: Direct engine requested in bundle.engine setting at [TEST_TMP_DIR]/databricks.yml:3:11 but the existing state uses "terraform". Deploying on "terraform"; will attempt to migrate the state to the direct engine after this deploy. +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Files: 6 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 0 unchanged +Executing 'postdeploy' script +from postdeploy: hello +Error: failed to execute script: exit status 1 + +Removing empty terraform state; direct engine will be used on the next deploy (opted in via bundle.engine setting at [TEST_TMP_DIR]/databricks.yml:3:11)... + +Exit code: 1 diff --git a/acceptance/bundle/migrate/auto-migrate-postdeploy-script/script b/acceptance/bundle/migrate/auto-migrate-postdeploy-script/script new file mode 100644 index 00000000000..c2b6e831bcf --- /dev/null +++ b/acceptance/bundle/migrate/auto-migrate-postdeploy-script/script @@ -0,0 +1,22 @@ +# The deploy runs first, then the postdeploy script, then the migration. The migration is +# not gated on the script: the resources are deployed either way, so the state describes +# the same deployment whether or not the script succeeded. + +seed_empty_tfstate() { + mkdir -p .databricks/bundle/default/terraform + cat > .databricks/bundle/default/terraform/terraform.tfstate <<'JSON' +{"version": 4, "serial": 1, "lineage": "test-lineage", "resources": []} +JSON +} + +seed_empty_tfstate +title "postdeploy script output comes before the migration notice" +trace $CLI bundle deploy + +title "Same when the postdeploy script fails: the migration still runs" +rm -rf .databricks +seed_empty_tfstate +export POSTDEPLOY_EXITCODE=1 +trace errcode $CLI bundle deploy + +rm -f out.requests.txt diff --git a/acceptance/bundle/migrate/auto-migrate-postdeploy-script/test.toml b/acceptance/bundle/migrate/auto-migrate-postdeploy-script/test.toml new file mode 100644 index 00000000000..322b96cacba --- /dev/null +++ b/acceptance/bundle/migrate/auto-migrate-postdeploy-script/test.toml @@ -0,0 +1 @@ +Env.POSTDEPLOY_EXITCODE = "0" diff --git a/acceptance/bundle/resource_deps/create_error/script b/acceptance/bundle/resource_deps/create_error/script index d475fdaea78..55b0d2ace42 100644 --- a/acceptance/bundle/resource_deps/create_error/script +++ b/acceptance/bundle/resource_deps/create_error/script @@ -1,6 +1,6 @@ echo "*" > .gitignore trace $CLI bundle plan 2>&1 -musterr $CLI bundle deploy &> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt +musterr $CLI bundle deploy -q &> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt # -q: independent job races the failure trace print_requests.py --sort //jobs trace $CLI bundle summary @@ -12,7 +12,7 @@ title "Plan should still contain foo and bar" trace $CLI bundle plan rm out.requests.txt -musterr $CLI bundle deploy &> out.deploy2.$DATABRICKS_BUNDLE_ENGINE.txt +musterr $CLI bundle deploy -q &> out.deploy2.$DATABRICKS_BUNDLE_ENGINE.txt title "Expecting no difference in the output between first and second deploy" diff.py out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt out.deploy2.$DATABRICKS_BUNDLE_ENGINE.txt rm out.deploy2.$DATABRICKS_BUNDLE_ENGINE.txt diff --git a/acceptance/bundle/resource_deps/missing_ingestion_definition/out.deploy.direct.txt b/acceptance/bundle/resource_deps/missing_ingestion_definition/out.deploy.direct.txt index b99c41ca46b..558fced9e22 100644 --- a/acceptance/bundle/resource_deps/missing_ingestion_definition/out.deploy.direct.txt +++ b/acceptance/bundle/resource_deps/missing_ingestion_definition/out.deploy.direct.txt @@ -1,4 +1,5 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Created pipelines.foo Error: cannot create resources.pipelines.bar: cannot resolve "${resources.pipelines.foo.ingestion_definition.connection_name}": ingestion_definition: cannot access nil value Files: 7 uploaded, 0 deleted diff --git a/acceptance/bundle/resource_deps/missing_map_key/out.deploy.direct.txt b/acceptance/bundle/resource_deps/missing_map_key/out.deploy.direct.txt index fe12fac4f34..b7e43c67f70 100644 --- a/acceptance/bundle/resource_deps/missing_map_key/out.deploy.direct.txt +++ b/acceptance/bundle/resource_deps/missing_map_key/out.deploy.direct.txt @@ -1,4 +1,5 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/state/default/files... +Created jobs.test Error: cannot create resources.jobs.bar: cannot resolve "${resources.jobs.test.tasks[0].new_cluster.custom_tags.missing_tag}": tasks[0].new_cluster.custom_tags.missing_tag: key "missing_tag" not found in map Files: 7 uploaded, 0 deleted diff --git a/acceptance/bundle/resources/job_runs/basic/output.txt b/acceptance/bundle/resources/job_runs/basic/output.txt index 64625c59277..4e98b174beb 100644 --- a/acceptance/bundle/resources/job_runs/basic/output.txt +++ b/acceptance/bundle/resources/job_runs/basic/output.txt @@ -33,10 +33,10 @@ Resources: >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-basic/default/files... +Created jobs.my_job Output from job_runs.my_run: id=[MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?o=[NUMID] Output from job_runs.my_run: id=[MY_RUN_ID]: SUCCESS Created job_runs.my_run -Created jobs.my_job Files: 4 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged diff --git a/acceptance/bundle/resources/job_runs/failed_run/output.txt b/acceptance/bundle/resources/job_runs/failed_run/output.txt index f8b8dbf4398..5c010b28bc2 100644 --- a/acceptance/bundle/resources/job_runs/failed_run/output.txt +++ b/acceptance/bundle/resources/job_runs/failed_run/output.txt @@ -2,6 +2,7 @@ === a run that finishes FAILED fails the deploy >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... +Created jobs.my_job Output from job_runs.my_run: id=[MY_RUN_ID]: Run URL: [RUN_URL] Error: cannot create resources.job_runs.my_run: waiting after creating id=[MY_RUN_ID]: run did not succeed: FAILED: Task main failed with message: Workload failed, see run output for details. task "main": RuntimeError: intentional failure diff --git a/acceptance/bundle/resources/job_runs/interrupted_run/output.txt b/acceptance/bundle/resources/job_runs/interrupted_run/output.txt index 2e1d9cacd7a..0d27e0ba334 100644 --- a/acceptance/bundle/resources/job_runs/interrupted_run/output.txt +++ b/acceptance/bundle/resources/job_runs/interrupted_run/output.txt @@ -2,6 +2,7 @@ === the deploy stops waiting before the run finishes >>> errcode [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-interrupted-run/default/files... +Created jobs.my_job Error: cannot create resources.job_runs.my_run: waiting after creating id=[MY_RUN_ID]: Fault injected by test. (403 INJECTED) Endpoint: GET [DATABRICKS_URL]/api/2.2/jobs/runs/get?run_id=[MY_RUN_ID] diff --git a/acceptance/bundle/resources/job_runs/job_parameters/output.txt b/acceptance/bundle/resources/job_runs/job_parameters/output.txt index e35d03c12c6..f5b8951cb5f 100644 --- a/acceptance/bundle/resources/job_runs/job_parameters/output.txt +++ b/acceptance/bundle/resources/job_runs/job_parameters/output.txt @@ -2,10 +2,10 @@ === deploy triggers the run with only the overridden parameter >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-job-parameters/default/files... +Created jobs.my_job Output from job_runs.my_run: id=[MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?o=[NUMID] Output from job_runs.my_run: id=[MY_RUN_ID]: SUCCESS Created job_runs.my_run -Created jobs.my_job Files: 4 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged diff --git a/acceptance/bundle/resources/job_runs/on_bundle_deploy/output.txt b/acceptance/bundle/resources/job_runs/on_bundle_deploy/output.txt index 39123f3370f..f342619d618 100644 --- a/acceptance/bundle/resources/job_runs/on_bundle_deploy/output.txt +++ b/acceptance/bundle/resources/job_runs/on_bundle_deploy/output.txt @@ -2,10 +2,10 @@ === first deploy triggers a run >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-bundle-deploy/default/files... +Created jobs.my_job Output from job_runs.my_run: id=[MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?o=[NUMID] Output from job_runs.my_run: id=[MY_RUN_ID]: SUCCESS Created job_runs.my_run -Created jobs.my_job Files: 5 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged diff --git a/acceptance/bundle/resources/job_runs/redeploy/output.txt b/acceptance/bundle/resources/job_runs/redeploy/output.txt index 06129f79373..a06600f5eca 100644 --- a/acceptance/bundle/resources/job_runs/redeploy/output.txt +++ b/acceptance/bundle/resources/job_runs/redeploy/output.txt @@ -2,10 +2,10 @@ === initial deploy triggers the first run >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-redeploy/default/files... +Created jobs.my_job Output from job_runs.my_run: id=[MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?o=[NUMID] Output from job_runs.my_run: id=[MY_RUN_ID]: SUCCESS Created job_runs.my_run -Created jobs.my_job Files: 4 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged diff --git a/acceptance/bundle/resources/job_runs/retried_run_now/output.txt b/acceptance/bundle/resources/job_runs/retried_run_now/output.txt index 1997c56d031..fca04b36f78 100644 --- a/acceptance/bundle/resources/job_runs/retried_run_now/output.txt +++ b/acceptance/bundle/resources/job_runs/retried_run_now/output.txt @@ -2,10 +2,10 @@ === the deploy succeeds, though its run-now was answered with a 503 >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-retried-run-now/default/files... +Created jobs.my_job Output from job_runs.my_run: id=[MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?o=[NUMID] Output from job_runs.my_run: id=[MY_RUN_ID]: SUCCESS Created job_runs.my_run -Created jobs.my_job Files: 5 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged diff --git a/acceptance/bundle/resources/job_runs/wait/output.txt b/acceptance/bundle/resources/job_runs/wait/output.txt index 80f36fe3ce8..034ecf1835c 100644 --- a/acceptance/bundle/resources/job_runs/wait/output.txt +++ b/acceptance/bundle/resources/job_runs/wait/output.txt @@ -2,11 +2,11 @@ === the deploy waits for the run to finish >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... +Created jobs.my_job Output from job_runs.my_run: id=[MY_RUN_ID]: Run URL: [RUN_URL] Output from job_runs.my_run: id=[MY_RUN_ID]: SUCCESS Created job_runs.my_run Created jobs.downstream_job -Created jobs.my_job Files: 6 uploaded, 0 deleted Resources: 3 created, 0 changed, 0 deleted, 0 unchanged diff --git a/acceptance/bundle/resources/permissions/pipelines/504/create/output.txt b/acceptance/bundle/resources/permissions/pipelines/504/create/output.txt index a38fa993af1..15c605d3a7f 100644 --- a/acceptance/bundle/resources/permissions/pipelines/504/create/output.txt +++ b/acceptance/bundle/resources/permissions/pipelines/504/create/output.txt @@ -1,6 +1,6 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... -Warn: deploying resources.pipelines.foo.permissions: retrying after 504 Gateway Timeout from PUT /api/2.0/permissions/pipelines/[UUID] Created pipelines.foo +Warn: deploying resources.pipelines.foo.permissions: retrying after 504 Gateway Timeout from PUT /api/2.0/permissions/pipelines/[UUID] Created pipelines.foo.permissions Files: 5 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged diff --git a/acceptance/bundle/resources/postgres_branches/without_branch_id/out.deploy.direct.txt b/acceptance/bundle/resources/postgres_branches/without_branch_id/out.deploy.direct.txt index 9d65ab09313..3118fd0daea 100644 --- a/acceptance/bundle/resources/postgres_branches/without_branch_id/out.deploy.direct.txt +++ b/acceptance/bundle/resources/postgres_branches/without_branch_id/out.deploy.direct.txt @@ -3,6 +3,7 @@ Warning: required field "branch_id" is not set in databricks.yml:21:7 Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/deploy-postgres-branch-no-id-[UNIQUE_NAME]/default/files... +Created postgres_projects.my_project Error: cannot create resources.postgres_branches.main: Field 'branch_id' is required, expected non-default value (not "")! (400 INVALID_PARAMETER_VALUE) Endpoint: POST [DATABRICKS_URL]/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches diff --git a/acceptance/bundle/resources/postgres_databases/live_errors/bad_database_id/output.txt b/acceptance/bundle/resources/postgres_databases/live_errors/bad_database_id/output.txt index c0c02c1c82a..7034fd62db6 100644 --- a/acceptance/bundle/resources/postgres_databases/live_errors/bad_database_id/output.txt +++ b/acceptance/bundle/resources/postgres_databases/live_errors/bad_database_id/output.txt @@ -1,4 +1,7 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/bad-database-id-[UNIQUE_NAME]/default/files... +Created postgres_branches.main +Created postgres_projects.my_project +Created postgres_roles.owner Error: cannot create resources.postgres_databases.my_database: Field database_id must match pattern ^[a-z]([a-z0-9-]{0,61}[a-z0-9])?$, got 'Invalid_DB_ID'. (400 INVALID_PARAMETER_VALUE) Endpoint: POST [DATABRICKS_URL]/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/main/databases?database_id=Invalid_DB_ID diff --git a/acceptance/bundle/resources/postgres_databases/live_errors/bad_role_ref/output.txt b/acceptance/bundle/resources/postgres_databases/live_errors/bad_role_ref/output.txt index 1f95583baca..207518f3b3f 100644 --- a/acceptance/bundle/resources/postgres_databases/live_errors/bad_role_ref/output.txt +++ b/acceptance/bundle/resources/postgres_databases/live_errors/bad_role_ref/output.txt @@ -1,4 +1,6 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/bad-role-ref-[UNIQUE_NAME]/default/files... +Created postgres_branches.main +Created postgres_projects.my_project Error: cannot create resources.postgres_databases.my_database: role not found; role_id:"does-not-exist" [TraceId: [TRACE_ID]] (404 NOT_FOUND) Endpoint: POST [DATABRICKS_URL]/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/main/databases?database_id=my-database diff --git a/acceptance/bundle/resources/postgres_endpoints/without_endpoint_id/out.deploy.direct.txt b/acceptance/bundle/resources/postgres_endpoints/without_endpoint_id/out.deploy.direct.txt index e2137454122..6aebf11934d 100644 --- a/acceptance/bundle/resources/postgres_endpoints/without_endpoint_id/out.deploy.direct.txt +++ b/acceptance/bundle/resources/postgres_endpoints/without_endpoint_id/out.deploy.direct.txt @@ -3,6 +3,8 @@ Warning: required field "endpoint_id" is not set in databricks.yml:27:7 Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/deploy-postgres-endpoint-no-id-[UNIQUE_NAME]/default/files... +Created postgres_branches.main +Created postgres_projects.my_project Error: cannot create resources.postgres_endpoints.primary: Field 'endpoint_id' is required, expected non-default value (not "")! (400 INVALID_PARAMETER_VALUE) Endpoint: POST [DATABRICKS_URL]/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]/branches/main/endpoints diff --git a/acceptance/bundle/resources/volumes/set-storage-location/out.deploy.direct.txt b/acceptance/bundle/resources/volumes/set-storage-location/out.deploy.direct.txt index c5237fefbd5..9d7f571c16c 100644 --- a/acceptance/bundle/resources/volumes/set-storage-location/out.deploy.direct.txt +++ b/acceptance/bundle/resources/volumes/set-storage-location/out.deploy.direct.txt @@ -1,6 +1,7 @@ >>> musterr [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/testbundle-[UNIQUE_NAME]/default/files... +Created schemas.myschema Error: cannot create resources.volumes.volume1: CreateVolume storage_location can not be provided. (400 INVALID_PARAMETER_VALUE) Endpoint: POST [DATABRICKS_URL]/api/2.1/unity-catalog/volumes diff --git a/acceptance/internal/config.go b/acceptance/internal/config.go index 66908c03e18..dcecbc76ff4 100644 --- a/acceptance/internal/config.go +++ b/acceptance/internal/config.go @@ -85,6 +85,17 @@ type TestConfig struct { // List of request headers to include when recording requests. IncludeRequestHeaders []string + // Map of name -> regexp. Each run of consecutive output lines matching one of the + // regexps is sorted before comparison. Use for output whose line order is not + // deterministic, e.g. "bundle deploy" reporting resources as they are applied in + // parallel. Entries are keyed by name so an inner test.toml can replace an + // inherited pattern by reusing its name, or switch it off via SortLinesOn. + SortLines map[string]string + + // Map of SortLines name -> whether to apply it. If a name is not listed, defaults + // to true; set it to false to drop an inherited pattern for this test. + SortLinesOn map[string]bool + // List of gitignore patterns to ignore when checking output files Ignore []string diff --git a/acceptance/selftest/sortlines/out.test.toml b/acceptance/selftest/sortlines/out.test.toml new file mode 100644 index 00000000000..8fdd3560ef2 --- /dev/null +++ b/acceptance/selftest/sortlines/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = [] diff --git a/acceptance/selftest/sortlines/output.txt b/acceptance/selftest/sortlines/output.txt new file mode 100644 index 00000000000..013e56e8224 --- /dev/null +++ b/acceptance/selftest/sortlines/output.txt @@ -0,0 +1,9 @@ +Fruit: quince +Fruit: zucchini +Not a fruit: rock +Fruit: apple +Fruit: banana +Not a fruit: spoon +Fruit: melon +Created jobs.z +Created jobs.a diff --git a/acceptance/selftest/sortlines/override/out.test.toml b/acceptance/selftest/sortlines/override/out.test.toml new file mode 100644 index 00000000000..8fdd3560ef2 --- /dev/null +++ b/acceptance/selftest/sortlines/override/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = [] diff --git a/acceptance/selftest/sortlines/override/output.txt b/acceptance/selftest/sortlines/override/output.txt new file mode 100644 index 00000000000..339e6b00366 --- /dev/null +++ b/acceptance/selftest/sortlines/override/output.txt @@ -0,0 +1,4 @@ +Fruit: pear +Fruit: apple +Veg: carrot +Veg: turnip diff --git a/acceptance/selftest/sortlines/override/script b/acceptance/selftest/sortlines/override/script new file mode 100644 index 00000000000..4a51a9f1add --- /dev/null +++ b/acceptance/selftest/sortlines/override/script @@ -0,0 +1,4 @@ +echo "Fruit: pear" +echo "Fruit: apple" +echo "Veg: turnip" +echo "Veg: carrot" diff --git a/acceptance/selftest/sortlines/override/test.toml b/acceptance/selftest/sortlines/override/test.toml new file mode 100644 index 00000000000..36c7a8748da --- /dev/null +++ b/acceptance/selftest/sortlines/override/test.toml @@ -0,0 +1,3 @@ +# Reusing the inherited name replaces the parent's pattern rather than adding to it: +# "Fruit:" lines are left alone from here on, and "Veg:" lines are sorted instead. +SortLines.fruit = '^Veg: ' diff --git a/acceptance/selftest/sortlines/script b/acceptance/selftest/sortlines/script new file mode 100644 index 00000000000..afe77ccac2c --- /dev/null +++ b/acceptance/selftest/sortlines/script @@ -0,0 +1,15 @@ +# Each run of consecutive "Fruit:" lines is sorted on its own, and a non-matching line ends +# the run. The values are picked so that this is visible in the output: the first block +# sorts late in the alphabet and the second sorts early, so if the two blocks were sorted +# together, "apple" and "banana" would move up past "Not a fruit: rock". +echo "Fruit: zucchini" +echo "Fruit: quince" +echo "Not a fruit: rock" +echo "Fruit: banana" +echo "Fruit: apple" +echo "Not a fruit: spoon" +echo "Fruit: melon" + +# deploy_actions is disabled here, so these keep the order they were printed in. +echo "Created jobs.z" +echo "Created jobs.a" diff --git a/acceptance/selftest/sortlines/test.toml b/acceptance/selftest/sortlines/test.toml new file mode 100644 index 00000000000..5375590369b --- /dev/null +++ b/acceptance/selftest/sortlines/test.toml @@ -0,0 +1,9 @@ +# Runs once: the sorting is independent of the deployment engine. +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = [] + +# Only "Fruit:" lines are sorted; every other line delimits the runs. +SortLines.fruit = '^Fruit: ' + +# The inherited deploy_actions pattern is irrelevant here, and switching it off +# exercises SortLinesOn. +SortLinesOn.deploy_actions = false diff --git a/acceptance/test.toml b/acceptance/test.toml index 5fca2e77234..0e3258c8aa1 100644 --- a/acceptance/test.toml +++ b/acceptance/test.toml @@ -19,6 +19,12 @@ Env.PYTHONDONTWRITEBYTECODE = "1" Env.PYTHONUNBUFFERED = "1" Env.PYTHONUTF8 = "1" +# The direct engine reports resources as soon as each one is applied, and resources are +# applied in parallel, so the order of these lines is not deterministic. Sort each run of +# them before comparing, so tests pin which resources were applied, not the order. +# A test that needs the raw order sets SortLinesOn.deploy_actions = false. +SortLines.deploy_actions = '^(Created|Updated|Deleted|Recreated|Resized) \S+$' + EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrixExclude.noplantf = ["DATABRICKS_BUNDLE_ENGINE=terraform", "READPLAN=1"] EnvRepl.DATABRICKS_BUNDLE_ENGINE = false diff --git a/bundle/deployplan/action.go b/bundle/deployplan/action.go index e855dbb2197..05669a029b3 100644 --- a/bundle/deployplan/action.go +++ b/bundle/deployplan/action.go @@ -67,6 +67,16 @@ func (a ActionType) StringShort() string { return items[0] } +// AppliedLine renders the user-facing line reporting that action has been applied +// to resourceKey, e.g. "Created jobs.foo". The past-tense verb is the short action +// name plus "d" (create->Created, delete->Deleted, ...), capitalized to match the +// sentence case of other output. "bundle plan" keeps the lower-case present tense, +// so the two are still distinguishable at a glance. +func AppliedLine(resourceKey string, action ActionType) string { + verb := action.StringShort() + "d" + return strings.ToUpper(verb[:1]) + verb[1:] + " " + strings.TrimPrefix(resourceKey, "resources.") +} + // GetHigherAction returns the action with higher severity between a and b. // Actions are ordered by severity in actionOrder map. func GetHigherAction(a, b ActionType) ActionType { diff --git a/bundle/direct/bundle_apply.go b/bundle/direct/bundle_apply.go index 424ae2bdec9..43edcba485f 100644 --- a/bundle/direct/bundle_apply.go +++ b/bundle/direct/bundle_apply.go @@ -9,6 +9,7 @@ import ( "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/bundle/terraform_dabs_map" + "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/logdiag" "github.com/databricks/cli/libs/structs/structaccess" @@ -16,7 +17,11 @@ import ( "github.com/databricks/databricks-sdk-go" ) -func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.WorkspaceClient, plan *deployplan.Plan) { +// Apply deploys every node in plan, respecting dependency order. When reportApplied +// is set, each resource is reported as soon as it is applied, so a long deploy shows +// what it has done and a failing one still reports the resources it did apply. Nodes +// run in parallel, so the lines come out in completion order, which varies per run. +func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.WorkspaceClient, plan *deployplan.Plan, reportApplied bool) { if plan == nil { panic("Planning is not done") } @@ -111,6 +116,9 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err)) return false } + if reportApplied { + cmdio.LogString(ctx, deployplan.AppliedLine(resourceKey, action)) + } return true } @@ -139,6 +147,12 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err)) return false } + + // Reported before the remote-state refresh below: the resource is already + // deployed at this point, so the line is accurate even if the refresh fails. + if reportApplied { + cmdio.LogString(ctx, deployplan.AppliedLine(resourceKey, action)) + } } // TODO: Note, we only really need remote state if there are remote references. diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index f16088e4518..ebbe10cdd5f 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -5,7 +5,6 @@ import ( "errors" "fmt" "slices" - "strings" "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/artifacts" @@ -90,7 +89,7 @@ func deployCore(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan, st // should report the engine that ran. b.Metrics.StateEngine = stateEngine.ThisOrDefault() if stateEngine.IsDirect() { - b.DeploymentBundle.Apply(ctx, b.WorkspaceClient(ctx), plan) + b.DeploymentBundle.Apply(ctx, b.WorkspaceClient(ctx), plan, reportPerResource(b)) state, err = b.DeploymentBundle.StateDB.Finalize(ctx) // Capture the finalized state for deploy telemetry. It carries each // resource's state-size in bytes (from the WAL replay Finalize just @@ -118,6 +117,12 @@ func deployCore(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan, st ) } +// reportPerResource reports whether the deploy should list resources individually +// (-q and -qq drop those lines). +func reportPerResource(b *bundle.Bundle) bool { + return b.Quiet < bundle.QuietSummary +} + // logFileSummary reports what the file sync did. Separate from the resource summary // because a deploy that only changes business logic (a .py or .sql file) leaves every // resource unchanged, so without this line its summary is all zeros and looks like a @@ -132,22 +137,20 @@ func logFileSummary(ctx context.Context, b *bundle.Bundle) { // logDeploySummary prints the per-resource actions that were applied followed by the // resource summary line. -q drops the per-resource lines, -qq drops the summary too. -// The past-tense verb is the short action name plus "d" (create→Created, -// delete→Deleted, ...), capitalized to match the sentence case of other output. -// "bundle plan" keeps the lower-case present tense, so the two are still -// distinguishable at a glance. -func logDeploySummary(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan) { +// The direct engine prints its own lines as it goes, so only the terraform engine +// reports them from the plan here. +func logDeploySummary(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan, stateEngine engine.EngineType) { if b.Quiet >= bundle.QuietAll { return } - if b.Quiet < bundle.QuietSummary { + // The direct engine already printed these lines as each resource was applied. + if reportPerResource(b) && !stateEngine.IsDirect() { for _, action := range plan.GetActions() { if action.ActionType == deployplan.Skip || action.ActionType == deployplan.Undefined { continue } - verb := action.ActionType.StringShort() + "d" - cmdio.LogString(ctx, strings.ToUpper(verb[:1])+verb[1:]+" "+strings.TrimPrefix(action.ResourceKey, "resources.")) + cmdio.LogString(ctx, deployplan.AppliedLine(action.ResourceKey, action.ActionType)) } } @@ -319,7 +322,7 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand // still propagates. Earlier failures report the files only, since the plan // counts would then describe what was intended rather than what was applied. filesReported = true - logDeploySummary(ctx, b, plan) + logDeploySummary(ctx, b, plan, stateEngine) bundle.ApplyContext(ctx, b, scripts.Execute(config.ScriptPostDeploy)) diff --git a/bundle/phases/destroy.go b/bundle/phases/destroy.go index e19dd11fb56..6ad118677ab 100644 --- a/bundle/phases/destroy.go +++ b/bundle/phases/destroy.go @@ -136,7 +136,9 @@ func approvalForDestroy(ctx context.Context, b *bundle.Bundle, plan *deployplan. func destroyCore(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan, engine engine.EngineType) { if engine.IsDirect() { - b.DeploymentBundle.Apply(ctx, b.WorkspaceClient(ctx), plan) + // Not reported per resource: destroy names them up front for consent and then + // reports only a count, so there is no per-resource output to report into. + b.DeploymentBundle.Apply(ctx, b.WorkspaceClient(ctx), plan, false) } else { // Core destructive mutators for destroy. These require informed user consent. bundle.ApplyContext(ctx, b, terraform.Apply()) diff --git a/libs/testdiff/sortlines.go b/libs/testdiff/sortlines.go new file mode 100644 index 00000000000..332a11f1c27 --- /dev/null +++ b/libs/testdiff/sortlines.go @@ -0,0 +1,33 @@ +package testdiff + +import ( + "regexp" + "slices" + "strings" +) + +// SortLineRuns sorts each run of consecutive lines matching pattern, leaving every +// other line where it is. Use it for output whose line order is genuinely +// nondeterministic — for example "bundle deploy" reporting resources as they are +// applied, which happens in parallel — so that a test pins the set of lines without +// pinning an order the command does not guarantee. +func SortLineRuns(input string, pattern *regexp.Regexp) string { + lines := strings.Split(input, "\n") + start := -1 + for i := range lines { + if pattern.MatchString(lines[i]) { + if start < 0 { + start = i + } + continue + } + if start >= 0 { + slices.Sort(lines[start:i]) + start = -1 + } + } + if start >= 0 { + slices.Sort(lines[start:]) + } + return strings.Join(lines, "\n") +} diff --git a/libs/testdiff/sortlines_test.go b/libs/testdiff/sortlines_test.go new file mode 100644 index 00000000000..951d8b3f617 --- /dev/null +++ b/libs/testdiff/sortlines_test.go @@ -0,0 +1,55 @@ +package testdiff + +import ( + "regexp" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestSortLineRuns(t *testing.T) { + pattern := regexp.MustCompile(`^(Created|Updated|Deleted) `) + + for _, tc := range []struct { + name string + input string + want string + }{ + { + name: "sorts a run and leaves surrounding lines alone", + input: "Uploading files...\nCreated volumes.v\nCreated jobs.a\nUpdated jobs.b\nFiles: 1 uploaded\n", + want: "Uploading files...\nCreated jobs.a\nCreated volumes.v\nUpdated jobs.b\nFiles: 1 uploaded\n", + }, + { + // Runs are sorted independently: a non-matching line between them is a + // boundary, so lines never move across it. + name: "does not merge runs separated by other output", + input: "Created jobs.z\nError: boom\nCreated jobs.a\n", + want: "Created jobs.z\nError: boom\nCreated jobs.a\n", + }, + { + // Same, with runs long enough to sort: each is ordered on its own, and the + // second run stays below the boundary even though it sorts first overall. + name: "sorts each run independently", + input: "Updated jobs.z\nUpdated jobs.y\nError: boom\nCreated jobs.b\nCreated jobs.a\n", + want: "Updated jobs.y\nUpdated jobs.z\nError: boom\nCreated jobs.a\nCreated jobs.b\n", + }, + { + name: "sorts a run that ends the input", + input: ">>> deploy\nDeleted jobs.b\nDeleted jobs.a", + want: ">>> deploy\nDeleted jobs.a\nDeleted jobs.b", + }, + { + name: "leaves input without matches untouched", + input: "Plan: 1 to add\n\nResources: 1 created\n", + want: "Plan: 1 to add\n\nResources: 1 created\n", + }, + } { + t.Run(tc.name, func(t *testing.T) { + got := SortLineRuns(tc.input, pattern) + assert.Equal(t, tc.want, got) + // Sorting is idempotent, so goldens stay stable when re-recorded. + assert.Equal(t, tc.want, SortLineRuns(got, pattern)) + }) + } +}