diff --git a/.nextchanges/bundles/job-runs-on-file-change.md b/.nextchanges/bundles/job-runs-on-file-change.md new file mode 100644 index 0000000000..95869a25cc --- /dev/null +++ b/.nextchanges/bundles/job-runs-on-file-change.md @@ -0,0 +1 @@ +direct: `resources.job_runs` can set `lifecycle.triggers.on_file_change` to a path or glob under the sync root to re-fire the run when matched file contents change (content hash), or when matches appear or disappear. Removing the trigger does not recreate the existing run. diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index de252dc416..63b82c1d59 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -905,15 +905,18 @@ resources.job_runs.*.jar_params[*] string ALL resources.job_runs.*.job_id int64 ALL resources.job_runs.*.job_parameters map[string]string ALL resources.job_runs.*.job_parameters.* string ALL -resources.job_runs.*.lifecycle *dresources.JobRunLifecycleState STATE resources.job_runs.*.lifecycle *resources.JobRunLifecycle INPUT +resources.job_runs.*.lifecycle dresources.JobRunLifecycleState STATE resources.job_runs.*.lifecycle resources.Lifecycle INPUT resources.job_runs.*.lifecycle.prevent_destroy bool INPUT -resources.job_runs.*.lifecycle.triggers *dresources.JobRunTriggersState STATE resources.job_runs.*.lifecycle.triggers []resources.JobRunTrigger INPUT +resources.job_runs.*.lifecycle.triggers dresources.JobRunTriggersState STATE resources.job_runs.*.lifecycle.triggers.on_bundle_deploy string STATE +resources.job_runs.*.lifecycle.triggers.on_file_change map[string]string STATE +resources.job_runs.*.lifecycle.triggers.on_file_change.* string STATE resources.job_runs.*.lifecycle.triggers[*] resources.JobRunTrigger INPUT resources.job_runs.*.lifecycle.triggers[*].on_bundle_deploy *bool INPUT +resources.job_runs.*.lifecycle.triggers[*].on_file_change *string INPUT resources.job_runs.*.modified_status string INPUT resources.job_runs.*.notebook_params map[string]string ALL resources.job_runs.*.notebook_params.* string ALL @@ -936,6 +939,8 @@ resources.job_runs.*.python_params []string ALL resources.job_runs.*.python_params[*] string ALL resources.job_runs.*.queue *jobs.QueueSettings ALL resources.job_runs.*.queue.enabled bool ALL +resources.job_runs.*.resolved_file_triggers map[string]string INPUT +resources.job_runs.*.resolved_file_triggers.* string INPUT resources.job_runs.*.resolved_job_id int64 INPUT resources.job_runs.*.result_state jobs.RunResultState REMOTE STATE resources.job_runs.*.run_id int64 REMOTE diff --git a/acceptance/bundle/resources/job_runs/failed_run/output.txt b/acceptance/bundle/resources/job_runs/failed_run/output.txt index f8b8dbf439..5b8b6dbcd5 100644 --- a/acceptance/bundle/resources/job_runs/failed_run/output.txt +++ b/acceptance/bundle/resources/job_runs/failed_run/output.txt @@ -27,6 +27,9 @@ Files: 7 uploaded, 0 deleted "new_state": { "value": { "job_id": [MY_JOB_ID], + "lifecycle": { + "triggers": {} + }, "result_state": "SUCCESS" } }, diff --git a/acceptance/bundle/resources/job_runs/on_bundle_deploy/output.txt b/acceptance/bundle/resources/job_runs/on_bundle_deploy/output.txt index 39123f3370..6cd8d44a41 100644 --- a/acceptance/bundle/resources/job_runs/on_bundle_deploy/output.txt +++ b/acceptance/bundle/resources/job_runs/on_bundle_deploy/output.txt @@ -30,20 +30,6 @@ Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle": { - "action": "recreate", - "reason": "immutable", - "old": { - "triggers": { - "on_bundle_deploy": "[UUID]" - } - }, - "new": { - "triggers": { - "on_bundle_deploy": "[UUID]" - } - } - }, "lifecycle.triggers.on_bundle_deploy": { "action": "recreate", "reason": "immutable", @@ -108,14 +94,10 @@ Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle": { + "lifecycle.triggers.on_bundle_deploy": { "action": "skip", "reason": "trigger removed", - "old": { - "triggers": { - "on_bundle_deploy": "[UUID]" - } - } + "old": "[UUID]" }, "result_state": { "action": "skip", diff --git a/acceptance/bundle/resources/job_runs/on_file_change/databricks.yml b/acceptance/bundle/resources/job_runs/on_file_change/databricks.yml new file mode 100644 index 0000000000..0a6df75047 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change/databricks.yml @@ -0,0 +1,18 @@ +bundle: + name: job-runs-on-file-change + +resources: + jobs: + my_job: + name: my-job + tasks: + - task_key: main + notebook_task: + notebook_path: /Workspace/test + + job_runs: + my_run: + job_id: ${resources.jobs.my_job.id} + lifecycle: + triggers: + - on_file_change: seed.txt diff --git a/acceptance/bundle/resources/job_runs/on_file_change/out.test.toml b/acceptance/bundle/resources/job_runs/on_file_change/out.test.toml new file mode 100644 index 0000000000..57b0f61685 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change/out.test.toml @@ -0,0 +1,3 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/job_runs/on_file_change/output.txt b/acceptance/bundle/resources/job_runs/on_file_change/output.txt new file mode 100644 index 0000000000..978e4b4394 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change/output.txt @@ -0,0 +1,207 @@ + +=== first deploy triggers a run +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change/default/files... +Output from job_runs.my_run: id=[MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?o=[NUMID] +Output from job_runs.my_run: id=[MY_RUN_ID]: SUCCESS +Created job_runs.my_run +Created jobs.my_job +Files: 6 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +>>> read_id.py my_job +[MY_JOB_ID] + +>>> print_requests.py //jobs/run-now +{ + "method": "POST", + "path": "/api/2.2/jobs/run-now", + "body": { + "idempotency_token": "[UUID]", + "job_id": [MY_JOB_ID] + } +} + +=== redeploy with unchanged file plans nothing +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change/default/files... +Files: 2 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 2 unchanged + +>>> print_requests.py //jobs/run-now + +=== touch without content change plans nothing +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change/default/files... +Files: 3 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 2 unchanged + +>>> print_requests.py //jobs/run-now + +=== editing the file re-fires +>>> [CLI] bundle plan +recreate job_runs.my_run + +Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged + +>>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json +{ + "lifecycle.triggers.on_file_change": { + "action": "recreate", + "reason": "immutable", + "old": { + "seed.txt": "[FILE_HASH][0]" + }, + "new": { + "seed.txt": "[FILE_HASH][1]" + } + }, + "lifecycle.triggers.on_file_change['seed.txt']": { + "action": "recreate", + "reason": "immutable", + "old": "[FILE_HASH][0]", + "new": "[FILE_HASH][1]" + }, + "result_state": { + "action": "skip", + "reason": "remote_already_set", + "new": "SUCCESS", + "remote": "SUCCESS" + } +} + +=== bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change/default/files... +Output from job_runs.my_run: id=[MY_RUN_ID_2]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_2]?o=[NUMID] +Output from job_runs.my_run: id=[MY_RUN_ID_2]: SUCCESS +Recreated job_runs.my_run +Files: 4 uploaded, 0 deleted +Resources: 1 created, 0 changed, 1 deleted, 1 unchanged + +>>> print_requests.py --keep //jobs/runs/delete +{ + "method": "POST", + "path": "/api/2.2/jobs/runs/delete", + "body": { + "run_id": [MY_RUN_ID] + } +} + +>>> print_requests.py //jobs/run-now +{ + "method": "POST", + "path": "/api/2.2/jobs/run-now", + "body": { + "idempotency_token": "[UUID]", + "job_id": [MY_JOB_ID] + } +} + +=== deleting the file warns and re-fires +>>> [CLI] bundle plan +Warning: lifecycle.triggers.on_file_change: no files match "seed.txt" + in databricks.yml:18:29 + +recreate job_runs.my_run + +Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged + +>>> changes +Warning: lifecycle.triggers.on_file_change: no files match "seed.txt" + in databricks.yml:18:29 + +{ + "lifecycle.triggers.on_file_change": { + "action": "recreate", + "reason": "immutable", + "old": { + "seed.txt": "[FILE_HASH][1]" + }, + "new": { + "seed.txt": "" + } + }, + "lifecycle.triggers.on_file_change['seed.txt']": { + "action": "recreate", + "reason": "immutable", + "old": "[FILE_HASH][1]", + "new": "" + }, + "result_state": { + "action": "skip", + "reason": "remote_already_set", + "new": "SUCCESS", + "remote": "SUCCESS" + } +} + +>>> [CLI] bundle deploy +Warning: lifecycle.triggers.on_file_change: no files match "seed.txt" + in databricks.yml:18:29 + +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change/default/files... +Output from job_runs.my_run: id=[MY_RUN_ID_3]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_3]?o=[NUMID] +Output from job_runs.my_run: id=[MY_RUN_ID_3]: SUCCESS +Recreated job_runs.my_run +Files: 3 uploaded, 1 deleted +Resources: 1 created, 0 changed, 1 deleted, 1 unchanged + +>>> print_requests.py //jobs/run-now +{ + "method": "POST", + "path": "/api/2.2/jobs/run-now", + "body": { + "idempotency_token": "[UUID]", + "job_id": [MY_JOB_ID] + } +} + +=== restoring the file re-fires again +>>> [CLI] bundle plan +recreate job_runs.my_run + +Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change/default/files... +Output from job_runs.my_run: id=[MY_RUN_ID_4]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_4]?o=[NUMID] +Output from job_runs.my_run: id=[MY_RUN_ID_4]: SUCCESS +Recreated job_runs.my_run +Files: 3 uploaded, 1 deleted +Resources: 1 created, 0 changed, 1 deleted, 1 unchanged + +>>> print_requests.py //jobs/run-now +{ + "method": "POST", + "path": "/api/2.2/jobs/run-now", + "body": { + "idempotency_token": "[UUID]", + "job_id": [MY_JOB_ID] + } +} + +=== removing on_file_change is unchanged and does not run +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change/default/files... +Files: 3 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 2 unchanged + +>>> print_requests.py //jobs/run-now + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.job_runs.my_run + delete resources.jobs.my_job + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change/default + +Destroy: 2 deleted diff --git a/acceptance/bundle/resources/job_runs/on_file_change/script b/acceptance/bundle/resources/job_runs/on_file_change/script new file mode 100644 index 0000000000..ef432d7603 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change/script @@ -0,0 +1,64 @@ +cleanup() { + trace $CLI bundle destroy --auto-approve + rm -f out.requests.txt +} +trap cleanup EXIT + +changes() { + $CLI bundle plan -o json | jq '.plan["resources.job_runs.my_run"].changes' +} + +title "first deploy triggers a run" +trace $CLI bundle deploy +trace read_id.py my_job +# Name the first run so the recreated one becomes [MY_RUN_ID_2]. +read_id.py my_run > /dev/null +trace print_requests.py //jobs/run-now + +title "redeploy with unchanged file plans nothing" +trace $CLI bundle plan +trace $CLI bundle deploy +trace print_requests.py //jobs/run-now + +title "touch without content change plans nothing" +python -c "import os; os.utime('seed.txt', None)" +trace $CLI bundle plan +trace $CLI bundle deploy +trace print_requests.py //jobs/run-now + +title "editing the file re-fires" +update_file.py seed.txt "v1" "v2" +trace $CLI bundle plan +# Save the plan so the READPLAN=1 variant deploys the hashes computed here. +# The deploy is not traced: readplanarg makes the command line differ per variant. +$CLI bundle plan -o json > tmp.plan.json +trace jq '.plan["resources.job_runs.my_run"].changes' tmp.plan.json +title "bundle deploy\n" +$CLI bundle deploy $(readplanarg tmp.plan.json) +read_id.py my_run > /dev/null +trace print_requests.py --keep //jobs/runs/delete +trace print_requests.py //jobs/run-now + +title "deleting the file warns and re-fires" +mv seed.txt tmp.seed.txt +trace $CLI bundle plan +trace changes +trace $CLI bundle deploy +read_id.py my_run > /dev/null +trace print_requests.py //jobs/run-now + +title "restoring the file re-fires again" +mv tmp.seed.txt seed.txt +trace $CLI bundle plan +trace $CLI bundle deploy +read_id.py my_run > /dev/null +trace print_requests.py //jobs/run-now + +title "removing on_file_change is unchanged and does not run" +update_file.py databricks.yml " lifecycle: + triggers: + - on_file_change: seed.txt +" "" +trace $CLI bundle plan +trace $CLI bundle deploy +trace print_requests.py //jobs/run-now diff --git a/acceptance/bundle/resources/job_runs/on_file_change/seed.txt b/acceptance/bundle/resources/job_runs/on_file_change/seed.txt new file mode 100644 index 0000000000..626799f0f8 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change/seed.txt @@ -0,0 +1 @@ +v1 diff --git a/acceptance/bundle/resources/job_runs/on_file_change/test.toml b/acceptance/bundle/resources/job_runs/on_file_change/test.toml new file mode 100644 index 0000000000..7a950e9fd0 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change/test.toml @@ -0,0 +1,14 @@ +# Deploy both by re-planning and from a plan saved on disk, so the hashes +# computed during planning survive plan serialization. +EnvMatrix.READPLAN = ["", "1"] + +Ignore = ["tmp.plan.json", "tmp.seed.txt"] + +# Content hashes are stable but unreadable, and the parent's `\d{8,}` rule +# mangles any digit run inside them. Order=1 runs before it; Distinct keeps +# different hashes distinguishable. +[[Repls]] +Old = '[0-9a-f]{64}' +New = '[FILE_HASH]' +Order = 1 +Distinct = true diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/absolute_pattern.yml b/acceptance/bundle/resources/job_runs/on_file_change_errors/absolute_pattern.yml new file mode 100644 index 0000000000..e0d709d3fa --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/absolute_pattern.yml @@ -0,0 +1,18 @@ +bundle: + name: job-runs-on-file-change-errors + +resources: + jobs: + my_job: + name: my-job + tasks: + - task_key: main + notebook_task: + notebook_path: /Workspace/test + + job_runs: + my_run: + job_id: ${resources.jobs.my_job.id} + lifecycle: + triggers: + - on_file_change: /tmp/seed.txt diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/allexcluded/a.txt b/acceptance/bundle/resources/job_runs/on_file_change_errors/allexcluded/a.txt new file mode 100644 index 0000000000..7898192261 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/allexcluded/a.txt @@ -0,0 +1 @@ +a diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/databricks.yml b/acceptance/bundle/resources/job_runs/on_file_change_errors/databricks.yml new file mode 100644 index 0000000000..0ce9819887 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/databricks.yml @@ -0,0 +1,24 @@ +bundle: + name: job-runs-on-file-change-errors + +sync: + exclude: + - mixed/excluded.txt + # Literal path: the script rewrites the trigger by substring, and allexcluded/* would hit this line. + - allexcluded/a.txt + +resources: + jobs: + my_job: + name: my-job + tasks: + - task_key: main + notebook_task: + notebook_path: /Workspace/test + + job_runs: + my_run: + job_id: ${resources.jobs.my_job.id} + lifecycle: + triggers: + - on_file_change: ../shared/*.txt diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/dirs/nested/keep.txt b/acceptance/bundle/resources/job_runs/on_file_change_errors/dirs/nested/keep.txt new file mode 100644 index 0000000000..79c53955ef --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/dirs/nested/keep.txt @@ -0,0 +1 @@ +nested diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/mixed/excluded.txt b/acceptance/bundle/resources/job_runs/on_file_change_errors/mixed/excluded.txt new file mode 100644 index 0000000000..bbde3dc9de --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/mixed/excluded.txt @@ -0,0 +1 @@ +excluded diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/mixed/nested/keep.txt b/acceptance/bundle/resources/job_runs/on_file_change_errors/mixed/nested/keep.txt new file mode 100644 index 0000000000..2fa992c0b8 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/mixed/nested/keep.txt @@ -0,0 +1 @@ +keep diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/out.test.toml b/acceptance/bundle/resources/job_runs/on_file_change_errors/out.test.toml new file mode 100644 index 0000000000..0938e67898 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt new file mode 100644 index 0000000000..efcd8f002b --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt @@ -0,0 +1,91 @@ + +=== a pattern that escapes the sync root is rejected +>>> musterr [CLI] bundle validate +Error: lifecycle.triggers.on_file_change: pattern "../shared/*.txt" is not under the sync root + in databricks.yml:24:29 + +Name: job-runs-on-file-change-errors +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-errors/default + +Found 1 error + +=== a glob that only matches directories warns +>>> [CLI] bundle validate +Warning: lifecycle.triggers.on_file_change: pattern "dirs/*" matches only directories or files excluded from sync, so nothing is hashed + in databricks.yml:24:29 + +Name: job-runs-on-file-change-errors +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-errors/default + +Found 1 warning + +=== a glob matching a directory and an excluded file warns +>>> [CLI] bundle validate +Warning: lifecycle.triggers.on_file_change: pattern "mixed/*" matches only directories or files excluded from sync, so nothing is hashed + in databricks.yml:24:29 + +Name: job-runs-on-file-change-errors +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-errors/default + +Found 1 warning + +=== a glob matching only excluded files warns +>>> [CLI] bundle validate +Warning: lifecycle.triggers.on_file_change: pattern "allexcluded/*" matches only directories or files excluded from sync, so nothing is hashed + in databricks.yml:24:29 + +Name: job-runs-on-file-change-errors +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-errors/default + +Found 1 warning + +=== a pattern that matches nothing warns +>>> [CLI] bundle validate +Warning: lifecycle.triggers.on_file_change: no files match "missing/*.txt" + in databricks.yml:24:29 + +Name: job-runs-on-file-change-errors +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-errors/default + +Found 1 warning + +=== ** is not supported +>>> musterr [CLI] bundle validate +Error: lifecycle.triggers.on_file_change: ** in "src/**" is not supported; use * for a single directory level + in databricks.yml:24:29 + +Name: job-runs-on-file-change-errors +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-errors/default + +Found 1 error + +=== an absolute pattern is rejected +>>> musterr [CLI] bundle validate +Error: lifecycle.triggers.on_file_change: pattern "/tmp/seed.txt" must be relative to the defining YAML file + in databricks.yml:18:29 + +Name: job-runs-on-file-change-errors +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-errors/default + +Found 1 error diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/script b/acceptance/bundle/resources/job_runs/on_file_change_errors/script new file mode 100644 index 0000000000..faaa5de8d3 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/script @@ -0,0 +1,31 @@ +title "a pattern that escapes the sync root is rejected" +trace musterr $CLI bundle validate + +title "a glob that only matches directories warns" +# dirs/* matches dirs/nested, a directory, so nothing is left to hash. +update_file.py databricks.yml "../shared/*.txt" "dirs/*" +trace $CLI bundle validate + +title "a glob matching a directory and an excluded file warns" +# mixed/excluded.txt is dropped by sync.exclude; mixed/nested is a directory. +update_file.py databricks.yml "dirs/*" "mixed/*" +trace $CLI bundle validate + +title "a glob matching only excluded files warns" +# Every match is dropped by sync.exclude. +update_file.py databricks.yml "mixed/*" "allexcluded/*" +trace $CLI bundle validate + +title "a pattern that matches nothing warns" +update_file.py databricks.yml "allexcluded/*" "missing/*.txt" +trace $CLI bundle validate + +title "** is not supported" +update_file.py databricks.yml "missing/*.txt" "src/**" +trace musterr $CLI bundle validate + +title "an absolute pattern is rejected" +# Copied from a committed file rather than passed to update_file.py: Git Bash on +# Windows rewrites a leading-slash argument into a Windows path. +cp absolute_pattern.yml databricks.yml +trace musterr $CLI bundle validate diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/test.toml b/acceptance/bundle/resources/job_runs/on_file_change_errors/test.toml new file mode 100644 index 0000000000..7b88d613d4 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/test.toml @@ -0,0 +1,2 @@ +# The bundle is never deployed here, so there are no requests worth recording. +RecordRequests = false diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/databricks.yml b/acceptance/bundle/resources/job_runs/on_file_change_glob/databricks.yml new file mode 100644 index 0000000000..d47fcf4404 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/databricks.yml @@ -0,0 +1,19 @@ +bundle: + name: job-runs-on-file-change-glob + +resources: + jobs: + my_job: + name: my-job + tasks: + - task_key: main + notebook_task: + notebook_path: /Workspace/test + + job_runs: + my_run: + job_id: ${resources.jobs.my_job.id} + lifecycle: + triggers: + - on_file_change: migrations/* + - on_file_change: mysubdir/*.txt diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/dot_gitignore b/acceptance/bundle/resources/job_runs/on_file_change_glob/dot_gitignore new file mode 100644 index 0000000000..57433d43f4 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/dot_gitignore @@ -0,0 +1,2 @@ +# Matches migrations/* but sync skips it, so a content edit must not re-fire. +migrations/ignored.txt diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/001.txt b/acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/001.txt new file mode 100644 index 0000000000..5626abf0f7 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/001.txt @@ -0,0 +1 @@ +one diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/002.txt b/acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/002.txt new file mode 100644 index 0000000000..f719efd430 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/002.txt @@ -0,0 +1 @@ +two diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/archive/003.txt b/acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/archive/003.txt new file mode 100644 index 0000000000..2bdf67abb1 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/archive/003.txt @@ -0,0 +1 @@ +three diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/ignored.txt b/acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/ignored.txt new file mode 100644 index 0000000000..ea10ec85c1 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/ignored.txt @@ -0,0 +1 @@ +ignored diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/mysubdir/a.txt b/acceptance/bundle/resources/job_runs/on_file_change_glob/mysubdir/a.txt new file mode 100644 index 0000000000..62e0af52c1 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/mysubdir/a.txt @@ -0,0 +1 @@ +sub diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/out.test.toml b/acceptance/bundle/resources/job_runs/on_file_change_glob/out.test.toml new file mode 100644 index 0000000000..57b0f61685 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/out.test.toml @@ -0,0 +1,3 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt new file mode 100644 index 0000000000..ffd878ba0c --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt @@ -0,0 +1,215 @@ + +=== deploy hashes the matched files +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-glob/default/files... +Output from job_runs.my_run: id=[MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?o=[NUMID] +Output from job_runs.my_run: id=[MY_RUN_ID]: SUCCESS +Created job_runs.my_run +Created jobs.my_job +Files: 10 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +>>> read_id.py my_job +[MY_JOB_ID] + +>>> print_requests.py //jobs/run-now +{ + "method": "POST", + "path": "/api/2.2/jobs/run-now", + "body": { + "idempotency_token": "[UUID]", + "job_id": [MY_JOB_ID] + } +} + +=== editing one matched file re-fires + +>>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json +{ + "lifecycle.triggers.on_file_change": { + "action": "recreate", + "reason": "immutable", + "old": { + "migrations/001.txt": "[FILE_HASH][0]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][2]" + }, + "new": { + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][2]" + } + }, + "lifecycle.triggers.on_file_change['migrations/001.txt']": { + "action": "recreate", + "reason": "immutable", + "old": "[FILE_HASH][0]", + "new": "[FILE_HASH][3]" + }, + "result_state": { + "action": "skip", + "reason": "remote_already_set", + "new": "SUCCESS", + "remote": "SUCCESS" + } +} + +=== bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-glob/default/files... +Output from job_runs.my_run: id=[MY_RUN_ID_2]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_2]?o=[NUMID] +Output from job_runs.my_run: id=[MY_RUN_ID_2]: SUCCESS +Recreated job_runs.my_run +Files: 4 uploaded, 0 deleted +Resources: 1 created, 0 changed, 1 deleted, 1 unchanged + +>>> print_requests.py --keep //jobs/runs/delete +{ + "method": "POST", + "path": "/api/2.2/jobs/runs/delete", + "body": { + "run_id": [MY_RUN_ID] + } +} + +>>> print_requests.py //jobs/run-now +{ + "method": "POST", + "path": "/api/2.2/jobs/run-now", + "body": { + "idempotency_token": "[UUID]", + "job_id": [MY_JOB_ID] + } +} + +=== editing a file in a subdirectory re-fires + +>>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json +{ + "lifecycle.triggers.on_file_change": { + "action": "recreate", + "reason": "immutable", + "old": { + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][2]" + }, + "new": { + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][4]" + } + }, + "lifecycle.triggers.on_file_change['mysubdir/a.txt']": { + "action": "recreate", + "reason": "immutable", + "old": "[FILE_HASH][2]", + "new": "[FILE_HASH][4]" + }, + "result_state": { + "action": "skip", + "reason": "remote_already_set", + "new": "SUCCESS", + "remote": "SUCCESS" + } +} + +=== a new matching file re-fires + +>>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json +{ + "lifecycle.triggers.on_file_change": { + "action": "recreate", + "reason": "immutable", + "old": { + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][2]" + }, + "new": { + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "migrations/003.txt": "[FILE_HASH][5]", + "mysubdir/a.txt": "[FILE_HASH][2]" + } + }, + "lifecycle.triggers.on_file_change['migrations/003.txt']": { + "action": "recreate", + "reason": "immutable", + "new": "[FILE_HASH][5]" + }, + "result_state": { + "action": "skip", + "reason": "remote_already_set", + "new": "SUCCESS", + "remote": "SUCCESS" + } +} + +=== removing a matched file re-fires + +>>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json +{ + "lifecycle.triggers.on_file_change": { + "action": "recreate", + "reason": "immutable", + "old": { + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][2]" + }, + "new": { + "migrations/001.txt": "[FILE_HASH][3]", + "mysubdir/a.txt": "[FILE_HASH][2]" + } + }, + "lifecycle.triggers.on_file_change['migrations/002.txt']": { + "action": "recreate", + "reason": "immutable", + "old": "[FILE_HASH][1]" + }, + "result_state": { + "action": "skip", + "reason": "remote_already_set", + "new": "SUCCESS", + "remote": "SUCCESS" + } +} + +=== editing a file below the glob does not re-fire + +>>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json +{ + "lifecycle.triggers.on_file_change": { + "action": "skip", + "reason": "missing_in_remote", + "old": { + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][2]" + }, + "new": { + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][2]" + } + }, + "result_state": { + "action": "skip", + "reason": "remote_already_set", + "new": "SUCCESS", + "remote": "SUCCESS" + } +} + +=== editing a gitignored file does not re-fire +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.job_runs.my_run + delete resources.jobs.my_job + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-glob/default + +Destroy: 2 deleted diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/script b/acceptance/bundle/resources/job_runs/on_file_change_glob/script new file mode 100644 index 0000000000..f0130f2637 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/script @@ -0,0 +1,59 @@ +# Fixture is named dot_gitignore so a real .gitignore does not affect git while developing. +mv dot_gitignore .gitignore + +cleanup() { + trace $CLI bundle destroy --auto-approve + rm -f out.requests.txt +} +trap cleanup EXIT + +# Plan once, print JSON changes, leave tmp.plan.json for readplanarg. +changes() { + $CLI bundle plan -o json > tmp.plan.json + trace jq '.plan["resources.job_runs.my_run"].changes' tmp.plan.json +} + +title "deploy hashes the matched files" +trace $CLI bundle deploy +trace read_id.py my_job +read_id.py my_run > /dev/null +trace print_requests.py //jobs/run-now + +title "editing one matched file re-fires\n" +update_file.py migrations/001.txt "one" "one-edited" +changes +# The deploy is not traced: readplanarg makes the command line differ per variant. +title "bundle deploy\n" +$CLI bundle deploy $(readplanarg tmp.plan.json) +read_id.py my_run > /dev/null +trace print_requests.py --keep //jobs/runs/delete +trace print_requests.py //jobs/run-now + +title "editing a file in a subdirectory re-fires\n" +# printf keeps LF on Windows, where update_file.py's text-mode rewrite would +# restore CRLF and leave the file hashing differently than the deploy above. +printf 'sub-edited\n' > mysubdir/a.txt +changes +printf 'sub\n' > mysubdir/a.txt + +title "a new matching file re-fires\n" +printf 'three\n' > migrations/003.txt +changes +rm migrations/003.txt + +title "removing a matched file re-fires\n" +mv migrations/002.txt tmp.002.txt +changes +mv tmp.002.txt migrations/002.txt + +title "editing a file below the glob does not re-fire\n" +# migrations/* matches the archive directory itself, which is skipped: the glob +# does not descend into it. +update_file.py migrations/archive/003.txt "three" "three-edited" +changes +update_file.py migrations/archive/003.txt "three-edited" "three" + +title "editing a gitignored file does not re-fire" +update_file.py migrations/ignored.txt "ignored" "ignored-edited" +trace $CLI bundle plan +update_file.py migrations/ignored.txt "ignored-edited" "ignored" diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml b/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml new file mode 100644 index 0000000000..98c98631e5 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml @@ -0,0 +1,13 @@ +# Deploy both by re-planning and from a plan saved on disk, so the hashes +# computed during planning survive plan serialization. +EnvMatrix.READPLAN = ["", "1"] + +Ignore = ["tmp.plan.json", "tmp.002.txt", ".gitignore"] + +# See on_file_change/test.toml: mask content hashes before the parent's +# `\d{8,}` rule can mangle them, keeping distinct hashes distinguishable. +[[Repls]] +Old = '[0-9a-f]{64}' +New = '[FILE_HASH]' +Order = 1 +Distinct = true diff --git a/acceptance/bundle/resources/job_runs/on_file_change_include/databricks.yml b/acceptance/bundle/resources/job_runs/on_file_change_include/databricks.yml new file mode 100644 index 0000000000..3d8b74d28f --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_include/databricks.yml @@ -0,0 +1,14 @@ +bundle: + name: job-runs-on-file-change-include + +include: + - resources/job_run.yml + +resources: + jobs: + my_job: + name: my-job + tasks: + - task_key: main + notebook_task: + notebook_path: /Workspace/test diff --git a/acceptance/bundle/resources/job_runs/on_file_change_include/out.test.toml b/acceptance/bundle/resources/job_runs/on_file_change_include/out.test.toml new file mode 100644 index 0000000000..0938e67898 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_include/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/job_runs/on_file_change_include/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_include/output.txt new file mode 100644 index 0000000000..52de054e73 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_include/output.txt @@ -0,0 +1,29 @@ + +=== first deploy +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-include/default/files... +Output from job_runs.my_run: id=[NUMID]: Run URL: [DATABRICKS_URL]/jobs/[NUMID]/runs/[NUMID]?o=[NUMID] +Output from job_runs.my_run: id=[NUMID]: SUCCESS +Created job_runs.my_run +Created jobs.my_job +Files: 7 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +=== editing a same-named file at the bundle root does not re-fire +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +=== editing the file next to the included YAML re-fires +>>> [CLI] bundle plan +recreate job_runs.my_run + +Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.job_runs.my_run + delete resources.jobs.my_job + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-include/default + +Destroy: 2 deleted diff --git a/acceptance/bundle/resources/job_runs/on_file_change_include/resources/job_run.yml b/acceptance/bundle/resources/job_runs/on_file_change_include/resources/job_run.yml new file mode 100644 index 0000000000..091489142f --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_include/resources/job_run.yml @@ -0,0 +1,7 @@ +resources: + job_runs: + my_run: + job_id: ${resources.jobs.my_job.id} + lifecycle: + triggers: + - on_file_change: watched.txt diff --git a/acceptance/bundle/resources/job_runs/on_file_change_include/resources/watched.txt b/acceptance/bundle/resources/job_runs/on_file_change_include/resources/watched.txt new file mode 100644 index 0000000000..8b4e79d3c4 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_include/resources/watched.txt @@ -0,0 +1 @@ +include-v1 diff --git a/acceptance/bundle/resources/job_runs/on_file_change_include/script b/acceptance/bundle/resources/job_runs/on_file_change_include/script new file mode 100644 index 0000000000..65a8cea7a5 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_include/script @@ -0,0 +1,17 @@ +cleanup() { + trace $CLI bundle destroy --auto-approve +} +trap cleanup EXIT + +title "first deploy" +trace $CLI bundle deploy + +title "editing a same-named file at the bundle root does not re-fire" +update_file.py watched.txt "root-v1" "root-v2" +trace $CLI bundle plan | contains.py "2 unchanged" "!recreate" +update_file.py watched.txt "root-v2" "root-v1" + +title "editing the file next to the included YAML re-fires" +update_file.py resources/watched.txt "include-v1" "include-v2" +trace $CLI bundle plan | contains.py "recreate job_runs.my_run" +update_file.py resources/watched.txt "include-v2" "include-v1" diff --git a/acceptance/bundle/resources/job_runs/on_file_change_include/test.toml b/acceptance/bundle/resources/job_runs/on_file_change_include/test.toml new file mode 100644 index 0000000000..c81bf4f943 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_include/test.toml @@ -0,0 +1,3 @@ +# Path resolution only; the HTTP traffic is the same as on_file_change. +RecordRequests = false +Ignore = [".databricks"] diff --git a/acceptance/bundle/resources/job_runs/on_file_change_include/watched.txt b/acceptance/bundle/resources/job_runs/on_file_change_include/watched.txt new file mode 100644 index 0000000000..d168853099 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_include/watched.txt @@ -0,0 +1 @@ +root-v1 diff --git a/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/bundle/databricks.yml b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/bundle/databricks.yml new file mode 100644 index 0000000000..2bdd1c06d1 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/bundle/databricks.yml @@ -0,0 +1,19 @@ +bundle: + name: job-runs-on-file-change-include-sync-root + +include: + - resources/job_run.yml + +sync: + paths: + - . + - ../shared.txt + +resources: + jobs: + my_job: + name: my-job + tasks: + - task_key: main + notebook_task: + notebook_path: /Workspace/test diff --git a/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/bundle/resources/job_run.yml b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/bundle/resources/job_run.yml new file mode 100644 index 0000000000..091489142f --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/bundle/resources/job_run.yml @@ -0,0 +1,7 @@ +resources: + job_runs: + my_run: + job_id: ${resources.jobs.my_job.id} + lifecycle: + triggers: + - on_file_change: watched.txt diff --git a/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/bundle/resources/watched.txt b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/bundle/resources/watched.txt new file mode 100644 index 0000000000..8b4e79d3c4 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/bundle/resources/watched.txt @@ -0,0 +1 @@ +include-v1 diff --git a/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/bundle/watched.txt b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/bundle/watched.txt new file mode 100644 index 0000000000..6f22941eac --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/bundle/watched.txt @@ -0,0 +1 @@ +bundle-v1 diff --git a/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/out.test.toml b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/out.test.toml new file mode 100644 index 0000000000..0938e67898 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/output.txt new file mode 100644 index 0000000000..6c475c0dbf --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/output.txt @@ -0,0 +1,33 @@ + +=== first deploy +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-include-sync-root/default/files... +Output from job_runs.my_run: id=[NUMID]: Run URL: [DATABRICKS_URL]/jobs/[NUMID]/runs/[NUMID]?o=[NUMID] +Output from job_runs.my_run: id=[NUMID]: SUCCESS +Created job_runs.my_run +Created jobs.my_job +Files: 5 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +=== editing a same-named file in the bundle does not re-fire +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +=== editing a file at the sync root does not re-fire +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +=== editing the file next to the included YAML re-fires +>>> [CLI] bundle plan +recreate job_runs.my_run + +Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.job_runs.my_run + delete resources.jobs.my_job + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-include-sync-root/default + +Destroy: 2 deleted diff --git a/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/script b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/script new file mode 100644 index 0000000000..5a608ea918 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/script @@ -0,0 +1,22 @@ +cleanup() { + withdir bundle trace $CLI bundle destroy --auto-approve +} +trap cleanup EXIT + +title "first deploy" +withdir bundle trace $CLI bundle deploy + +title "editing a same-named file in the bundle does not re-fire" +update_file.py bundle/watched.txt "bundle-v1" "bundle-v2" +withdir bundle trace $CLI bundle plan | contains.py "2 unchanged" "!recreate" +update_file.py bundle/watched.txt "bundle-v2" "bundle-v1" + +title "editing a file at the sync root does not re-fire" +update_file.py shared.txt "parent-v1" "parent-v2" +withdir bundle trace $CLI bundle plan | contains.py "2 unchanged" "!recreate" +update_file.py shared.txt "parent-v2" "parent-v1" + +title "editing the file next to the included YAML re-fires" +update_file.py bundle/resources/watched.txt "include-v1" "include-v2" +withdir bundle trace $CLI bundle plan | contains.py "recreate job_runs.my_run" +update_file.py bundle/resources/watched.txt "include-v2" "include-v1" diff --git a/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/shared.txt b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/shared.txt new file mode 100644 index 0000000000..4ddcc71c37 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/shared.txt @@ -0,0 +1 @@ +parent-v1 diff --git a/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/test.toml b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/test.toml new file mode 100644 index 0000000000..e1290bad9f --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/test.toml @@ -0,0 +1,2 @@ +RecordRequests = false +Ignore = [".databricks"] diff --git a/acceptance/bundle/resources/job_runs/on_file_change_sync_root/bundle/databricks.yml b/acceptance/bundle/resources/job_runs/on_file_change_sync_root/bundle/databricks.yml new file mode 100644 index 0000000000..772a7ef92e --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_sync_root/bundle/databricks.yml @@ -0,0 +1,23 @@ +bundle: + name: job-runs-on-file-change-sync-root + +sync: + paths: + - . + - ../shared.txt + +resources: + jobs: + my_job: + name: my-job + tasks: + - task_key: main + notebook_task: + notebook_path: /Workspace/test + + job_runs: + my_run: + job_id: ${resources.jobs.my_job.id} + lifecycle: + triggers: + - on_file_change: ../shared.txt diff --git a/acceptance/bundle/resources/job_runs/on_file_change_sync_root/bundle/shared.txt b/acceptance/bundle/resources/job_runs/on_file_change_sync_root/bundle/shared.txt new file mode 100644 index 0000000000..6f22941eac --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_sync_root/bundle/shared.txt @@ -0,0 +1 @@ +bundle-v1 diff --git a/acceptance/bundle/resources/job_runs/on_file_change_sync_root/out.test.toml b/acceptance/bundle/resources/job_runs/on_file_change_sync_root/out.test.toml new file mode 100644 index 0000000000..0938e67898 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_sync_root/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/job_runs/on_file_change_sync_root/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_sync_root/output.txt new file mode 100644 index 0000000000..6a52f127ce --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_sync_root/output.txt @@ -0,0 +1,29 @@ + +=== first deploy +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-sync-root/default/files... +Output from job_runs.my_run: id=[NUMID]: Run URL: [DATABRICKS_URL]/jobs/[NUMID]/runs/[NUMID]?o=[NUMID] +Output from job_runs.my_run: id=[NUMID]: SUCCESS +Created job_runs.my_run +Created jobs.my_job +Files: 3 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +=== editing a same-named file in the bundle does not re-fire +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +=== editing the file at the sync root re-fires +>>> [CLI] bundle plan +recreate job_runs.my_run + +Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.job_runs.my_run + delete resources.jobs.my_job + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-sync-root/default + +Destroy: 2 deleted diff --git a/acceptance/bundle/resources/job_runs/on_file_change_sync_root/script b/acceptance/bundle/resources/job_runs/on_file_change_sync_root/script new file mode 100644 index 0000000000..dcbc475840 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_sync_root/script @@ -0,0 +1,17 @@ +cleanup() { + withdir bundle trace $CLI bundle destroy --auto-approve +} +trap cleanup EXIT + +title "first deploy" +withdir bundle trace $CLI bundle deploy + +title "editing a same-named file in the bundle does not re-fire" +update_file.py bundle/shared.txt "bundle-v1" "bundle-v2" +withdir bundle trace $CLI bundle plan | contains.py "2 unchanged" "!recreate" +update_file.py bundle/shared.txt "bundle-v2" "bundle-v1" + +title "editing the file at the sync root re-fires" +update_file.py shared.txt "sync-v1" "sync-v2" +withdir bundle trace $CLI bundle plan | contains.py "recreate job_runs.my_run" +update_file.py shared.txt "sync-v2" "sync-v1" diff --git a/acceptance/bundle/resources/job_runs/on_file_change_sync_root/shared.txt b/acceptance/bundle/resources/job_runs/on_file_change_sync_root/shared.txt new file mode 100644 index 0000000000..04f175f2a1 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_sync_root/shared.txt @@ -0,0 +1 @@ +sync-v1 diff --git a/acceptance/bundle/resources/job_runs/on_file_change_sync_root/test.toml b/acceptance/bundle/resources/job_runs/on_file_change_sync_root/test.toml new file mode 100644 index 0000000000..e1290bad9f --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_sync_root/test.toml @@ -0,0 +1,2 @@ +RecordRequests = false +Ignore = [".databricks"] diff --git a/acceptance/bundle/resources/job_runs/redeploy/output.txt b/acceptance/bundle/resources/job_runs/redeploy/output.txt index 06129f7937..8532692bdd 100644 --- a/acceptance/bundle/resources/job_runs/redeploy/output.txt +++ b/acceptance/bundle/resources/job_runs/redeploy/output.txt @@ -59,6 +59,9 @@ Resources: "job_parameters": { "env": "prod" }, + "lifecycle": { + "triggers": {} + }, "result_state": "SUCCESS" } }, diff --git a/bundle/config/mutator/normalize_paths_test.go b/bundle/config/mutator/normalize_paths_test.go index 8bee6c5064..2399a03aa8 100644 --- a/bundle/config/mutator/normalize_paths_test.go +++ b/bundle/config/mutator/normalize_paths_test.go @@ -53,6 +53,44 @@ func TestNormalizePaths(t *testing.T) { require.Equal(t, "src/notebook.py", newValue.MustString()) } +func TestNormalizePaths_jobRunOnFileChange(t *testing.T) { + tmpDir := t.TempDir() + pattern := "../data/*.txt" + m := NormalizePaths() + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + JobRuns: map[string]*resources.JobRun{ + "run1": { + Lifecycle: &resources.JobRunLifecycle{ + Triggers: []resources.JobRunTrigger{ + {OnFileChange: &pattern}, + }, + }, + }, + }, + }, + }, + BundleRootPath: tmpDir, + } + + location := dyn.Location{File: filepath.Join(tmpDir, "resources", "run.yml")} + path := dyn.MustPathFromString("resources.job_runs.run1.lifecycle.triggers[0].on_file_change") + err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { + return dyn.MapByPath(v, path, func(path dyn.Path, value dyn.Value) (dyn.Value, error) { + return dyn.NewValue(value.MustString(), []dyn.Location{location}), nil + }) + }) + require.NoError(t, err) + + diags := bundle.Apply(t.Context(), b, m) + require.NoError(t, diags.Error()) + + newValue, err := dyn.GetByPath(b.Config.Value(), path) + require.NoError(t, err) + require.Equal(t, "data/*.txt", newValue.MustString()) +} + func TestNormalizePath_absolutePath(t *testing.T) { value, err := normalizePath("/notebook.py", dyn.Location{}, "/tmp") assert.NoError(t, err) diff --git a/bundle/config/mutator/paths/job_run_paths_visitor.go b/bundle/config/mutator/paths/job_run_paths_visitor.go new file mode 100644 index 0000000000..3bc56bacf7 --- /dev/null +++ b/bundle/config/mutator/paths/job_run_paths_visitor.go @@ -0,0 +1,24 @@ +package paths + +import ( + "github.com/databricks/cli/libs/dyn" +) + +// VisitJobRunPaths visits local paths on job_runs so NormalizePaths can rewrite +// them relative to the bundle root. Not used by TranslatePaths: hashing still +// needs a local glob, not a workspace path. +func VisitJobRunPaths(value dyn.Value, fn VisitFunc) (dyn.Value, error) { + pattern := dyn.NewPattern( + dyn.Key("resources"), + dyn.Key("job_runs"), + dyn.AnyKey(), + dyn.Key("lifecycle"), + dyn.Key("triggers"), + dyn.AnyIndex(), + dyn.Key("on_file_change"), + ) + + return dyn.MapByPattern(value, pattern, func(path dyn.Path, value dyn.Value) (dyn.Value, error) { + return fn(path, TranslateModeLocalRelative, value) + }) +} diff --git a/bundle/config/mutator/paths/job_run_paths_visitor_test.go b/bundle/config/mutator/paths/job_run_paths_visitor_test.go new file mode 100644 index 0000000000..0a006fd9df --- /dev/null +++ b/bundle/config/mutator/paths/job_run_paths_visitor_test.go @@ -0,0 +1,34 @@ +package paths + +import ( + "testing" + + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/libs/dyn" + "github.com/stretchr/testify/assert" +) + +func TestVisitJobRunPaths(t *testing.T) { + watched := "watched.txt" + root := config.Root{ + Resources: config.Resources{ + JobRuns: map[string]*resources.JobRun{ + "run0": { + Lifecycle: &resources.JobRunLifecycle{ + Triggers: []resources.JobRunTrigger{ + {OnFileChange: &watched}, + }, + }, + }, + }, + }, + } + + actual := collectVisitedPaths(t, root, VisitJobRunPaths) + expected := []dyn.Path{ + dyn.MustPathFromString("resources.job_runs.run0.lifecycle.triggers[0].on_file_change"), + } + + assert.ElementsMatch(t, expected, actual) +} diff --git a/bundle/config/mutator/paths/visitor.go b/bundle/config/mutator/paths/visitor.go index bdf42188fd..cadc52b75f 100644 --- a/bundle/config/mutator/paths/visitor.go +++ b/bundle/config/mutator/paths/visitor.go @@ -10,6 +10,7 @@ type VisitFunc func(path dyn.Path, mode TranslateMode, value dyn.Value) (dyn.Val func VisitPaths(root dyn.Value, fn VisitFunc) (dyn.Value, error) { visitors := []func(dyn.Value, VisitFunc) (dyn.Value, error){ VisitJobPaths, + VisitJobRunPaths, VisitJobLibrariesPaths, VisitAppPaths, VisitArtifactPaths, diff --git a/bundle/config/mutator/resolve_job_run_file_triggers.go b/bundle/config/mutator/resolve_job_run_file_triggers.go new file mode 100644 index 0000000000..3cfb7a42ee --- /dev/null +++ b/bundle/config/mutator/resolve_job_run_file_triggers.go @@ -0,0 +1,188 @@ +package mutator + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "maps" + "os" + pathlib "path" + "path/filepath" + "strings" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/libs/diag" + libsync "github.com/databricks/cli/libs/sync" +) + +// missingFileHash marks a pattern with no matching file so appear/disappear recreates. +const missingFileHash = "" + +type resolveJobRunFileTriggers struct{} + +// ResolveJobRunFileTriggers expands on_file_change globs and stores per-file +// content hashes on each job_run for PrepareState to copy into local state. +func ResolveJobRunFileTriggers() bundle.Mutator { + return &resolveJobRunFileTriggers{} +} + +func (*resolveJobRunFileTriggers) Name() string { + return "ResolveJobRunFileTriggers" +} + +func (*resolveJobRunFileTriggers) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + syncable, diags := syncableRelPaths(ctx, b) + if diags.HasError() { + return diags + } + for name, jr := range b.Config.Resources.JobRuns { + if jr == nil || jr.Lifecycle == nil { + continue + } + out := make(map[string]string) + for i, t := range jr.Lifecycle.Triggers { + if t.OnFileChange == nil { + continue + } + path := fmt.Sprintf("resources.job_runs.%s.lifecycle.triggers[%d].on_file_change", name, i) + hashes, d := resolveFileTrigger(b, path, *t.OnFileChange, syncable) + diags = diags.Extend(d) + maps.Copy(out, hashes) + } + jr.ResolvedFileTriggers = out + } + return diags +} + +// syncableRelPaths is the set of relative paths sync would upload. +func syncableRelPaths(ctx context.Context, b *bundle.Bundle) (map[string]struct{}, diag.Diagnostics) { + var diags diag.Diagnostics + needs := false + for _, jr := range b.Config.Resources.JobRuns { + if jr != nil && jr.HasOnFileChange() { + needs = true + break + } + } + if !needs { + return nil, diags + } + + out, err := listSyncableRelPaths(ctx, b) + if err != nil { + return nil, diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: list sync files: %s", err), + }) + } + return out, diags +} + +func listSyncableRelPaths(ctx context.Context, b *bundle.Bundle) (map[string]struct{}, error) { + fl, err := libsync.NewFileList(ctx, b.WorktreeRoot, b.SyncRoot, b.Config.Sync.Paths, b.Config.Sync.Include, b.Config.Sync.Exclude) + if err != nil { + return nil, err + } + files, err := fl.Files(ctx) + if err != nil { + return nil, err + } + out := make(map[string]struct{}, len(files)) + for _, f := range files { + out[filepath.ToSlash(f.Relative)] = struct{}{} + } + return out, nil +} + +// fileTriggerDiag reports a diagnostic against the on_file_change entry at loc. +func fileTriggerDiag(b *bundle.Bundle, loc string, severity diag.Severity, format string, args ...any) diag.Diagnostic { + return diag.Diagnostic{ + Severity: severity, + Summary: "lifecycle.triggers.on_file_change: " + fmt.Sprintf(format, args...), + Locations: b.Config.GetLocations(loc), + } +} + +func resolveFileTrigger(b *bundle.Bundle, loc, pattern string, syncable map[string]struct{}) (map[string]string, diag.Diagnostics) { + var diags diag.Diagnostics + out := make(map[string]string) + // filepath.Glob treats ** as two *, so doublestar-style patterns match less than expected. + if strings.Contains(pattern, "**") { + return out, diags.Append(fileTriggerDiag(b, loc, diag.Error, "** in %q is not supported; use * for a single directory level", pattern)) + } + // filepath.Join would otherwise rebase an absolute pattern under the bundle + // root (Join("/bundle", "/etc/passwd") is "/bundle/etc/passwd"). A POSIX path + // is absolute on Windows too, so check both flavours like NormalizePaths does. + if filepath.IsAbs(pattern) || pathlib.IsAbs(pattern) { + return out, diags.Append(fileTriggerDiag(b, loc, diag.Error, "pattern %q must be relative to the defining YAML file", pattern)) + } + // NormalizePaths has already rewritten YAML-relative globs to be bundle-root + // relative. Join that onto the bundle root, then require the result stay + // under the sync root (an ancestor of the bundle when sync.paths uses ..). + joined := filepath.Join(b.BundleRootPath, filepath.FromSlash(pattern)) + relPattern, err := filepath.Rel(b.SyncRootPath, joined) + if err != nil || !filepath.IsLocal(relPattern) { + return out, diags.Append(fileTriggerDiag(b, loc, diag.Error, "pattern %q is not under the sync root", pattern)) + } + matches, err := filepath.Glob(joined) + if err != nil { + return out, diags.Append(fileTriggerDiag(b, loc, diag.Error, "invalid pattern %q: %s", pattern, err)) + } + for _, match := range matches { + info, err := os.Stat(match) + if err != nil { + diags = diags.Append(fileTriggerDiag(b, loc, diag.Error, "stat %q: %s", match, err)) + continue + } + // A glob like migrations/* routinely matches subdirectories; there is + // nothing to hash and nothing for the user to fix, so skip them quietly. + if !info.Mode().IsRegular() { + continue + } + rel, err := filepath.Rel(b.SyncRootPath, match) + if err != nil || !filepath.IsLocal(rel) { + diags = diags.Append(fileTriggerDiag(b, loc, diag.Error, "matched path %q is not under the sync root", match)) + continue + } + // Honor .gitignore and sync.exclude the same way sync does. + if _, ok := syncable[filepath.ToSlash(rel)]; !ok { + continue + } + hash, err := hashFile(match) + if err != nil { + diags = diags.Append(fileTriggerDiag(b, loc, diag.Error, "hash %q: %s", match, err)) + continue + } + out[filepath.ToSlash(rel)] = hash + } + // A pattern that hashes nothing is a warning, not an error: every such case + // re-arms once a matching file appears. Record the placeholder under the + // pattern's own sync-root-relative key so that appearance is a hash change + // rather than a key swap. Skip it when a match failed to be read, since the + // error already says the fingerprint is incomplete. + if len(out) == 0 && !diags.HasError() { + out[filepath.ToSlash(relPattern)] = missingFileHash + if len(matches) == 0 { + diags = diags.Append(fileTriggerDiag(b, loc, diag.Warning, "no files match %q", pattern)) + } else { + diags = diags.Append(fileTriggerDiag(b, loc, diag.Warning, "pattern %q matches only directories or files excluded from sync, so nothing is hashed", pattern)) + } + } + return out, diags +} + +func hashFile(path string) (string, error) { + f, err := os.Open(path) + if err != nil { + return "", err + } + defer f.Close() + + h := sha256.New() + if _, err := io.Copy(h, f); err != nil { + return "", err + } + return hex.EncodeToString(h.Sum(nil)), nil +} diff --git a/bundle/config/mutator/resolve_job_run_file_triggers_test.go b/bundle/config/mutator/resolve_job_run_file_triggers_test.go new file mode 100644 index 0000000000..9271589bb4 --- /dev/null +++ b/bundle/config/mutator/resolve_job_run_file_triggers_test.go @@ -0,0 +1,105 @@ +package mutator_test + +import ( + "crypto/sha256" + "encoding/hex" + "os" + "path/filepath" + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/mutator" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/libs/vfs" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestResolveJobRunFileTriggers(t *testing.T) { + t.Run("hashes file contents with sha256", func(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "a.txt"), []byte("hello"), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "b.txt"), []byte("world"), 0o644)) + + pattern := "*.txt" + b := bundleWithFileTrigger(dir, pattern) + + diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) + require.False(t, diags.HasError()) + + hashes := b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers + require.Len(t, hashes, 2) + assert.Equal(t, contentHash("hello"), hashes["a.txt"]) + assert.Equal(t, contentHash("world"), hashes["b.txt"]) + }) + + t.Run("rejects an absolute pattern", func(t *testing.T) { + dir := t.TempDir() + pattern := "/etc/passwd" + b := bundleWithFileTrigger(dir, pattern) + + diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) + require.True(t, diags.HasError()) + require.Equal(t, `lifecycle.triggers.on_file_change: pattern "/etc/passwd" must be relative to the defining YAML file`, diags[0].Summary) + assert.Empty(t, b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers) + }) + + t.Run("missing pattern is keyed relative to the sync root", func(t *testing.T) { + parent := t.TempDir() + bundleDir := filepath.Join(parent, "bundle") + require.NoError(t, os.Mkdir(bundleDir, 0o755)) + + pattern := "../missing.txt" + b := bundleWithFileTrigger(parent, pattern) + b.BundleRootPath = bundleDir + + diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) + require.False(t, diags.HasError()) + assert.Equal(t, map[string]string{"missing.txt": ""}, b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers) + }) + + t.Run("globs from the bundle root when the sync root is an ancestor", func(t *testing.T) { + parent := t.TempDir() + bundleDir := filepath.Join(parent, "bundle") + require.NoError(t, os.Mkdir(bundleDir, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(parent, "shared.txt"), []byte("from-sync-root"), 0o644)) + + pattern := "../shared.txt" + b := bundleWithFileTrigger(parent, pattern) + b.BundleRootPath = bundleDir + + diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) + require.False(t, diags.HasError()) + assert.Equal(t, contentHash("from-sync-root"), b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers["shared.txt"]) + }) +} + +func bundleWithFileTrigger(syncRoot, pattern string) *bundle.Bundle { + root := vfs.MustNew(syncRoot) + return &bundle.Bundle{ + BundleRootPath: syncRoot, + SyncRootPath: syncRoot, + SyncRoot: root, + WorktreeRoot: root, + Config: config.Root{ + Sync: config.Sync{Paths: []string{"."}}, + Resources: config.Resources{ + JobRuns: map[string]*resources.JobRun{ + "my_run": { + Lifecycle: &resources.JobRunLifecycle{ + Triggers: []resources.JobRunTrigger{ + {OnFileChange: &pattern}, + }, + }, + }, + }, + }, + }, + } +} + +func contentHash(content string) string { + sum := sha256.Sum256([]byte(content)) + return hex.EncodeToString(sum[:]) +} diff --git a/bundle/config/mutator/validate_job_run_triggers.go b/bundle/config/mutator/validate_job_run_triggers.go index a7c5ada925..fa31032c68 100644 --- a/bundle/config/mutator/validate_job_run_triggers.go +++ b/bundle/config/mutator/validate_job_run_triggers.go @@ -3,6 +3,7 @@ package mutator import ( "context" "fmt" + "strings" "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" @@ -25,32 +26,61 @@ func (*validateJobRunTriggers) Apply(_ context.Context, b *bundle.Bundle) diag.D if jr == nil || jr.Lifecycle == nil { continue } - // Recreate-every-deploy cannot coexist with prevent_destroy. - if jr.HasOnBundleDeploy() && jr.Lifecycle.PreventDestroy { - diags = diags.Append(diag.Diagnostic{ - Severity: diag.Error, - Summary: "lifecycle.triggers.on_bundle_deploy is incompatible with lifecycle.prevent_destroy", - Locations: b.Config.GetLocations(fmt.Sprintf("resources.job_runs.%s.lifecycle", name)), - }) + if jr.Lifecycle.PreventDestroy { + if summary := preventDestroyError(jr.HasOnBundleDeploy(), jr.HasOnFileChange()); summary != "" { + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: summary, + Locations: b.Config.GetLocations(fmt.Sprintf("resources.job_runs.%s.lifecycle", name)), + }) + } } for i, t := range jr.Lifecycle.Triggers { path := fmt.Sprintf("resources.job_runs.%s.lifecycle.triggers[%d]", name, i) - if t.OnBundleDeploy == nil { + if t.OnBundleDeploy == nil && t.OnFileChange == nil { diags = diags.Append(diag.Diagnostic{ Severity: diag.Error, - Summary: "lifecycle.triggers entry must set on_bundle_deploy: true", + Summary: "lifecycle.triggers entry must set on_bundle_deploy or on_file_change", Locations: b.Config.GetLocations(path), }) continue } - if !*t.OnBundleDeploy { + if t.OnBundleDeploy != nil && t.OnFileChange != nil { + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: "lifecycle.triggers entry must set only one of on_bundle_deploy or on_file_change", + Locations: b.Config.GetLocations(path), + }) + continue + } + if t.OnBundleDeploy != nil && !*t.OnBundleDeploy { diags = diags.Append(diag.Diagnostic{ Severity: diag.Error, Summary: "lifecycle.triggers.on_bundle_deploy must be true when set", Locations: b.Config.GetLocations(path + ".on_bundle_deploy"), }) } + if t.OnFileChange != nil && strings.TrimSpace(*t.OnFileChange) == "" { + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: "lifecycle.triggers.on_file_change must be non-empty when set", + Locations: b.Config.GetLocations(path + ".on_file_change"), + }) + } } } return diags } + +func preventDestroyError(onBundleDeploy, onFileChange bool) string { + switch { + case onBundleDeploy && onFileChange: + return "lifecycle.triggers.on_bundle_deploy and on_file_change are incompatible with lifecycle.prevent_destroy" + case onBundleDeploy: + return "lifecycle.triggers.on_bundle_deploy is incompatible with lifecycle.prevent_destroy" + case onFileChange: + return "lifecycle.triggers.on_file_change is incompatible with lifecycle.prevent_destroy" + default: + return "" + } +} diff --git a/bundle/config/mutator/validate_job_run_triggers_test.go b/bundle/config/mutator/validate_job_run_triggers_test.go index 9c22545747..07a7276a41 100644 --- a/bundle/config/mutator/validate_job_run_triggers_test.go +++ b/bundle/config/mutator/validate_job_run_triggers_test.go @@ -14,6 +14,10 @@ func TestValidateJobRunTriggers(t *testing.T) { trueVal := true falseVal := false + fileChange := "seed.txt" + emptyFile := "" + whitespaceFile := " \t" + tests := []struct { name string triggers []resources.JobRunTrigger @@ -26,12 +30,32 @@ func TestValidateJobRunTriggers(t *testing.T) { {OnBundleDeploy: &trueVal}, }, }, + { + name: "on_file_change set", + triggers: []resources.JobRunTrigger{ + {OnFileChange: &fileChange}, + }, + }, + { + name: "both triggers as separate entries", + triggers: []resources.JobRunTrigger{ + {OnFileChange: &fileChange}, + {OnBundleDeploy: &trueVal}, + }, + }, { name: "empty entry", triggers: []resources.JobRunTrigger{ {}, }, - summary: "lifecycle.triggers entry must set on_bundle_deploy: true", + summary: "lifecycle.triggers entry must set on_bundle_deploy or on_file_change", + }, + { + name: "both keys on one entry", + triggers: []resources.JobRunTrigger{ + {OnBundleDeploy: &trueVal, OnFileChange: &fileChange}, + }, + summary: "lifecycle.triggers entry must set only one of on_bundle_deploy or on_file_change", }, { name: "on_bundle_deploy false", @@ -40,6 +64,20 @@ func TestValidateJobRunTriggers(t *testing.T) { }, summary: "lifecycle.triggers.on_bundle_deploy must be true when set", }, + { + name: "on_file_change empty", + triggers: []resources.JobRunTrigger{ + {OnFileChange: &emptyFile}, + }, + summary: "lifecycle.triggers.on_file_change must be non-empty when set", + }, + { + name: "on_file_change whitespace", + triggers: []resources.JobRunTrigger{ + {OnFileChange: &whitespaceFile}, + }, + summary: "lifecycle.triggers.on_file_change must be non-empty when set", + }, { name: "on_bundle_deploy with prevent_destroy", triggers: []resources.JobRunTrigger{ @@ -48,6 +86,23 @@ func TestValidateJobRunTriggers(t *testing.T) { preventDestroy: true, summary: "lifecycle.triggers.on_bundle_deploy is incompatible with lifecycle.prevent_destroy", }, + { + name: "on_file_change with prevent_destroy", + triggers: []resources.JobRunTrigger{ + {OnFileChange: &fileChange}, + }, + preventDestroy: true, + summary: "lifecycle.triggers.on_file_change is incompatible with lifecycle.prevent_destroy", + }, + { + name: "both triggers with prevent_destroy", + triggers: []resources.JobRunTrigger{ + {OnFileChange: &fileChange}, + {OnBundleDeploy: &trueVal}, + }, + preventDestroy: true, + summary: "lifecycle.triggers.on_bundle_deploy and on_file_change are incompatible with lifecycle.prevent_destroy", + }, { name: "prevent_destroy alone", preventDestroy: true, diff --git a/bundle/config/resources/job_run.go b/bundle/config/resources/job_run.go index 56f3c39a35..512077e273 100644 --- a/bundle/config/resources/job_run.go +++ b/bundle/config/resources/job_run.go @@ -27,6 +27,10 @@ type JobRun struct { // the run URL. Keeping it separate from RunNow.JobId (a ${resources.jobs.*.id} // reference) lets state loading preserve that reference and its plan dependency. ResolvedJobID int64 `json:"resolved_job_id,omitempty" bundle:"internal"` + + // ResolvedFileTriggers is path → content hash for on_file_change, computed + // under SyncRoot before plan. bundle:"internal" keeps it out of schema. + ResolvedFileTriggers map[string]string `json:"resolved_file_triggers,omitempty" bundle:"internal"` } // HasOnBundleDeploy reports whether any trigger re-fires on every deploy. @@ -42,6 +46,19 @@ func (r *JobRun) HasOnBundleDeploy() bool { return false } +// HasOnFileChange reports whether any trigger re-fires when matched files change. +func (r *JobRun) HasOnFileChange() bool { + if r.Lifecycle == nil { + return false + } + for _, t := range r.Lifecycle.Triggers { + if t.OnFileChange != nil { + return true + } + } + return false +} + func (r *JobRun) UnmarshalJSON(b []byte) error { return marshal.Unmarshal(b, r) } diff --git a/bundle/config/resources/lifecycle.go b/bundle/config/resources/lifecycle.go index 88d130239e..1f5d576d83 100644 --- a/bundle/config/resources/lifecycle.go +++ b/bundle/config/resources/lifecycle.go @@ -36,5 +36,6 @@ type JobRunLifecycle struct { // JobRunTrigger is one lifecycle.triggers entry. type JobRunTrigger struct { - OnBundleDeploy *bool `json:"on_bundle_deploy,omitempty"` + OnBundleDeploy *bool `json:"on_bundle_deploy,omitempty"` + OnFileChange *string `json:"on_file_change,omitempty"` // path or glob relative to the defining YAML file; must resolve under the sync root } diff --git a/bundle/direct/dresources/job_run.go b/bundle/direct/dresources/job_run.go index 9eb671f430..078ae1b1aa 100644 --- a/bundle/direct/dresources/job_run.go +++ b/bundle/direct/dresources/job_run.go @@ -30,11 +30,20 @@ const jobRunTimeout = 24 * time.Hour type JobRunTriggersState struct { // Fresh UUID each plan while armed so Old!=New forces recreate. OnBundleDeploy string `json:"on_bundle_deploy,omitempty"` + // Content hashes from ResolveJobRunFileTriggers; any change recreates. + OnFileChange map[string]string `json:"on_file_change,omitempty"` } -// JobRunLifecycleState holds local-only lifecycle fields persisted in state. +// JobRunLifecycleState is the local-only trigger fingerprint. Nested by value, +// not by pointer: structdiff cannot descend into a nil pointer and would report +// the whole subtree at "lifecycle" instead of the leaf that actually changed. type JobRunLifecycleState struct { - Triggers *JobRunTriggersState `json:"triggers,omitempty"` + Triggers JobRunTriggersState `json:"triggers"` +} + +func emptyJobRunLifecycleState() JobRunLifecycleState { + var empty JobRunLifecycleState + return empty } // JobRunState is the RunNow request plus the outcome required for planning. @@ -44,9 +53,9 @@ type JobRunState struct { // Always SUCCESS during planning and cleared before persistence. ResultState jobs.RunResultState `json:"result_state,omitempty"` - // Local-only; listed in knownMissingInRemoteType. Nested under lifecycle to - // mirror config and avoid colliding with a future Jobs API field. - Lifecycle *JobRunLifecycleState `json:"lifecycle,omitempty"` + // Local-only. Nested under lifecycle to mirror config and avoid colliding + // with a future Jobs API field. + Lifecycle JobRunLifecycleState `json:"lifecycle"` } func (s *JobRunState) UnmarshalJSON(b []byte) error { @@ -57,8 +66,8 @@ func (s JobRunState) MarshalJSON() ([]byte, error) { return marshal.Marshal(s) } -// JobRunRemote embeds RunNow so every StateType path is a valid RemoteType path -// (see TestRemoteSuperset), plus the run's output-only fields for a faithful view. +// JobRunRemote is the RunNow request plus the run's output-only fields. It has no +// lifecycle: GetRun never returns the fingerprints (see knownMissingInRemoteType). type JobRunRemote struct { jobs.RunNow @@ -97,12 +106,13 @@ func (*ResourceJobRun) PrepareState(input *resources.JobRun) *JobRunState { state := &JobRunState{ RunNow: input.RunNow, ResultState: jobs.RunResultStateSuccess, - Lifecycle: nil, + Lifecycle: emptyJobRunLifecycleState(), } if input.HasOnBundleDeploy() { - state.Lifecycle = &JobRunLifecycleState{ - Triggers: &JobRunTriggersState{OnBundleDeploy: uuid.NewString()}, - } + state.Lifecycle.Triggers.OnBundleDeploy = uuid.NewString() + } + if len(input.ResolvedFileTriggers) > 0 { + state.Lifecycle.Triggers.OnFileChange = input.ResolvedFileTriggers } return state } @@ -178,13 +188,13 @@ func (r *ResourceJobRun) DoRead(ctx context.Context, id string) (*JobRunRemote, } // RemapState extracts the fields used for diffing: the RunNow request and the -// outcome the run reached. +// outcome the run reached. Lifecycle has no remote counterpart, so it stays empty +// and the planner skips it as missing_in_remote. func (*ResourceJobRun) RemapState(remote *JobRunRemote) *JobRunState { return &JobRunState{ RunNow: remote.RunNow, ResultState: remote.ResultState, - // Local-only lifecycle fingerprints stay unset on the remapped remote. - Lifecycle: nil, + Lifecycle: emptyJobRunLifecycleState(), } } @@ -372,14 +382,16 @@ func reportRunLine(ctx context.Context, runID int64, msg string) { // Clearing a trigger skips its local-only fingerprint without re-firing the run. func (*ResourceJobRun) OverrideChangeDesc(_ context.Context, path *structpath.PathNode, change *ChangeDesc, remote *JobRunRemote) error { switch path.String() { - case "lifecycle", "lifecycle.triggers", "lifecycle.triggers.on_bundle_deploy": - // A cleared trigger sets New empty; structdiff may report it at lifecycle, - // lifecycle.triggers, or the leaf. + case "lifecycle.triggers.on_bundle_deploy": if change.New == nil || change.New == "" { change.Action = deployplan.Skip change.Reason = "trigger removed" } - return nil + case "lifecycle.triggers.on_file_change": + if change.New == nil { + change.Action = deployplan.Skip + change.Reason = "trigger removed" + } case "result_state": // The planner passes no remote state when the run could not be read. if remote == nil || runIsTerminal(remote.State.LifeCycleState) { @@ -387,10 +399,8 @@ func (*ResourceJobRun) OverrideChangeDesc(_ context.Context, path *structpath.Pa } change.Action = deployplan.Skip change.Reason = "run in progress" - return nil - default: - return nil } + return nil } // DoDelete deletes the run via jobs/runs/delete, on both destroy and the diff --git a/bundle/direct/dresources/job_run_test.go b/bundle/direct/dresources/job_run_test.go index 0ffc1c4804..f9f2199931 100644 --- a/bundle/direct/dresources/job_run_test.go +++ b/bundle/direct/dresources/job_run_test.go @@ -10,6 +10,7 @@ import ( "testing" "time" + "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/cli/libs/testserver" "github.com/databricks/databricks-sdk-go" @@ -244,6 +245,15 @@ func TestJobRunWaitAbandonedLinksTheRun(t *testing.T) { require.ErrorContains(t, err, testRunPageLink) } +// State written before lifecycle existed has no such key, and must still load. +func TestJobRunStateUnmarshalWithoutLifecycle(t *testing.T) { + var state JobRunState + + require.NoError(t, json.Unmarshal([]byte(`{}`), &state)) + + assert.Nil(t, state.Lifecycle.Triggers.OnFileChange) +} + // The planner diffs RemapState(remote) against PrepareState(config), so a run // that did not end in SUCCESS has to surface as a difference on result_state. func TestJobRunRemapStateCarriesTheOutcome(t *testing.T) { @@ -259,6 +269,7 @@ func TestJobRunRemapStateCarriesTheOutcome(t *testing.T) { state := (&ResourceJobRun{}).RemapState(remote) assert.Equal(t, outcome, state.ResultState) + assert.Equal(t, emptyJobRunLifecycleState(), state.Lifecycle) }) } } @@ -357,3 +368,29 @@ func TestJobRunDeleteLeavesFinishedRunAlone(t *testing.T) { assert.False(t, cancelled.Load(), "a run that already finished has nothing to cancel") } + +func TestJobRunOverrideChangeDescTriggerRemoved(t *testing.T) { + r := &ResourceJobRun{} + for _, tt := range []struct { + name string + path string + new any + action deployplan.ActionType + }{ + {"cleared on_bundle_deploy string", "lifecycle.triggers.on_bundle_deploy", "", deployplan.Skip}, + {"nil on_bundle_deploy", "lifecycle.triggers.on_bundle_deploy", nil, deployplan.Skip}, + {"rotated on_bundle_deploy", "lifecycle.triggers.on_bundle_deploy", "uuid", deployplan.Recreate}, + {"cleared on_file_change", "lifecycle.triggers.on_file_change", nil, deployplan.Skip}, + {"changed on_file_change map", "lifecycle.triggers.on_file_change", map[string]string{"a.txt": "h"}, deployplan.Recreate}, + // A file dropping out of the map is a real change, so the skip must not + // extend to paths below on_file_change. + {"cleared on_file_change child", "lifecycle.triggers.on_file_change['a.txt']", nil, deployplan.Recreate}, + {"result_state with unreadable remote", "result_state", nil, deployplan.Recreate}, + } { + t.Run(tt.name, func(t *testing.T) { + change := &ChangeDesc{Action: deployplan.Recreate, New: tt.new} + require.NoError(t, r.OverrideChangeDesc(t.Context(), structpath.MustParsePath(tt.path), change, nil)) + assert.Equal(t, tt.action, change.Action) + }) + } +} diff --git a/bundle/direct/dresources/type_test.go b/bundle/direct/dresources/type_test.go index 2d5516d59c..7ad9872c37 100644 --- a/bundle/direct/dresources/type_test.go +++ b/bundle/direct/dresources/type_test.go @@ -19,6 +19,11 @@ var knownMissingInRemoteType = map[string][]string{ "external_locations": { "skip_validation", }, + "job_runs": { + // Deliberate omission: the trigger fingerprints are client-side only, so + // GetRun has nothing to report them under. + "lifecycle", + }, "model_serving_endpoints": { "rate_limits", }, @@ -56,10 +61,6 @@ var knownMissingInRemoteType = map[string][]string{ "vector_search_endpoints": { "usage_policy_id", }, - "job_runs": { - // Local-only trigger fingerprints under lifecycle. - "lifecycle", - }, } // commonMissingInStateType lists fields that are commonly missing across all resource types. diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index e47f89c505..8b17164066 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -1017,6 +1017,9 @@ resources: "on_bundle_deploy": "description": |- If true, re-fire the run on every bundle deploy. Incompatible with lifecycle.prevent_destroy. + "on_file_change": + "description": |- + Path or glob relative to the defining YAML file. It must resolve under the sync root. Re-fire the run when a matched file's content hash changes, or when the set of matches appears or disappears. Only files the bundle syncs are hashed, so .gitignore and sync.exclude apply. Use * to match a single directory level; ** is not supported. Incompatible with lifecycle.prevent_destroy. "python_named_params": "description": |- PLACEHOLDER diff --git a/bundle/phases/initialize.go b/bundle/phases/initialize.go index b15e1c30df..d2ba4e76ef 100644 --- a/bundle/phases/initialize.go +++ b/bundle/phases/initialize.go @@ -191,6 +191,9 @@ func Initialize(ctx context.Context, b *bundle.Bundle) { // Reject invalid job_runs.lifecycle.triggers (empty, false, prevent_destroy). mutator.ValidateJobRunTriggers(), + // Expand on_file_change globs and hash matched files into ResolvedFileTriggers. + mutator.ResolveJobRunFileTriggers(), + // Reads (dynamic): * (strings) (searches for ${resources.*} references) // Warns (TF engine) or errors (direct engine) when a cross-resource reference // points to a Terraform-only field with no DABs equivalent. diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 425956ab06..593447629b 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -1394,6 +1394,10 @@ "on_bundle_deploy": { "description": "If true, re-fire the run on every bundle deploy. Incompatible with lifecycle.prevent_destroy.", "$ref": "#/$defs/bool" + }, + "on_file_change": { + "description": "Path or glob relative to the defining YAML file. It must resolve under the sync root. Re-fire the run when a matched file's content hash changes, or when the set of matches appears or disappears. Only files the bundle syncs are hashed, so .gitignore and sync.exclude apply. Use * to match a single directory level; ** is not supported. Incompatible with lifecycle.prevent_destroy.", + "$ref": "#/$defs/string" } }, "additionalProperties": false