From 459d58e6e3c442c5cd7558cc772eda39de451cfd Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Tue, 18 Aug 2026 12:25:26 +0000 Subject: [PATCH 01/26] job_runs: add lifecycle.triggers.on_file_change Re-fire a run when matched file contents change, using stable hash fingerprints so mtime-only updates (touch) do not recreate. --- .../bundles/job-runs-on-file-change.md | 1 + .../job_runs/on_file_change/databricks.yml | 18 ++ .../job_runs/on_file_change/out.test.toml | 2 + .../job_runs/on_file_change/output.txt | 90 +++++++++ .../resources/job_runs/on_file_change/script | 34 ++++ .../job_runs/on_file_change/seed.txt | 1 + .../mutator/resolve_job_run_file_triggers.go | 177 ++++++++++++++++++ ...lve_job_run_file_triggers_internal_test.go | 73 ++++++++ .../resolve_job_run_file_triggers_test.go | 143 ++++++++++++++ .../mutator/validate_job_run_triggers.go | 26 ++- .../mutator/validate_job_run_triggers_test.go | 50 ++++- bundle/config/resources/job_run.go | 29 +++ bundle/config/resources/lifecycle.go | 9 +- bundle/direct/dresources/job_run.go | 28 ++- bundle/direct/dresources/resources.yml | 6 + bundle/internal/schema/annotations.yml | 3 + bundle/phases/initialize.go | 6 +- bundle/phases/plan.go | 4 +- bundle/schema/jsonschema.json | 4 + 19 files changed, 689 insertions(+), 15 deletions(-) create mode 100644 .nextchanges/bundles/job-runs-on-file-change.md create mode 100644 acceptance/bundle/resources/job_runs/on_file_change/databricks.yml create mode 100644 acceptance/bundle/resources/job_runs/on_file_change/out.test.toml create mode 100644 acceptance/bundle/resources/job_runs/on_file_change/output.txt create mode 100644 acceptance/bundle/resources/job_runs/on_file_change/script create mode 100644 acceptance/bundle/resources/job_runs/on_file_change/seed.txt create mode 100644 bundle/config/mutator/resolve_job_run_file_triggers.go create mode 100644 bundle/config/mutator/resolve_job_run_file_triggers_internal_test.go create mode 100644 bundle/config/mutator/resolve_job_run_file_triggers_test.go diff --git a/.nextchanges/bundles/job-runs-on-file-change.md b/.nextchanges/bundles/job-runs-on-file-change.md new file mode 100644 index 00000000000..00682403a0d --- /dev/null +++ b/.nextchanges/bundles/job-runs-on-file-change.md @@ -0,0 +1 @@ +direct: `resources.job_runs` can set `lifecycle.triggers.on_file_change` to a path or glob to re-fire the run when matched file contents change. Removing the trigger does not recreate the existing run. diff --git a/acceptance/bundle/resources/job_runs/on_file_change/databricks.yml b/acceptance/bundle/resources/job_runs/on_file_change/databricks.yml new file mode 100644 index 00000000000..0a6df75047a --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change/databricks.yml @@ -0,0 +1,18 @@ +bundle: + name: job-runs-on-file-change + +resources: + jobs: + my_job: + name: my-job + tasks: + - task_key: main + notebook_task: + notebook_path: /Workspace/test + + job_runs: + my_run: + job_id: ${resources.jobs.my_job.id} + lifecycle: + triggers: + - on_file_change: seed.txt diff --git a/acceptance/bundle/resources/job_runs/on_file_change/out.test.toml b/acceptance/bundle/resources/job_runs/on_file_change/out.test.toml new file mode 100644 index 00000000000..0938e678987 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/job_runs/on_file_change/output.txt b/acceptance/bundle/resources/job_runs/on_file_change/output.txt new file mode 100644 index 00000000000..bcf5962f8fa --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change/output.txt @@ -0,0 +1,90 @@ + +=== first deploy triggers a run +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change/default/files... +job run [MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?o=[NUMID] +job run [MY_RUN_ID]: SUCCESS +Created job_runs.my_run +Created jobs.my_job +Files: 5 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +>>> read_id.py my_job +[MY_JOB_ID] + +>>> print_requests.py //jobs/run-now +{ + "method": "POST", + "path": "/api/2.2/jobs/run-now", + "body": { + "idempotency_token": "[UUID]", + "job_id": [MY_JOB_ID] + } +} + +=== redeploy with unchanged file plans nothing +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change/default/files... +Files: 2 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 2 unchanged + +>>> print_requests.py //jobs/run-now + +=== editing the file re-fires +>>> [CLI] bundle plan +recreate job_runs.my_run + +Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change/default/files... +job run [MY_RUN_ID_2]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_2]?o=[NUMID] +job run [MY_RUN_ID_2]: SUCCESS +Recreated job_runs.my_run +Files: 3 uploaded, 0 deleted +Resources: 1 created, 0 changed, 1 deleted, 1 unchanged + +>>> print_requests.py --keep //jobs/runs/delete +{ + "method": "POST", + "path": "/api/2.2/jobs/runs/delete", + "body": { + "run_id": [MY_RUN_ID] + } +} + +>>> print_requests.py //jobs/run-now +{ + "method": "POST", + "path": "/api/2.2/jobs/run-now", + "body": { + "idempotency_token": "[UUID]", + "job_id": [MY_JOB_ID] + } +} + +=== removing on_file_change rewrites state without a run +>>> [CLI] bundle plan +update job_runs.my_run + +Plan: 0 to add, 1 to change, 0 to delete, 1 unchanged + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change/default/files... +Updated job_runs.my_run +Files: 3 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 1 unchanged + +>>> print_requests.py //jobs/run-now + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.job_runs.my_run + delete resources.jobs.my_job + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change/default + +Destroy: 2 deleted diff --git a/acceptance/bundle/resources/job_runs/on_file_change/script b/acceptance/bundle/resources/job_runs/on_file_change/script new file mode 100644 index 00000000000..ff162e11f0b --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change/script @@ -0,0 +1,34 @@ +cleanup() { + trace $CLI bundle destroy --auto-approve + rm -f out.requests.txt +} +trap cleanup EXIT + +title "first deploy triggers a run" +trace $CLI bundle deploy +trace read_id.py my_job +# Name the first run so the recreated one becomes [MY_RUN_ID_2]. +read_id.py my_run > /dev/null +trace print_requests.py //jobs/run-now + +title "redeploy with unchanged file plans nothing" +trace $CLI bundle plan +trace $CLI bundle deploy +trace print_requests.py //jobs/run-now + +title "editing the file re-fires" +update_file.py seed.txt "v1" "v2" +trace $CLI bundle plan +trace $CLI bundle deploy +read_id.py my_run > /dev/null +trace print_requests.py --keep //jobs/runs/delete +trace print_requests.py //jobs/run-now + +title "removing on_file_change rewrites state without a run" +update_file.py databricks.yml " lifecycle: + triggers: + - on_file_change: seed.txt +" "" +trace $CLI bundle plan +trace $CLI bundle deploy +trace print_requests.py //jobs/run-now diff --git a/acceptance/bundle/resources/job_runs/on_file_change/seed.txt b/acceptance/bundle/resources/job_runs/on_file_change/seed.txt new file mode 100644 index 00000000000..626799f0f85 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change/seed.txt @@ -0,0 +1 @@ +v1 diff --git a/bundle/config/mutator/resolve_job_run_file_triggers.go b/bundle/config/mutator/resolve_job_run_file_triggers.go new file mode 100644 index 00000000000..a82ed654be3 --- /dev/null +++ b/bundle/config/mutator/resolve_job_run_file_triggers.go @@ -0,0 +1,177 @@ +package mutator + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "os" + "path/filepath" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/libs/diag" +) + +// missingFileSentinelSize marks a pattern with no matching file so the next +// plan can distinguish "still missing" from "file appeared". +const missingFileSentinelSize = int64(-1) + +type resolveJobRunFileTriggers struct{} + +// ResolveJobRunFileTriggers expands on_file_change globs and stores per-file +// fingerprints on each job_run for PrepareState to copy into local state. +func ResolveJobRunFileTriggers() bundle.Mutator { + return &resolveJobRunFileTriggers{} +} + +func (*resolveJobRunFileTriggers) Name() string { + return "ResolveJobRunFileTriggers" +} + +func (*resolveJobRunFileTriggers) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { + var diags diag.Diagnostics + for name, jr := range b.Config.Resources.JobRuns { + if jr == nil { + continue + } + patterns := jr.OnFileChangePatterns() + if len(patterns) == 0 { + continue + } + fps, d := resolveFileTriggers(b, name, patterns, previousFileTriggers(b, name)) + diags = diags.Extend(d) + jr.ResolvedFileTriggers = fps + } + return diags +} + +// previousFileTriggers reads on_file_change fingerprints from deployment state +// when it is open (plan/deploy after StatePull). Used so unchanged content keeps +// a stable fingerprint across mtime-only updates (e.g. touch). +func previousFileTriggers(b *bundle.Bundle, name string) map[string]resources.JobRunFileFingerprint { + if b.DeploymentBundle.StateDB.Path == "" { + return nil + } + entry, ok := b.DeploymentBundle.StateDB.GetResourceEntry("resources.job_runs." + name) + if !ok || len(entry.State) == 0 { + return nil + } + var state struct { + Lifecycle *struct { + Triggers *struct { + OnFileChange map[string]resources.JobRunFileFingerprint `json:"on_file_change"` + } `json:"triggers"` + } `json:"lifecycle"` + } + if err := json.Unmarshal(entry.State, &state); err != nil { + return nil + } + if state.Lifecycle == nil || state.Lifecycle.Triggers == nil { + return nil + } + return state.Lifecycle.Triggers.OnFileChange +} + +func resolveFileTriggers(b *bundle.Bundle, name string, patterns []string, prev map[string]resources.JobRunFileFingerprint) (map[string]resources.JobRunFileFingerprint, diag.Diagnostics) { + var diags diag.Diagnostics + out := make(map[string]resources.JobRunFileFingerprint) + for _, pattern := range patterns { + path := fmt.Sprintf("resources.job_runs.%s.lifecycle.triggers", name) + matches, err := filepath.Glob(filepath.Join(b.SyncRootPath, filepath.FromSlash(pattern))) + if err != nil { + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: invalid pattern %q: %s", pattern, err), + Locations: b.Config.GetLocations(path), + }) + continue + } + if len(matches) == 0 { + // Distinct state when the path/glob matches nothing (design doc). + out[filepath.ToSlash(pattern)] = resources.JobRunFileFingerprint{ + Size: missingFileSentinelSize, + } + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Warning, + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: no files match %q", pattern), + Locations: b.Config.GetLocations(path), + }) + continue + } + for _, match := range matches { + info, err := os.Stat(match) + if err != nil { + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: stat %q: %s", match, err), + Locations: b.Config.GetLocations(path), + }) + continue + } + if !info.Mode().IsRegular() { + continue + } + rel, err := filepath.Rel(b.SyncRootPath, match) + if err != nil { + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: relative path for %q: %s", match, err), + Locations: b.Config.GetLocations(path), + }) + continue + } + key := filepath.ToSlash(rel) + fp, err := fingerprintFile(match, info, prev[key]) + if err != nil { + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: hash %q: %s", match, err), + Locations: b.Config.GetLocations(path), + }) + continue + } + out[key] = fp + } + } + return out, diags +} + +// fingerprintFile builds a content fingerprint. If prev has the same size and +// mtime, it is reused without reading the file. If content hash matches prev, +// prev is reused so mtime-only updates (touch) do not change planned state. +func fingerprintFile(path string, info os.FileInfo, prev resources.JobRunFileFingerprint) (resources.JobRunFileFingerprint, error) { + size := info.Size() + mtime := info.ModTime().UnixNano() + if prev.Hash != "" && prev.Size == size && prev.MtimeNs == mtime { + return prev, nil + } + hash, err := hashFile(path) + if err != nil { + return resources.JobRunFileFingerprint{}, err + } + if prev.Hash != "" && prev.Hash == hash { + return prev, nil + } + return resources.JobRunFileFingerprint{ + Hash: hash, + Size: size, + MtimeNs: mtime, + }, nil +} + +func hashFile(path string) (string, error) { + f, err := os.Open(path) + if err != nil { + return "", err + } + defer f.Close() + + h := sha256.New() + if _, err := io.Copy(h, f); err != nil { + return "", err + } + return hex.EncodeToString(h.Sum(nil)), nil +} diff --git a/bundle/config/mutator/resolve_job_run_file_triggers_internal_test.go b/bundle/config/mutator/resolve_job_run_file_triggers_internal_test.go new file mode 100644 index 00000000000..e82bd22945b --- /dev/null +++ b/bundle/config/mutator/resolve_job_run_file_triggers_internal_test.go @@ -0,0 +1,73 @@ +package mutator + +import ( + "os" + "path/filepath" + "testing" + "time" + + "github.com/databricks/cli/bundle/config/resources" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestFingerprintFileContentStableAcrossTouch(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "seed.txt") + require.NoError(t, os.WriteFile(path, []byte("v1"), 0o644)) + + info, err := os.Stat(path) + require.NoError(t, err) + first, err := fingerprintFile(path, info, resources.JobRunFileFingerprint{}) + require.NoError(t, err) + require.NotEmpty(t, first.Hash) + + // Advance mtime without changing contents (touch). + require.NoError(t, os.Chtimes(path, time.Now().Add(time.Minute), time.Now().Add(time.Minute))) + info, err = os.Stat(path) + require.NoError(t, err) + assert.NotEqual(t, first.MtimeNs, info.ModTime().UnixNano()) + + second, err := fingerprintFile(path, info, first) + require.NoError(t, err) + assert.Equal(t, first, second, "unchanged content must reuse the previous fingerprint") +} + +func TestFingerprintFileFastPathSkipsWhenMtimeAndSizeMatch(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "seed.txt") + require.NoError(t, os.WriteFile(path, []byte("v1"), 0o644)) + + info, err := os.Stat(path) + require.NoError(t, err) + prev := resources.JobRunFileFingerprint{ + Hash: "not-the-real-hash", + Size: info.Size(), + MtimeNs: info.ModTime().UnixNano(), + } + + got, err := fingerprintFile(path, info, prev) + require.NoError(t, err) + assert.Equal(t, prev, got, "matching size+mtime must reuse prev without re-hashing") +} + +func TestFingerprintFileContentChange(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "seed.txt") + require.NoError(t, os.WriteFile(path, []byte("v1"), 0o644)) + + info, err := os.Stat(path) + require.NoError(t, err) + first, err := fingerprintFile(path, info, resources.JobRunFileFingerprint{}) + require.NoError(t, err) + + // Different size so the mtime+size fast path cannot reuse prev. + require.NoError(t, os.WriteFile(path, []byte("v2-changed"), 0o644)) + info, err = os.Stat(path) + require.NoError(t, err) + second, err := fingerprintFile(path, info, first) + require.NoError(t, err) + assert.NotEqual(t, first.Hash, second.Hash) + assert.Equal(t, info.Size(), second.Size) + assert.Equal(t, info.ModTime().UnixNano(), second.MtimeNs) +} diff --git a/bundle/config/mutator/resolve_job_run_file_triggers_test.go b/bundle/config/mutator/resolve_job_run_file_triggers_test.go new file mode 100644 index 00000000000..9a9cf77bf53 --- /dev/null +++ b/bundle/config/mutator/resolve_job_run_file_triggers_test.go @@ -0,0 +1,143 @@ +package mutator_test + +import ( + "crypto/sha256" + "encoding/hex" + "os" + "path/filepath" + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/mutator" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/libs/diag" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestResolveJobRunFileTriggers(t *testing.T) { + t.Run("matches files and fills fingerprints", func(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "a.txt"), []byte("hello"), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "b.txt"), []byte("world"), 0o644)) + + pattern := "*.txt" + b := bundleWithFileTrigger(dir, pattern) + + diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) + require.False(t, diags.HasError()) + + fps := b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers + require.Len(t, fps, 2) + + assertFingerprint(t, fps["a.txt"], "hello") + assertFingerprint(t, fps["b.txt"], "world") + }) + + t.Run("no matches warns and stores sentinel", func(t *testing.T) { + dir := t.TempDir() + pattern := "missing.txt" + b := bundleWithFileTrigger(dir, pattern) + + diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) + require.False(t, diags.HasError()) + require.Len(t, diags, 1) + assert.Equal(t, diag.Warning, diags[0].Severity) + assert.Contains(t, diags[0].Summary, `no files match "missing.txt"`) + + fps := b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers + require.Len(t, fps, 1) + fp := fps["missing.txt"] + assert.Empty(t, fp.Hash) + assert.Equal(t, int64(-1), fp.Size) + assert.Zero(t, fp.MtimeNs) + }) + + t.Run("no file triggers is a no-op", func(t *testing.T) { + dir := t.TempDir() + on := true + b := &bundle.Bundle{ + SyncRootPath: dir, + Config: config.Root{ + Resources: config.Resources{ + JobRuns: map[string]*resources.JobRun{ + "my_run": { + Lifecycle: &resources.JobRunLifecycle{ + Triggers: []resources.JobRunTrigger{ + {OnBundleDeploy: &on}, + }, + }, + }, + }, + }, + }, + } + + diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) + assert.Empty(t, diags) + assert.Nil(t, b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers) + }) + + t.Run("multiple patterns merge into one map", func(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "a.txt"), []byte("aaa"), 0o644)) + require.NoError(t, os.MkdirAll(filepath.Join(dir, "subdir"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "subdir", "x.py"), []byte("bbb"), 0o644)) + + patA := "a.txt" + patB := "subdir/*.py" + b := &bundle.Bundle{ + SyncRootPath: dir, + Config: config.Root{ + Resources: config.Resources{ + JobRuns: map[string]*resources.JobRun{ + "my_run": { + Lifecycle: &resources.JobRunLifecycle{ + Triggers: []resources.JobRunTrigger{ + {OnFileChange: &patA}, + {OnFileChange: &patB}, + }, + }, + }, + }, + }, + }, + } + + diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) + require.False(t, diags.HasError()) + + fps := b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers + require.Len(t, fps, 2) + assertFingerprint(t, fps["a.txt"], "aaa") + assertFingerprint(t, fps["subdir/x.py"], "bbb") + }) +} + +func bundleWithFileTrigger(syncRoot, pattern string) *bundle.Bundle { + return &bundle.Bundle{ + SyncRootPath: syncRoot, + Config: config.Root{ + Resources: config.Resources{ + JobRuns: map[string]*resources.JobRun{ + "my_run": { + Lifecycle: &resources.JobRunLifecycle{ + Triggers: []resources.JobRunTrigger{ + {OnFileChange: &pattern}, + }, + }, + }, + }, + }, + }, + } +} + +func assertFingerprint(t *testing.T, fp resources.JobRunFileFingerprint, content string) { + t.Helper() + sum := sha256.Sum256([]byte(content)) + assert.Equal(t, hex.EncodeToString(sum[:]), fp.Hash) + assert.Equal(t, int64(len(content)), fp.Size) + assert.NotZero(t, fp.MtimeNs) +} diff --git a/bundle/config/mutator/validate_job_run_triggers.go b/bundle/config/mutator/validate_job_run_triggers.go index a7c5ada925e..c185d3b5781 100644 --- a/bundle/config/mutator/validate_job_run_triggers.go +++ b/bundle/config/mutator/validate_job_run_triggers.go @@ -3,6 +3,7 @@ package mutator import ( "context" "fmt" + "strings" "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" @@ -26,30 +27,45 @@ func (*validateJobRunTriggers) Apply(_ context.Context, b *bundle.Bundle) diag.D continue } // Recreate-every-deploy cannot coexist with prevent_destroy. - if jr.HasOnBundleDeploy() && jr.Lifecycle.PreventDestroy { + if (jr.HasOnBundleDeploy() || jr.HasOnFileChange()) && jr.Lifecycle.PreventDestroy { diags = diags.Append(diag.Diagnostic{ Severity: diag.Error, - Summary: "lifecycle.triggers.on_bundle_deploy is incompatible with lifecycle.prevent_destroy", + Summary: "lifecycle.triggers.on_bundle_deploy or on_file_change is incompatible with lifecycle.prevent_destroy", Locations: b.Config.GetLocations(fmt.Sprintf("resources.job_runs.%s.lifecycle", name)), }) } for i, t := range jr.Lifecycle.Triggers { path := fmt.Sprintf("resources.job_runs.%s.lifecycle.triggers[%d]", name, i) - if t.OnBundleDeploy == nil { + if t.OnBundleDeploy == nil && t.OnFileChange == nil { diags = diags.Append(diag.Diagnostic{ Severity: diag.Error, - Summary: "lifecycle.triggers entry must set on_bundle_deploy: true", + Summary: "lifecycle.triggers entry must set on_bundle_deploy or on_file_change", Locations: b.Config.GetLocations(path), }) continue } - if !*t.OnBundleDeploy { + if t.OnBundleDeploy != nil && t.OnFileChange != nil { + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: "lifecycle.triggers entry must set only one of on_bundle_deploy or on_file_change", + Locations: b.Config.GetLocations(path), + }) + continue + } + if t.OnBundleDeploy != nil && !*t.OnBundleDeploy { diags = diags.Append(diag.Diagnostic{ Severity: diag.Error, Summary: "lifecycle.triggers.on_bundle_deploy must be true when set", Locations: b.Config.GetLocations(path + ".on_bundle_deploy"), }) } + if t.OnFileChange != nil && strings.TrimSpace(*t.OnFileChange) == "" { + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: "lifecycle.triggers.on_file_change must be non-empty when set", + Locations: b.Config.GetLocations(path + ".on_file_change"), + }) + } } } return diags diff --git a/bundle/config/mutator/validate_job_run_triggers_test.go b/bundle/config/mutator/validate_job_run_triggers_test.go index 9c225457471..7e8772d8d8b 100644 --- a/bundle/config/mutator/validate_job_run_triggers_test.go +++ b/bundle/config/mutator/validate_job_run_triggers_test.go @@ -14,6 +14,10 @@ func TestValidateJobRunTriggers(t *testing.T) { trueVal := true falseVal := false + fileChange := "seed.txt" + emptyFile := "" + whitespaceFile := " \t" + tests := []struct { name string triggers []resources.JobRunTrigger @@ -26,12 +30,32 @@ func TestValidateJobRunTriggers(t *testing.T) { {OnBundleDeploy: &trueVal}, }, }, + { + name: "on_file_change set", + triggers: []resources.JobRunTrigger{ + {OnFileChange: &fileChange}, + }, + }, + { + name: "both triggers as separate entries", + triggers: []resources.JobRunTrigger{ + {OnFileChange: &fileChange}, + {OnBundleDeploy: &trueVal}, + }, + }, { name: "empty entry", triggers: []resources.JobRunTrigger{ {}, }, - summary: "lifecycle.triggers entry must set on_bundle_deploy: true", + summary: "lifecycle.triggers entry must set on_bundle_deploy or on_file_change", + }, + { + name: "both keys on one entry", + triggers: []resources.JobRunTrigger{ + {OnBundleDeploy: &trueVal, OnFileChange: &fileChange}, + }, + summary: "lifecycle.triggers entry must set only one of on_bundle_deploy or on_file_change", }, { name: "on_bundle_deploy false", @@ -40,13 +64,35 @@ func TestValidateJobRunTriggers(t *testing.T) { }, summary: "lifecycle.triggers.on_bundle_deploy must be true when set", }, + { + name: "on_file_change empty", + triggers: []resources.JobRunTrigger{ + {OnFileChange: &emptyFile}, + }, + summary: "lifecycle.triggers.on_file_change must be non-empty when set", + }, + { + name: "on_file_change whitespace", + triggers: []resources.JobRunTrigger{ + {OnFileChange: &whitespaceFile}, + }, + summary: "lifecycle.triggers.on_file_change must be non-empty when set", + }, { name: "on_bundle_deploy with prevent_destroy", triggers: []resources.JobRunTrigger{ {OnBundleDeploy: &trueVal}, }, preventDestroy: true, - summary: "lifecycle.triggers.on_bundle_deploy is incompatible with lifecycle.prevent_destroy", + summary: "lifecycle.triggers.on_bundle_deploy or on_file_change is incompatible with lifecycle.prevent_destroy", + }, + { + name: "on_file_change with prevent_destroy", + triggers: []resources.JobRunTrigger{ + {OnFileChange: &fileChange}, + }, + preventDestroy: true, + summary: "lifecycle.triggers.on_bundle_deploy or on_file_change is incompatible with lifecycle.prevent_destroy", }, { name: "prevent_destroy alone", diff --git a/bundle/config/resources/job_run.go b/bundle/config/resources/job_run.go index 56f3c39a356..cf66c71f728 100644 --- a/bundle/config/resources/job_run.go +++ b/bundle/config/resources/job_run.go @@ -27,6 +27,10 @@ type JobRun struct { // the run URL. Keeping it separate from RunNow.JobId (a ${resources.jobs.*.id} // reference) lets state loading preserve that reference and its plan dependency. ResolvedJobID int64 `json:"resolved_job_id,omitempty" bundle:"internal"` + + // ResolvedFileTriggers is the per-file fingerprint map for on_file_change, + // computed under SyncRoot before plan. bundle:"internal" keeps it out of schema. + ResolvedFileTriggers map[string]JobRunFileFingerprint `json:"resolved_file_triggers,omitempty" bundle:"internal"` } // HasOnBundleDeploy reports whether any trigger re-fires on every deploy. @@ -42,6 +46,31 @@ func (r *JobRun) HasOnBundleDeploy() bool { return false } +func (r *JobRun) HasOnFileChange() bool { + if r.Lifecycle == nil { + return false + } + for _, t := range r.Lifecycle.Triggers { + if t.OnFileChange != nil { + return true + } + } + return false +} + +func (r *JobRun) OnFileChangePatterns() []string { + if r.Lifecycle == nil { + return nil + } + var patterns []string + for _, t := range r.Lifecycle.Triggers { + if t.OnFileChange != nil { + patterns = append(patterns, *t.OnFileChange) + } + } + return patterns +} + func (r *JobRun) UnmarshalJSON(b []byte) error { return marshal.Unmarshal(b, r) } diff --git a/bundle/config/resources/lifecycle.go b/bundle/config/resources/lifecycle.go index 88d130239ee..59ccb682b81 100644 --- a/bundle/config/resources/lifecycle.go +++ b/bundle/config/resources/lifecycle.go @@ -36,5 +36,12 @@ type JobRunLifecycle struct { // JobRunTrigger is one lifecycle.triggers entry. type JobRunTrigger struct { - OnBundleDeploy *bool `json:"on_bundle_deploy,omitempty"` + OnBundleDeploy *bool `json:"on_bundle_deploy,omitempty"` + OnFileChange *string `json:"on_file_change,omitempty"` // path or glob under sync root +} + +type JobRunFileFingerprint struct { + Hash string `json:"hash"` + Size int64 `json:"size"` + MtimeNs int64 `json:"mtime_ns"` } diff --git a/bundle/direct/dresources/job_run.go b/bundle/direct/dresources/job_run.go index 9eb671f430c..4e57a4bcdf4 100644 --- a/bundle/direct/dresources/job_run.go +++ b/bundle/direct/dresources/job_run.go @@ -26,10 +26,20 @@ import ( // jobRunTimeout matches the timeout `bundle run` allows a run (bundle/run/job.go). const jobRunTimeout = 24 * time.Hour +// jobRunTriggerLocalPaths are local-only fingerprints: clearing one is skip, not recreate. +var jobRunTriggerLocalPaths = []string{ + "lifecycle", + "lifecycle.triggers", + "lifecycle.triggers.on_bundle_deploy", + "lifecycle.triggers.on_file_change", +} + // JobRunTriggersState is the persisted fingerprint of lifecycle.triggers. type JobRunTriggersState struct { // Fresh UUID each plan while armed so Old!=New forces recreate. OnBundleDeploy string `json:"on_bundle_deploy,omitempty"` + // Per-file fingerprints from ResolveJobRunFileTriggers; change to recreate. + OnFileChange map[string]resources.JobRunFileFingerprint `json:"on_file_change,omitempty"` } // JobRunLifecycleState holds local-only lifecycle fields persisted in state. @@ -97,12 +107,19 @@ func (*ResourceJobRun) PrepareState(input *resources.JobRun) *JobRunState { state := &JobRunState{ RunNow: input.RunNow, ResultState: jobs.RunResultStateSuccess, - Lifecycle: nil, } + var triggers *JobRunTriggersState if input.HasOnBundleDeploy() { - state.Lifecycle = &JobRunLifecycleState{ - Triggers: &JobRunTriggersState{OnBundleDeploy: uuid.NewString()}, + triggers = &JobRunTriggersState{OnBundleDeploy: uuid.NewString()} + } + if len(input.ResolvedFileTriggers) > 0 { + if triggers == nil { + triggers = &JobRunTriggersState{} } + triggers.OnFileChange = input.ResolvedFileTriggers + } + if triggers != nil { + state.Lifecycle = &JobRunLifecycleState{Triggers: triggers} } return state } @@ -371,8 +388,7 @@ func reportRunLine(ctx context.Context, runID int64, msg string) { // result_state either, so the lifecycle state is what tells the two apart. // Clearing a trigger skips its local-only fingerprint without re-firing the run. func (*ResourceJobRun) OverrideChangeDesc(_ context.Context, path *structpath.PathNode, change *ChangeDesc, remote *JobRunRemote) error { - switch path.String() { - case "lifecycle", "lifecycle.triggers", "lifecycle.triggers.on_bundle_deploy": + if slices.Contains(jobRunTriggerLocalPaths, path.String()) { // A cleared trigger sets New empty; structdiff may report it at lifecycle, // lifecycle.triggers, or the leaf. if change.New == nil || change.New == "" { @@ -380,6 +396,8 @@ func (*ResourceJobRun) OverrideChangeDesc(_ context.Context, path *structpath.Pa change.Reason = "trigger removed" } return nil + } + switch path.String() { case "result_state": // The planner passes no remote state when the run could not be read. if remote == nil || runIsTerminal(remote.State.LifeCycleState) { diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index 472a4a0def8..2907eb60f2d 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -182,6 +182,12 @@ resources: recreate_on_changes: - reason: immutable + # mtime is stored for the resolve fast path; content identity is the hash. + # Ignoring mtime-only local drift keeps touch from recreating the run. + ignore_local_changes: + - field: lifecycle.triggers.on_file_change[*].mtime_ns + reason: mtime_only_fingerprint + pipelines: recreate_on_changes: - field: storage diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index e47f89c505a..b94e79b5933 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -1017,6 +1017,9 @@ resources: "on_bundle_deploy": "description": |- If true, re-fire the run on every bundle deploy. Incompatible with lifecycle.prevent_destroy. + "on_file_change": + "description": |- + Path or glob under the sync root. Re-fire the run when any matched file's content changes. Incompatible with lifecycle.prevent_destroy. "python_named_params": "description": |- PLACEHOLDER diff --git a/bundle/phases/initialize.go b/bundle/phases/initialize.go index b15e1c30df6..9eba6f466e0 100644 --- a/bundle/phases/initialize.go +++ b/bundle/phases/initialize.go @@ -26,7 +26,8 @@ import ( func Initialize(ctx context.Context, b *bundle.Bundle) { log.Info(ctx, "Phase: initialize") - bundle.ApplySeqContext(ctx, b, + bundle.ApplySeqContext( + ctx, b, // Reads (dynamic): resource.*.* // Checks that none of resources.. is nil. Raises error otherwise. validate.AllResourcesHaveValues(), @@ -191,6 +192,9 @@ func Initialize(ctx context.Context, b *bundle.Bundle) { // Reject invalid job_runs.lifecycle.triggers (empty, false, prevent_destroy). mutator.ValidateJobRunTriggers(), + // Expand on_file_change globs and hash matched files into ResolvedFileTriggers. + mutator.ResolveJobRunFileTriggers(), + // Reads (dynamic): * (strings) (searches for ${resources.*} references) // Warns (TF engine) or errors (direct engine) when a cross-resource reference // points to a Terraform-only field with no DABs equivalent. diff --git a/bundle/phases/plan.go b/bundle/phases/plan.go index 3db0864c2c4..92ca524cc4f 100644 --- a/bundle/phases/plan.go +++ b/bundle/phases/plan.go @@ -21,7 +21,8 @@ import ( // PreDeployChecks is common set of mutators between "bundle plan" and "bundle deploy". // Note, it is not run in "bundle migrate" so it must not modify the config func PreDeployChecks(ctx context.Context, b *bundle.Bundle, isPlan bool, engine engine.EngineType) { - bundle.ApplySeqContext(ctx, b, + bundle.ApplySeqContext( + ctx, b, terraform.CheckDashboardsModifiedRemotely(isPlan, engine), resourcemutator.SecretScopeFixups(engine), deploy.StatePull(), @@ -30,6 +31,7 @@ func PreDeployChecks(ctx context.Context, b *bundle.Bundle, isPlan bool, engine mutator.ValidateLifecycleStarted(engine), mutator.ValidateCascadeOnDestroy(engine), mutator.ValidateJobRunTriggers(), + mutator.ResolveJobRunFileTriggers(), statemgmt.CheckRunningResource(engine), ) } diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 425956ab06d..3605d5c3496 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -1394,6 +1394,10 @@ "on_bundle_deploy": { "description": "If true, re-fire the run on every bundle deploy. Incompatible with lifecycle.prevent_destroy.", "$ref": "#/$defs/bool" + }, + "on_file_change": { + "description": "Path or glob under the sync root. Re-fire the run when any matched file's content changes. Incompatible with lifecycle.prevent_destroy.", + "$ref": "#/$defs/string" } }, "additionalProperties": false From 9306589745bc0c12e9545da76ce95fb601712446 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Tue, 18 Aug 2026 12:41:58 +0000 Subject: [PATCH 02/26] job_runs: fix exhaustruct and regenerate refschema for on_file_change Satisfy the linter on PrepareState literals after adding OnFileChange, and refresh out.fields.txt so validate-generated stays green. --- acceptance/bundle/refschema/out.fields.txt | 11 +++++++++++ bundle/direct/dresources/job_run.go | 11 +++++++++-- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index de252dc4162..1088fe02a73 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -912,8 +912,14 @@ resources.job_runs.*.lifecycle.prevent_destroy bool INPUT resources.job_runs.*.lifecycle.triggers *dresources.JobRunTriggersState STATE resources.job_runs.*.lifecycle.triggers []resources.JobRunTrigger INPUT resources.job_runs.*.lifecycle.triggers.on_bundle_deploy string STATE +resources.job_runs.*.lifecycle.triggers.on_file_change map[string]resources.JobRunFileFingerprint STATE +resources.job_runs.*.lifecycle.triggers.on_file_change.* resources.JobRunFileFingerprint STATE +resources.job_runs.*.lifecycle.triggers.on_file_change.*.hash string STATE +resources.job_runs.*.lifecycle.triggers.on_file_change.*.mtime_ns int64 STATE +resources.job_runs.*.lifecycle.triggers.on_file_change.*.size int64 STATE resources.job_runs.*.lifecycle.triggers[*] resources.JobRunTrigger INPUT resources.job_runs.*.lifecycle.triggers[*].on_bundle_deploy *bool INPUT +resources.job_runs.*.lifecycle.triggers[*].on_file_change *string INPUT resources.job_runs.*.modified_status string INPUT resources.job_runs.*.notebook_params map[string]string ALL resources.job_runs.*.notebook_params.* string ALL @@ -936,6 +942,11 @@ resources.job_runs.*.python_params []string ALL resources.job_runs.*.python_params[*] string ALL resources.job_runs.*.queue *jobs.QueueSettings ALL resources.job_runs.*.queue.enabled bool ALL +resources.job_runs.*.resolved_file_triggers map[string]resources.JobRunFileFingerprint INPUT +resources.job_runs.*.resolved_file_triggers.* resources.JobRunFileFingerprint INPUT +resources.job_runs.*.resolved_file_triggers.*.hash string INPUT +resources.job_runs.*.resolved_file_triggers.*.mtime_ns int64 INPUT +resources.job_runs.*.resolved_file_triggers.*.size int64 INPUT resources.job_runs.*.resolved_job_id int64 INPUT resources.job_runs.*.result_state jobs.RunResultState REMOTE STATE resources.job_runs.*.run_id int64 REMOTE diff --git a/bundle/direct/dresources/job_run.go b/bundle/direct/dresources/job_run.go index 4e57a4bcdf4..f66991f1ca8 100644 --- a/bundle/direct/dresources/job_run.go +++ b/bundle/direct/dresources/job_run.go @@ -107,14 +107,21 @@ func (*ResourceJobRun) PrepareState(input *resources.JobRun) *JobRunState { state := &JobRunState{ RunNow: input.RunNow, ResultState: jobs.RunResultStateSuccess, + Lifecycle: nil, } var triggers *JobRunTriggersState if input.HasOnBundleDeploy() { - triggers = &JobRunTriggersState{OnBundleDeploy: uuid.NewString()} + triggers = &JobRunTriggersState{ + OnBundleDeploy: uuid.NewString(), + OnFileChange: nil, + } } if len(input.ResolvedFileTriggers) > 0 { if triggers == nil { - triggers = &JobRunTriggersState{} + triggers = &JobRunTriggersState{ + OnBundleDeploy: "", + OnFileChange: nil, + } } triggers.OnFileChange = input.ResolvedFileTriggers } From c7cf6162c1e0707caf4bfbad5ee4da3319feba8f Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Tue, 18 Aug 2026 13:03:41 +0000 Subject: [PATCH 03/26] job_runs: tighten on_file_change path and directory checks Reject patterns outside the sync root and directory-only matches so the trigger cannot fingerprint escaped paths or silently disarm. --- .../bundles/job-runs-on-file-change.md | 2 +- .../mutator/resolve_job_run_file_triggers.go | 31 ++++++++++++++++--- .../resolve_job_run_file_triggers_test.go | 25 +++++++++++++++ bundle/internal/schema/annotations.yml | 2 +- bundle/schema/jsonschema.json | 2 +- 5 files changed, 55 insertions(+), 7 deletions(-) diff --git a/.nextchanges/bundles/job-runs-on-file-change.md b/.nextchanges/bundles/job-runs-on-file-change.md index 00682403a0d..eadceda6268 100644 --- a/.nextchanges/bundles/job-runs-on-file-change.md +++ b/.nextchanges/bundles/job-runs-on-file-change.md @@ -1 +1 @@ -direct: `resources.job_runs` can set `lifecycle.triggers.on_file_change` to a path or glob to re-fire the run when matched file contents change. Removing the trigger does not recreate the existing run. +direct: `resources.job_runs` can set `lifecycle.triggers.on_file_change` to a path or glob under the sync root to re-fire the run when matched file contents change, or when matches appear or disappear. Removing the trigger does not recreate the existing run. diff --git a/bundle/config/mutator/resolve_job_run_file_triggers.go b/bundle/config/mutator/resolve_job_run_file_triggers.go index a82ed654be3..f0338b2d50a 100644 --- a/bundle/config/mutator/resolve_job_run_file_triggers.go +++ b/bundle/config/mutator/resolve_job_run_file_triggers.go @@ -80,7 +80,17 @@ func resolveFileTriggers(b *bundle.Bundle, name string, patterns []string, prev out := make(map[string]resources.JobRunFileFingerprint) for _, pattern := range patterns { path := fmt.Sprintf("resources.job_runs.%s.lifecycle.triggers", name) - matches, err := filepath.Glob(filepath.Join(b.SyncRootPath, filepath.FromSlash(pattern))) + localPattern := filepath.FromSlash(pattern) + // Keep fingerprints under SyncRoot; same IsLocal gate as translate_paths. + if !filepath.IsLocal(localPattern) { + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: pattern %q is not under the sync root", pattern), + Locations: b.Config.GetLocations(path), + }) + continue + } + matches, err := filepath.Glob(filepath.Join(b.SyncRootPath, localPattern)) if err != nil { diags = diags.Append(diag.Diagnostic{ Severity: diag.Error, @@ -90,7 +100,7 @@ func resolveFileTriggers(b *bundle.Bundle, name string, patterns []string, prev continue } if len(matches) == 0 { - // Distinct state when the path/glob matches nothing (design doc). + // Distinct state when the path/glob matches nothing so appear/disappear recreates. out[filepath.ToSlash(pattern)] = resources.JobRunFileFingerprint{ Size: missingFileSentinelSize, } @@ -101,6 +111,8 @@ func resolveFileTriggers(b *bundle.Bundle, name string, patterns []string, prev }) continue } + regularMatches := 0 + sawNonRegular := false for _, match := range matches { info, err := os.Stat(match) if err != nil { @@ -112,13 +124,15 @@ func resolveFileTriggers(b *bundle.Bundle, name string, patterns []string, prev continue } if !info.Mode().IsRegular() { + sawNonRegular = true continue } + regularMatches++ rel, err := filepath.Rel(b.SyncRootPath, match) - if err != nil { + if err != nil || !filepath.IsLocal(rel) { diags = diags.Append(diag.Diagnostic{ Severity: diag.Error, - Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: relative path for %q: %s", match, err), + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: matched path %q is not under the sync root", match), Locations: b.Config.GetLocations(path), }) continue @@ -135,6 +149,15 @@ func resolveFileTriggers(b *bundle.Bundle, name string, patterns []string, prev } out[key] = fp } + // A directory-only match would otherwise leave ResolvedFileTriggers empty + // and silently disarm the trigger while config still sets on_file_change. + if regularMatches == 0 && sawNonRegular { + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: pattern %q matches no regular files", pattern), + Locations: b.Config.GetLocations(path), + }) + } } return out, diags } diff --git a/bundle/config/mutator/resolve_job_run_file_triggers_test.go b/bundle/config/mutator/resolve_job_run_file_triggers_test.go index 9a9cf77bf53..06266d75894 100644 --- a/bundle/config/mutator/resolve_job_run_file_triggers_test.go +++ b/bundle/config/mutator/resolve_job_run_file_triggers_test.go @@ -113,6 +113,31 @@ func TestResolveJobRunFileTriggers(t *testing.T) { assertFingerprint(t, fps["a.txt"], "aaa") assertFingerprint(t, fps["subdir/x.py"], "bbb") }) + + t.Run("pattern outside sync root is an error", func(t *testing.T) { + dir := t.TempDir() + pattern := "../outside.txt" + b := bundleWithFileTrigger(dir, pattern) + + diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) + require.True(t, diags.HasError()) + require.Len(t, diags, 1) + assert.Contains(t, diags[0].Summary, `not under the sync root`) + assert.Empty(t, b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers) + }) + + t.Run("directory-only match is an error", func(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(dir, "migrations"), 0o755)) + pattern := "migrations" + b := bundleWithFileTrigger(dir, pattern) + + diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) + require.True(t, diags.HasError()) + require.Len(t, diags, 1) + assert.Contains(t, diags[0].Summary, `matches no regular files`) + assert.Empty(t, b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers) + }) } func bundleWithFileTrigger(syncRoot, pattern string) *bundle.Bundle { diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index b94e79b5933..85de818a711 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -1019,7 +1019,7 @@ resources: If true, re-fire the run on every bundle deploy. Incompatible with lifecycle.prevent_destroy. "on_file_change": "description": |- - Path or glob under the sync root. Re-fire the run when any matched file's content changes. Incompatible with lifecycle.prevent_destroy. + Path or glob under the sync root (filepath.Glob; `**` is not supported). Re-fire the run when a matched file's content changes, or when the set of matches appears or disappears. Incompatible with lifecycle.prevent_destroy. "python_named_params": "description": |- PLACEHOLDER diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 3605d5c3496..431308f46ad 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -1396,7 +1396,7 @@ "$ref": "#/$defs/bool" }, "on_file_change": { - "description": "Path or glob under the sync root. Re-fire the run when any matched file's content changes. Incompatible with lifecycle.prevent_destroy.", + "description": "Path or glob under the sync root (filepath.Glob; `**` is not supported). Re-fire the run when a matched file's content changes, or when the set of matches appears or disappears. Incompatible with lifecycle.prevent_destroy.", "$ref": "#/$defs/string" } }, From 6e63c23723761939f13d467ceee61ea014bffbeb Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Tue, 18 Aug 2026 13:16:41 +0000 Subject: [PATCH 04/26] job_runs: store on_file_change as path-to-hash only Drop size/mtime fingerprints and the resolve fast path so content identity alone drives recreate, and touch no longer needs a planner exception. --- .../bundles/job-runs-on-file-change.md | 2 +- acceptance/bundle/refschema/out.fields.txt | 14 +- .../job_runs/on_file_change/output.txt | 11 + .../resources/job_runs/on_file_change/script | 6 + .../mutator/resolve_job_run_file_triggers.go | 204 +++++++----------- ...lve_job_run_file_triggers_internal_test.go | 73 ------- .../resolve_job_run_file_triggers_test.go | 48 +++-- bundle/config/resources/job_run.go | 19 +- bundle/config/resources/lifecycle.go | 6 - bundle/direct/dresources/job_run.go | 4 +- bundle/direct/dresources/resources.yml | 6 - bundle/internal/schema/annotations.yml | 2 +- bundle/schema/jsonschema.json | 2 +- 13 files changed, 131 insertions(+), 266 deletions(-) delete mode 100644 bundle/config/mutator/resolve_job_run_file_triggers_internal_test.go diff --git a/.nextchanges/bundles/job-runs-on-file-change.md b/.nextchanges/bundles/job-runs-on-file-change.md index eadceda6268..95869a25ccf 100644 --- a/.nextchanges/bundles/job-runs-on-file-change.md +++ b/.nextchanges/bundles/job-runs-on-file-change.md @@ -1 +1 @@ -direct: `resources.job_runs` can set `lifecycle.triggers.on_file_change` to a path or glob under the sync root to re-fire the run when matched file contents change, or when matches appear or disappear. Removing the trigger does not recreate the existing run. +direct: `resources.job_runs` can set `lifecycle.triggers.on_file_change` to a path or glob under the sync root to re-fire the run when matched file contents change (content hash), or when matches appear or disappear. Removing the trigger does not recreate the existing run. diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index 1088fe02a73..82933993eee 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -912,11 +912,8 @@ resources.job_runs.*.lifecycle.prevent_destroy bool INPUT resources.job_runs.*.lifecycle.triggers *dresources.JobRunTriggersState STATE resources.job_runs.*.lifecycle.triggers []resources.JobRunTrigger INPUT resources.job_runs.*.lifecycle.triggers.on_bundle_deploy string STATE -resources.job_runs.*.lifecycle.triggers.on_file_change map[string]resources.JobRunFileFingerprint STATE -resources.job_runs.*.lifecycle.triggers.on_file_change.* resources.JobRunFileFingerprint STATE -resources.job_runs.*.lifecycle.triggers.on_file_change.*.hash string STATE -resources.job_runs.*.lifecycle.triggers.on_file_change.*.mtime_ns int64 STATE -resources.job_runs.*.lifecycle.triggers.on_file_change.*.size int64 STATE +resources.job_runs.*.lifecycle.triggers.on_file_change map[string]string STATE +resources.job_runs.*.lifecycle.triggers.on_file_change.* string STATE resources.job_runs.*.lifecycle.triggers[*] resources.JobRunTrigger INPUT resources.job_runs.*.lifecycle.triggers[*].on_bundle_deploy *bool INPUT resources.job_runs.*.lifecycle.triggers[*].on_file_change *string INPUT @@ -942,11 +939,8 @@ resources.job_runs.*.python_params []string ALL resources.job_runs.*.python_params[*] string ALL resources.job_runs.*.queue *jobs.QueueSettings ALL resources.job_runs.*.queue.enabled bool ALL -resources.job_runs.*.resolved_file_triggers map[string]resources.JobRunFileFingerprint INPUT -resources.job_runs.*.resolved_file_triggers.* resources.JobRunFileFingerprint INPUT -resources.job_runs.*.resolved_file_triggers.*.hash string INPUT -resources.job_runs.*.resolved_file_triggers.*.mtime_ns int64 INPUT -resources.job_runs.*.resolved_file_triggers.*.size int64 INPUT +resources.job_runs.*.resolved_file_triggers map[string]string INPUT +resources.job_runs.*.resolved_file_triggers.* string INPUT resources.job_runs.*.resolved_job_id int64 INPUT resources.job_runs.*.result_state jobs.RunResultState REMOTE STATE resources.job_runs.*.run_id int64 REMOTE diff --git a/acceptance/bundle/resources/job_runs/on_file_change/output.txt b/acceptance/bundle/resources/job_runs/on_file_change/output.txt index bcf5962f8fa..f356fe7fb61 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change/output.txt @@ -33,6 +33,17 @@ Resources: 0 created, 0 changed, 0 deleted, 2 unchanged >>> print_requests.py //jobs/run-now +=== touch without content change plans nothing +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change/default/files... +Files: 3 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 2 unchanged + +>>> print_requests.py //jobs/run-now + === editing the file re-fires >>> [CLI] bundle plan recreate job_runs.my_run diff --git a/acceptance/bundle/resources/job_runs/on_file_change/script b/acceptance/bundle/resources/job_runs/on_file_change/script index ff162e11f0b..0506dd5e106 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change/script +++ b/acceptance/bundle/resources/job_runs/on_file_change/script @@ -16,6 +16,12 @@ trace $CLI bundle plan trace $CLI bundle deploy trace print_requests.py //jobs/run-now +title "touch without content change plans nothing" +python -c "import os; os.utime('seed.txt', None)" +trace $CLI bundle plan +trace $CLI bundle deploy +trace print_requests.py //jobs/run-now + title "editing the file re-fires" update_file.py seed.txt "v1" "v2" trace $CLI bundle plan diff --git a/bundle/config/mutator/resolve_job_run_file_triggers.go b/bundle/config/mutator/resolve_job_run_file_triggers.go index f0338b2d50a..55a55a94b95 100644 --- a/bundle/config/mutator/resolve_job_run_file_triggers.go +++ b/bundle/config/mutator/resolve_job_run_file_triggers.go @@ -4,25 +4,23 @@ import ( "context" "crypto/sha256" "encoding/hex" - "encoding/json" "fmt" "io" "os" "path/filepath" + "strings" "github.com/databricks/cli/bundle" - "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/libs/diag" ) -// missingFileSentinelSize marks a pattern with no matching file so the next -// plan can distinguish "still missing" from "file appeared". -const missingFileSentinelSize = int64(-1) +// missingFileHash marks a pattern with no matching file so appear/disappear recreates. +const missingFileHash = "" type resolveJobRunFileTriggers struct{} // ResolveJobRunFileTriggers expands on_file_change globs and stores per-file -// fingerprints on each job_run for PrepareState to copy into local state. +// content hashes on each job_run for PrepareState to copy into local state. func ResolveJobRunFileTriggers() bundle.Mutator { return &resolveJobRunFileTriggers{} } @@ -34,155 +32,105 @@ func (*resolveJobRunFileTriggers) Name() string { func (*resolveJobRunFileTriggers) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { var diags diag.Diagnostics for name, jr := range b.Config.Resources.JobRuns { - if jr == nil { + if jr == nil || jr.Lifecycle == nil { continue } - patterns := jr.OnFileChangePatterns() - if len(patterns) == 0 { - continue + out := make(map[string]string) + for i, t := range jr.Lifecycle.Triggers { + if t.OnFileChange == nil { + continue + } + path := fmt.Sprintf("resources.job_runs.%s.lifecycle.triggers[%d].on_file_change", name, i) + hashes, d := resolveFileTrigger(b, path, strings.TrimSpace(*t.OnFileChange)) + diags = diags.Extend(d) + for k, v := range hashes { + out[k] = v + } + } + if len(out) == 0 { + jr.ResolvedFileTriggers = nil + } else { + jr.ResolvedFileTriggers = out } - fps, d := resolveFileTriggers(b, name, patterns, previousFileTriggers(b, name)) - diags = diags.Extend(d) - jr.ResolvedFileTriggers = fps } return diags } -// previousFileTriggers reads on_file_change fingerprints from deployment state -// when it is open (plan/deploy after StatePull). Used so unchanged content keeps -// a stable fingerprint across mtime-only updates (e.g. touch). -func previousFileTriggers(b *bundle.Bundle, name string) map[string]resources.JobRunFileFingerprint { - if b.DeploymentBundle.StateDB.Path == "" { - return nil - } - entry, ok := b.DeploymentBundle.StateDB.GetResourceEntry("resources.job_runs." + name) - if !ok || len(entry.State) == 0 { - return nil - } - var state struct { - Lifecycle *struct { - Triggers *struct { - OnFileChange map[string]resources.JobRunFileFingerprint `json:"on_file_change"` - } `json:"triggers"` - } `json:"lifecycle"` +func resolveFileTrigger(b *bundle.Bundle, loc, pattern string) (map[string]string, diag.Diagnostics) { + var diags diag.Diagnostics + out := make(map[string]string) + localPattern := filepath.FromSlash(pattern) + // Keep hashes under SyncRoot; same IsLocal gate as translate_paths. + if !filepath.IsLocal(localPattern) { + return out, diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: pattern %q is not under the sync root", pattern), + Locations: b.Config.GetLocations(loc), + }) } - if err := json.Unmarshal(entry.State, &state); err != nil { - return nil + matches, err := filepath.Glob(filepath.Join(b.SyncRootPath, localPattern)) + if err != nil { + return out, diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: invalid pattern %q: %s", pattern, err), + Locations: b.Config.GetLocations(loc), + }) } - if state.Lifecycle == nil || state.Lifecycle.Triggers == nil { - return nil + if len(matches) == 0 { + out[filepath.ToSlash(pattern)] = missingFileHash + return out, diags.Append(diag.Diagnostic{ + Severity: diag.Warning, + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: no files match %q", pattern), + Locations: b.Config.GetLocations(loc), + }) } - return state.Lifecycle.Triggers.OnFileChange -} - -func resolveFileTriggers(b *bundle.Bundle, name string, patterns []string, prev map[string]resources.JobRunFileFingerprint) (map[string]resources.JobRunFileFingerprint, diag.Diagnostics) { - var diags diag.Diagnostics - out := make(map[string]resources.JobRunFileFingerprint) - for _, pattern := range patterns { - path := fmt.Sprintf("resources.job_runs.%s.lifecycle.triggers", name) - localPattern := filepath.FromSlash(pattern) - // Keep fingerprints under SyncRoot; same IsLocal gate as translate_paths. - if !filepath.IsLocal(localPattern) { + regularMatches := 0 + sawNonRegular := false + for _, match := range matches { + info, err := os.Stat(match) + if err != nil { diags = diags.Append(diag.Diagnostic{ Severity: diag.Error, - Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: pattern %q is not under the sync root", pattern), - Locations: b.Config.GetLocations(path), + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: stat %q: %s", match, err), + Locations: b.Config.GetLocations(loc), }) continue } - matches, err := filepath.Glob(filepath.Join(b.SyncRootPath, localPattern)) - if err != nil { - diags = diags.Append(diag.Diagnostic{ - Severity: diag.Error, - Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: invalid pattern %q: %s", pattern, err), - Locations: b.Config.GetLocations(path), - }) + if !info.Mode().IsRegular() { + sawNonRegular = true continue } - if len(matches) == 0 { - // Distinct state when the path/glob matches nothing so appear/disappear recreates. - out[filepath.ToSlash(pattern)] = resources.JobRunFileFingerprint{ - Size: missingFileSentinelSize, - } + regularMatches++ + rel, err := filepath.Rel(b.SyncRootPath, match) + if err != nil || !filepath.IsLocal(rel) { diags = diags.Append(diag.Diagnostic{ - Severity: diag.Warning, - Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: no files match %q", pattern), - Locations: b.Config.GetLocations(path), + Severity: diag.Error, + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: matched path %q is not under the sync root", match), + Locations: b.Config.GetLocations(loc), }) continue } - regularMatches := 0 - sawNonRegular := false - for _, match := range matches { - info, err := os.Stat(match) - if err != nil { - diags = diags.Append(diag.Diagnostic{ - Severity: diag.Error, - Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: stat %q: %s", match, err), - Locations: b.Config.GetLocations(path), - }) - continue - } - if !info.Mode().IsRegular() { - sawNonRegular = true - continue - } - regularMatches++ - rel, err := filepath.Rel(b.SyncRootPath, match) - if err != nil || !filepath.IsLocal(rel) { - diags = diags.Append(diag.Diagnostic{ - Severity: diag.Error, - Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: matched path %q is not under the sync root", match), - Locations: b.Config.GetLocations(path), - }) - continue - } - key := filepath.ToSlash(rel) - fp, err := fingerprintFile(match, info, prev[key]) - if err != nil { - diags = diags.Append(diag.Diagnostic{ - Severity: diag.Error, - Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: hash %q: %s", match, err), - Locations: b.Config.GetLocations(path), - }) - continue - } - out[key] = fp - } - // A directory-only match would otherwise leave ResolvedFileTriggers empty - // and silently disarm the trigger while config still sets on_file_change. - if regularMatches == 0 && sawNonRegular { + hash, err := hashFile(match) + if err != nil { diags = diags.Append(diag.Diagnostic{ Severity: diag.Error, - Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: pattern %q matches no regular files", pattern), - Locations: b.Config.GetLocations(path), + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: hash %q: %s", match, err), + Locations: b.Config.GetLocations(loc), }) + continue } + out[filepath.ToSlash(rel)] = hash } - return out, diags -} - -// fingerprintFile builds a content fingerprint. If prev has the same size and -// mtime, it is reused without reading the file. If content hash matches prev, -// prev is reused so mtime-only updates (touch) do not change planned state. -func fingerprintFile(path string, info os.FileInfo, prev resources.JobRunFileFingerprint) (resources.JobRunFileFingerprint, error) { - size := info.Size() - mtime := info.ModTime().UnixNano() - if prev.Hash != "" && prev.Size == size && prev.MtimeNs == mtime { - return prev, nil + // A directory-only match would otherwise leave ResolvedFileTriggers empty + // and silently disarm the trigger while config still sets on_file_change. + if regularMatches == 0 && sawNonRegular { + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: pattern %q matches no regular files", pattern), + Locations: b.Config.GetLocations(loc), + }) } - hash, err := hashFile(path) - if err != nil { - return resources.JobRunFileFingerprint{}, err - } - if prev.Hash != "" && prev.Hash == hash { - return prev, nil - } - return resources.JobRunFileFingerprint{ - Hash: hash, - Size: size, - MtimeNs: mtime, - }, nil + return out, diags } func hashFile(path string) (string, error) { diff --git a/bundle/config/mutator/resolve_job_run_file_triggers_internal_test.go b/bundle/config/mutator/resolve_job_run_file_triggers_internal_test.go deleted file mode 100644 index e82bd22945b..00000000000 --- a/bundle/config/mutator/resolve_job_run_file_triggers_internal_test.go +++ /dev/null @@ -1,73 +0,0 @@ -package mutator - -import ( - "os" - "path/filepath" - "testing" - "time" - - "github.com/databricks/cli/bundle/config/resources" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestFingerprintFileContentStableAcrossTouch(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, "seed.txt") - require.NoError(t, os.WriteFile(path, []byte("v1"), 0o644)) - - info, err := os.Stat(path) - require.NoError(t, err) - first, err := fingerprintFile(path, info, resources.JobRunFileFingerprint{}) - require.NoError(t, err) - require.NotEmpty(t, first.Hash) - - // Advance mtime without changing contents (touch). - require.NoError(t, os.Chtimes(path, time.Now().Add(time.Minute), time.Now().Add(time.Minute))) - info, err = os.Stat(path) - require.NoError(t, err) - assert.NotEqual(t, first.MtimeNs, info.ModTime().UnixNano()) - - second, err := fingerprintFile(path, info, first) - require.NoError(t, err) - assert.Equal(t, first, second, "unchanged content must reuse the previous fingerprint") -} - -func TestFingerprintFileFastPathSkipsWhenMtimeAndSizeMatch(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, "seed.txt") - require.NoError(t, os.WriteFile(path, []byte("v1"), 0o644)) - - info, err := os.Stat(path) - require.NoError(t, err) - prev := resources.JobRunFileFingerprint{ - Hash: "not-the-real-hash", - Size: info.Size(), - MtimeNs: info.ModTime().UnixNano(), - } - - got, err := fingerprintFile(path, info, prev) - require.NoError(t, err) - assert.Equal(t, prev, got, "matching size+mtime must reuse prev without re-hashing") -} - -func TestFingerprintFileContentChange(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, "seed.txt") - require.NoError(t, os.WriteFile(path, []byte("v1"), 0o644)) - - info, err := os.Stat(path) - require.NoError(t, err) - first, err := fingerprintFile(path, info, resources.JobRunFileFingerprint{}) - require.NoError(t, err) - - // Different size so the mtime+size fast path cannot reuse prev. - require.NoError(t, os.WriteFile(path, []byte("v2-changed"), 0o644)) - info, err = os.Stat(path) - require.NoError(t, err) - second, err := fingerprintFile(path, info, first) - require.NoError(t, err) - assert.NotEqual(t, first.Hash, second.Hash) - assert.Equal(t, info.Size(), second.Size) - assert.Equal(t, info.ModTime().UnixNano(), second.MtimeNs) -} diff --git a/bundle/config/mutator/resolve_job_run_file_triggers_test.go b/bundle/config/mutator/resolve_job_run_file_triggers_test.go index 06266d75894..3ec663c89bd 100644 --- a/bundle/config/mutator/resolve_job_run_file_triggers_test.go +++ b/bundle/config/mutator/resolve_job_run_file_triggers_test.go @@ -17,7 +17,7 @@ import ( ) func TestResolveJobRunFileTriggers(t *testing.T) { - t.Run("matches files and fills fingerprints", func(t *testing.T) { + t.Run("matches files and fills hashes", func(t *testing.T) { dir := t.TempDir() require.NoError(t, os.WriteFile(filepath.Join(dir, "a.txt"), []byte("hello"), 0o644)) require.NoError(t, os.WriteFile(filepath.Join(dir, "b.txt"), []byte("world"), 0o644)) @@ -28,14 +28,13 @@ func TestResolveJobRunFileTriggers(t *testing.T) { diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) require.False(t, diags.HasError()) - fps := b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers - require.Len(t, fps, 2) - - assertFingerprint(t, fps["a.txt"], "hello") - assertFingerprint(t, fps["b.txt"], "world") + hashes := b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers + require.Len(t, hashes, 2) + assert.Equal(t, contentHash("hello"), hashes["a.txt"]) + assert.Equal(t, contentHash("world"), hashes["b.txt"]) }) - t.Run("no matches warns and stores sentinel", func(t *testing.T) { + t.Run("no matches warns and stores empty hash", func(t *testing.T) { dir := t.TempDir() pattern := "missing.txt" b := bundleWithFileTrigger(dir, pattern) @@ -46,12 +45,9 @@ func TestResolveJobRunFileTriggers(t *testing.T) { assert.Equal(t, diag.Warning, diags[0].Severity) assert.Contains(t, diags[0].Summary, `no files match "missing.txt"`) - fps := b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers - require.Len(t, fps, 1) - fp := fps["missing.txt"] - assert.Empty(t, fp.Hash) - assert.Equal(t, int64(-1), fp.Size) - assert.Zero(t, fp.MtimeNs) + hashes := b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers + require.Len(t, hashes, 1) + assert.Empty(t, hashes["missing.txt"]) }) t.Run("no file triggers is a no-op", func(t *testing.T) { @@ -108,10 +104,10 @@ func TestResolveJobRunFileTriggers(t *testing.T) { diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) require.False(t, diags.HasError()) - fps := b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers - require.Len(t, fps, 2) - assertFingerprint(t, fps["a.txt"], "aaa") - assertFingerprint(t, fps["subdir/x.py"], "bbb") + hashes := b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers + require.Len(t, hashes, 2) + assert.Equal(t, contentHash("aaa"), hashes["a.txt"]) + assert.Equal(t, contentHash("bbb"), hashes["subdir/x.py"]) }) t.Run("pattern outside sync root is an error", func(t *testing.T) { @@ -138,6 +134,17 @@ func TestResolveJobRunFileTriggers(t *testing.T) { assert.Contains(t, diags[0].Summary, `matches no regular files`) assert.Empty(t, b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers) }) + + t.Run("trims pattern whitespace", func(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "seed.txt"), []byte("v1"), 0o644)) + pattern := " seed.txt " + b := bundleWithFileTrigger(dir, pattern) + + diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) + require.False(t, diags.HasError()) + assert.Equal(t, contentHash("v1"), b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers["seed.txt"]) + }) } func bundleWithFileTrigger(syncRoot, pattern string) *bundle.Bundle { @@ -159,10 +166,7 @@ func bundleWithFileTrigger(syncRoot, pattern string) *bundle.Bundle { } } -func assertFingerprint(t *testing.T, fp resources.JobRunFileFingerprint, content string) { - t.Helper() +func contentHash(content string) string { sum := sha256.Sum256([]byte(content)) - assert.Equal(t, hex.EncodeToString(sum[:]), fp.Hash) - assert.Equal(t, int64(len(content)), fp.Size) - assert.NotZero(t, fp.MtimeNs) + return hex.EncodeToString(sum[:]) } diff --git a/bundle/config/resources/job_run.go b/bundle/config/resources/job_run.go index cf66c71f728..f4b16963453 100644 --- a/bundle/config/resources/job_run.go +++ b/bundle/config/resources/job_run.go @@ -28,9 +28,9 @@ type JobRun struct { // reference) lets state loading preserve that reference and its plan dependency. ResolvedJobID int64 `json:"resolved_job_id,omitempty" bundle:"internal"` - // ResolvedFileTriggers is the per-file fingerprint map for on_file_change, - // computed under SyncRoot before plan. bundle:"internal" keeps it out of schema. - ResolvedFileTriggers map[string]JobRunFileFingerprint `json:"resolved_file_triggers,omitempty" bundle:"internal"` + // ResolvedFileTriggers is path → content hash for on_file_change, computed + // under SyncRoot before plan. bundle:"internal" keeps it out of schema. + ResolvedFileTriggers map[string]string `json:"resolved_file_triggers,omitempty" bundle:"internal"` } // HasOnBundleDeploy reports whether any trigger re-fires on every deploy. @@ -58,19 +58,6 @@ func (r *JobRun) HasOnFileChange() bool { return false } -func (r *JobRun) OnFileChangePatterns() []string { - if r.Lifecycle == nil { - return nil - } - var patterns []string - for _, t := range r.Lifecycle.Triggers { - if t.OnFileChange != nil { - patterns = append(patterns, *t.OnFileChange) - } - } - return patterns -} - func (r *JobRun) UnmarshalJSON(b []byte) error { return marshal.Unmarshal(b, r) } diff --git a/bundle/config/resources/lifecycle.go b/bundle/config/resources/lifecycle.go index 59ccb682b81..a4e881d298c 100644 --- a/bundle/config/resources/lifecycle.go +++ b/bundle/config/resources/lifecycle.go @@ -39,9 +39,3 @@ type JobRunTrigger struct { OnBundleDeploy *bool `json:"on_bundle_deploy,omitempty"` OnFileChange *string `json:"on_file_change,omitempty"` // path or glob under sync root } - -type JobRunFileFingerprint struct { - Hash string `json:"hash"` - Size int64 `json:"size"` - MtimeNs int64 `json:"mtime_ns"` -} diff --git a/bundle/direct/dresources/job_run.go b/bundle/direct/dresources/job_run.go index f66991f1ca8..5b97cc9e22b 100644 --- a/bundle/direct/dresources/job_run.go +++ b/bundle/direct/dresources/job_run.go @@ -38,8 +38,8 @@ var jobRunTriggerLocalPaths = []string{ type JobRunTriggersState struct { // Fresh UUID each plan while armed so Old!=New forces recreate. OnBundleDeploy string `json:"on_bundle_deploy,omitempty"` - // Per-file fingerprints from ResolveJobRunFileTriggers; change to recreate. - OnFileChange map[string]resources.JobRunFileFingerprint `json:"on_file_change,omitempty"` + // Path → content hash from ResolveJobRunFileTriggers; change to recreate. + OnFileChange map[string]string `json:"on_file_change,omitempty"` } // JobRunLifecycleState holds local-only lifecycle fields persisted in state. diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index 2907eb60f2d..472a4a0def8 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -182,12 +182,6 @@ resources: recreate_on_changes: - reason: immutable - # mtime is stored for the resolve fast path; content identity is the hash. - # Ignoring mtime-only local drift keeps touch from recreating the run. - ignore_local_changes: - - field: lifecycle.triggers.on_file_change[*].mtime_ns - reason: mtime_only_fingerprint - pipelines: recreate_on_changes: - field: storage diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index 85de818a711..60234453787 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -1019,7 +1019,7 @@ resources: If true, re-fire the run on every bundle deploy. Incompatible with lifecycle.prevent_destroy. "on_file_change": "description": |- - Path or glob under the sync root (filepath.Glob; `**` is not supported). Re-fire the run when a matched file's content changes, or when the set of matches appears or disappears. Incompatible with lifecycle.prevent_destroy. + Path or glob under the sync root (filepath.Glob; `**` is not supported). Re-fire the run when a matched file's content hash changes, or when the set of matches appears or disappears. Incompatible with lifecycle.prevent_destroy. "python_named_params": "description": |- PLACEHOLDER diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 431308f46ad..af58d318973 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -1396,7 +1396,7 @@ "$ref": "#/$defs/bool" }, "on_file_change": { - "description": "Path or glob under the sync root (filepath.Glob; `**` is not supported). Re-fire the run when a matched file's content changes, or when the set of matches appears or disappears. Incompatible with lifecycle.prevent_destroy.", + "description": "Path or glob under the sync root (filepath.Glob; `**` is not supported). Re-fire the run when a matched file's content hash changes, or when the set of matches appears or disappears. Incompatible with lifecycle.prevent_destroy.", "$ref": "#/$defs/string" } }, From 88f08fdeaacc4902f56afe9b90e05808bd76f6f0 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Tue, 18 Aug 2026 13:23:44 +0000 Subject: [PATCH 05/26] job_runs: use maps.Copy in on_file_change hash merge The modernize linter rejects the manual m[k]=v copy loop, failing CI lint. --- bundle/config/mutator/resolve_job_run_file_triggers.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/bundle/config/mutator/resolve_job_run_file_triggers.go b/bundle/config/mutator/resolve_job_run_file_triggers.go index 55a55a94b95..ead17c59e17 100644 --- a/bundle/config/mutator/resolve_job_run_file_triggers.go +++ b/bundle/config/mutator/resolve_job_run_file_triggers.go @@ -6,6 +6,7 @@ import ( "encoding/hex" "fmt" "io" + "maps" "os" "path/filepath" "strings" @@ -43,9 +44,7 @@ func (*resolveJobRunFileTriggers) Apply(_ context.Context, b *bundle.Bundle) dia path := fmt.Sprintf("resources.job_runs.%s.lifecycle.triggers[%d].on_file_change", name, i) hashes, d := resolveFileTrigger(b, path, strings.TrimSpace(*t.OnFileChange)) diags = diags.Extend(d) - for k, v := range hashes { - out[k] = v - } + maps.Copy(out, hashes) } if len(out) == 0 { jr.ResolvedFileTriggers = nil From 8977dcd65ba667e0c683598585f5c7c54d66ac60 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Wed, 19 Aug 2026 08:39:23 +0000 Subject: [PATCH 06/26] job_runs: drop filepath.Glob jargon from on_file_change schema docs --- bundle/internal/schema/annotations.yml | 2 +- bundle/schema/jsonschema.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index 60234453787..ff2d1fd2626 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -1019,7 +1019,7 @@ resources: If true, re-fire the run on every bundle deploy. Incompatible with lifecycle.prevent_destroy. "on_file_change": "description": |- - Path or glob under the sync root (filepath.Glob; `**` is not supported). Re-fire the run when a matched file's content hash changes, or when the set of matches appears or disappears. Incompatible with lifecycle.prevent_destroy. + Path or glob under the sync root. Re-fire the run when a matched file's content hash changes, or when the set of matches appears or disappears. Incompatible with lifecycle.prevent_destroy. "python_named_params": "description": |- PLACEHOLDER diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index af58d318973..11fbf411f7a 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -1396,7 +1396,7 @@ "$ref": "#/$defs/bool" }, "on_file_change": { - "description": "Path or glob under the sync root (filepath.Glob; `**` is not supported). Re-fire the run when a matched file's content hash changes, or when the set of matches appears or disappears. Incompatible with lifecycle.prevent_destroy.", + "description": "Path or glob under the sync root. Re-fire the run when a matched file's content hash changes, or when the set of matches appears or disappears. Incompatible with lifecycle.prevent_destroy.", "$ref": "#/$defs/string" } }, From 2db8ec518e54185e5cf44998d5789bbc457955bb Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Wed, 19 Aug 2026 11:38:20 +0000 Subject: [PATCH 07/26] acc: refresh on_file_change goldens for resource-key progress lines --- .../bundle/resources/job_runs/on_file_change/output.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/acceptance/bundle/resources/job_runs/on_file_change/output.txt b/acceptance/bundle/resources/job_runs/on_file_change/output.txt index f356fe7fb61..a98eb3a6ca7 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change/output.txt @@ -2,8 +2,8 @@ === first deploy triggers a run >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change/default/files... -job run [MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?o=[NUMID] -job run [MY_RUN_ID]: SUCCESS +Output from job_runs.my_run: id=[MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?o=[NUMID] +Output from job_runs.my_run: id=[MY_RUN_ID]: SUCCESS Created job_runs.my_run Created jobs.my_job Files: 5 uploaded, 0 deleted @@ -52,8 +52,8 @@ Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change/default/files... -job run [MY_RUN_ID_2]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_2]?o=[NUMID] -job run [MY_RUN_ID_2]: SUCCESS +Output from job_runs.my_run: id=[MY_RUN_ID_2]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_2]?o=[NUMID] +Output from job_runs.my_run: id=[MY_RUN_ID_2]: SUCCESS Recreated job_runs.my_run Files: 3 uploaded, 0 deleted Resources: 1 created, 0 changed, 1 deleted, 1 unchanged From 6fd75c8c961fc6629d39d8bfe5f10c848b60bc9c Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Wed, 19 Aug 2026 14:50:14 +0000 Subject: [PATCH 08/26] acc: cover on_file_change globs, errors, and appear/disappear --- .../job_runs/on_file_change/out.test.toml | 1 + .../job_runs/on_file_change/output.txt | 131 +++++++++++++- .../resources/job_runs/on_file_change/script | 26 ++- .../job_runs/on_file_change/test.toml | 14 ++ .../on_file_change_errors/databricks.yml | 18 ++ .../dirs/nested/keep.txt | 1 + .../on_file_change_errors/out.test.toml | 2 + .../job_runs/on_file_change_errors/output.txt | 39 +++++ .../job_runs/on_file_change_errors/script | 12 ++ .../job_runs/on_file_change_errors/test.toml | 2 + .../on_file_change_glob/databricks.yml | 18 ++ .../on_file_change_glob/migrations/001.txt | 1 + .../on_file_change_glob/migrations/002.txt | 1 + .../migrations/archive/003.txt | 1 + .../on_file_change_glob/out.test.toml | 2 + .../job_runs/on_file_change_glob/output.txt | 164 ++++++++++++++++++ .../job_runs/on_file_change_glob/script | 47 +++++ .../job_runs/on_file_change_glob/test.toml | 9 + bundle/phases/plan.go | 4 +- 19 files changed, 486 insertions(+), 7 deletions(-) create mode 100644 acceptance/bundle/resources/job_runs/on_file_change/test.toml create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_errors/databricks.yml create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_errors/dirs/nested/keep.txt create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_errors/out.test.toml create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_errors/script create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_errors/test.toml create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_glob/databricks.yml create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/001.txt create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/002.txt create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/archive/003.txt create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_glob/out.test.toml create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_glob/script create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml diff --git a/acceptance/bundle/resources/job_runs/on_file_change/out.test.toml b/acceptance/bundle/resources/job_runs/on_file_change/out.test.toml index 0938e678987..57b0f616850 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change/out.test.toml +++ b/acceptance/bundle/resources/job_runs/on_file_change/out.test.toml @@ -1,2 +1,3 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/job_runs/on_file_change/output.txt b/acceptance/bundle/resources/job_runs/on_file_change/output.txt index a98eb3a6ca7..d03307b90f2 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change/output.txt @@ -6,7 +6,7 @@ Output from job_runs.my_run: id=[MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[MY_ Output from job_runs.my_run: id=[MY_RUN_ID]: SUCCESS Created job_runs.my_run Created jobs.my_job -Files: 5 uploaded, 0 deleted +Files: 6 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged >>> read_id.py my_job @@ -50,12 +50,46 @@ recreate job_runs.my_run Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged ->>> [CLI] bundle deploy +>>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json +{ + "lifecycle": { + "action": "recreate", + "reason": "immutable", + "old": { + "triggers": { + "on_file_change": { + "seed.txt": "[FILE_HASH][0]" + } + } + }, + "new": { + "triggers": { + "on_file_change": { + "seed.txt": "[FILE_HASH][1]" + } + } + } + }, + "lifecycle.triggers.on_file_change['seed.txt']": { + "action": "recreate", + "reason": "immutable", + "old": "[FILE_HASH][0]", + "new": "[FILE_HASH][1]" + }, + "result_state": { + "action": "skip", + "reason": "remote_already_set", + "new": "SUCCESS", + "remote": "SUCCESS" + } +} + +=== bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change/default/files... Output from job_runs.my_run: id=[MY_RUN_ID_2]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_2]?o=[NUMID] Output from job_runs.my_run: id=[MY_RUN_ID_2]: SUCCESS Recreated job_runs.my_run -Files: 3 uploaded, 0 deleted +Files: 4 uploaded, 0 deleted Resources: 1 created, 0 changed, 1 deleted, 1 unchanged >>> print_requests.py --keep //jobs/runs/delete @@ -77,6 +111,97 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged } } +=== deleting the file warns and re-fires +>>> [CLI] bundle plan +Warning: lifecycle.triggers.on_file_change: no files match "seed.txt" + in databricks.yml:18:29 + +recreate job_runs.my_run + +Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged + +>>> changes +Warning: lifecycle.triggers.on_file_change: no files match "seed.txt" + in databricks.yml:18:29 + +{ + "lifecycle": { + "action": "recreate", + "reason": "immutable", + "old": { + "triggers": { + "on_file_change": { + "seed.txt": "[FILE_HASH][1]" + } + } + }, + "new": { + "triggers": { + "on_file_change": { + "seed.txt": "" + } + } + } + }, + "lifecycle.triggers.on_file_change['seed.txt']": { + "action": "recreate", + "reason": "immutable", + "old": "[FILE_HASH][1]", + "new": "" + }, + "result_state": { + "action": "skip", + "reason": "remote_already_set", + "new": "SUCCESS", + "remote": "SUCCESS" + } +} + +>>> [CLI] bundle deploy +Warning: lifecycle.triggers.on_file_change: no files match "seed.txt" + in databricks.yml:18:29 + +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change/default/files... +Output from job_runs.my_run: id=[MY_RUN_ID_3]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_3]?o=[NUMID] +Output from job_runs.my_run: id=[MY_RUN_ID_3]: SUCCESS +Recreated job_runs.my_run +Files: 3 uploaded, 1 deleted +Resources: 1 created, 0 changed, 1 deleted, 1 unchanged + +>>> print_requests.py //jobs/run-now +{ + "method": "POST", + "path": "/api/2.2/jobs/run-now", + "body": { + "idempotency_token": "[UUID]", + "job_id": [MY_JOB_ID] + } +} + +=== restoring the file re-fires again +>>> [CLI] bundle plan +recreate job_runs.my_run + +Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change/default/files... +Output from job_runs.my_run: id=[MY_RUN_ID_4]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_4]?o=[NUMID] +Output from job_runs.my_run: id=[MY_RUN_ID_4]: SUCCESS +Recreated job_runs.my_run +Files: 3 uploaded, 1 deleted +Resources: 1 created, 0 changed, 1 deleted, 1 unchanged + +>>> print_requests.py //jobs/run-now +{ + "method": "POST", + "path": "/api/2.2/jobs/run-now", + "body": { + "idempotency_token": "[UUID]", + "job_id": [MY_JOB_ID] + } +} + === removing on_file_change rewrites state without a run >>> [CLI] bundle plan update job_runs.my_run diff --git a/acceptance/bundle/resources/job_runs/on_file_change/script b/acceptance/bundle/resources/job_runs/on_file_change/script index 0506dd5e106..fd1e8800c42 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change/script +++ b/acceptance/bundle/resources/job_runs/on_file_change/script @@ -4,6 +4,10 @@ cleanup() { } trap cleanup EXIT +changes() { + $CLI bundle plan -o json | jq '.plan["resources.job_runs.my_run"].changes' +} + title "first deploy triggers a run" trace $CLI bundle deploy trace read_id.py my_job @@ -25,11 +29,31 @@ trace print_requests.py //jobs/run-now title "editing the file re-fires" update_file.py seed.txt "v1" "v2" trace $CLI bundle plan -trace $CLI bundle deploy +# Save the plan so the READPLAN=1 variant deploys the hashes computed here. +# The deploy is not traced: readplanarg makes the command line differ per variant. +$CLI bundle plan -o json > tmp.plan.json +trace jq '.plan["resources.job_runs.my_run"].changes' tmp.plan.json +title "bundle deploy\n" +$CLI bundle deploy $(readplanarg tmp.plan.json) read_id.py my_run > /dev/null trace print_requests.py --keep //jobs/runs/delete trace print_requests.py //jobs/run-now +title "deleting the file warns and re-fires" +mv seed.txt tmp.seed.txt +trace $CLI bundle plan +trace changes +trace $CLI bundle deploy +read_id.py my_run > /dev/null +trace print_requests.py //jobs/run-now + +title "restoring the file re-fires again" +mv tmp.seed.txt seed.txt +trace $CLI bundle plan +trace $CLI bundle deploy +read_id.py my_run > /dev/null +trace print_requests.py //jobs/run-now + title "removing on_file_change rewrites state without a run" update_file.py databricks.yml " lifecycle: triggers: diff --git a/acceptance/bundle/resources/job_runs/on_file_change/test.toml b/acceptance/bundle/resources/job_runs/on_file_change/test.toml new file mode 100644 index 00000000000..7a950e9fd0c --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change/test.toml @@ -0,0 +1,14 @@ +# Deploy both by re-planning and from a plan saved on disk, so the hashes +# computed during planning survive plan serialization. +EnvMatrix.READPLAN = ["", "1"] + +Ignore = ["tmp.plan.json", "tmp.seed.txt"] + +# Content hashes are stable but unreadable, and the parent's `\d{8,}` rule +# mangles any digit run inside them. Order=1 runs before it; Distinct keeps +# different hashes distinguishable. +[[Repls]] +Old = '[0-9a-f]{64}' +New = '[FILE_HASH]' +Order = 1 +Distinct = true diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/databricks.yml b/acceptance/bundle/resources/job_runs/on_file_change_errors/databricks.yml new file mode 100644 index 00000000000..44c5433e52a --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/databricks.yml @@ -0,0 +1,18 @@ +bundle: + name: job-runs-on-file-change-errors + +resources: + jobs: + my_job: + name: my-job + tasks: + - task_key: main + notebook_task: + notebook_path: /Workspace/test + + job_runs: + my_run: + job_id: ${resources.jobs.my_job.id} + lifecycle: + triggers: + - on_file_change: ../shared/*.txt diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/dirs/nested/keep.txt b/acceptance/bundle/resources/job_runs/on_file_change_errors/dirs/nested/keep.txt new file mode 100644 index 00000000000..79c53955ef8 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/dirs/nested/keep.txt @@ -0,0 +1 @@ +nested diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/out.test.toml b/acceptance/bundle/resources/job_runs/on_file_change_errors/out.test.toml new file mode 100644 index 00000000000..0938e678987 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt new file mode 100644 index 00000000000..c7d3a98e1cb --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt @@ -0,0 +1,39 @@ + +=== a pattern outside the sync root is rejected +>>> musterr [CLI] bundle validate +Error: lifecycle.triggers.on_file_change: pattern "../shared/*.txt" is not under the sync root + in databricks.yml:18:29 + +Name: job-runs-on-file-change-errors +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-errors/default + +Found 1 error + +=== a glob that only matches directories is rejected +>>> musterr [CLI] bundle validate +Error: lifecycle.triggers.on_file_change: pattern "dirs/*" matches no regular files + in databricks.yml:18:29 + +Name: job-runs-on-file-change-errors +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-errors/default + +Found 1 error + +=== a pattern that matches nothing warns +>>> [CLI] bundle validate +Warning: lifecycle.triggers.on_file_change: no files match "missing/*.txt" + in databricks.yml:18:29 + +Name: job-runs-on-file-change-errors +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-errors/default + +Found 1 warning diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/script b/acceptance/bundle/resources/job_runs/on_file_change_errors/script new file mode 100644 index 00000000000..39596f0c772 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/script @@ -0,0 +1,12 @@ +title "a pattern outside the sync root is rejected" +trace musterr $CLI bundle validate + +title "a glob that only matches directories is rejected" +# dirs/* matches dirs/nested, a directory: nothing is left to hash, so the +# trigger would silently never fire. +update_file.py databricks.yml "../shared/*.txt" "dirs/*" +trace musterr $CLI bundle validate + +title "a pattern that matches nothing warns" +update_file.py databricks.yml "dirs/*" "missing/*.txt" +trace $CLI bundle validate diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/test.toml b/acceptance/bundle/resources/job_runs/on_file_change_errors/test.toml new file mode 100644 index 00000000000..7b88d613d4c --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/test.toml @@ -0,0 +1,2 @@ +# The bundle is never deployed here, so there are no requests worth recording. +RecordRequests = false diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/databricks.yml b/acceptance/bundle/resources/job_runs/on_file_change_glob/databricks.yml new file mode 100644 index 00000000000..3b148538f38 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/databricks.yml @@ -0,0 +1,18 @@ +bundle: + name: job-runs-on-file-change-glob + +resources: + jobs: + my_job: + name: my-job + tasks: + - task_key: main + notebook_task: + notebook_path: /Workspace/test + + job_runs: + my_run: + job_id: ${resources.jobs.my_job.id} + lifecycle: + triggers: + - on_file_change: migrations/* diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/001.txt b/acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/001.txt new file mode 100644 index 00000000000..5626abf0f72 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/001.txt @@ -0,0 +1 @@ +one diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/002.txt b/acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/002.txt new file mode 100644 index 00000000000..f719efd430d --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/002.txt @@ -0,0 +1 @@ +two diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/archive/003.txt b/acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/archive/003.txt new file mode 100644 index 00000000000..2bdf67abb16 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/archive/003.txt @@ -0,0 +1 @@ +three diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/out.test.toml b/acceptance/bundle/resources/job_runs/on_file_change_glob/out.test.toml new file mode 100644 index 00000000000..0938e678987 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt new file mode 100644 index 00000000000..798b459a9bd --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt @@ -0,0 +1,164 @@ + +=== deploy hashes the matched files +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-glob/default/files... +Output from job_runs.my_run: id=[MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?o=[NUMID] +Output from job_runs.my_run: id=[MY_RUN_ID]: SUCCESS +Created job_runs.my_run +Created jobs.my_job +Files: 8 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +>>> read_id.py my_job +[MY_JOB_ID] + +>>> print_requests.py //jobs/run-now +{ + "method": "POST", + "path": "/api/2.2/jobs/run-now", + "body": { + "idempotency_token": "[UUID]", + "job_id": [MY_JOB_ID] + } +} + +=== editing one matched file re-fires +>>> [CLI] bundle plan +recreate job_runs.my_run + +Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged + +>>> changes +{ + "lifecycle": { + "action": "recreate", + "reason": "immutable", + "old": { + "triggers": { + "on_file_change": { + "migrations/001.txt": "[FILE_HASH][0]", + "migrations/002.txt": "[FILE_HASH][1]" + } + } + }, + "new": { + "triggers": { + "on_file_change": { + "migrations/001.txt": "[FILE_HASH][2]", + "migrations/002.txt": "[FILE_HASH][1]" + } + } + } + }, + "lifecycle.triggers.on_file_change['migrations/001.txt']": { + "action": "recreate", + "reason": "immutable", + "old": "[FILE_HASH][0]", + "new": "[FILE_HASH][2]" + }, + "result_state": { + "action": "skip", + "reason": "remote_already_set", + "new": "SUCCESS", + "remote": "SUCCESS" + } +} + +=== a new matching file re-fires +>>> [CLI] bundle plan +recreate job_runs.my_run + +Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged + +>>> changes +{ + "lifecycle": { + "action": "recreate", + "reason": "immutable", + "old": { + "triggers": { + "on_file_change": { + "migrations/001.txt": "[FILE_HASH][0]", + "migrations/002.txt": "[FILE_HASH][1]" + } + } + }, + "new": { + "triggers": { + "on_file_change": { + "migrations/001.txt": "[FILE_HASH][0]", + "migrations/002.txt": "[FILE_HASH][1]", + "migrations/003.txt": "[FILE_HASH][3]" + } + } + } + }, + "lifecycle.triggers.on_file_change['migrations/003.txt']": { + "action": "recreate", + "reason": "immutable", + "new": "[FILE_HASH][3]" + }, + "result_state": { + "action": "skip", + "reason": "remote_already_set", + "new": "SUCCESS", + "remote": "SUCCESS" + } +} + +=== removing a matched file re-fires +>>> [CLI] bundle plan +recreate job_runs.my_run + +Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged + +>>> changes +{ + "lifecycle": { + "action": "recreate", + "reason": "immutable", + "old": { + "triggers": { + "on_file_change": { + "migrations/001.txt": "[FILE_HASH][0]", + "migrations/002.txt": "[FILE_HASH][1]" + } + } + }, + "new": { + "triggers": { + "on_file_change": { + "migrations/001.txt": "[FILE_HASH][0]" + } + } + } + }, + "lifecycle.triggers.on_file_change['migrations/002.txt']": { + "action": "recreate", + "reason": "immutable", + "old": "[FILE_HASH][1]" + }, + "result_state": { + "action": "skip", + "reason": "remote_already_set", + "new": "SUCCESS", + "remote": "SUCCESS" + } +} + +=== editing a file below the glob does not re-fire +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +=== ** is not recursive: it matches the same files as * +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.job_runs.my_run + delete resources.jobs.my_job + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-glob/default + +Destroy: 2 deleted diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/script b/acceptance/bundle/resources/job_runs/on_file_change_glob/script new file mode 100644 index 00000000000..8feeebcd067 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/script @@ -0,0 +1,47 @@ +cleanup() { + trace $CLI bundle destroy --auto-approve + rm -f out.requests.txt +} +trap cleanup EXIT + +changes() { + $CLI bundle plan -o json | jq '.plan["resources.job_runs.my_run"].changes' +} + +title "deploy hashes the matched files" +trace $CLI bundle deploy +trace read_id.py my_job +read_id.py my_run > /dev/null +trace print_requests.py //jobs/run-now + +# Each phase below plans against the state deployed above and then restores the +# tree, so every plan is compared against the same baseline. + +title "editing one matched file re-fires" +update_file.py migrations/001.txt "one" "one-edited" +trace $CLI bundle plan +trace changes +update_file.py migrations/001.txt "one-edited" "one" + +title "a new matching file re-fires" +printf 'three\n' > migrations/003.txt +trace $CLI bundle plan +trace changes +rm migrations/003.txt + +title "removing a matched file re-fires" +mv migrations/002.txt tmp.002.txt +trace $CLI bundle plan +trace changes +mv tmp.002.txt migrations/002.txt + +title "editing a file below the glob does not re-fire" +# migrations/* matches the archive directory itself, which is skipped: the glob +# does not descend into it. +update_file.py migrations/archive/003.txt "three" "three-edited" +trace $CLI bundle plan +update_file.py migrations/archive/003.txt "three-edited" "three" + +title "** is not recursive: it matches the same files as *" +update_file.py databricks.yml "migrations/*" "migrations/**" +trace $CLI bundle plan diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml b/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml new file mode 100644 index 00000000000..1c349066b94 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml @@ -0,0 +1,9 @@ +Ignore = ["tmp.002.txt"] + +# See on_file_change/test.toml: mask content hashes before the parent's +# `\d{8,}` rule can mangle them, keeping distinct hashes distinguishable. +[[Repls]] +Old = '[0-9a-f]{64}' +New = '[FILE_HASH]' +Order = 1 +Distinct = true diff --git a/bundle/phases/plan.go b/bundle/phases/plan.go index 92ca524cc4f..3db0864c2c4 100644 --- a/bundle/phases/plan.go +++ b/bundle/phases/plan.go @@ -21,8 +21,7 @@ import ( // PreDeployChecks is common set of mutators between "bundle plan" and "bundle deploy". // Note, it is not run in "bundle migrate" so it must not modify the config func PreDeployChecks(ctx context.Context, b *bundle.Bundle, isPlan bool, engine engine.EngineType) { - bundle.ApplySeqContext( - ctx, b, + bundle.ApplySeqContext(ctx, b, terraform.CheckDashboardsModifiedRemotely(isPlan, engine), resourcemutator.SecretScopeFixups(engine), deploy.StatePull(), @@ -31,7 +30,6 @@ func PreDeployChecks(ctx context.Context, b *bundle.Bundle, isPlan bool, engine mutator.ValidateLifecycleStarted(engine), mutator.ValidateCascadeOnDestroy(engine), mutator.ValidateJobRunTriggers(), - mutator.ResolveJobRunFileTriggers(), statemgmt.CheckRunningResource(engine), ) } From 2806e126f769cf304e0ea779c3ca3e41f5b5d70a Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Wed, 19 Aug 2026 19:58:11 +0000 Subject: [PATCH 09/26] acc: expand on_file_change glob coverage and skip ignored files Honor .gitignore and sync.exclude when hashing matches. Cover mysubdir globs, READPLAN, and keep LF on Windows so restored files hash cleanly. --- .../job_runs/on_file_change_glob/.gitignore | 2 + .../on_file_change_glob/databricks.yml | 1 + .../on_file_change_glob/mysubdir/a.txt | 1 + .../on_file_change_glob/out.test.toml | 1 + .../job_runs/on_file_change_glob/output.txt | 183 +++++++++++++++--- .../job_runs/on_file_change_glob/script | 39 ++-- .../job_runs/on_file_change_glob/test.toml | 6 +- .../mutator/resolve_job_run_file_triggers.go | 65 ++++++- .../resolve_job_run_file_triggers_test.go | 40 ++++ 9 files changed, 287 insertions(+), 51 deletions(-) create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_glob/.gitignore create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_glob/mysubdir/a.txt diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/.gitignore b/acceptance/bundle/resources/job_runs/on_file_change_glob/.gitignore new file mode 100644 index 00000000000..57433d43f47 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/.gitignore @@ -0,0 +1,2 @@ +# Matches migrations/* but sync skips it, so a content edit must not re-fire. +migrations/ignored.txt diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/databricks.yml b/acceptance/bundle/resources/job_runs/on_file_change_glob/databricks.yml index 3b148538f38..d47fcf44049 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/databricks.yml +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/databricks.yml @@ -16,3 +16,4 @@ resources: lifecycle: triggers: - on_file_change: migrations/* + - on_file_change: mysubdir/*.txt diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/mysubdir/a.txt b/acceptance/bundle/resources/job_runs/on_file_change_glob/mysubdir/a.txt new file mode 100644 index 00000000000..62e0af52c19 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/mysubdir/a.txt @@ -0,0 +1 @@ +sub diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/out.test.toml b/acceptance/bundle/resources/job_runs/on_file_change_glob/out.test.toml index 0938e678987..57b0f616850 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/out.test.toml +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/out.test.toml @@ -1,2 +1,3 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt index 798b459a9bd..d424482b961 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt @@ -6,7 +6,7 @@ Output from job_runs.my_run: id=[MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[MY_ Output from job_runs.my_run: id=[MY_RUN_ID]: SUCCESS Created job_runs.my_run Created jobs.my_job -Files: 8 uploaded, 0 deleted +Files: 10 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged >>> read_id.py my_job @@ -23,12 +23,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged } === editing one matched file re-fires ->>> [CLI] bundle plan -recreate job_runs.my_run - -Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged - ->>> changes +>>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { "lifecycle": { "action": "recreate", @@ -37,15 +32,17 @@ Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged "triggers": { "on_file_change": { "migrations/001.txt": "[FILE_HASH][0]", - "migrations/002.txt": "[FILE_HASH][1]" + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][2]" } } }, "new": { "triggers": { "on_file_change": { - "migrations/001.txt": "[FILE_HASH][2]", - "migrations/002.txt": "[FILE_HASH][1]" + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][2]" } } } @@ -54,7 +51,7 @@ Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged "action": "recreate", "reason": "immutable", "old": "[FILE_HASH][0]", - "new": "[FILE_HASH][2]" + "new": "[FILE_HASH][3]" }, "result_state": { "action": "skip", @@ -64,13 +61,35 @@ Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged } } -=== a new matching file re-fires ->>> [CLI] bundle plan -recreate job_runs.my_run +=== bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-glob/default/files... +Output from job_runs.my_run: id=[MY_RUN_ID_2]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_2]?o=[NUMID] +Output from job_runs.my_run: id=[MY_RUN_ID_2]: SUCCESS +Recreated job_runs.my_run +Files: 4 uploaded, 0 deleted +Resources: 1 created, 0 changed, 1 deleted, 1 unchanged + +>>> print_requests.py --keep //jobs/runs/delete +{ + "method": "POST", + "path": "/api/2.2/jobs/runs/delete", + "body": { + "run_id": [MY_RUN_ID] + } +} -Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged +>>> print_requests.py //jobs/run-now +{ + "method": "POST", + "path": "/api/2.2/jobs/run-now", + "body": { + "idempotency_token": "[UUID]", + "job_id": [MY_JOB_ID] + } +} ->>> changes +=== editing a file in a subdirectory re-fires +>>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { "lifecycle": { "action": "recreate", @@ -78,17 +97,58 @@ Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged "old": { "triggers": { "on_file_change": { - "migrations/001.txt": "[FILE_HASH][0]", - "migrations/002.txt": "[FILE_HASH][1]" + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][2]" } } }, "new": { "triggers": { "on_file_change": { - "migrations/001.txt": "[FILE_HASH][0]", + "migrations/001.txt": "[FILE_HASH][3]", "migrations/002.txt": "[FILE_HASH][1]", - "migrations/003.txt": "[FILE_HASH][3]" + "mysubdir/a.txt": "[FILE_HASH][4]" + } + } + } + }, + "lifecycle.triggers.on_file_change['mysubdir/a.txt']": { + "action": "recreate", + "reason": "immutable", + "old": "[FILE_HASH][2]", + "new": "[FILE_HASH][4]" + }, + "result_state": { + "action": "skip", + "reason": "remote_already_set", + "new": "SUCCESS", + "remote": "SUCCESS" + } +} + +=== a new matching file re-fires +>>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json +{ + "lifecycle": { + "action": "recreate", + "reason": "immutable", + "old": { + "triggers": { + "on_file_change": { + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][2]" + } + } + }, + "new": { + "triggers": { + "on_file_change": { + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "migrations/003.txt": "[FILE_HASH][5]", + "mysubdir/a.txt": "[FILE_HASH][2]" } } } @@ -96,7 +156,7 @@ Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged "lifecycle.triggers.on_file_change['migrations/003.txt']": { "action": "recreate", "reason": "immutable", - "new": "[FILE_HASH][3]" + "new": "[FILE_HASH][5]" }, "result_state": { "action": "skip", @@ -107,12 +167,7 @@ Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged } === removing a matched file re-fires ->>> [CLI] bundle plan -recreate job_runs.my_run - -Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged - ->>> changes +>>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { "lifecycle": { "action": "recreate", @@ -120,15 +175,17 @@ Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged "old": { "triggers": { "on_file_change": { - "migrations/001.txt": "[FILE_HASH][0]", - "migrations/002.txt": "[FILE_HASH][1]" + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][2]" } } }, "new": { "triggers": { "on_file_change": { - "migrations/001.txt": "[FILE_HASH][0]" + "migrations/001.txt": "[FILE_HASH][3]", + "mysubdir/a.txt": "[FILE_HASH][2]" } } } @@ -147,12 +204,74 @@ Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged } === editing a file below the glob does not re-fire +>>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json +{ + "lifecycle": { + "action": "skip", + "reason": "missing_in_remote", + "old": { + "triggers": { + "on_file_change": { + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][2]" + } + } + }, + "new": { + "triggers": { + "on_file_change": { + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][2]" + } + } + } + }, + "result_state": { + "action": "skip", + "reason": "remote_already_set", + "new": "SUCCESS", + "remote": "SUCCESS" + } +} + +=== editing a gitignored file does not re-fire >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged === ** is not recursive: it matches the same files as * ->>> [CLI] bundle plan -Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged +>>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json +{ + "lifecycle": { + "action": "skip", + "reason": "missing_in_remote", + "old": { + "triggers": { + "on_file_change": { + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][2]" + } + } + }, + "new": { + "triggers": { + "on_file_change": { + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][2]" + } + } + } + }, + "result_state": { + "action": "skip", + "reason": "remote_already_set", + "new": "SUCCESS", + "remote": "SUCCESS" + } +} >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/script b/acceptance/bundle/resources/job_runs/on_file_change_glob/script index 8feeebcd067..983cb9ac401 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/script +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/script @@ -4,8 +4,10 @@ cleanup() { } trap cleanup EXIT +# Plan once, print JSON changes, leave tmp.plan.json for readplanarg. changes() { - $CLI bundle plan -o json | jq '.plan["resources.job_runs.my_run"].changes' + $CLI bundle plan -o json > tmp.plan.json + trace jq '.plan["resources.job_runs.my_run"].changes' tmp.plan.json } title "deploy hashes the matched files" @@ -14,34 +16,45 @@ trace read_id.py my_job read_id.py my_run > /dev/null trace print_requests.py //jobs/run-now -# Each phase below plans against the state deployed above and then restores the -# tree, so every plan is compared against the same baseline. - title "editing one matched file re-fires" update_file.py migrations/001.txt "one" "one-edited" -trace $CLI bundle plan -trace changes -update_file.py migrations/001.txt "one-edited" "one" +changes +# The deploy is not traced: readplanarg makes the command line differ per variant. +title "bundle deploy\n" +$CLI bundle deploy $(readplanarg tmp.plan.json) +read_id.py my_run > /dev/null +trace print_requests.py --keep //jobs/runs/delete +trace print_requests.py //jobs/run-now + +title "editing a file in a subdirectory re-fires" +# printf keeps LF on Windows, where update_file.py's text-mode rewrite would +# restore CRLF and leave the file hashing differently than the deploy above. +printf 'sub-edited\n' > mysubdir/a.txt +changes +printf 'sub\n' > mysubdir/a.txt title "a new matching file re-fires" printf 'three\n' > migrations/003.txt -trace $CLI bundle plan -trace changes +changes rm migrations/003.txt title "removing a matched file re-fires" mv migrations/002.txt tmp.002.txt -trace $CLI bundle plan -trace changes +changes mv tmp.002.txt migrations/002.txt title "editing a file below the glob does not re-fire" # migrations/* matches the archive directory itself, which is skipped: the glob # does not descend into it. update_file.py migrations/archive/003.txt "three" "three-edited" -trace $CLI bundle plan +changes update_file.py migrations/archive/003.txt "three-edited" "three" +title "editing a gitignored file does not re-fire" +update_file.py migrations/ignored.txt "ignored" "ignored-edited" +trace $CLI bundle plan +update_file.py migrations/ignored.txt "ignored-edited" "ignored" + title "** is not recursive: it matches the same files as *" update_file.py databricks.yml "migrations/*" "migrations/**" -trace $CLI bundle plan +changes diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml b/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml index 1c349066b94..3516572564d 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml @@ -1,4 +1,8 @@ -Ignore = ["tmp.002.txt"] +# Deploy both by re-planning and from a plan saved on disk, so the hashes +# computed during planning survive plan serialization. +EnvMatrix.READPLAN = ["", "1"] + +Ignore = ["tmp.plan.json", "tmp.002.txt"] # See on_file_change/test.toml: mask content hashes before the parent's # `\d{8,}` rule can mangle them, keeping distinct hashes distinguishable. diff --git a/bundle/config/mutator/resolve_job_run_file_triggers.go b/bundle/config/mutator/resolve_job_run_file_triggers.go index ead17c59e17..d74b9841c57 100644 --- a/bundle/config/mutator/resolve_job_run_file_triggers.go +++ b/bundle/config/mutator/resolve_job_run_file_triggers.go @@ -13,6 +13,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" + libsync "github.com/databricks/cli/libs/sync" ) // missingFileHash marks a pattern with no matching file so appear/disappear recreates. @@ -30,8 +31,13 @@ func (*resolveJobRunFileTriggers) Name() string { return "ResolveJobRunFileTriggers" } -func (*resolveJobRunFileTriggers) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { +func (*resolveJobRunFileTriggers) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { var diags diag.Diagnostics + syncable, d := syncableRelPaths(ctx, b) + if d.HasError() { + return d + } + diags = diags.Extend(d) for name, jr := range b.Config.Resources.JobRuns { if jr == nil || jr.Lifecycle == nil { continue @@ -42,7 +48,7 @@ func (*resolveJobRunFileTriggers) Apply(_ context.Context, b *bundle.Bundle) dia continue } path := fmt.Sprintf("resources.job_runs.%s.lifecycle.triggers[%d].on_file_change", name, i) - hashes, d := resolveFileTrigger(b, path, strings.TrimSpace(*t.OnFileChange)) + hashes, d := resolveFileTrigger(b, path, strings.TrimSpace(*t.OnFileChange), syncable) diags = diags.Extend(d) maps.Copy(out, hashes) } @@ -55,7 +61,42 @@ func (*resolveJobRunFileTriggers) Apply(_ context.Context, b *bundle.Bundle) dia return diags } -func resolveFileTrigger(b *bundle.Bundle, loc, pattern string) (map[string]string, diag.Diagnostics) { +// syncableRelPaths is the set of relative paths sync would upload. +func syncableRelPaths(ctx context.Context, b *bundle.Bundle) (map[string]struct{}, diag.Diagnostics) { + var diags diag.Diagnostics + needs := false + for _, jr := range b.Config.Resources.JobRuns { + if jr != nil && jr.HasOnFileChange() { + needs = true + break + } + } + if !needs { + return nil, diags + } + + fl, err := libsync.NewFileList(ctx, b.WorktreeRoot, b.SyncRoot, b.Config.Sync.Paths, b.Config.Sync.Include, b.Config.Sync.Exclude) + if err != nil { + return nil, diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: list sync files: %s", err), + }) + } + files, err := fl.Files(ctx) + if err != nil { + return nil, diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: list sync files: %s", err), + }) + } + out := make(map[string]struct{}, len(files)) + for _, f := range files { + out[filepath.ToSlash(f.Relative)] = struct{}{} + } + return out, diags +} + +func resolveFileTrigger(b *bundle.Bundle, loc, pattern string, syncable map[string]struct{}) (map[string]string, diag.Diagnostics) { var diags diag.Diagnostics out := make(map[string]string) localPattern := filepath.FromSlash(pattern) @@ -84,6 +125,7 @@ func resolveFileTrigger(b *bundle.Bundle, loc, pattern string) (map[string]strin }) } regularMatches := 0 + ignoredMatches := 0 sawNonRegular := false for _, match := range matches { info, err := os.Stat(match) @@ -99,7 +141,6 @@ func resolveFileTrigger(b *bundle.Bundle, loc, pattern string) (map[string]strin sawNonRegular = true continue } - regularMatches++ rel, err := filepath.Rel(b.SyncRootPath, match) if err != nil || !filepath.IsLocal(rel) { diags = diags.Append(diag.Diagnostic{ @@ -109,6 +150,12 @@ func resolveFileTrigger(b *bundle.Bundle, loc, pattern string) (map[string]strin }) continue } + // Same membership as sync: .gitignore and sync.exclude drop a glob match. + if _, ok := syncable[filepath.ToSlash(rel)]; !ok { + ignoredMatches++ + continue + } + regularMatches++ hash, err := hashFile(match) if err != nil { diags = diags.Append(diag.Diagnostic{ @@ -122,13 +169,21 @@ func resolveFileTrigger(b *bundle.Bundle, loc, pattern string) (map[string]strin } // A directory-only match would otherwise leave ResolvedFileTriggers empty // and silently disarm the trigger while config still sets on_file_change. - if regularMatches == 0 && sawNonRegular { + if regularMatches == 0 && sawNonRegular && ignoredMatches == 0 { diags = diags.Append(diag.Diagnostic{ Severity: diag.Error, Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: pattern %q matches no regular files", pattern), Locations: b.Config.GetLocations(loc), }) } + if len(out) == 0 && ignoredMatches > 0 { + out[filepath.ToSlash(pattern)] = missingFileHash + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Warning, + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: no files match %q", pattern), + Locations: b.Config.GetLocations(loc), + }) + } return out, diags } diff --git a/bundle/config/mutator/resolve_job_run_file_triggers_test.go b/bundle/config/mutator/resolve_job_run_file_triggers_test.go index 3ec663c89bd..344fed25657 100644 --- a/bundle/config/mutator/resolve_job_run_file_triggers_test.go +++ b/bundle/config/mutator/resolve_job_run_file_triggers_test.go @@ -12,6 +12,7 @@ import ( "github.com/databricks/cli/bundle/config/mutator" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/vfs" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -85,7 +86,10 @@ func TestResolveJobRunFileTriggers(t *testing.T) { patB := "subdir/*.py" b := &bundle.Bundle{ SyncRootPath: dir, + SyncRoot: vfs.MustNew(dir), + WorktreeRoot: vfs.MustNew(dir), Config: config.Root{ + Sync: config.Sync{Paths: []string{"."}}, Resources: config.Resources{ JobRuns: map[string]*resources.JobRun{ "my_run": { @@ -135,6 +139,38 @@ func TestResolveJobRunFileTriggers(t *testing.T) { assert.Empty(t, b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers) }) + t.Run("skips gitignored files", func(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, ".gitignore"), []byte("skip.txt\n"), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "keep.txt"), []byte("keep"), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "skip.txt"), []byte("skip"), 0o644)) + + b := bundleWithFileTrigger(dir, "*.txt") + diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) + require.False(t, diags.HasError()) + + hashes := b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers + require.Len(t, hashes, 1) + assert.Equal(t, contentHash("keep"), hashes["keep.txt"]) + assert.NotContains(t, hashes, "skip.txt") + }) + + t.Run("skips sync.exclude files", func(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "keep.txt"), []byte("keep"), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "skip.txt"), []byte("skip"), 0o644)) + + b := bundleWithFileTrigger(dir, "*.txt") + b.Config.Sync.Exclude = []string{"skip.txt"} + diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) + require.False(t, diags.HasError()) + + hashes := b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers + require.Len(t, hashes, 1) + assert.Equal(t, contentHash("keep"), hashes["keep.txt"]) + assert.NotContains(t, hashes, "skip.txt") + }) + t.Run("trims pattern whitespace", func(t *testing.T) { dir := t.TempDir() require.NoError(t, os.WriteFile(filepath.Join(dir, "seed.txt"), []byte("v1"), 0o644)) @@ -148,9 +184,13 @@ func TestResolveJobRunFileTriggers(t *testing.T) { } func bundleWithFileTrigger(syncRoot, pattern string) *bundle.Bundle { + root := vfs.MustNew(syncRoot) return &bundle.Bundle{ SyncRootPath: syncRoot, + SyncRoot: root, + WorktreeRoot: root, Config: config.Root{ + Sync: config.Sync{Paths: []string{"."}}, Resources: config.Resources{ JobRuns: map[string]*resources.JobRun{ "my_run": { From 3e19c167b512f040a7ef87d1e5f379fa02d83d1b Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Wed, 19 Aug 2026 20:10:03 +0000 Subject: [PATCH 10/26] acc: force-add gitignored on_file_change fixture The glob test edits migrations/ignored.txt, but the test .gitignore excluded it from the commit so CI clones would miss the file. --- .../job_runs/on_file_change_glob/migrations/ignored.txt | 1 + 1 file changed, 1 insertion(+) create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/ignored.txt diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/ignored.txt b/acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/ignored.txt new file mode 100644 index 00000000000..ea10ec85c10 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/ignored.txt @@ -0,0 +1 @@ +ignored From 7e9e1a03e44ccb5da2ad6da4ea40ec70fcec617c Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Thu, 20 Aug 2026 08:11:43 +0000 Subject: [PATCH 11/26] acc: drop READPLAN from the on_file_change glob test Plan serialization is already covered by on_file_change; the glob cases only assert the hash map. --- .../resources/job_runs/on_file_change_glob/out.test.toml | 1 - .../bundle/resources/job_runs/on_file_change_glob/output.txt | 2 +- .../bundle/resources/job_runs/on_file_change_glob/script | 5 +---- .../bundle/resources/job_runs/on_file_change_glob/test.toml | 4 ---- 4 files changed, 2 insertions(+), 10 deletions(-) diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/out.test.toml b/acceptance/bundle/resources/job_runs/on_file_change_glob/out.test.toml index 57b0f616850..0938e678987 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/out.test.toml +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/out.test.toml @@ -1,3 +1,2 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt index d424482b961..bd7d08557f5 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt @@ -61,7 +61,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged } } -=== bundle deploy +>>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-glob/default/files... Output from job_runs.my_run: id=[MY_RUN_ID_2]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_2]?o=[NUMID] Output from job_runs.my_run: id=[MY_RUN_ID_2]: SUCCESS diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/script b/acceptance/bundle/resources/job_runs/on_file_change_glob/script index 983cb9ac401..23390d23a55 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/script +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/script @@ -4,7 +4,6 @@ cleanup() { } trap cleanup EXIT -# Plan once, print JSON changes, leave tmp.plan.json for readplanarg. changes() { $CLI bundle plan -o json > tmp.plan.json trace jq '.plan["resources.job_runs.my_run"].changes' tmp.plan.json @@ -19,9 +18,7 @@ trace print_requests.py //jobs/run-now title "editing one matched file re-fires" update_file.py migrations/001.txt "one" "one-edited" changes -# The deploy is not traced: readplanarg makes the command line differ per variant. -title "bundle deploy\n" -$CLI bundle deploy $(readplanarg tmp.plan.json) +trace $CLI bundle deploy read_id.py my_run > /dev/null trace print_requests.py --keep //jobs/runs/delete trace print_requests.py //jobs/run-now diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml b/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml index 3516572564d..d372fd54eae 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml @@ -1,7 +1,3 @@ -# Deploy both by re-planning and from a plan saved on disk, so the hashes -# computed during planning survive plan serialization. -EnvMatrix.READPLAN = ["", "1"] - Ignore = ["tmp.plan.json", "tmp.002.txt"] # See on_file_change/test.toml: mask content hashes before the parent's From 6bdcffc815c2763a43b5f69ac5a2c7435898eb40 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Thu, 20 Aug 2026 12:34:44 +0000 Subject: [PATCH 12/26] acc: restore READPLAN on the on_file_change glob test Cover deploying glob hashes from a saved plan, not only from a live re-plan. --- .../resources/job_runs/on_file_change_glob/out.test.toml | 1 + .../bundle/resources/job_runs/on_file_change_glob/output.txt | 2 +- .../bundle/resources/job_runs/on_file_change_glob/script | 5 ++++- .../bundle/resources/job_runs/on_file_change_glob/test.toml | 4 ++++ 4 files changed, 10 insertions(+), 2 deletions(-) diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/out.test.toml b/acceptance/bundle/resources/job_runs/on_file_change_glob/out.test.toml index 0938e678987..57b0f616850 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/out.test.toml +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/out.test.toml @@ -1,2 +1,3 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt index bd7d08557f5..d424482b961 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt @@ -61,7 +61,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged } } ->>> [CLI] bundle deploy +=== bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-glob/default/files... Output from job_runs.my_run: id=[MY_RUN_ID_2]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_2]?o=[NUMID] Output from job_runs.my_run: id=[MY_RUN_ID_2]: SUCCESS diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/script b/acceptance/bundle/resources/job_runs/on_file_change_glob/script index 23390d23a55..983cb9ac401 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/script +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/script @@ -4,6 +4,7 @@ cleanup() { } trap cleanup EXIT +# Plan once, print JSON changes, leave tmp.plan.json for readplanarg. changes() { $CLI bundle plan -o json > tmp.plan.json trace jq '.plan["resources.job_runs.my_run"].changes' tmp.plan.json @@ -18,7 +19,9 @@ trace print_requests.py //jobs/run-now title "editing one matched file re-fires" update_file.py migrations/001.txt "one" "one-edited" changes -trace $CLI bundle deploy +# The deploy is not traced: readplanarg makes the command line differ per variant. +title "bundle deploy\n" +$CLI bundle deploy $(readplanarg tmp.plan.json) read_id.py my_run > /dev/null trace print_requests.py --keep //jobs/runs/delete trace print_requests.py //jobs/run-now diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml b/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml index d372fd54eae..3516572564d 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml @@ -1,3 +1,7 @@ +# Deploy both by re-planning and from a plan saved on disk, so the hashes +# computed during planning survive plan serialization. +EnvMatrix.READPLAN = ["", "1"] + Ignore = ["tmp.plan.json", "tmp.002.txt"] # See on_file_change/test.toml: mask content hashes before the parent's From 5ac0e4b9380134d3c79b51e11c1b08fbf69db964 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Thu, 20 Aug 2026 14:08:42 +0000 Subject: [PATCH 13/26] job_runs: address on_file_change review comments Keep the gitignore fixture as dot_gitignore so it does not affect the repo while developing, drop unit tests that duplicate acceptance coverage, and allocate trigger state once when a trigger is armed. --- .../{.gitignore => dot_gitignore} | 0 .../job_runs/on_file_change_glob/script | 3 + .../job_runs/on_file_change_glob/test.toml | 2 +- .../resolve_job_run_file_triggers_test.go | 139 +----------------- bundle/direct/dresources/job_run.go | 23 +-- 5 files changed, 11 insertions(+), 156 deletions(-) rename acceptance/bundle/resources/job_runs/on_file_change_glob/{.gitignore => dot_gitignore} (100%) diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/.gitignore b/acceptance/bundle/resources/job_runs/on_file_change_glob/dot_gitignore similarity index 100% rename from acceptance/bundle/resources/job_runs/on_file_change_glob/.gitignore rename to acceptance/bundle/resources/job_runs/on_file_change_glob/dot_gitignore diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/script b/acceptance/bundle/resources/job_runs/on_file_change_glob/script index 983cb9ac401..dd737f242b5 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/script +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/script @@ -1,3 +1,6 @@ +# Fixture is named dot_gitignore so a real .gitignore does not affect git while developing. +mv dot_gitignore .gitignore + cleanup() { trace $CLI bundle destroy --auto-approve rm -f out.requests.txt diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml b/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml index 3516572564d..98c98631e5b 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml @@ -2,7 +2,7 @@ # computed during planning survive plan serialization. EnvMatrix.READPLAN = ["", "1"] -Ignore = ["tmp.plan.json", "tmp.002.txt"] +Ignore = ["tmp.plan.json", "tmp.002.txt", ".gitignore"] # See on_file_change/test.toml: mask content hashes before the parent's # `\d{8,}` rule can mangle them, keeping distinct hashes distinguishable. diff --git a/bundle/config/mutator/resolve_job_run_file_triggers_test.go b/bundle/config/mutator/resolve_job_run_file_triggers_test.go index 344fed25657..1cc54eecf4f 100644 --- a/bundle/config/mutator/resolve_job_run_file_triggers_test.go +++ b/bundle/config/mutator/resolve_job_run_file_triggers_test.go @@ -11,14 +11,13 @@ import ( "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/mutator" "github.com/databricks/cli/bundle/config/resources" - "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/vfs" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestResolveJobRunFileTriggers(t *testing.T) { - t.Run("matches files and fills hashes", func(t *testing.T) { + t.Run("hashes file contents with sha256", func(t *testing.T) { dir := t.TempDir() require.NoError(t, os.WriteFile(filepath.Join(dir, "a.txt"), []byte("hello"), 0o644)) require.NoError(t, os.WriteFile(filepath.Join(dir, "b.txt"), []byte("world"), 0o644)) @@ -35,142 +34,6 @@ func TestResolveJobRunFileTriggers(t *testing.T) { assert.Equal(t, contentHash("world"), hashes["b.txt"]) }) - t.Run("no matches warns and stores empty hash", func(t *testing.T) { - dir := t.TempDir() - pattern := "missing.txt" - b := bundleWithFileTrigger(dir, pattern) - - diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) - require.False(t, diags.HasError()) - require.Len(t, diags, 1) - assert.Equal(t, diag.Warning, diags[0].Severity) - assert.Contains(t, diags[0].Summary, `no files match "missing.txt"`) - - hashes := b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers - require.Len(t, hashes, 1) - assert.Empty(t, hashes["missing.txt"]) - }) - - t.Run("no file triggers is a no-op", func(t *testing.T) { - dir := t.TempDir() - on := true - b := &bundle.Bundle{ - SyncRootPath: dir, - Config: config.Root{ - Resources: config.Resources{ - JobRuns: map[string]*resources.JobRun{ - "my_run": { - Lifecycle: &resources.JobRunLifecycle{ - Triggers: []resources.JobRunTrigger{ - {OnBundleDeploy: &on}, - }, - }, - }, - }, - }, - }, - } - - diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) - assert.Empty(t, diags) - assert.Nil(t, b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers) - }) - - t.Run("multiple patterns merge into one map", func(t *testing.T) { - dir := t.TempDir() - require.NoError(t, os.WriteFile(filepath.Join(dir, "a.txt"), []byte("aaa"), 0o644)) - require.NoError(t, os.MkdirAll(filepath.Join(dir, "subdir"), 0o755)) - require.NoError(t, os.WriteFile(filepath.Join(dir, "subdir", "x.py"), []byte("bbb"), 0o644)) - - patA := "a.txt" - patB := "subdir/*.py" - b := &bundle.Bundle{ - SyncRootPath: dir, - SyncRoot: vfs.MustNew(dir), - WorktreeRoot: vfs.MustNew(dir), - Config: config.Root{ - Sync: config.Sync{Paths: []string{"."}}, - Resources: config.Resources{ - JobRuns: map[string]*resources.JobRun{ - "my_run": { - Lifecycle: &resources.JobRunLifecycle{ - Triggers: []resources.JobRunTrigger{ - {OnFileChange: &patA}, - {OnFileChange: &patB}, - }, - }, - }, - }, - }, - }, - } - - diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) - require.False(t, diags.HasError()) - - hashes := b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers - require.Len(t, hashes, 2) - assert.Equal(t, contentHash("aaa"), hashes["a.txt"]) - assert.Equal(t, contentHash("bbb"), hashes["subdir/x.py"]) - }) - - t.Run("pattern outside sync root is an error", func(t *testing.T) { - dir := t.TempDir() - pattern := "../outside.txt" - b := bundleWithFileTrigger(dir, pattern) - - diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) - require.True(t, diags.HasError()) - require.Len(t, diags, 1) - assert.Contains(t, diags[0].Summary, `not under the sync root`) - assert.Empty(t, b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers) - }) - - t.Run("directory-only match is an error", func(t *testing.T) { - dir := t.TempDir() - require.NoError(t, os.MkdirAll(filepath.Join(dir, "migrations"), 0o755)) - pattern := "migrations" - b := bundleWithFileTrigger(dir, pattern) - - diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) - require.True(t, diags.HasError()) - require.Len(t, diags, 1) - assert.Contains(t, diags[0].Summary, `matches no regular files`) - assert.Empty(t, b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers) - }) - - t.Run("skips gitignored files", func(t *testing.T) { - dir := t.TempDir() - require.NoError(t, os.WriteFile(filepath.Join(dir, ".gitignore"), []byte("skip.txt\n"), 0o644)) - require.NoError(t, os.WriteFile(filepath.Join(dir, "keep.txt"), []byte("keep"), 0o644)) - require.NoError(t, os.WriteFile(filepath.Join(dir, "skip.txt"), []byte("skip"), 0o644)) - - b := bundleWithFileTrigger(dir, "*.txt") - diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) - require.False(t, diags.HasError()) - - hashes := b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers - require.Len(t, hashes, 1) - assert.Equal(t, contentHash("keep"), hashes["keep.txt"]) - assert.NotContains(t, hashes, "skip.txt") - }) - - t.Run("skips sync.exclude files", func(t *testing.T) { - dir := t.TempDir() - require.NoError(t, os.WriteFile(filepath.Join(dir, "keep.txt"), []byte("keep"), 0o644)) - require.NoError(t, os.WriteFile(filepath.Join(dir, "skip.txt"), []byte("skip"), 0o644)) - - b := bundleWithFileTrigger(dir, "*.txt") - b.Config.Sync.Exclude = []string{"skip.txt"} - diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) - require.False(t, diags.HasError()) - - hashes := b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers - require.Len(t, hashes, 1) - assert.Equal(t, contentHash("keep"), hashes["keep.txt"]) - assert.NotContains(t, hashes, "skip.txt") - }) - t.Run("trims pattern whitespace", func(t *testing.T) { dir := t.TempDir() require.NoError(t, os.WriteFile(filepath.Join(dir, "seed.txt"), []byte("v1"), 0o644)) diff --git a/bundle/direct/dresources/job_run.go b/bundle/direct/dresources/job_run.go index 5b97cc9e22b..8117af9aa22 100644 --- a/bundle/direct/dresources/job_run.go +++ b/bundle/direct/dresources/job_run.go @@ -109,25 +109,14 @@ func (*ResourceJobRun) PrepareState(input *resources.JobRun) *JobRunState { ResultState: jobs.RunResultStateSuccess, Lifecycle: nil, } - var triggers *JobRunTriggersState - if input.HasOnBundleDeploy() { - triggers = &JobRunTriggersState{ - OnBundleDeploy: uuid.NewString(), - OnFileChange: nil, - } + if !input.HasOnBundleDeploy() && len(input.ResolvedFileTriggers) == 0 { + return state } - if len(input.ResolvedFileTriggers) > 0 { - if triggers == nil { - triggers = &JobRunTriggersState{ - OnBundleDeploy: "", - OnFileChange: nil, - } - } - triggers.OnFileChange = input.ResolvedFileTriggers - } - if triggers != nil { - state.Lifecycle = &JobRunLifecycleState{Triggers: triggers} + triggers := &JobRunTriggersState{OnFileChange: input.ResolvedFileTriggers} + if input.HasOnBundleDeploy() { + triggers.OnBundleDeploy = uuid.NewString() } + state.Lifecycle = &JobRunLifecycleState{Triggers: triggers} return state } From 0a62f66fb78781b9e1d38206a19da0e799e2fac3 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Thu, 20 Aug 2026 14:16:35 +0000 Subject: [PATCH 14/26] job_runs: list every trigger field to satisfy exhaustruct The previous simplification omitted OnBundleDeploy from the struct literal, which the linter rejects. --- bundle/direct/dresources/job_run.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/bundle/direct/dresources/job_run.go b/bundle/direct/dresources/job_run.go index 8117af9aa22..ac65e5db697 100644 --- a/bundle/direct/dresources/job_run.go +++ b/bundle/direct/dresources/job_run.go @@ -112,11 +112,16 @@ func (*ResourceJobRun) PrepareState(input *resources.JobRun) *JobRunState { if !input.HasOnBundleDeploy() && len(input.ResolvedFileTriggers) == 0 { return state } - triggers := &JobRunTriggersState{OnFileChange: input.ResolvedFileTriggers} + onBundleDeploy := "" if input.HasOnBundleDeploy() { - triggers.OnBundleDeploy = uuid.NewString() + onBundleDeploy = uuid.NewString() + } + state.Lifecycle = &JobRunLifecycleState{ + Triggers: &JobRunTriggersState{ + OnBundleDeploy: onBundleDeploy, + OnFileChange: input.ResolvedFileTriggers, + }, } - state.Lifecycle = &JobRunLifecycleState{Triggers: triggers} return state } From 5c3bed337e4d37e1f4bf4f050b6041b434f9cb4e Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Fri, 21 Aug 2026 12:31:38 +0000 Subject: [PATCH 15/26] job_runs: keep empty nested lifecycle on state and remote Always persist the same lifecycle.triggers shape so trigger fingerprints compare at the same paths. Wrap on_file_change hashes in files so dropping the trigger updates instead of recreating. --- acceptance/bundle/refschema/out.fields.txt | 11 +- .../resources/job_runs/failed_run/output.txt | 10 ++ .../job_runs/interrupted_run/output.txt | 5 + .../job_runs/on_bundle_deploy/output.txt | 22 +-- .../job_runs/on_file_change/output.txt | 32 +--- .../job_runs/on_file_change_glob/output.txt | 144 ++++++------------ .../resources/job_runs/redeploy/output.txt | 10 ++ bundle/direct/dresources/job_run.go | 86 +++++++---- bundle/direct/dresources/job_run_test.go | 16 +- bundle/direct/dresources/resources.yml | 3 + bundle/direct/dresources/type_test.go | 4 - 11 files changed, 167 insertions(+), 176 deletions(-) diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index 82933993eee..7302df55893 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -905,15 +905,16 @@ resources.job_runs.*.jar_params[*] string ALL resources.job_runs.*.job_id int64 ALL resources.job_runs.*.job_parameters map[string]string ALL resources.job_runs.*.job_parameters.* string ALL -resources.job_runs.*.lifecycle *dresources.JobRunLifecycleState STATE +resources.job_runs.*.lifecycle *dresources.JobRunLifecycleState REMOTE STATE resources.job_runs.*.lifecycle *resources.JobRunLifecycle INPUT resources.job_runs.*.lifecycle resources.Lifecycle INPUT resources.job_runs.*.lifecycle.prevent_destroy bool INPUT -resources.job_runs.*.lifecycle.triggers *dresources.JobRunTriggersState STATE +resources.job_runs.*.lifecycle.triggers *dresources.JobRunTriggersState REMOTE STATE resources.job_runs.*.lifecycle.triggers []resources.JobRunTrigger INPUT -resources.job_runs.*.lifecycle.triggers.on_bundle_deploy string STATE -resources.job_runs.*.lifecycle.triggers.on_file_change map[string]string STATE -resources.job_runs.*.lifecycle.triggers.on_file_change.* string STATE +resources.job_runs.*.lifecycle.triggers.on_bundle_deploy string REMOTE STATE +resources.job_runs.*.lifecycle.triggers.on_file_change *dresources.JobRunFileTriggerState REMOTE STATE +resources.job_runs.*.lifecycle.triggers.on_file_change.files map[string]string REMOTE STATE +resources.job_runs.*.lifecycle.triggers.on_file_change.files.* string REMOTE STATE resources.job_runs.*.lifecycle.triggers[*] resources.JobRunTrigger INPUT resources.job_runs.*.lifecycle.triggers[*].on_bundle_deploy *bool INPUT resources.job_runs.*.lifecycle.triggers[*].on_file_change *string INPUT diff --git a/acceptance/bundle/resources/job_runs/failed_run/output.txt b/acceptance/bundle/resources/job_runs/failed_run/output.txt index f8b8dbf4398..0c6bddb0d9b 100644 --- a/acceptance/bundle/resources/job_runs/failed_run/output.txt +++ b/acceptance/bundle/resources/job_runs/failed_run/output.txt @@ -27,11 +27,21 @@ Files: 7 uploaded, 0 deleted "new_state": { "value": { "job_id": [MY_JOB_ID], + "lifecycle": { + "triggers": { + "on_file_change": {} + } + }, "result_state": "SUCCESS" } }, "remote_state": { "job_id": [MY_JOB_ID], + "lifecycle": { + "triggers": { + "on_file_change": {} + } + }, "result_state": "FAILED", "run_id": [MY_RUN_ID], "run_name": "test-job-[UNIQUE_NAME]", diff --git a/acceptance/bundle/resources/job_runs/interrupted_run/output.txt b/acceptance/bundle/resources/job_runs/interrupted_run/output.txt index 2e1d9cacd7a..da185c3704d 100644 --- a/acceptance/bundle/resources/job_runs/interrupted_run/output.txt +++ b/acceptance/bundle/resources/job_runs/interrupted_run/output.txt @@ -42,6 +42,11 @@ Exit code: 1 "action": "skip", "remote_state": { "job_id": [MY_JOB_ID], + "lifecycle": { + "triggers": { + "on_file_change": {} + } + }, "run_id": [MY_RUN_ID], "run_name": "my-job", "run_page_url": "[DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?o=[NUMID]", diff --git a/acceptance/bundle/resources/job_runs/on_bundle_deploy/output.txt b/acceptance/bundle/resources/job_runs/on_bundle_deploy/output.txt index 39123f3370f..6cd8d44a418 100644 --- a/acceptance/bundle/resources/job_runs/on_bundle_deploy/output.txt +++ b/acceptance/bundle/resources/job_runs/on_bundle_deploy/output.txt @@ -30,20 +30,6 @@ Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle": { - "action": "recreate", - "reason": "immutable", - "old": { - "triggers": { - "on_bundle_deploy": "[UUID]" - } - }, - "new": { - "triggers": { - "on_bundle_deploy": "[UUID]" - } - } - }, "lifecycle.triggers.on_bundle_deploy": { "action": "recreate", "reason": "immutable", @@ -108,14 +94,10 @@ Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle": { + "lifecycle.triggers.on_bundle_deploy": { "action": "skip", "reason": "trigger removed", - "old": { - "triggers": { - "on_bundle_deploy": "[UUID]" - } - } + "old": "[UUID]" }, "result_state": { "action": "skip", diff --git a/acceptance/bundle/resources/job_runs/on_file_change/output.txt b/acceptance/bundle/resources/job_runs/on_file_change/output.txt index d03307b90f2..9b662e1936d 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change/output.txt @@ -52,25 +52,17 @@ Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle": { + "lifecycle.triggers.on_file_change.files": { "action": "recreate", "reason": "immutable", "old": { - "triggers": { - "on_file_change": { - "seed.txt": "[FILE_HASH][0]" - } - } + "seed.txt": "[FILE_HASH][0]" }, "new": { - "triggers": { - "on_file_change": { - "seed.txt": "[FILE_HASH][1]" - } - } + "seed.txt": "[FILE_HASH][1]" } }, - "lifecycle.triggers.on_file_change['seed.txt']": { + "lifecycle.triggers.on_file_change.files['seed.txt']": { "action": "recreate", "reason": "immutable", "old": "[FILE_HASH][0]", @@ -125,25 +117,17 @@ Warning: lifecycle.triggers.on_file_change: no files match "seed.txt" in databricks.yml:18:29 { - "lifecycle": { + "lifecycle.triggers.on_file_change.files": { "action": "recreate", "reason": "immutable", "old": { - "triggers": { - "on_file_change": { - "seed.txt": "[FILE_HASH][1]" - } - } + "seed.txt": "[FILE_HASH][1]" }, "new": { - "triggers": { - "on_file_change": { - "seed.txt": "" - } - } + "seed.txt": "" } }, - "lifecycle.triggers.on_file_change['seed.txt']": { + "lifecycle.triggers.on_file_change.files['seed.txt']": { "action": "recreate", "reason": "immutable", "old": "[FILE_HASH][1]", diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt index d424482b961..5b4ef23178c 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt @@ -25,29 +25,21 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged === editing one matched file re-fires >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle": { + "lifecycle.triggers.on_file_change.files": { "action": "recreate", "reason": "immutable", "old": { - "triggers": { - "on_file_change": { - "migrations/001.txt": "[FILE_HASH][0]", - "migrations/002.txt": "[FILE_HASH][1]", - "mysubdir/a.txt": "[FILE_HASH][2]" - } - } + "migrations/001.txt": "[FILE_HASH][0]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][2]" }, "new": { - "triggers": { - "on_file_change": { - "migrations/001.txt": "[FILE_HASH][3]", - "migrations/002.txt": "[FILE_HASH][1]", - "mysubdir/a.txt": "[FILE_HASH][2]" - } - } + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][2]" } }, - "lifecycle.triggers.on_file_change['migrations/001.txt']": { + "lifecycle.triggers.on_file_change.files['migrations/001.txt']": { "action": "recreate", "reason": "immutable", "old": "[FILE_HASH][0]", @@ -91,29 +83,21 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged === editing a file in a subdirectory re-fires >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle": { + "lifecycle.triggers.on_file_change.files": { "action": "recreate", "reason": "immutable", "old": { - "triggers": { - "on_file_change": { - "migrations/001.txt": "[FILE_HASH][3]", - "migrations/002.txt": "[FILE_HASH][1]", - "mysubdir/a.txt": "[FILE_HASH][2]" - } - } + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][2]" }, "new": { - "triggers": { - "on_file_change": { - "migrations/001.txt": "[FILE_HASH][3]", - "migrations/002.txt": "[FILE_HASH][1]", - "mysubdir/a.txt": "[FILE_HASH][4]" - } - } + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][4]" } }, - "lifecycle.triggers.on_file_change['mysubdir/a.txt']": { + "lifecycle.triggers.on_file_change.files['mysubdir/a.txt']": { "action": "recreate", "reason": "immutable", "old": "[FILE_HASH][2]", @@ -130,30 +114,22 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged === a new matching file re-fires >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle": { + "lifecycle.triggers.on_file_change.files": { "action": "recreate", "reason": "immutable", "old": { - "triggers": { - "on_file_change": { - "migrations/001.txt": "[FILE_HASH][3]", - "migrations/002.txt": "[FILE_HASH][1]", - "mysubdir/a.txt": "[FILE_HASH][2]" - } - } + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][2]" }, "new": { - "triggers": { - "on_file_change": { - "migrations/001.txt": "[FILE_HASH][3]", - "migrations/002.txt": "[FILE_HASH][1]", - "migrations/003.txt": "[FILE_HASH][5]", - "mysubdir/a.txt": "[FILE_HASH][2]" - } - } + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "migrations/003.txt": "[FILE_HASH][5]", + "mysubdir/a.txt": "[FILE_HASH][2]" } }, - "lifecycle.triggers.on_file_change['migrations/003.txt']": { + "lifecycle.triggers.on_file_change.files['migrations/003.txt']": { "action": "recreate", "reason": "immutable", "new": "[FILE_HASH][5]" @@ -169,28 +145,20 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged === removing a matched file re-fires >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle": { + "lifecycle.triggers.on_file_change.files": { "action": "recreate", "reason": "immutable", "old": { - "triggers": { - "on_file_change": { - "migrations/001.txt": "[FILE_HASH][3]", - "migrations/002.txt": "[FILE_HASH][1]", - "mysubdir/a.txt": "[FILE_HASH][2]" - } - } + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][2]" }, "new": { - "triggers": { - "on_file_change": { - "migrations/001.txt": "[FILE_HASH][3]", - "mysubdir/a.txt": "[FILE_HASH][2]" - } - } + "migrations/001.txt": "[FILE_HASH][3]", + "mysubdir/a.txt": "[FILE_HASH][2]" } }, - "lifecycle.triggers.on_file_change['migrations/002.txt']": { + "lifecycle.triggers.on_file_change.files['migrations/002.txt']": { "action": "recreate", "reason": "immutable", "old": "[FILE_HASH][1]" @@ -206,26 +174,18 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged === editing a file below the glob does not re-fire >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle": { + "lifecycle.triggers.on_file_change.files": { "action": "skip", - "reason": "missing_in_remote", + "reason": "not_returned_by_api", "old": { - "triggers": { - "on_file_change": { - "migrations/001.txt": "[FILE_HASH][3]", - "migrations/002.txt": "[FILE_HASH][1]", - "mysubdir/a.txt": "[FILE_HASH][2]" - } - } + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][2]" }, "new": { - "triggers": { - "on_file_change": { - "migrations/001.txt": "[FILE_HASH][3]", - "migrations/002.txt": "[FILE_HASH][1]", - "mysubdir/a.txt": "[FILE_HASH][2]" - } - } + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][2]" } }, "result_state": { @@ -243,26 +203,18 @@ Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged === ** is not recursive: it matches the same files as * >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle": { + "lifecycle.triggers.on_file_change.files": { "action": "skip", - "reason": "missing_in_remote", + "reason": "not_returned_by_api", "old": { - "triggers": { - "on_file_change": { - "migrations/001.txt": "[FILE_HASH][3]", - "migrations/002.txt": "[FILE_HASH][1]", - "mysubdir/a.txt": "[FILE_HASH][2]" - } - } + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][2]" }, "new": { - "triggers": { - "on_file_change": { - "migrations/001.txt": "[FILE_HASH][3]", - "migrations/002.txt": "[FILE_HASH][1]", - "mysubdir/a.txt": "[FILE_HASH][2]" - } - } + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][2]" } }, "result_state": { diff --git a/acceptance/bundle/resources/job_runs/redeploy/output.txt b/acceptance/bundle/resources/job_runs/redeploy/output.txt index 06129f79373..60111c98a93 100644 --- a/acceptance/bundle/resources/job_runs/redeploy/output.txt +++ b/acceptance/bundle/resources/job_runs/redeploy/output.txt @@ -59,6 +59,11 @@ Resources: "job_parameters": { "env": "prod" }, + "lifecycle": { + "triggers": { + "on_file_change": {} + } + }, "result_state": "SUCCESS" } }, @@ -67,6 +72,11 @@ Resources: "job_parameters": { "env": "dev" }, + "lifecycle": { + "triggers": { + "on_file_change": {} + } + }, "result_state": "SUCCESS", "run_id": [MY_RUN_ID], "run_name": "my-job", diff --git a/bundle/direct/dresources/job_run.go b/bundle/direct/dresources/job_run.go index ac65e5db697..815091a721a 100644 --- a/bundle/direct/dresources/job_run.go +++ b/bundle/direct/dresources/job_run.go @@ -3,6 +3,7 @@ package dresources import ( "cmp" "context" + "encoding/json" "errors" "fmt" "slices" @@ -28,10 +29,12 @@ const jobRunTimeout = 24 * time.Hour // jobRunTriggerLocalPaths are local-only fingerprints: clearing one is skip, not recreate. var jobRunTriggerLocalPaths = []string{ - "lifecycle", - "lifecycle.triggers", "lifecycle.triggers.on_bundle_deploy", - "lifecycle.triggers.on_file_change", + "lifecycle.triggers.on_file_change.files", +} + +func isJobRunTriggerPath(path string) bool { + return slices.Contains(jobRunTriggerLocalPaths, path) } // JobRunTriggersState is the persisted fingerprint of lifecycle.triggers. @@ -39,12 +42,44 @@ type JobRunTriggersState struct { // Fresh UUID each plan while armed so Old!=New forces recreate. OnBundleDeploy string `json:"on_bundle_deploy,omitempty"` // Path → content hash from ResolveJobRunFileTriggers; change to recreate. - OnFileChange map[string]string `json:"on_file_change,omitempty"` + OnFileChange *JobRunFileTriggerState `json:"on_file_change"` +} + +// JobRunFileTriggerState is always present so remote and state share one shape. +// Files is nil when the trigger is off, so dropping the trigger diffs here and +// a file appear/disappear diffs inside the map. +type JobRunFileTriggerState struct { + Files map[string]string `json:"files,omitempty"` +} + +// UnmarshalJSON accepts the wrapped map and the older path-to-hash map. +func (s *JobRunFileTriggerState) UnmarshalJSON(b []byte) error { + var raw map[string]json.RawMessage + if err := json.Unmarshal(b, &raw); err != nil { + return err + } + if files, ok := raw["files"]; ok && len(files) > 0 && files[0] == '{' { + return json.Unmarshal(files, &s.Files) + } + if len(raw) == 0 { + s.Files = nil + return nil + } + return json.Unmarshal(b, &s.Files) } -// JobRunLifecycleState holds local-only lifecycle fields persisted in state. +// JobRunLifecycleState is the local-only trigger fingerprint, also present (empty) on remote. type JobRunLifecycleState struct { - Triggers *JobRunTriggersState `json:"triggers,omitempty"` + Triggers *JobRunTriggersState `json:"triggers"` +} + +func newJobRunLifecycleState() *JobRunLifecycleState { + return &JobRunLifecycleState{ + Triggers: &JobRunTriggersState{ + OnBundleDeploy: "", + OnFileChange: &JobRunFileTriggerState{Files: nil}, + }, + } } // JobRunState is the RunNow request plus the outcome required for planning. @@ -54,12 +89,13 @@ type JobRunState struct { // Always SUCCESS during planning and cleared before persistence. ResultState jobs.RunResultState `json:"result_state,omitempty"` - // Local-only; listed in knownMissingInRemoteType. Nested under lifecycle to - // mirror config and avoid colliding with a future Jobs API field. - Lifecycle *JobRunLifecycleState `json:"lifecycle,omitempty"` + // Local-only. Nested under lifecycle to mirror config and avoid colliding + // with a future Jobs API field. + Lifecycle *JobRunLifecycleState `json:"lifecycle"` } func (s *JobRunState) UnmarshalJSON(b []byte) error { + s.Lifecycle = newJobRunLifecycleState() return marshal.Unmarshal(b, s) } @@ -76,6 +112,9 @@ type JobRunRemote struct { // compare the two. See "RemapState is a dumb copy" in README.md. ResultState jobs.RunResultState `json:"result_state,omitempty"` + // Always the empty fingerprint: GetRun does not return triggers. + Lifecycle *JobRunLifecycleState `json:"lifecycle"` + RunId int64 `json:"run_id,omitempty"` RunName string `json:"run_name,omitempty"` State *jobs.RunState `json:"state,omitempty"` @@ -86,6 +125,7 @@ type JobRunRemote struct { // Custom marshaler needed because embedded RunNow's MarshalJSON would otherwise // take over and drop the additional fields. func (s *JobRunRemote) UnmarshalJSON(b []byte) error { + s.Lifecycle = newJobRunLifecycleState() return marshal.Unmarshal(b, s) } @@ -107,20 +147,13 @@ func (*ResourceJobRun) PrepareState(input *resources.JobRun) *JobRunState { state := &JobRunState{ RunNow: input.RunNow, ResultState: jobs.RunResultStateSuccess, - Lifecycle: nil, - } - if !input.HasOnBundleDeploy() && len(input.ResolvedFileTriggers) == 0 { - return state + Lifecycle: newJobRunLifecycleState(), } - onBundleDeploy := "" if input.HasOnBundleDeploy() { - onBundleDeploy = uuid.NewString() + state.Lifecycle.Triggers.OnBundleDeploy = uuid.NewString() } - state.Lifecycle = &JobRunLifecycleState{ - Triggers: &JobRunTriggersState{ - OnBundleDeploy: onBundleDeploy, - OnFileChange: input.ResolvedFileTriggers, - }, + if len(input.ResolvedFileTriggers) > 0 { + state.Lifecycle.Triggers.OnFileChange.Files = input.ResolvedFileTriggers } return state } @@ -161,6 +194,7 @@ func makeJobRunRemote(run *jobs.Run) *JobRunRemote { ForceSendFields: nil, }, ResultState: run.State.ResultState, + Lifecycle: newJobRunLifecycleState(), RunId: run.RunId, RunName: run.RunName, // Rebuilt, not copied: the SDK records explicitly-sent fields in @@ -201,8 +235,7 @@ func (*ResourceJobRun) RemapState(remote *JobRunRemote) *JobRunState { return &JobRunState{ RunNow: remote.RunNow, ResultState: remote.ResultState, - // Local-only lifecycle fingerprints stay unset on the remapped remote. - Lifecycle: nil, + Lifecycle: remote.Lifecycle, } } @@ -389,10 +422,11 @@ func reportRunLine(ctx context.Context, runID int64, msg string) { // result_state either, so the lifecycle state is what tells the two apart. // Clearing a trigger skips its local-only fingerprint without re-firing the run. func (*ResourceJobRun) OverrideChangeDesc(_ context.Context, path *structpath.PathNode, change *ChangeDesc, remote *JobRunRemote) error { - if slices.Contains(jobRunTriggerLocalPaths, path.String()) { - // A cleared trigger sets New empty; structdiff may report it at lifecycle, - // lifecycle.triggers, or the leaf. - if change.New == nil || change.New == "" { + pathString := path.String() + if isJobRunTriggerPath(pathString) { + removed := pathString == "lifecycle.triggers.on_bundle_deploy" && (change.New == nil || change.New == "") + removed = removed || pathString == "lifecycle.triggers.on_file_change.files" && change.New == nil + if removed { change.Action = deployplan.Skip change.Reason = "trigger removed" } diff --git a/bundle/direct/dresources/job_run_test.go b/bundle/direct/dresources/job_run_test.go index 0ffc1c48040..66f61646116 100644 --- a/bundle/direct/dresources/job_run_test.go +++ b/bundle/direct/dresources/job_run_test.go @@ -244,6 +244,18 @@ func TestJobRunWaitAbandonedLinksTheRun(t *testing.T) { require.ErrorContains(t, err, testRunPageLink) } +// State written before lifecycle existed has no such key, and must still load. +func TestJobRunStateUnmarshalWithoutLifecycle(t *testing.T) { + var state JobRunState + + require.NoError(t, json.Unmarshal([]byte(`{}`), &state)) + + require.NotNil(t, state.Lifecycle) + require.NotNil(t, state.Lifecycle.Triggers) + require.NotNil(t, state.Lifecycle.Triggers.OnFileChange) + assert.Nil(t, state.Lifecycle.Triggers.OnFileChange.Files) +} + // The planner diffs RemapState(remote) against PrepareState(config), so a run // that did not end in SUCCESS has to surface as a difference on result_state. func TestJobRunRemapStateCarriesTheOutcome(t *testing.T) { @@ -254,11 +266,13 @@ func TestJobRunRemapStateCarriesTheOutcome(t *testing.T) { "", } { t.Run(string(outcome), func(t *testing.T) { - remote := &JobRunRemote{RunId: 123, ResultState: outcome} + lifecycle := newJobRunLifecycleState() + remote := &JobRunRemote{RunId: 123, ResultState: outcome, Lifecycle: lifecycle} state := (&ResourceJobRun{}).RemapState(remote) assert.Equal(t, outcome, state.ResultState) + assert.Same(t, lifecycle, state.Lifecycle) }) } } diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index 472a4a0def8..99cf7a2ba8b 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -175,6 +175,9 @@ resources: reason: not_returned_by_api - field: queue reason: not_returned_by_api + # Local-only trigger fingerprints; GetRun does not return them. + - field: lifecycle + reason: not_returned_by_api # A run is immutable and fire-once, so any change recreates it. Omitting # `field` matches every field (root; see TestFieldRuleOmittedIsRoot). # `field: ""` would instead match nothing. The one exception is a run that is diff --git a/bundle/direct/dresources/type_test.go b/bundle/direct/dresources/type_test.go index 2d5516d59c7..ca84850c33d 100644 --- a/bundle/direct/dresources/type_test.go +++ b/bundle/direct/dresources/type_test.go @@ -56,10 +56,6 @@ var knownMissingInRemoteType = map[string][]string{ "vector_search_endpoints": { "usage_policy_id", }, - "job_runs": { - // Local-only trigger fingerprints under lifecycle. - "lifecycle", - }, } // commonMissingInStateType lists fields that are commonly missing across all resource types. From 31e72863eb77a97d933630e0098e6af34915ffe6 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Fri, 21 Aug 2026 13:02:42 +0000 Subject: [PATCH 16/26] job_runs: keep the lifecycle fingerprint out of the remote type Nest lifecycle by value so structdiff descends to the leaf that changed instead of reporting the whole subtree, and let RemapState supply the empty shape. GetRun never returns the fingerprints, so the remote type no longer advertises a lifecycle it cannot fill. --- acceptance/bundle/refschema/out.fields.txt | 12 ++--- .../resources/job_runs/failed_run/output.txt | 5 --- .../job_runs/interrupted_run/output.txt | 5 --- .../job_runs/on_file_change_glob/output.txt | 4 +- .../resources/job_runs/redeploy/output.txt | 5 --- bundle/direct/dresources/job_run.go | 45 +++++++++---------- bundle/direct/dresources/job_run_test.go | 8 +--- bundle/direct/dresources/resources.yml | 3 -- bundle/direct/dresources/type_test.go | 5 +++ 9 files changed, 37 insertions(+), 55 deletions(-) diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index 7302df55893..d1af0279654 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -905,16 +905,16 @@ resources.job_runs.*.jar_params[*] string ALL resources.job_runs.*.job_id int64 ALL resources.job_runs.*.job_parameters map[string]string ALL resources.job_runs.*.job_parameters.* string ALL -resources.job_runs.*.lifecycle *dresources.JobRunLifecycleState REMOTE STATE resources.job_runs.*.lifecycle *resources.JobRunLifecycle INPUT +resources.job_runs.*.lifecycle dresources.JobRunLifecycleState STATE resources.job_runs.*.lifecycle resources.Lifecycle INPUT resources.job_runs.*.lifecycle.prevent_destroy bool INPUT -resources.job_runs.*.lifecycle.triggers *dresources.JobRunTriggersState REMOTE STATE resources.job_runs.*.lifecycle.triggers []resources.JobRunTrigger INPUT -resources.job_runs.*.lifecycle.triggers.on_bundle_deploy string REMOTE STATE -resources.job_runs.*.lifecycle.triggers.on_file_change *dresources.JobRunFileTriggerState REMOTE STATE -resources.job_runs.*.lifecycle.triggers.on_file_change.files map[string]string REMOTE STATE -resources.job_runs.*.lifecycle.triggers.on_file_change.files.* string REMOTE STATE +resources.job_runs.*.lifecycle.triggers dresources.JobRunTriggersState STATE +resources.job_runs.*.lifecycle.triggers.on_bundle_deploy string STATE +resources.job_runs.*.lifecycle.triggers.on_file_change dresources.JobRunFileTriggerState STATE +resources.job_runs.*.lifecycle.triggers.on_file_change.files map[string]string STATE +resources.job_runs.*.lifecycle.triggers.on_file_change.files.* string STATE resources.job_runs.*.lifecycle.triggers[*] resources.JobRunTrigger INPUT resources.job_runs.*.lifecycle.triggers[*].on_bundle_deploy *bool INPUT resources.job_runs.*.lifecycle.triggers[*].on_file_change *string INPUT diff --git a/acceptance/bundle/resources/job_runs/failed_run/output.txt b/acceptance/bundle/resources/job_runs/failed_run/output.txt index 0c6bddb0d9b..37f1a831681 100644 --- a/acceptance/bundle/resources/job_runs/failed_run/output.txt +++ b/acceptance/bundle/resources/job_runs/failed_run/output.txt @@ -37,11 +37,6 @@ Files: 7 uploaded, 0 deleted }, "remote_state": { "job_id": [MY_JOB_ID], - "lifecycle": { - "triggers": { - "on_file_change": {} - } - }, "result_state": "FAILED", "run_id": [MY_RUN_ID], "run_name": "test-job-[UNIQUE_NAME]", diff --git a/acceptance/bundle/resources/job_runs/interrupted_run/output.txt b/acceptance/bundle/resources/job_runs/interrupted_run/output.txt index da185c3704d..2e1d9cacd7a 100644 --- a/acceptance/bundle/resources/job_runs/interrupted_run/output.txt +++ b/acceptance/bundle/resources/job_runs/interrupted_run/output.txt @@ -42,11 +42,6 @@ Exit code: 1 "action": "skip", "remote_state": { "job_id": [MY_JOB_ID], - "lifecycle": { - "triggers": { - "on_file_change": {} - } - }, "run_id": [MY_RUN_ID], "run_name": "my-job", "run_page_url": "[DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?o=[NUMID]", diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt index 5b4ef23178c..69a7e5ece94 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt @@ -176,7 +176,7 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged { "lifecycle.triggers.on_file_change.files": { "action": "skip", - "reason": "not_returned_by_api", + "reason": "missing_in_remote", "old": { "migrations/001.txt": "[FILE_HASH][3]", "migrations/002.txt": "[FILE_HASH][1]", @@ -205,7 +205,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged { "lifecycle.triggers.on_file_change.files": { "action": "skip", - "reason": "not_returned_by_api", + "reason": "missing_in_remote", "old": { "migrations/001.txt": "[FILE_HASH][3]", "migrations/002.txt": "[FILE_HASH][1]", diff --git a/acceptance/bundle/resources/job_runs/redeploy/output.txt b/acceptance/bundle/resources/job_runs/redeploy/output.txt index 60111c98a93..a80685733ad 100644 --- a/acceptance/bundle/resources/job_runs/redeploy/output.txt +++ b/acceptance/bundle/resources/job_runs/redeploy/output.txt @@ -72,11 +72,6 @@ Resources: "job_parameters": { "env": "dev" }, - "lifecycle": { - "triggers": { - "on_file_change": {} - } - }, "result_state": "SUCCESS", "run_id": [MY_RUN_ID], "run_name": "my-job", diff --git a/bundle/direct/dresources/job_run.go b/bundle/direct/dresources/job_run.go index 815091a721a..9a937787e02 100644 --- a/bundle/direct/dresources/job_run.go +++ b/bundle/direct/dresources/job_run.go @@ -41,13 +41,12 @@ func isJobRunTriggerPath(path string) bool { type JobRunTriggersState struct { // Fresh UUID each plan while armed so Old!=New forces recreate. OnBundleDeploy string `json:"on_bundle_deploy,omitempty"` - // Path → content hash from ResolveJobRunFileTriggers; change to recreate. - OnFileChange *JobRunFileTriggerState `json:"on_file_change"` + // Content hashes from ResolveJobRunFileTriggers; any change recreates. + OnFileChange JobRunFileTriggerState `json:"on_file_change"` } -// JobRunFileTriggerState is always present so remote and state share one shape. -// Files is nil when the trigger is off, so dropping the trigger diffs here and -// a file appear/disappear diffs inside the map. +// JobRunFileTriggerState wraps the hashes so turning the trigger off (Files nil) +// diffs at one path, distinct from the per-file entries a changed file produces. type JobRunFileTriggerState struct { Files map[string]string `json:"files,omitempty"` } @@ -68,16 +67,21 @@ func (s *JobRunFileTriggerState) UnmarshalJSON(b []byte) error { return json.Unmarshal(b, &s.Files) } -// JobRunLifecycleState is the local-only trigger fingerprint, also present (empty) on remote. +// JobRunLifecycleState is the local-only trigger fingerprint. Nested by value, +// not by pointer: structdiff cannot descend into a nil pointer and would report +// the whole subtree at "lifecycle" instead of the leaf that actually changed. type JobRunLifecycleState struct { - Triggers *JobRunTriggersState `json:"triggers"` + Triggers JobRunTriggersState `json:"triggers"` } -func newJobRunLifecycleState() *JobRunLifecycleState { - return &JobRunLifecycleState{ - Triggers: &JobRunTriggersState{ +// Zero value spelled out field by field, as exhaustruct requires. +func emptyJobRunLifecycleState() JobRunLifecycleState { + return JobRunLifecycleState{ + Triggers: JobRunTriggersState{ OnBundleDeploy: "", - OnFileChange: &JobRunFileTriggerState{Files: nil}, + OnFileChange: JobRunFileTriggerState{ + Files: nil, + }, }, } } @@ -91,11 +95,10 @@ type JobRunState struct { // Local-only. Nested under lifecycle to mirror config and avoid colliding // with a future Jobs API field. - Lifecycle *JobRunLifecycleState `json:"lifecycle"` + Lifecycle JobRunLifecycleState `json:"lifecycle"` } func (s *JobRunState) UnmarshalJSON(b []byte) error { - s.Lifecycle = newJobRunLifecycleState() return marshal.Unmarshal(b, s) } @@ -103,8 +106,8 @@ func (s JobRunState) MarshalJSON() ([]byte, error) { return marshal.Marshal(s) } -// JobRunRemote embeds RunNow so every StateType path is a valid RemoteType path -// (see TestRemoteSuperset), plus the run's output-only fields for a faithful view. +// JobRunRemote is the RunNow request plus the run's output-only fields. It has no +// lifecycle: GetRun never returns the fingerprints (see knownMissingInRemoteType). type JobRunRemote struct { jobs.RunNow @@ -112,9 +115,6 @@ type JobRunRemote struct { // compare the two. See "RemapState is a dumb copy" in README.md. ResultState jobs.RunResultState `json:"result_state,omitempty"` - // Always the empty fingerprint: GetRun does not return triggers. - Lifecycle *JobRunLifecycleState `json:"lifecycle"` - RunId int64 `json:"run_id,omitempty"` RunName string `json:"run_name,omitempty"` State *jobs.RunState `json:"state,omitempty"` @@ -125,7 +125,6 @@ type JobRunRemote struct { // Custom marshaler needed because embedded RunNow's MarshalJSON would otherwise // take over and drop the additional fields. func (s *JobRunRemote) UnmarshalJSON(b []byte) error { - s.Lifecycle = newJobRunLifecycleState() return marshal.Unmarshal(b, s) } @@ -147,7 +146,7 @@ func (*ResourceJobRun) PrepareState(input *resources.JobRun) *JobRunState { state := &JobRunState{ RunNow: input.RunNow, ResultState: jobs.RunResultStateSuccess, - Lifecycle: newJobRunLifecycleState(), + Lifecycle: emptyJobRunLifecycleState(), } if input.HasOnBundleDeploy() { state.Lifecycle.Triggers.OnBundleDeploy = uuid.NewString() @@ -194,7 +193,6 @@ func makeJobRunRemote(run *jobs.Run) *JobRunRemote { ForceSendFields: nil, }, ResultState: run.State.ResultState, - Lifecycle: newJobRunLifecycleState(), RunId: run.RunId, RunName: run.RunName, // Rebuilt, not copied: the SDK records explicitly-sent fields in @@ -230,12 +228,13 @@ func (r *ResourceJobRun) DoRead(ctx context.Context, id string) (*JobRunRemote, } // RemapState extracts the fields used for diffing: the RunNow request and the -// outcome the run reached. +// outcome the run reached. Lifecycle has no remote counterpart, so it stays empty +// and the planner skips it as missing_in_remote. func (*ResourceJobRun) RemapState(remote *JobRunRemote) *JobRunState { return &JobRunState{ RunNow: remote.RunNow, ResultState: remote.ResultState, - Lifecycle: remote.Lifecycle, + Lifecycle: emptyJobRunLifecycleState(), } } diff --git a/bundle/direct/dresources/job_run_test.go b/bundle/direct/dresources/job_run_test.go index 66f61646116..34174454ab7 100644 --- a/bundle/direct/dresources/job_run_test.go +++ b/bundle/direct/dresources/job_run_test.go @@ -250,9 +250,6 @@ func TestJobRunStateUnmarshalWithoutLifecycle(t *testing.T) { require.NoError(t, json.Unmarshal([]byte(`{}`), &state)) - require.NotNil(t, state.Lifecycle) - require.NotNil(t, state.Lifecycle.Triggers) - require.NotNil(t, state.Lifecycle.Triggers.OnFileChange) assert.Nil(t, state.Lifecycle.Triggers.OnFileChange.Files) } @@ -266,13 +263,12 @@ func TestJobRunRemapStateCarriesTheOutcome(t *testing.T) { "", } { t.Run(string(outcome), func(t *testing.T) { - lifecycle := newJobRunLifecycleState() - remote := &JobRunRemote{RunId: 123, ResultState: outcome, Lifecycle: lifecycle} + remote := &JobRunRemote{RunId: 123, ResultState: outcome} state := (&ResourceJobRun{}).RemapState(remote) assert.Equal(t, outcome, state.ResultState) - assert.Same(t, lifecycle, state.Lifecycle) + assert.Equal(t, emptyJobRunLifecycleState(), state.Lifecycle) }) } } diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index 99cf7a2ba8b..472a4a0def8 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -175,9 +175,6 @@ resources: reason: not_returned_by_api - field: queue reason: not_returned_by_api - # Local-only trigger fingerprints; GetRun does not return them. - - field: lifecycle - reason: not_returned_by_api # A run is immutable and fire-once, so any change recreates it. Omitting # `field` matches every field (root; see TestFieldRuleOmittedIsRoot). # `field: ""` would instead match nothing. The one exception is a run that is diff --git a/bundle/direct/dresources/type_test.go b/bundle/direct/dresources/type_test.go index ca84850c33d..7ad9872c373 100644 --- a/bundle/direct/dresources/type_test.go +++ b/bundle/direct/dresources/type_test.go @@ -19,6 +19,11 @@ var knownMissingInRemoteType = map[string][]string{ "external_locations": { "skip_validation", }, + "job_runs": { + // Deliberate omission: the trigger fingerprints are client-side only, so + // GetRun has nothing to report them under. + "lifecycle", + }, "model_serving_endpoints": { "rate_limits", }, From 07e6c7958ff2a65539da28e1ce8f58b1ef1209ec Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Mon, 24 Aug 2026 07:55:05 +0000 Subject: [PATCH 17/26] job_runs: store on_file_change hashes on the trigger itself Drop the files wrapper so plan diffs land on lifecycle.triggers.on_file_change, matching the config path without an extra state-only level. --- acceptance/bundle/refschema/out.fields.txt | 5 ++- .../job_runs/on_file_change/output.txt | 8 ++--- .../job_runs/on_file_change_glob/output.txt | 20 +++++------ bundle/direct/dresources/job_run.go | 35 +++---------------- bundle/direct/dresources/job_run_test.go | 2 +- 5 files changed, 22 insertions(+), 48 deletions(-) diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index d1af0279654..63b82c1d595 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -912,9 +912,8 @@ resources.job_runs.*.lifecycle.prevent_destroy bool INPUT resources.job_runs.*.lifecycle.triggers []resources.JobRunTrigger INPUT resources.job_runs.*.lifecycle.triggers dresources.JobRunTriggersState STATE resources.job_runs.*.lifecycle.triggers.on_bundle_deploy string STATE -resources.job_runs.*.lifecycle.triggers.on_file_change dresources.JobRunFileTriggerState STATE -resources.job_runs.*.lifecycle.triggers.on_file_change.files map[string]string STATE -resources.job_runs.*.lifecycle.triggers.on_file_change.files.* string STATE +resources.job_runs.*.lifecycle.triggers.on_file_change map[string]string STATE +resources.job_runs.*.lifecycle.triggers.on_file_change.* string STATE resources.job_runs.*.lifecycle.triggers[*] resources.JobRunTrigger INPUT resources.job_runs.*.lifecycle.triggers[*].on_bundle_deploy *bool INPUT resources.job_runs.*.lifecycle.triggers[*].on_file_change *string INPUT diff --git a/acceptance/bundle/resources/job_runs/on_file_change/output.txt b/acceptance/bundle/resources/job_runs/on_file_change/output.txt index 9b662e1936d..28deeee38a4 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change/output.txt @@ -52,7 +52,7 @@ Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle.triggers.on_file_change.files": { + "lifecycle.triggers.on_file_change": { "action": "recreate", "reason": "immutable", "old": { @@ -62,7 +62,7 @@ Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged "seed.txt": "[FILE_HASH][1]" } }, - "lifecycle.triggers.on_file_change.files['seed.txt']": { + "lifecycle.triggers.on_file_change['seed.txt']": { "action": "recreate", "reason": "immutable", "old": "[FILE_HASH][0]", @@ -117,7 +117,7 @@ Warning: lifecycle.triggers.on_file_change: no files match "seed.txt" in databricks.yml:18:29 { - "lifecycle.triggers.on_file_change.files": { + "lifecycle.triggers.on_file_change": { "action": "recreate", "reason": "immutable", "old": { @@ -127,7 +127,7 @@ Warning: lifecycle.triggers.on_file_change: no files match "seed.txt" "seed.txt": "" } }, - "lifecycle.triggers.on_file_change.files['seed.txt']": { + "lifecycle.triggers.on_file_change['seed.txt']": { "action": "recreate", "reason": "immutable", "old": "[FILE_HASH][1]", diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt index 69a7e5ece94..a04e3710fb4 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt @@ -25,7 +25,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged === editing one matched file re-fires >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle.triggers.on_file_change.files": { + "lifecycle.triggers.on_file_change": { "action": "recreate", "reason": "immutable", "old": { @@ -39,7 +39,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "mysubdir/a.txt": "[FILE_HASH][2]" } }, - "lifecycle.triggers.on_file_change.files['migrations/001.txt']": { + "lifecycle.triggers.on_file_change['migrations/001.txt']": { "action": "recreate", "reason": "immutable", "old": "[FILE_HASH][0]", @@ -83,7 +83,7 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged === editing a file in a subdirectory re-fires >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle.triggers.on_file_change.files": { + "lifecycle.triggers.on_file_change": { "action": "recreate", "reason": "immutable", "old": { @@ -97,7 +97,7 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged "mysubdir/a.txt": "[FILE_HASH][4]" } }, - "lifecycle.triggers.on_file_change.files['mysubdir/a.txt']": { + "lifecycle.triggers.on_file_change['mysubdir/a.txt']": { "action": "recreate", "reason": "immutable", "old": "[FILE_HASH][2]", @@ -114,7 +114,7 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged === a new matching file re-fires >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle.triggers.on_file_change.files": { + "lifecycle.triggers.on_file_change": { "action": "recreate", "reason": "immutable", "old": { @@ -129,7 +129,7 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged "mysubdir/a.txt": "[FILE_HASH][2]" } }, - "lifecycle.triggers.on_file_change.files['migrations/003.txt']": { + "lifecycle.triggers.on_file_change['migrations/003.txt']": { "action": "recreate", "reason": "immutable", "new": "[FILE_HASH][5]" @@ -145,7 +145,7 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged === removing a matched file re-fires >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle.triggers.on_file_change.files": { + "lifecycle.triggers.on_file_change": { "action": "recreate", "reason": "immutable", "old": { @@ -158,7 +158,7 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged "mysubdir/a.txt": "[FILE_HASH][2]" } }, - "lifecycle.triggers.on_file_change.files['migrations/002.txt']": { + "lifecycle.triggers.on_file_change['migrations/002.txt']": { "action": "recreate", "reason": "immutable", "old": "[FILE_HASH][1]" @@ -174,7 +174,7 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged === editing a file below the glob does not re-fire >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle.triggers.on_file_change.files": { + "lifecycle.triggers.on_file_change": { "action": "skip", "reason": "missing_in_remote", "old": { @@ -203,7 +203,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged === ** is not recursive: it matches the same files as * >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle.triggers.on_file_change.files": { + "lifecycle.triggers.on_file_change": { "action": "skip", "reason": "missing_in_remote", "old": { diff --git a/bundle/direct/dresources/job_run.go b/bundle/direct/dresources/job_run.go index 9a937787e02..9d1a272a7a3 100644 --- a/bundle/direct/dresources/job_run.go +++ b/bundle/direct/dresources/job_run.go @@ -3,7 +3,6 @@ package dresources import ( "cmp" "context" - "encoding/json" "errors" "fmt" "slices" @@ -30,7 +29,7 @@ const jobRunTimeout = 24 * time.Hour // jobRunTriggerLocalPaths are local-only fingerprints: clearing one is skip, not recreate. var jobRunTriggerLocalPaths = []string{ "lifecycle.triggers.on_bundle_deploy", - "lifecycle.triggers.on_file_change.files", + "lifecycle.triggers.on_file_change", } func isJobRunTriggerPath(path string) bool { @@ -42,29 +41,7 @@ type JobRunTriggersState struct { // Fresh UUID each plan while armed so Old!=New forces recreate. OnBundleDeploy string `json:"on_bundle_deploy,omitempty"` // Content hashes from ResolveJobRunFileTriggers; any change recreates. - OnFileChange JobRunFileTriggerState `json:"on_file_change"` -} - -// JobRunFileTriggerState wraps the hashes so turning the trigger off (Files nil) -// diffs at one path, distinct from the per-file entries a changed file produces. -type JobRunFileTriggerState struct { - Files map[string]string `json:"files,omitempty"` -} - -// UnmarshalJSON accepts the wrapped map and the older path-to-hash map. -func (s *JobRunFileTriggerState) UnmarshalJSON(b []byte) error { - var raw map[string]json.RawMessage - if err := json.Unmarshal(b, &raw); err != nil { - return err - } - if files, ok := raw["files"]; ok && len(files) > 0 && files[0] == '{' { - return json.Unmarshal(files, &s.Files) - } - if len(raw) == 0 { - s.Files = nil - return nil - } - return json.Unmarshal(b, &s.Files) + OnFileChange map[string]string `json:"on_file_change,omitempty"` } // JobRunLifecycleState is the local-only trigger fingerprint. Nested by value, @@ -79,9 +56,7 @@ func emptyJobRunLifecycleState() JobRunLifecycleState { return JobRunLifecycleState{ Triggers: JobRunTriggersState{ OnBundleDeploy: "", - OnFileChange: JobRunFileTriggerState{ - Files: nil, - }, + OnFileChange: nil, }, } } @@ -152,7 +127,7 @@ func (*ResourceJobRun) PrepareState(input *resources.JobRun) *JobRunState { state.Lifecycle.Triggers.OnBundleDeploy = uuid.NewString() } if len(input.ResolvedFileTriggers) > 0 { - state.Lifecycle.Triggers.OnFileChange.Files = input.ResolvedFileTriggers + state.Lifecycle.Triggers.OnFileChange = input.ResolvedFileTriggers } return state } @@ -424,7 +399,7 @@ func (*ResourceJobRun) OverrideChangeDesc(_ context.Context, path *structpath.Pa pathString := path.String() if isJobRunTriggerPath(pathString) { removed := pathString == "lifecycle.triggers.on_bundle_deploy" && (change.New == nil || change.New == "") - removed = removed || pathString == "lifecycle.triggers.on_file_change.files" && change.New == nil + removed = removed || pathString == "lifecycle.triggers.on_file_change" && change.New == nil if removed { change.Action = deployplan.Skip change.Reason = "trigger removed" diff --git a/bundle/direct/dresources/job_run_test.go b/bundle/direct/dresources/job_run_test.go index 34174454ab7..804968f7707 100644 --- a/bundle/direct/dresources/job_run_test.go +++ b/bundle/direct/dresources/job_run_test.go @@ -250,7 +250,7 @@ func TestJobRunStateUnmarshalWithoutLifecycle(t *testing.T) { require.NoError(t, json.Unmarshal([]byte(`{}`), &state)) - assert.Nil(t, state.Lifecycle.Triggers.OnFileChange.Files) + assert.Nil(t, state.Lifecycle.Triggers.OnFileChange) } // The planner diffs RemapState(remote) against PrepareState(config), so a run From 580aaea3815d410b1e821e0b68373e98112c5d92 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Mon, 24 Aug 2026 08:27:10 +0000 Subject: [PATCH 18/26] acc: refresh job_runs goldens for the omitted empty trigger Flattening on_file_change to a map made an unarmed trigger drop out of the serialized state, so the three plans that print an unarmed job_run now report an empty triggers object. --- acceptance/bundle/resources/job_runs/failed_run/output.txt | 4 +--- acceptance/bundle/resources/job_runs/redeploy/output.txt | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/acceptance/bundle/resources/job_runs/failed_run/output.txt b/acceptance/bundle/resources/job_runs/failed_run/output.txt index 37f1a831681..5b8b6dbcd5f 100644 --- a/acceptance/bundle/resources/job_runs/failed_run/output.txt +++ b/acceptance/bundle/resources/job_runs/failed_run/output.txt @@ -28,9 +28,7 @@ Files: 7 uploaded, 0 deleted "value": { "job_id": [MY_JOB_ID], "lifecycle": { - "triggers": { - "on_file_change": {} - } + "triggers": {} }, "result_state": "SUCCESS" } diff --git a/acceptance/bundle/resources/job_runs/redeploy/output.txt b/acceptance/bundle/resources/job_runs/redeploy/output.txt index a80685733ad..8532692bdd9 100644 --- a/acceptance/bundle/resources/job_runs/redeploy/output.txt +++ b/acceptance/bundle/resources/job_runs/redeploy/output.txt @@ -60,9 +60,7 @@ Resources: "env": "prod" }, "lifecycle": { - "triggers": { - "on_file_change": {} - } + "triggers": {} }, "result_state": "SUCCESS" } From 3718f1c0fd4fb9d9d150f497d673b00dd1e2b3a5 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Mon, 24 Aug 2026 09:37:16 +0000 Subject: [PATCH 19/26] job_runs: error when on_file_change hashes nothing Directory-only and sync-excluded matches silently disarmed the trigger. Treat them as errors, warn that ** is not recursive, and cover both in acc. --- .../on_file_change_errors/allexcluded/a.txt | 1 + .../on_file_change_errors/databricks.yml | 6 ++++ .../on_file_change_errors/mixed/excluded.txt | 1 + .../mixed/nested/keep.txt | 1 + .../job_runs/on_file_change_errors/output.txt | 34 ++++++++++++++++--- .../job_runs/on_file_change_errors/script | 15 ++++++-- .../job_runs/on_file_change_glob/output.txt | 6 +++- .../job_runs/on_file_change_glob/script | 4 ++- .../mutator/resolve_job_run_file_triggers.go | 28 +++++++-------- 9 files changed, 73 insertions(+), 23 deletions(-) create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_errors/allexcluded/a.txt create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_errors/mixed/excluded.txt create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_errors/mixed/nested/keep.txt diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/allexcluded/a.txt b/acceptance/bundle/resources/job_runs/on_file_change_errors/allexcluded/a.txt new file mode 100644 index 00000000000..78981922613 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/allexcluded/a.txt @@ -0,0 +1 @@ +a diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/databricks.yml b/acceptance/bundle/resources/job_runs/on_file_change_errors/databricks.yml index 44c5433e52a..0ce9819887c 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_errors/databricks.yml +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/databricks.yml @@ -1,6 +1,12 @@ bundle: name: job-runs-on-file-change-errors +sync: + exclude: + - mixed/excluded.txt + # Literal path: the script rewrites the trigger by substring, and allexcluded/* would hit this line. + - allexcluded/a.txt + resources: jobs: my_job: diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/mixed/excluded.txt b/acceptance/bundle/resources/job_runs/on_file_change_errors/mixed/excluded.txt new file mode 100644 index 00000000000..bbde3dc9de9 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/mixed/excluded.txt @@ -0,0 +1 @@ +excluded diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/mixed/nested/keep.txt b/acceptance/bundle/resources/job_runs/on_file_change_errors/mixed/nested/keep.txt new file mode 100644 index 00000000000..2fa992c0b8b --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/mixed/nested/keep.txt @@ -0,0 +1 @@ +keep diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt index c7d3a98e1cb..381dc3e114f 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt @@ -2,7 +2,7 @@ === a pattern outside the sync root is rejected >>> musterr [CLI] bundle validate Error: lifecycle.triggers.on_file_change: pattern "../shared/*.txt" is not under the sync root - in databricks.yml:18:29 + in databricks.yml:25:29 Name: job-runs-on-file-change-errors Target: default @@ -14,8 +14,34 @@ Found 1 error === a glob that only matches directories is rejected >>> musterr [CLI] bundle validate -Error: lifecycle.triggers.on_file_change: pattern "dirs/*" matches no regular files - in databricks.yml:18:29 +Error: lifecycle.triggers.on_file_change: pattern "dirs/*" matches only directories or files excluded from sync, leaving nothing to hash + in databricks.yml:25:29 + +Name: job-runs-on-file-change-errors +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-errors/default + +Found 1 error + +=== a glob matching only a directory and an excluded file is rejected +>>> musterr [CLI] bundle validate +Error: lifecycle.triggers.on_file_change: pattern "mixed/*" matches only directories or files excluded from sync, leaving nothing to hash + in databricks.yml:25:29 + +Name: job-runs-on-file-change-errors +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-errors/default + +Found 1 error + +=== a glob matching only excluded files is rejected +>>> musterr [CLI] bundle validate +Error: lifecycle.triggers.on_file_change: pattern "allexcluded/*" matches only directories or files excluded from sync, leaving nothing to hash + in databricks.yml:25:29 Name: job-runs-on-file-change-errors Target: default @@ -28,7 +54,7 @@ Found 1 error === a pattern that matches nothing warns >>> [CLI] bundle validate Warning: lifecycle.triggers.on_file_change: no files match "missing/*.txt" - in databricks.yml:18:29 + in databricks.yml:25:29 Name: job-runs-on-file-change-errors Target: default diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/script b/acceptance/bundle/resources/job_runs/on_file_change_errors/script index 39596f0c772..48b9c5c7fc4 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_errors/script +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/script @@ -2,11 +2,20 @@ title "a pattern outside the sync root is rejected" trace musterr $CLI bundle validate title "a glob that only matches directories is rejected" -# dirs/* matches dirs/nested, a directory: nothing is left to hash, so the -# trigger would silently never fire. +# dirs/* matches dirs/nested, a directory, so nothing is left to hash. update_file.py databricks.yml "../shared/*.txt" "dirs/*" trace musterr $CLI bundle validate +title "a glob matching only a directory and an excluded file is rejected" +# mixed/excluded.txt is dropped by sync.exclude; mixed/nested is a directory. +update_file.py databricks.yml "dirs/*" "mixed/*" +trace musterr $CLI bundle validate + +title "a glob matching only excluded files is rejected" +# Every match is dropped by sync.exclude; unlike a missing pattern, this cannot re-arm later. +update_file.py databricks.yml "mixed/*" "allexcluded/*" +trace musterr $CLI bundle validate + title "a pattern that matches nothing warns" -update_file.py databricks.yml "dirs/*" "missing/*.txt" +update_file.py databricks.yml "allexcluded/*" "missing/*.txt" trace $CLI bundle validate diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt index a04e3710fb4..2f66f33e664 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt @@ -200,7 +200,11 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged -=== ** is not recursive: it matches the same files as * +=== ** is not recursive: it warns and matches the same files as * +Warning: lifecycle.triggers.on_file_change: ** in "migrations/**" is not recursive and matches the same files as * + in databricks.yml:18:29 + + >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { "lifecycle.triggers.on_file_change": { diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/script b/acceptance/bundle/resources/job_runs/on_file_change_glob/script index dd737f242b5..0f7107a43db 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/script +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/script @@ -58,6 +58,8 @@ update_file.py migrations/ignored.txt "ignored" "ignored-edited" trace $CLI bundle plan update_file.py migrations/ignored.txt "ignored-edited" "ignored" -title "** is not recursive: it matches the same files as *" +# Warning is on stderr of the untraced plan in changes(); trailing newline keeps it off the title. +title "** is not recursive: it warns and matches the same files as *\n" update_file.py databricks.yml "migrations/*" "migrations/**" changes +update_file.py databricks.yml "migrations/**" "migrations/*" diff --git a/bundle/config/mutator/resolve_job_run_file_triggers.go b/bundle/config/mutator/resolve_job_run_file_triggers.go index d74b9841c57..fbbabb60815 100644 --- a/bundle/config/mutator/resolve_job_run_file_triggers.go +++ b/bundle/config/mutator/resolve_job_run_file_triggers.go @@ -108,6 +108,14 @@ func resolveFileTrigger(b *bundle.Bundle, loc, pattern string, syncable map[stri Locations: b.Config.GetLocations(loc), }) } + // filepath.Glob treats ** as two *, so doublestar-style patterns match less than expected. + if strings.Contains(pattern, "**") { + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Warning, + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: ** in %q is not recursive and matches the same files as *", pattern), + Locations: b.Config.GetLocations(loc), + }) + } matches, err := filepath.Glob(filepath.Join(b.SyncRootPath, localPattern)) if err != nil { return out, diags.Append(diag.Diagnostic{ @@ -150,7 +158,7 @@ func resolveFileTrigger(b *bundle.Bundle, loc, pattern string, syncable map[stri }) continue } - // Same membership as sync: .gitignore and sync.exclude drop a glob match. + // Honor .gitignore and sync.exclude the same way sync does. if _, ok := syncable[filepath.ToSlash(rel)]; !ok { ignoredMatches++ continue @@ -167,20 +175,12 @@ func resolveFileTrigger(b *bundle.Bundle, loc, pattern string, syncable map[stri } out[filepath.ToSlash(rel)] = hash } - // A directory-only match would otherwise leave ResolvedFileTriggers empty - // and silently disarm the trigger while config still sets on_file_change. - if regularMatches == 0 && sawNonRegular && ignoredMatches == 0 { - diags = diags.Append(diag.Diagnostic{ + // Directories or excluded files would leave hashes empty and disarm the trigger. + // Unlike a missing-file warning, this cannot re-arm when a file appears later. + if regularMatches == 0 && (sawNonRegular || ignoredMatches > 0) { + return out, diags.Append(diag.Diagnostic{ Severity: diag.Error, - Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: pattern %q matches no regular files", pattern), - Locations: b.Config.GetLocations(loc), - }) - } - if len(out) == 0 && ignoredMatches > 0 { - out[filepath.ToSlash(pattern)] = missingFileHash - diags = diags.Append(diag.Diagnostic{ - Severity: diag.Warning, - Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: no files match %q", pattern), + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: pattern %q matches only directories or files excluded from sync, leaving nothing to hash", pattern), Locations: b.Config.GetLocations(loc), }) } From db83a4699cd01f87c32259dbc4648df4f3d05318 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Mon, 24 Aug 2026 11:58:36 +0000 Subject: [PATCH 20/26] acc: fix on_file_change_errors diagnostic line numbers The goldens were captured against a one-line-taller databricks.yml, so CI failed after the source settled on line 24. --- .../job_runs/on_file_change_errors/output.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt index 381dc3e114f..15934e8eeb6 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt @@ -2,7 +2,7 @@ === a pattern outside the sync root is rejected >>> musterr [CLI] bundle validate Error: lifecycle.triggers.on_file_change: pattern "../shared/*.txt" is not under the sync root - in databricks.yml:25:29 + in databricks.yml:24:29 Name: job-runs-on-file-change-errors Target: default @@ -15,7 +15,7 @@ Found 1 error === a glob that only matches directories is rejected >>> musterr [CLI] bundle validate Error: lifecycle.triggers.on_file_change: pattern "dirs/*" matches only directories or files excluded from sync, leaving nothing to hash - in databricks.yml:25:29 + in databricks.yml:24:29 Name: job-runs-on-file-change-errors Target: default @@ -28,7 +28,7 @@ Found 1 error === a glob matching only a directory and an excluded file is rejected >>> musterr [CLI] bundle validate Error: lifecycle.triggers.on_file_change: pattern "mixed/*" matches only directories or files excluded from sync, leaving nothing to hash - in databricks.yml:25:29 + in databricks.yml:24:29 Name: job-runs-on-file-change-errors Target: default @@ -41,7 +41,7 @@ Found 1 error === a glob matching only excluded files is rejected >>> musterr [CLI] bundle validate Error: lifecycle.triggers.on_file_change: pattern "allexcluded/*" matches only directories or files excluded from sync, leaving nothing to hash - in databricks.yml:25:29 + in databricks.yml:24:29 Name: job-runs-on-file-change-errors Target: default @@ -54,7 +54,7 @@ Found 1 error === a pattern that matches nothing warns >>> [CLI] bundle validate Warning: lifecycle.triggers.on_file_change: no files match "missing/*.txt" - in databricks.yml:25:29 + in databricks.yml:24:29 Name: job-runs-on-file-change-errors Target: default From a9f1c7d75f0036ce17bcfbc8fd429c19cbe7dbc9 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Mon, 24 Aug 2026 13:11:35 +0000 Subject: [PATCH 21/26] acc: removing on_file_change is unchanged, not an update Clearing a trigger is now a skip, so the plan reports the run as unchanged and the deploy makes no API call. --- .../bundle/resources/job_runs/on_file_change/output.txt | 9 +++------ .../bundle/resources/job_runs/on_file_change/script | 2 +- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/acceptance/bundle/resources/job_runs/on_file_change/output.txt b/acceptance/bundle/resources/job_runs/on_file_change/output.txt index 28deeee38a4..978e4b4394a 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change/output.txt @@ -186,17 +186,14 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged } } -=== removing on_file_change rewrites state without a run +=== removing on_file_change is unchanged and does not run >>> [CLI] bundle plan -update job_runs.my_run - -Plan: 0 to add, 1 to change, 0 to delete, 1 unchanged +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change/default/files... -Updated job_runs.my_run Files: 3 uploaded, 0 deleted -Resources: 0 created, 1 changed, 0 deleted, 1 unchanged +Resources: 0 created, 0 changed, 0 deleted, 2 unchanged >>> print_requests.py //jobs/run-now diff --git a/acceptance/bundle/resources/job_runs/on_file_change/script b/acceptance/bundle/resources/job_runs/on_file_change/script index fd1e8800c42..ef432d76033 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change/script +++ b/acceptance/bundle/resources/job_runs/on_file_change/script @@ -54,7 +54,7 @@ trace $CLI bundle deploy read_id.py my_run > /dev/null trace print_requests.py //jobs/run-now -title "removing on_file_change rewrites state without a run" +title "removing on_file_change is unchanged and does not run" update_file.py databricks.yml " lifecycle: triggers: - on_file_change: seed.txt From f02044641391a43bb6c0fa192839566503c6c9f4 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Mon, 24 Aug 2026 16:56:03 +0000 Subject: [PATCH 22/26] job_runs: resolve on_file_change relative to the defining YAML file Patterns must still land under the sync root. Reject **, warn when a glob matches directories, and skip only when the trigger itself is cleared. --- .../job_runs/on_file_change_errors/output.txt | 29 ++++++--- .../job_runs/on_file_change_errors/script | 12 ++-- .../job_runs/on_file_change_glob/output.txt | 59 ++++++++++--------- .../job_runs/on_file_change_glob/script | 16 ++--- .../on_file_change_include/databricks.yml | 14 +++++ .../on_file_change_include/out.test.toml | 2 + .../on_file_change_include/output.txt | 29 +++++++++ .../resources/job_run.yml | 7 +++ .../resources/watched.txt | 1 + .../job_runs/on_file_change_include/script | 17 ++++++ .../job_runs/on_file_change_include/test.toml | 3 + .../on_file_change_include/watched.txt | 1 + .../bundle/databricks.yml | 19 ++++++ .../bundle/resources/job_run.yml | 7 +++ .../bundle/resources/watched.txt | 1 + .../bundle/watched.txt | 1 + .../out.test.toml | 2 + .../output.txt | 33 +++++++++++ .../on_file_change_include_sync_root/script | 22 +++++++ .../shared.txt | 1 + .../test.toml | 2 + .../bundle/databricks.yml | 23 ++++++++ .../bundle/shared.txt | 1 + .../on_file_change_sync_root/out.test.toml | 2 + .../on_file_change_sync_root/output.txt | 29 +++++++++ .../job_runs/on_file_change_sync_root/script | 17 ++++++ .../on_file_change_sync_root/shared.txt | 1 + .../on_file_change_sync_root/test.toml | 2 + bundle/config/mutator/normalize_paths_test.go | 38 ++++++++++++ .../mutator/paths/job_run_paths_visitor.go | 24 ++++++++ .../paths/job_run_paths_visitor_test.go | 34 +++++++++++ bundle/config/mutator/paths/visitor.go | 1 + .../mutator/resolve_job_run_file_triggers.go | 46 +++++++++------ .../resolve_job_run_file_triggers_test.go | 22 ++++++- bundle/config/resources/lifecycle.go | 2 +- bundle/direct/dresources/job_run.go | 34 +++++++---- bundle/direct/dresources/job_run_test.go | 27 +++++++++ bundle/internal/schema/annotations.yml | 2 +- bundle/schema/jsonschema.json | 2 +- 39 files changed, 499 insertions(+), 86 deletions(-) create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_include/databricks.yml create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_include/out.test.toml create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_include/output.txt create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_include/resources/job_run.yml create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_include/resources/watched.txt create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_include/script create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_include/test.toml create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_include/watched.txt create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/bundle/databricks.yml create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/bundle/resources/job_run.yml create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/bundle/resources/watched.txt create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/bundle/watched.txt create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/out.test.toml create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/output.txt create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/script create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/shared.txt create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/test.toml create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_sync_root/bundle/databricks.yml create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_sync_root/bundle/shared.txt create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_sync_root/out.test.toml create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_sync_root/output.txt create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_sync_root/script create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_sync_root/shared.txt create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_sync_root/test.toml create mode 100644 bundle/config/mutator/paths/job_run_paths_visitor.go create mode 100644 bundle/config/mutator/paths/job_run_paths_visitor_test.go diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt index 15934e8eeb6..12e4730d8d9 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt @@ -1,5 +1,5 @@ -=== a pattern outside the sync root is rejected +=== a pattern that escapes the sync root is rejected >>> musterr [CLI] bundle validate Error: lifecycle.triggers.on_file_change: pattern "../shared/*.txt" is not under the sync root in databricks.yml:24:29 @@ -12,9 +12,9 @@ Workspace: Found 1 error -=== a glob that only matches directories is rejected ->>> musterr [CLI] bundle validate -Error: lifecycle.triggers.on_file_change: pattern "dirs/*" matches only directories or files excluded from sync, leaving nothing to hash +=== a glob that only matches directories warns +>>> [CLI] bundle validate +Warning: lifecycle.triggers.on_file_change: pattern "dirs/*" matches only directories in databricks.yml:24:29 Name: job-runs-on-file-change-errors @@ -23,11 +23,11 @@ Workspace: User: [USERNAME] Path: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-errors/default -Found 1 error +Found 1 warning -=== a glob matching only a directory and an excluded file is rejected +=== a glob matching a directory and an excluded file is rejected >>> musterr [CLI] bundle validate -Error: lifecycle.triggers.on_file_change: pattern "mixed/*" matches only directories or files excluded from sync, leaving nothing to hash +Error: lifecycle.triggers.on_file_change: pattern "mixed/*" matches only files excluded from sync, leaving nothing to hash in databricks.yml:24:29 Name: job-runs-on-file-change-errors @@ -40,7 +40,7 @@ Found 1 error === a glob matching only excluded files is rejected >>> musterr [CLI] bundle validate -Error: lifecycle.triggers.on_file_change: pattern "allexcluded/*" matches only directories or files excluded from sync, leaving nothing to hash +Error: lifecycle.triggers.on_file_change: pattern "allexcluded/*" matches only files excluded from sync, leaving nothing to hash in databricks.yml:24:29 Name: job-runs-on-file-change-errors @@ -63,3 +63,16 @@ Workspace: Path: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-errors/default Found 1 warning + +=== ** is not supported +>>> musterr [CLI] bundle validate +Error: lifecycle.triggers.on_file_change: ** in "src/**" is not supported; use * for a single directory level + in databricks.yml:24:29 + +Name: job-runs-on-file-change-errors +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-errors/default + +Found 1 error diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/script b/acceptance/bundle/resources/job_runs/on_file_change_errors/script index 48b9c5c7fc4..4b6e4ead73b 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_errors/script +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/script @@ -1,12 +1,12 @@ -title "a pattern outside the sync root is rejected" +title "a pattern that escapes the sync root is rejected" trace musterr $CLI bundle validate -title "a glob that only matches directories is rejected" +title "a glob that only matches directories warns" # dirs/* matches dirs/nested, a directory, so nothing is left to hash. update_file.py databricks.yml "../shared/*.txt" "dirs/*" -trace musterr $CLI bundle validate +trace $CLI bundle validate -title "a glob matching only a directory and an excluded file is rejected" +title "a glob matching a directory and an excluded file is rejected" # mixed/excluded.txt is dropped by sync.exclude; mixed/nested is a directory. update_file.py databricks.yml "dirs/*" "mixed/*" trace musterr $CLI bundle validate @@ -19,3 +19,7 @@ trace musterr $CLI bundle validate title "a pattern that matches nothing warns" update_file.py databricks.yml "allexcluded/*" "missing/*.txt" trace $CLI bundle validate + +title "** is not supported" +update_file.py databricks.yml "missing/*.txt" "src/**" +trace musterr $CLI bundle validate diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt index 2f66f33e664..390dda9f513 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt @@ -1,6 +1,9 @@ === deploy hashes the matched files >>> [CLI] bundle deploy +Warning: lifecycle.triggers.on_file_change: pattern "migrations/*" also matched directories, which are not hashed + in databricks.yml:18:29 + Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-glob/default/files... Output from job_runs.my_run: id=[MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?o=[NUMID] Output from job_runs.my_run: id=[MY_RUN_ID]: SUCCESS @@ -23,6 +26,10 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged } === editing one matched file re-fires +Warning: lifecycle.triggers.on_file_change: pattern "migrations/*" also matched directories, which are not hashed + in databricks.yml:18:29 + + >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { "lifecycle.triggers.on_file_change": { @@ -54,6 +61,9 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged } === bundle deploy +Warning: lifecycle.triggers.on_file_change: pattern "migrations/*" also matched directories, which are not hashed + in databricks.yml:18:29 + Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-glob/default/files... Output from job_runs.my_run: id=[MY_RUN_ID_2]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_2]?o=[NUMID] Output from job_runs.my_run: id=[MY_RUN_ID_2]: SUCCESS @@ -81,6 +91,10 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged } === editing a file in a subdirectory re-fires +Warning: lifecycle.triggers.on_file_change: pattern "migrations/*" also matched directories, which are not hashed + in databricks.yml:18:29 + + >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { "lifecycle.triggers.on_file_change": { @@ -112,6 +126,10 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged } === a new matching file re-fires +Warning: lifecycle.triggers.on_file_change: pattern "migrations/*" also matched directories, which are not hashed + in databricks.yml:18:29 + + >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { "lifecycle.triggers.on_file_change": { @@ -143,6 +161,10 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged } === removing a matched file re-fires +Warning: lifecycle.triggers.on_file_change: pattern "migrations/*" also matched directories, which are not hashed + in databricks.yml:18:29 + + >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { "lifecycle.triggers.on_file_change": { @@ -172,6 +194,10 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged } === editing a file below the glob does not re-fire +Warning: lifecycle.triggers.on_file_change: pattern "migrations/*" also matched directories, which are not hashed + in databricks.yml:18:29 + + >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { "lifecycle.triggers.on_file_change": { @@ -198,38 +224,15 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged === editing a gitignored file does not re-fire >>> [CLI] bundle plan -Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged - -=== ** is not recursive: it warns and matches the same files as * -Warning: lifecycle.triggers.on_file_change: ** in "migrations/**" is not recursive and matches the same files as * +Warning: lifecycle.triggers.on_file_change: pattern "migrations/*" also matched directories, which are not hashed in databricks.yml:18:29 - ->>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json -{ - "lifecycle.triggers.on_file_change": { - "action": "skip", - "reason": "missing_in_remote", - "old": { - "migrations/001.txt": "[FILE_HASH][3]", - "migrations/002.txt": "[FILE_HASH][1]", - "mysubdir/a.txt": "[FILE_HASH][2]" - }, - "new": { - "migrations/001.txt": "[FILE_HASH][3]", - "migrations/002.txt": "[FILE_HASH][1]", - "mysubdir/a.txt": "[FILE_HASH][2]" - } - }, - "result_state": { - "action": "skip", - "reason": "remote_already_set", - "new": "SUCCESS", - "remote": "SUCCESS" - } -} +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged >>> [CLI] bundle destroy --auto-approve +Warning: lifecycle.triggers.on_file_change: pattern "migrations/*" also matched directories, which are not hashed + in databricks.yml:18:29 + The following resources will be deleted: delete resources.job_runs.my_run delete resources.jobs.my_job diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/script b/acceptance/bundle/resources/job_runs/on_file_change_glob/script index 0f7107a43db..f0130f2637b 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/script +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/script @@ -19,7 +19,7 @@ trace read_id.py my_job read_id.py my_run > /dev/null trace print_requests.py //jobs/run-now -title "editing one matched file re-fires" +title "editing one matched file re-fires\n" update_file.py migrations/001.txt "one" "one-edited" changes # The deploy is not traced: readplanarg makes the command line differ per variant. @@ -29,24 +29,24 @@ read_id.py my_run > /dev/null trace print_requests.py --keep //jobs/runs/delete trace print_requests.py //jobs/run-now -title "editing a file in a subdirectory re-fires" +title "editing a file in a subdirectory re-fires\n" # printf keeps LF on Windows, where update_file.py's text-mode rewrite would # restore CRLF and leave the file hashing differently than the deploy above. printf 'sub-edited\n' > mysubdir/a.txt changes printf 'sub\n' > mysubdir/a.txt -title "a new matching file re-fires" +title "a new matching file re-fires\n" printf 'three\n' > migrations/003.txt changes rm migrations/003.txt -title "removing a matched file re-fires" +title "removing a matched file re-fires\n" mv migrations/002.txt tmp.002.txt changes mv tmp.002.txt migrations/002.txt -title "editing a file below the glob does not re-fire" +title "editing a file below the glob does not re-fire\n" # migrations/* matches the archive directory itself, which is skipped: the glob # does not descend into it. update_file.py migrations/archive/003.txt "three" "three-edited" @@ -57,9 +57,3 @@ title "editing a gitignored file does not re-fire" update_file.py migrations/ignored.txt "ignored" "ignored-edited" trace $CLI bundle plan update_file.py migrations/ignored.txt "ignored-edited" "ignored" - -# Warning is on stderr of the untraced plan in changes(); trailing newline keeps it off the title. -title "** is not recursive: it warns and matches the same files as *\n" -update_file.py databricks.yml "migrations/*" "migrations/**" -changes -update_file.py databricks.yml "migrations/**" "migrations/*" diff --git a/acceptance/bundle/resources/job_runs/on_file_change_include/databricks.yml b/acceptance/bundle/resources/job_runs/on_file_change_include/databricks.yml new file mode 100644 index 00000000000..3d8b74d28f8 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_include/databricks.yml @@ -0,0 +1,14 @@ +bundle: + name: job-runs-on-file-change-include + +include: + - resources/job_run.yml + +resources: + jobs: + my_job: + name: my-job + tasks: + - task_key: main + notebook_task: + notebook_path: /Workspace/test diff --git a/acceptance/bundle/resources/job_runs/on_file_change_include/out.test.toml b/acceptance/bundle/resources/job_runs/on_file_change_include/out.test.toml new file mode 100644 index 00000000000..0938e678987 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_include/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/job_runs/on_file_change_include/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_include/output.txt new file mode 100644 index 00000000000..52de054e734 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_include/output.txt @@ -0,0 +1,29 @@ + +=== first deploy +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-include/default/files... +Output from job_runs.my_run: id=[NUMID]: Run URL: [DATABRICKS_URL]/jobs/[NUMID]/runs/[NUMID]?o=[NUMID] +Output from job_runs.my_run: id=[NUMID]: SUCCESS +Created job_runs.my_run +Created jobs.my_job +Files: 7 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +=== editing a same-named file at the bundle root does not re-fire +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +=== editing the file next to the included YAML re-fires +>>> [CLI] bundle plan +recreate job_runs.my_run + +Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.job_runs.my_run + delete resources.jobs.my_job + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-include/default + +Destroy: 2 deleted diff --git a/acceptance/bundle/resources/job_runs/on_file_change_include/resources/job_run.yml b/acceptance/bundle/resources/job_runs/on_file_change_include/resources/job_run.yml new file mode 100644 index 00000000000..091489142f5 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_include/resources/job_run.yml @@ -0,0 +1,7 @@ +resources: + job_runs: + my_run: + job_id: ${resources.jobs.my_job.id} + lifecycle: + triggers: + - on_file_change: watched.txt diff --git a/acceptance/bundle/resources/job_runs/on_file_change_include/resources/watched.txt b/acceptance/bundle/resources/job_runs/on_file_change_include/resources/watched.txt new file mode 100644 index 00000000000..8b4e79d3c47 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_include/resources/watched.txt @@ -0,0 +1 @@ +include-v1 diff --git a/acceptance/bundle/resources/job_runs/on_file_change_include/script b/acceptance/bundle/resources/job_runs/on_file_change_include/script new file mode 100644 index 00000000000..65a8cea7a5b --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_include/script @@ -0,0 +1,17 @@ +cleanup() { + trace $CLI bundle destroy --auto-approve +} +trap cleanup EXIT + +title "first deploy" +trace $CLI bundle deploy + +title "editing a same-named file at the bundle root does not re-fire" +update_file.py watched.txt "root-v1" "root-v2" +trace $CLI bundle plan | contains.py "2 unchanged" "!recreate" +update_file.py watched.txt "root-v2" "root-v1" + +title "editing the file next to the included YAML re-fires" +update_file.py resources/watched.txt "include-v1" "include-v2" +trace $CLI bundle plan | contains.py "recreate job_runs.my_run" +update_file.py resources/watched.txt "include-v2" "include-v1" diff --git a/acceptance/bundle/resources/job_runs/on_file_change_include/test.toml b/acceptance/bundle/resources/job_runs/on_file_change_include/test.toml new file mode 100644 index 00000000000..c81bf4f943a --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_include/test.toml @@ -0,0 +1,3 @@ +# Path resolution only; the HTTP traffic is the same as on_file_change. +RecordRequests = false +Ignore = [".databricks"] diff --git a/acceptance/bundle/resources/job_runs/on_file_change_include/watched.txt b/acceptance/bundle/resources/job_runs/on_file_change_include/watched.txt new file mode 100644 index 00000000000..d1688530992 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_include/watched.txt @@ -0,0 +1 @@ +root-v1 diff --git a/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/bundle/databricks.yml b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/bundle/databricks.yml new file mode 100644 index 00000000000..2bdd1c06d13 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/bundle/databricks.yml @@ -0,0 +1,19 @@ +bundle: + name: job-runs-on-file-change-include-sync-root + +include: + - resources/job_run.yml + +sync: + paths: + - . + - ../shared.txt + +resources: + jobs: + my_job: + name: my-job + tasks: + - task_key: main + notebook_task: + notebook_path: /Workspace/test diff --git a/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/bundle/resources/job_run.yml b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/bundle/resources/job_run.yml new file mode 100644 index 00000000000..091489142f5 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/bundle/resources/job_run.yml @@ -0,0 +1,7 @@ +resources: + job_runs: + my_run: + job_id: ${resources.jobs.my_job.id} + lifecycle: + triggers: + - on_file_change: watched.txt diff --git a/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/bundle/resources/watched.txt b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/bundle/resources/watched.txt new file mode 100644 index 00000000000..8b4e79d3c47 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/bundle/resources/watched.txt @@ -0,0 +1 @@ +include-v1 diff --git a/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/bundle/watched.txt b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/bundle/watched.txt new file mode 100644 index 00000000000..6f22941eacf --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/bundle/watched.txt @@ -0,0 +1 @@ +bundle-v1 diff --git a/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/out.test.toml b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/out.test.toml new file mode 100644 index 00000000000..0938e678987 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/output.txt new file mode 100644 index 00000000000..6c475c0dbfc --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/output.txt @@ -0,0 +1,33 @@ + +=== first deploy +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-include-sync-root/default/files... +Output from job_runs.my_run: id=[NUMID]: Run URL: [DATABRICKS_URL]/jobs/[NUMID]/runs/[NUMID]?o=[NUMID] +Output from job_runs.my_run: id=[NUMID]: SUCCESS +Created job_runs.my_run +Created jobs.my_job +Files: 5 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +=== editing a same-named file in the bundle does not re-fire +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +=== editing a file at the sync root does not re-fire +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +=== editing the file next to the included YAML re-fires +>>> [CLI] bundle plan +recreate job_runs.my_run + +Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.job_runs.my_run + delete resources.jobs.my_job + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-include-sync-root/default + +Destroy: 2 deleted diff --git a/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/script b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/script new file mode 100644 index 00000000000..5a608ea9188 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/script @@ -0,0 +1,22 @@ +cleanup() { + withdir bundle trace $CLI bundle destroy --auto-approve +} +trap cleanup EXIT + +title "first deploy" +withdir bundle trace $CLI bundle deploy + +title "editing a same-named file in the bundle does not re-fire" +update_file.py bundle/watched.txt "bundle-v1" "bundle-v2" +withdir bundle trace $CLI bundle plan | contains.py "2 unchanged" "!recreate" +update_file.py bundle/watched.txt "bundle-v2" "bundle-v1" + +title "editing a file at the sync root does not re-fire" +update_file.py shared.txt "parent-v1" "parent-v2" +withdir bundle trace $CLI bundle plan | contains.py "2 unchanged" "!recreate" +update_file.py shared.txt "parent-v2" "parent-v1" + +title "editing the file next to the included YAML re-fires" +update_file.py bundle/resources/watched.txt "include-v1" "include-v2" +withdir bundle trace $CLI bundle plan | contains.py "recreate job_runs.my_run" +update_file.py bundle/resources/watched.txt "include-v2" "include-v1" diff --git a/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/shared.txt b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/shared.txt new file mode 100644 index 00000000000..4ddcc71c374 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/shared.txt @@ -0,0 +1 @@ +parent-v1 diff --git a/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/test.toml b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/test.toml new file mode 100644 index 00000000000..e1290bad9f4 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/test.toml @@ -0,0 +1,2 @@ +RecordRequests = false +Ignore = [".databricks"] diff --git a/acceptance/bundle/resources/job_runs/on_file_change_sync_root/bundle/databricks.yml b/acceptance/bundle/resources/job_runs/on_file_change_sync_root/bundle/databricks.yml new file mode 100644 index 00000000000..772a7ef92e7 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_sync_root/bundle/databricks.yml @@ -0,0 +1,23 @@ +bundle: + name: job-runs-on-file-change-sync-root + +sync: + paths: + - . + - ../shared.txt + +resources: + jobs: + my_job: + name: my-job + tasks: + - task_key: main + notebook_task: + notebook_path: /Workspace/test + + job_runs: + my_run: + job_id: ${resources.jobs.my_job.id} + lifecycle: + triggers: + - on_file_change: ../shared.txt diff --git a/acceptance/bundle/resources/job_runs/on_file_change_sync_root/bundle/shared.txt b/acceptance/bundle/resources/job_runs/on_file_change_sync_root/bundle/shared.txt new file mode 100644 index 00000000000..6f22941eacf --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_sync_root/bundle/shared.txt @@ -0,0 +1 @@ +bundle-v1 diff --git a/acceptance/bundle/resources/job_runs/on_file_change_sync_root/out.test.toml b/acceptance/bundle/resources/job_runs/on_file_change_sync_root/out.test.toml new file mode 100644 index 00000000000..0938e678987 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_sync_root/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/job_runs/on_file_change_sync_root/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_sync_root/output.txt new file mode 100644 index 00000000000..6a52f127ce8 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_sync_root/output.txt @@ -0,0 +1,29 @@ + +=== first deploy +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-sync-root/default/files... +Output from job_runs.my_run: id=[NUMID]: Run URL: [DATABRICKS_URL]/jobs/[NUMID]/runs/[NUMID]?o=[NUMID] +Output from job_runs.my_run: id=[NUMID]: SUCCESS +Created job_runs.my_run +Created jobs.my_job +Files: 3 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +=== editing a same-named file in the bundle does not re-fire +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +=== editing the file at the sync root re-fires +>>> [CLI] bundle plan +recreate job_runs.my_run + +Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.job_runs.my_run + delete resources.jobs.my_job + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-sync-root/default + +Destroy: 2 deleted diff --git a/acceptance/bundle/resources/job_runs/on_file_change_sync_root/script b/acceptance/bundle/resources/job_runs/on_file_change_sync_root/script new file mode 100644 index 00000000000..dcbc475840b --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_sync_root/script @@ -0,0 +1,17 @@ +cleanup() { + withdir bundle trace $CLI bundle destroy --auto-approve +} +trap cleanup EXIT + +title "first deploy" +withdir bundle trace $CLI bundle deploy + +title "editing a same-named file in the bundle does not re-fire" +update_file.py bundle/shared.txt "bundle-v1" "bundle-v2" +withdir bundle trace $CLI bundle plan | contains.py "2 unchanged" "!recreate" +update_file.py bundle/shared.txt "bundle-v2" "bundle-v1" + +title "editing the file at the sync root re-fires" +update_file.py shared.txt "sync-v1" "sync-v2" +withdir bundle trace $CLI bundle plan | contains.py "recreate job_runs.my_run" +update_file.py shared.txt "sync-v2" "sync-v1" diff --git a/acceptance/bundle/resources/job_runs/on_file_change_sync_root/shared.txt b/acceptance/bundle/resources/job_runs/on_file_change_sync_root/shared.txt new file mode 100644 index 00000000000..04f175f2a1c --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_sync_root/shared.txt @@ -0,0 +1 @@ +sync-v1 diff --git a/acceptance/bundle/resources/job_runs/on_file_change_sync_root/test.toml b/acceptance/bundle/resources/job_runs/on_file_change_sync_root/test.toml new file mode 100644 index 00000000000..e1290bad9f4 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_sync_root/test.toml @@ -0,0 +1,2 @@ +RecordRequests = false +Ignore = [".databricks"] diff --git a/bundle/config/mutator/normalize_paths_test.go b/bundle/config/mutator/normalize_paths_test.go index 8bee6c5064b..2399a03aa83 100644 --- a/bundle/config/mutator/normalize_paths_test.go +++ b/bundle/config/mutator/normalize_paths_test.go @@ -53,6 +53,44 @@ func TestNormalizePaths(t *testing.T) { require.Equal(t, "src/notebook.py", newValue.MustString()) } +func TestNormalizePaths_jobRunOnFileChange(t *testing.T) { + tmpDir := t.TempDir() + pattern := "../data/*.txt" + m := NormalizePaths() + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + JobRuns: map[string]*resources.JobRun{ + "run1": { + Lifecycle: &resources.JobRunLifecycle{ + Triggers: []resources.JobRunTrigger{ + {OnFileChange: &pattern}, + }, + }, + }, + }, + }, + }, + BundleRootPath: tmpDir, + } + + location := dyn.Location{File: filepath.Join(tmpDir, "resources", "run.yml")} + path := dyn.MustPathFromString("resources.job_runs.run1.lifecycle.triggers[0].on_file_change") + err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { + return dyn.MapByPath(v, path, func(path dyn.Path, value dyn.Value) (dyn.Value, error) { + return dyn.NewValue(value.MustString(), []dyn.Location{location}), nil + }) + }) + require.NoError(t, err) + + diags := bundle.Apply(t.Context(), b, m) + require.NoError(t, diags.Error()) + + newValue, err := dyn.GetByPath(b.Config.Value(), path) + require.NoError(t, err) + require.Equal(t, "data/*.txt", newValue.MustString()) +} + func TestNormalizePath_absolutePath(t *testing.T) { value, err := normalizePath("/notebook.py", dyn.Location{}, "/tmp") assert.NoError(t, err) diff --git a/bundle/config/mutator/paths/job_run_paths_visitor.go b/bundle/config/mutator/paths/job_run_paths_visitor.go new file mode 100644 index 00000000000..3bc56bacf72 --- /dev/null +++ b/bundle/config/mutator/paths/job_run_paths_visitor.go @@ -0,0 +1,24 @@ +package paths + +import ( + "github.com/databricks/cli/libs/dyn" +) + +// VisitJobRunPaths visits local paths on job_runs so NormalizePaths can rewrite +// them relative to the bundle root. Not used by TranslatePaths: hashing still +// needs a local glob, not a workspace path. +func VisitJobRunPaths(value dyn.Value, fn VisitFunc) (dyn.Value, error) { + pattern := dyn.NewPattern( + dyn.Key("resources"), + dyn.Key("job_runs"), + dyn.AnyKey(), + dyn.Key("lifecycle"), + dyn.Key("triggers"), + dyn.AnyIndex(), + dyn.Key("on_file_change"), + ) + + return dyn.MapByPattern(value, pattern, func(path dyn.Path, value dyn.Value) (dyn.Value, error) { + return fn(path, TranslateModeLocalRelative, value) + }) +} diff --git a/bundle/config/mutator/paths/job_run_paths_visitor_test.go b/bundle/config/mutator/paths/job_run_paths_visitor_test.go new file mode 100644 index 00000000000..0a006fd9df1 --- /dev/null +++ b/bundle/config/mutator/paths/job_run_paths_visitor_test.go @@ -0,0 +1,34 @@ +package paths + +import ( + "testing" + + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/libs/dyn" + "github.com/stretchr/testify/assert" +) + +func TestVisitJobRunPaths(t *testing.T) { + watched := "watched.txt" + root := config.Root{ + Resources: config.Resources{ + JobRuns: map[string]*resources.JobRun{ + "run0": { + Lifecycle: &resources.JobRunLifecycle{ + Triggers: []resources.JobRunTrigger{ + {OnFileChange: &watched}, + }, + }, + }, + }, + }, + } + + actual := collectVisitedPaths(t, root, VisitJobRunPaths) + expected := []dyn.Path{ + dyn.MustPathFromString("resources.job_runs.run0.lifecycle.triggers[0].on_file_change"), + } + + assert.ElementsMatch(t, expected, actual) +} diff --git a/bundle/config/mutator/paths/visitor.go b/bundle/config/mutator/paths/visitor.go index bdf42188fde..cadc52b75fe 100644 --- a/bundle/config/mutator/paths/visitor.go +++ b/bundle/config/mutator/paths/visitor.go @@ -10,6 +10,7 @@ type VisitFunc func(path dyn.Path, mode TranslateMode, value dyn.Value) (dyn.Val func VisitPaths(root dyn.Value, fn VisitFunc) (dyn.Value, error) { visitors := []func(dyn.Value, VisitFunc) (dyn.Value, error){ VisitJobPaths, + VisitJobRunPaths, VisitJobLibrariesPaths, VisitAppPaths, VisitArtifactPaths, diff --git a/bundle/config/mutator/resolve_job_run_file_triggers.go b/bundle/config/mutator/resolve_job_run_file_triggers.go index fbbabb60815..2b3f11a1a88 100644 --- a/bundle/config/mutator/resolve_job_run_file_triggers.go +++ b/bundle/config/mutator/resolve_job_run_file_triggers.go @@ -52,11 +52,7 @@ func (*resolveJobRunFileTriggers) Apply(ctx context.Context, b *bundle.Bundle) d diags = diags.Extend(d) maps.Copy(out, hashes) } - if len(out) == 0 { - jr.ResolvedFileTriggers = nil - } else { - jr.ResolvedFileTriggers = out - } + jr.ResolvedFileTriggers = out } return diags } @@ -100,23 +96,27 @@ func resolveFileTrigger(b *bundle.Bundle, loc, pattern string, syncable map[stri var diags diag.Diagnostics out := make(map[string]string) localPattern := filepath.FromSlash(pattern) - // Keep hashes under SyncRoot; same IsLocal gate as translate_paths. - if !filepath.IsLocal(localPattern) { + // filepath.Glob treats ** as two *, so doublestar-style patterns match less than expected. + if strings.Contains(pattern, "**") { return out, diags.Append(diag.Diagnostic{ Severity: diag.Error, - Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: pattern %q is not under the sync root", pattern), + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: ** in %q is not supported; use * for a single directory level", pattern), Locations: b.Config.GetLocations(loc), }) } - // filepath.Glob treats ** as two *, so doublestar-style patterns match less than expected. - if strings.Contains(pattern, "**") { - diags = diags.Append(diag.Diagnostic{ - Severity: diag.Warning, - Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: ** in %q is not recursive and matches the same files as *", pattern), + // NormalizePaths has already rewritten YAML-relative globs to be bundle-root + // relative. Join that onto the bundle root, then require the result stay + // under the sync root (an ancestor of the bundle when sync.paths uses ..). + joined := filepath.Join(b.BundleRootPath, localPattern) + relPattern, err := filepath.Rel(b.SyncRootPath, joined) + if err != nil || !filepath.IsLocal(relPattern) { + return out, diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: pattern %q is not under the sync root", pattern), Locations: b.Config.GetLocations(loc), }) } - matches, err := filepath.Glob(filepath.Join(b.SyncRootPath, localPattern)) + matches, err := filepath.Glob(joined) if err != nil { return out, diags.Append(diag.Diagnostic{ Severity: diag.Error, @@ -175,12 +175,22 @@ func resolveFileTrigger(b *bundle.Bundle, loc, pattern string, syncable map[stri } out[filepath.ToSlash(rel)] = hash } - // Directories or excluded files would leave hashes empty and disarm the trigger. - // Unlike a missing-file warning, this cannot re-arm when a file appears later. - if regularMatches == 0 && (sawNonRegular || ignoredMatches > 0) { + if regularMatches == 0 && ignoredMatches > 0 { return out, diags.Append(diag.Diagnostic{ Severity: diag.Error, - Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: pattern %q matches only directories or files excluded from sync, leaving nothing to hash", pattern), + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: pattern %q matches only files excluded from sync, leaving nothing to hash", pattern), + Locations: b.Config.GetLocations(loc), + }) + } + if sawNonRegular { + msg := fmt.Sprintf("lifecycle.triggers.on_file_change: pattern %q also matched directories, which are not hashed", pattern) + if regularMatches == 0 { + out[filepath.ToSlash(pattern)] = missingFileHash + msg = fmt.Sprintf("lifecycle.triggers.on_file_change: pattern %q matches only directories", pattern) + } + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Warning, + Summary: msg, Locations: b.Config.GetLocations(loc), }) } diff --git a/bundle/config/mutator/resolve_job_run_file_triggers_test.go b/bundle/config/mutator/resolve_job_run_file_triggers_test.go index 1cc54eecf4f..0630a4b35f6 100644 --- a/bundle/config/mutator/resolve_job_run_file_triggers_test.go +++ b/bundle/config/mutator/resolve_job_run_file_triggers_test.go @@ -44,14 +44,30 @@ func TestResolveJobRunFileTriggers(t *testing.T) { require.False(t, diags.HasError()) assert.Equal(t, contentHash("v1"), b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers["seed.txt"]) }) + + t.Run("globs from the bundle root when the sync root is an ancestor", func(t *testing.T) { + parent := t.TempDir() + bundleDir := filepath.Join(parent, "bundle") + require.NoError(t, os.Mkdir(bundleDir, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(parent, "shared.txt"), []byte("from-sync-root"), 0o644)) + + pattern := "../shared.txt" + b := bundleWithFileTrigger(parent, pattern) + b.BundleRootPath = bundleDir + + diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) + require.False(t, diags.HasError()) + assert.Equal(t, contentHash("from-sync-root"), b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers["shared.txt"]) + }) } func bundleWithFileTrigger(syncRoot, pattern string) *bundle.Bundle { root := vfs.MustNew(syncRoot) return &bundle.Bundle{ - SyncRootPath: syncRoot, - SyncRoot: root, - WorktreeRoot: root, + BundleRootPath: syncRoot, + SyncRootPath: syncRoot, + SyncRoot: root, + WorktreeRoot: root, Config: config.Root{ Sync: config.Sync{Paths: []string{"."}}, Resources: config.Resources{ diff --git a/bundle/config/resources/lifecycle.go b/bundle/config/resources/lifecycle.go index a4e881d298c..1f5d576d835 100644 --- a/bundle/config/resources/lifecycle.go +++ b/bundle/config/resources/lifecycle.go @@ -37,5 +37,5 @@ type JobRunLifecycle struct { // JobRunTrigger is one lifecycle.triggers entry. type JobRunTrigger struct { OnBundleDeploy *bool `json:"on_bundle_deploy,omitempty"` - OnFileChange *string `json:"on_file_change,omitempty"` // path or glob under sync root + OnFileChange *string `json:"on_file_change,omitempty"` // path or glob relative to the defining YAML file; must resolve under the sync root } diff --git a/bundle/direct/dresources/job_run.go b/bundle/direct/dresources/job_run.go index 9d1a272a7a3..872720ffcd4 100644 --- a/bundle/direct/dresources/job_run.go +++ b/bundle/direct/dresources/job_run.go @@ -26,14 +26,29 @@ import ( // jobRunTimeout matches the timeout `bundle run` allows a run (bundle/run/job.go). const jobRunTimeout = 24 * time.Hour -// jobRunTriggerLocalPaths are local-only fingerprints: clearing one is skip, not recreate. -var jobRunTriggerLocalPaths = []string{ - "lifecycle.triggers.on_bundle_deploy", - "lifecycle.triggers.on_file_change", +// Parsed so HasPrefix catches parent and child paths; skip is only when the +// trigger itself is cleared, not when a file hash under on_file_change changes. +var ( + jobRunTriggersPath = structpath.MustParsePath("lifecycle.triggers") + jobRunOnBundleDeployPath = structpath.MustParsePath("lifecycle.triggers.on_bundle_deploy") + jobRunOnFileChangePath = structpath.MustParsePath("lifecycle.triggers.on_file_change") +) + +func samePath(a, b *structpath.PathNode) bool { + return a.HasPrefix(b) && b.HasPrefix(a) } -func isJobRunTriggerPath(path string) bool { - return slices.Contains(jobRunTriggerLocalPaths, path) +func jobRunTriggerRemoved(path *structpath.PathNode, change *ChangeDesc) bool { + switch { + case samePath(path, jobRunOnBundleDeployPath): + return change.New == nil || change.New == "" + case samePath(path, jobRunOnFileChangePath): + return change.New == nil + case samePath(path, jobRunTriggersPath): + return change.New == nil + default: + return false + } } // JobRunTriggersState is the persisted fingerprint of lifecycle.triggers. @@ -396,11 +411,8 @@ func reportRunLine(ctx context.Context, runID int64, msg string) { // result_state either, so the lifecycle state is what tells the two apart. // Clearing a trigger skips its local-only fingerprint without re-firing the run. func (*ResourceJobRun) OverrideChangeDesc(_ context.Context, path *structpath.PathNode, change *ChangeDesc, remote *JobRunRemote) error { - pathString := path.String() - if isJobRunTriggerPath(pathString) { - removed := pathString == "lifecycle.triggers.on_bundle_deploy" && (change.New == nil || change.New == "") - removed = removed || pathString == "lifecycle.triggers.on_file_change" && change.New == nil - if removed { + if path.HasPrefix(jobRunTriggersPath) { + if jobRunTriggerRemoved(path, change) { change.Action = deployplan.Skip change.Reason = "trigger removed" } diff --git a/bundle/direct/dresources/job_run_test.go b/bundle/direct/dresources/job_run_test.go index 804968f7707..68738b93e18 100644 --- a/bundle/direct/dresources/job_run_test.go +++ b/bundle/direct/dresources/job_run_test.go @@ -10,6 +10,7 @@ import ( "testing" "time" + "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/cli/libs/testserver" "github.com/databricks/databricks-sdk-go" @@ -367,3 +368,29 @@ func TestJobRunDeleteLeavesFinishedRunAlone(t *testing.T) { assert.False(t, cancelled.Load(), "a run that already finished has nothing to cancel") } + +func TestJobRunOverrideChangeDescTriggerRemoved(t *testing.T) { + r := &ResourceJobRun{} + for _, tt := range []struct { + name string + path string + new any + action deployplan.ActionType + }{ + {"cleared on_bundle_deploy string", "lifecycle.triggers.on_bundle_deploy", "", deployplan.Skip}, + {"nil on_bundle_deploy", "lifecycle.triggers.on_bundle_deploy", nil, deployplan.Skip}, + {"rotated on_bundle_deploy", "lifecycle.triggers.on_bundle_deploy", "uuid", deployplan.Recreate}, + {"cleared on_file_change", "lifecycle.triggers.on_file_change", nil, deployplan.Skip}, + {"changed on_file_change map", "lifecycle.triggers.on_file_change", map[string]string{"a.txt": "h"}, deployplan.Recreate}, + {"cleared on_file_change child", "lifecycle.triggers.on_file_change['a.txt']", nil, deployplan.Recreate}, + {"cleared triggers parent", "lifecycle.triggers", nil, deployplan.Skip}, + {"changed triggers parent", "lifecycle.triggers", JobRunTriggersState{OnBundleDeploy: "uuid"}, deployplan.Recreate}, + {"result_state unchanged", "result_state", nil, deployplan.Recreate}, + } { + t.Run(tt.name, func(t *testing.T) { + change := &ChangeDesc{Action: deployplan.Recreate, New: tt.new} + require.NoError(t, r.OverrideChangeDesc(t.Context(), structpath.MustParsePath(tt.path), change, nil)) + assert.Equal(t, tt.action, change.Action) + }) + } +} diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index ff2d1fd2626..c3e76eb7b8d 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -1019,7 +1019,7 @@ resources: If true, re-fire the run on every bundle deploy. Incompatible with lifecycle.prevent_destroy. "on_file_change": "description": |- - Path or glob under the sync root. Re-fire the run when a matched file's content hash changes, or when the set of matches appears or disappears. Incompatible with lifecycle.prevent_destroy. + Path or glob relative to the defining YAML file. It must resolve under the sync root. Re-fire the run when a matched file's content hash changes, or when the set of matches appears or disappears. Incompatible with lifecycle.prevent_destroy. "python_named_params": "description": |- PLACEHOLDER diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 11fbf411f7a..3829ae5c406 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -1396,7 +1396,7 @@ "$ref": "#/$defs/bool" }, "on_file_change": { - "description": "Path or glob under the sync root. Re-fire the run when a matched file's content hash changes, or when the set of matches appears or disappears. Incompatible with lifecycle.prevent_destroy.", + "description": "Path or glob relative to the defining YAML file. It must resolve under the sync root. Re-fire the run when a matched file's content hash changes, or when the set of matches appears or disappears. Incompatible with lifecycle.prevent_destroy.", "$ref": "#/$defs/string" } }, From 8501706a1353bb3cecf6bb117942fe812328ed8a Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Tue, 25 Aug 2026 08:30:03 +0000 Subject: [PATCH 23/26] job_runs: reject absolute on_file_change patterns filepath.Join rebases an absolute glob under the bundle root, so reject those up front. Key missing-file placeholders to the sync root, and skip a cleared trigger from path.String() instead of prefix matching. --- .../job_runs/on_file_change_errors/output.txt | 13 ++++++ .../job_runs/on_file_change_errors/script | 4 ++ .../mutator/resolve_job_run_file_triggers.go | 20 ++++++--- .../resolve_job_run_file_triggers_test.go | 22 ++++++++-- .../mutator/validate_job_run_triggers.go | 28 +++++++++---- .../mutator/validate_job_run_triggers_test.go | 13 +++++- bundle/config/resources/job_run.go | 1 + bundle/direct/dresources/job_run.go | 42 ++++--------------- bundle/phases/initialize.go | 3 +- 9 files changed, 93 insertions(+), 53 deletions(-) diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt index 12e4730d8d9..37ea391136b 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt @@ -76,3 +76,16 @@ Workspace: Path: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-errors/default Found 1 error + +=== an absolute pattern is rejected +>>> musterr [CLI] bundle validate +Error: lifecycle.triggers.on_file_change: pattern "/tmp/seed.txt" must be relative to the defining YAML file + in databricks.yml:24:29 + +Name: job-runs-on-file-change-errors +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-errors/default + +Found 1 error diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/script b/acceptance/bundle/resources/job_runs/on_file_change_errors/script index 4b6e4ead73b..86b10b70bf0 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_errors/script +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/script @@ -23,3 +23,7 @@ trace $CLI bundle validate title "** is not supported" update_file.py databricks.yml "missing/*.txt" "src/**" trace musterr $CLI bundle validate + +title "an absolute pattern is rejected" +update_file.py databricks.yml "src/**" "/tmp/seed.txt" +trace musterr $CLI bundle validate diff --git a/bundle/config/mutator/resolve_job_run_file_triggers.go b/bundle/config/mutator/resolve_job_run_file_triggers.go index 2b3f11a1a88..dc615f030c8 100644 --- a/bundle/config/mutator/resolve_job_run_file_triggers.go +++ b/bundle/config/mutator/resolve_job_run_file_triggers.go @@ -8,6 +8,7 @@ import ( "io" "maps" "os" + pathlib "path" "path/filepath" "strings" @@ -48,7 +49,7 @@ func (*resolveJobRunFileTriggers) Apply(ctx context.Context, b *bundle.Bundle) d continue } path := fmt.Sprintf("resources.job_runs.%s.lifecycle.triggers[%d].on_file_change", name, i) - hashes, d := resolveFileTrigger(b, path, strings.TrimSpace(*t.OnFileChange), syncable) + hashes, d := resolveFileTrigger(b, path, *t.OnFileChange, syncable) diags = diags.Extend(d) maps.Copy(out, hashes) } @@ -95,7 +96,6 @@ func syncableRelPaths(ctx context.Context, b *bundle.Bundle) (map[string]struct{ func resolveFileTrigger(b *bundle.Bundle, loc, pattern string, syncable map[string]struct{}) (map[string]string, diag.Diagnostics) { var diags diag.Diagnostics out := make(map[string]string) - localPattern := filepath.FromSlash(pattern) // filepath.Glob treats ** as two *, so doublestar-style patterns match less than expected. if strings.Contains(pattern, "**") { return out, diags.Append(diag.Diagnostic{ @@ -104,10 +104,19 @@ func resolveFileTrigger(b *bundle.Bundle, loc, pattern string, syncable map[stri Locations: b.Config.GetLocations(loc), }) } + // filepath.Join would otherwise rebase an absolute pattern under the bundle + // root (Join("/bundle", "/etc/passwd") is "/bundle/etc/passwd"). + if filepath.IsAbs(filepath.FromSlash(pattern)) || pathlib.IsAbs(pattern) { + return out, diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: pattern %q must be relative to the defining YAML file", pattern), + Locations: b.Config.GetLocations(loc), + }) + } // NormalizePaths has already rewritten YAML-relative globs to be bundle-root // relative. Join that onto the bundle root, then require the result stay // under the sync root (an ancestor of the bundle when sync.paths uses ..). - joined := filepath.Join(b.BundleRootPath, localPattern) + joined := filepath.Join(b.BundleRootPath, filepath.FromSlash(pattern)) relPattern, err := filepath.Rel(b.SyncRootPath, joined) if err != nil || !filepath.IsLocal(relPattern) { return out, diags.Append(diag.Diagnostic{ @@ -116,6 +125,7 @@ func resolveFileTrigger(b *bundle.Bundle, loc, pattern string, syncable map[stri Locations: b.Config.GetLocations(loc), }) } + key := filepath.ToSlash(relPattern) matches, err := filepath.Glob(joined) if err != nil { return out, diags.Append(diag.Diagnostic{ @@ -125,7 +135,7 @@ func resolveFileTrigger(b *bundle.Bundle, loc, pattern string, syncable map[stri }) } if len(matches) == 0 { - out[filepath.ToSlash(pattern)] = missingFileHash + out[key] = missingFileHash return out, diags.Append(diag.Diagnostic{ Severity: diag.Warning, Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: no files match %q", pattern), @@ -185,7 +195,7 @@ func resolveFileTrigger(b *bundle.Bundle, loc, pattern string, syncable map[stri if sawNonRegular { msg := fmt.Sprintf("lifecycle.triggers.on_file_change: pattern %q also matched directories, which are not hashed", pattern) if regularMatches == 0 { - out[filepath.ToSlash(pattern)] = missingFileHash + out[key] = missingFileHash msg = fmt.Sprintf("lifecycle.triggers.on_file_change: pattern %q matches only directories", pattern) } diags = diags.Append(diag.Diagnostic{ diff --git a/bundle/config/mutator/resolve_job_run_file_triggers_test.go b/bundle/config/mutator/resolve_job_run_file_triggers_test.go index 0630a4b35f6..9271589bb48 100644 --- a/bundle/config/mutator/resolve_job_run_file_triggers_test.go +++ b/bundle/config/mutator/resolve_job_run_file_triggers_test.go @@ -34,15 +34,29 @@ func TestResolveJobRunFileTriggers(t *testing.T) { assert.Equal(t, contentHash("world"), hashes["b.txt"]) }) - t.Run("trims pattern whitespace", func(t *testing.T) { + t.Run("rejects an absolute pattern", func(t *testing.T) { dir := t.TempDir() - require.NoError(t, os.WriteFile(filepath.Join(dir, "seed.txt"), []byte("v1"), 0o644)) - pattern := " seed.txt " + pattern := "/etc/passwd" b := bundleWithFileTrigger(dir, pattern) + diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) + require.True(t, diags.HasError()) + require.Equal(t, `lifecycle.triggers.on_file_change: pattern "/etc/passwd" must be relative to the defining YAML file`, diags[0].Summary) + assert.Empty(t, b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers) + }) + + t.Run("missing pattern is keyed relative to the sync root", func(t *testing.T) { + parent := t.TempDir() + bundleDir := filepath.Join(parent, "bundle") + require.NoError(t, os.Mkdir(bundleDir, 0o755)) + + pattern := "../missing.txt" + b := bundleWithFileTrigger(parent, pattern) + b.BundleRootPath = bundleDir + diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) require.False(t, diags.HasError()) - assert.Equal(t, contentHash("v1"), b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers["seed.txt"]) + assert.Equal(t, map[string]string{"missing.txt": ""}, b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers) }) t.Run("globs from the bundle root when the sync root is an ancestor", func(t *testing.T) { diff --git a/bundle/config/mutator/validate_job_run_triggers.go b/bundle/config/mutator/validate_job_run_triggers.go index c185d3b5781..fa31032c689 100644 --- a/bundle/config/mutator/validate_job_run_triggers.go +++ b/bundle/config/mutator/validate_job_run_triggers.go @@ -26,13 +26,14 @@ func (*validateJobRunTriggers) Apply(_ context.Context, b *bundle.Bundle) diag.D if jr == nil || jr.Lifecycle == nil { continue } - // Recreate-every-deploy cannot coexist with prevent_destroy. - if (jr.HasOnBundleDeploy() || jr.HasOnFileChange()) && jr.Lifecycle.PreventDestroy { - diags = diags.Append(diag.Diagnostic{ - Severity: diag.Error, - Summary: "lifecycle.triggers.on_bundle_deploy or on_file_change is incompatible with lifecycle.prevent_destroy", - Locations: b.Config.GetLocations(fmt.Sprintf("resources.job_runs.%s.lifecycle", name)), - }) + if jr.Lifecycle.PreventDestroy { + if summary := preventDestroyError(jr.HasOnBundleDeploy(), jr.HasOnFileChange()); summary != "" { + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: summary, + Locations: b.Config.GetLocations(fmt.Sprintf("resources.job_runs.%s.lifecycle", name)), + }) + } } for i, t := range jr.Lifecycle.Triggers { path := fmt.Sprintf("resources.job_runs.%s.lifecycle.triggers[%d]", name, i) @@ -70,3 +71,16 @@ func (*validateJobRunTriggers) Apply(_ context.Context, b *bundle.Bundle) diag.D } return diags } + +func preventDestroyError(onBundleDeploy, onFileChange bool) string { + switch { + case onBundleDeploy && onFileChange: + return "lifecycle.triggers.on_bundle_deploy and on_file_change are incompatible with lifecycle.prevent_destroy" + case onBundleDeploy: + return "lifecycle.triggers.on_bundle_deploy is incompatible with lifecycle.prevent_destroy" + case onFileChange: + return "lifecycle.triggers.on_file_change is incompatible with lifecycle.prevent_destroy" + default: + return "" + } +} diff --git a/bundle/config/mutator/validate_job_run_triggers_test.go b/bundle/config/mutator/validate_job_run_triggers_test.go index 7e8772d8d8b..07a7276a41c 100644 --- a/bundle/config/mutator/validate_job_run_triggers_test.go +++ b/bundle/config/mutator/validate_job_run_triggers_test.go @@ -84,7 +84,7 @@ func TestValidateJobRunTriggers(t *testing.T) { {OnBundleDeploy: &trueVal}, }, preventDestroy: true, - summary: "lifecycle.triggers.on_bundle_deploy or on_file_change is incompatible with lifecycle.prevent_destroy", + summary: "lifecycle.triggers.on_bundle_deploy is incompatible with lifecycle.prevent_destroy", }, { name: "on_file_change with prevent_destroy", @@ -92,7 +92,16 @@ func TestValidateJobRunTriggers(t *testing.T) { {OnFileChange: &fileChange}, }, preventDestroy: true, - summary: "lifecycle.triggers.on_bundle_deploy or on_file_change is incompatible with lifecycle.prevent_destroy", + summary: "lifecycle.triggers.on_file_change is incompatible with lifecycle.prevent_destroy", + }, + { + name: "both triggers with prevent_destroy", + triggers: []resources.JobRunTrigger{ + {OnFileChange: &fileChange}, + {OnBundleDeploy: &trueVal}, + }, + preventDestroy: true, + summary: "lifecycle.triggers.on_bundle_deploy and on_file_change are incompatible with lifecycle.prevent_destroy", }, { name: "prevent_destroy alone", diff --git a/bundle/config/resources/job_run.go b/bundle/config/resources/job_run.go index f4b16963453..512077e2738 100644 --- a/bundle/config/resources/job_run.go +++ b/bundle/config/resources/job_run.go @@ -46,6 +46,7 @@ func (r *JobRun) HasOnBundleDeploy() bool { return false } +// HasOnFileChange reports whether any trigger re-fires when matched files change. func (r *JobRun) HasOnFileChange() bool { if r.Lifecycle == nil { return false diff --git a/bundle/direct/dresources/job_run.go b/bundle/direct/dresources/job_run.go index 872720ffcd4..d7574bdc2ac 100644 --- a/bundle/direct/dresources/job_run.go +++ b/bundle/direct/dresources/job_run.go @@ -26,31 +26,6 @@ import ( // jobRunTimeout matches the timeout `bundle run` allows a run (bundle/run/job.go). const jobRunTimeout = 24 * time.Hour -// Parsed so HasPrefix catches parent and child paths; skip is only when the -// trigger itself is cleared, not when a file hash under on_file_change changes. -var ( - jobRunTriggersPath = structpath.MustParsePath("lifecycle.triggers") - jobRunOnBundleDeployPath = structpath.MustParsePath("lifecycle.triggers.on_bundle_deploy") - jobRunOnFileChangePath = structpath.MustParsePath("lifecycle.triggers.on_file_change") -) - -func samePath(a, b *structpath.PathNode) bool { - return a.HasPrefix(b) && b.HasPrefix(a) -} - -func jobRunTriggerRemoved(path *structpath.PathNode, change *ChangeDesc) bool { - switch { - case samePath(path, jobRunOnBundleDeployPath): - return change.New == nil || change.New == "" - case samePath(path, jobRunOnFileChangePath): - return change.New == nil - case samePath(path, jobRunTriggersPath): - return change.New == nil - default: - return false - } -} - // JobRunTriggersState is the persisted fingerprint of lifecycle.triggers. type JobRunTriggersState struct { // Fresh UUID each plan while armed so Old!=New forces recreate. @@ -411,14 +386,17 @@ func reportRunLine(ctx context.Context, runID int64, msg string) { // result_state either, so the lifecycle state is what tells the two apart. // Clearing a trigger skips its local-only fingerprint without re-firing the run. func (*ResourceJobRun) OverrideChangeDesc(_ context.Context, path *structpath.PathNode, change *ChangeDesc, remote *JobRunRemote) error { - if path.HasPrefix(jobRunTriggersPath) { - if jobRunTriggerRemoved(path, change) { + switch path.String() { + case "lifecycle.triggers.on_bundle_deploy": + if change.New == nil || change.New == "" { + change.Action = deployplan.Skip + change.Reason = "trigger removed" + } + case "lifecycle.triggers.on_file_change", "lifecycle.triggers": + if change.New == nil { change.Action = deployplan.Skip change.Reason = "trigger removed" } - return nil - } - switch path.String() { case "result_state": // The planner passes no remote state when the run could not be read. if remote == nil || runIsTerminal(remote.State.LifeCycleState) { @@ -426,10 +404,8 @@ func (*ResourceJobRun) OverrideChangeDesc(_ context.Context, path *structpath.Pa } change.Action = deployplan.Skip change.Reason = "run in progress" - return nil - default: - return nil } + return nil } // DoDelete deletes the run via jobs/runs/delete, on both destroy and the diff --git a/bundle/phases/initialize.go b/bundle/phases/initialize.go index 9eba6f466e0..d2ba4e76efb 100644 --- a/bundle/phases/initialize.go +++ b/bundle/phases/initialize.go @@ -26,8 +26,7 @@ import ( func Initialize(ctx context.Context, b *bundle.Bundle) { log.Info(ctx, "Phase: initialize") - bundle.ApplySeqContext( - ctx, b, + bundle.ApplySeqContext(ctx, b, // Reads (dynamic): resource.*.* // Checks that none of resources.. is nil. Raises error otherwise. validate.AllResourcesHaveValues(), From 03c484a8911f6986b9c15c43e397426475d3fc6f Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Tue, 25 Aug 2026 09:13:00 +0000 Subject: [PATCH 24/26] job_runs: warn instead of erroring when on_file_change hashes nothing Every "nothing to hash" case re-arms once a matching file appears, now that the placeholder is always recorded, so none of them justify failing the deploy. Directories matched alongside real files are skipped silently: a glob like migrations/* routinely matches a subdirectory, and warning about it on every command was noise the user cannot act on. Also drop the unreachable "lifecycle.triggers" branch in OverrideChangeDesc (structdiff descends into the value struct and never reports a change at that path) and document the ** and sync-skipping rules in the schema. --- .../job_runs/on_file_change_errors/output.txt | 18 +-- .../job_runs/on_file_change_errors/script | 10 +- .../job_runs/on_file_change_glob/output.txt | 27 ---- .../mutator/resolve_job_run_file_triggers.go | 126 +++++++----------- bundle/direct/dresources/job_run.go | 11 +- bundle/direct/dresources/job_run_test.go | 6 +- bundle/internal/schema/annotations.yml | 2 +- bundle/schema/jsonschema.json | 2 +- 8 files changed, 68 insertions(+), 134 deletions(-) diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt index 37ea391136b..246a713e321 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt @@ -14,7 +14,7 @@ Found 1 error === a glob that only matches directories warns >>> [CLI] bundle validate -Warning: lifecycle.triggers.on_file_change: pattern "dirs/*" matches only directories +Warning: lifecycle.triggers.on_file_change: pattern "dirs/*" matches only directories or files excluded from sync, so nothing is hashed in databricks.yml:24:29 Name: job-runs-on-file-change-errors @@ -25,9 +25,9 @@ Workspace: Found 1 warning -=== a glob matching a directory and an excluded file is rejected ->>> musterr [CLI] bundle validate -Error: lifecycle.triggers.on_file_change: pattern "mixed/*" matches only files excluded from sync, leaving nothing to hash +=== a glob matching a directory and an excluded file warns +>>> [CLI] bundle validate +Warning: lifecycle.triggers.on_file_change: pattern "mixed/*" matches only directories or files excluded from sync, so nothing is hashed in databricks.yml:24:29 Name: job-runs-on-file-change-errors @@ -36,11 +36,11 @@ Workspace: User: [USERNAME] Path: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-errors/default -Found 1 error +Found 1 warning -=== a glob matching only excluded files is rejected ->>> musterr [CLI] bundle validate -Error: lifecycle.triggers.on_file_change: pattern "allexcluded/*" matches only files excluded from sync, leaving nothing to hash +=== a glob matching only excluded files warns +>>> [CLI] bundle validate +Warning: lifecycle.triggers.on_file_change: pattern "allexcluded/*" matches only directories or files excluded from sync, so nothing is hashed in databricks.yml:24:29 Name: job-runs-on-file-change-errors @@ -49,7 +49,7 @@ Workspace: User: [USERNAME] Path: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-errors/default -Found 1 error +Found 1 warning === a pattern that matches nothing warns >>> [CLI] bundle validate diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/script b/acceptance/bundle/resources/job_runs/on_file_change_errors/script index 86b10b70bf0..bca06ed87d4 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_errors/script +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/script @@ -6,15 +6,15 @@ title "a glob that only matches directories warns" update_file.py databricks.yml "../shared/*.txt" "dirs/*" trace $CLI bundle validate -title "a glob matching a directory and an excluded file is rejected" +title "a glob matching a directory and an excluded file warns" # mixed/excluded.txt is dropped by sync.exclude; mixed/nested is a directory. update_file.py databricks.yml "dirs/*" "mixed/*" -trace musterr $CLI bundle validate +trace $CLI bundle validate -title "a glob matching only excluded files is rejected" -# Every match is dropped by sync.exclude; unlike a missing pattern, this cannot re-arm later. +title "a glob matching only excluded files warns" +# Every match is dropped by sync.exclude. update_file.py databricks.yml "mixed/*" "allexcluded/*" -trace musterr $CLI bundle validate +trace $CLI bundle validate title "a pattern that matches nothing warns" update_file.py databricks.yml "allexcluded/*" "missing/*.txt" diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt index 390dda9f513..ffd878ba0cd 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt @@ -1,9 +1,6 @@ === deploy hashes the matched files >>> [CLI] bundle deploy -Warning: lifecycle.triggers.on_file_change: pattern "migrations/*" also matched directories, which are not hashed - in databricks.yml:18:29 - Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-glob/default/files... Output from job_runs.my_run: id=[MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?o=[NUMID] Output from job_runs.my_run: id=[MY_RUN_ID]: SUCCESS @@ -26,9 +23,6 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged } === editing one matched file re-fires -Warning: lifecycle.triggers.on_file_change: pattern "migrations/*" also matched directories, which are not hashed - in databricks.yml:18:29 - >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { @@ -61,9 +55,6 @@ Warning: lifecycle.triggers.on_file_change: pattern "migrations/*" also matched } === bundle deploy -Warning: lifecycle.triggers.on_file_change: pattern "migrations/*" also matched directories, which are not hashed - in databricks.yml:18:29 - Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-glob/default/files... Output from job_runs.my_run: id=[MY_RUN_ID_2]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_2]?o=[NUMID] Output from job_runs.my_run: id=[MY_RUN_ID_2]: SUCCESS @@ -91,9 +82,6 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged } === editing a file in a subdirectory re-fires -Warning: lifecycle.triggers.on_file_change: pattern "migrations/*" also matched directories, which are not hashed - in databricks.yml:18:29 - >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { @@ -126,9 +114,6 @@ Warning: lifecycle.triggers.on_file_change: pattern "migrations/*" also matched } === a new matching file re-fires -Warning: lifecycle.triggers.on_file_change: pattern "migrations/*" also matched directories, which are not hashed - in databricks.yml:18:29 - >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { @@ -161,9 +146,6 @@ Warning: lifecycle.triggers.on_file_change: pattern "migrations/*" also matched } === removing a matched file re-fires -Warning: lifecycle.triggers.on_file_change: pattern "migrations/*" also matched directories, which are not hashed - in databricks.yml:18:29 - >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { @@ -194,9 +176,6 @@ Warning: lifecycle.triggers.on_file_change: pattern "migrations/*" also matched } === editing a file below the glob does not re-fire -Warning: lifecycle.triggers.on_file_change: pattern "migrations/*" also matched directories, which are not hashed - in databricks.yml:18:29 - >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { @@ -224,15 +203,9 @@ Warning: lifecycle.triggers.on_file_change: pattern "migrations/*" also matched === editing a gitignored file does not re-fire >>> [CLI] bundle plan -Warning: lifecycle.triggers.on_file_change: pattern "migrations/*" also matched directories, which are not hashed - in databricks.yml:18:29 - Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged >>> [CLI] bundle destroy --auto-approve -Warning: lifecycle.triggers.on_file_change: pattern "migrations/*" also matched directories, which are not hashed - in databricks.yml:18:29 - The following resources will be deleted: delete resources.job_runs.my_run delete resources.jobs.my_job diff --git a/bundle/config/mutator/resolve_job_run_file_triggers.go b/bundle/config/mutator/resolve_job_run_file_triggers.go index dc615f030c8..3cfb7a42ee1 100644 --- a/bundle/config/mutator/resolve_job_run_file_triggers.go +++ b/bundle/config/mutator/resolve_job_run_file_triggers.go @@ -33,12 +33,10 @@ func (*resolveJobRunFileTriggers) Name() string { } func (*resolveJobRunFileTriggers) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - var diags diag.Diagnostics - syncable, d := syncableRelPaths(ctx, b) - if d.HasError() { - return d + syncable, diags := syncableRelPaths(ctx, b) + if diags.HasError() { + return diags } - diags = diags.Extend(d) for name, jr := range b.Config.Resources.JobRuns { if jr == nil || jr.Lifecycle == nil { continue @@ -72,25 +70,39 @@ func syncableRelPaths(ctx context.Context, b *bundle.Bundle) (map[string]struct{ return nil, diags } - fl, err := libsync.NewFileList(ctx, b.WorktreeRoot, b.SyncRoot, b.Config.Sync.Paths, b.Config.Sync.Include, b.Config.Sync.Exclude) + out, err := listSyncableRelPaths(ctx, b) if err != nil { return nil, diags.Append(diag.Diagnostic{ Severity: diag.Error, Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: list sync files: %s", err), }) } + return out, diags +} + +func listSyncableRelPaths(ctx context.Context, b *bundle.Bundle) (map[string]struct{}, error) { + fl, err := libsync.NewFileList(ctx, b.WorktreeRoot, b.SyncRoot, b.Config.Sync.Paths, b.Config.Sync.Include, b.Config.Sync.Exclude) + if err != nil { + return nil, err + } files, err := fl.Files(ctx) if err != nil { - return nil, diags.Append(diag.Diagnostic{ - Severity: diag.Error, - Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: list sync files: %s", err), - }) + return nil, err } out := make(map[string]struct{}, len(files)) for _, f := range files { out[filepath.ToSlash(f.Relative)] = struct{}{} } - return out, diags + return out, nil +} + +// fileTriggerDiag reports a diagnostic against the on_file_change entry at loc. +func fileTriggerDiag(b *bundle.Bundle, loc string, severity diag.Severity, format string, args ...any) diag.Diagnostic { + return diag.Diagnostic{ + Severity: severity, + Summary: "lifecycle.triggers.on_file_change: " + fmt.Sprintf(format, args...), + Locations: b.Config.GetLocations(loc), + } } func resolveFileTrigger(b *bundle.Bundle, loc, pattern string, syncable map[string]struct{}) (map[string]string, diag.Diagnostics) { @@ -98,20 +110,13 @@ func resolveFileTrigger(b *bundle.Bundle, loc, pattern string, syncable map[stri out := make(map[string]string) // filepath.Glob treats ** as two *, so doublestar-style patterns match less than expected. if strings.Contains(pattern, "**") { - return out, diags.Append(diag.Diagnostic{ - Severity: diag.Error, - Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: ** in %q is not supported; use * for a single directory level", pattern), - Locations: b.Config.GetLocations(loc), - }) + return out, diags.Append(fileTriggerDiag(b, loc, diag.Error, "** in %q is not supported; use * for a single directory level", pattern)) } // filepath.Join would otherwise rebase an absolute pattern under the bundle - // root (Join("/bundle", "/etc/passwd") is "/bundle/etc/passwd"). - if filepath.IsAbs(filepath.FromSlash(pattern)) || pathlib.IsAbs(pattern) { - return out, diags.Append(diag.Diagnostic{ - Severity: diag.Error, - Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: pattern %q must be relative to the defining YAML file", pattern), - Locations: b.Config.GetLocations(loc), - }) + // root (Join("/bundle", "/etc/passwd") is "/bundle/etc/passwd"). A POSIX path + // is absolute on Windows too, so check both flavours like NormalizePaths does. + if filepath.IsAbs(pattern) || pathlib.IsAbs(pattern) { + return out, diags.Append(fileTriggerDiag(b, loc, diag.Error, "pattern %q must be relative to the defining YAML file", pattern)) } // NormalizePaths has already rewritten YAML-relative globs to be bundle-root // relative. Join that onto the bundle root, then require the result stay @@ -119,90 +124,51 @@ func resolveFileTrigger(b *bundle.Bundle, loc, pattern string, syncable map[stri joined := filepath.Join(b.BundleRootPath, filepath.FromSlash(pattern)) relPattern, err := filepath.Rel(b.SyncRootPath, joined) if err != nil || !filepath.IsLocal(relPattern) { - return out, diags.Append(diag.Diagnostic{ - Severity: diag.Error, - Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: pattern %q is not under the sync root", pattern), - Locations: b.Config.GetLocations(loc), - }) + return out, diags.Append(fileTriggerDiag(b, loc, diag.Error, "pattern %q is not under the sync root", pattern)) } - key := filepath.ToSlash(relPattern) matches, err := filepath.Glob(joined) if err != nil { - return out, diags.Append(diag.Diagnostic{ - Severity: diag.Error, - Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: invalid pattern %q: %s", pattern, err), - Locations: b.Config.GetLocations(loc), - }) - } - if len(matches) == 0 { - out[key] = missingFileHash - return out, diags.Append(diag.Diagnostic{ - Severity: diag.Warning, - Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: no files match %q", pattern), - Locations: b.Config.GetLocations(loc), - }) + return out, diags.Append(fileTriggerDiag(b, loc, diag.Error, "invalid pattern %q: %s", pattern, err)) } - regularMatches := 0 - ignoredMatches := 0 - sawNonRegular := false for _, match := range matches { info, err := os.Stat(match) if err != nil { - diags = diags.Append(diag.Diagnostic{ - Severity: diag.Error, - Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: stat %q: %s", match, err), - Locations: b.Config.GetLocations(loc), - }) + diags = diags.Append(fileTriggerDiag(b, loc, diag.Error, "stat %q: %s", match, err)) continue } + // A glob like migrations/* routinely matches subdirectories; there is + // nothing to hash and nothing for the user to fix, so skip them quietly. if !info.Mode().IsRegular() { - sawNonRegular = true continue } rel, err := filepath.Rel(b.SyncRootPath, match) if err != nil || !filepath.IsLocal(rel) { - diags = diags.Append(diag.Diagnostic{ - Severity: diag.Error, - Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: matched path %q is not under the sync root", match), - Locations: b.Config.GetLocations(loc), - }) + diags = diags.Append(fileTriggerDiag(b, loc, diag.Error, "matched path %q is not under the sync root", match)) continue } // Honor .gitignore and sync.exclude the same way sync does. if _, ok := syncable[filepath.ToSlash(rel)]; !ok { - ignoredMatches++ continue } - regularMatches++ hash, err := hashFile(match) if err != nil { - diags = diags.Append(diag.Diagnostic{ - Severity: diag.Error, - Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: hash %q: %s", match, err), - Locations: b.Config.GetLocations(loc), - }) + diags = diags.Append(fileTriggerDiag(b, loc, diag.Error, "hash %q: %s", match, err)) continue } out[filepath.ToSlash(rel)] = hash } - if regularMatches == 0 && ignoredMatches > 0 { - return out, diags.Append(diag.Diagnostic{ - Severity: diag.Error, - Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: pattern %q matches only files excluded from sync, leaving nothing to hash", pattern), - Locations: b.Config.GetLocations(loc), - }) - } - if sawNonRegular { - msg := fmt.Sprintf("lifecycle.triggers.on_file_change: pattern %q also matched directories, which are not hashed", pattern) - if regularMatches == 0 { - out[key] = missingFileHash - msg = fmt.Sprintf("lifecycle.triggers.on_file_change: pattern %q matches only directories", pattern) + // A pattern that hashes nothing is a warning, not an error: every such case + // re-arms once a matching file appears. Record the placeholder under the + // pattern's own sync-root-relative key so that appearance is a hash change + // rather than a key swap. Skip it when a match failed to be read, since the + // error already says the fingerprint is incomplete. + if len(out) == 0 && !diags.HasError() { + out[filepath.ToSlash(relPattern)] = missingFileHash + if len(matches) == 0 { + diags = diags.Append(fileTriggerDiag(b, loc, diag.Warning, "no files match %q", pattern)) + } else { + diags = diags.Append(fileTriggerDiag(b, loc, diag.Warning, "pattern %q matches only directories or files excluded from sync, so nothing is hashed", pattern)) } - diags = diags.Append(diag.Diagnostic{ - Severity: diag.Warning, - Summary: msg, - Locations: b.Config.GetLocations(loc), - }) } return out, diags } diff --git a/bundle/direct/dresources/job_run.go b/bundle/direct/dresources/job_run.go index d7574bdc2ac..078ae1b1aa6 100644 --- a/bundle/direct/dresources/job_run.go +++ b/bundle/direct/dresources/job_run.go @@ -41,14 +41,9 @@ type JobRunLifecycleState struct { Triggers JobRunTriggersState `json:"triggers"` } -// Zero value spelled out field by field, as exhaustruct requires. func emptyJobRunLifecycleState() JobRunLifecycleState { - return JobRunLifecycleState{ - Triggers: JobRunTriggersState{ - OnBundleDeploy: "", - OnFileChange: nil, - }, - } + var empty JobRunLifecycleState + return empty } // JobRunState is the RunNow request plus the outcome required for planning. @@ -392,7 +387,7 @@ func (*ResourceJobRun) OverrideChangeDesc(_ context.Context, path *structpath.Pa change.Action = deployplan.Skip change.Reason = "trigger removed" } - case "lifecycle.triggers.on_file_change", "lifecycle.triggers": + case "lifecycle.triggers.on_file_change": if change.New == nil { change.Action = deployplan.Skip change.Reason = "trigger removed" diff --git a/bundle/direct/dresources/job_run_test.go b/bundle/direct/dresources/job_run_test.go index 68738b93e18..f9f21999316 100644 --- a/bundle/direct/dresources/job_run_test.go +++ b/bundle/direct/dresources/job_run_test.go @@ -382,10 +382,10 @@ func TestJobRunOverrideChangeDescTriggerRemoved(t *testing.T) { {"rotated on_bundle_deploy", "lifecycle.triggers.on_bundle_deploy", "uuid", deployplan.Recreate}, {"cleared on_file_change", "lifecycle.triggers.on_file_change", nil, deployplan.Skip}, {"changed on_file_change map", "lifecycle.triggers.on_file_change", map[string]string{"a.txt": "h"}, deployplan.Recreate}, + // A file dropping out of the map is a real change, so the skip must not + // extend to paths below on_file_change. {"cleared on_file_change child", "lifecycle.triggers.on_file_change['a.txt']", nil, deployplan.Recreate}, - {"cleared triggers parent", "lifecycle.triggers", nil, deployplan.Skip}, - {"changed triggers parent", "lifecycle.triggers", JobRunTriggersState{OnBundleDeploy: "uuid"}, deployplan.Recreate}, - {"result_state unchanged", "result_state", nil, deployplan.Recreate}, + {"result_state with unreadable remote", "result_state", nil, deployplan.Recreate}, } { t.Run(tt.name, func(t *testing.T) { change := &ChangeDesc{Action: deployplan.Recreate, New: tt.new} diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index c3e76eb7b8d..8b17164066d 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -1019,7 +1019,7 @@ resources: If true, re-fire the run on every bundle deploy. Incompatible with lifecycle.prevent_destroy. "on_file_change": "description": |- - Path or glob relative to the defining YAML file. It must resolve under the sync root. Re-fire the run when a matched file's content hash changes, or when the set of matches appears or disappears. Incompatible with lifecycle.prevent_destroy. + Path or glob relative to the defining YAML file. It must resolve under the sync root. Re-fire the run when a matched file's content hash changes, or when the set of matches appears or disappears. Only files the bundle syncs are hashed, so .gitignore and sync.exclude apply. Use * to match a single directory level; ** is not supported. Incompatible with lifecycle.prevent_destroy. "python_named_params": "description": |- PLACEHOLDER diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 3829ae5c406..593447629ba 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -1396,7 +1396,7 @@ "$ref": "#/$defs/bool" }, "on_file_change": { - "description": "Path or glob relative to the defining YAML file. It must resolve under the sync root. Re-fire the run when a matched file's content hash changes, or when the set of matches appears or disappears. Incompatible with lifecycle.prevent_destroy.", + "description": "Path or glob relative to the defining YAML file. It must resolve under the sync root. Re-fire the run when a matched file's content hash changes, or when the set of matches appears or disappears. Only files the bundle syncs are hashed, so .gitignore and sync.exclude apply. Use * to match a single directory level; ** is not supported. Incompatible with lifecycle.prevent_destroy.", "$ref": "#/$defs/string" } }, From c433182c3a88e6cd001ef5c6ee2cbb344f17a0cd Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Tue, 25 Aug 2026 09:40:05 +0000 Subject: [PATCH 25/26] acc: stop Git Bash rewriting the on_file_change absolute pattern Windows CI converted /tmp/seed.txt to %TEMP%/seed.txt, so the golden expecting the POSIX path failed. MSYS_NO_PATHCONV keeps the argument intact. --- .../bundle/resources/job_runs/on_file_change_errors/test.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/test.toml b/acceptance/bundle/resources/job_runs/on_file_change_errors/test.toml index 7b88d613d4c..af98ba376da 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_errors/test.toml +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/test.toml @@ -1,2 +1,5 @@ +# Git Bash otherwise rewrites /tmp/seed.txt to %TEMP%/seed.txt. +Env.MSYS_NO_PATHCONV = "1" + # The bundle is never deployed here, so there are no requests worth recording. RecordRequests = false From b0437a39e860e14d72e8b61df2d22d49a26ca445 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Tue, 25 Aug 2026 10:09:43 +0000 Subject: [PATCH 26/26] acc: read the absolute on_file_change pattern from a committed file MSYS_NO_PATHCONV also stops Git Bash converting the path of a shebang helper, so update_file.py could not be found on Windows. Keep the leading-slash pattern out of argv entirely instead. --- .../on_file_change_errors/absolute_pattern.yml | 18 ++++++++++++++++++ .../job_runs/on_file_change_errors/output.txt | 2 +- .../job_runs/on_file_change_errors/script | 4 +++- .../job_runs/on_file_change_errors/test.toml | 3 --- 4 files changed, 22 insertions(+), 5 deletions(-) create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_errors/absolute_pattern.yml diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/absolute_pattern.yml b/acceptance/bundle/resources/job_runs/on_file_change_errors/absolute_pattern.yml new file mode 100644 index 00000000000..e0d709d3fac --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/absolute_pattern.yml @@ -0,0 +1,18 @@ +bundle: + name: job-runs-on-file-change-errors + +resources: + jobs: + my_job: + name: my-job + tasks: + - task_key: main + notebook_task: + notebook_path: /Workspace/test + + job_runs: + my_run: + job_id: ${resources.jobs.my_job.id} + lifecycle: + triggers: + - on_file_change: /tmp/seed.txt diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt index 246a713e321..efcd8f002b4 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt @@ -80,7 +80,7 @@ Found 1 error === an absolute pattern is rejected >>> musterr [CLI] bundle validate Error: lifecycle.triggers.on_file_change: pattern "/tmp/seed.txt" must be relative to the defining YAML file - in databricks.yml:24:29 + in databricks.yml:18:29 Name: job-runs-on-file-change-errors Target: default diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/script b/acceptance/bundle/resources/job_runs/on_file_change_errors/script index bca06ed87d4..faaa5de8d3c 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_errors/script +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/script @@ -25,5 +25,7 @@ update_file.py databricks.yml "missing/*.txt" "src/**" trace musterr $CLI bundle validate title "an absolute pattern is rejected" -update_file.py databricks.yml "src/**" "/tmp/seed.txt" +# Copied from a committed file rather than passed to update_file.py: Git Bash on +# Windows rewrites a leading-slash argument into a Windows path. +cp absolute_pattern.yml databricks.yml trace musterr $CLI bundle validate diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/test.toml b/acceptance/bundle/resources/job_runs/on_file_change_errors/test.toml index af98ba376da..7b88d613d4c 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_errors/test.toml +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/test.toml @@ -1,5 +1,2 @@ -# Git Bash otherwise rewrites /tmp/seed.txt to %TEMP%/seed.txt. -Env.MSYS_NO_PATHCONV = "1" - # The bundle is never deployed here, so there are no requests worth recording. RecordRequests = false