From be0eca098ce840163341b3d3a832d64e4af13b68 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Tue, 18 Aug 2026 12:25:26 +0000 Subject: [PATCH 01/75] job_runs: add lifecycle.triggers.on_file_change Re-fire a run when matched file contents change, using stable hash fingerprints so mtime-only updates (touch) do not recreate. --- .../bundles/job-runs-on-file-change.md | 1 + .../job_runs/on_file_change/databricks.yml | 18 ++ .../job_runs/on_file_change/out.test.toml | 2 + .../job_runs/on_file_change/output.txt | 90 +++++++++ .../resources/job_runs/on_file_change/script | 34 ++++ .../job_runs/on_file_change/seed.txt | 1 + .../mutator/resolve_job_run_file_triggers.go | 177 ++++++++++++++++++ ...lve_job_run_file_triggers_internal_test.go | 73 ++++++++ .../resolve_job_run_file_triggers_test.go | 143 ++++++++++++++ .../mutator/validate_job_run_triggers.go | 26 ++- .../mutator/validate_job_run_triggers_test.go | 50 ++++- bundle/config/resources/job_run.go | 29 +++ bundle/config/resources/lifecycle.go | 9 +- bundle/direct/dresources/job_run.go | 31 ++- bundle/direct/dresources/job_run_test.go | 76 ++++++++ bundle/direct/dresources/resources.yml | 6 + bundle/internal/schema/annotations.yml | 3 + bundle/phases/initialize.go | 6 +- bundle/phases/plan.go | 4 +- bundle/schema/jsonschema.json | 4 + 20 files changed, 767 insertions(+), 16 deletions(-) create mode 100644 .nextchanges/bundles/job-runs-on-file-change.md create mode 100644 acceptance/bundle/resources/job_runs/on_file_change/databricks.yml create mode 100644 acceptance/bundle/resources/job_runs/on_file_change/out.test.toml create mode 100644 acceptance/bundle/resources/job_runs/on_file_change/output.txt create mode 100644 acceptance/bundle/resources/job_runs/on_file_change/script create mode 100644 acceptance/bundle/resources/job_runs/on_file_change/seed.txt create mode 100644 bundle/config/mutator/resolve_job_run_file_triggers.go create mode 100644 bundle/config/mutator/resolve_job_run_file_triggers_internal_test.go create mode 100644 bundle/config/mutator/resolve_job_run_file_triggers_test.go diff --git a/.nextchanges/bundles/job-runs-on-file-change.md b/.nextchanges/bundles/job-runs-on-file-change.md new file mode 100644 index 00000000000..00682403a0d --- /dev/null +++ b/.nextchanges/bundles/job-runs-on-file-change.md @@ -0,0 +1 @@ +direct: `resources.job_runs` can set `lifecycle.triggers.on_file_change` to a path or glob to re-fire the run when matched file contents change. Removing the trigger does not recreate the existing run. diff --git a/acceptance/bundle/resources/job_runs/on_file_change/databricks.yml b/acceptance/bundle/resources/job_runs/on_file_change/databricks.yml new file mode 100644 index 00000000000..0a6df75047a --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change/databricks.yml @@ -0,0 +1,18 @@ +bundle: + name: job-runs-on-file-change + +resources: + jobs: + my_job: + name: my-job + tasks: + - task_key: main + notebook_task: + notebook_path: /Workspace/test + + job_runs: + my_run: + job_id: ${resources.jobs.my_job.id} + lifecycle: + triggers: + - on_file_change: seed.txt diff --git a/acceptance/bundle/resources/job_runs/on_file_change/out.test.toml b/acceptance/bundle/resources/job_runs/on_file_change/out.test.toml new file mode 100644 index 00000000000..0938e678987 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/job_runs/on_file_change/output.txt b/acceptance/bundle/resources/job_runs/on_file_change/output.txt new file mode 100644 index 00000000000..bcf5962f8fa --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change/output.txt @@ -0,0 +1,90 @@ + +=== first deploy triggers a run +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change/default/files... +job run [MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?o=[NUMID] +job run [MY_RUN_ID]: SUCCESS +Created job_runs.my_run +Created jobs.my_job +Files: 5 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +>>> read_id.py my_job +[MY_JOB_ID] + +>>> print_requests.py //jobs/run-now +{ + "method": "POST", + "path": "/api/2.2/jobs/run-now", + "body": { + "idempotency_token": "[UUID]", + "job_id": [MY_JOB_ID] + } +} + +=== redeploy with unchanged file plans nothing +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change/default/files... +Files: 2 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 2 unchanged + +>>> print_requests.py //jobs/run-now + +=== editing the file re-fires +>>> [CLI] bundle plan +recreate job_runs.my_run + +Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change/default/files... +job run [MY_RUN_ID_2]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_2]?o=[NUMID] +job run [MY_RUN_ID_2]: SUCCESS +Recreated job_runs.my_run +Files: 3 uploaded, 0 deleted +Resources: 1 created, 0 changed, 1 deleted, 1 unchanged + +>>> print_requests.py --keep //jobs/runs/delete +{ + "method": "POST", + "path": "/api/2.2/jobs/runs/delete", + "body": { + "run_id": [MY_RUN_ID] + } +} + +>>> print_requests.py //jobs/run-now +{ + "method": "POST", + "path": "/api/2.2/jobs/run-now", + "body": { + "idempotency_token": "[UUID]", + "job_id": [MY_JOB_ID] + } +} + +=== removing on_file_change rewrites state without a run +>>> [CLI] bundle plan +update job_runs.my_run + +Plan: 0 to add, 1 to change, 0 to delete, 1 unchanged + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change/default/files... +Updated job_runs.my_run +Files: 3 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 1 unchanged + +>>> print_requests.py //jobs/run-now + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.job_runs.my_run + delete resources.jobs.my_job + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change/default + +Destroy: 2 deleted diff --git a/acceptance/bundle/resources/job_runs/on_file_change/script b/acceptance/bundle/resources/job_runs/on_file_change/script new file mode 100644 index 00000000000..ff162e11f0b --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change/script @@ -0,0 +1,34 @@ +cleanup() { + trace $CLI bundle destroy --auto-approve + rm -f out.requests.txt +} +trap cleanup EXIT + +title "first deploy triggers a run" +trace $CLI bundle deploy +trace read_id.py my_job +# Name the first run so the recreated one becomes [MY_RUN_ID_2]. +read_id.py my_run > /dev/null +trace print_requests.py //jobs/run-now + +title "redeploy with unchanged file plans nothing" +trace $CLI bundle plan +trace $CLI bundle deploy +trace print_requests.py //jobs/run-now + +title "editing the file re-fires" +update_file.py seed.txt "v1" "v2" +trace $CLI bundle plan +trace $CLI bundle deploy +read_id.py my_run > /dev/null +trace print_requests.py --keep //jobs/runs/delete +trace print_requests.py //jobs/run-now + +title "removing on_file_change rewrites state without a run" +update_file.py databricks.yml " lifecycle: + triggers: + - on_file_change: seed.txt +" "" +trace $CLI bundle plan +trace $CLI bundle deploy +trace print_requests.py //jobs/run-now diff --git a/acceptance/bundle/resources/job_runs/on_file_change/seed.txt b/acceptance/bundle/resources/job_runs/on_file_change/seed.txt new file mode 100644 index 00000000000..626799f0f85 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change/seed.txt @@ -0,0 +1 @@ +v1 diff --git a/bundle/config/mutator/resolve_job_run_file_triggers.go b/bundle/config/mutator/resolve_job_run_file_triggers.go new file mode 100644 index 00000000000..a82ed654be3 --- /dev/null +++ b/bundle/config/mutator/resolve_job_run_file_triggers.go @@ -0,0 +1,177 @@ +package mutator + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "os" + "path/filepath" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/libs/diag" +) + +// missingFileSentinelSize marks a pattern with no matching file so the next +// plan can distinguish "still missing" from "file appeared". +const missingFileSentinelSize = int64(-1) + +type resolveJobRunFileTriggers struct{} + +// ResolveJobRunFileTriggers expands on_file_change globs and stores per-file +// fingerprints on each job_run for PrepareState to copy into local state. +func ResolveJobRunFileTriggers() bundle.Mutator { + return &resolveJobRunFileTriggers{} +} + +func (*resolveJobRunFileTriggers) Name() string { + return "ResolveJobRunFileTriggers" +} + +func (*resolveJobRunFileTriggers) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { + var diags diag.Diagnostics + for name, jr := range b.Config.Resources.JobRuns { + if jr == nil { + continue + } + patterns := jr.OnFileChangePatterns() + if len(patterns) == 0 { + continue + } + fps, d := resolveFileTriggers(b, name, patterns, previousFileTriggers(b, name)) + diags = diags.Extend(d) + jr.ResolvedFileTriggers = fps + } + return diags +} + +// previousFileTriggers reads on_file_change fingerprints from deployment state +// when it is open (plan/deploy after StatePull). Used so unchanged content keeps +// a stable fingerprint across mtime-only updates (e.g. touch). +func previousFileTriggers(b *bundle.Bundle, name string) map[string]resources.JobRunFileFingerprint { + if b.DeploymentBundle.StateDB.Path == "" { + return nil + } + entry, ok := b.DeploymentBundle.StateDB.GetResourceEntry("resources.job_runs." + name) + if !ok || len(entry.State) == 0 { + return nil + } + var state struct { + Lifecycle *struct { + Triggers *struct { + OnFileChange map[string]resources.JobRunFileFingerprint `json:"on_file_change"` + } `json:"triggers"` + } `json:"lifecycle"` + } + if err := json.Unmarshal(entry.State, &state); err != nil { + return nil + } + if state.Lifecycle == nil || state.Lifecycle.Triggers == nil { + return nil + } + return state.Lifecycle.Triggers.OnFileChange +} + +func resolveFileTriggers(b *bundle.Bundle, name string, patterns []string, prev map[string]resources.JobRunFileFingerprint) (map[string]resources.JobRunFileFingerprint, diag.Diagnostics) { + var diags diag.Diagnostics + out := make(map[string]resources.JobRunFileFingerprint) + for _, pattern := range patterns { + path := fmt.Sprintf("resources.job_runs.%s.lifecycle.triggers", name) + matches, err := filepath.Glob(filepath.Join(b.SyncRootPath, filepath.FromSlash(pattern))) + if err != nil { + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: invalid pattern %q: %s", pattern, err), + Locations: b.Config.GetLocations(path), + }) + continue + } + if len(matches) == 0 { + // Distinct state when the path/glob matches nothing (design doc). + out[filepath.ToSlash(pattern)] = resources.JobRunFileFingerprint{ + Size: missingFileSentinelSize, + } + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Warning, + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: no files match %q", pattern), + Locations: b.Config.GetLocations(path), + }) + continue + } + for _, match := range matches { + info, err := os.Stat(match) + if err != nil { + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: stat %q: %s", match, err), + Locations: b.Config.GetLocations(path), + }) + continue + } + if !info.Mode().IsRegular() { + continue + } + rel, err := filepath.Rel(b.SyncRootPath, match) + if err != nil { + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: relative path for %q: %s", match, err), + Locations: b.Config.GetLocations(path), + }) + continue + } + key := filepath.ToSlash(rel) + fp, err := fingerprintFile(match, info, prev[key]) + if err != nil { + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: hash %q: %s", match, err), + Locations: b.Config.GetLocations(path), + }) + continue + } + out[key] = fp + } + } + return out, diags +} + +// fingerprintFile builds a content fingerprint. If prev has the same size and +// mtime, it is reused without reading the file. If content hash matches prev, +// prev is reused so mtime-only updates (touch) do not change planned state. +func fingerprintFile(path string, info os.FileInfo, prev resources.JobRunFileFingerprint) (resources.JobRunFileFingerprint, error) { + size := info.Size() + mtime := info.ModTime().UnixNano() + if prev.Hash != "" && prev.Size == size && prev.MtimeNs == mtime { + return prev, nil + } + hash, err := hashFile(path) + if err != nil { + return resources.JobRunFileFingerprint{}, err + } + if prev.Hash != "" && prev.Hash == hash { + return prev, nil + } + return resources.JobRunFileFingerprint{ + Hash: hash, + Size: size, + MtimeNs: mtime, + }, nil +} + +func hashFile(path string) (string, error) { + f, err := os.Open(path) + if err != nil { + return "", err + } + defer f.Close() + + h := sha256.New() + if _, err := io.Copy(h, f); err != nil { + return "", err + } + return hex.EncodeToString(h.Sum(nil)), nil +} diff --git a/bundle/config/mutator/resolve_job_run_file_triggers_internal_test.go b/bundle/config/mutator/resolve_job_run_file_triggers_internal_test.go new file mode 100644 index 00000000000..e82bd22945b --- /dev/null +++ b/bundle/config/mutator/resolve_job_run_file_triggers_internal_test.go @@ -0,0 +1,73 @@ +package mutator + +import ( + "os" + "path/filepath" + "testing" + "time" + + "github.com/databricks/cli/bundle/config/resources" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestFingerprintFileContentStableAcrossTouch(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "seed.txt") + require.NoError(t, os.WriteFile(path, []byte("v1"), 0o644)) + + info, err := os.Stat(path) + require.NoError(t, err) + first, err := fingerprintFile(path, info, resources.JobRunFileFingerprint{}) + require.NoError(t, err) + require.NotEmpty(t, first.Hash) + + // Advance mtime without changing contents (touch). + require.NoError(t, os.Chtimes(path, time.Now().Add(time.Minute), time.Now().Add(time.Minute))) + info, err = os.Stat(path) + require.NoError(t, err) + assert.NotEqual(t, first.MtimeNs, info.ModTime().UnixNano()) + + second, err := fingerprintFile(path, info, first) + require.NoError(t, err) + assert.Equal(t, first, second, "unchanged content must reuse the previous fingerprint") +} + +func TestFingerprintFileFastPathSkipsWhenMtimeAndSizeMatch(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "seed.txt") + require.NoError(t, os.WriteFile(path, []byte("v1"), 0o644)) + + info, err := os.Stat(path) + require.NoError(t, err) + prev := resources.JobRunFileFingerprint{ + Hash: "not-the-real-hash", + Size: info.Size(), + MtimeNs: info.ModTime().UnixNano(), + } + + got, err := fingerprintFile(path, info, prev) + require.NoError(t, err) + assert.Equal(t, prev, got, "matching size+mtime must reuse prev without re-hashing") +} + +func TestFingerprintFileContentChange(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "seed.txt") + require.NoError(t, os.WriteFile(path, []byte("v1"), 0o644)) + + info, err := os.Stat(path) + require.NoError(t, err) + first, err := fingerprintFile(path, info, resources.JobRunFileFingerprint{}) + require.NoError(t, err) + + // Different size so the mtime+size fast path cannot reuse prev. + require.NoError(t, os.WriteFile(path, []byte("v2-changed"), 0o644)) + info, err = os.Stat(path) + require.NoError(t, err) + second, err := fingerprintFile(path, info, first) + require.NoError(t, err) + assert.NotEqual(t, first.Hash, second.Hash) + assert.Equal(t, info.Size(), second.Size) + assert.Equal(t, info.ModTime().UnixNano(), second.MtimeNs) +} diff --git a/bundle/config/mutator/resolve_job_run_file_triggers_test.go b/bundle/config/mutator/resolve_job_run_file_triggers_test.go new file mode 100644 index 00000000000..9a9cf77bf53 --- /dev/null +++ b/bundle/config/mutator/resolve_job_run_file_triggers_test.go @@ -0,0 +1,143 @@ +package mutator_test + +import ( + "crypto/sha256" + "encoding/hex" + "os" + "path/filepath" + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/mutator" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/libs/diag" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestResolveJobRunFileTriggers(t *testing.T) { + t.Run("matches files and fills fingerprints", func(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "a.txt"), []byte("hello"), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "b.txt"), []byte("world"), 0o644)) + + pattern := "*.txt" + b := bundleWithFileTrigger(dir, pattern) + + diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) + require.False(t, diags.HasError()) + + fps := b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers + require.Len(t, fps, 2) + + assertFingerprint(t, fps["a.txt"], "hello") + assertFingerprint(t, fps["b.txt"], "world") + }) + + t.Run("no matches warns and stores sentinel", func(t *testing.T) { + dir := t.TempDir() + pattern := "missing.txt" + b := bundleWithFileTrigger(dir, pattern) + + diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) + require.False(t, diags.HasError()) + require.Len(t, diags, 1) + assert.Equal(t, diag.Warning, diags[0].Severity) + assert.Contains(t, diags[0].Summary, `no files match "missing.txt"`) + + fps := b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers + require.Len(t, fps, 1) + fp := fps["missing.txt"] + assert.Empty(t, fp.Hash) + assert.Equal(t, int64(-1), fp.Size) + assert.Zero(t, fp.MtimeNs) + }) + + t.Run("no file triggers is a no-op", func(t *testing.T) { + dir := t.TempDir() + on := true + b := &bundle.Bundle{ + SyncRootPath: dir, + Config: config.Root{ + Resources: config.Resources{ + JobRuns: map[string]*resources.JobRun{ + "my_run": { + Lifecycle: &resources.JobRunLifecycle{ + Triggers: []resources.JobRunTrigger{ + {OnBundleDeploy: &on}, + }, + }, + }, + }, + }, + }, + } + + diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) + assert.Empty(t, diags) + assert.Nil(t, b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers) + }) + + t.Run("multiple patterns merge into one map", func(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "a.txt"), []byte("aaa"), 0o644)) + require.NoError(t, os.MkdirAll(filepath.Join(dir, "subdir"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "subdir", "x.py"), []byte("bbb"), 0o644)) + + patA := "a.txt" + patB := "subdir/*.py" + b := &bundle.Bundle{ + SyncRootPath: dir, + Config: config.Root{ + Resources: config.Resources{ + JobRuns: map[string]*resources.JobRun{ + "my_run": { + Lifecycle: &resources.JobRunLifecycle{ + Triggers: []resources.JobRunTrigger{ + {OnFileChange: &patA}, + {OnFileChange: &patB}, + }, + }, + }, + }, + }, + }, + } + + diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) + require.False(t, diags.HasError()) + + fps := b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers + require.Len(t, fps, 2) + assertFingerprint(t, fps["a.txt"], "aaa") + assertFingerprint(t, fps["subdir/x.py"], "bbb") + }) +} + +func bundleWithFileTrigger(syncRoot, pattern string) *bundle.Bundle { + return &bundle.Bundle{ + SyncRootPath: syncRoot, + Config: config.Root{ + Resources: config.Resources{ + JobRuns: map[string]*resources.JobRun{ + "my_run": { + Lifecycle: &resources.JobRunLifecycle{ + Triggers: []resources.JobRunTrigger{ + {OnFileChange: &pattern}, + }, + }, + }, + }, + }, + }, + } +} + +func assertFingerprint(t *testing.T, fp resources.JobRunFileFingerprint, content string) { + t.Helper() + sum := sha256.Sum256([]byte(content)) + assert.Equal(t, hex.EncodeToString(sum[:]), fp.Hash) + assert.Equal(t, int64(len(content)), fp.Size) + assert.NotZero(t, fp.MtimeNs) +} diff --git a/bundle/config/mutator/validate_job_run_triggers.go b/bundle/config/mutator/validate_job_run_triggers.go index a7c5ada925e..c185d3b5781 100644 --- a/bundle/config/mutator/validate_job_run_triggers.go +++ b/bundle/config/mutator/validate_job_run_triggers.go @@ -3,6 +3,7 @@ package mutator import ( "context" "fmt" + "strings" "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" @@ -26,30 +27,45 @@ func (*validateJobRunTriggers) Apply(_ context.Context, b *bundle.Bundle) diag.D continue } // Recreate-every-deploy cannot coexist with prevent_destroy. - if jr.HasOnBundleDeploy() && jr.Lifecycle.PreventDestroy { + if (jr.HasOnBundleDeploy() || jr.HasOnFileChange()) && jr.Lifecycle.PreventDestroy { diags = diags.Append(diag.Diagnostic{ Severity: diag.Error, - Summary: "lifecycle.triggers.on_bundle_deploy is incompatible with lifecycle.prevent_destroy", + Summary: "lifecycle.triggers.on_bundle_deploy or on_file_change is incompatible with lifecycle.prevent_destroy", Locations: b.Config.GetLocations(fmt.Sprintf("resources.job_runs.%s.lifecycle", name)), }) } for i, t := range jr.Lifecycle.Triggers { path := fmt.Sprintf("resources.job_runs.%s.lifecycle.triggers[%d]", name, i) - if t.OnBundleDeploy == nil { + if t.OnBundleDeploy == nil && t.OnFileChange == nil { diags = diags.Append(diag.Diagnostic{ Severity: diag.Error, - Summary: "lifecycle.triggers entry must set on_bundle_deploy: true", + Summary: "lifecycle.triggers entry must set on_bundle_deploy or on_file_change", Locations: b.Config.GetLocations(path), }) continue } - if !*t.OnBundleDeploy { + if t.OnBundleDeploy != nil && t.OnFileChange != nil { + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: "lifecycle.triggers entry must set only one of on_bundle_deploy or on_file_change", + Locations: b.Config.GetLocations(path), + }) + continue + } + if t.OnBundleDeploy != nil && !*t.OnBundleDeploy { diags = diags.Append(diag.Diagnostic{ Severity: diag.Error, Summary: "lifecycle.triggers.on_bundle_deploy must be true when set", Locations: b.Config.GetLocations(path + ".on_bundle_deploy"), }) } + if t.OnFileChange != nil && strings.TrimSpace(*t.OnFileChange) == "" { + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: "lifecycle.triggers.on_file_change must be non-empty when set", + Locations: b.Config.GetLocations(path + ".on_file_change"), + }) + } } } return diags diff --git a/bundle/config/mutator/validate_job_run_triggers_test.go b/bundle/config/mutator/validate_job_run_triggers_test.go index 9c225457471..7e8772d8d8b 100644 --- a/bundle/config/mutator/validate_job_run_triggers_test.go +++ b/bundle/config/mutator/validate_job_run_triggers_test.go @@ -14,6 +14,10 @@ func TestValidateJobRunTriggers(t *testing.T) { trueVal := true falseVal := false + fileChange := "seed.txt" + emptyFile := "" + whitespaceFile := " \t" + tests := []struct { name string triggers []resources.JobRunTrigger @@ -26,12 +30,32 @@ func TestValidateJobRunTriggers(t *testing.T) { {OnBundleDeploy: &trueVal}, }, }, + { + name: "on_file_change set", + triggers: []resources.JobRunTrigger{ + {OnFileChange: &fileChange}, + }, + }, + { + name: "both triggers as separate entries", + triggers: []resources.JobRunTrigger{ + {OnFileChange: &fileChange}, + {OnBundleDeploy: &trueVal}, + }, + }, { name: "empty entry", triggers: []resources.JobRunTrigger{ {}, }, - summary: "lifecycle.triggers entry must set on_bundle_deploy: true", + summary: "lifecycle.triggers entry must set on_bundle_deploy or on_file_change", + }, + { + name: "both keys on one entry", + triggers: []resources.JobRunTrigger{ + {OnBundleDeploy: &trueVal, OnFileChange: &fileChange}, + }, + summary: "lifecycle.triggers entry must set only one of on_bundle_deploy or on_file_change", }, { name: "on_bundle_deploy false", @@ -40,13 +64,35 @@ func TestValidateJobRunTriggers(t *testing.T) { }, summary: "lifecycle.triggers.on_bundle_deploy must be true when set", }, + { + name: "on_file_change empty", + triggers: []resources.JobRunTrigger{ + {OnFileChange: &emptyFile}, + }, + summary: "lifecycle.triggers.on_file_change must be non-empty when set", + }, + { + name: "on_file_change whitespace", + triggers: []resources.JobRunTrigger{ + {OnFileChange: &whitespaceFile}, + }, + summary: "lifecycle.triggers.on_file_change must be non-empty when set", + }, { name: "on_bundle_deploy with prevent_destroy", triggers: []resources.JobRunTrigger{ {OnBundleDeploy: &trueVal}, }, preventDestroy: true, - summary: "lifecycle.triggers.on_bundle_deploy is incompatible with lifecycle.prevent_destroy", + summary: "lifecycle.triggers.on_bundle_deploy or on_file_change is incompatible with lifecycle.prevent_destroy", + }, + { + name: "on_file_change with prevent_destroy", + triggers: []resources.JobRunTrigger{ + {OnFileChange: &fileChange}, + }, + preventDestroy: true, + summary: "lifecycle.triggers.on_bundle_deploy or on_file_change is incompatible with lifecycle.prevent_destroy", }, { name: "prevent_destroy alone", diff --git a/bundle/config/resources/job_run.go b/bundle/config/resources/job_run.go index 56f3c39a356..cf66c71f728 100644 --- a/bundle/config/resources/job_run.go +++ b/bundle/config/resources/job_run.go @@ -27,6 +27,10 @@ type JobRun struct { // the run URL. Keeping it separate from RunNow.JobId (a ${resources.jobs.*.id} // reference) lets state loading preserve that reference and its plan dependency. ResolvedJobID int64 `json:"resolved_job_id,omitempty" bundle:"internal"` + + // ResolvedFileTriggers is the per-file fingerprint map for on_file_change, + // computed under SyncRoot before plan. bundle:"internal" keeps it out of schema. + ResolvedFileTriggers map[string]JobRunFileFingerprint `json:"resolved_file_triggers,omitempty" bundle:"internal"` } // HasOnBundleDeploy reports whether any trigger re-fires on every deploy. @@ -42,6 +46,31 @@ func (r *JobRun) HasOnBundleDeploy() bool { return false } +func (r *JobRun) HasOnFileChange() bool { + if r.Lifecycle == nil { + return false + } + for _, t := range r.Lifecycle.Triggers { + if t.OnFileChange != nil { + return true + } + } + return false +} + +func (r *JobRun) OnFileChangePatterns() []string { + if r.Lifecycle == nil { + return nil + } + var patterns []string + for _, t := range r.Lifecycle.Triggers { + if t.OnFileChange != nil { + patterns = append(patterns, *t.OnFileChange) + } + } + return patterns +} + func (r *JobRun) UnmarshalJSON(b []byte) error { return marshal.Unmarshal(b, r) } diff --git a/bundle/config/resources/lifecycle.go b/bundle/config/resources/lifecycle.go index 88d130239ee..59ccb682b81 100644 --- a/bundle/config/resources/lifecycle.go +++ b/bundle/config/resources/lifecycle.go @@ -36,5 +36,12 @@ type JobRunLifecycle struct { // JobRunTrigger is one lifecycle.triggers entry. type JobRunTrigger struct { - OnBundleDeploy *bool `json:"on_bundle_deploy,omitempty"` + OnBundleDeploy *bool `json:"on_bundle_deploy,omitempty"` + OnFileChange *string `json:"on_file_change,omitempty"` // path or glob under sync root +} + +type JobRunFileFingerprint struct { + Hash string `json:"hash"` + Size int64 `json:"size"` + MtimeNs int64 `json:"mtime_ns"` } diff --git a/bundle/direct/dresources/job_run.go b/bundle/direct/dresources/job_run.go index b391fd706d4..219cf2ab740 100644 --- a/bundle/direct/dresources/job_run.go +++ b/bundle/direct/dresources/job_run.go @@ -26,10 +26,21 @@ import ( // jobRunTimeout matches the timeout `bundle run` allows a run (bundle/run/job.go). const jobRunTimeout = 24 * time.Hour +// jobRunTriggerLocalPaths is shared by OverrideChangeDesc and DoUpdate so +// clearing a trigger stays a state-only update in both places. +var jobRunTriggerLocalPaths = []string{ + "lifecycle", + "lifecycle.triggers", + "lifecycle.triggers.on_bundle_deploy", + "lifecycle.triggers.on_file_change", +} + // JobRunTriggersState is the persisted fingerprint of lifecycle.triggers. type JobRunTriggersState struct { // Fresh UUID each plan while armed so Old!=New forces recreate. OnBundleDeploy string `json:"on_bundle_deploy,omitempty"` + // Per-file fingerprints from ResolveJobRunFileTriggers; change to recreate. + OnFileChange map[string]resources.JobRunFileFingerprint `json:"on_file_change,omitempty"` } // JobRunLifecycleState holds local-only lifecycle fields persisted in state. @@ -97,12 +108,19 @@ func (*ResourceJobRun) PrepareState(input *resources.JobRun) *JobRunState { state := &JobRunState{ RunNow: input.RunNow, ResultState: jobs.RunResultStateSuccess, - Lifecycle: nil, } + var triggers *JobRunTriggersState if input.HasOnBundleDeploy() { - state.Lifecycle = &JobRunLifecycleState{ - Triggers: &JobRunTriggersState{OnBundleDeploy: uuid.NewString()}, + triggers = &JobRunTriggersState{OnBundleDeploy: uuid.NewString()} + } + if len(input.ResolvedFileTriggers) > 0 { + if triggers == nil { + triggers = &JobRunTriggersState{} } + triggers.OnFileChange = input.ResolvedFileTriggers + } + if triggers != nil { + state.Lifecycle = &JobRunLifecycleState{Triggers: triggers} } return state } @@ -370,7 +388,7 @@ func reportRunLine(ctx context.Context, runID int64, msg string) { func (r *ResourceJobRun) DoUpdate(ctx context.Context, id string, config *JobRunState, entry *PlanEntry) (*JobRunRemote, error) { // Clearing a trigger only drops its local-only fingerprint from state; wait on // the run only when some other field changed. - if !entry.Changes.HasChangeExcept("lifecycle", "lifecycle.triggers", "lifecycle.triggers.on_bundle_deploy") { + if !entry.Changes.HasChangeExcept(jobRunTriggerLocalPaths...) { config.ResultState = "" return nil, nil } @@ -386,8 +404,7 @@ func (r *ResourceJobRun) DoUpdate(ctx context.Context, id string, config *JobRun // Clearing a trigger downgrades the recreate to a state-only update so the // fingerprint is dropped from state without re-firing the run. func (*ResourceJobRun) OverrideChangeDesc(_ context.Context, path *structpath.PathNode, change *ChangeDesc, remote *JobRunRemote) error { - switch path.String() { - case "lifecycle", "lifecycle.triggers", "lifecycle.triggers.on_bundle_deploy": + if slices.Contains(jobRunTriggerLocalPaths, path.String()) { // A cleared trigger sets New empty; structdiff may report it at lifecycle, // lifecycle.triggers, or the leaf. DoUpdate treats these paths as no-ops. if change.New == nil || change.New == "" { @@ -395,6 +412,8 @@ func (*ResourceJobRun) OverrideChangeDesc(_ context.Context, path *structpath.Pa change.Reason = "trigger removed" } return nil + } + switch path.String() { case "result_state": // The planner passes no remote state when the run could not be read. if remote == nil || runIsTerminal(remote.State.LifeCycleState) { diff --git a/bundle/direct/dresources/job_run_test.go b/bundle/direct/dresources/job_run_test.go index 27c1b93fac1..e374e7b0130 100644 --- a/bundle/direct/dresources/job_run_test.go +++ b/bundle/direct/dresources/job_run_test.go @@ -376,6 +376,41 @@ func TestJobRunPrepareStateOnBundleDeploy(t *testing.T) { }) } +func TestJobRunPrepareStateOnFileChange(t *testing.T) { + fps := map[string]resources.JobRunFileFingerprint{ + "a.txt": {Hash: "abc", Size: 3, MtimeNs: 1}, + } + + t.Run("unset", func(t *testing.T) { + state := (&ResourceJobRun{}).PrepareState(&resources.JobRun{}) + assert.Nil(t, state.Lifecycle) + }) + + t.Run("armed", func(t *testing.T) { + state := (&ResourceJobRun{}).PrepareState(&resources.JobRun{ + ResolvedFileTriggers: fps, + }) + require.NotNil(t, state.Lifecycle) + require.NotNil(t, state.Lifecycle.Triggers) + assert.Equal(t, fps, state.Lifecycle.Triggers.OnFileChange) + assert.Empty(t, state.Lifecycle.Triggers.OnBundleDeploy) + }) + + t.Run("both triggers", func(t *testing.T) { + on := true + state := (&ResourceJobRun{}).PrepareState(&resources.JobRun{ + Lifecycle: &resources.JobRunLifecycle{ + Triggers: []resources.JobRunTrigger{{OnBundleDeploy: &on}}, + }, + ResolvedFileTriggers: fps, + }) + require.NotNil(t, state.Lifecycle) + require.NotNil(t, state.Lifecycle.Triggers) + assert.NotEmpty(t, state.Lifecycle.Triggers.OnBundleDeploy) + assert.Equal(t, fps, state.Lifecycle.Triggers.OnFileChange) + }) +} + func TestJobRunOverrideChangeDescTriggerRemoved(t *testing.T) { r := &ResourceJobRun{} @@ -401,6 +436,19 @@ func TestJobRunOverrideChangeDescTriggerRemoved(t *testing.T) { assert.Equal(t, "trigger removed", change.Reason) }) + t.Run("clearing on_file_change leaf downgrades to update", func(t *testing.T) { + change := &ChangeDesc{ + Action: deployplan.Recreate, + Old: map[string]resources.JobRunFileFingerprint{ + "a.txt": {Hash: "abc", Size: 3, MtimeNs: 1}, + }, + New: nil, + } + require.NoError(t, r.OverrideChangeDesc(t.Context(), structpath.MustParsePath("lifecycle.triggers.on_file_change"), change, nil)) + assert.Equal(t, deployplan.Update, change.Action) + assert.Equal(t, "trigger removed", change.Reason) + }) + t.Run("fresh fingerprint still recreates", func(t *testing.T) { change := &ChangeDesc{ Action: deployplan.Recreate, @@ -410,6 +458,20 @@ func TestJobRunOverrideChangeDescTriggerRemoved(t *testing.T) { require.NoError(t, r.OverrideChangeDesc(t.Context(), structpath.MustParsePath("lifecycle.triggers.on_bundle_deploy"), change, nil)) assert.Equal(t, deployplan.Recreate, change.Action) }) + + t.Run("changed on_file_change fingerprint still recreates", func(t *testing.T) { + change := &ChangeDesc{ + Action: deployplan.Recreate, + Old: map[string]resources.JobRunFileFingerprint{ + "a.txt": {Hash: "old", Size: 1, MtimeNs: 1}, + }, + New: map[string]resources.JobRunFileFingerprint{ + "a.txt": {Hash: "new", Size: 1, MtimeNs: 2}, + }, + } + require.NoError(t, r.OverrideChangeDesc(t.Context(), structpath.MustParsePath("lifecycle.triggers.on_file_change"), change, nil)) + assert.Equal(t, deployplan.Recreate, change.Action) + }) } // The planner diffs RemapState(remote) against PrepareState(config), so a run @@ -449,6 +511,20 @@ func TestJobRunIgnoresEveryRequestField(t *testing.T) { assert.False(t, ignoresRemoteChanges(ignored, "result_state"), "result_state must stay comparable against the remote") } +func TestJobRunIgnoresMtimeOnlyFileTriggerDrift(t *testing.T) { + adapters, err := InitAll(nil) + require.NoError(t, err) + ignored := adapters["job_runs"].ResourceConfig().IgnoreLocalChanges + path := structpath.MustParsePath("lifecycle.triggers.on_file_change['seed.txt'].mtime_ns") + assert.True(t, slices.ContainsFunc(ignored, func(r FieldRule) bool { + return path.HasPatternPrefix(r.Field) + }), "mtime-only fingerprint drift must be ignored") + hashPath := structpath.MustParsePath("lifecycle.triggers.on_file_change['seed.txt'].hash") + assert.False(t, slices.ContainsFunc(ignored, func(r FieldRule) bool { + return hashPath.HasPatternPrefix(r.Field) + }), "hash changes must still recreate") +} + // ignoresRemoteChanges reports whether the rules suppress remote drift on field. func ignoresRemoteChanges(rules []FieldRule, field string) bool { path := structpath.MustParsePath(field) diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index e98a1ac2330..5559f7fa5bf 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -182,6 +182,12 @@ resources: recreate_on_changes: - reason: immutable + # mtime is stored for the resolve fast path; content identity is the hash. + # Ignoring mtime-only local drift keeps touch from recreating the run. + ignore_local_changes: + - field: lifecycle.triggers.on_file_change[*].mtime_ns + reason: mtime_only_fingerprint + pipelines: recreate_on_changes: - field: storage diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index a47e8151681..bfecaef7c00 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -990,6 +990,9 @@ resources: "on_bundle_deploy": "description": |- If true, re-fire the run on every bundle deploy. Incompatible with lifecycle.prevent_destroy. + "on_file_change": + "description": |- + Path or glob under the sync root. Re-fire the run when any matched file's content changes. Incompatible with lifecycle.prevent_destroy. "python_named_params": "description": |- PLACEHOLDER diff --git a/bundle/phases/initialize.go b/bundle/phases/initialize.go index b15e1c30df6..9eba6f466e0 100644 --- a/bundle/phases/initialize.go +++ b/bundle/phases/initialize.go @@ -26,7 +26,8 @@ import ( func Initialize(ctx context.Context, b *bundle.Bundle) { log.Info(ctx, "Phase: initialize") - bundle.ApplySeqContext(ctx, b, + bundle.ApplySeqContext( + ctx, b, // Reads (dynamic): resource.*.* // Checks that none of resources.. is nil. Raises error otherwise. validate.AllResourcesHaveValues(), @@ -191,6 +192,9 @@ func Initialize(ctx context.Context, b *bundle.Bundle) { // Reject invalid job_runs.lifecycle.triggers (empty, false, prevent_destroy). mutator.ValidateJobRunTriggers(), + // Expand on_file_change globs and hash matched files into ResolvedFileTriggers. + mutator.ResolveJobRunFileTriggers(), + // Reads (dynamic): * (strings) (searches for ${resources.*} references) // Warns (TF engine) or errors (direct engine) when a cross-resource reference // points to a Terraform-only field with no DABs equivalent. diff --git a/bundle/phases/plan.go b/bundle/phases/plan.go index 3db0864c2c4..92ca524cc4f 100644 --- a/bundle/phases/plan.go +++ b/bundle/phases/plan.go @@ -21,7 +21,8 @@ import ( // PreDeployChecks is common set of mutators between "bundle plan" and "bundle deploy". // Note, it is not run in "bundle migrate" so it must not modify the config func PreDeployChecks(ctx context.Context, b *bundle.Bundle, isPlan bool, engine engine.EngineType) { - bundle.ApplySeqContext(ctx, b, + bundle.ApplySeqContext( + ctx, b, terraform.CheckDashboardsModifiedRemotely(isPlan, engine), resourcemutator.SecretScopeFixups(engine), deploy.StatePull(), @@ -30,6 +31,7 @@ func PreDeployChecks(ctx context.Context, b *bundle.Bundle, isPlan bool, engine mutator.ValidateLifecycleStarted(engine), mutator.ValidateCascadeOnDestroy(engine), mutator.ValidateJobRunTriggers(), + mutator.ResolveJobRunFileTriggers(), statemgmt.CheckRunningResource(engine), ) } diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index e0c710428a3..a520561d933 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -1300,6 +1300,10 @@ "on_bundle_deploy": { "description": "If true, re-fire the run on every bundle deploy. Incompatible with lifecycle.prevent_destroy.", "$ref": "#/$defs/bool" + }, + "on_file_change": { + "description": "Path or glob under the sync root. Re-fire the run when any matched file's content changes. Incompatible with lifecycle.prevent_destroy.", + "$ref": "#/$defs/string" } }, "additionalProperties": false From 926b8cf555626bfaf34bb2c10799997d59b59100 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Tue, 18 Aug 2026 12:41:58 +0000 Subject: [PATCH 02/75] job_runs: fix exhaustruct and regenerate refschema for on_file_change Satisfy the linter on PrepareState literals after adding OnFileChange, and refresh out.fields.txt so validate-generated stays green. --- acceptance/bundle/refschema/out.fields.txt | 11 +++++++++++ bundle/direct/dresources/job_run.go | 11 +++++++++-- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index 49418540acf..8d956f2740c 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -852,8 +852,14 @@ resources.job_runs.*.lifecycle.prevent_destroy bool INPUT resources.job_runs.*.lifecycle.triggers *dresources.JobRunTriggersState STATE resources.job_runs.*.lifecycle.triggers []resources.JobRunTrigger INPUT resources.job_runs.*.lifecycle.triggers.on_bundle_deploy string STATE +resources.job_runs.*.lifecycle.triggers.on_file_change map[string]resources.JobRunFileFingerprint STATE +resources.job_runs.*.lifecycle.triggers.on_file_change.* resources.JobRunFileFingerprint STATE +resources.job_runs.*.lifecycle.triggers.on_file_change.*.hash string STATE +resources.job_runs.*.lifecycle.triggers.on_file_change.*.mtime_ns int64 STATE +resources.job_runs.*.lifecycle.triggers.on_file_change.*.size int64 STATE resources.job_runs.*.lifecycle.triggers[*] resources.JobRunTrigger INPUT resources.job_runs.*.lifecycle.triggers[*].on_bundle_deploy *bool INPUT +resources.job_runs.*.lifecycle.triggers[*].on_file_change *string INPUT resources.job_runs.*.modified_status string INPUT resources.job_runs.*.notebook_params map[string]string ALL resources.job_runs.*.notebook_params.* string ALL @@ -876,6 +882,11 @@ resources.job_runs.*.python_params []string ALL resources.job_runs.*.python_params[*] string ALL resources.job_runs.*.queue *jobs.QueueSettings ALL resources.job_runs.*.queue.enabled bool ALL +resources.job_runs.*.resolved_file_triggers map[string]resources.JobRunFileFingerprint INPUT +resources.job_runs.*.resolved_file_triggers.* resources.JobRunFileFingerprint INPUT +resources.job_runs.*.resolved_file_triggers.*.hash string INPUT +resources.job_runs.*.resolved_file_triggers.*.mtime_ns int64 INPUT +resources.job_runs.*.resolved_file_triggers.*.size int64 INPUT resources.job_runs.*.resolved_job_id int64 INPUT resources.job_runs.*.result_state jobs.RunResultState REMOTE STATE resources.job_runs.*.run_id int64 REMOTE diff --git a/bundle/direct/dresources/job_run.go b/bundle/direct/dresources/job_run.go index 219cf2ab740..db202c70954 100644 --- a/bundle/direct/dresources/job_run.go +++ b/bundle/direct/dresources/job_run.go @@ -108,14 +108,21 @@ func (*ResourceJobRun) PrepareState(input *resources.JobRun) *JobRunState { state := &JobRunState{ RunNow: input.RunNow, ResultState: jobs.RunResultStateSuccess, + Lifecycle: nil, } var triggers *JobRunTriggersState if input.HasOnBundleDeploy() { - triggers = &JobRunTriggersState{OnBundleDeploy: uuid.NewString()} + triggers = &JobRunTriggersState{ + OnBundleDeploy: uuid.NewString(), + OnFileChange: nil, + } } if len(input.ResolvedFileTriggers) > 0 { if triggers == nil { - triggers = &JobRunTriggersState{} + triggers = &JobRunTriggersState{ + OnBundleDeploy: "", + OnFileChange: nil, + } } triggers.OnFileChange = input.ResolvedFileTriggers } From f181266ab18ae3238634ec5d4c59b549920d586b Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Tue, 18 Aug 2026 13:03:41 +0000 Subject: [PATCH 03/75] job_runs: tighten on_file_change path and directory checks Reject patterns outside the sync root and directory-only matches so the trigger cannot fingerprint escaped paths or silently disarm. --- .../bundles/job-runs-on-file-change.md | 2 +- .../mutator/resolve_job_run_file_triggers.go | 31 ++++++++++++++++--- .../resolve_job_run_file_triggers_test.go | 25 +++++++++++++++ bundle/internal/schema/annotations.yml | 2 +- bundle/schema/jsonschema.json | 2 +- 5 files changed, 55 insertions(+), 7 deletions(-) diff --git a/.nextchanges/bundles/job-runs-on-file-change.md b/.nextchanges/bundles/job-runs-on-file-change.md index 00682403a0d..eadceda6268 100644 --- a/.nextchanges/bundles/job-runs-on-file-change.md +++ b/.nextchanges/bundles/job-runs-on-file-change.md @@ -1 +1 @@ -direct: `resources.job_runs` can set `lifecycle.triggers.on_file_change` to a path or glob to re-fire the run when matched file contents change. Removing the trigger does not recreate the existing run. +direct: `resources.job_runs` can set `lifecycle.triggers.on_file_change` to a path or glob under the sync root to re-fire the run when matched file contents change, or when matches appear or disappear. Removing the trigger does not recreate the existing run. diff --git a/bundle/config/mutator/resolve_job_run_file_triggers.go b/bundle/config/mutator/resolve_job_run_file_triggers.go index a82ed654be3..f0338b2d50a 100644 --- a/bundle/config/mutator/resolve_job_run_file_triggers.go +++ b/bundle/config/mutator/resolve_job_run_file_triggers.go @@ -80,7 +80,17 @@ func resolveFileTriggers(b *bundle.Bundle, name string, patterns []string, prev out := make(map[string]resources.JobRunFileFingerprint) for _, pattern := range patterns { path := fmt.Sprintf("resources.job_runs.%s.lifecycle.triggers", name) - matches, err := filepath.Glob(filepath.Join(b.SyncRootPath, filepath.FromSlash(pattern))) + localPattern := filepath.FromSlash(pattern) + // Keep fingerprints under SyncRoot; same IsLocal gate as translate_paths. + if !filepath.IsLocal(localPattern) { + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: pattern %q is not under the sync root", pattern), + Locations: b.Config.GetLocations(path), + }) + continue + } + matches, err := filepath.Glob(filepath.Join(b.SyncRootPath, localPattern)) if err != nil { diags = diags.Append(diag.Diagnostic{ Severity: diag.Error, @@ -90,7 +100,7 @@ func resolveFileTriggers(b *bundle.Bundle, name string, patterns []string, prev continue } if len(matches) == 0 { - // Distinct state when the path/glob matches nothing (design doc). + // Distinct state when the path/glob matches nothing so appear/disappear recreates. out[filepath.ToSlash(pattern)] = resources.JobRunFileFingerprint{ Size: missingFileSentinelSize, } @@ -101,6 +111,8 @@ func resolveFileTriggers(b *bundle.Bundle, name string, patterns []string, prev }) continue } + regularMatches := 0 + sawNonRegular := false for _, match := range matches { info, err := os.Stat(match) if err != nil { @@ -112,13 +124,15 @@ func resolveFileTriggers(b *bundle.Bundle, name string, patterns []string, prev continue } if !info.Mode().IsRegular() { + sawNonRegular = true continue } + regularMatches++ rel, err := filepath.Rel(b.SyncRootPath, match) - if err != nil { + if err != nil || !filepath.IsLocal(rel) { diags = diags.Append(diag.Diagnostic{ Severity: diag.Error, - Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: relative path for %q: %s", match, err), + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: matched path %q is not under the sync root", match), Locations: b.Config.GetLocations(path), }) continue @@ -135,6 +149,15 @@ func resolveFileTriggers(b *bundle.Bundle, name string, patterns []string, prev } out[key] = fp } + // A directory-only match would otherwise leave ResolvedFileTriggers empty + // and silently disarm the trigger while config still sets on_file_change. + if regularMatches == 0 && sawNonRegular { + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: pattern %q matches no regular files", pattern), + Locations: b.Config.GetLocations(path), + }) + } } return out, diags } diff --git a/bundle/config/mutator/resolve_job_run_file_triggers_test.go b/bundle/config/mutator/resolve_job_run_file_triggers_test.go index 9a9cf77bf53..06266d75894 100644 --- a/bundle/config/mutator/resolve_job_run_file_triggers_test.go +++ b/bundle/config/mutator/resolve_job_run_file_triggers_test.go @@ -113,6 +113,31 @@ func TestResolveJobRunFileTriggers(t *testing.T) { assertFingerprint(t, fps["a.txt"], "aaa") assertFingerprint(t, fps["subdir/x.py"], "bbb") }) + + t.Run("pattern outside sync root is an error", func(t *testing.T) { + dir := t.TempDir() + pattern := "../outside.txt" + b := bundleWithFileTrigger(dir, pattern) + + diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) + require.True(t, diags.HasError()) + require.Len(t, diags, 1) + assert.Contains(t, diags[0].Summary, `not under the sync root`) + assert.Empty(t, b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers) + }) + + t.Run("directory-only match is an error", func(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(dir, "migrations"), 0o755)) + pattern := "migrations" + b := bundleWithFileTrigger(dir, pattern) + + diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) + require.True(t, diags.HasError()) + require.Len(t, diags, 1) + assert.Contains(t, diags[0].Summary, `matches no regular files`) + assert.Empty(t, b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers) + }) } func bundleWithFileTrigger(syncRoot, pattern string) *bundle.Bundle { diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index bfecaef7c00..8f8dc45d066 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -992,7 +992,7 @@ resources: If true, re-fire the run on every bundle deploy. Incompatible with lifecycle.prevent_destroy. "on_file_change": "description": |- - Path or glob under the sync root. Re-fire the run when any matched file's content changes. Incompatible with lifecycle.prevent_destroy. + Path or glob under the sync root (filepath.Glob; `**` is not supported). Re-fire the run when a matched file's content changes, or when the set of matches appears or disappears. Incompatible with lifecycle.prevent_destroy. "python_named_params": "description": |- PLACEHOLDER diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index a520561d933..69f9dcca0db 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -1302,7 +1302,7 @@ "$ref": "#/$defs/bool" }, "on_file_change": { - "description": "Path or glob under the sync root. Re-fire the run when any matched file's content changes. Incompatible with lifecycle.prevent_destroy.", + "description": "Path or glob under the sync root (filepath.Glob; `**` is not supported). Re-fire the run when a matched file's content changes, or when the set of matches appears or disappears. Incompatible with lifecycle.prevent_destroy.", "$ref": "#/$defs/string" } }, From 7255a17c25ea0627645c0801a5d5865f6e0f71a4 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Tue, 18 Aug 2026 13:16:41 +0000 Subject: [PATCH 04/75] job_runs: store on_file_change as path-to-hash only Drop size/mtime fingerprints and the resolve fast path so content identity alone drives recreate, and touch no longer needs a planner exception. --- .../bundles/job-runs-on-file-change.md | 2 +- acceptance/bundle/refschema/out.fields.txt | 14 +- .../job_runs/on_file_change/output.txt | 11 + .../resources/job_runs/on_file_change/script | 6 + .../mutator/resolve_job_run_file_triggers.go | 204 +++++++----------- ...lve_job_run_file_triggers_internal_test.go | 73 ------- .../resolve_job_run_file_triggers_test.go | 48 +++-- bundle/config/resources/job_run.go | 19 +- bundle/config/resources/lifecycle.go | 6 - bundle/direct/dresources/job_run.go | 4 +- bundle/direct/dresources/job_run_test.go | 42 +--- bundle/direct/dresources/resources.yml | 6 - bundle/internal/schema/annotations.yml | 2 +- bundle/schema/jsonschema.json | 2 +- 14 files changed, 141 insertions(+), 298 deletions(-) delete mode 100644 bundle/config/mutator/resolve_job_run_file_triggers_internal_test.go diff --git a/.nextchanges/bundles/job-runs-on-file-change.md b/.nextchanges/bundles/job-runs-on-file-change.md index eadceda6268..95869a25ccf 100644 --- a/.nextchanges/bundles/job-runs-on-file-change.md +++ b/.nextchanges/bundles/job-runs-on-file-change.md @@ -1 +1 @@ -direct: `resources.job_runs` can set `lifecycle.triggers.on_file_change` to a path or glob under the sync root to re-fire the run when matched file contents change, or when matches appear or disappear. Removing the trigger does not recreate the existing run. +direct: `resources.job_runs` can set `lifecycle.triggers.on_file_change` to a path or glob under the sync root to re-fire the run when matched file contents change (content hash), or when matches appear or disappear. Removing the trigger does not recreate the existing run. diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index 8d956f2740c..b84842ffdc6 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -852,11 +852,8 @@ resources.job_runs.*.lifecycle.prevent_destroy bool INPUT resources.job_runs.*.lifecycle.triggers *dresources.JobRunTriggersState STATE resources.job_runs.*.lifecycle.triggers []resources.JobRunTrigger INPUT resources.job_runs.*.lifecycle.triggers.on_bundle_deploy string STATE -resources.job_runs.*.lifecycle.triggers.on_file_change map[string]resources.JobRunFileFingerprint STATE -resources.job_runs.*.lifecycle.triggers.on_file_change.* resources.JobRunFileFingerprint STATE -resources.job_runs.*.lifecycle.triggers.on_file_change.*.hash string STATE -resources.job_runs.*.lifecycle.triggers.on_file_change.*.mtime_ns int64 STATE -resources.job_runs.*.lifecycle.triggers.on_file_change.*.size int64 STATE +resources.job_runs.*.lifecycle.triggers.on_file_change map[string]string STATE +resources.job_runs.*.lifecycle.triggers.on_file_change.* string STATE resources.job_runs.*.lifecycle.triggers[*] resources.JobRunTrigger INPUT resources.job_runs.*.lifecycle.triggers[*].on_bundle_deploy *bool INPUT resources.job_runs.*.lifecycle.triggers[*].on_file_change *string INPUT @@ -882,11 +879,8 @@ resources.job_runs.*.python_params []string ALL resources.job_runs.*.python_params[*] string ALL resources.job_runs.*.queue *jobs.QueueSettings ALL resources.job_runs.*.queue.enabled bool ALL -resources.job_runs.*.resolved_file_triggers map[string]resources.JobRunFileFingerprint INPUT -resources.job_runs.*.resolved_file_triggers.* resources.JobRunFileFingerprint INPUT -resources.job_runs.*.resolved_file_triggers.*.hash string INPUT -resources.job_runs.*.resolved_file_triggers.*.mtime_ns int64 INPUT -resources.job_runs.*.resolved_file_triggers.*.size int64 INPUT +resources.job_runs.*.resolved_file_triggers map[string]string INPUT +resources.job_runs.*.resolved_file_triggers.* string INPUT resources.job_runs.*.resolved_job_id int64 INPUT resources.job_runs.*.result_state jobs.RunResultState REMOTE STATE resources.job_runs.*.run_id int64 REMOTE diff --git a/acceptance/bundle/resources/job_runs/on_file_change/output.txt b/acceptance/bundle/resources/job_runs/on_file_change/output.txt index bcf5962f8fa..f356fe7fb61 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change/output.txt @@ -33,6 +33,17 @@ Resources: 0 created, 0 changed, 0 deleted, 2 unchanged >>> print_requests.py //jobs/run-now +=== touch without content change plans nothing +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change/default/files... +Files: 3 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 2 unchanged + +>>> print_requests.py //jobs/run-now + === editing the file re-fires >>> [CLI] bundle plan recreate job_runs.my_run diff --git a/acceptance/bundle/resources/job_runs/on_file_change/script b/acceptance/bundle/resources/job_runs/on_file_change/script index ff162e11f0b..0506dd5e106 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change/script +++ b/acceptance/bundle/resources/job_runs/on_file_change/script @@ -16,6 +16,12 @@ trace $CLI bundle plan trace $CLI bundle deploy trace print_requests.py //jobs/run-now +title "touch without content change plans nothing" +python -c "import os; os.utime('seed.txt', None)" +trace $CLI bundle plan +trace $CLI bundle deploy +trace print_requests.py //jobs/run-now + title "editing the file re-fires" update_file.py seed.txt "v1" "v2" trace $CLI bundle plan diff --git a/bundle/config/mutator/resolve_job_run_file_triggers.go b/bundle/config/mutator/resolve_job_run_file_triggers.go index f0338b2d50a..55a55a94b95 100644 --- a/bundle/config/mutator/resolve_job_run_file_triggers.go +++ b/bundle/config/mutator/resolve_job_run_file_triggers.go @@ -4,25 +4,23 @@ import ( "context" "crypto/sha256" "encoding/hex" - "encoding/json" "fmt" "io" "os" "path/filepath" + "strings" "github.com/databricks/cli/bundle" - "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/libs/diag" ) -// missingFileSentinelSize marks a pattern with no matching file so the next -// plan can distinguish "still missing" from "file appeared". -const missingFileSentinelSize = int64(-1) +// missingFileHash marks a pattern with no matching file so appear/disappear recreates. +const missingFileHash = "" type resolveJobRunFileTriggers struct{} // ResolveJobRunFileTriggers expands on_file_change globs and stores per-file -// fingerprints on each job_run for PrepareState to copy into local state. +// content hashes on each job_run for PrepareState to copy into local state. func ResolveJobRunFileTriggers() bundle.Mutator { return &resolveJobRunFileTriggers{} } @@ -34,155 +32,105 @@ func (*resolveJobRunFileTriggers) Name() string { func (*resolveJobRunFileTriggers) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { var diags diag.Diagnostics for name, jr := range b.Config.Resources.JobRuns { - if jr == nil { + if jr == nil || jr.Lifecycle == nil { continue } - patterns := jr.OnFileChangePatterns() - if len(patterns) == 0 { - continue + out := make(map[string]string) + for i, t := range jr.Lifecycle.Triggers { + if t.OnFileChange == nil { + continue + } + path := fmt.Sprintf("resources.job_runs.%s.lifecycle.triggers[%d].on_file_change", name, i) + hashes, d := resolveFileTrigger(b, path, strings.TrimSpace(*t.OnFileChange)) + diags = diags.Extend(d) + for k, v := range hashes { + out[k] = v + } + } + if len(out) == 0 { + jr.ResolvedFileTriggers = nil + } else { + jr.ResolvedFileTriggers = out } - fps, d := resolveFileTriggers(b, name, patterns, previousFileTriggers(b, name)) - diags = diags.Extend(d) - jr.ResolvedFileTriggers = fps } return diags } -// previousFileTriggers reads on_file_change fingerprints from deployment state -// when it is open (plan/deploy after StatePull). Used so unchanged content keeps -// a stable fingerprint across mtime-only updates (e.g. touch). -func previousFileTriggers(b *bundle.Bundle, name string) map[string]resources.JobRunFileFingerprint { - if b.DeploymentBundle.StateDB.Path == "" { - return nil - } - entry, ok := b.DeploymentBundle.StateDB.GetResourceEntry("resources.job_runs." + name) - if !ok || len(entry.State) == 0 { - return nil - } - var state struct { - Lifecycle *struct { - Triggers *struct { - OnFileChange map[string]resources.JobRunFileFingerprint `json:"on_file_change"` - } `json:"triggers"` - } `json:"lifecycle"` +func resolveFileTrigger(b *bundle.Bundle, loc, pattern string) (map[string]string, diag.Diagnostics) { + var diags diag.Diagnostics + out := make(map[string]string) + localPattern := filepath.FromSlash(pattern) + // Keep hashes under SyncRoot; same IsLocal gate as translate_paths. + if !filepath.IsLocal(localPattern) { + return out, diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: pattern %q is not under the sync root", pattern), + Locations: b.Config.GetLocations(loc), + }) } - if err := json.Unmarshal(entry.State, &state); err != nil { - return nil + matches, err := filepath.Glob(filepath.Join(b.SyncRootPath, localPattern)) + if err != nil { + return out, diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: invalid pattern %q: %s", pattern, err), + Locations: b.Config.GetLocations(loc), + }) } - if state.Lifecycle == nil || state.Lifecycle.Triggers == nil { - return nil + if len(matches) == 0 { + out[filepath.ToSlash(pattern)] = missingFileHash + return out, diags.Append(diag.Diagnostic{ + Severity: diag.Warning, + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: no files match %q", pattern), + Locations: b.Config.GetLocations(loc), + }) } - return state.Lifecycle.Triggers.OnFileChange -} - -func resolveFileTriggers(b *bundle.Bundle, name string, patterns []string, prev map[string]resources.JobRunFileFingerprint) (map[string]resources.JobRunFileFingerprint, diag.Diagnostics) { - var diags diag.Diagnostics - out := make(map[string]resources.JobRunFileFingerprint) - for _, pattern := range patterns { - path := fmt.Sprintf("resources.job_runs.%s.lifecycle.triggers", name) - localPattern := filepath.FromSlash(pattern) - // Keep fingerprints under SyncRoot; same IsLocal gate as translate_paths. - if !filepath.IsLocal(localPattern) { + regularMatches := 0 + sawNonRegular := false + for _, match := range matches { + info, err := os.Stat(match) + if err != nil { diags = diags.Append(diag.Diagnostic{ Severity: diag.Error, - Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: pattern %q is not under the sync root", pattern), - Locations: b.Config.GetLocations(path), + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: stat %q: %s", match, err), + Locations: b.Config.GetLocations(loc), }) continue } - matches, err := filepath.Glob(filepath.Join(b.SyncRootPath, localPattern)) - if err != nil { - diags = diags.Append(diag.Diagnostic{ - Severity: diag.Error, - Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: invalid pattern %q: %s", pattern, err), - Locations: b.Config.GetLocations(path), - }) + if !info.Mode().IsRegular() { + sawNonRegular = true continue } - if len(matches) == 0 { - // Distinct state when the path/glob matches nothing so appear/disappear recreates. - out[filepath.ToSlash(pattern)] = resources.JobRunFileFingerprint{ - Size: missingFileSentinelSize, - } + regularMatches++ + rel, err := filepath.Rel(b.SyncRootPath, match) + if err != nil || !filepath.IsLocal(rel) { diags = diags.Append(diag.Diagnostic{ - Severity: diag.Warning, - Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: no files match %q", pattern), - Locations: b.Config.GetLocations(path), + Severity: diag.Error, + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: matched path %q is not under the sync root", match), + Locations: b.Config.GetLocations(loc), }) continue } - regularMatches := 0 - sawNonRegular := false - for _, match := range matches { - info, err := os.Stat(match) - if err != nil { - diags = diags.Append(diag.Diagnostic{ - Severity: diag.Error, - Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: stat %q: %s", match, err), - Locations: b.Config.GetLocations(path), - }) - continue - } - if !info.Mode().IsRegular() { - sawNonRegular = true - continue - } - regularMatches++ - rel, err := filepath.Rel(b.SyncRootPath, match) - if err != nil || !filepath.IsLocal(rel) { - diags = diags.Append(diag.Diagnostic{ - Severity: diag.Error, - Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: matched path %q is not under the sync root", match), - Locations: b.Config.GetLocations(path), - }) - continue - } - key := filepath.ToSlash(rel) - fp, err := fingerprintFile(match, info, prev[key]) - if err != nil { - diags = diags.Append(diag.Diagnostic{ - Severity: diag.Error, - Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: hash %q: %s", match, err), - Locations: b.Config.GetLocations(path), - }) - continue - } - out[key] = fp - } - // A directory-only match would otherwise leave ResolvedFileTriggers empty - // and silently disarm the trigger while config still sets on_file_change. - if regularMatches == 0 && sawNonRegular { + hash, err := hashFile(match) + if err != nil { diags = diags.Append(diag.Diagnostic{ Severity: diag.Error, - Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: pattern %q matches no regular files", pattern), - Locations: b.Config.GetLocations(path), + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: hash %q: %s", match, err), + Locations: b.Config.GetLocations(loc), }) + continue } + out[filepath.ToSlash(rel)] = hash } - return out, diags -} - -// fingerprintFile builds a content fingerprint. If prev has the same size and -// mtime, it is reused without reading the file. If content hash matches prev, -// prev is reused so mtime-only updates (touch) do not change planned state. -func fingerprintFile(path string, info os.FileInfo, prev resources.JobRunFileFingerprint) (resources.JobRunFileFingerprint, error) { - size := info.Size() - mtime := info.ModTime().UnixNano() - if prev.Hash != "" && prev.Size == size && prev.MtimeNs == mtime { - return prev, nil + // A directory-only match would otherwise leave ResolvedFileTriggers empty + // and silently disarm the trigger while config still sets on_file_change. + if regularMatches == 0 && sawNonRegular { + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: pattern %q matches no regular files", pattern), + Locations: b.Config.GetLocations(loc), + }) } - hash, err := hashFile(path) - if err != nil { - return resources.JobRunFileFingerprint{}, err - } - if prev.Hash != "" && prev.Hash == hash { - return prev, nil - } - return resources.JobRunFileFingerprint{ - Hash: hash, - Size: size, - MtimeNs: mtime, - }, nil + return out, diags } func hashFile(path string) (string, error) { diff --git a/bundle/config/mutator/resolve_job_run_file_triggers_internal_test.go b/bundle/config/mutator/resolve_job_run_file_triggers_internal_test.go deleted file mode 100644 index e82bd22945b..00000000000 --- a/bundle/config/mutator/resolve_job_run_file_triggers_internal_test.go +++ /dev/null @@ -1,73 +0,0 @@ -package mutator - -import ( - "os" - "path/filepath" - "testing" - "time" - - "github.com/databricks/cli/bundle/config/resources" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestFingerprintFileContentStableAcrossTouch(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, "seed.txt") - require.NoError(t, os.WriteFile(path, []byte("v1"), 0o644)) - - info, err := os.Stat(path) - require.NoError(t, err) - first, err := fingerprintFile(path, info, resources.JobRunFileFingerprint{}) - require.NoError(t, err) - require.NotEmpty(t, first.Hash) - - // Advance mtime without changing contents (touch). - require.NoError(t, os.Chtimes(path, time.Now().Add(time.Minute), time.Now().Add(time.Minute))) - info, err = os.Stat(path) - require.NoError(t, err) - assert.NotEqual(t, first.MtimeNs, info.ModTime().UnixNano()) - - second, err := fingerprintFile(path, info, first) - require.NoError(t, err) - assert.Equal(t, first, second, "unchanged content must reuse the previous fingerprint") -} - -func TestFingerprintFileFastPathSkipsWhenMtimeAndSizeMatch(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, "seed.txt") - require.NoError(t, os.WriteFile(path, []byte("v1"), 0o644)) - - info, err := os.Stat(path) - require.NoError(t, err) - prev := resources.JobRunFileFingerprint{ - Hash: "not-the-real-hash", - Size: info.Size(), - MtimeNs: info.ModTime().UnixNano(), - } - - got, err := fingerprintFile(path, info, prev) - require.NoError(t, err) - assert.Equal(t, prev, got, "matching size+mtime must reuse prev without re-hashing") -} - -func TestFingerprintFileContentChange(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, "seed.txt") - require.NoError(t, os.WriteFile(path, []byte("v1"), 0o644)) - - info, err := os.Stat(path) - require.NoError(t, err) - first, err := fingerprintFile(path, info, resources.JobRunFileFingerprint{}) - require.NoError(t, err) - - // Different size so the mtime+size fast path cannot reuse prev. - require.NoError(t, os.WriteFile(path, []byte("v2-changed"), 0o644)) - info, err = os.Stat(path) - require.NoError(t, err) - second, err := fingerprintFile(path, info, first) - require.NoError(t, err) - assert.NotEqual(t, first.Hash, second.Hash) - assert.Equal(t, info.Size(), second.Size) - assert.Equal(t, info.ModTime().UnixNano(), second.MtimeNs) -} diff --git a/bundle/config/mutator/resolve_job_run_file_triggers_test.go b/bundle/config/mutator/resolve_job_run_file_triggers_test.go index 06266d75894..3ec663c89bd 100644 --- a/bundle/config/mutator/resolve_job_run_file_triggers_test.go +++ b/bundle/config/mutator/resolve_job_run_file_triggers_test.go @@ -17,7 +17,7 @@ import ( ) func TestResolveJobRunFileTriggers(t *testing.T) { - t.Run("matches files and fills fingerprints", func(t *testing.T) { + t.Run("matches files and fills hashes", func(t *testing.T) { dir := t.TempDir() require.NoError(t, os.WriteFile(filepath.Join(dir, "a.txt"), []byte("hello"), 0o644)) require.NoError(t, os.WriteFile(filepath.Join(dir, "b.txt"), []byte("world"), 0o644)) @@ -28,14 +28,13 @@ func TestResolveJobRunFileTriggers(t *testing.T) { diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) require.False(t, diags.HasError()) - fps := b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers - require.Len(t, fps, 2) - - assertFingerprint(t, fps["a.txt"], "hello") - assertFingerprint(t, fps["b.txt"], "world") + hashes := b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers + require.Len(t, hashes, 2) + assert.Equal(t, contentHash("hello"), hashes["a.txt"]) + assert.Equal(t, contentHash("world"), hashes["b.txt"]) }) - t.Run("no matches warns and stores sentinel", func(t *testing.T) { + t.Run("no matches warns and stores empty hash", func(t *testing.T) { dir := t.TempDir() pattern := "missing.txt" b := bundleWithFileTrigger(dir, pattern) @@ -46,12 +45,9 @@ func TestResolveJobRunFileTriggers(t *testing.T) { assert.Equal(t, diag.Warning, diags[0].Severity) assert.Contains(t, diags[0].Summary, `no files match "missing.txt"`) - fps := b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers - require.Len(t, fps, 1) - fp := fps["missing.txt"] - assert.Empty(t, fp.Hash) - assert.Equal(t, int64(-1), fp.Size) - assert.Zero(t, fp.MtimeNs) + hashes := b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers + require.Len(t, hashes, 1) + assert.Empty(t, hashes["missing.txt"]) }) t.Run("no file triggers is a no-op", func(t *testing.T) { @@ -108,10 +104,10 @@ func TestResolveJobRunFileTriggers(t *testing.T) { diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) require.False(t, diags.HasError()) - fps := b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers - require.Len(t, fps, 2) - assertFingerprint(t, fps["a.txt"], "aaa") - assertFingerprint(t, fps["subdir/x.py"], "bbb") + hashes := b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers + require.Len(t, hashes, 2) + assert.Equal(t, contentHash("aaa"), hashes["a.txt"]) + assert.Equal(t, contentHash("bbb"), hashes["subdir/x.py"]) }) t.Run("pattern outside sync root is an error", func(t *testing.T) { @@ -138,6 +134,17 @@ func TestResolveJobRunFileTriggers(t *testing.T) { assert.Contains(t, diags[0].Summary, `matches no regular files`) assert.Empty(t, b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers) }) + + t.Run("trims pattern whitespace", func(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "seed.txt"), []byte("v1"), 0o644)) + pattern := " seed.txt " + b := bundleWithFileTrigger(dir, pattern) + + diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) + require.False(t, diags.HasError()) + assert.Equal(t, contentHash("v1"), b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers["seed.txt"]) + }) } func bundleWithFileTrigger(syncRoot, pattern string) *bundle.Bundle { @@ -159,10 +166,7 @@ func bundleWithFileTrigger(syncRoot, pattern string) *bundle.Bundle { } } -func assertFingerprint(t *testing.T, fp resources.JobRunFileFingerprint, content string) { - t.Helper() +func contentHash(content string) string { sum := sha256.Sum256([]byte(content)) - assert.Equal(t, hex.EncodeToString(sum[:]), fp.Hash) - assert.Equal(t, int64(len(content)), fp.Size) - assert.NotZero(t, fp.MtimeNs) + return hex.EncodeToString(sum[:]) } diff --git a/bundle/config/resources/job_run.go b/bundle/config/resources/job_run.go index cf66c71f728..f4b16963453 100644 --- a/bundle/config/resources/job_run.go +++ b/bundle/config/resources/job_run.go @@ -28,9 +28,9 @@ type JobRun struct { // reference) lets state loading preserve that reference and its plan dependency. ResolvedJobID int64 `json:"resolved_job_id,omitempty" bundle:"internal"` - // ResolvedFileTriggers is the per-file fingerprint map for on_file_change, - // computed under SyncRoot before plan. bundle:"internal" keeps it out of schema. - ResolvedFileTriggers map[string]JobRunFileFingerprint `json:"resolved_file_triggers,omitempty" bundle:"internal"` + // ResolvedFileTriggers is path → content hash for on_file_change, computed + // under SyncRoot before plan. bundle:"internal" keeps it out of schema. + ResolvedFileTriggers map[string]string `json:"resolved_file_triggers,omitempty" bundle:"internal"` } // HasOnBundleDeploy reports whether any trigger re-fires on every deploy. @@ -58,19 +58,6 @@ func (r *JobRun) HasOnFileChange() bool { return false } -func (r *JobRun) OnFileChangePatterns() []string { - if r.Lifecycle == nil { - return nil - } - var patterns []string - for _, t := range r.Lifecycle.Triggers { - if t.OnFileChange != nil { - patterns = append(patterns, *t.OnFileChange) - } - } - return patterns -} - func (r *JobRun) UnmarshalJSON(b []byte) error { return marshal.Unmarshal(b, r) } diff --git a/bundle/config/resources/lifecycle.go b/bundle/config/resources/lifecycle.go index 59ccb682b81..a4e881d298c 100644 --- a/bundle/config/resources/lifecycle.go +++ b/bundle/config/resources/lifecycle.go @@ -39,9 +39,3 @@ type JobRunTrigger struct { OnBundleDeploy *bool `json:"on_bundle_deploy,omitempty"` OnFileChange *string `json:"on_file_change,omitempty"` // path or glob under sync root } - -type JobRunFileFingerprint struct { - Hash string `json:"hash"` - Size int64 `json:"size"` - MtimeNs int64 `json:"mtime_ns"` -} diff --git a/bundle/direct/dresources/job_run.go b/bundle/direct/dresources/job_run.go index db202c70954..c7bc11d62a0 100644 --- a/bundle/direct/dresources/job_run.go +++ b/bundle/direct/dresources/job_run.go @@ -39,8 +39,8 @@ var jobRunTriggerLocalPaths = []string{ type JobRunTriggersState struct { // Fresh UUID each plan while armed so Old!=New forces recreate. OnBundleDeploy string `json:"on_bundle_deploy,omitempty"` - // Per-file fingerprints from ResolveJobRunFileTriggers; change to recreate. - OnFileChange map[string]resources.JobRunFileFingerprint `json:"on_file_change,omitempty"` + // Path → content hash from ResolveJobRunFileTriggers; change to recreate. + OnFileChange map[string]string `json:"on_file_change,omitempty"` } // JobRunLifecycleState holds local-only lifecycle fields persisted in state. diff --git a/bundle/direct/dresources/job_run_test.go b/bundle/direct/dresources/job_run_test.go index e374e7b0130..2c5815d3a20 100644 --- a/bundle/direct/dresources/job_run_test.go +++ b/bundle/direct/dresources/job_run_test.go @@ -377,9 +377,7 @@ func TestJobRunPrepareStateOnBundleDeploy(t *testing.T) { } func TestJobRunPrepareStateOnFileChange(t *testing.T) { - fps := map[string]resources.JobRunFileFingerprint{ - "a.txt": {Hash: "abc", Size: 3, MtimeNs: 1}, - } + hashes := map[string]string{"a.txt": "abc"} t.Run("unset", func(t *testing.T) { state := (&ResourceJobRun{}).PrepareState(&resources.JobRun{}) @@ -388,11 +386,11 @@ func TestJobRunPrepareStateOnFileChange(t *testing.T) { t.Run("armed", func(t *testing.T) { state := (&ResourceJobRun{}).PrepareState(&resources.JobRun{ - ResolvedFileTriggers: fps, + ResolvedFileTriggers: hashes, }) require.NotNil(t, state.Lifecycle) require.NotNil(t, state.Lifecycle.Triggers) - assert.Equal(t, fps, state.Lifecycle.Triggers.OnFileChange) + assert.Equal(t, hashes, state.Lifecycle.Triggers.OnFileChange) assert.Empty(t, state.Lifecycle.Triggers.OnBundleDeploy) }) @@ -402,12 +400,12 @@ func TestJobRunPrepareStateOnFileChange(t *testing.T) { Lifecycle: &resources.JobRunLifecycle{ Triggers: []resources.JobRunTrigger{{OnBundleDeploy: &on}}, }, - ResolvedFileTriggers: fps, + ResolvedFileTriggers: hashes, }) require.NotNil(t, state.Lifecycle) require.NotNil(t, state.Lifecycle.Triggers) assert.NotEmpty(t, state.Lifecycle.Triggers.OnBundleDeploy) - assert.Equal(t, fps, state.Lifecycle.Triggers.OnFileChange) + assert.Equal(t, hashes, state.Lifecycle.Triggers.OnFileChange) }) } @@ -439,10 +437,8 @@ func TestJobRunOverrideChangeDescTriggerRemoved(t *testing.T) { t.Run("clearing on_file_change leaf downgrades to update", func(t *testing.T) { change := &ChangeDesc{ Action: deployplan.Recreate, - Old: map[string]resources.JobRunFileFingerprint{ - "a.txt": {Hash: "abc", Size: 3, MtimeNs: 1}, - }, - New: nil, + Old: map[string]string{"a.txt": "abc"}, + New: nil, } require.NoError(t, r.OverrideChangeDesc(t.Context(), structpath.MustParsePath("lifecycle.triggers.on_file_change"), change, nil)) assert.Equal(t, deployplan.Update, change.Action) @@ -459,15 +455,11 @@ func TestJobRunOverrideChangeDescTriggerRemoved(t *testing.T) { assert.Equal(t, deployplan.Recreate, change.Action) }) - t.Run("changed on_file_change fingerprint still recreates", func(t *testing.T) { + t.Run("changed on_file_change hash still recreates", func(t *testing.T) { change := &ChangeDesc{ Action: deployplan.Recreate, - Old: map[string]resources.JobRunFileFingerprint{ - "a.txt": {Hash: "old", Size: 1, MtimeNs: 1}, - }, - New: map[string]resources.JobRunFileFingerprint{ - "a.txt": {Hash: "new", Size: 1, MtimeNs: 2}, - }, + Old: map[string]string{"a.txt": "old"}, + New: map[string]string{"a.txt": "new"}, } require.NoError(t, r.OverrideChangeDesc(t.Context(), structpath.MustParsePath("lifecycle.triggers.on_file_change"), change, nil)) assert.Equal(t, deployplan.Recreate, change.Action) @@ -511,20 +503,6 @@ func TestJobRunIgnoresEveryRequestField(t *testing.T) { assert.False(t, ignoresRemoteChanges(ignored, "result_state"), "result_state must stay comparable against the remote") } -func TestJobRunIgnoresMtimeOnlyFileTriggerDrift(t *testing.T) { - adapters, err := InitAll(nil) - require.NoError(t, err) - ignored := adapters["job_runs"].ResourceConfig().IgnoreLocalChanges - path := structpath.MustParsePath("lifecycle.triggers.on_file_change['seed.txt'].mtime_ns") - assert.True(t, slices.ContainsFunc(ignored, func(r FieldRule) bool { - return path.HasPatternPrefix(r.Field) - }), "mtime-only fingerprint drift must be ignored") - hashPath := structpath.MustParsePath("lifecycle.triggers.on_file_change['seed.txt'].hash") - assert.False(t, slices.ContainsFunc(ignored, func(r FieldRule) bool { - return hashPath.HasPatternPrefix(r.Field) - }), "hash changes must still recreate") -} - // ignoresRemoteChanges reports whether the rules suppress remote drift on field. func ignoresRemoteChanges(rules []FieldRule, field string) bool { path := structpath.MustParsePath(field) diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index 5559f7fa5bf..e98a1ac2330 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -182,12 +182,6 @@ resources: recreate_on_changes: - reason: immutable - # mtime is stored for the resolve fast path; content identity is the hash. - # Ignoring mtime-only local drift keeps touch from recreating the run. - ignore_local_changes: - - field: lifecycle.triggers.on_file_change[*].mtime_ns - reason: mtime_only_fingerprint - pipelines: recreate_on_changes: - field: storage diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index 8f8dc45d066..d48a3e2053a 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -992,7 +992,7 @@ resources: If true, re-fire the run on every bundle deploy. Incompatible with lifecycle.prevent_destroy. "on_file_change": "description": |- - Path or glob under the sync root (filepath.Glob; `**` is not supported). Re-fire the run when a matched file's content changes, or when the set of matches appears or disappears. Incompatible with lifecycle.prevent_destroy. + Path or glob under the sync root (filepath.Glob; `**` is not supported). Re-fire the run when a matched file's content hash changes, or when the set of matches appears or disappears. Incompatible with lifecycle.prevent_destroy. "python_named_params": "description": |- PLACEHOLDER diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 69f9dcca0db..9f56ca61a18 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -1302,7 +1302,7 @@ "$ref": "#/$defs/bool" }, "on_file_change": { - "description": "Path or glob under the sync root (filepath.Glob; `**` is not supported). Re-fire the run when a matched file's content changes, or when the set of matches appears or disappears. Incompatible with lifecycle.prevent_destroy.", + "description": "Path or glob under the sync root (filepath.Glob; `**` is not supported). Re-fire the run when a matched file's content hash changes, or when the set of matches appears or disappears. Incompatible with lifecycle.prevent_destroy.", "$ref": "#/$defs/string" } }, From a02fd3d71dd8ff3cd1b7d3d1fcd6b44ac47e2942 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Tue, 18 Aug 2026 13:23:44 +0000 Subject: [PATCH 05/75] job_runs: use maps.Copy in on_file_change hash merge The modernize linter rejects the manual m[k]=v copy loop, failing CI lint. --- bundle/config/mutator/resolve_job_run_file_triggers.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/bundle/config/mutator/resolve_job_run_file_triggers.go b/bundle/config/mutator/resolve_job_run_file_triggers.go index 55a55a94b95..ead17c59e17 100644 --- a/bundle/config/mutator/resolve_job_run_file_triggers.go +++ b/bundle/config/mutator/resolve_job_run_file_triggers.go @@ -6,6 +6,7 @@ import ( "encoding/hex" "fmt" "io" + "maps" "os" "path/filepath" "strings" @@ -43,9 +44,7 @@ func (*resolveJobRunFileTriggers) Apply(_ context.Context, b *bundle.Bundle) dia path := fmt.Sprintf("resources.job_runs.%s.lifecycle.triggers[%d].on_file_change", name, i) hashes, d := resolveFileTrigger(b, path, strings.TrimSpace(*t.OnFileChange)) diags = diags.Extend(d) - for k, v := range hashes { - out[k] = v - } + maps.Copy(out, hashes) } if len(out) == 0 { jr.ResolvedFileTriggers = nil From 43cf3cf247d32f3ab101af3a1f19e8b461ecdf81 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Wed, 19 Aug 2026 08:39:23 +0000 Subject: [PATCH 06/75] job_runs: drop filepath.Glob jargon from on_file_change schema docs --- bundle/internal/schema/annotations.yml | 2 +- bundle/schema/jsonschema.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index d48a3e2053a..1fd6b452415 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -992,7 +992,7 @@ resources: If true, re-fire the run on every bundle deploy. Incompatible with lifecycle.prevent_destroy. "on_file_change": "description": |- - Path or glob under the sync root (filepath.Glob; `**` is not supported). Re-fire the run when a matched file's content hash changes, or when the set of matches appears or disappears. Incompatible with lifecycle.prevent_destroy. + Path or glob under the sync root. Re-fire the run when a matched file's content hash changes, or when the set of matches appears or disappears. Incompatible with lifecycle.prevent_destroy. "python_named_params": "description": |- PLACEHOLDER diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 9f56ca61a18..39d31e92167 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -1302,7 +1302,7 @@ "$ref": "#/$defs/bool" }, "on_file_change": { - "description": "Path or glob under the sync root (filepath.Glob; `**` is not supported). Re-fire the run when a matched file's content hash changes, or when the set of matches appears or disappears. Incompatible with lifecycle.prevent_destroy.", + "description": "Path or glob under the sync root. Re-fire the run when a matched file's content hash changes, or when the set of matches appears or disappears. Incompatible with lifecycle.prevent_destroy.", "$ref": "#/$defs/string" } }, From 47994d80e858b23a92e457f7e215fd69810ab6ed Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Wed, 19 Aug 2026 11:38:20 +0000 Subject: [PATCH 07/75] acc: refresh on_file_change goldens for resource-key progress lines --- .../bundle/resources/job_runs/on_file_change/output.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/acceptance/bundle/resources/job_runs/on_file_change/output.txt b/acceptance/bundle/resources/job_runs/on_file_change/output.txt index f356fe7fb61..a98eb3a6ca7 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change/output.txt @@ -2,8 +2,8 @@ === first deploy triggers a run >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change/default/files... -job run [MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?o=[NUMID] -job run [MY_RUN_ID]: SUCCESS +Output from job_runs.my_run: id=[MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?o=[NUMID] +Output from job_runs.my_run: id=[MY_RUN_ID]: SUCCESS Created job_runs.my_run Created jobs.my_job Files: 5 uploaded, 0 deleted @@ -52,8 +52,8 @@ Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change/default/files... -job run [MY_RUN_ID_2]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_2]?o=[NUMID] -job run [MY_RUN_ID_2]: SUCCESS +Output from job_runs.my_run: id=[MY_RUN_ID_2]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_2]?o=[NUMID] +Output from job_runs.my_run: id=[MY_RUN_ID_2]: SUCCESS Recreated job_runs.my_run Files: 3 uploaded, 0 deleted Resources: 1 created, 0 changed, 1 deleted, 1 unchanged From 132ba7bee6452fc728245821aa30b5d56f3ee5eb Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Wed, 19 Aug 2026 14:50:14 +0000 Subject: [PATCH 08/75] acc: cover on_file_change globs, errors, and appear/disappear --- .../job_runs/on_file_change/out.test.toml | 1 + .../job_runs/on_file_change/output.txt | 131 +++++++++++++- .../resources/job_runs/on_file_change/script | 26 ++- .../job_runs/on_file_change/test.toml | 14 ++ .../on_file_change_errors/databricks.yml | 18 ++ .../dirs/nested/keep.txt | 1 + .../on_file_change_errors/out.test.toml | 2 + .../job_runs/on_file_change_errors/output.txt | 39 +++++ .../job_runs/on_file_change_errors/script | 12 ++ .../job_runs/on_file_change_errors/test.toml | 2 + .../on_file_change_glob/databricks.yml | 18 ++ .../on_file_change_glob/migrations/001.txt | 1 + .../on_file_change_glob/migrations/002.txt | 1 + .../migrations/archive/003.txt | 1 + .../on_file_change_glob/out.test.toml | 2 + .../job_runs/on_file_change_glob/output.txt | 164 ++++++++++++++++++ .../job_runs/on_file_change_glob/script | 47 +++++ .../job_runs/on_file_change_glob/test.toml | 9 + bundle/phases/plan.go | 4 +- 19 files changed, 486 insertions(+), 7 deletions(-) create mode 100644 acceptance/bundle/resources/job_runs/on_file_change/test.toml create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_errors/databricks.yml create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_errors/dirs/nested/keep.txt create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_errors/out.test.toml create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_errors/script create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_errors/test.toml create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_glob/databricks.yml create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/001.txt create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/002.txt create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/archive/003.txt create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_glob/out.test.toml create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_glob/script create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml diff --git a/acceptance/bundle/resources/job_runs/on_file_change/out.test.toml b/acceptance/bundle/resources/job_runs/on_file_change/out.test.toml index 0938e678987..57b0f616850 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change/out.test.toml +++ b/acceptance/bundle/resources/job_runs/on_file_change/out.test.toml @@ -1,2 +1,3 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/job_runs/on_file_change/output.txt b/acceptance/bundle/resources/job_runs/on_file_change/output.txt index a98eb3a6ca7..d03307b90f2 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change/output.txt @@ -6,7 +6,7 @@ Output from job_runs.my_run: id=[MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[MY_ Output from job_runs.my_run: id=[MY_RUN_ID]: SUCCESS Created job_runs.my_run Created jobs.my_job -Files: 5 uploaded, 0 deleted +Files: 6 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged >>> read_id.py my_job @@ -50,12 +50,46 @@ recreate job_runs.my_run Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged ->>> [CLI] bundle deploy +>>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json +{ + "lifecycle": { + "action": "recreate", + "reason": "immutable", + "old": { + "triggers": { + "on_file_change": { + "seed.txt": "[FILE_HASH][0]" + } + } + }, + "new": { + "triggers": { + "on_file_change": { + "seed.txt": "[FILE_HASH][1]" + } + } + } + }, + "lifecycle.triggers.on_file_change['seed.txt']": { + "action": "recreate", + "reason": "immutable", + "old": "[FILE_HASH][0]", + "new": "[FILE_HASH][1]" + }, + "result_state": { + "action": "skip", + "reason": "remote_already_set", + "new": "SUCCESS", + "remote": "SUCCESS" + } +} + +=== bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change/default/files... Output from job_runs.my_run: id=[MY_RUN_ID_2]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_2]?o=[NUMID] Output from job_runs.my_run: id=[MY_RUN_ID_2]: SUCCESS Recreated job_runs.my_run -Files: 3 uploaded, 0 deleted +Files: 4 uploaded, 0 deleted Resources: 1 created, 0 changed, 1 deleted, 1 unchanged >>> print_requests.py --keep //jobs/runs/delete @@ -77,6 +111,97 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged } } +=== deleting the file warns and re-fires +>>> [CLI] bundle plan +Warning: lifecycle.triggers.on_file_change: no files match "seed.txt" + in databricks.yml:18:29 + +recreate job_runs.my_run + +Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged + +>>> changes +Warning: lifecycle.triggers.on_file_change: no files match "seed.txt" + in databricks.yml:18:29 + +{ + "lifecycle": { + "action": "recreate", + "reason": "immutable", + "old": { + "triggers": { + "on_file_change": { + "seed.txt": "[FILE_HASH][1]" + } + } + }, + "new": { + "triggers": { + "on_file_change": { + "seed.txt": "" + } + } + } + }, + "lifecycle.triggers.on_file_change['seed.txt']": { + "action": "recreate", + "reason": "immutable", + "old": "[FILE_HASH][1]", + "new": "" + }, + "result_state": { + "action": "skip", + "reason": "remote_already_set", + "new": "SUCCESS", + "remote": "SUCCESS" + } +} + +>>> [CLI] bundle deploy +Warning: lifecycle.triggers.on_file_change: no files match "seed.txt" + in databricks.yml:18:29 + +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change/default/files... +Output from job_runs.my_run: id=[MY_RUN_ID_3]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_3]?o=[NUMID] +Output from job_runs.my_run: id=[MY_RUN_ID_3]: SUCCESS +Recreated job_runs.my_run +Files: 3 uploaded, 1 deleted +Resources: 1 created, 0 changed, 1 deleted, 1 unchanged + +>>> print_requests.py //jobs/run-now +{ + "method": "POST", + "path": "/api/2.2/jobs/run-now", + "body": { + "idempotency_token": "[UUID]", + "job_id": [MY_JOB_ID] + } +} + +=== restoring the file re-fires again +>>> [CLI] bundle plan +recreate job_runs.my_run + +Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change/default/files... +Output from job_runs.my_run: id=[MY_RUN_ID_4]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_4]?o=[NUMID] +Output from job_runs.my_run: id=[MY_RUN_ID_4]: SUCCESS +Recreated job_runs.my_run +Files: 3 uploaded, 1 deleted +Resources: 1 created, 0 changed, 1 deleted, 1 unchanged + +>>> print_requests.py //jobs/run-now +{ + "method": "POST", + "path": "/api/2.2/jobs/run-now", + "body": { + "idempotency_token": "[UUID]", + "job_id": [MY_JOB_ID] + } +} + === removing on_file_change rewrites state without a run >>> [CLI] bundle plan update job_runs.my_run diff --git a/acceptance/bundle/resources/job_runs/on_file_change/script b/acceptance/bundle/resources/job_runs/on_file_change/script index 0506dd5e106..fd1e8800c42 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change/script +++ b/acceptance/bundle/resources/job_runs/on_file_change/script @@ -4,6 +4,10 @@ cleanup() { } trap cleanup EXIT +changes() { + $CLI bundle plan -o json | jq '.plan["resources.job_runs.my_run"].changes' +} + title "first deploy triggers a run" trace $CLI bundle deploy trace read_id.py my_job @@ -25,11 +29,31 @@ trace print_requests.py //jobs/run-now title "editing the file re-fires" update_file.py seed.txt "v1" "v2" trace $CLI bundle plan -trace $CLI bundle deploy +# Save the plan so the READPLAN=1 variant deploys the hashes computed here. +# The deploy is not traced: readplanarg makes the command line differ per variant. +$CLI bundle plan -o json > tmp.plan.json +trace jq '.plan["resources.job_runs.my_run"].changes' tmp.plan.json +title "bundle deploy\n" +$CLI bundle deploy $(readplanarg tmp.plan.json) read_id.py my_run > /dev/null trace print_requests.py --keep //jobs/runs/delete trace print_requests.py //jobs/run-now +title "deleting the file warns and re-fires" +mv seed.txt tmp.seed.txt +trace $CLI bundle plan +trace changes +trace $CLI bundle deploy +read_id.py my_run > /dev/null +trace print_requests.py //jobs/run-now + +title "restoring the file re-fires again" +mv tmp.seed.txt seed.txt +trace $CLI bundle plan +trace $CLI bundle deploy +read_id.py my_run > /dev/null +trace print_requests.py //jobs/run-now + title "removing on_file_change rewrites state without a run" update_file.py databricks.yml " lifecycle: triggers: diff --git a/acceptance/bundle/resources/job_runs/on_file_change/test.toml b/acceptance/bundle/resources/job_runs/on_file_change/test.toml new file mode 100644 index 00000000000..7a950e9fd0c --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change/test.toml @@ -0,0 +1,14 @@ +# Deploy both by re-planning and from a plan saved on disk, so the hashes +# computed during planning survive plan serialization. +EnvMatrix.READPLAN = ["", "1"] + +Ignore = ["tmp.plan.json", "tmp.seed.txt"] + +# Content hashes are stable but unreadable, and the parent's `\d{8,}` rule +# mangles any digit run inside them. Order=1 runs before it; Distinct keeps +# different hashes distinguishable. +[[Repls]] +Old = '[0-9a-f]{64}' +New = '[FILE_HASH]' +Order = 1 +Distinct = true diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/databricks.yml b/acceptance/bundle/resources/job_runs/on_file_change_errors/databricks.yml new file mode 100644 index 00000000000..44c5433e52a --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/databricks.yml @@ -0,0 +1,18 @@ +bundle: + name: job-runs-on-file-change-errors + +resources: + jobs: + my_job: + name: my-job + tasks: + - task_key: main + notebook_task: + notebook_path: /Workspace/test + + job_runs: + my_run: + job_id: ${resources.jobs.my_job.id} + lifecycle: + triggers: + - on_file_change: ../shared/*.txt diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/dirs/nested/keep.txt b/acceptance/bundle/resources/job_runs/on_file_change_errors/dirs/nested/keep.txt new file mode 100644 index 00000000000..79c53955ef8 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/dirs/nested/keep.txt @@ -0,0 +1 @@ +nested diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/out.test.toml b/acceptance/bundle/resources/job_runs/on_file_change_errors/out.test.toml new file mode 100644 index 00000000000..0938e678987 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt new file mode 100644 index 00000000000..c7d3a98e1cb --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt @@ -0,0 +1,39 @@ + +=== a pattern outside the sync root is rejected +>>> musterr [CLI] bundle validate +Error: lifecycle.triggers.on_file_change: pattern "../shared/*.txt" is not under the sync root + in databricks.yml:18:29 + +Name: job-runs-on-file-change-errors +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-errors/default + +Found 1 error + +=== a glob that only matches directories is rejected +>>> musterr [CLI] bundle validate +Error: lifecycle.triggers.on_file_change: pattern "dirs/*" matches no regular files + in databricks.yml:18:29 + +Name: job-runs-on-file-change-errors +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-errors/default + +Found 1 error + +=== a pattern that matches nothing warns +>>> [CLI] bundle validate +Warning: lifecycle.triggers.on_file_change: no files match "missing/*.txt" + in databricks.yml:18:29 + +Name: job-runs-on-file-change-errors +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-errors/default + +Found 1 warning diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/script b/acceptance/bundle/resources/job_runs/on_file_change_errors/script new file mode 100644 index 00000000000..39596f0c772 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/script @@ -0,0 +1,12 @@ +title "a pattern outside the sync root is rejected" +trace musterr $CLI bundle validate + +title "a glob that only matches directories is rejected" +# dirs/* matches dirs/nested, a directory: nothing is left to hash, so the +# trigger would silently never fire. +update_file.py databricks.yml "../shared/*.txt" "dirs/*" +trace musterr $CLI bundle validate + +title "a pattern that matches nothing warns" +update_file.py databricks.yml "dirs/*" "missing/*.txt" +trace $CLI bundle validate diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/test.toml b/acceptance/bundle/resources/job_runs/on_file_change_errors/test.toml new file mode 100644 index 00000000000..7b88d613d4c --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/test.toml @@ -0,0 +1,2 @@ +# The bundle is never deployed here, so there are no requests worth recording. +RecordRequests = false diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/databricks.yml b/acceptance/bundle/resources/job_runs/on_file_change_glob/databricks.yml new file mode 100644 index 00000000000..3b148538f38 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/databricks.yml @@ -0,0 +1,18 @@ +bundle: + name: job-runs-on-file-change-glob + +resources: + jobs: + my_job: + name: my-job + tasks: + - task_key: main + notebook_task: + notebook_path: /Workspace/test + + job_runs: + my_run: + job_id: ${resources.jobs.my_job.id} + lifecycle: + triggers: + - on_file_change: migrations/* diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/001.txt b/acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/001.txt new file mode 100644 index 00000000000..5626abf0f72 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/001.txt @@ -0,0 +1 @@ +one diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/002.txt b/acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/002.txt new file mode 100644 index 00000000000..f719efd430d --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/002.txt @@ -0,0 +1 @@ +two diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/archive/003.txt b/acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/archive/003.txt new file mode 100644 index 00000000000..2bdf67abb16 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/archive/003.txt @@ -0,0 +1 @@ +three diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/out.test.toml b/acceptance/bundle/resources/job_runs/on_file_change_glob/out.test.toml new file mode 100644 index 00000000000..0938e678987 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt new file mode 100644 index 00000000000..798b459a9bd --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt @@ -0,0 +1,164 @@ + +=== deploy hashes the matched files +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-glob/default/files... +Output from job_runs.my_run: id=[MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?o=[NUMID] +Output from job_runs.my_run: id=[MY_RUN_ID]: SUCCESS +Created job_runs.my_run +Created jobs.my_job +Files: 8 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +>>> read_id.py my_job +[MY_JOB_ID] + +>>> print_requests.py //jobs/run-now +{ + "method": "POST", + "path": "/api/2.2/jobs/run-now", + "body": { + "idempotency_token": "[UUID]", + "job_id": [MY_JOB_ID] + } +} + +=== editing one matched file re-fires +>>> [CLI] bundle plan +recreate job_runs.my_run + +Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged + +>>> changes +{ + "lifecycle": { + "action": "recreate", + "reason": "immutable", + "old": { + "triggers": { + "on_file_change": { + "migrations/001.txt": "[FILE_HASH][0]", + "migrations/002.txt": "[FILE_HASH][1]" + } + } + }, + "new": { + "triggers": { + "on_file_change": { + "migrations/001.txt": "[FILE_HASH][2]", + "migrations/002.txt": "[FILE_HASH][1]" + } + } + } + }, + "lifecycle.triggers.on_file_change['migrations/001.txt']": { + "action": "recreate", + "reason": "immutable", + "old": "[FILE_HASH][0]", + "new": "[FILE_HASH][2]" + }, + "result_state": { + "action": "skip", + "reason": "remote_already_set", + "new": "SUCCESS", + "remote": "SUCCESS" + } +} + +=== a new matching file re-fires +>>> [CLI] bundle plan +recreate job_runs.my_run + +Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged + +>>> changes +{ + "lifecycle": { + "action": "recreate", + "reason": "immutable", + "old": { + "triggers": { + "on_file_change": { + "migrations/001.txt": "[FILE_HASH][0]", + "migrations/002.txt": "[FILE_HASH][1]" + } + } + }, + "new": { + "triggers": { + "on_file_change": { + "migrations/001.txt": "[FILE_HASH][0]", + "migrations/002.txt": "[FILE_HASH][1]", + "migrations/003.txt": "[FILE_HASH][3]" + } + } + } + }, + "lifecycle.triggers.on_file_change['migrations/003.txt']": { + "action": "recreate", + "reason": "immutable", + "new": "[FILE_HASH][3]" + }, + "result_state": { + "action": "skip", + "reason": "remote_already_set", + "new": "SUCCESS", + "remote": "SUCCESS" + } +} + +=== removing a matched file re-fires +>>> [CLI] bundle plan +recreate job_runs.my_run + +Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged + +>>> changes +{ + "lifecycle": { + "action": "recreate", + "reason": "immutable", + "old": { + "triggers": { + "on_file_change": { + "migrations/001.txt": "[FILE_HASH][0]", + "migrations/002.txt": "[FILE_HASH][1]" + } + } + }, + "new": { + "triggers": { + "on_file_change": { + "migrations/001.txt": "[FILE_HASH][0]" + } + } + } + }, + "lifecycle.triggers.on_file_change['migrations/002.txt']": { + "action": "recreate", + "reason": "immutable", + "old": "[FILE_HASH][1]" + }, + "result_state": { + "action": "skip", + "reason": "remote_already_set", + "new": "SUCCESS", + "remote": "SUCCESS" + } +} + +=== editing a file below the glob does not re-fire +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +=== ** is not recursive: it matches the same files as * +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.job_runs.my_run + delete resources.jobs.my_job + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-glob/default + +Destroy: 2 deleted diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/script b/acceptance/bundle/resources/job_runs/on_file_change_glob/script new file mode 100644 index 00000000000..8feeebcd067 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/script @@ -0,0 +1,47 @@ +cleanup() { + trace $CLI bundle destroy --auto-approve + rm -f out.requests.txt +} +trap cleanup EXIT + +changes() { + $CLI bundle plan -o json | jq '.plan["resources.job_runs.my_run"].changes' +} + +title "deploy hashes the matched files" +trace $CLI bundle deploy +trace read_id.py my_job +read_id.py my_run > /dev/null +trace print_requests.py //jobs/run-now + +# Each phase below plans against the state deployed above and then restores the +# tree, so every plan is compared against the same baseline. + +title "editing one matched file re-fires" +update_file.py migrations/001.txt "one" "one-edited" +trace $CLI bundle plan +trace changes +update_file.py migrations/001.txt "one-edited" "one" + +title "a new matching file re-fires" +printf 'three\n' > migrations/003.txt +trace $CLI bundle plan +trace changes +rm migrations/003.txt + +title "removing a matched file re-fires" +mv migrations/002.txt tmp.002.txt +trace $CLI bundle plan +trace changes +mv tmp.002.txt migrations/002.txt + +title "editing a file below the glob does not re-fire" +# migrations/* matches the archive directory itself, which is skipped: the glob +# does not descend into it. +update_file.py migrations/archive/003.txt "three" "three-edited" +trace $CLI bundle plan +update_file.py migrations/archive/003.txt "three-edited" "three" + +title "** is not recursive: it matches the same files as *" +update_file.py databricks.yml "migrations/*" "migrations/**" +trace $CLI bundle plan diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml b/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml new file mode 100644 index 00000000000..1c349066b94 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml @@ -0,0 +1,9 @@ +Ignore = ["tmp.002.txt"] + +# See on_file_change/test.toml: mask content hashes before the parent's +# `\d{8,}` rule can mangle them, keeping distinct hashes distinguishable. +[[Repls]] +Old = '[0-9a-f]{64}' +New = '[FILE_HASH]' +Order = 1 +Distinct = true diff --git a/bundle/phases/plan.go b/bundle/phases/plan.go index 92ca524cc4f..3db0864c2c4 100644 --- a/bundle/phases/plan.go +++ b/bundle/phases/plan.go @@ -21,8 +21,7 @@ import ( // PreDeployChecks is common set of mutators between "bundle plan" and "bundle deploy". // Note, it is not run in "bundle migrate" so it must not modify the config func PreDeployChecks(ctx context.Context, b *bundle.Bundle, isPlan bool, engine engine.EngineType) { - bundle.ApplySeqContext( - ctx, b, + bundle.ApplySeqContext(ctx, b, terraform.CheckDashboardsModifiedRemotely(isPlan, engine), resourcemutator.SecretScopeFixups(engine), deploy.StatePull(), @@ -31,7 +30,6 @@ func PreDeployChecks(ctx context.Context, b *bundle.Bundle, isPlan bool, engine mutator.ValidateLifecycleStarted(engine), mutator.ValidateCascadeOnDestroy(engine), mutator.ValidateJobRunTriggers(), - mutator.ResolveJobRunFileTriggers(), statemgmt.CheckRunningResource(engine), ) } From 75eaf87409d30c529ea0342a4a23985f9f7f121b Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Wed, 19 Aug 2026 19:58:11 +0000 Subject: [PATCH 09/75] acc: expand on_file_change glob coverage and skip ignored files Honor .gitignore and sync.exclude when hashing matches. Cover mysubdir globs, READPLAN, and keep LF on Windows so restored files hash cleanly. --- .../job_runs/on_file_change_glob/.gitignore | 2 + .../on_file_change_glob/databricks.yml | 1 + .../on_file_change_glob/mysubdir/a.txt | 1 + .../on_file_change_glob/out.test.toml | 1 + .../job_runs/on_file_change_glob/output.txt | 183 +++++++++++++++--- .../job_runs/on_file_change_glob/script | 39 ++-- .../job_runs/on_file_change_glob/test.toml | 6 +- .../mutator/resolve_job_run_file_triggers.go | 65 ++++++- .../resolve_job_run_file_triggers_test.go | 40 ++++ 9 files changed, 287 insertions(+), 51 deletions(-) create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_glob/.gitignore create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_glob/mysubdir/a.txt diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/.gitignore b/acceptance/bundle/resources/job_runs/on_file_change_glob/.gitignore new file mode 100644 index 00000000000..57433d43f47 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/.gitignore @@ -0,0 +1,2 @@ +# Matches migrations/* but sync skips it, so a content edit must not re-fire. +migrations/ignored.txt diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/databricks.yml b/acceptance/bundle/resources/job_runs/on_file_change_glob/databricks.yml index 3b148538f38..d47fcf44049 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/databricks.yml +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/databricks.yml @@ -16,3 +16,4 @@ resources: lifecycle: triggers: - on_file_change: migrations/* + - on_file_change: mysubdir/*.txt diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/mysubdir/a.txt b/acceptance/bundle/resources/job_runs/on_file_change_glob/mysubdir/a.txt new file mode 100644 index 00000000000..62e0af52c19 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/mysubdir/a.txt @@ -0,0 +1 @@ +sub diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/out.test.toml b/acceptance/bundle/resources/job_runs/on_file_change_glob/out.test.toml index 0938e678987..57b0f616850 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/out.test.toml +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/out.test.toml @@ -1,2 +1,3 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt index 798b459a9bd..d424482b961 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt @@ -6,7 +6,7 @@ Output from job_runs.my_run: id=[MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[MY_ Output from job_runs.my_run: id=[MY_RUN_ID]: SUCCESS Created job_runs.my_run Created jobs.my_job -Files: 8 uploaded, 0 deleted +Files: 10 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged >>> read_id.py my_job @@ -23,12 +23,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged } === editing one matched file re-fires ->>> [CLI] bundle plan -recreate job_runs.my_run - -Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged - ->>> changes +>>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { "lifecycle": { "action": "recreate", @@ -37,15 +32,17 @@ Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged "triggers": { "on_file_change": { "migrations/001.txt": "[FILE_HASH][0]", - "migrations/002.txt": "[FILE_HASH][1]" + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][2]" } } }, "new": { "triggers": { "on_file_change": { - "migrations/001.txt": "[FILE_HASH][2]", - "migrations/002.txt": "[FILE_HASH][1]" + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][2]" } } } @@ -54,7 +51,7 @@ Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged "action": "recreate", "reason": "immutable", "old": "[FILE_HASH][0]", - "new": "[FILE_HASH][2]" + "new": "[FILE_HASH][3]" }, "result_state": { "action": "skip", @@ -64,13 +61,35 @@ Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged } } -=== a new matching file re-fires ->>> [CLI] bundle plan -recreate job_runs.my_run +=== bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-glob/default/files... +Output from job_runs.my_run: id=[MY_RUN_ID_2]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_2]?o=[NUMID] +Output from job_runs.my_run: id=[MY_RUN_ID_2]: SUCCESS +Recreated job_runs.my_run +Files: 4 uploaded, 0 deleted +Resources: 1 created, 0 changed, 1 deleted, 1 unchanged + +>>> print_requests.py --keep //jobs/runs/delete +{ + "method": "POST", + "path": "/api/2.2/jobs/runs/delete", + "body": { + "run_id": [MY_RUN_ID] + } +} -Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged +>>> print_requests.py //jobs/run-now +{ + "method": "POST", + "path": "/api/2.2/jobs/run-now", + "body": { + "idempotency_token": "[UUID]", + "job_id": [MY_JOB_ID] + } +} ->>> changes +=== editing a file in a subdirectory re-fires +>>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { "lifecycle": { "action": "recreate", @@ -78,17 +97,58 @@ Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged "old": { "triggers": { "on_file_change": { - "migrations/001.txt": "[FILE_HASH][0]", - "migrations/002.txt": "[FILE_HASH][1]" + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][2]" } } }, "new": { "triggers": { "on_file_change": { - "migrations/001.txt": "[FILE_HASH][0]", + "migrations/001.txt": "[FILE_HASH][3]", "migrations/002.txt": "[FILE_HASH][1]", - "migrations/003.txt": "[FILE_HASH][3]" + "mysubdir/a.txt": "[FILE_HASH][4]" + } + } + } + }, + "lifecycle.triggers.on_file_change['mysubdir/a.txt']": { + "action": "recreate", + "reason": "immutable", + "old": "[FILE_HASH][2]", + "new": "[FILE_HASH][4]" + }, + "result_state": { + "action": "skip", + "reason": "remote_already_set", + "new": "SUCCESS", + "remote": "SUCCESS" + } +} + +=== a new matching file re-fires +>>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json +{ + "lifecycle": { + "action": "recreate", + "reason": "immutable", + "old": { + "triggers": { + "on_file_change": { + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][2]" + } + } + }, + "new": { + "triggers": { + "on_file_change": { + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "migrations/003.txt": "[FILE_HASH][5]", + "mysubdir/a.txt": "[FILE_HASH][2]" } } } @@ -96,7 +156,7 @@ Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged "lifecycle.triggers.on_file_change['migrations/003.txt']": { "action": "recreate", "reason": "immutable", - "new": "[FILE_HASH][3]" + "new": "[FILE_HASH][5]" }, "result_state": { "action": "skip", @@ -107,12 +167,7 @@ Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged } === removing a matched file re-fires ->>> [CLI] bundle plan -recreate job_runs.my_run - -Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged - ->>> changes +>>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { "lifecycle": { "action": "recreate", @@ -120,15 +175,17 @@ Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged "old": { "triggers": { "on_file_change": { - "migrations/001.txt": "[FILE_HASH][0]", - "migrations/002.txt": "[FILE_HASH][1]" + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][2]" } } }, "new": { "triggers": { "on_file_change": { - "migrations/001.txt": "[FILE_HASH][0]" + "migrations/001.txt": "[FILE_HASH][3]", + "mysubdir/a.txt": "[FILE_HASH][2]" } } } @@ -147,12 +204,74 @@ Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged } === editing a file below the glob does not re-fire +>>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json +{ + "lifecycle": { + "action": "skip", + "reason": "missing_in_remote", + "old": { + "triggers": { + "on_file_change": { + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][2]" + } + } + }, + "new": { + "triggers": { + "on_file_change": { + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][2]" + } + } + } + }, + "result_state": { + "action": "skip", + "reason": "remote_already_set", + "new": "SUCCESS", + "remote": "SUCCESS" + } +} + +=== editing a gitignored file does not re-fire >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged === ** is not recursive: it matches the same files as * ->>> [CLI] bundle plan -Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged +>>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json +{ + "lifecycle": { + "action": "skip", + "reason": "missing_in_remote", + "old": { + "triggers": { + "on_file_change": { + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][2]" + } + } + }, + "new": { + "triggers": { + "on_file_change": { + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][2]" + } + } + } + }, + "result_state": { + "action": "skip", + "reason": "remote_already_set", + "new": "SUCCESS", + "remote": "SUCCESS" + } +} >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/script b/acceptance/bundle/resources/job_runs/on_file_change_glob/script index 8feeebcd067..983cb9ac401 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/script +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/script @@ -4,8 +4,10 @@ cleanup() { } trap cleanup EXIT +# Plan once, print JSON changes, leave tmp.plan.json for readplanarg. changes() { - $CLI bundle plan -o json | jq '.plan["resources.job_runs.my_run"].changes' + $CLI bundle plan -o json > tmp.plan.json + trace jq '.plan["resources.job_runs.my_run"].changes' tmp.plan.json } title "deploy hashes the matched files" @@ -14,34 +16,45 @@ trace read_id.py my_job read_id.py my_run > /dev/null trace print_requests.py //jobs/run-now -# Each phase below plans against the state deployed above and then restores the -# tree, so every plan is compared against the same baseline. - title "editing one matched file re-fires" update_file.py migrations/001.txt "one" "one-edited" -trace $CLI bundle plan -trace changes -update_file.py migrations/001.txt "one-edited" "one" +changes +# The deploy is not traced: readplanarg makes the command line differ per variant. +title "bundle deploy\n" +$CLI bundle deploy $(readplanarg tmp.plan.json) +read_id.py my_run > /dev/null +trace print_requests.py --keep //jobs/runs/delete +trace print_requests.py //jobs/run-now + +title "editing a file in a subdirectory re-fires" +# printf keeps LF on Windows, where update_file.py's text-mode rewrite would +# restore CRLF and leave the file hashing differently than the deploy above. +printf 'sub-edited\n' > mysubdir/a.txt +changes +printf 'sub\n' > mysubdir/a.txt title "a new matching file re-fires" printf 'three\n' > migrations/003.txt -trace $CLI bundle plan -trace changes +changes rm migrations/003.txt title "removing a matched file re-fires" mv migrations/002.txt tmp.002.txt -trace $CLI bundle plan -trace changes +changes mv tmp.002.txt migrations/002.txt title "editing a file below the glob does not re-fire" # migrations/* matches the archive directory itself, which is skipped: the glob # does not descend into it. update_file.py migrations/archive/003.txt "three" "three-edited" -trace $CLI bundle plan +changes update_file.py migrations/archive/003.txt "three-edited" "three" +title "editing a gitignored file does not re-fire" +update_file.py migrations/ignored.txt "ignored" "ignored-edited" +trace $CLI bundle plan +update_file.py migrations/ignored.txt "ignored-edited" "ignored" + title "** is not recursive: it matches the same files as *" update_file.py databricks.yml "migrations/*" "migrations/**" -trace $CLI bundle plan +changes diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml b/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml index 1c349066b94..3516572564d 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml @@ -1,4 +1,8 @@ -Ignore = ["tmp.002.txt"] +# Deploy both by re-planning and from a plan saved on disk, so the hashes +# computed during planning survive plan serialization. +EnvMatrix.READPLAN = ["", "1"] + +Ignore = ["tmp.plan.json", "tmp.002.txt"] # See on_file_change/test.toml: mask content hashes before the parent's # `\d{8,}` rule can mangle them, keeping distinct hashes distinguishable. diff --git a/bundle/config/mutator/resolve_job_run_file_triggers.go b/bundle/config/mutator/resolve_job_run_file_triggers.go index ead17c59e17..d74b9841c57 100644 --- a/bundle/config/mutator/resolve_job_run_file_triggers.go +++ b/bundle/config/mutator/resolve_job_run_file_triggers.go @@ -13,6 +13,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" + libsync "github.com/databricks/cli/libs/sync" ) // missingFileHash marks a pattern with no matching file so appear/disappear recreates. @@ -30,8 +31,13 @@ func (*resolveJobRunFileTriggers) Name() string { return "ResolveJobRunFileTriggers" } -func (*resolveJobRunFileTriggers) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { +func (*resolveJobRunFileTriggers) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { var diags diag.Diagnostics + syncable, d := syncableRelPaths(ctx, b) + if d.HasError() { + return d + } + diags = diags.Extend(d) for name, jr := range b.Config.Resources.JobRuns { if jr == nil || jr.Lifecycle == nil { continue @@ -42,7 +48,7 @@ func (*resolveJobRunFileTriggers) Apply(_ context.Context, b *bundle.Bundle) dia continue } path := fmt.Sprintf("resources.job_runs.%s.lifecycle.triggers[%d].on_file_change", name, i) - hashes, d := resolveFileTrigger(b, path, strings.TrimSpace(*t.OnFileChange)) + hashes, d := resolveFileTrigger(b, path, strings.TrimSpace(*t.OnFileChange), syncable) diags = diags.Extend(d) maps.Copy(out, hashes) } @@ -55,7 +61,42 @@ func (*resolveJobRunFileTriggers) Apply(_ context.Context, b *bundle.Bundle) dia return diags } -func resolveFileTrigger(b *bundle.Bundle, loc, pattern string) (map[string]string, diag.Diagnostics) { +// syncableRelPaths is the set of relative paths sync would upload. +func syncableRelPaths(ctx context.Context, b *bundle.Bundle) (map[string]struct{}, diag.Diagnostics) { + var diags diag.Diagnostics + needs := false + for _, jr := range b.Config.Resources.JobRuns { + if jr != nil && jr.HasOnFileChange() { + needs = true + break + } + } + if !needs { + return nil, diags + } + + fl, err := libsync.NewFileList(ctx, b.WorktreeRoot, b.SyncRoot, b.Config.Sync.Paths, b.Config.Sync.Include, b.Config.Sync.Exclude) + if err != nil { + return nil, diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: list sync files: %s", err), + }) + } + files, err := fl.Files(ctx) + if err != nil { + return nil, diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: list sync files: %s", err), + }) + } + out := make(map[string]struct{}, len(files)) + for _, f := range files { + out[filepath.ToSlash(f.Relative)] = struct{}{} + } + return out, diags +} + +func resolveFileTrigger(b *bundle.Bundle, loc, pattern string, syncable map[string]struct{}) (map[string]string, diag.Diagnostics) { var diags diag.Diagnostics out := make(map[string]string) localPattern := filepath.FromSlash(pattern) @@ -84,6 +125,7 @@ func resolveFileTrigger(b *bundle.Bundle, loc, pattern string) (map[string]strin }) } regularMatches := 0 + ignoredMatches := 0 sawNonRegular := false for _, match := range matches { info, err := os.Stat(match) @@ -99,7 +141,6 @@ func resolveFileTrigger(b *bundle.Bundle, loc, pattern string) (map[string]strin sawNonRegular = true continue } - regularMatches++ rel, err := filepath.Rel(b.SyncRootPath, match) if err != nil || !filepath.IsLocal(rel) { diags = diags.Append(diag.Diagnostic{ @@ -109,6 +150,12 @@ func resolveFileTrigger(b *bundle.Bundle, loc, pattern string) (map[string]strin }) continue } + // Same membership as sync: .gitignore and sync.exclude drop a glob match. + if _, ok := syncable[filepath.ToSlash(rel)]; !ok { + ignoredMatches++ + continue + } + regularMatches++ hash, err := hashFile(match) if err != nil { diags = diags.Append(diag.Diagnostic{ @@ -122,13 +169,21 @@ func resolveFileTrigger(b *bundle.Bundle, loc, pattern string) (map[string]strin } // A directory-only match would otherwise leave ResolvedFileTriggers empty // and silently disarm the trigger while config still sets on_file_change. - if regularMatches == 0 && sawNonRegular { + if regularMatches == 0 && sawNonRegular && ignoredMatches == 0 { diags = diags.Append(diag.Diagnostic{ Severity: diag.Error, Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: pattern %q matches no regular files", pattern), Locations: b.Config.GetLocations(loc), }) } + if len(out) == 0 && ignoredMatches > 0 { + out[filepath.ToSlash(pattern)] = missingFileHash + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Warning, + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: no files match %q", pattern), + Locations: b.Config.GetLocations(loc), + }) + } return out, diags } diff --git a/bundle/config/mutator/resolve_job_run_file_triggers_test.go b/bundle/config/mutator/resolve_job_run_file_triggers_test.go index 3ec663c89bd..344fed25657 100644 --- a/bundle/config/mutator/resolve_job_run_file_triggers_test.go +++ b/bundle/config/mutator/resolve_job_run_file_triggers_test.go @@ -12,6 +12,7 @@ import ( "github.com/databricks/cli/bundle/config/mutator" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/vfs" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -85,7 +86,10 @@ func TestResolveJobRunFileTriggers(t *testing.T) { patB := "subdir/*.py" b := &bundle.Bundle{ SyncRootPath: dir, + SyncRoot: vfs.MustNew(dir), + WorktreeRoot: vfs.MustNew(dir), Config: config.Root{ + Sync: config.Sync{Paths: []string{"."}}, Resources: config.Resources{ JobRuns: map[string]*resources.JobRun{ "my_run": { @@ -135,6 +139,38 @@ func TestResolveJobRunFileTriggers(t *testing.T) { assert.Empty(t, b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers) }) + t.Run("skips gitignored files", func(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, ".gitignore"), []byte("skip.txt\n"), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "keep.txt"), []byte("keep"), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "skip.txt"), []byte("skip"), 0o644)) + + b := bundleWithFileTrigger(dir, "*.txt") + diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) + require.False(t, diags.HasError()) + + hashes := b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers + require.Len(t, hashes, 1) + assert.Equal(t, contentHash("keep"), hashes["keep.txt"]) + assert.NotContains(t, hashes, "skip.txt") + }) + + t.Run("skips sync.exclude files", func(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "keep.txt"), []byte("keep"), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "skip.txt"), []byte("skip"), 0o644)) + + b := bundleWithFileTrigger(dir, "*.txt") + b.Config.Sync.Exclude = []string{"skip.txt"} + diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) + require.False(t, diags.HasError()) + + hashes := b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers + require.Len(t, hashes, 1) + assert.Equal(t, contentHash("keep"), hashes["keep.txt"]) + assert.NotContains(t, hashes, "skip.txt") + }) + t.Run("trims pattern whitespace", func(t *testing.T) { dir := t.TempDir() require.NoError(t, os.WriteFile(filepath.Join(dir, "seed.txt"), []byte("v1"), 0o644)) @@ -148,9 +184,13 @@ func TestResolveJobRunFileTriggers(t *testing.T) { } func bundleWithFileTrigger(syncRoot, pattern string) *bundle.Bundle { + root := vfs.MustNew(syncRoot) return &bundle.Bundle{ SyncRootPath: syncRoot, + SyncRoot: root, + WorktreeRoot: root, Config: config.Root{ + Sync: config.Sync{Paths: []string{"."}}, Resources: config.Resources{ JobRuns: map[string]*resources.JobRun{ "my_run": { From 4e43352cf8a38d3f3535e5895b21191288a6e359 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Wed, 19 Aug 2026 20:10:03 +0000 Subject: [PATCH 10/75] acc: force-add gitignored on_file_change fixture The glob test edits migrations/ignored.txt, but the test .gitignore excluded it from the commit so CI clones would miss the file. --- .../job_runs/on_file_change_glob/migrations/ignored.txt | 1 + 1 file changed, 1 insertion(+) create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/ignored.txt diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/ignored.txt b/acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/ignored.txt new file mode 100644 index 00000000000..ea10ec85c10 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/ignored.txt @@ -0,0 +1 @@ +ignored From a7a3275042c4371401da2746fa10690951f9be6c Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Thu, 20 Aug 2026 08:11:43 +0000 Subject: [PATCH 11/75] acc: drop READPLAN from the on_file_change glob test Plan serialization is already covered by on_file_change; the glob cases only assert the hash map. --- .../resources/job_runs/on_file_change_glob/out.test.toml | 1 - .../bundle/resources/job_runs/on_file_change_glob/output.txt | 2 +- .../bundle/resources/job_runs/on_file_change_glob/script | 5 +---- .../bundle/resources/job_runs/on_file_change_glob/test.toml | 4 ---- 4 files changed, 2 insertions(+), 10 deletions(-) diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/out.test.toml b/acceptance/bundle/resources/job_runs/on_file_change_glob/out.test.toml index 57b0f616850..0938e678987 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/out.test.toml +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/out.test.toml @@ -1,3 +1,2 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt index d424482b961..bd7d08557f5 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt @@ -61,7 +61,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged } } -=== bundle deploy +>>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-glob/default/files... Output from job_runs.my_run: id=[MY_RUN_ID_2]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_2]?o=[NUMID] Output from job_runs.my_run: id=[MY_RUN_ID_2]: SUCCESS diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/script b/acceptance/bundle/resources/job_runs/on_file_change_glob/script index 983cb9ac401..23390d23a55 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/script +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/script @@ -4,7 +4,6 @@ cleanup() { } trap cleanup EXIT -# Plan once, print JSON changes, leave tmp.plan.json for readplanarg. changes() { $CLI bundle plan -o json > tmp.plan.json trace jq '.plan["resources.job_runs.my_run"].changes' tmp.plan.json @@ -19,9 +18,7 @@ trace print_requests.py //jobs/run-now title "editing one matched file re-fires" update_file.py migrations/001.txt "one" "one-edited" changes -# The deploy is not traced: readplanarg makes the command line differ per variant. -title "bundle deploy\n" -$CLI bundle deploy $(readplanarg tmp.plan.json) +trace $CLI bundle deploy read_id.py my_run > /dev/null trace print_requests.py --keep //jobs/runs/delete trace print_requests.py //jobs/run-now diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml b/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml index 3516572564d..d372fd54eae 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml @@ -1,7 +1,3 @@ -# Deploy both by re-planning and from a plan saved on disk, so the hashes -# computed during planning survive plan serialization. -EnvMatrix.READPLAN = ["", "1"] - Ignore = ["tmp.plan.json", "tmp.002.txt"] # See on_file_change/test.toml: mask content hashes before the parent's From 786a2d260ff7ca2e29236ff06e7b0384a8c403ff Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Thu, 20 Aug 2026 12:34:44 +0000 Subject: [PATCH 12/75] acc: restore READPLAN on the on_file_change glob test Cover deploying glob hashes from a saved plan, not only from a live re-plan. --- .../resources/job_runs/on_file_change_glob/out.test.toml | 1 + .../bundle/resources/job_runs/on_file_change_glob/output.txt | 2 +- .../bundle/resources/job_runs/on_file_change_glob/script | 5 ++++- .../bundle/resources/job_runs/on_file_change_glob/test.toml | 4 ++++ 4 files changed, 10 insertions(+), 2 deletions(-) diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/out.test.toml b/acceptance/bundle/resources/job_runs/on_file_change_glob/out.test.toml index 0938e678987..57b0f616850 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/out.test.toml +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/out.test.toml @@ -1,2 +1,3 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt index bd7d08557f5..d424482b961 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt @@ -61,7 +61,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged } } ->>> [CLI] bundle deploy +=== bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-glob/default/files... Output from job_runs.my_run: id=[MY_RUN_ID_2]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_2]?o=[NUMID] Output from job_runs.my_run: id=[MY_RUN_ID_2]: SUCCESS diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/script b/acceptance/bundle/resources/job_runs/on_file_change_glob/script index 23390d23a55..983cb9ac401 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/script +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/script @@ -4,6 +4,7 @@ cleanup() { } trap cleanup EXIT +# Plan once, print JSON changes, leave tmp.plan.json for readplanarg. changes() { $CLI bundle plan -o json > tmp.plan.json trace jq '.plan["resources.job_runs.my_run"].changes' tmp.plan.json @@ -18,7 +19,9 @@ trace print_requests.py //jobs/run-now title "editing one matched file re-fires" update_file.py migrations/001.txt "one" "one-edited" changes -trace $CLI bundle deploy +# The deploy is not traced: readplanarg makes the command line differ per variant. +title "bundle deploy\n" +$CLI bundle deploy $(readplanarg tmp.plan.json) read_id.py my_run > /dev/null trace print_requests.py --keep //jobs/runs/delete trace print_requests.py //jobs/run-now diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml b/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml index d372fd54eae..3516572564d 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml @@ -1,3 +1,7 @@ +# Deploy both by re-planning and from a plan saved on disk, so the hashes +# computed during planning survive plan serialization. +EnvMatrix.READPLAN = ["", "1"] + Ignore = ["tmp.plan.json", "tmp.002.txt"] # See on_file_change/test.toml: mask content hashes before the parent's From 76c0d7082428fa40f77ed04863dbf3153fbf24e1 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Thu, 20 Aug 2026 14:08:42 +0000 Subject: [PATCH 13/75] job_runs: address on_file_change review comments Keep the gitignore fixture as dot_gitignore so it does not affect the repo while developing, drop unit tests that duplicate acceptance coverage, and allocate trigger state once when a trigger is armed. --- .../{.gitignore => dot_gitignore} | 0 .../job_runs/on_file_change_glob/script | 3 + .../job_runs/on_file_change_glob/test.toml | 2 +- .../resolve_job_run_file_triggers_test.go | 139 +----------------- bundle/direct/dresources/job_run.go | 23 +-- bundle/direct/dresources/job_run_test.go | 36 ++--- 6 files changed, 23 insertions(+), 180 deletions(-) rename acceptance/bundle/resources/job_runs/on_file_change_glob/{.gitignore => dot_gitignore} (100%) diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/.gitignore b/acceptance/bundle/resources/job_runs/on_file_change_glob/dot_gitignore similarity index 100% rename from acceptance/bundle/resources/job_runs/on_file_change_glob/.gitignore rename to acceptance/bundle/resources/job_runs/on_file_change_glob/dot_gitignore diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/script b/acceptance/bundle/resources/job_runs/on_file_change_glob/script index 983cb9ac401..dd737f242b5 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/script +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/script @@ -1,3 +1,6 @@ +# Fixture is named dot_gitignore so a real .gitignore does not affect git while developing. +mv dot_gitignore .gitignore + cleanup() { trace $CLI bundle destroy --auto-approve rm -f out.requests.txt diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml b/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml index 3516572564d..98c98631e5b 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml @@ -2,7 +2,7 @@ # computed during planning survive plan serialization. EnvMatrix.READPLAN = ["", "1"] -Ignore = ["tmp.plan.json", "tmp.002.txt"] +Ignore = ["tmp.plan.json", "tmp.002.txt", ".gitignore"] # See on_file_change/test.toml: mask content hashes before the parent's # `\d{8,}` rule can mangle them, keeping distinct hashes distinguishable. diff --git a/bundle/config/mutator/resolve_job_run_file_triggers_test.go b/bundle/config/mutator/resolve_job_run_file_triggers_test.go index 344fed25657..1cc54eecf4f 100644 --- a/bundle/config/mutator/resolve_job_run_file_triggers_test.go +++ b/bundle/config/mutator/resolve_job_run_file_triggers_test.go @@ -11,14 +11,13 @@ import ( "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/mutator" "github.com/databricks/cli/bundle/config/resources" - "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/vfs" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestResolveJobRunFileTriggers(t *testing.T) { - t.Run("matches files and fills hashes", func(t *testing.T) { + t.Run("hashes file contents with sha256", func(t *testing.T) { dir := t.TempDir() require.NoError(t, os.WriteFile(filepath.Join(dir, "a.txt"), []byte("hello"), 0o644)) require.NoError(t, os.WriteFile(filepath.Join(dir, "b.txt"), []byte("world"), 0o644)) @@ -35,142 +34,6 @@ func TestResolveJobRunFileTriggers(t *testing.T) { assert.Equal(t, contentHash("world"), hashes["b.txt"]) }) - t.Run("no matches warns and stores empty hash", func(t *testing.T) { - dir := t.TempDir() - pattern := "missing.txt" - b := bundleWithFileTrigger(dir, pattern) - - diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) - require.False(t, diags.HasError()) - require.Len(t, diags, 1) - assert.Equal(t, diag.Warning, diags[0].Severity) - assert.Contains(t, diags[0].Summary, `no files match "missing.txt"`) - - hashes := b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers - require.Len(t, hashes, 1) - assert.Empty(t, hashes["missing.txt"]) - }) - - t.Run("no file triggers is a no-op", func(t *testing.T) { - dir := t.TempDir() - on := true - b := &bundle.Bundle{ - SyncRootPath: dir, - Config: config.Root{ - Resources: config.Resources{ - JobRuns: map[string]*resources.JobRun{ - "my_run": { - Lifecycle: &resources.JobRunLifecycle{ - Triggers: []resources.JobRunTrigger{ - {OnBundleDeploy: &on}, - }, - }, - }, - }, - }, - }, - } - - diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) - assert.Empty(t, diags) - assert.Nil(t, b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers) - }) - - t.Run("multiple patterns merge into one map", func(t *testing.T) { - dir := t.TempDir() - require.NoError(t, os.WriteFile(filepath.Join(dir, "a.txt"), []byte("aaa"), 0o644)) - require.NoError(t, os.MkdirAll(filepath.Join(dir, "subdir"), 0o755)) - require.NoError(t, os.WriteFile(filepath.Join(dir, "subdir", "x.py"), []byte("bbb"), 0o644)) - - patA := "a.txt" - patB := "subdir/*.py" - b := &bundle.Bundle{ - SyncRootPath: dir, - SyncRoot: vfs.MustNew(dir), - WorktreeRoot: vfs.MustNew(dir), - Config: config.Root{ - Sync: config.Sync{Paths: []string{"."}}, - Resources: config.Resources{ - JobRuns: map[string]*resources.JobRun{ - "my_run": { - Lifecycle: &resources.JobRunLifecycle{ - Triggers: []resources.JobRunTrigger{ - {OnFileChange: &patA}, - {OnFileChange: &patB}, - }, - }, - }, - }, - }, - }, - } - - diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) - require.False(t, diags.HasError()) - - hashes := b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers - require.Len(t, hashes, 2) - assert.Equal(t, contentHash("aaa"), hashes["a.txt"]) - assert.Equal(t, contentHash("bbb"), hashes["subdir/x.py"]) - }) - - t.Run("pattern outside sync root is an error", func(t *testing.T) { - dir := t.TempDir() - pattern := "../outside.txt" - b := bundleWithFileTrigger(dir, pattern) - - diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) - require.True(t, diags.HasError()) - require.Len(t, diags, 1) - assert.Contains(t, diags[0].Summary, `not under the sync root`) - assert.Empty(t, b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers) - }) - - t.Run("directory-only match is an error", func(t *testing.T) { - dir := t.TempDir() - require.NoError(t, os.MkdirAll(filepath.Join(dir, "migrations"), 0o755)) - pattern := "migrations" - b := bundleWithFileTrigger(dir, pattern) - - diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) - require.True(t, diags.HasError()) - require.Len(t, diags, 1) - assert.Contains(t, diags[0].Summary, `matches no regular files`) - assert.Empty(t, b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers) - }) - - t.Run("skips gitignored files", func(t *testing.T) { - dir := t.TempDir() - require.NoError(t, os.WriteFile(filepath.Join(dir, ".gitignore"), []byte("skip.txt\n"), 0o644)) - require.NoError(t, os.WriteFile(filepath.Join(dir, "keep.txt"), []byte("keep"), 0o644)) - require.NoError(t, os.WriteFile(filepath.Join(dir, "skip.txt"), []byte("skip"), 0o644)) - - b := bundleWithFileTrigger(dir, "*.txt") - diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) - require.False(t, diags.HasError()) - - hashes := b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers - require.Len(t, hashes, 1) - assert.Equal(t, contentHash("keep"), hashes["keep.txt"]) - assert.NotContains(t, hashes, "skip.txt") - }) - - t.Run("skips sync.exclude files", func(t *testing.T) { - dir := t.TempDir() - require.NoError(t, os.WriteFile(filepath.Join(dir, "keep.txt"), []byte("keep"), 0o644)) - require.NoError(t, os.WriteFile(filepath.Join(dir, "skip.txt"), []byte("skip"), 0o644)) - - b := bundleWithFileTrigger(dir, "*.txt") - b.Config.Sync.Exclude = []string{"skip.txt"} - diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) - require.False(t, diags.HasError()) - - hashes := b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers - require.Len(t, hashes, 1) - assert.Equal(t, contentHash("keep"), hashes["keep.txt"]) - assert.NotContains(t, hashes, "skip.txt") - }) - t.Run("trims pattern whitespace", func(t *testing.T) { dir := t.TempDir() require.NoError(t, os.WriteFile(filepath.Join(dir, "seed.txt"), []byte("v1"), 0o644)) diff --git a/bundle/direct/dresources/job_run.go b/bundle/direct/dresources/job_run.go index c7bc11d62a0..ab85857e731 100644 --- a/bundle/direct/dresources/job_run.go +++ b/bundle/direct/dresources/job_run.go @@ -110,25 +110,14 @@ func (*ResourceJobRun) PrepareState(input *resources.JobRun) *JobRunState { ResultState: jobs.RunResultStateSuccess, Lifecycle: nil, } - var triggers *JobRunTriggersState - if input.HasOnBundleDeploy() { - triggers = &JobRunTriggersState{ - OnBundleDeploy: uuid.NewString(), - OnFileChange: nil, - } + if !input.HasOnBundleDeploy() && len(input.ResolvedFileTriggers) == 0 { + return state } - if len(input.ResolvedFileTriggers) > 0 { - if triggers == nil { - triggers = &JobRunTriggersState{ - OnBundleDeploy: "", - OnFileChange: nil, - } - } - triggers.OnFileChange = input.ResolvedFileTriggers - } - if triggers != nil { - state.Lifecycle = &JobRunLifecycleState{Triggers: triggers} + triggers := &JobRunTriggersState{OnFileChange: input.ResolvedFileTriggers} + if input.HasOnBundleDeploy() { + triggers.OnBundleDeploy = uuid.NewString() } + state.Lifecycle = &JobRunLifecycleState{Triggers: triggers} return state } diff --git a/bundle/direct/dresources/job_run_test.go b/bundle/direct/dresources/job_run_test.go index 2c5815d3a20..72de912e9d6 100644 --- a/bundle/direct/dresources/job_run_test.go +++ b/bundle/direct/dresources/job_run_test.go @@ -354,36 +354,24 @@ func TestJobRunPrepareStateRequiresSuccess(t *testing.T) { } func TestJobRunPrepareStateOnBundleDeploy(t *testing.T) { - t.Run("unset", func(t *testing.T) { - state := (&ResourceJobRun{}).PrepareState(&resources.JobRun{}) - assert.Nil(t, state.Lifecycle) - }) - - t.Run("armed", func(t *testing.T) { - on := true - input := &resources.JobRun{ - Lifecycle: &resources.JobRunLifecycle{ - Triggers: []resources.JobRunTrigger{{OnBundleDeploy: &on}}, - }, - } - first := (&ResourceJobRun{}).PrepareState(input) - require.NotNil(t, first.Lifecycle) - require.NotNil(t, first.Lifecycle.Triggers) - assert.NotEmpty(t, first.Lifecycle.Triggers.OnBundleDeploy) + on := true + input := &resources.JobRun{ + Lifecycle: &resources.JobRunLifecycle{ + Triggers: []resources.JobRunTrigger{{OnBundleDeploy: &on}}, + }, + } + first := (&ResourceJobRun{}).PrepareState(input) + require.NotNil(t, first.Lifecycle) + require.NotNil(t, first.Lifecycle.Triggers) + assert.NotEmpty(t, first.Lifecycle.Triggers.OnBundleDeploy) - second := (&ResourceJobRun{}).PrepareState(input) - assert.NotEqual(t, first.Lifecycle.Triggers.OnBundleDeploy, second.Lifecycle.Triggers.OnBundleDeploy) - }) + second := (&ResourceJobRun{}).PrepareState(input) + assert.NotEqual(t, first.Lifecycle.Triggers.OnBundleDeploy, second.Lifecycle.Triggers.OnBundleDeploy) } func TestJobRunPrepareStateOnFileChange(t *testing.T) { hashes := map[string]string{"a.txt": "abc"} - t.Run("unset", func(t *testing.T) { - state := (&ResourceJobRun{}).PrepareState(&resources.JobRun{}) - assert.Nil(t, state.Lifecycle) - }) - t.Run("armed", func(t *testing.T) { state := (&ResourceJobRun{}).PrepareState(&resources.JobRun{ ResolvedFileTriggers: hashes, From e1fbadc2cbab7c6cf605df8ec9be6134be88994a Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Thu, 20 Aug 2026 14:16:35 +0000 Subject: [PATCH 14/75] job_runs: list every trigger field to satisfy exhaustruct The previous simplification omitted OnBundleDeploy from the struct literal, which the linter rejects. --- bundle/direct/dresources/job_run.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/bundle/direct/dresources/job_run.go b/bundle/direct/dresources/job_run.go index ab85857e731..63ed81cb78a 100644 --- a/bundle/direct/dresources/job_run.go +++ b/bundle/direct/dresources/job_run.go @@ -113,11 +113,16 @@ func (*ResourceJobRun) PrepareState(input *resources.JobRun) *JobRunState { if !input.HasOnBundleDeploy() && len(input.ResolvedFileTriggers) == 0 { return state } - triggers := &JobRunTriggersState{OnFileChange: input.ResolvedFileTriggers} + onBundleDeploy := "" if input.HasOnBundleDeploy() { - triggers.OnBundleDeploy = uuid.NewString() + onBundleDeploy = uuid.NewString() + } + state.Lifecycle = &JobRunLifecycleState{ + Triggers: &JobRunTriggersState{ + OnBundleDeploy: onBundleDeploy, + OnFileChange: input.ResolvedFileTriggers, + }, } - state.Lifecycle = &JobRunLifecycleState{Triggers: triggers} return state } From eb7c422e37ce9a7ed71ca837b47996b51836d437 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Fri, 21 Aug 2026 12:31:38 +0000 Subject: [PATCH 15/75] job_runs: keep empty nested lifecycle on state and remote Always persist the same lifecycle.triggers shape so trigger fingerprints compare at the same paths. Wrap on_file_change hashes in files so dropping the trigger updates instead of recreating. --- acceptance/bundle/refschema/out.fields.txt | 11 +- .../resources/job_runs/failed_run/output.txt | 10 ++ .../job_runs/interrupted_run/output.txt | 10 ++ .../job_runs/on_bundle_deploy/output.txt | 22 +-- .../job_runs/on_file_change/output.txt | 32 +--- .../job_runs/on_file_change_glob/output.txt | 144 ++++++------------ .../resources/job_runs/redeploy/output.txt | 10 ++ bundle/direct/dresources/job_run.go | 101 ++++++++---- bundle/direct/dresources/job_run_test.go | 94 +++++++++--- bundle/direct/dresources/resources.yml | 3 + bundle/direct/dresources/type_test.go | 4 - 11 files changed, 244 insertions(+), 197 deletions(-) diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index b84842ffdc6..d6da3840c2a 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -845,15 +845,16 @@ resources.job_runs.*.jar_params[*] string ALL resources.job_runs.*.job_id int64 ALL resources.job_runs.*.job_parameters map[string]string ALL resources.job_runs.*.job_parameters.* string ALL -resources.job_runs.*.lifecycle *dresources.JobRunLifecycleState STATE +resources.job_runs.*.lifecycle *dresources.JobRunLifecycleState REMOTE STATE resources.job_runs.*.lifecycle *resources.JobRunLifecycle INPUT resources.job_runs.*.lifecycle resources.Lifecycle INPUT resources.job_runs.*.lifecycle.prevent_destroy bool INPUT -resources.job_runs.*.lifecycle.triggers *dresources.JobRunTriggersState STATE +resources.job_runs.*.lifecycle.triggers *dresources.JobRunTriggersState REMOTE STATE resources.job_runs.*.lifecycle.triggers []resources.JobRunTrigger INPUT -resources.job_runs.*.lifecycle.triggers.on_bundle_deploy string STATE -resources.job_runs.*.lifecycle.triggers.on_file_change map[string]string STATE -resources.job_runs.*.lifecycle.triggers.on_file_change.* string STATE +resources.job_runs.*.lifecycle.triggers.on_bundle_deploy string REMOTE STATE +resources.job_runs.*.lifecycle.triggers.on_file_change *dresources.JobRunFileTriggerState REMOTE STATE +resources.job_runs.*.lifecycle.triggers.on_file_change.files map[string]string REMOTE STATE +resources.job_runs.*.lifecycle.triggers.on_file_change.files.* string REMOTE STATE resources.job_runs.*.lifecycle.triggers[*] resources.JobRunTrigger INPUT resources.job_runs.*.lifecycle.triggers[*].on_bundle_deploy *bool INPUT resources.job_runs.*.lifecycle.triggers[*].on_file_change *string INPUT diff --git a/acceptance/bundle/resources/job_runs/failed_run/output.txt b/acceptance/bundle/resources/job_runs/failed_run/output.txt index f8b8dbf4398..0c6bddb0d9b 100644 --- a/acceptance/bundle/resources/job_runs/failed_run/output.txt +++ b/acceptance/bundle/resources/job_runs/failed_run/output.txt @@ -27,11 +27,21 @@ Files: 7 uploaded, 0 deleted "new_state": { "value": { "job_id": [MY_JOB_ID], + "lifecycle": { + "triggers": { + "on_file_change": {} + } + }, "result_state": "SUCCESS" } }, "remote_state": { "job_id": [MY_JOB_ID], + "lifecycle": { + "triggers": { + "on_file_change": {} + } + }, "result_state": "FAILED", "run_id": [MY_RUN_ID], "run_name": "test-job-[UNIQUE_NAME]", diff --git a/acceptance/bundle/resources/job_runs/interrupted_run/output.txt b/acceptance/bundle/resources/job_runs/interrupted_run/output.txt index 25aaf20b64e..47dfb8c7617 100644 --- a/acceptance/bundle/resources/job_runs/interrupted_run/output.txt +++ b/acceptance/bundle/resources/job_runs/interrupted_run/output.txt @@ -43,11 +43,21 @@ Exit code: 1 "new_state": { "value": { "job_id": [MY_JOB_ID], + "lifecycle": { + "triggers": { + "on_file_change": {} + } + }, "result_state": "SUCCESS" } }, "remote_state": { "job_id": [MY_JOB_ID], + "lifecycle": { + "triggers": { + "on_file_change": {} + } + }, "run_id": [MY_RUN_ID], "run_name": "my-job", "run_page_url": "[DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?o=[NUMID]", diff --git a/acceptance/bundle/resources/job_runs/on_bundle_deploy/output.txt b/acceptance/bundle/resources/job_runs/on_bundle_deploy/output.txt index d9524633590..ec8b65a220b 100644 --- a/acceptance/bundle/resources/job_runs/on_bundle_deploy/output.txt +++ b/acceptance/bundle/resources/job_runs/on_bundle_deploy/output.txt @@ -30,20 +30,6 @@ Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle": { - "action": "recreate", - "reason": "immutable", - "old": { - "triggers": { - "on_bundle_deploy": "[UUID]" - } - }, - "new": { - "triggers": { - "on_bundle_deploy": "[UUID]" - } - } - }, "lifecycle.triggers.on_bundle_deploy": { "action": "recreate", "reason": "immutable", @@ -110,14 +96,10 @@ Plan: 0 to add, 1 to change, 0 to delete, 1 unchanged >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle": { + "lifecycle.triggers.on_bundle_deploy": { "action": "update", "reason": "trigger removed", - "old": { - "triggers": { - "on_bundle_deploy": "[UUID]" - } - } + "old": "[UUID]" }, "result_state": { "action": "skip", diff --git a/acceptance/bundle/resources/job_runs/on_file_change/output.txt b/acceptance/bundle/resources/job_runs/on_file_change/output.txt index d03307b90f2..9b662e1936d 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change/output.txt @@ -52,25 +52,17 @@ Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle": { + "lifecycle.triggers.on_file_change.files": { "action": "recreate", "reason": "immutable", "old": { - "triggers": { - "on_file_change": { - "seed.txt": "[FILE_HASH][0]" - } - } + "seed.txt": "[FILE_HASH][0]" }, "new": { - "triggers": { - "on_file_change": { - "seed.txt": "[FILE_HASH][1]" - } - } + "seed.txt": "[FILE_HASH][1]" } }, - "lifecycle.triggers.on_file_change['seed.txt']": { + "lifecycle.triggers.on_file_change.files['seed.txt']": { "action": "recreate", "reason": "immutable", "old": "[FILE_HASH][0]", @@ -125,25 +117,17 @@ Warning: lifecycle.triggers.on_file_change: no files match "seed.txt" in databricks.yml:18:29 { - "lifecycle": { + "lifecycle.triggers.on_file_change.files": { "action": "recreate", "reason": "immutable", "old": { - "triggers": { - "on_file_change": { - "seed.txt": "[FILE_HASH][1]" - } - } + "seed.txt": "[FILE_HASH][1]" }, "new": { - "triggers": { - "on_file_change": { - "seed.txt": "" - } - } + "seed.txt": "" } }, - "lifecycle.triggers.on_file_change['seed.txt']": { + "lifecycle.triggers.on_file_change.files['seed.txt']": { "action": "recreate", "reason": "immutable", "old": "[FILE_HASH][1]", diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt index d424482b961..5b4ef23178c 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt @@ -25,29 +25,21 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged === editing one matched file re-fires >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle": { + "lifecycle.triggers.on_file_change.files": { "action": "recreate", "reason": "immutable", "old": { - "triggers": { - "on_file_change": { - "migrations/001.txt": "[FILE_HASH][0]", - "migrations/002.txt": "[FILE_HASH][1]", - "mysubdir/a.txt": "[FILE_HASH][2]" - } - } + "migrations/001.txt": "[FILE_HASH][0]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][2]" }, "new": { - "triggers": { - "on_file_change": { - "migrations/001.txt": "[FILE_HASH][3]", - "migrations/002.txt": "[FILE_HASH][1]", - "mysubdir/a.txt": "[FILE_HASH][2]" - } - } + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][2]" } }, - "lifecycle.triggers.on_file_change['migrations/001.txt']": { + "lifecycle.triggers.on_file_change.files['migrations/001.txt']": { "action": "recreate", "reason": "immutable", "old": "[FILE_HASH][0]", @@ -91,29 +83,21 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged === editing a file in a subdirectory re-fires >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle": { + "lifecycle.triggers.on_file_change.files": { "action": "recreate", "reason": "immutable", "old": { - "triggers": { - "on_file_change": { - "migrations/001.txt": "[FILE_HASH][3]", - "migrations/002.txt": "[FILE_HASH][1]", - "mysubdir/a.txt": "[FILE_HASH][2]" - } - } + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][2]" }, "new": { - "triggers": { - "on_file_change": { - "migrations/001.txt": "[FILE_HASH][3]", - "migrations/002.txt": "[FILE_HASH][1]", - "mysubdir/a.txt": "[FILE_HASH][4]" - } - } + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][4]" } }, - "lifecycle.triggers.on_file_change['mysubdir/a.txt']": { + "lifecycle.triggers.on_file_change.files['mysubdir/a.txt']": { "action": "recreate", "reason": "immutable", "old": "[FILE_HASH][2]", @@ -130,30 +114,22 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged === a new matching file re-fires >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle": { + "lifecycle.triggers.on_file_change.files": { "action": "recreate", "reason": "immutable", "old": { - "triggers": { - "on_file_change": { - "migrations/001.txt": "[FILE_HASH][3]", - "migrations/002.txt": "[FILE_HASH][1]", - "mysubdir/a.txt": "[FILE_HASH][2]" - } - } + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][2]" }, "new": { - "triggers": { - "on_file_change": { - "migrations/001.txt": "[FILE_HASH][3]", - "migrations/002.txt": "[FILE_HASH][1]", - "migrations/003.txt": "[FILE_HASH][5]", - "mysubdir/a.txt": "[FILE_HASH][2]" - } - } + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "migrations/003.txt": "[FILE_HASH][5]", + "mysubdir/a.txt": "[FILE_HASH][2]" } }, - "lifecycle.triggers.on_file_change['migrations/003.txt']": { + "lifecycle.triggers.on_file_change.files['migrations/003.txt']": { "action": "recreate", "reason": "immutable", "new": "[FILE_HASH][5]" @@ -169,28 +145,20 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged === removing a matched file re-fires >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle": { + "lifecycle.triggers.on_file_change.files": { "action": "recreate", "reason": "immutable", "old": { - "triggers": { - "on_file_change": { - "migrations/001.txt": "[FILE_HASH][3]", - "migrations/002.txt": "[FILE_HASH][1]", - "mysubdir/a.txt": "[FILE_HASH][2]" - } - } + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][2]" }, "new": { - "triggers": { - "on_file_change": { - "migrations/001.txt": "[FILE_HASH][3]", - "mysubdir/a.txt": "[FILE_HASH][2]" - } - } + "migrations/001.txt": "[FILE_HASH][3]", + "mysubdir/a.txt": "[FILE_HASH][2]" } }, - "lifecycle.triggers.on_file_change['migrations/002.txt']": { + "lifecycle.triggers.on_file_change.files['migrations/002.txt']": { "action": "recreate", "reason": "immutable", "old": "[FILE_HASH][1]" @@ -206,26 +174,18 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged === editing a file below the glob does not re-fire >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle": { + "lifecycle.triggers.on_file_change.files": { "action": "skip", - "reason": "missing_in_remote", + "reason": "not_returned_by_api", "old": { - "triggers": { - "on_file_change": { - "migrations/001.txt": "[FILE_HASH][3]", - "migrations/002.txt": "[FILE_HASH][1]", - "mysubdir/a.txt": "[FILE_HASH][2]" - } - } + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][2]" }, "new": { - "triggers": { - "on_file_change": { - "migrations/001.txt": "[FILE_HASH][3]", - "migrations/002.txt": "[FILE_HASH][1]", - "mysubdir/a.txt": "[FILE_HASH][2]" - } - } + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][2]" } }, "result_state": { @@ -243,26 +203,18 @@ Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged === ** is not recursive: it matches the same files as * >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle": { + "lifecycle.triggers.on_file_change.files": { "action": "skip", - "reason": "missing_in_remote", + "reason": "not_returned_by_api", "old": { - "triggers": { - "on_file_change": { - "migrations/001.txt": "[FILE_HASH][3]", - "migrations/002.txt": "[FILE_HASH][1]", - "mysubdir/a.txt": "[FILE_HASH][2]" - } - } + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][2]" }, "new": { - "triggers": { - "on_file_change": { - "migrations/001.txt": "[FILE_HASH][3]", - "migrations/002.txt": "[FILE_HASH][1]", - "mysubdir/a.txt": "[FILE_HASH][2]" - } - } + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][2]" } }, "result_state": { diff --git a/acceptance/bundle/resources/job_runs/redeploy/output.txt b/acceptance/bundle/resources/job_runs/redeploy/output.txt index 06129f79373..60111c98a93 100644 --- a/acceptance/bundle/resources/job_runs/redeploy/output.txt +++ b/acceptance/bundle/resources/job_runs/redeploy/output.txt @@ -59,6 +59,11 @@ Resources: "job_parameters": { "env": "prod" }, + "lifecycle": { + "triggers": { + "on_file_change": {} + } + }, "result_state": "SUCCESS" } }, @@ -67,6 +72,11 @@ Resources: "job_parameters": { "env": "dev" }, + "lifecycle": { + "triggers": { + "on_file_change": {} + } + }, "result_state": "SUCCESS", "run_id": [MY_RUN_ID], "run_name": "my-job", diff --git a/bundle/direct/dresources/job_run.go b/bundle/direct/dresources/job_run.go index 63ed81cb78a..e27e89a7e3c 100644 --- a/bundle/direct/dresources/job_run.go +++ b/bundle/direct/dresources/job_run.go @@ -3,6 +3,7 @@ package dresources import ( "cmp" "context" + "encoding/json" "errors" "fmt" "slices" @@ -26,13 +27,24 @@ import ( // jobRunTimeout matches the timeout `bundle run` allows a run (bundle/run/job.go). const jobRunTimeout = 24 * time.Hour -// jobRunTriggerLocalPaths is shared by OverrideChangeDesc and DoUpdate so -// clearing a trigger stays a state-only update in both places. +// jobRunTriggerLocalPaths are state-only: DoUpdate must not wait, OverrideChangeDesc +// downgrades a cleared trigger to update. var jobRunTriggerLocalPaths = []string{ - "lifecycle", - "lifecycle.triggers", "lifecycle.triggers.on_bundle_deploy", - "lifecycle.triggers.on_file_change", + "lifecycle.triggers.on_file_change.files", +} + +func isJobRunTriggerPath(path string) bool { + return slices.Contains(jobRunTriggerLocalPaths, path) +} + +func hasJobRunNonTriggerChanges(changes Changes) bool { + for path, change := range changes { + if change.Action != deployplan.Skip && !isJobRunTriggerPath(path) { + return true + } + } + return false } // JobRunTriggersState is the persisted fingerprint of lifecycle.triggers. @@ -40,12 +52,44 @@ type JobRunTriggersState struct { // Fresh UUID each plan while armed so Old!=New forces recreate. OnBundleDeploy string `json:"on_bundle_deploy,omitempty"` // Path → content hash from ResolveJobRunFileTriggers; change to recreate. - OnFileChange map[string]string `json:"on_file_change,omitempty"` + OnFileChange *JobRunFileTriggerState `json:"on_file_change"` +} + +// JobRunFileTriggerState is always present so remote and state share one shape. +// Files is nil when the trigger is off, so dropping the trigger diffs here and +// a file appear/disappear diffs inside the map. +type JobRunFileTriggerState struct { + Files map[string]string `json:"files,omitempty"` +} + +// UnmarshalJSON accepts the wrapped map and the older path-to-hash map. +func (s *JobRunFileTriggerState) UnmarshalJSON(b []byte) error { + var raw map[string]json.RawMessage + if err := json.Unmarshal(b, &raw); err != nil { + return err + } + if files, ok := raw["files"]; ok && len(files) > 0 && files[0] == '{' { + return json.Unmarshal(files, &s.Files) + } + if len(raw) == 0 { + s.Files = nil + return nil + } + return json.Unmarshal(b, &s.Files) } -// JobRunLifecycleState holds local-only lifecycle fields persisted in state. +// JobRunLifecycleState is the local-only trigger fingerprint, also present (empty) on remote. type JobRunLifecycleState struct { - Triggers *JobRunTriggersState `json:"triggers,omitempty"` + Triggers *JobRunTriggersState `json:"triggers"` +} + +func newJobRunLifecycleState() *JobRunLifecycleState { + return &JobRunLifecycleState{ + Triggers: &JobRunTriggersState{ + OnBundleDeploy: "", + OnFileChange: &JobRunFileTriggerState{Files: nil}, + }, + } } // JobRunState is the RunNow request plus the outcome required for planning. @@ -55,12 +99,13 @@ type JobRunState struct { // Always SUCCESS during planning and cleared before persistence. ResultState jobs.RunResultState `json:"result_state,omitempty"` - // Local-only; listed in knownMissingInRemoteType. Nested under lifecycle to - // mirror config and avoid colliding with a future Jobs API field. - Lifecycle *JobRunLifecycleState `json:"lifecycle,omitempty"` + // Local-only. Nested under lifecycle to mirror config and avoid colliding + // with a future Jobs API field. + Lifecycle *JobRunLifecycleState `json:"lifecycle"` } func (s *JobRunState) UnmarshalJSON(b []byte) error { + s.Lifecycle = newJobRunLifecycleState() return marshal.Unmarshal(b, s) } @@ -77,6 +122,9 @@ type JobRunRemote struct { // compare the two. See "RemapState is a dumb copy" in README.md. ResultState jobs.RunResultState `json:"result_state,omitempty"` + // Always the empty fingerprint: GetRun does not return triggers. + Lifecycle *JobRunLifecycleState `json:"lifecycle"` + RunId int64 `json:"run_id,omitempty"` RunName string `json:"run_name,omitempty"` State *jobs.RunState `json:"state,omitempty"` @@ -87,6 +135,7 @@ type JobRunRemote struct { // Custom marshaler needed because embedded RunNow's MarshalJSON would otherwise // take over and drop the additional fields. func (s *JobRunRemote) UnmarshalJSON(b []byte) error { + s.Lifecycle = newJobRunLifecycleState() return marshal.Unmarshal(b, s) } @@ -108,20 +157,13 @@ func (*ResourceJobRun) PrepareState(input *resources.JobRun) *JobRunState { state := &JobRunState{ RunNow: input.RunNow, ResultState: jobs.RunResultStateSuccess, - Lifecycle: nil, + Lifecycle: newJobRunLifecycleState(), } - if !input.HasOnBundleDeploy() && len(input.ResolvedFileTriggers) == 0 { - return state - } - onBundleDeploy := "" if input.HasOnBundleDeploy() { - onBundleDeploy = uuid.NewString() + state.Lifecycle.Triggers.OnBundleDeploy = uuid.NewString() } - state.Lifecycle = &JobRunLifecycleState{ - Triggers: &JobRunTriggersState{ - OnBundleDeploy: onBundleDeploy, - OnFileChange: input.ResolvedFileTriggers, - }, + if len(input.ResolvedFileTriggers) > 0 { + state.Lifecycle.Triggers.OnFileChange.Files = input.ResolvedFileTriggers } return state } @@ -162,6 +204,7 @@ func makeJobRunRemote(run *jobs.Run) *JobRunRemote { ForceSendFields: nil, }, ResultState: run.State.ResultState, + Lifecycle: newJobRunLifecycleState(), RunId: run.RunId, RunName: run.RunName, // Rebuilt, not copied: the SDK records explicitly-sent fields in @@ -202,8 +245,7 @@ func (*ResourceJobRun) RemapState(remote *JobRunRemote) *JobRunState { return &JobRunState{ RunNow: remote.RunNow, ResultState: remote.ResultState, - // Local-only lifecycle fingerprints stay unset on the remapped remote. - Lifecycle: nil, + Lifecycle: remote.Lifecycle, } } @@ -389,7 +431,7 @@ func reportRunLine(ctx context.Context, runID int64, msg string) { func (r *ResourceJobRun) DoUpdate(ctx context.Context, id string, config *JobRunState, entry *PlanEntry) (*JobRunRemote, error) { // Clearing a trigger only drops its local-only fingerprint from state; wait on // the run only when some other field changed. - if !entry.Changes.HasChangeExcept(jobRunTriggerLocalPaths...) { + if !hasJobRunNonTriggerChanges(entry.Changes) { config.ResultState = "" return nil, nil } @@ -405,10 +447,11 @@ func (r *ResourceJobRun) DoUpdate(ctx context.Context, id string, config *JobRun // Clearing a trigger downgrades the recreate to a state-only update so the // fingerprint is dropped from state without re-firing the run. func (*ResourceJobRun) OverrideChangeDesc(_ context.Context, path *structpath.PathNode, change *ChangeDesc, remote *JobRunRemote) error { - if slices.Contains(jobRunTriggerLocalPaths, path.String()) { - // A cleared trigger sets New empty; structdiff may report it at lifecycle, - // lifecycle.triggers, or the leaf. DoUpdate treats these paths as no-ops. - if change.New == nil || change.New == "" { + pathString := path.String() + if isJobRunTriggerPath(pathString) { + removed := pathString == "lifecycle.triggers.on_bundle_deploy" && (change.New == nil || change.New == "") + removed = removed || pathString == "lifecycle.triggers.on_file_change.files" && change.New == nil + if removed { change.Action = deployplan.Update change.Reason = "trigger removed" } diff --git a/bundle/direct/dresources/job_run_test.go b/bundle/direct/dresources/job_run_test.go index 72de912e9d6..a42c661883b 100644 --- a/bundle/direct/dresources/job_run_test.go +++ b/bundle/direct/dresources/job_run_test.go @@ -351,6 +351,10 @@ func TestJobRunPrepareStateRequiresSuccess(t *testing.T) { state := (&ResourceJobRun{}).PrepareState(&resources.JobRun{RunNow: jobs.RunNow{JobId: 456}}) assert.Equal(t, jobs.RunResultStateSuccess, state.ResultState) + require.NotNil(t, state.Lifecycle) + require.NotNil(t, state.Lifecycle.Triggers) + assert.NotNil(t, state.Lifecycle.Triggers.OnFileChange) + assert.Nil(t, state.Lifecycle.Triggers.OnFileChange.Files) } func TestJobRunPrepareStateOnBundleDeploy(t *testing.T) { @@ -378,7 +382,7 @@ func TestJobRunPrepareStateOnFileChange(t *testing.T) { }) require.NotNil(t, state.Lifecycle) require.NotNil(t, state.Lifecycle.Triggers) - assert.Equal(t, hashes, state.Lifecycle.Triggers.OnFileChange) + assert.Equal(t, hashes, state.Lifecycle.Triggers.OnFileChange.Files) assert.Empty(t, state.Lifecycle.Triggers.OnBundleDeploy) }) @@ -393,24 +397,54 @@ func TestJobRunPrepareStateOnFileChange(t *testing.T) { require.NotNil(t, state.Lifecycle) require.NotNil(t, state.Lifecycle.Triggers) assert.NotEmpty(t, state.Lifecycle.Triggers.OnBundleDeploy) - assert.Equal(t, hashes, state.Lifecycle.Triggers.OnFileChange) + assert.Equal(t, hashes, state.Lifecycle.Triggers.OnFileChange.Files) }) } -func TestJobRunOverrideChangeDescTriggerRemoved(t *testing.T) { - r := &ResourceJobRun{} +func TestJobRunStateUnmarshalLifecycleCompatibility(t *testing.T) { + t.Run("missing lifecycle gets empty shape", func(t *testing.T) { + var state JobRunState + require.NoError(t, json.Unmarshal([]byte(`{}`), &state)) + require.NotNil(t, state.Lifecycle) + require.NotNil(t, state.Lifecycle.Triggers) + require.NotNil(t, state.Lifecycle.Triggers.OnFileChange) + assert.Nil(t, state.Lifecycle.Triggers.OnFileChange.Files) + }) - t.Run("clearing lifecycle downgrades to update", func(t *testing.T) { - change := &ChangeDesc{ - Action: deployplan.Recreate, - Old: &JobRunLifecycleState{Triggers: &JobRunTriggersState{OnBundleDeploy: "old"}}, - New: nil, - } - require.NoError(t, r.OverrideChangeDesc(t.Context(), structpath.MustParsePath("lifecycle"), change, nil)) - assert.Equal(t, deployplan.Update, change.Action) - assert.Equal(t, "trigger removed", change.Reason) + t.Run("old path-to-hash map is preserved", func(t *testing.T) { + var state JobRunState + require.NoError(t, json.Unmarshal([]byte(`{ + "lifecycle": { + "triggers": { + "on_file_change": { + "a.txt": "abc" + } + } + } + }`), &state)) + assert.Equal(t, map[string]string{"a.txt": "abc"}, state.Lifecycle.Triggers.OnFileChange.Files) }) + t.Run("new wrapped map is preserved", func(t *testing.T) { + var state JobRunState + require.NoError(t, json.Unmarshal([]byte(`{ + "lifecycle": { + "triggers": { + "on_file_change": { + "files": { + "a.txt": "abc" + } + } + } + } + }`), &state)) + assert.Equal(t, map[string]string{"a.txt": "abc"}, state.Lifecycle.Triggers.OnFileChange.Files) + }) +} + +func TestJobRunOverrideChangeDescTriggerRemoved(t *testing.T) { + r := &ResourceJobRun{} + t.Run("clearing on_bundle_deploy leaf downgrades to update", func(t *testing.T) { change := &ChangeDesc{ Action: deployplan.Recreate, @@ -422,13 +456,13 @@ func TestJobRunOverrideChangeDescTriggerRemoved(t *testing.T) { assert.Equal(t, "trigger removed", change.Reason) }) - t.Run("clearing on_file_change leaf downgrades to update", func(t *testing.T) { + t.Run("clearing on_file_change files downgrades to update", func(t *testing.T) { change := &ChangeDesc{ Action: deployplan.Recreate, Old: map[string]string{"a.txt": "abc"}, New: nil, } - require.NoError(t, r.OverrideChangeDesc(t.Context(), structpath.MustParsePath("lifecycle.triggers.on_file_change"), change, nil)) + require.NoError(t, r.OverrideChangeDesc(t.Context(), structpath.MustParsePath("lifecycle.triggers.on_file_change.files"), change, nil)) assert.Equal(t, deployplan.Update, change.Action) assert.Equal(t, "trigger removed", change.Reason) }) @@ -446,10 +480,30 @@ func TestJobRunOverrideChangeDescTriggerRemoved(t *testing.T) { t.Run("changed on_file_change hash still recreates", func(t *testing.T) { change := &ChangeDesc{ Action: deployplan.Recreate, - Old: map[string]string{"a.txt": "old"}, - New: map[string]string{"a.txt": "new"}, + Old: "old", + New: "new", + } + require.NoError(t, r.OverrideChangeDesc(t.Context(), structpath.MustParsePath("lifecycle.triggers.on_file_change.files['a.txt']"), change, nil)) + assert.Equal(t, deployplan.Recreate, change.Action) + }) + + t.Run("removed matched file still recreates", func(t *testing.T) { + change := &ChangeDesc{ + Action: deployplan.Recreate, + Old: "old", + New: nil, + } + require.NoError(t, r.OverrideChangeDesc(t.Context(), structpath.MustParsePath("lifecycle.triggers.on_file_change.files['a.txt']"), change, nil)) + assert.Equal(t, deployplan.Recreate, change.Action) + }) + + t.Run("missing file fingerprint still recreates", func(t *testing.T) { + change := &ChangeDesc{ + Action: deployplan.Recreate, + Old: "old", + New: "", } - require.NoError(t, r.OverrideChangeDesc(t.Context(), structpath.MustParsePath("lifecycle.triggers.on_file_change"), change, nil)) + require.NoError(t, r.OverrideChangeDesc(t.Context(), structpath.MustParsePath("lifecycle.triggers.on_file_change.files['a.txt']"), change, nil)) assert.Equal(t, deployplan.Recreate, change.Action) }) } @@ -464,11 +518,13 @@ func TestJobRunRemapStateCarriesTheOutcome(t *testing.T) { "", } { t.Run(string(outcome), func(t *testing.T) { - remote := &JobRunRemote{RunId: 123, ResultState: outcome} + lifecycle := newJobRunLifecycleState() + remote := &JobRunRemote{RunId: 123, ResultState: outcome, Lifecycle: lifecycle} state := (&ResourceJobRun{}).RemapState(remote) assert.Equal(t, outcome, state.ResultState) + assert.Same(t, lifecycle, state.Lifecycle) }) } } diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index e98a1ac2330..06485a31054 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -175,6 +175,9 @@ resources: reason: not_returned_by_api - field: queue reason: not_returned_by_api + # Local-only trigger fingerprints; GetRun does not return them. + - field: lifecycle + reason: not_returned_by_api # A run is immutable and fire-once, so any change recreates it. Omitting # `field` matches every field (root; see TestFieldRuleOmittedIsRoot). # `field: ""` would instead match nothing. The one exception is a run that is diff --git a/bundle/direct/dresources/type_test.go b/bundle/direct/dresources/type_test.go index 2d5516d59c7..ca84850c33d 100644 --- a/bundle/direct/dresources/type_test.go +++ b/bundle/direct/dresources/type_test.go @@ -56,10 +56,6 @@ var knownMissingInRemoteType = map[string][]string{ "vector_search_endpoints": { "usage_policy_id", }, - "job_runs": { - // Local-only trigger fingerprints under lifecycle. - "lifecycle", - }, } // commonMissingInStateType lists fields that are commonly missing across all resource types. From 451d13c921a16c8f73f7e1752559f341cec44458 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Fri, 21 Aug 2026 13:02:42 +0000 Subject: [PATCH 16/75] job_runs: keep the lifecycle fingerprint out of the remote type Nest lifecycle by value so structdiff descends to the leaf that changed instead of reporting the whole subtree, and let RemapState supply the empty shape. GetRun never returns the fingerprints, so the remote type no longer advertises a lifecycle it cannot fill. --- acceptance/bundle/refschema/out.fields.txt | 12 ++--- .../resources/job_runs/failed_run/output.txt | 5 --- .../job_runs/interrupted_run/output.txt | 5 --- .../job_runs/on_file_change_glob/output.txt | 4 +- .../resources/job_runs/redeploy/output.txt | 5 --- bundle/direct/dresources/job_run.go | 45 +++++++++---------- bundle/direct/dresources/job_run_test.go | 17 +------ bundle/direct/dresources/resources.yml | 3 -- bundle/direct/dresources/type_test.go | 5 +++ 9 files changed, 37 insertions(+), 64 deletions(-) diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index d6da3840c2a..691c5589bcc 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -845,16 +845,16 @@ resources.job_runs.*.jar_params[*] string ALL resources.job_runs.*.job_id int64 ALL resources.job_runs.*.job_parameters map[string]string ALL resources.job_runs.*.job_parameters.* string ALL -resources.job_runs.*.lifecycle *dresources.JobRunLifecycleState REMOTE STATE resources.job_runs.*.lifecycle *resources.JobRunLifecycle INPUT +resources.job_runs.*.lifecycle dresources.JobRunLifecycleState STATE resources.job_runs.*.lifecycle resources.Lifecycle INPUT resources.job_runs.*.lifecycle.prevent_destroy bool INPUT -resources.job_runs.*.lifecycle.triggers *dresources.JobRunTriggersState REMOTE STATE resources.job_runs.*.lifecycle.triggers []resources.JobRunTrigger INPUT -resources.job_runs.*.lifecycle.triggers.on_bundle_deploy string REMOTE STATE -resources.job_runs.*.lifecycle.triggers.on_file_change *dresources.JobRunFileTriggerState REMOTE STATE -resources.job_runs.*.lifecycle.triggers.on_file_change.files map[string]string REMOTE STATE -resources.job_runs.*.lifecycle.triggers.on_file_change.files.* string REMOTE STATE +resources.job_runs.*.lifecycle.triggers dresources.JobRunTriggersState STATE +resources.job_runs.*.lifecycle.triggers.on_bundle_deploy string STATE +resources.job_runs.*.lifecycle.triggers.on_file_change dresources.JobRunFileTriggerState STATE +resources.job_runs.*.lifecycle.triggers.on_file_change.files map[string]string STATE +resources.job_runs.*.lifecycle.triggers.on_file_change.files.* string STATE resources.job_runs.*.lifecycle.triggers[*] resources.JobRunTrigger INPUT resources.job_runs.*.lifecycle.triggers[*].on_bundle_deploy *bool INPUT resources.job_runs.*.lifecycle.triggers[*].on_file_change *string INPUT diff --git a/acceptance/bundle/resources/job_runs/failed_run/output.txt b/acceptance/bundle/resources/job_runs/failed_run/output.txt index 0c6bddb0d9b..37f1a831681 100644 --- a/acceptance/bundle/resources/job_runs/failed_run/output.txt +++ b/acceptance/bundle/resources/job_runs/failed_run/output.txt @@ -37,11 +37,6 @@ Files: 7 uploaded, 0 deleted }, "remote_state": { "job_id": [MY_JOB_ID], - "lifecycle": { - "triggers": { - "on_file_change": {} - } - }, "result_state": "FAILED", "run_id": [MY_RUN_ID], "run_name": "test-job-[UNIQUE_NAME]", diff --git a/acceptance/bundle/resources/job_runs/interrupted_run/output.txt b/acceptance/bundle/resources/job_runs/interrupted_run/output.txt index 47dfb8c7617..df52920c879 100644 --- a/acceptance/bundle/resources/job_runs/interrupted_run/output.txt +++ b/acceptance/bundle/resources/job_runs/interrupted_run/output.txt @@ -53,11 +53,6 @@ Exit code: 1 }, "remote_state": { "job_id": [MY_JOB_ID], - "lifecycle": { - "triggers": { - "on_file_change": {} - } - }, "run_id": [MY_RUN_ID], "run_name": "my-job", "run_page_url": "[DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?o=[NUMID]", diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt index 5b4ef23178c..69a7e5ece94 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt @@ -176,7 +176,7 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged { "lifecycle.triggers.on_file_change.files": { "action": "skip", - "reason": "not_returned_by_api", + "reason": "missing_in_remote", "old": { "migrations/001.txt": "[FILE_HASH][3]", "migrations/002.txt": "[FILE_HASH][1]", @@ -205,7 +205,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged { "lifecycle.triggers.on_file_change.files": { "action": "skip", - "reason": "not_returned_by_api", + "reason": "missing_in_remote", "old": { "migrations/001.txt": "[FILE_HASH][3]", "migrations/002.txt": "[FILE_HASH][1]", diff --git a/acceptance/bundle/resources/job_runs/redeploy/output.txt b/acceptance/bundle/resources/job_runs/redeploy/output.txt index 60111c98a93..a80685733ad 100644 --- a/acceptance/bundle/resources/job_runs/redeploy/output.txt +++ b/acceptance/bundle/resources/job_runs/redeploy/output.txt @@ -72,11 +72,6 @@ Resources: "job_parameters": { "env": "dev" }, - "lifecycle": { - "triggers": { - "on_file_change": {} - } - }, "result_state": "SUCCESS", "run_id": [MY_RUN_ID], "run_name": "my-job", diff --git a/bundle/direct/dresources/job_run.go b/bundle/direct/dresources/job_run.go index e27e89a7e3c..a300318dabe 100644 --- a/bundle/direct/dresources/job_run.go +++ b/bundle/direct/dresources/job_run.go @@ -51,13 +51,12 @@ func hasJobRunNonTriggerChanges(changes Changes) bool { type JobRunTriggersState struct { // Fresh UUID each plan while armed so Old!=New forces recreate. OnBundleDeploy string `json:"on_bundle_deploy,omitempty"` - // Path → content hash from ResolveJobRunFileTriggers; change to recreate. - OnFileChange *JobRunFileTriggerState `json:"on_file_change"` + // Content hashes from ResolveJobRunFileTriggers; any change recreates. + OnFileChange JobRunFileTriggerState `json:"on_file_change"` } -// JobRunFileTriggerState is always present so remote and state share one shape. -// Files is nil when the trigger is off, so dropping the trigger diffs here and -// a file appear/disappear diffs inside the map. +// JobRunFileTriggerState wraps the hashes so turning the trigger off (Files nil) +// diffs at one path, distinct from the per-file entries a changed file produces. type JobRunFileTriggerState struct { Files map[string]string `json:"files,omitempty"` } @@ -78,16 +77,21 @@ func (s *JobRunFileTriggerState) UnmarshalJSON(b []byte) error { return json.Unmarshal(b, &s.Files) } -// JobRunLifecycleState is the local-only trigger fingerprint, also present (empty) on remote. +// JobRunLifecycleState is the local-only trigger fingerprint. Nested by value, +// not by pointer: structdiff cannot descend into a nil pointer and would report +// the whole subtree at "lifecycle" instead of the leaf that actually changed. type JobRunLifecycleState struct { - Triggers *JobRunTriggersState `json:"triggers"` + Triggers JobRunTriggersState `json:"triggers"` } -func newJobRunLifecycleState() *JobRunLifecycleState { - return &JobRunLifecycleState{ - Triggers: &JobRunTriggersState{ +// Zero value spelled out field by field, as exhaustruct requires. +func emptyJobRunLifecycleState() JobRunLifecycleState { + return JobRunLifecycleState{ + Triggers: JobRunTriggersState{ OnBundleDeploy: "", - OnFileChange: &JobRunFileTriggerState{Files: nil}, + OnFileChange: JobRunFileTriggerState{ + Files: nil, + }, }, } } @@ -101,11 +105,10 @@ type JobRunState struct { // Local-only. Nested under lifecycle to mirror config and avoid colliding // with a future Jobs API field. - Lifecycle *JobRunLifecycleState `json:"lifecycle"` + Lifecycle JobRunLifecycleState `json:"lifecycle"` } func (s *JobRunState) UnmarshalJSON(b []byte) error { - s.Lifecycle = newJobRunLifecycleState() return marshal.Unmarshal(b, s) } @@ -113,8 +116,8 @@ func (s JobRunState) MarshalJSON() ([]byte, error) { return marshal.Marshal(s) } -// JobRunRemote embeds RunNow so every StateType path is a valid RemoteType path -// (see TestRemoteSuperset), plus the run's output-only fields for a faithful view. +// JobRunRemote is the RunNow request plus the run's output-only fields. It has no +// lifecycle: GetRun never returns the fingerprints (see knownMissingInRemoteType). type JobRunRemote struct { jobs.RunNow @@ -122,9 +125,6 @@ type JobRunRemote struct { // compare the two. See "RemapState is a dumb copy" in README.md. ResultState jobs.RunResultState `json:"result_state,omitempty"` - // Always the empty fingerprint: GetRun does not return triggers. - Lifecycle *JobRunLifecycleState `json:"lifecycle"` - RunId int64 `json:"run_id,omitempty"` RunName string `json:"run_name,omitempty"` State *jobs.RunState `json:"state,omitempty"` @@ -135,7 +135,6 @@ type JobRunRemote struct { // Custom marshaler needed because embedded RunNow's MarshalJSON would otherwise // take over and drop the additional fields. func (s *JobRunRemote) UnmarshalJSON(b []byte) error { - s.Lifecycle = newJobRunLifecycleState() return marshal.Unmarshal(b, s) } @@ -157,7 +156,7 @@ func (*ResourceJobRun) PrepareState(input *resources.JobRun) *JobRunState { state := &JobRunState{ RunNow: input.RunNow, ResultState: jobs.RunResultStateSuccess, - Lifecycle: newJobRunLifecycleState(), + Lifecycle: emptyJobRunLifecycleState(), } if input.HasOnBundleDeploy() { state.Lifecycle.Triggers.OnBundleDeploy = uuid.NewString() @@ -204,7 +203,6 @@ func makeJobRunRemote(run *jobs.Run) *JobRunRemote { ForceSendFields: nil, }, ResultState: run.State.ResultState, - Lifecycle: newJobRunLifecycleState(), RunId: run.RunId, RunName: run.RunName, // Rebuilt, not copied: the SDK records explicitly-sent fields in @@ -240,12 +238,13 @@ func (r *ResourceJobRun) DoRead(ctx context.Context, id string) (*JobRunRemote, } // RemapState extracts the fields used for diffing: the RunNow request and the -// outcome the run reached. +// outcome the run reached. Lifecycle has no remote counterpart, so it stays empty +// and the planner skips it as missing_in_remote. func (*ResourceJobRun) RemapState(remote *JobRunRemote) *JobRunState { return &JobRunState{ RunNow: remote.RunNow, ResultState: remote.ResultState, - Lifecycle: remote.Lifecycle, + Lifecycle: emptyJobRunLifecycleState(), } } diff --git a/bundle/direct/dresources/job_run_test.go b/bundle/direct/dresources/job_run_test.go index a42c661883b..78ec09d3604 100644 --- a/bundle/direct/dresources/job_run_test.go +++ b/bundle/direct/dresources/job_run_test.go @@ -351,9 +351,6 @@ func TestJobRunPrepareStateRequiresSuccess(t *testing.T) { state := (&ResourceJobRun{}).PrepareState(&resources.JobRun{RunNow: jobs.RunNow{JobId: 456}}) assert.Equal(t, jobs.RunResultStateSuccess, state.ResultState) - require.NotNil(t, state.Lifecycle) - require.NotNil(t, state.Lifecycle.Triggers) - assert.NotNil(t, state.Lifecycle.Triggers.OnFileChange) assert.Nil(t, state.Lifecycle.Triggers.OnFileChange.Files) } @@ -365,8 +362,6 @@ func TestJobRunPrepareStateOnBundleDeploy(t *testing.T) { }, } first := (&ResourceJobRun{}).PrepareState(input) - require.NotNil(t, first.Lifecycle) - require.NotNil(t, first.Lifecycle.Triggers) assert.NotEmpty(t, first.Lifecycle.Triggers.OnBundleDeploy) second := (&ResourceJobRun{}).PrepareState(input) @@ -380,8 +375,6 @@ func TestJobRunPrepareStateOnFileChange(t *testing.T) { state := (&ResourceJobRun{}).PrepareState(&resources.JobRun{ ResolvedFileTriggers: hashes, }) - require.NotNil(t, state.Lifecycle) - require.NotNil(t, state.Lifecycle.Triggers) assert.Equal(t, hashes, state.Lifecycle.Triggers.OnFileChange.Files) assert.Empty(t, state.Lifecycle.Triggers.OnBundleDeploy) }) @@ -394,8 +387,6 @@ func TestJobRunPrepareStateOnFileChange(t *testing.T) { }, ResolvedFileTriggers: hashes, }) - require.NotNil(t, state.Lifecycle) - require.NotNil(t, state.Lifecycle.Triggers) assert.NotEmpty(t, state.Lifecycle.Triggers.OnBundleDeploy) assert.Equal(t, hashes, state.Lifecycle.Triggers.OnFileChange.Files) }) @@ -405,9 +396,6 @@ func TestJobRunStateUnmarshalLifecycleCompatibility(t *testing.T) { t.Run("missing lifecycle gets empty shape", func(t *testing.T) { var state JobRunState require.NoError(t, json.Unmarshal([]byte(`{}`), &state)) - require.NotNil(t, state.Lifecycle) - require.NotNil(t, state.Lifecycle.Triggers) - require.NotNil(t, state.Lifecycle.Triggers.OnFileChange) assert.Nil(t, state.Lifecycle.Triggers.OnFileChange.Files) }) @@ -518,13 +506,12 @@ func TestJobRunRemapStateCarriesTheOutcome(t *testing.T) { "", } { t.Run(string(outcome), func(t *testing.T) { - lifecycle := newJobRunLifecycleState() - remote := &JobRunRemote{RunId: 123, ResultState: outcome, Lifecycle: lifecycle} + remote := &JobRunRemote{RunId: 123, ResultState: outcome} state := (&ResourceJobRun{}).RemapState(remote) assert.Equal(t, outcome, state.ResultState) - assert.Same(t, lifecycle, state.Lifecycle) + assert.Equal(t, emptyJobRunLifecycleState(), state.Lifecycle) }) } } diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index 06485a31054..e98a1ac2330 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -175,9 +175,6 @@ resources: reason: not_returned_by_api - field: queue reason: not_returned_by_api - # Local-only trigger fingerprints; GetRun does not return them. - - field: lifecycle - reason: not_returned_by_api # A run is immutable and fire-once, so any change recreates it. Omitting # `field` matches every field (root; see TestFieldRuleOmittedIsRoot). # `field: ""` would instead match nothing. The one exception is a run that is diff --git a/bundle/direct/dresources/type_test.go b/bundle/direct/dresources/type_test.go index ca84850c33d..7ad9872c373 100644 --- a/bundle/direct/dresources/type_test.go +++ b/bundle/direct/dresources/type_test.go @@ -19,6 +19,11 @@ var knownMissingInRemoteType = map[string][]string{ "external_locations": { "skip_validation", }, + "job_runs": { + // Deliberate omission: the trigger fingerprints are client-side only, so + // GetRun has nothing to report them under. + "lifecycle", + }, "model_serving_endpoints": { "rate_limits", }, From 9bc1c426b6a3e106700bc9edfe3f8769afd942eb Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Mon, 24 Aug 2026 07:55:05 +0000 Subject: [PATCH 17/75] job_runs: store on_file_change hashes on the trigger itself Drop the files wrapper so plan diffs land on lifecycle.triggers.on_file_change, matching the config path without an extra state-only level. --- acceptance/bundle/refschema/out.fields.txt | 5 +-- .../job_runs/on_file_change/output.txt | 8 ++-- .../job_runs/on_file_change_glob/output.txt | 20 +++++----- bundle/direct/dresources/job_run.go | 35 +++-------------- bundle/direct/dresources/job_run_test.go | 38 ++++++------------- 5 files changed, 32 insertions(+), 74 deletions(-) diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index 691c5589bcc..878453cb964 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -852,9 +852,8 @@ resources.job_runs.*.lifecycle.prevent_destroy bool INPUT resources.job_runs.*.lifecycle.triggers []resources.JobRunTrigger INPUT resources.job_runs.*.lifecycle.triggers dresources.JobRunTriggersState STATE resources.job_runs.*.lifecycle.triggers.on_bundle_deploy string STATE -resources.job_runs.*.lifecycle.triggers.on_file_change dresources.JobRunFileTriggerState STATE -resources.job_runs.*.lifecycle.triggers.on_file_change.files map[string]string STATE -resources.job_runs.*.lifecycle.triggers.on_file_change.files.* string STATE +resources.job_runs.*.lifecycle.triggers.on_file_change map[string]string STATE +resources.job_runs.*.lifecycle.triggers.on_file_change.* string STATE resources.job_runs.*.lifecycle.triggers[*] resources.JobRunTrigger INPUT resources.job_runs.*.lifecycle.triggers[*].on_bundle_deploy *bool INPUT resources.job_runs.*.lifecycle.triggers[*].on_file_change *string INPUT diff --git a/acceptance/bundle/resources/job_runs/on_file_change/output.txt b/acceptance/bundle/resources/job_runs/on_file_change/output.txt index 9b662e1936d..28deeee38a4 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change/output.txt @@ -52,7 +52,7 @@ Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle.triggers.on_file_change.files": { + "lifecycle.triggers.on_file_change": { "action": "recreate", "reason": "immutable", "old": { @@ -62,7 +62,7 @@ Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged "seed.txt": "[FILE_HASH][1]" } }, - "lifecycle.triggers.on_file_change.files['seed.txt']": { + "lifecycle.triggers.on_file_change['seed.txt']": { "action": "recreate", "reason": "immutable", "old": "[FILE_HASH][0]", @@ -117,7 +117,7 @@ Warning: lifecycle.triggers.on_file_change: no files match "seed.txt" in databricks.yml:18:29 { - "lifecycle.triggers.on_file_change.files": { + "lifecycle.triggers.on_file_change": { "action": "recreate", "reason": "immutable", "old": { @@ -127,7 +127,7 @@ Warning: lifecycle.triggers.on_file_change: no files match "seed.txt" "seed.txt": "" } }, - "lifecycle.triggers.on_file_change.files['seed.txt']": { + "lifecycle.triggers.on_file_change['seed.txt']": { "action": "recreate", "reason": "immutable", "old": "[FILE_HASH][1]", diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt index 69a7e5ece94..a04e3710fb4 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt @@ -25,7 +25,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged === editing one matched file re-fires >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle.triggers.on_file_change.files": { + "lifecycle.triggers.on_file_change": { "action": "recreate", "reason": "immutable", "old": { @@ -39,7 +39,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "mysubdir/a.txt": "[FILE_HASH][2]" } }, - "lifecycle.triggers.on_file_change.files['migrations/001.txt']": { + "lifecycle.triggers.on_file_change['migrations/001.txt']": { "action": "recreate", "reason": "immutable", "old": "[FILE_HASH][0]", @@ -83,7 +83,7 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged === editing a file in a subdirectory re-fires >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle.triggers.on_file_change.files": { + "lifecycle.triggers.on_file_change": { "action": "recreate", "reason": "immutable", "old": { @@ -97,7 +97,7 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged "mysubdir/a.txt": "[FILE_HASH][4]" } }, - "lifecycle.triggers.on_file_change.files['mysubdir/a.txt']": { + "lifecycle.triggers.on_file_change['mysubdir/a.txt']": { "action": "recreate", "reason": "immutable", "old": "[FILE_HASH][2]", @@ -114,7 +114,7 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged === a new matching file re-fires >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle.triggers.on_file_change.files": { + "lifecycle.triggers.on_file_change": { "action": "recreate", "reason": "immutable", "old": { @@ -129,7 +129,7 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged "mysubdir/a.txt": "[FILE_HASH][2]" } }, - "lifecycle.triggers.on_file_change.files['migrations/003.txt']": { + "lifecycle.triggers.on_file_change['migrations/003.txt']": { "action": "recreate", "reason": "immutable", "new": "[FILE_HASH][5]" @@ -145,7 +145,7 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged === removing a matched file re-fires >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle.triggers.on_file_change.files": { + "lifecycle.triggers.on_file_change": { "action": "recreate", "reason": "immutable", "old": { @@ -158,7 +158,7 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged "mysubdir/a.txt": "[FILE_HASH][2]" } }, - "lifecycle.triggers.on_file_change.files['migrations/002.txt']": { + "lifecycle.triggers.on_file_change['migrations/002.txt']": { "action": "recreate", "reason": "immutable", "old": "[FILE_HASH][1]" @@ -174,7 +174,7 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged === editing a file below the glob does not re-fire >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle.triggers.on_file_change.files": { + "lifecycle.triggers.on_file_change": { "action": "skip", "reason": "missing_in_remote", "old": { @@ -203,7 +203,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged === ** is not recursive: it matches the same files as * >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle.triggers.on_file_change.files": { + "lifecycle.triggers.on_file_change": { "action": "skip", "reason": "missing_in_remote", "old": { diff --git a/bundle/direct/dresources/job_run.go b/bundle/direct/dresources/job_run.go index a300318dabe..6ac69e7a3a2 100644 --- a/bundle/direct/dresources/job_run.go +++ b/bundle/direct/dresources/job_run.go @@ -3,7 +3,6 @@ package dresources import ( "cmp" "context" - "encoding/json" "errors" "fmt" "slices" @@ -31,7 +30,7 @@ const jobRunTimeout = 24 * time.Hour // downgrades a cleared trigger to update. var jobRunTriggerLocalPaths = []string{ "lifecycle.triggers.on_bundle_deploy", - "lifecycle.triggers.on_file_change.files", + "lifecycle.triggers.on_file_change", } func isJobRunTriggerPath(path string) bool { @@ -52,29 +51,7 @@ type JobRunTriggersState struct { // Fresh UUID each plan while armed so Old!=New forces recreate. OnBundleDeploy string `json:"on_bundle_deploy,omitempty"` // Content hashes from ResolveJobRunFileTriggers; any change recreates. - OnFileChange JobRunFileTriggerState `json:"on_file_change"` -} - -// JobRunFileTriggerState wraps the hashes so turning the trigger off (Files nil) -// diffs at one path, distinct from the per-file entries a changed file produces. -type JobRunFileTriggerState struct { - Files map[string]string `json:"files,omitempty"` -} - -// UnmarshalJSON accepts the wrapped map and the older path-to-hash map. -func (s *JobRunFileTriggerState) UnmarshalJSON(b []byte) error { - var raw map[string]json.RawMessage - if err := json.Unmarshal(b, &raw); err != nil { - return err - } - if files, ok := raw["files"]; ok && len(files) > 0 && files[0] == '{' { - return json.Unmarshal(files, &s.Files) - } - if len(raw) == 0 { - s.Files = nil - return nil - } - return json.Unmarshal(b, &s.Files) + OnFileChange map[string]string `json:"on_file_change,omitempty"` } // JobRunLifecycleState is the local-only trigger fingerprint. Nested by value, @@ -89,9 +66,7 @@ func emptyJobRunLifecycleState() JobRunLifecycleState { return JobRunLifecycleState{ Triggers: JobRunTriggersState{ OnBundleDeploy: "", - OnFileChange: JobRunFileTriggerState{ - Files: nil, - }, + OnFileChange: nil, }, } } @@ -162,7 +137,7 @@ func (*ResourceJobRun) PrepareState(input *resources.JobRun) *JobRunState { state.Lifecycle.Triggers.OnBundleDeploy = uuid.NewString() } if len(input.ResolvedFileTriggers) > 0 { - state.Lifecycle.Triggers.OnFileChange.Files = input.ResolvedFileTriggers + state.Lifecycle.Triggers.OnFileChange = input.ResolvedFileTriggers } return state } @@ -449,7 +424,7 @@ func (*ResourceJobRun) OverrideChangeDesc(_ context.Context, path *structpath.Pa pathString := path.String() if isJobRunTriggerPath(pathString) { removed := pathString == "lifecycle.triggers.on_bundle_deploy" && (change.New == nil || change.New == "") - removed = removed || pathString == "lifecycle.triggers.on_file_change.files" && change.New == nil + removed = removed || pathString == "lifecycle.triggers.on_file_change" && change.New == nil if removed { change.Action = deployplan.Update change.Reason = "trigger removed" diff --git a/bundle/direct/dresources/job_run_test.go b/bundle/direct/dresources/job_run_test.go index 78ec09d3604..448bdf6b634 100644 --- a/bundle/direct/dresources/job_run_test.go +++ b/bundle/direct/dresources/job_run_test.go @@ -351,7 +351,7 @@ func TestJobRunPrepareStateRequiresSuccess(t *testing.T) { state := (&ResourceJobRun{}).PrepareState(&resources.JobRun{RunNow: jobs.RunNow{JobId: 456}}) assert.Equal(t, jobs.RunResultStateSuccess, state.ResultState) - assert.Nil(t, state.Lifecycle.Triggers.OnFileChange.Files) + assert.Nil(t, state.Lifecycle.Triggers.OnFileChange) } func TestJobRunPrepareStateOnBundleDeploy(t *testing.T) { @@ -375,7 +375,7 @@ func TestJobRunPrepareStateOnFileChange(t *testing.T) { state := (&ResourceJobRun{}).PrepareState(&resources.JobRun{ ResolvedFileTriggers: hashes, }) - assert.Equal(t, hashes, state.Lifecycle.Triggers.OnFileChange.Files) + assert.Equal(t, hashes, state.Lifecycle.Triggers.OnFileChange) assert.Empty(t, state.Lifecycle.Triggers.OnBundleDeploy) }) @@ -388,7 +388,7 @@ func TestJobRunPrepareStateOnFileChange(t *testing.T) { ResolvedFileTriggers: hashes, }) assert.NotEmpty(t, state.Lifecycle.Triggers.OnBundleDeploy) - assert.Equal(t, hashes, state.Lifecycle.Triggers.OnFileChange.Files) + assert.Equal(t, hashes, state.Lifecycle.Triggers.OnFileChange) }) } @@ -396,10 +396,10 @@ func TestJobRunStateUnmarshalLifecycleCompatibility(t *testing.T) { t.Run("missing lifecycle gets empty shape", func(t *testing.T) { var state JobRunState require.NoError(t, json.Unmarshal([]byte(`{}`), &state)) - assert.Nil(t, state.Lifecycle.Triggers.OnFileChange.Files) + assert.Nil(t, state.Lifecycle.Triggers.OnFileChange) }) - t.Run("old path-to-hash map is preserved", func(t *testing.T) { + t.Run("path-to-hash map is preserved", func(t *testing.T) { var state JobRunState require.NoError(t, json.Unmarshal([]byte(`{ "lifecycle": { @@ -410,23 +410,7 @@ func TestJobRunStateUnmarshalLifecycleCompatibility(t *testing.T) { } } }`), &state)) - assert.Equal(t, map[string]string{"a.txt": "abc"}, state.Lifecycle.Triggers.OnFileChange.Files) - }) - - t.Run("new wrapped map is preserved", func(t *testing.T) { - var state JobRunState - require.NoError(t, json.Unmarshal([]byte(`{ - "lifecycle": { - "triggers": { - "on_file_change": { - "files": { - "a.txt": "abc" - } - } - } - } - }`), &state)) - assert.Equal(t, map[string]string{"a.txt": "abc"}, state.Lifecycle.Triggers.OnFileChange.Files) + assert.Equal(t, map[string]string{"a.txt": "abc"}, state.Lifecycle.Triggers.OnFileChange) }) } @@ -444,13 +428,13 @@ func TestJobRunOverrideChangeDescTriggerRemoved(t *testing.T) { assert.Equal(t, "trigger removed", change.Reason) }) - t.Run("clearing on_file_change files downgrades to update", func(t *testing.T) { + t.Run("clearing on_file_change downgrades to update", func(t *testing.T) { change := &ChangeDesc{ Action: deployplan.Recreate, Old: map[string]string{"a.txt": "abc"}, New: nil, } - require.NoError(t, r.OverrideChangeDesc(t.Context(), structpath.MustParsePath("lifecycle.triggers.on_file_change.files"), change, nil)) + require.NoError(t, r.OverrideChangeDesc(t.Context(), structpath.MustParsePath("lifecycle.triggers.on_file_change"), change, nil)) assert.Equal(t, deployplan.Update, change.Action) assert.Equal(t, "trigger removed", change.Reason) }) @@ -471,7 +455,7 @@ func TestJobRunOverrideChangeDescTriggerRemoved(t *testing.T) { Old: "old", New: "new", } - require.NoError(t, r.OverrideChangeDesc(t.Context(), structpath.MustParsePath("lifecycle.triggers.on_file_change.files['a.txt']"), change, nil)) + require.NoError(t, r.OverrideChangeDesc(t.Context(), structpath.MustParsePath("lifecycle.triggers.on_file_change['a.txt']"), change, nil)) assert.Equal(t, deployplan.Recreate, change.Action) }) @@ -481,7 +465,7 @@ func TestJobRunOverrideChangeDescTriggerRemoved(t *testing.T) { Old: "old", New: nil, } - require.NoError(t, r.OverrideChangeDesc(t.Context(), structpath.MustParsePath("lifecycle.triggers.on_file_change.files['a.txt']"), change, nil)) + require.NoError(t, r.OverrideChangeDesc(t.Context(), structpath.MustParsePath("lifecycle.triggers.on_file_change['a.txt']"), change, nil)) assert.Equal(t, deployplan.Recreate, change.Action) }) @@ -491,7 +475,7 @@ func TestJobRunOverrideChangeDescTriggerRemoved(t *testing.T) { Old: "old", New: "", } - require.NoError(t, r.OverrideChangeDesc(t.Context(), structpath.MustParsePath("lifecycle.triggers.on_file_change.files['a.txt']"), change, nil)) + require.NoError(t, r.OverrideChangeDesc(t.Context(), structpath.MustParsePath("lifecycle.triggers.on_file_change['a.txt']"), change, nil)) assert.Equal(t, deployplan.Recreate, change.Action) }) } From 5a213cfe72603b55dd215dded026d8c26f34033c Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Mon, 24 Aug 2026 08:17:19 +0000 Subject: [PATCH 18/75] job_runs: drop unit tests already covered by acceptance The trigger, PrepareState and OverrideChangeDesc cases are asserted end to end by on_bundle_deploy and on_file_change, so the unit copies only duplicated goldens. Keep the ones acceptance cannot reach without stubbing GetRun. --- bundle/direct/dresources/job_run_test.go | 212 +---------------------- 1 file changed, 6 insertions(+), 206 deletions(-) diff --git a/bundle/direct/dresources/job_run_test.go b/bundle/direct/dresources/job_run_test.go index 448bdf6b634..ad23ca605bc 100644 --- a/bundle/direct/dresources/job_run_test.go +++ b/bundle/direct/dresources/job_run_test.go @@ -11,8 +11,6 @@ import ( "time" "github.com/databricks/cli/bundle/config/resources" - "github.com/databricks/cli/bundle/deployplan" - "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/cli/libs/testserver" "github.com/databricks/databricks-sdk-go" @@ -68,28 +66,6 @@ func waitForTestRun(t *testing.T, ctx context.Context, client *databricks.Worksp return r.WaitAfterCreate(ctx, "123", &JobRunState{}) } -func TestJobRunWaitSucceeds(t *testing.T) { - client := jobRunClient(t, &jobs.RunState{ - LifeCycleState: jobs.RunLifeCycleStateTerminated, - ResultState: jobs.RunResultStateSuccess, - }) - - remote, err := waitForTestRun(t, t.Context(), client) - - require.NoError(t, err) - require.NotNil(t, remote.State) - assert.Equal(t, jobs.RunResultStateSuccess, remote.State.ResultState) -} - -func TestReportRunLineIncludesResourceKey(t *testing.T) { - ctx, stderr := cmdio.NewTestContextWithStderr(t.Context()) - ctx = WithResourceKey(ctx, "job_runs.my_run") - - reportRunLine(ctx, 123, "SUCCESS") - - assert.Equal(t, "Output from job_runs.my_run: id=123: SUCCESS\n", stderr.String()) -} - func TestJobRunWaitFailsOnFailedResult(t *testing.T) { client := jobRunClient(t, &jobs.RunState{ LifeCycleState: jobs.RunLifeCycleStateTerminated, @@ -102,36 +78,6 @@ func TestJobRunWaitFailsOnFailedResult(t *testing.T) { require.ErrorContains(t, err, "did not succeed: FAILED: task failed") } -func TestJobRunWaitReportsFailedTask(t *testing.T) { - failed := &jobs.RunState{ - LifeCycleState: jobs.RunLifeCycleStateTerminated, - ResultState: jobs.RunResultStateFailed, - } - server := testserver.New(t) - server.Handle("GET", "/api/2.2/jobs/runs/get", func(req testserver.Request) any { - return jobs.Run{ - RunId: 123, - JobId: 456, - State: failed, - Tasks: []jobs.RunTask{ - {TaskKey: "ok", RunId: 998, State: &jobs.RunState{ - LifeCycleState: jobs.RunLifeCycleStateTerminated, - ResultState: jobs.RunResultStateSuccess, - }}, - {TaskKey: "main", RunId: 999, State: failed}, - }, - } - }) - server.Handle("GET", "/api/2.2/jobs/runs/get-output", func(req testserver.Request) any { - return jobs.RunOutput{Error: "notebook not found"} - }) - - _, err := waitForTestRun(t, t.Context(), jobRunClientFor(t, server)) - - require.ErrorContains(t, err, `task "main": notebook not found`) - assert.NotContains(t, err.Error(), `task "ok"`) -} - // Without the deprecated per-task state, a failed task is told apart from a // skipped one by its termination details. func TestJobRunWaitReportsFailedTaskWithoutDeprecatedState(t *testing.T) { @@ -257,39 +203,6 @@ func TestJobRunWaitFailsOnInternalError(t *testing.T) { require.ErrorContains(t, err, testRunPageLink) } -// A real workspace reports a run whose task failed as INTERNAL_ERROR in the -// deprecated life_cycle_state. The failing task still has to be named. -func TestJobRunWaitReportsFailedTaskOfInternalErrorRun(t *testing.T) { - server := testserver.New(t) - server.Handle("GET", "/api/2.2/jobs/runs/get", func(req testserver.Request) any { - return jobs.Run{ - RunId: 123, - JobId: 456, - RunPageUrl: testRunPageURL, - State: &jobs.RunState{ - LifeCycleState: jobs.RunLifeCycleStateInternalError, - ResultState: jobs.RunResultStateFailed, - StateMessage: "Task main failed with message: Workload failed, see run output for details.", - }, - Tasks: []jobs.RunTask{ - {TaskKey: "main", RunId: 999, State: &jobs.RunState{ - LifeCycleState: jobs.RunLifeCycleStateTerminated, - ResultState: jobs.RunResultStateFailed, - }}, - }, - } - }) - server.Handle("GET", "/api/2.2/jobs/runs/get-output", func(req testserver.Request) any { - return jobs.RunOutput{Error: "RuntimeError: intentional failure"} - }) - - _, err := waitForTestRun(t, t.Context(), jobRunClientFor(t, server)) - - require.ErrorContains(t, err, "run did not succeed: FAILED") - require.ErrorContains(t, err, `task "main": RuntimeError: intentional failure`) - require.ErrorContains(t, err, testRunPageLink) -} - func TestJobRunWaitReportsOnlyTheLastAttemptOfATask(t *testing.T) { failed := &jobs.RunState{ LifeCycleState: jobs.RunLifeCycleStateTerminated, @@ -332,64 +245,17 @@ func TestJobRunWaitAbandonedLinksTheRun(t *testing.T) { require.ErrorContains(t, err, testRunPageLink) } -// An abandoned wait leaves the run going with its id recorded, so the next deploy -// reads an empty outcome, which result_state drift catches. -func TestJobRunReadOfUnfinishedRunReportsNoResult(t *testing.T) { - client := jobRunClient(t, &jobs.RunState{LifeCycleState: jobs.RunLifeCycleStateRunning}) - - remote, err := (&ResourceJobRun{}).New(client).DoRead(t.Context(), "123") - - require.NoError(t, err) - require.NotNil(t, remote.State) - assert.Equal(t, jobs.RunLifeCycleStateRunning, remote.State.LifeCycleState) - assert.Empty(t, remote.ResultState) -} - -// PrepareState records the outcome the run must reach, the same for every run, -// so the planner has something to compare the remote against. -func TestJobRunPrepareStateRequiresSuccess(t *testing.T) { - state := (&ResourceJobRun{}).PrepareState(&resources.JobRun{RunNow: jobs.RunNow{JobId: 456}}) - - assert.Equal(t, jobs.RunResultStateSuccess, state.ResultState) - assert.Nil(t, state.Lifecycle.Triggers.OnFileChange) -} - -func TestJobRunPrepareStateOnBundleDeploy(t *testing.T) { +func TestJobRunPrepareStateBothTriggers(t *testing.T) { on := true - input := &resources.JobRun{ + hashes := map[string]string{"a.txt": "abc"} + state := (&ResourceJobRun{}).PrepareState(&resources.JobRun{ Lifecycle: &resources.JobRunLifecycle{ Triggers: []resources.JobRunTrigger{{OnBundleDeploy: &on}}, }, - } - first := (&ResourceJobRun{}).PrepareState(input) - assert.NotEmpty(t, first.Lifecycle.Triggers.OnBundleDeploy) - - second := (&ResourceJobRun{}).PrepareState(input) - assert.NotEqual(t, first.Lifecycle.Triggers.OnBundleDeploy, second.Lifecycle.Triggers.OnBundleDeploy) -} - -func TestJobRunPrepareStateOnFileChange(t *testing.T) { - hashes := map[string]string{"a.txt": "abc"} - - t.Run("armed", func(t *testing.T) { - state := (&ResourceJobRun{}).PrepareState(&resources.JobRun{ - ResolvedFileTriggers: hashes, - }) - assert.Equal(t, hashes, state.Lifecycle.Triggers.OnFileChange) - assert.Empty(t, state.Lifecycle.Triggers.OnBundleDeploy) - }) - - t.Run("both triggers", func(t *testing.T) { - on := true - state := (&ResourceJobRun{}).PrepareState(&resources.JobRun{ - Lifecycle: &resources.JobRunLifecycle{ - Triggers: []resources.JobRunTrigger{{OnBundleDeploy: &on}}, - }, - ResolvedFileTriggers: hashes, - }) - assert.NotEmpty(t, state.Lifecycle.Triggers.OnBundleDeploy) - assert.Equal(t, hashes, state.Lifecycle.Triggers.OnFileChange) + ResolvedFileTriggers: hashes, }) + assert.NotEmpty(t, state.Lifecycle.Triggers.OnBundleDeploy) + assert.Equal(t, hashes, state.Lifecycle.Triggers.OnFileChange) } func TestJobRunStateUnmarshalLifecycleCompatibility(t *testing.T) { @@ -414,72 +280,6 @@ func TestJobRunStateUnmarshalLifecycleCompatibility(t *testing.T) { }) } -func TestJobRunOverrideChangeDescTriggerRemoved(t *testing.T) { - r := &ResourceJobRun{} - - t.Run("clearing on_bundle_deploy leaf downgrades to update", func(t *testing.T) { - change := &ChangeDesc{ - Action: deployplan.Recreate, - Old: "old", - New: "", - } - require.NoError(t, r.OverrideChangeDesc(t.Context(), structpath.MustParsePath("lifecycle.triggers.on_bundle_deploy"), change, nil)) - assert.Equal(t, deployplan.Update, change.Action) - assert.Equal(t, "trigger removed", change.Reason) - }) - - t.Run("clearing on_file_change downgrades to update", func(t *testing.T) { - change := &ChangeDesc{ - Action: deployplan.Recreate, - Old: map[string]string{"a.txt": "abc"}, - New: nil, - } - require.NoError(t, r.OverrideChangeDesc(t.Context(), structpath.MustParsePath("lifecycle.triggers.on_file_change"), change, nil)) - assert.Equal(t, deployplan.Update, change.Action) - assert.Equal(t, "trigger removed", change.Reason) - }) - - t.Run("fresh fingerprint still recreates", func(t *testing.T) { - change := &ChangeDesc{ - Action: deployplan.Recreate, - Old: "old", - New: "new", - } - require.NoError(t, r.OverrideChangeDesc(t.Context(), structpath.MustParsePath("lifecycle.triggers.on_bundle_deploy"), change, nil)) - assert.Equal(t, deployplan.Recreate, change.Action) - }) - - t.Run("changed on_file_change hash still recreates", func(t *testing.T) { - change := &ChangeDesc{ - Action: deployplan.Recreate, - Old: "old", - New: "new", - } - require.NoError(t, r.OverrideChangeDesc(t.Context(), structpath.MustParsePath("lifecycle.triggers.on_file_change['a.txt']"), change, nil)) - assert.Equal(t, deployplan.Recreate, change.Action) - }) - - t.Run("removed matched file still recreates", func(t *testing.T) { - change := &ChangeDesc{ - Action: deployplan.Recreate, - Old: "old", - New: nil, - } - require.NoError(t, r.OverrideChangeDesc(t.Context(), structpath.MustParsePath("lifecycle.triggers.on_file_change['a.txt']"), change, nil)) - assert.Equal(t, deployplan.Recreate, change.Action) - }) - - t.Run("missing file fingerprint still recreates", func(t *testing.T) { - change := &ChangeDesc{ - Action: deployplan.Recreate, - Old: "old", - New: "", - } - require.NoError(t, r.OverrideChangeDesc(t.Context(), structpath.MustParsePath("lifecycle.triggers.on_file_change['a.txt']"), change, nil)) - assert.Equal(t, deployplan.Recreate, change.Action) - }) -} - // The planner diffs RemapState(remote) against PrepareState(config), so a run // that did not end in SUCCESS has to surface as a difference on result_state. func TestJobRunRemapStateCarriesTheOutcome(t *testing.T) { From be936946bb743998957a6a6343049d87e5099555 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Mon, 24 Aug 2026 08:27:10 +0000 Subject: [PATCH 19/75] acc: refresh job_runs goldens for the omitted empty trigger Flattening on_file_change to a map made an unarmed trigger drop out of the serialized state, so the three plans that print an unarmed job_run now report an empty triggers object. --- acceptance/bundle/resources/job_runs/failed_run/output.txt | 4 +--- .../bundle/resources/job_runs/interrupted_run/output.txt | 4 +--- acceptance/bundle/resources/job_runs/redeploy/output.txt | 4 +--- 3 files changed, 3 insertions(+), 9 deletions(-) diff --git a/acceptance/bundle/resources/job_runs/failed_run/output.txt b/acceptance/bundle/resources/job_runs/failed_run/output.txt index 37f1a831681..5b8b6dbcd5f 100644 --- a/acceptance/bundle/resources/job_runs/failed_run/output.txt +++ b/acceptance/bundle/resources/job_runs/failed_run/output.txt @@ -28,9 +28,7 @@ Files: 7 uploaded, 0 deleted "value": { "job_id": [MY_JOB_ID], "lifecycle": { - "triggers": { - "on_file_change": {} - } + "triggers": {} }, "result_state": "SUCCESS" } diff --git a/acceptance/bundle/resources/job_runs/interrupted_run/output.txt b/acceptance/bundle/resources/job_runs/interrupted_run/output.txt index df52920c879..31b53fb4032 100644 --- a/acceptance/bundle/resources/job_runs/interrupted_run/output.txt +++ b/acceptance/bundle/resources/job_runs/interrupted_run/output.txt @@ -44,9 +44,7 @@ Exit code: 1 "value": { "job_id": [MY_JOB_ID], "lifecycle": { - "triggers": { - "on_file_change": {} - } + "triggers": {} }, "result_state": "SUCCESS" } diff --git a/acceptance/bundle/resources/job_runs/redeploy/output.txt b/acceptance/bundle/resources/job_runs/redeploy/output.txt index a80685733ad..8532692bdd9 100644 --- a/acceptance/bundle/resources/job_runs/redeploy/output.txt +++ b/acceptance/bundle/resources/job_runs/redeploy/output.txt @@ -60,9 +60,7 @@ Resources: "env": "prod" }, "lifecycle": { - "triggers": { - "on_file_change": {} - } + "triggers": {} }, "result_state": "SUCCESS" } From d5042f3c49c1ce854cf4139ccd19ce0ad9d14e03 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Mon, 24 Aug 2026 08:42:42 +0000 Subject: [PATCH 20/75] job_runs: cover cancel-before-delete with an acceptance test The unit test staged an asynchronous cancellation by hand; the acceptance test shows the real request order a user's destroy produces. Also drops two unit tests that the on_file_change acceptance tests already cover. --- .../destroy_unfinished_run/databricks.yml | 15 +++ .../destroy_unfinished_run/out.test.toml | 2 + .../destroy_unfinished_run/output.txt | 44 ++++++++ .../job_runs/destroy_unfinished_run/script | 18 ++++ bundle/direct/dresources/job_run_test.go | 100 +++--------------- 5 files changed, 94 insertions(+), 85 deletions(-) create mode 100644 acceptance/bundle/resources/job_runs/destroy_unfinished_run/databricks.yml create mode 100644 acceptance/bundle/resources/job_runs/destroy_unfinished_run/out.test.toml create mode 100644 acceptance/bundle/resources/job_runs/destroy_unfinished_run/output.txt create mode 100644 acceptance/bundle/resources/job_runs/destroy_unfinished_run/script diff --git a/acceptance/bundle/resources/job_runs/destroy_unfinished_run/databricks.yml b/acceptance/bundle/resources/job_runs/destroy_unfinished_run/databricks.yml new file mode 100644 index 00000000000..177a9ca2227 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/destroy_unfinished_run/databricks.yml @@ -0,0 +1,15 @@ +bundle: + name: job-runs-destroy-unfinished-run + +resources: + jobs: + my_job: + name: my-job + tasks: + - task_key: main + notebook_task: + notebook_path: /Workspace/test + + job_runs: + my_run: + job_id: ${resources.jobs.my_job.id} diff --git a/acceptance/bundle/resources/job_runs/destroy_unfinished_run/out.test.toml b/acceptance/bundle/resources/job_runs/destroy_unfinished_run/out.test.toml new file mode 100644 index 00000000000..0938e678987 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/destroy_unfinished_run/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/job_runs/destroy_unfinished_run/output.txt b/acceptance/bundle/resources/job_runs/destroy_unfinished_run/output.txt new file mode 100644 index 00000000000..1b1fd310b0c --- /dev/null +++ b/acceptance/bundle/resources/job_runs/destroy_unfinished_run/output.txt @@ -0,0 +1,44 @@ + +=== the deploy stops waiting before the run finishes +>>> errcode [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-destroy-unfinished-run/default/files... +Error: cannot create resources.job_runs.my_run: waiting after creating id=[MY_RUN_ID]: Fault injected by test. (403 INJECTED) + +Endpoint: GET [DATABRICKS_URL]/api/2.2/jobs/runs/get?run_id=[MY_RUN_ID] +HTTP Status: 403 Forbidden +API error_code: INJECTED +API message: Fault injected by test. + +Files: 4 uploaded, 0 deleted + +Exit code: 1 + +>>> read_id.py my_run +[MY_RUN_ID] + +=== destroy cancels the run before deleting it +>>> [CLI] bundle destroy --auto-approve +Warn: planning resources.job_runs.my_run: reading resources.job_runs.my_run id="[MY_RUN_ID]": Fault injected by test. +The following resources will be deleted: + delete resources.job_runs.my_run + delete resources.jobs.my_job + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/job-runs-destroy-unfinished-run/default + +Destroy: 2 deleted + +>>> print_requests.py //jobs/runs +{ + "method": "POST", + "path": "/api/2.2/jobs/runs/cancel", + "body": { + "run_id": [MY_RUN_ID] + } +} +{ + "method": "POST", + "path": "/api/2.2/jobs/runs/delete", + "body": { + "run_id": [MY_RUN_ID] + } +} diff --git a/acceptance/bundle/resources/job_runs/destroy_unfinished_run/script b/acceptance/bundle/resources/job_runs/destroy_unfinished_run/script new file mode 100644 index 00000000000..3b9914b1037 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/destroy_unfinished_run/script @@ -0,0 +1,18 @@ +cleanup() { + rm -f out.requests.txt +} +trap cleanup EXIT + +# The fake workspace settles a run on its first successful poll, so both the +# deploy's wait and the refresh destroy plans with are faulted: the run is still +# going when the delete reads it, which is what an interrupted deploy leaves behind. +fault.py "GET /api/2.2/jobs/runs/get" 403 0 2 + +title "the deploy stops waiting before the run finishes" +trace errcode $CLI bundle deploy +trace read_id.py my_run + +# jobs/runs/delete rejects an active run, so the run is cancelled first. +title "destroy cancels the run before deleting it" +trace $CLI bundle destroy --auto-approve +trace print_requests.py //jobs/runs diff --git a/bundle/direct/dresources/job_run_test.go b/bundle/direct/dresources/job_run_test.go index ad23ca605bc..804968f7707 100644 --- a/bundle/direct/dresources/job_run_test.go +++ b/bundle/direct/dresources/job_run_test.go @@ -10,7 +10,6 @@ import ( "testing" "time" - "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/cli/libs/testserver" "github.com/databricks/databricks-sdk-go" @@ -245,39 +244,13 @@ func TestJobRunWaitAbandonedLinksTheRun(t *testing.T) { require.ErrorContains(t, err, testRunPageLink) } -func TestJobRunPrepareStateBothTriggers(t *testing.T) { - on := true - hashes := map[string]string{"a.txt": "abc"} - state := (&ResourceJobRun{}).PrepareState(&resources.JobRun{ - Lifecycle: &resources.JobRunLifecycle{ - Triggers: []resources.JobRunTrigger{{OnBundleDeploy: &on}}, - }, - ResolvedFileTriggers: hashes, - }) - assert.NotEmpty(t, state.Lifecycle.Triggers.OnBundleDeploy) - assert.Equal(t, hashes, state.Lifecycle.Triggers.OnFileChange) -} +// State written before lifecycle existed has no such key, and must still load. +func TestJobRunStateUnmarshalWithoutLifecycle(t *testing.T) { + var state JobRunState -func TestJobRunStateUnmarshalLifecycleCompatibility(t *testing.T) { - t.Run("missing lifecycle gets empty shape", func(t *testing.T) { - var state JobRunState - require.NoError(t, json.Unmarshal([]byte(`{}`), &state)) - assert.Nil(t, state.Lifecycle.Triggers.OnFileChange) - }) + require.NoError(t, json.Unmarshal([]byte(`{}`), &state)) - t.Run("path-to-hash map is preserved", func(t *testing.T) { - var state JobRunState - require.NoError(t, json.Unmarshal([]byte(`{ - "lifecycle": { - "triggers": { - "on_file_change": { - "a.txt": "abc" - } - } - } - }`), &state)) - assert.Equal(t, map[string]string{"a.txt": "abc"}, state.Lifecycle.Triggers.OnFileChange) - }) + assert.Nil(t, state.Lifecycle.Triggers.OnFileChange) } // The planner diffs RemapState(remote) against PrepareState(config), so a run @@ -372,68 +345,25 @@ func TestJobRunCreateSendsAFreshIdempotencyToken(t *testing.T) { assert.Empty(t, config.IdempotencyToken) } -// jobRunDeletion records what the fake workspace saw while a run was deleted. -type jobRunDeletion struct { - cancelled atomic.Bool - settled atomic.Bool - settledAtDelete atomic.Bool -} - -// jobRunDeleteClient returns a client for a run in the given state, whose cancel -// settles one poll late the way the API's asynchronous cancellation does. -func jobRunDeleteClient(t *testing.T, state *jobs.RunState) (*databricks.WorkspaceClient, *jobRunDeletion) { - t.Helper() - var deletion jobRunDeletion - cancelled := &jobs.RunState{ - LifeCycleState: jobs.RunLifeCycleStateTerminated, - ResultState: jobs.RunResultStateCanceled, - } - +func TestJobRunDeleteLeavesFinishedRunAlone(t *testing.T) { + var cancelled atomic.Bool server := testserver.New(t) server.Handle("GET", "/api/2.2/jobs/runs/get", func(req testserver.Request) any { - current := state - switch { - case deletion.settled.Load(): - current = cancelled - case deletion.cancelled.Load(): - // Report the run's old state once more, then settle on the next poll. - deletion.settled.Store(true) - } - return jobs.Run{RunId: 123, JobId: 456, State: current} + return jobs.Run{RunId: 123, JobId: 456, State: &jobs.RunState{ + LifeCycleState: jobs.RunLifeCycleStateTerminated, + ResultState: jobs.RunResultStateSuccess, + }} }) server.Handle("POST", "/api/2.2/jobs/runs/cancel", func(req testserver.Request) any { - deletion.cancelled.Store(true) + cancelled.Store(true) return testserver.Response{} }) server.Handle("POST", "/api/2.2/jobs/runs/delete", func(req testserver.Request) any { - deletion.settledAtDelete.Store(deletion.settled.Load()) return testserver.Response{} }) - return jobRunClientFor(t, server), &deletion -} - -func deleteTestRun(t *testing.T, client *databricks.WorkspaceClient) error { - t.Helper() - return (&ResourceJobRun{}).New(client).DoDelete(t.Context(), "123", &JobRunState{}) -} - -func TestJobRunDeleteCancelsUnfinishedRun(t *testing.T) { - // An interrupted wait leaves the run going, and jobs/runs/delete rejects it. - client, deletion := jobRunDeleteClient(t, &jobs.RunState{LifeCycleState: jobs.RunLifeCycleStateRunning}) - - require.NoError(t, deleteTestRun(t, client)) - - assert.True(t, deletion.cancelled.Load(), "expected the run to be cancelled") - assert.True(t, deletion.settledAtDelete.Load(), "expected the delete to wait for the cancellation to settle") -} - -func TestJobRunDeleteLeavesFinishedRunAlone(t *testing.T) { - client, deletion := jobRunDeleteClient(t, &jobs.RunState{ - LifeCycleState: jobs.RunLifeCycleStateTerminated, - ResultState: jobs.RunResultStateSuccess, - }) + r := (&ResourceJobRun{}).New(jobRunClientFor(t, server)) - require.NoError(t, deleteTestRun(t, client)) + require.NoError(t, r.DoDelete(t.Context(), "123", &JobRunState{})) - assert.False(t, deletion.cancelled.Load(), "a run that already finished has nothing to cancel") + assert.False(t, cancelled.Load(), "a run that already finished has nothing to cancel") } From 9e0b681bc8fc0fc48f496cec7546de43e4fa11bd Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Fri, 21 Aug 2026 15:53:19 +0000 Subject: [PATCH 21/75] Add job_runs lifecycle.triggers.on_value_change Re-fire a run when a resolved expression changes, including resource refs that become deploy-graph edges. Removing the trigger is a state-only update. --- .../bundles/job-runs-on-value-change.md | 1 + .../job_runs/on_value_change/databricks.yml | 25 ++++ .../job_runs/on_value_change/out.test.toml | 2 + .../job_runs/on_value_change/output.txt | 140 ++++++++++++++++++ .../resources/job_runs/on_value_change/script | 40 +++++ .../on_value_change_ref/databricks.yml | 32 ++++ .../on_value_change_ref/out.test.toml | 3 + .../job_runs/on_value_change_ref/output.txt | 76 ++++++++++ .../job_runs/on_value_change_ref/script | 29 ++++ .../job_runs/on_value_change_ref/test.toml | 5 + .../mutator/validate_job_run_triggers.go | 23 ++- .../mutator/validate_job_run_triggers_test.go | 48 +++++- bundle/config/resources/job_run.go | 12 ++ bundle/config/resources/lifecycle.go | 17 ++- bundle/direct/bundle_plan.go | 1 + bundle/direct/dresources/job_run.go | 58 ++++++++ bundle/direct/dresources/job_run_test.go | 42 ++++++ bundle/internal/schema/annotations.yml | 3 + bundle/schema/jsonschema.json | 4 + 19 files changed, 548 insertions(+), 13 deletions(-) create mode 100644 .nextchanges/bundles/job-runs-on-value-change.md create mode 100644 acceptance/bundle/resources/job_runs/on_value_change/databricks.yml create mode 100644 acceptance/bundle/resources/job_runs/on_value_change/out.test.toml create mode 100644 acceptance/bundle/resources/job_runs/on_value_change/output.txt create mode 100644 acceptance/bundle/resources/job_runs/on_value_change/script create mode 100644 acceptance/bundle/resources/job_runs/on_value_change_ref/databricks.yml create mode 100644 acceptance/bundle/resources/job_runs/on_value_change_ref/out.test.toml create mode 100644 acceptance/bundle/resources/job_runs/on_value_change_ref/output.txt create mode 100644 acceptance/bundle/resources/job_runs/on_value_change_ref/script create mode 100644 acceptance/bundle/resources/job_runs/on_value_change_ref/test.toml diff --git a/.nextchanges/bundles/job-runs-on-value-change.md b/.nextchanges/bundles/job-runs-on-value-change.md new file mode 100644 index 00000000000..94d0b06290f --- /dev/null +++ b/.nextchanges/bundles/job-runs-on-value-change.md @@ -0,0 +1 @@ +direct: `resources.job_runs` can set `lifecycle.triggers.on_value_change` to an interpolated expression (for example `${resources.jobs.foo.id}`) to re-fire the run when that value changes. Removing the trigger does not recreate the existing run. diff --git a/acceptance/bundle/resources/job_runs/on_value_change/databricks.yml b/acceptance/bundle/resources/job_runs/on_value_change/databricks.yml new file mode 100644 index 00000000000..b0ed0322b9c --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_value_change/databricks.yml @@ -0,0 +1,25 @@ +bundle: + name: job-runs-on-value-change + +variables: + stamp: + default: v1 + other: + default: a1 + +resources: + jobs: + my_job: + name: my-job + tasks: + - task_key: main + notebook_task: + notebook_path: /Workspace/test + + job_runs: + my_run: + job_id: ${resources.jobs.my_job.id} + lifecycle: + triggers: + - on_value_change: ${var.stamp} + - on_value_change: ${var.other} diff --git a/acceptance/bundle/resources/job_runs/on_value_change/out.test.toml b/acceptance/bundle/resources/job_runs/on_value_change/out.test.toml new file mode 100644 index 00000000000..0938e678987 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_value_change/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/job_runs/on_value_change/output.txt b/acceptance/bundle/resources/job_runs/on_value_change/output.txt new file mode 100644 index 00000000000..970d1c9a6ac --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_value_change/output.txt @@ -0,0 +1,140 @@ + +=== first deploy triggers a run +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-value-change/default/files... +Output from job_runs.my_run: id=[MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?o=[NUMID] +Output from job_runs.my_run: id=[MY_RUN_ID]: SUCCESS +Created job_runs.my_run +Created jobs.my_job +Files: 4 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +>>> read_id.py my_job +[MY_JOB_ID] + +>>> print_requests.py //jobs/run-now +{ + "method": "POST", + "path": "/api/2.2/jobs/run-now", + "body": { + "idempotency_token": "[UUID]", + "job_id": [MY_JOB_ID] + } +} + +=== redeploy with unchanged values plans nothing +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-value-change/default/files... +Files: 2 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 2 unchanged + +>>> print_requests.py //jobs/run-now + +=== changing one value re-fires +>>> [CLI] bundle plan +recreate job_runs.my_run + +Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged + +>>> changes +{ + "lifecycle.triggers.on_value_change": { + "action": "recreate", + "reason": "immutable", + "old": { + "a1": "a1", + "v1": "v1" + }, + "new": { + "a1": "a1", + "v2": "v2" + } + }, + "lifecycle.triggers.on_value_change['v1']": { + "action": "recreate", + "reason": "immutable", + "old": "v1" + }, + "lifecycle.triggers.on_value_change['v2']": { + "action": "recreate", + "reason": "immutable", + "new": "v2" + }, + "result_state": { + "action": "skip", + "reason": "remote_already_set", + "new": "SUCCESS", + "remote": "SUCCESS" + } +} + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-value-change/default/files... +Output from job_runs.my_run: id=[MY_RUN_ID_2]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_2]?o=[NUMID] +Output from job_runs.my_run: id=[MY_RUN_ID_2]: SUCCESS +Recreated job_runs.my_run +Files: 3 uploaded, 0 deleted +Resources: 1 created, 0 changed, 1 deleted, 1 unchanged + +>>> print_requests.py --keep //jobs/runs/delete +{ + "method": "POST", + "path": "/api/2.2/jobs/runs/delete", + "body": { + "run_id": [MY_RUN_ID] + } +} + +>>> print_requests.py //jobs/run-now +{ + "method": "POST", + "path": "/api/2.2/jobs/run-now", + "body": { + "idempotency_token": "[UUID]", + "job_id": [MY_JOB_ID] + } +} + +=== removing on_value_change rewrites state without a run +>>> [CLI] bundle plan +update job_runs.my_run + +Plan: 0 to add, 1 to change, 0 to delete, 1 unchanged + +>>> changes +{ + "lifecycle.triggers.on_value_change": { + "action": "update", + "reason": "trigger removed", + "old": { + "a1": "a1", + "v2": "v2" + } + }, + "result_state": { + "action": "skip", + "reason": "remote_already_set", + "new": "SUCCESS", + "remote": "SUCCESS" + } +} + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-value-change/default/files... +Updated job_runs.my_run +Files: 3 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 1 unchanged + +>>> print_requests.py //jobs/run-now + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.job_runs.my_run + delete resources.jobs.my_job + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-value-change/default + +Destroy: 2 deleted diff --git a/acceptance/bundle/resources/job_runs/on_value_change/script b/acceptance/bundle/resources/job_runs/on_value_change/script new file mode 100644 index 00000000000..a8421e880dd --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_value_change/script @@ -0,0 +1,40 @@ +cleanup() { + trace $CLI bundle destroy --auto-approve + rm -f out.requests.txt +} +trap cleanup EXIT + +changes() { + $CLI bundle plan -o json | jq '.plan["resources.job_runs.my_run"].changes' +} + +title "first deploy triggers a run" +trace $CLI bundle deploy +trace read_id.py my_job +read_id.py my_run > /dev/null +trace print_requests.py //jobs/run-now + +title "redeploy with unchanged values plans nothing" +trace $CLI bundle plan +trace $CLI bundle deploy +trace print_requests.py //jobs/run-now + +title "changing one value re-fires" +update_file.py databricks.yml "default: v1" "default: v2" +trace $CLI bundle plan +trace changes +trace $CLI bundle deploy +read_id.py my_run > /dev/null +trace print_requests.py --keep //jobs/runs/delete +trace print_requests.py //jobs/run-now + +title "removing on_value_change rewrites state without a run" +update_file.py databricks.yml ' lifecycle: + triggers: + - on_value_change: ${var.stamp} + - on_value_change: ${var.other} +' '' +trace $CLI bundle plan +trace changes +trace $CLI bundle deploy +trace print_requests.py //jobs/run-now diff --git a/acceptance/bundle/resources/job_runs/on_value_change_ref/databricks.yml b/acceptance/bundle/resources/job_runs/on_value_change_ref/databricks.yml new file mode 100644 index 00000000000..84de0fdaf82 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_value_change_ref/databricks.yml @@ -0,0 +1,32 @@ +bundle: + name: job-runs-on-value-change-ref + +resources: + jobs: + my_job: + name: my-job + tasks: + - task_key: main + notebook_task: + notebook_path: /Workspace/test + + other: + name: other-job + tasks: + - task_key: main + notebook_task: + notebook_path: /Workspace/test + + extra: + name: extra-job + tasks: + - task_key: main + notebook_task: + notebook_path: /Workspace/test + + job_runs: + my_run: + job_id: ${resources.jobs.my_job.id} + lifecycle: + triggers: + - on_value_change: ${resources.jobs.other.id}-${resources.jobs.extra.id} diff --git a/acceptance/bundle/resources/job_runs/on_value_change_ref/out.test.toml b/acceptance/bundle/resources/job_runs/on_value_change_ref/out.test.toml new file mode 100644 index 00000000000..57b0f616850 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_value_change_ref/out.test.toml @@ -0,0 +1,3 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/job_runs/on_value_change_ref/output.txt b/acceptance/bundle/resources/job_runs/on_value_change_ref/output.txt new file mode 100644 index 00000000000..f1521167db3 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_value_change_ref/output.txt @@ -0,0 +1,76 @@ + +=== plan depends on both jobs in the expression +>>> jq .plan["resources.job_runs.my_run"].depends_on tmp.plan.json +[ + { + "node": "resources.jobs.extra", + "label": "${resources.jobs.extra.id}" + }, + { + "node": "resources.jobs.my_job", + "label": "${resources.jobs.my_job.id}" + }, + { + "node": "resources.jobs.other", + "label": "${resources.jobs.other.id}" + } +] + +=== first deploy triggers a run +=== bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-value-change-ref/default/files... +Output from job_runs.my_run: id=[MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?o=[NUMID] +Output from job_runs.my_run: id=[MY_RUN_ID]: SUCCESS +Created job_runs.my_run +Created jobs.extra +Created jobs.my_job +Created jobs.other +Files: 6 uploaded, 0 deleted +Resources: 4 created, 0 changed, 0 deleted, 0 unchanged + +>>> read_id.py my_job +[MY_JOB_ID] + +>>> read_id.py other +[OTHER_ID] + +>>> read_id.py extra +[EXTRA_ID] + +>>> print_requests.py //jobs/run-now +{ + "method": "POST", + "path": "/api/2.2/jobs/run-now", + "body": { + "idempotency_token": "[UUID]", + "job_id": [MY_JOB_ID] + } +} + +=== state records the concatenated resolved ids +>>> print_state.py +{ + "${resources.jobs.other.id}-${resources.jobs.extra.id}": "[OTHER_ID]-[EXTRA_ID]" +} + +=== redeploy with unchanged ids plans nothing +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 4 unchanged + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-value-change-ref/default/files... +Files: 2 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 4 unchanged + +>>> print_requests.py //jobs/run-now + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.job_runs.my_run + delete resources.jobs.extra + delete resources.jobs.my_job + delete resources.jobs.other + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-value-change-ref/default + +Destroy: 4 deleted diff --git a/acceptance/bundle/resources/job_runs/on_value_change_ref/script b/acceptance/bundle/resources/job_runs/on_value_change_ref/script new file mode 100644 index 00000000000..7b0c981b10f --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_value_change_ref/script @@ -0,0 +1,29 @@ +cleanup() { + trace $CLI bundle destroy --auto-approve + rm -f out.requests.txt +} +trap cleanup EXIT + +title "plan depends on both jobs in the expression" +$CLI bundle plan -o json > tmp.plan.json +trace jq '.plan["resources.job_runs.my_run"].depends_on' tmp.plan.json | contains.py \ + 'resources.jobs.other' '${resources.jobs.other.id}' \ + 'resources.jobs.extra' '${resources.jobs.extra.id}' + +title "first deploy triggers a run" +# Not traced: readplanarg makes the command line differ per READPLAN variant. +title "bundle deploy\n" +$CLI bundle deploy $(readplanarg tmp.plan.json) +trace read_id.py my_job +trace read_id.py other +trace read_id.py extra +read_id.py my_run > /dev/null +trace print_requests.py //jobs/run-now + +title "state records the concatenated resolved ids" +trace print_state.py | jq '.state["resources.job_runs.my_run"].state.lifecycle.triggers.on_value_change' + +title "redeploy with unchanged ids plans nothing" +trace $CLI bundle plan +trace $CLI bundle deploy +trace print_requests.py //jobs/run-now diff --git a/acceptance/bundle/resources/job_runs/on_value_change_ref/test.toml b/acceptance/bundle/resources/job_runs/on_value_change_ref/test.toml new file mode 100644 index 00000000000..a698f7a65d1 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_value_change_ref/test.toml @@ -0,0 +1,5 @@ +# Deploy both by re-planning and from a plan saved on disk, so resource refs +# in on_value_change survive plan serialization. +EnvMatrix.READPLAN = ["", "1"] + +Ignore = ["tmp.plan.json"] diff --git a/bundle/config/mutator/validate_job_run_triggers.go b/bundle/config/mutator/validate_job_run_triggers.go index c185d3b5781..fa62b3110f6 100644 --- a/bundle/config/mutator/validate_job_run_triggers.go +++ b/bundle/config/mutator/validate_job_run_triggers.go @@ -26,28 +26,28 @@ func (*validateJobRunTriggers) Apply(_ context.Context, b *bundle.Bundle) diag.D if jr == nil || jr.Lifecycle == nil { continue } - // Recreate-every-deploy cannot coexist with prevent_destroy. - if (jr.HasOnBundleDeploy() || jr.HasOnFileChange()) && jr.Lifecycle.PreventDestroy { + if (jr.HasOnBundleDeploy() || jr.HasOnFileChange() || jr.HasOnValueChange()) && jr.Lifecycle.PreventDestroy { diags = diags.Append(diag.Diagnostic{ Severity: diag.Error, - Summary: "lifecycle.triggers.on_bundle_deploy or on_file_change is incompatible with lifecycle.prevent_destroy", + Summary: "lifecycle.triggers.on_bundle_deploy, on_file_change, or on_value_change is incompatible with lifecycle.prevent_destroy", Locations: b.Config.GetLocations(fmt.Sprintf("resources.job_runs.%s.lifecycle", name)), }) } for i, t := range jr.Lifecycle.Triggers { path := fmt.Sprintf("resources.job_runs.%s.lifecycle.triggers[%d]", name, i) - if t.OnBundleDeploy == nil && t.OnFileChange == nil { + switch t.ArmedCount() { + case 0: diags = diags.Append(diag.Diagnostic{ Severity: diag.Error, - Summary: "lifecycle.triggers entry must set on_bundle_deploy or on_file_change", + Summary: "lifecycle.triggers entry must set on_bundle_deploy, on_file_change, or on_value_change", Locations: b.Config.GetLocations(path), }) continue - } - if t.OnBundleDeploy != nil && t.OnFileChange != nil { + case 1: + default: diags = diags.Append(diag.Diagnostic{ Severity: diag.Error, - Summary: "lifecycle.triggers entry must set only one of on_bundle_deploy or on_file_change", + Summary: "lifecycle.triggers entry must set only one of on_bundle_deploy, on_file_change, or on_value_change", Locations: b.Config.GetLocations(path), }) continue @@ -66,6 +66,13 @@ func (*validateJobRunTriggers) Apply(_ context.Context, b *bundle.Bundle) diag.D Locations: b.Config.GetLocations(path + ".on_file_change"), }) } + if t.OnValueChange != nil && strings.TrimSpace(*t.OnValueChange) == "" { + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: "lifecycle.triggers.on_value_change must be non-empty when set", + Locations: b.Config.GetLocations(path + ".on_value_change"), + }) + } } } return diags diff --git a/bundle/config/mutator/validate_job_run_triggers_test.go b/bundle/config/mutator/validate_job_run_triggers_test.go index 7e8772d8d8b..e6b429f0f5a 100644 --- a/bundle/config/mutator/validate_job_run_triggers_test.go +++ b/bundle/config/mutator/validate_job_run_triggers_test.go @@ -18,6 +18,10 @@ func TestValidateJobRunTriggers(t *testing.T) { emptyFile := "" whitespaceFile := " \t" + valueChange := "${resources.jobs.foo.id}" + emptyValue := "" + whitespaceValue := " \t" + tests := []struct { name string triggers []resources.JobRunTrigger @@ -41,6 +45,7 @@ func TestValidateJobRunTriggers(t *testing.T) { triggers: []resources.JobRunTrigger{ {OnFileChange: &fileChange}, {OnBundleDeploy: &trueVal}, + {OnValueChange: &valueChange}, }, }, { @@ -48,14 +53,14 @@ func TestValidateJobRunTriggers(t *testing.T) { triggers: []resources.JobRunTrigger{ {}, }, - summary: "lifecycle.triggers entry must set on_bundle_deploy or on_file_change", + summary: "lifecycle.triggers entry must set on_bundle_deploy, on_file_change, or on_value_change", }, { name: "both keys on one entry", triggers: []resources.JobRunTrigger{ {OnBundleDeploy: &trueVal, OnFileChange: &fileChange}, }, - summary: "lifecycle.triggers entry must set only one of on_bundle_deploy or on_file_change", + summary: "lifecycle.triggers entry must set only one of on_bundle_deploy, on_file_change, or on_value_change", }, { name: "on_bundle_deploy false", @@ -84,7 +89,7 @@ func TestValidateJobRunTriggers(t *testing.T) { {OnBundleDeploy: &trueVal}, }, preventDestroy: true, - summary: "lifecycle.triggers.on_bundle_deploy or on_file_change is incompatible with lifecycle.prevent_destroy", + summary: "lifecycle.triggers.on_bundle_deploy, on_file_change, or on_value_change is incompatible with lifecycle.prevent_destroy", }, { name: "on_file_change with prevent_destroy", @@ -92,12 +97,47 @@ func TestValidateJobRunTriggers(t *testing.T) { {OnFileChange: &fileChange}, }, preventDestroy: true, - summary: "lifecycle.triggers.on_bundle_deploy or on_file_change is incompatible with lifecycle.prevent_destroy", + summary: "lifecycle.triggers.on_bundle_deploy, on_file_change, or on_value_change is incompatible with lifecycle.prevent_destroy", }, { name: "prevent_destroy alone", preventDestroy: true, }, + { + name: "on_value_change set", + triggers: []resources.JobRunTrigger{ + {OnValueChange: &valueChange}, + }, + }, + { + name: "on_value_change empty", + triggers: []resources.JobRunTrigger{ + {OnValueChange: &emptyValue}, + }, + summary: "lifecycle.triggers.on_value_change must be non-empty when set", + }, + { + name: "on_value_change whitespace", + triggers: []resources.JobRunTrigger{ + {OnValueChange: &whitespaceValue}, + }, + summary: "lifecycle.triggers.on_value_change must be non-empty when set", + }, + { + name: "on_value_change with prevent_destroy", + triggers: []resources.JobRunTrigger{ + {OnValueChange: &valueChange}, + }, + preventDestroy: true, + summary: "lifecycle.triggers.on_bundle_deploy, on_file_change, or on_value_change is incompatible with lifecycle.prevent_destroy", + }, + { + name: "on_value_change and on_file_change on one entry", + triggers: []resources.JobRunTrigger{ + {OnFileChange: &fileChange, OnValueChange: &valueChange}, + }, + summary: "lifecycle.triggers entry must set only one of on_bundle_deploy, on_file_change, or on_value_change", + }, } for _, tt := range tests { diff --git a/bundle/config/resources/job_run.go b/bundle/config/resources/job_run.go index f4b16963453..4200a65a2ef 100644 --- a/bundle/config/resources/job_run.go +++ b/bundle/config/resources/job_run.go @@ -58,6 +58,18 @@ func (r *JobRun) HasOnFileChange() bool { return false } +func (r *JobRun) HasOnValueChange() bool { + if r.Lifecycle == nil { + return false + } + for _, t := range r.Lifecycle.Triggers { + if t.OnValueChange != nil { + return true + } + } + return false +} + func (r *JobRun) UnmarshalJSON(b []byte) error { return marshal.Unmarshal(b, r) } diff --git a/bundle/config/resources/lifecycle.go b/bundle/config/resources/lifecycle.go index a4e881d298c..9bed0aaa641 100644 --- a/bundle/config/resources/lifecycle.go +++ b/bundle/config/resources/lifecycle.go @@ -37,5 +37,20 @@ type JobRunLifecycle struct { // JobRunTrigger is one lifecycle.triggers entry. type JobRunTrigger struct { OnBundleDeploy *bool `json:"on_bundle_deploy,omitempty"` - OnFileChange *string `json:"on_file_change,omitempty"` // path or glob under sync root + OnFileChange *string `json:"on_file_change,omitempty"` // path or glob under sync root + OnValueChange *string `json:"on_value_change,omitempty"` // interpolated expr; re-fire when the resolved value changes +} + +func (t JobRunTrigger) ArmedCount() int { + n := 0 + if t.OnBundleDeploy != nil { + n++ + } + if t.OnFileChange != nil { + n++ + } + if t.OnValueChange != nil { + n++ + } + return n } diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index 5b8829e3f59..11c34d5cdee 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -1007,6 +1007,7 @@ func (b *DeploymentBundle) makePlan(ctx context.Context, configRoot *config.Root } maps.Copy(refs, inputStructVar.Refs) + dresources.DropJobRunValueChangeConfigRefs(refs) var dependsOn []deployplan.DependsOnEntry for _, reference := range refs { diff --git a/bundle/direct/dresources/job_run.go b/bundle/direct/dresources/job_run.go index 6ac69e7a3a2..734742feb18 100644 --- a/bundle/direct/dresources/job_run.go +++ b/bundle/direct/dresources/job_run.go @@ -13,8 +13,11 @@ import ( "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/dyn/dynvar" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" "github.com/databricks/cli/libs/workspaceurls" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/marshal" @@ -31,6 +34,7 @@ const jobRunTimeout = 24 * time.Hour var jobRunTriggerLocalPaths = []string{ "lifecycle.triggers.on_bundle_deploy", "lifecycle.triggers.on_file_change", + "lifecycle.triggers.on_value_change", } func isJobRunTriggerPath(path string) bool { @@ -52,6 +56,8 @@ type JobRunTriggersState struct { OnBundleDeploy string `json:"on_bundle_deploy,omitempty"` // Content hashes from ResolveJobRunFileTriggers; any change recreates. OnFileChange map[string]string `json:"on_file_change,omitempty"` + // Resolved expression per watched value; any change recreates. + OnValueChange map[string]string `json:"on_value_change,omitempty"` } // JobRunLifecycleState is the local-only trigger fingerprint. Nested by value, @@ -67,6 +73,7 @@ func emptyJobRunLifecycleState() JobRunLifecycleState { Triggers: JobRunTriggersState{ OnBundleDeploy: "", OnFileChange: nil, + OnValueChange: nil, }, } } @@ -139,9 +146,59 @@ func (*ResourceJobRun) PrepareState(input *resources.JobRun) *JobRunState { if len(input.ResolvedFileTriggers) > 0 { state.Lifecycle.Triggers.OnFileChange = input.ResolvedFileTriggers } + if values := jobRunValueChangeState(input); len(values) > 0 { + state.Lifecycle.Triggers.OnValueChange = values + } return state } +func (*ResourceJobRun) PrepareInputConfig(input *resources.JobRun, _ string) (*structvar.StructVar, error) { + refs := map[string]string{} + for expr := range jobRunValueChangeState(input) { + if _, ok := dynvar.NewRef(dyn.V(expr)); !ok { + continue + } + path := structpath.NewStringKey(structpath.MustParsePath("lifecycle.triggers.on_value_change"), expr) + refs[path.String()] = expr + } + if len(refs) == 0 { + refs = nil + } + return &structvar.StructVar{Value: input, Refs: refs}, nil +} + +func jobRunValueChangeState(input *resources.JobRun) map[string]string { + if input.Lifecycle == nil { + return nil + } + out := make(map[string]string) + for _, t := range input.Lifecycle.Triggers { + if t.OnValueChange == nil { + continue + } + expr := strings.TrimSpace(*t.OnValueChange) + if expr == "" { + continue + } + out[expr] = expr + } + if len(out) == 0 { + return nil + } + return out +} + +// DropJobRunValueChangeConfigRefs drops lifecycle.triggers[N].on_value_change. +// ExtractReferences treats [0] on the triggers struct as a no-op, so that path +// is the wrapper, which cannot hold a resolved id. +func DropJobRunValueChangeConfigRefs(refs map[string]string) { + for k := range refs { + if strings.Contains(k, ".triggers[") && strings.HasSuffix(k, "].on_value_change") { + delete(refs, k) + } + } +} + // makeJobRunRemote maps the GetRun response into the RunNow-shaped remote: GET // nests the params under overriding_parameters and returns job_parameters as a // list, so both are flattened back into RunNow. @@ -425,6 +482,7 @@ func (*ResourceJobRun) OverrideChangeDesc(_ context.Context, path *structpath.Pa if isJobRunTriggerPath(pathString) { removed := pathString == "lifecycle.triggers.on_bundle_deploy" && (change.New == nil || change.New == "") removed = removed || pathString == "lifecycle.triggers.on_file_change" && change.New == nil + removed = removed || pathString == "lifecycle.triggers.on_value_change" && change.New == nil if removed { change.Action = deployplan.Update change.Reason = "trigger removed" diff --git a/bundle/direct/dresources/job_run_test.go b/bundle/direct/dresources/job_run_test.go index 804968f7707..da325b143b9 100644 --- a/bundle/direct/dresources/job_run_test.go +++ b/bundle/direct/dresources/job_run_test.go @@ -10,6 +10,7 @@ import ( "testing" "time" + "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/cli/libs/testserver" "github.com/databricks/databricks-sdk-go" @@ -253,6 +254,47 @@ func TestJobRunStateUnmarshalWithoutLifecycle(t *testing.T) { assert.Nil(t, state.Lifecycle.Triggers.OnFileChange) } +// The ref path is what the deploy graph keys the dependency on, so it has to be +// the state path of the watched expression, not the config path it came from. +func TestJobRunPrepareInputConfigOnValueChange(t *testing.T) { + expr := "${resources.jobs.foo.id}" + literal := "v1" + input := &resources.JobRun{ + Lifecycle: &resources.JobRunLifecycle{ + Triggers: []resources.JobRunTrigger{ + {OnValueChange: &expr}, + {OnValueChange: &literal}, + }, + }, + } + + sv, err := (&ResourceJobRun{}).PrepareInputConfig(input, "resources.job_runs.my_run") + + require.NoError(t, err) + assert.Same(t, input, sv.Value) + // Only the interpolated expression is a ref; the literal has nothing to resolve. + path := structpath.NewStringKey(structpath.MustParsePath("lifecycle.triggers.on_value_change"), expr) + assert.Equal(t, map[string]string{path.String(): expr}, sv.Refs) +} + +func TestDropJobRunValueChangeConfigRefs(t *testing.T) { + expr := "${resources.jobs.other.id}" + statePath := structpath.NewStringKey(structpath.MustParsePath("lifecycle.triggers.on_value_change"), expr).String() + refs := map[string]string{ + "lifecycle.triggers[0].on_value_change": expr, + "job_id": "${resources.jobs.my_job.id}", + statePath: expr, + } + + DropJobRunValueChangeConfigRefs(refs) + + // The config path is dropped; the state path that carries the resolved id stays. + assert.Equal(t, map[string]string{ + "job_id": "${resources.jobs.my_job.id}", + statePath: expr, + }, refs) +} + // The planner diffs RemapState(remote) against PrepareState(config), so a run // that did not end in SUCCESS has to surface as a difference on result_state. func TestJobRunRemapStateCarriesTheOutcome(t *testing.T) { diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index 1fd6b452415..111edc74670 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -993,6 +993,9 @@ resources: "on_file_change": "description": |- Path or glob under the sync root. Re-fire the run when a matched file's content hash changes, or when the set of matches appears or disappears. Incompatible with lifecycle.prevent_destroy. + "on_value_change": + "description": |- + Interpolated expression. Re-fire the run when the resolved value changes. Resource references in the expression become deployment dependencies. Incompatible with lifecycle.prevent_destroy. "python_named_params": "description": |- PLACEHOLDER diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 39d31e92167..4172c6a20e1 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -1304,6 +1304,10 @@ "on_file_change": { "description": "Path or glob under the sync root. Re-fire the run when a matched file's content hash changes, or when the set of matches appears or disappears. Incompatible with lifecycle.prevent_destroy.", "$ref": "#/$defs/string" + }, + "on_value_change": { + "description": "Interpolated expression. Re-fire the run when the resolved value changes. Resource references in the expression become deployment dependencies. Incompatible with lifecycle.prevent_destroy.", + "$ref": "#/$defs/string" } }, "additionalProperties": false From e4310883c723b74f3fa65c688ce906f5ae3cebcc Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Fri, 21 Aug 2026 16:13:05 +0000 Subject: [PATCH 22/75] Regenerate job_runs reference schema fields Include the on_value_change input and state paths required by generated-file validation. --- acceptance/bundle/refschema/out.fields.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index 878453cb964..36808c90cda 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -854,9 +854,12 @@ resources.job_runs.*.lifecycle.triggers dresources.JobRunTriggersState STATE resources.job_runs.*.lifecycle.triggers.on_bundle_deploy string STATE resources.job_runs.*.lifecycle.triggers.on_file_change map[string]string STATE resources.job_runs.*.lifecycle.triggers.on_file_change.* string STATE +resources.job_runs.*.lifecycle.triggers.on_value_change map[string]string STATE +resources.job_runs.*.lifecycle.triggers.on_value_change.* string STATE resources.job_runs.*.lifecycle.triggers[*] resources.JobRunTrigger INPUT resources.job_runs.*.lifecycle.triggers[*].on_bundle_deploy *bool INPUT resources.job_runs.*.lifecycle.triggers[*].on_file_change *string INPUT +resources.job_runs.*.lifecycle.triggers[*].on_value_change *string INPUT resources.job_runs.*.modified_status string INPUT resources.job_runs.*.notebook_params map[string]string ALL resources.job_runs.*.notebook_params.* string ALL From d2ca9357f7f2ca859efce08051e39f2eb24aa5c1 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Tue, 18 Aug 2026 12:25:26 +0000 Subject: [PATCH 23/75] job_runs: add lifecycle.triggers.on_file_change Re-fire a run when matched file contents change, using stable hash fingerprints so mtime-only updates (touch) do not recreate. --- .../bundles/job-runs-on-file-change.md | 1 + .../job_runs/on_file_change/databricks.yml | 18 ++ .../job_runs/on_file_change/out.test.toml | 2 + .../job_runs/on_file_change/output.txt | 90 +++++++++ .../resources/job_runs/on_file_change/script | 34 ++++ .../job_runs/on_file_change/seed.txt | 1 + .../mutator/resolve_job_run_file_triggers.go | 177 ++++++++++++++++++ ...lve_job_run_file_triggers_internal_test.go | 73 ++++++++ .../resolve_job_run_file_triggers_test.go | 143 ++++++++++++++ .../mutator/validate_job_run_triggers.go | 26 ++- .../mutator/validate_job_run_triggers_test.go | 50 ++++- bundle/config/resources/job_run.go | 29 +++ bundle/config/resources/lifecycle.go | 9 +- bundle/direct/dresources/job_run.go | 28 ++- bundle/direct/dresources/job_run_test.go | 76 ++++++++ bundle/direct/dresources/resources.yml | 6 + bundle/internal/schema/annotations.yml | 3 + bundle/phases/initialize.go | 6 +- bundle/phases/plan.go | 4 +- bundle/schema/jsonschema.json | 4 + 20 files changed, 765 insertions(+), 15 deletions(-) create mode 100644 .nextchanges/bundles/job-runs-on-file-change.md create mode 100644 acceptance/bundle/resources/job_runs/on_file_change/databricks.yml create mode 100644 acceptance/bundle/resources/job_runs/on_file_change/out.test.toml create mode 100644 acceptance/bundle/resources/job_runs/on_file_change/output.txt create mode 100644 acceptance/bundle/resources/job_runs/on_file_change/script create mode 100644 acceptance/bundle/resources/job_runs/on_file_change/seed.txt create mode 100644 bundle/config/mutator/resolve_job_run_file_triggers.go create mode 100644 bundle/config/mutator/resolve_job_run_file_triggers_internal_test.go create mode 100644 bundle/config/mutator/resolve_job_run_file_triggers_test.go diff --git a/.nextchanges/bundles/job-runs-on-file-change.md b/.nextchanges/bundles/job-runs-on-file-change.md new file mode 100644 index 00000000000..00682403a0d --- /dev/null +++ b/.nextchanges/bundles/job-runs-on-file-change.md @@ -0,0 +1 @@ +direct: `resources.job_runs` can set `lifecycle.triggers.on_file_change` to a path or glob to re-fire the run when matched file contents change. Removing the trigger does not recreate the existing run. diff --git a/acceptance/bundle/resources/job_runs/on_file_change/databricks.yml b/acceptance/bundle/resources/job_runs/on_file_change/databricks.yml new file mode 100644 index 00000000000..0a6df75047a --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change/databricks.yml @@ -0,0 +1,18 @@ +bundle: + name: job-runs-on-file-change + +resources: + jobs: + my_job: + name: my-job + tasks: + - task_key: main + notebook_task: + notebook_path: /Workspace/test + + job_runs: + my_run: + job_id: ${resources.jobs.my_job.id} + lifecycle: + triggers: + - on_file_change: seed.txt diff --git a/acceptance/bundle/resources/job_runs/on_file_change/out.test.toml b/acceptance/bundle/resources/job_runs/on_file_change/out.test.toml new file mode 100644 index 00000000000..0938e678987 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/job_runs/on_file_change/output.txt b/acceptance/bundle/resources/job_runs/on_file_change/output.txt new file mode 100644 index 00000000000..bcf5962f8fa --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change/output.txt @@ -0,0 +1,90 @@ + +=== first deploy triggers a run +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change/default/files... +job run [MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?o=[NUMID] +job run [MY_RUN_ID]: SUCCESS +Created job_runs.my_run +Created jobs.my_job +Files: 5 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +>>> read_id.py my_job +[MY_JOB_ID] + +>>> print_requests.py //jobs/run-now +{ + "method": "POST", + "path": "/api/2.2/jobs/run-now", + "body": { + "idempotency_token": "[UUID]", + "job_id": [MY_JOB_ID] + } +} + +=== redeploy with unchanged file plans nothing +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change/default/files... +Files: 2 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 2 unchanged + +>>> print_requests.py //jobs/run-now + +=== editing the file re-fires +>>> [CLI] bundle plan +recreate job_runs.my_run + +Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change/default/files... +job run [MY_RUN_ID_2]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_2]?o=[NUMID] +job run [MY_RUN_ID_2]: SUCCESS +Recreated job_runs.my_run +Files: 3 uploaded, 0 deleted +Resources: 1 created, 0 changed, 1 deleted, 1 unchanged + +>>> print_requests.py --keep //jobs/runs/delete +{ + "method": "POST", + "path": "/api/2.2/jobs/runs/delete", + "body": { + "run_id": [MY_RUN_ID] + } +} + +>>> print_requests.py //jobs/run-now +{ + "method": "POST", + "path": "/api/2.2/jobs/run-now", + "body": { + "idempotency_token": "[UUID]", + "job_id": [MY_JOB_ID] + } +} + +=== removing on_file_change rewrites state without a run +>>> [CLI] bundle plan +update job_runs.my_run + +Plan: 0 to add, 1 to change, 0 to delete, 1 unchanged + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change/default/files... +Updated job_runs.my_run +Files: 3 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 1 unchanged + +>>> print_requests.py //jobs/run-now + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.job_runs.my_run + delete resources.jobs.my_job + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change/default + +Destroy: 2 deleted diff --git a/acceptance/bundle/resources/job_runs/on_file_change/script b/acceptance/bundle/resources/job_runs/on_file_change/script new file mode 100644 index 00000000000..ff162e11f0b --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change/script @@ -0,0 +1,34 @@ +cleanup() { + trace $CLI bundle destroy --auto-approve + rm -f out.requests.txt +} +trap cleanup EXIT + +title "first deploy triggers a run" +trace $CLI bundle deploy +trace read_id.py my_job +# Name the first run so the recreated one becomes [MY_RUN_ID_2]. +read_id.py my_run > /dev/null +trace print_requests.py //jobs/run-now + +title "redeploy with unchanged file plans nothing" +trace $CLI bundle plan +trace $CLI bundle deploy +trace print_requests.py //jobs/run-now + +title "editing the file re-fires" +update_file.py seed.txt "v1" "v2" +trace $CLI bundle plan +trace $CLI bundle deploy +read_id.py my_run > /dev/null +trace print_requests.py --keep //jobs/runs/delete +trace print_requests.py //jobs/run-now + +title "removing on_file_change rewrites state without a run" +update_file.py databricks.yml " lifecycle: + triggers: + - on_file_change: seed.txt +" "" +trace $CLI bundle plan +trace $CLI bundle deploy +trace print_requests.py //jobs/run-now diff --git a/acceptance/bundle/resources/job_runs/on_file_change/seed.txt b/acceptance/bundle/resources/job_runs/on_file_change/seed.txt new file mode 100644 index 00000000000..626799f0f85 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change/seed.txt @@ -0,0 +1 @@ +v1 diff --git a/bundle/config/mutator/resolve_job_run_file_triggers.go b/bundle/config/mutator/resolve_job_run_file_triggers.go new file mode 100644 index 00000000000..a82ed654be3 --- /dev/null +++ b/bundle/config/mutator/resolve_job_run_file_triggers.go @@ -0,0 +1,177 @@ +package mutator + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "os" + "path/filepath" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/libs/diag" +) + +// missingFileSentinelSize marks a pattern with no matching file so the next +// plan can distinguish "still missing" from "file appeared". +const missingFileSentinelSize = int64(-1) + +type resolveJobRunFileTriggers struct{} + +// ResolveJobRunFileTriggers expands on_file_change globs and stores per-file +// fingerprints on each job_run for PrepareState to copy into local state. +func ResolveJobRunFileTriggers() bundle.Mutator { + return &resolveJobRunFileTriggers{} +} + +func (*resolveJobRunFileTriggers) Name() string { + return "ResolveJobRunFileTriggers" +} + +func (*resolveJobRunFileTriggers) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { + var diags diag.Diagnostics + for name, jr := range b.Config.Resources.JobRuns { + if jr == nil { + continue + } + patterns := jr.OnFileChangePatterns() + if len(patterns) == 0 { + continue + } + fps, d := resolveFileTriggers(b, name, patterns, previousFileTriggers(b, name)) + diags = diags.Extend(d) + jr.ResolvedFileTriggers = fps + } + return diags +} + +// previousFileTriggers reads on_file_change fingerprints from deployment state +// when it is open (plan/deploy after StatePull). Used so unchanged content keeps +// a stable fingerprint across mtime-only updates (e.g. touch). +func previousFileTriggers(b *bundle.Bundle, name string) map[string]resources.JobRunFileFingerprint { + if b.DeploymentBundle.StateDB.Path == "" { + return nil + } + entry, ok := b.DeploymentBundle.StateDB.GetResourceEntry("resources.job_runs." + name) + if !ok || len(entry.State) == 0 { + return nil + } + var state struct { + Lifecycle *struct { + Triggers *struct { + OnFileChange map[string]resources.JobRunFileFingerprint `json:"on_file_change"` + } `json:"triggers"` + } `json:"lifecycle"` + } + if err := json.Unmarshal(entry.State, &state); err != nil { + return nil + } + if state.Lifecycle == nil || state.Lifecycle.Triggers == nil { + return nil + } + return state.Lifecycle.Triggers.OnFileChange +} + +func resolveFileTriggers(b *bundle.Bundle, name string, patterns []string, prev map[string]resources.JobRunFileFingerprint) (map[string]resources.JobRunFileFingerprint, diag.Diagnostics) { + var diags diag.Diagnostics + out := make(map[string]resources.JobRunFileFingerprint) + for _, pattern := range patterns { + path := fmt.Sprintf("resources.job_runs.%s.lifecycle.triggers", name) + matches, err := filepath.Glob(filepath.Join(b.SyncRootPath, filepath.FromSlash(pattern))) + if err != nil { + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: invalid pattern %q: %s", pattern, err), + Locations: b.Config.GetLocations(path), + }) + continue + } + if len(matches) == 0 { + // Distinct state when the path/glob matches nothing (design doc). + out[filepath.ToSlash(pattern)] = resources.JobRunFileFingerprint{ + Size: missingFileSentinelSize, + } + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Warning, + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: no files match %q", pattern), + Locations: b.Config.GetLocations(path), + }) + continue + } + for _, match := range matches { + info, err := os.Stat(match) + if err != nil { + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: stat %q: %s", match, err), + Locations: b.Config.GetLocations(path), + }) + continue + } + if !info.Mode().IsRegular() { + continue + } + rel, err := filepath.Rel(b.SyncRootPath, match) + if err != nil { + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: relative path for %q: %s", match, err), + Locations: b.Config.GetLocations(path), + }) + continue + } + key := filepath.ToSlash(rel) + fp, err := fingerprintFile(match, info, prev[key]) + if err != nil { + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: hash %q: %s", match, err), + Locations: b.Config.GetLocations(path), + }) + continue + } + out[key] = fp + } + } + return out, diags +} + +// fingerprintFile builds a content fingerprint. If prev has the same size and +// mtime, it is reused without reading the file. If content hash matches prev, +// prev is reused so mtime-only updates (touch) do not change planned state. +func fingerprintFile(path string, info os.FileInfo, prev resources.JobRunFileFingerprint) (resources.JobRunFileFingerprint, error) { + size := info.Size() + mtime := info.ModTime().UnixNano() + if prev.Hash != "" && prev.Size == size && prev.MtimeNs == mtime { + return prev, nil + } + hash, err := hashFile(path) + if err != nil { + return resources.JobRunFileFingerprint{}, err + } + if prev.Hash != "" && prev.Hash == hash { + return prev, nil + } + return resources.JobRunFileFingerprint{ + Hash: hash, + Size: size, + MtimeNs: mtime, + }, nil +} + +func hashFile(path string) (string, error) { + f, err := os.Open(path) + if err != nil { + return "", err + } + defer f.Close() + + h := sha256.New() + if _, err := io.Copy(h, f); err != nil { + return "", err + } + return hex.EncodeToString(h.Sum(nil)), nil +} diff --git a/bundle/config/mutator/resolve_job_run_file_triggers_internal_test.go b/bundle/config/mutator/resolve_job_run_file_triggers_internal_test.go new file mode 100644 index 00000000000..e82bd22945b --- /dev/null +++ b/bundle/config/mutator/resolve_job_run_file_triggers_internal_test.go @@ -0,0 +1,73 @@ +package mutator + +import ( + "os" + "path/filepath" + "testing" + "time" + + "github.com/databricks/cli/bundle/config/resources" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestFingerprintFileContentStableAcrossTouch(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "seed.txt") + require.NoError(t, os.WriteFile(path, []byte("v1"), 0o644)) + + info, err := os.Stat(path) + require.NoError(t, err) + first, err := fingerprintFile(path, info, resources.JobRunFileFingerprint{}) + require.NoError(t, err) + require.NotEmpty(t, first.Hash) + + // Advance mtime without changing contents (touch). + require.NoError(t, os.Chtimes(path, time.Now().Add(time.Minute), time.Now().Add(time.Minute))) + info, err = os.Stat(path) + require.NoError(t, err) + assert.NotEqual(t, first.MtimeNs, info.ModTime().UnixNano()) + + second, err := fingerprintFile(path, info, first) + require.NoError(t, err) + assert.Equal(t, first, second, "unchanged content must reuse the previous fingerprint") +} + +func TestFingerprintFileFastPathSkipsWhenMtimeAndSizeMatch(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "seed.txt") + require.NoError(t, os.WriteFile(path, []byte("v1"), 0o644)) + + info, err := os.Stat(path) + require.NoError(t, err) + prev := resources.JobRunFileFingerprint{ + Hash: "not-the-real-hash", + Size: info.Size(), + MtimeNs: info.ModTime().UnixNano(), + } + + got, err := fingerprintFile(path, info, prev) + require.NoError(t, err) + assert.Equal(t, prev, got, "matching size+mtime must reuse prev without re-hashing") +} + +func TestFingerprintFileContentChange(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "seed.txt") + require.NoError(t, os.WriteFile(path, []byte("v1"), 0o644)) + + info, err := os.Stat(path) + require.NoError(t, err) + first, err := fingerprintFile(path, info, resources.JobRunFileFingerprint{}) + require.NoError(t, err) + + // Different size so the mtime+size fast path cannot reuse prev. + require.NoError(t, os.WriteFile(path, []byte("v2-changed"), 0o644)) + info, err = os.Stat(path) + require.NoError(t, err) + second, err := fingerprintFile(path, info, first) + require.NoError(t, err) + assert.NotEqual(t, first.Hash, second.Hash) + assert.Equal(t, info.Size(), second.Size) + assert.Equal(t, info.ModTime().UnixNano(), second.MtimeNs) +} diff --git a/bundle/config/mutator/resolve_job_run_file_triggers_test.go b/bundle/config/mutator/resolve_job_run_file_triggers_test.go new file mode 100644 index 00000000000..9a9cf77bf53 --- /dev/null +++ b/bundle/config/mutator/resolve_job_run_file_triggers_test.go @@ -0,0 +1,143 @@ +package mutator_test + +import ( + "crypto/sha256" + "encoding/hex" + "os" + "path/filepath" + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/mutator" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/libs/diag" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestResolveJobRunFileTriggers(t *testing.T) { + t.Run("matches files and fills fingerprints", func(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "a.txt"), []byte("hello"), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "b.txt"), []byte("world"), 0o644)) + + pattern := "*.txt" + b := bundleWithFileTrigger(dir, pattern) + + diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) + require.False(t, diags.HasError()) + + fps := b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers + require.Len(t, fps, 2) + + assertFingerprint(t, fps["a.txt"], "hello") + assertFingerprint(t, fps["b.txt"], "world") + }) + + t.Run("no matches warns and stores sentinel", func(t *testing.T) { + dir := t.TempDir() + pattern := "missing.txt" + b := bundleWithFileTrigger(dir, pattern) + + diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) + require.False(t, diags.HasError()) + require.Len(t, diags, 1) + assert.Equal(t, diag.Warning, diags[0].Severity) + assert.Contains(t, diags[0].Summary, `no files match "missing.txt"`) + + fps := b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers + require.Len(t, fps, 1) + fp := fps["missing.txt"] + assert.Empty(t, fp.Hash) + assert.Equal(t, int64(-1), fp.Size) + assert.Zero(t, fp.MtimeNs) + }) + + t.Run("no file triggers is a no-op", func(t *testing.T) { + dir := t.TempDir() + on := true + b := &bundle.Bundle{ + SyncRootPath: dir, + Config: config.Root{ + Resources: config.Resources{ + JobRuns: map[string]*resources.JobRun{ + "my_run": { + Lifecycle: &resources.JobRunLifecycle{ + Triggers: []resources.JobRunTrigger{ + {OnBundleDeploy: &on}, + }, + }, + }, + }, + }, + }, + } + + diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) + assert.Empty(t, diags) + assert.Nil(t, b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers) + }) + + t.Run("multiple patterns merge into one map", func(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "a.txt"), []byte("aaa"), 0o644)) + require.NoError(t, os.MkdirAll(filepath.Join(dir, "subdir"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "subdir", "x.py"), []byte("bbb"), 0o644)) + + patA := "a.txt" + patB := "subdir/*.py" + b := &bundle.Bundle{ + SyncRootPath: dir, + Config: config.Root{ + Resources: config.Resources{ + JobRuns: map[string]*resources.JobRun{ + "my_run": { + Lifecycle: &resources.JobRunLifecycle{ + Triggers: []resources.JobRunTrigger{ + {OnFileChange: &patA}, + {OnFileChange: &patB}, + }, + }, + }, + }, + }, + }, + } + + diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) + require.False(t, diags.HasError()) + + fps := b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers + require.Len(t, fps, 2) + assertFingerprint(t, fps["a.txt"], "aaa") + assertFingerprint(t, fps["subdir/x.py"], "bbb") + }) +} + +func bundleWithFileTrigger(syncRoot, pattern string) *bundle.Bundle { + return &bundle.Bundle{ + SyncRootPath: syncRoot, + Config: config.Root{ + Resources: config.Resources{ + JobRuns: map[string]*resources.JobRun{ + "my_run": { + Lifecycle: &resources.JobRunLifecycle{ + Triggers: []resources.JobRunTrigger{ + {OnFileChange: &pattern}, + }, + }, + }, + }, + }, + }, + } +} + +func assertFingerprint(t *testing.T, fp resources.JobRunFileFingerprint, content string) { + t.Helper() + sum := sha256.Sum256([]byte(content)) + assert.Equal(t, hex.EncodeToString(sum[:]), fp.Hash) + assert.Equal(t, int64(len(content)), fp.Size) + assert.NotZero(t, fp.MtimeNs) +} diff --git a/bundle/config/mutator/validate_job_run_triggers.go b/bundle/config/mutator/validate_job_run_triggers.go index a7c5ada925e..c185d3b5781 100644 --- a/bundle/config/mutator/validate_job_run_triggers.go +++ b/bundle/config/mutator/validate_job_run_triggers.go @@ -3,6 +3,7 @@ package mutator import ( "context" "fmt" + "strings" "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" @@ -26,30 +27,45 @@ func (*validateJobRunTriggers) Apply(_ context.Context, b *bundle.Bundle) diag.D continue } // Recreate-every-deploy cannot coexist with prevent_destroy. - if jr.HasOnBundleDeploy() && jr.Lifecycle.PreventDestroy { + if (jr.HasOnBundleDeploy() || jr.HasOnFileChange()) && jr.Lifecycle.PreventDestroy { diags = diags.Append(diag.Diagnostic{ Severity: diag.Error, - Summary: "lifecycle.triggers.on_bundle_deploy is incompatible with lifecycle.prevent_destroy", + Summary: "lifecycle.triggers.on_bundle_deploy or on_file_change is incompatible with lifecycle.prevent_destroy", Locations: b.Config.GetLocations(fmt.Sprintf("resources.job_runs.%s.lifecycle", name)), }) } for i, t := range jr.Lifecycle.Triggers { path := fmt.Sprintf("resources.job_runs.%s.lifecycle.triggers[%d]", name, i) - if t.OnBundleDeploy == nil { + if t.OnBundleDeploy == nil && t.OnFileChange == nil { diags = diags.Append(diag.Diagnostic{ Severity: diag.Error, - Summary: "lifecycle.triggers entry must set on_bundle_deploy: true", + Summary: "lifecycle.triggers entry must set on_bundle_deploy or on_file_change", Locations: b.Config.GetLocations(path), }) continue } - if !*t.OnBundleDeploy { + if t.OnBundleDeploy != nil && t.OnFileChange != nil { + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: "lifecycle.triggers entry must set only one of on_bundle_deploy or on_file_change", + Locations: b.Config.GetLocations(path), + }) + continue + } + if t.OnBundleDeploy != nil && !*t.OnBundleDeploy { diags = diags.Append(diag.Diagnostic{ Severity: diag.Error, Summary: "lifecycle.triggers.on_bundle_deploy must be true when set", Locations: b.Config.GetLocations(path + ".on_bundle_deploy"), }) } + if t.OnFileChange != nil && strings.TrimSpace(*t.OnFileChange) == "" { + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: "lifecycle.triggers.on_file_change must be non-empty when set", + Locations: b.Config.GetLocations(path + ".on_file_change"), + }) + } } } return diags diff --git a/bundle/config/mutator/validate_job_run_triggers_test.go b/bundle/config/mutator/validate_job_run_triggers_test.go index 9c225457471..7e8772d8d8b 100644 --- a/bundle/config/mutator/validate_job_run_triggers_test.go +++ b/bundle/config/mutator/validate_job_run_triggers_test.go @@ -14,6 +14,10 @@ func TestValidateJobRunTriggers(t *testing.T) { trueVal := true falseVal := false + fileChange := "seed.txt" + emptyFile := "" + whitespaceFile := " \t" + tests := []struct { name string triggers []resources.JobRunTrigger @@ -26,12 +30,32 @@ func TestValidateJobRunTriggers(t *testing.T) { {OnBundleDeploy: &trueVal}, }, }, + { + name: "on_file_change set", + triggers: []resources.JobRunTrigger{ + {OnFileChange: &fileChange}, + }, + }, + { + name: "both triggers as separate entries", + triggers: []resources.JobRunTrigger{ + {OnFileChange: &fileChange}, + {OnBundleDeploy: &trueVal}, + }, + }, { name: "empty entry", triggers: []resources.JobRunTrigger{ {}, }, - summary: "lifecycle.triggers entry must set on_bundle_deploy: true", + summary: "lifecycle.triggers entry must set on_bundle_deploy or on_file_change", + }, + { + name: "both keys on one entry", + triggers: []resources.JobRunTrigger{ + {OnBundleDeploy: &trueVal, OnFileChange: &fileChange}, + }, + summary: "lifecycle.triggers entry must set only one of on_bundle_deploy or on_file_change", }, { name: "on_bundle_deploy false", @@ -40,13 +64,35 @@ func TestValidateJobRunTriggers(t *testing.T) { }, summary: "lifecycle.triggers.on_bundle_deploy must be true when set", }, + { + name: "on_file_change empty", + triggers: []resources.JobRunTrigger{ + {OnFileChange: &emptyFile}, + }, + summary: "lifecycle.triggers.on_file_change must be non-empty when set", + }, + { + name: "on_file_change whitespace", + triggers: []resources.JobRunTrigger{ + {OnFileChange: &whitespaceFile}, + }, + summary: "lifecycle.triggers.on_file_change must be non-empty when set", + }, { name: "on_bundle_deploy with prevent_destroy", triggers: []resources.JobRunTrigger{ {OnBundleDeploy: &trueVal}, }, preventDestroy: true, - summary: "lifecycle.triggers.on_bundle_deploy is incompatible with lifecycle.prevent_destroy", + summary: "lifecycle.triggers.on_bundle_deploy or on_file_change is incompatible with lifecycle.prevent_destroy", + }, + { + name: "on_file_change with prevent_destroy", + triggers: []resources.JobRunTrigger{ + {OnFileChange: &fileChange}, + }, + preventDestroy: true, + summary: "lifecycle.triggers.on_bundle_deploy or on_file_change is incompatible with lifecycle.prevent_destroy", }, { name: "prevent_destroy alone", diff --git a/bundle/config/resources/job_run.go b/bundle/config/resources/job_run.go index 56f3c39a356..cf66c71f728 100644 --- a/bundle/config/resources/job_run.go +++ b/bundle/config/resources/job_run.go @@ -27,6 +27,10 @@ type JobRun struct { // the run URL. Keeping it separate from RunNow.JobId (a ${resources.jobs.*.id} // reference) lets state loading preserve that reference and its plan dependency. ResolvedJobID int64 `json:"resolved_job_id,omitempty" bundle:"internal"` + + // ResolvedFileTriggers is the per-file fingerprint map for on_file_change, + // computed under SyncRoot before plan. bundle:"internal" keeps it out of schema. + ResolvedFileTriggers map[string]JobRunFileFingerprint `json:"resolved_file_triggers,omitempty" bundle:"internal"` } // HasOnBundleDeploy reports whether any trigger re-fires on every deploy. @@ -42,6 +46,31 @@ func (r *JobRun) HasOnBundleDeploy() bool { return false } +func (r *JobRun) HasOnFileChange() bool { + if r.Lifecycle == nil { + return false + } + for _, t := range r.Lifecycle.Triggers { + if t.OnFileChange != nil { + return true + } + } + return false +} + +func (r *JobRun) OnFileChangePatterns() []string { + if r.Lifecycle == nil { + return nil + } + var patterns []string + for _, t := range r.Lifecycle.Triggers { + if t.OnFileChange != nil { + patterns = append(patterns, *t.OnFileChange) + } + } + return patterns +} + func (r *JobRun) UnmarshalJSON(b []byte) error { return marshal.Unmarshal(b, r) } diff --git a/bundle/config/resources/lifecycle.go b/bundle/config/resources/lifecycle.go index 88d130239ee..59ccb682b81 100644 --- a/bundle/config/resources/lifecycle.go +++ b/bundle/config/resources/lifecycle.go @@ -36,5 +36,12 @@ type JobRunLifecycle struct { // JobRunTrigger is one lifecycle.triggers entry. type JobRunTrigger struct { - OnBundleDeploy *bool `json:"on_bundle_deploy,omitempty"` + OnBundleDeploy *bool `json:"on_bundle_deploy,omitempty"` + OnFileChange *string `json:"on_file_change,omitempty"` // path or glob under sync root +} + +type JobRunFileFingerprint struct { + Hash string `json:"hash"` + Size int64 `json:"size"` + MtimeNs int64 `json:"mtime_ns"` } diff --git a/bundle/direct/dresources/job_run.go b/bundle/direct/dresources/job_run.go index 9eb671f430c..4e57a4bcdf4 100644 --- a/bundle/direct/dresources/job_run.go +++ b/bundle/direct/dresources/job_run.go @@ -26,10 +26,20 @@ import ( // jobRunTimeout matches the timeout `bundle run` allows a run (bundle/run/job.go). const jobRunTimeout = 24 * time.Hour +// jobRunTriggerLocalPaths are local-only fingerprints: clearing one is skip, not recreate. +var jobRunTriggerLocalPaths = []string{ + "lifecycle", + "lifecycle.triggers", + "lifecycle.triggers.on_bundle_deploy", + "lifecycle.triggers.on_file_change", +} + // JobRunTriggersState is the persisted fingerprint of lifecycle.triggers. type JobRunTriggersState struct { // Fresh UUID each plan while armed so Old!=New forces recreate. OnBundleDeploy string `json:"on_bundle_deploy,omitempty"` + // Per-file fingerprints from ResolveJobRunFileTriggers; change to recreate. + OnFileChange map[string]resources.JobRunFileFingerprint `json:"on_file_change,omitempty"` } // JobRunLifecycleState holds local-only lifecycle fields persisted in state. @@ -97,12 +107,19 @@ func (*ResourceJobRun) PrepareState(input *resources.JobRun) *JobRunState { state := &JobRunState{ RunNow: input.RunNow, ResultState: jobs.RunResultStateSuccess, - Lifecycle: nil, } + var triggers *JobRunTriggersState if input.HasOnBundleDeploy() { - state.Lifecycle = &JobRunLifecycleState{ - Triggers: &JobRunTriggersState{OnBundleDeploy: uuid.NewString()}, + triggers = &JobRunTriggersState{OnBundleDeploy: uuid.NewString()} + } + if len(input.ResolvedFileTriggers) > 0 { + if triggers == nil { + triggers = &JobRunTriggersState{} } + triggers.OnFileChange = input.ResolvedFileTriggers + } + if triggers != nil { + state.Lifecycle = &JobRunLifecycleState{Triggers: triggers} } return state } @@ -371,8 +388,7 @@ func reportRunLine(ctx context.Context, runID int64, msg string) { // result_state either, so the lifecycle state is what tells the two apart. // Clearing a trigger skips its local-only fingerprint without re-firing the run. func (*ResourceJobRun) OverrideChangeDesc(_ context.Context, path *structpath.PathNode, change *ChangeDesc, remote *JobRunRemote) error { - switch path.String() { - case "lifecycle", "lifecycle.triggers", "lifecycle.triggers.on_bundle_deploy": + if slices.Contains(jobRunTriggerLocalPaths, path.String()) { // A cleared trigger sets New empty; structdiff may report it at lifecycle, // lifecycle.triggers, or the leaf. if change.New == nil || change.New == "" { @@ -380,6 +396,8 @@ func (*ResourceJobRun) OverrideChangeDesc(_ context.Context, path *structpath.Pa change.Reason = "trigger removed" } return nil + } + switch path.String() { case "result_state": // The planner passes no remote state when the run could not be read. if remote == nil || runIsTerminal(remote.State.LifeCycleState) { diff --git a/bundle/direct/dresources/job_run_test.go b/bundle/direct/dresources/job_run_test.go index cef8cadecce..f90517206fa 100644 --- a/bundle/direct/dresources/job_run_test.go +++ b/bundle/direct/dresources/job_run_test.go @@ -376,6 +376,41 @@ func TestJobRunPrepareStateOnBundleDeploy(t *testing.T) { }) } +func TestJobRunPrepareStateOnFileChange(t *testing.T) { + fps := map[string]resources.JobRunFileFingerprint{ + "a.txt": {Hash: "abc", Size: 3, MtimeNs: 1}, + } + + t.Run("unset", func(t *testing.T) { + state := (&ResourceJobRun{}).PrepareState(&resources.JobRun{}) + assert.Nil(t, state.Lifecycle) + }) + + t.Run("armed", func(t *testing.T) { + state := (&ResourceJobRun{}).PrepareState(&resources.JobRun{ + ResolvedFileTriggers: fps, + }) + require.NotNil(t, state.Lifecycle) + require.NotNil(t, state.Lifecycle.Triggers) + assert.Equal(t, fps, state.Lifecycle.Triggers.OnFileChange) + assert.Empty(t, state.Lifecycle.Triggers.OnBundleDeploy) + }) + + t.Run("both triggers", func(t *testing.T) { + on := true + state := (&ResourceJobRun{}).PrepareState(&resources.JobRun{ + Lifecycle: &resources.JobRunLifecycle{ + Triggers: []resources.JobRunTrigger{{OnBundleDeploy: &on}}, + }, + ResolvedFileTriggers: fps, + }) + require.NotNil(t, state.Lifecycle) + require.NotNil(t, state.Lifecycle.Triggers) + assert.NotEmpty(t, state.Lifecycle.Triggers.OnBundleDeploy) + assert.Equal(t, fps, state.Lifecycle.Triggers.OnFileChange) + }) +} + func TestJobRunOverrideChangeDescTriggerRemoved(t *testing.T) { r := &ResourceJobRun{} @@ -401,6 +436,19 @@ func TestJobRunOverrideChangeDescTriggerRemoved(t *testing.T) { assert.Equal(t, "trigger removed", change.Reason) }) + t.Run("clearing on_file_change leaf downgrades to update", func(t *testing.T) { + change := &ChangeDesc{ + Action: deployplan.Recreate, + Old: map[string]resources.JobRunFileFingerprint{ + "a.txt": {Hash: "abc", Size: 3, MtimeNs: 1}, + }, + New: nil, + } + require.NoError(t, r.OverrideChangeDesc(t.Context(), structpath.MustParsePath("lifecycle.triggers.on_file_change"), change, nil)) + assert.Equal(t, deployplan.Update, change.Action) + assert.Equal(t, "trigger removed", change.Reason) + }) + t.Run("fresh fingerprint still recreates", func(t *testing.T) { change := &ChangeDesc{ Action: deployplan.Recreate, @@ -410,6 +458,20 @@ func TestJobRunOverrideChangeDescTriggerRemoved(t *testing.T) { require.NoError(t, r.OverrideChangeDesc(t.Context(), structpath.MustParsePath("lifecycle.triggers.on_bundle_deploy"), change, nil)) assert.Equal(t, deployplan.Recreate, change.Action) }) + + t.Run("changed on_file_change fingerprint still recreates", func(t *testing.T) { + change := &ChangeDesc{ + Action: deployplan.Recreate, + Old: map[string]resources.JobRunFileFingerprint{ + "a.txt": {Hash: "old", Size: 1, MtimeNs: 1}, + }, + New: map[string]resources.JobRunFileFingerprint{ + "a.txt": {Hash: "new", Size: 1, MtimeNs: 2}, + }, + } + require.NoError(t, r.OverrideChangeDesc(t.Context(), structpath.MustParsePath("lifecycle.triggers.on_file_change"), change, nil)) + assert.Equal(t, deployplan.Recreate, change.Action) + }) } // The planner diffs RemapState(remote) against PrepareState(config), so a run @@ -449,6 +511,20 @@ func TestJobRunIgnoresEveryRequestField(t *testing.T) { assert.False(t, ignoresRemoteChanges(ignored, "result_state"), "result_state must stay comparable against the remote") } +func TestJobRunIgnoresMtimeOnlyFileTriggerDrift(t *testing.T) { + adapters, err := InitAll(nil) + require.NoError(t, err) + ignored := adapters["job_runs"].ResourceConfig().IgnoreLocalChanges + path := structpath.MustParsePath("lifecycle.triggers.on_file_change['seed.txt'].mtime_ns") + assert.True(t, slices.ContainsFunc(ignored, func(r FieldRule) bool { + return path.HasPatternPrefix(r.Field) + }), "mtime-only fingerprint drift must be ignored") + hashPath := structpath.MustParsePath("lifecycle.triggers.on_file_change['seed.txt'].hash") + assert.False(t, slices.ContainsFunc(ignored, func(r FieldRule) bool { + return hashPath.HasPatternPrefix(r.Field) + }), "hash changes must still recreate") +} + // ignoresRemoteChanges reports whether the rules suppress remote drift on field. func ignoresRemoteChanges(rules []FieldRule, field string) bool { path := structpath.MustParsePath(field) diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index b0a9ed6b365..0d49749c15a 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -182,6 +182,12 @@ resources: recreate_on_changes: - reason: immutable + # mtime is stored for the resolve fast path; content identity is the hash. + # Ignoring mtime-only local drift keeps touch from recreating the run. + ignore_local_changes: + - field: lifecycle.triggers.on_file_change[*].mtime_ns + reason: mtime_only_fingerprint + pipelines: recreate_on_changes: - field: storage diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index a47e8151681..bfecaef7c00 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -990,6 +990,9 @@ resources: "on_bundle_deploy": "description": |- If true, re-fire the run on every bundle deploy. Incompatible with lifecycle.prevent_destroy. + "on_file_change": + "description": |- + Path or glob under the sync root. Re-fire the run when any matched file's content changes. Incompatible with lifecycle.prevent_destroy. "python_named_params": "description": |- PLACEHOLDER diff --git a/bundle/phases/initialize.go b/bundle/phases/initialize.go index b15e1c30df6..9eba6f466e0 100644 --- a/bundle/phases/initialize.go +++ b/bundle/phases/initialize.go @@ -26,7 +26,8 @@ import ( func Initialize(ctx context.Context, b *bundle.Bundle) { log.Info(ctx, "Phase: initialize") - bundle.ApplySeqContext(ctx, b, + bundle.ApplySeqContext( + ctx, b, // Reads (dynamic): resource.*.* // Checks that none of resources.. is nil. Raises error otherwise. validate.AllResourcesHaveValues(), @@ -191,6 +192,9 @@ func Initialize(ctx context.Context, b *bundle.Bundle) { // Reject invalid job_runs.lifecycle.triggers (empty, false, prevent_destroy). mutator.ValidateJobRunTriggers(), + // Expand on_file_change globs and hash matched files into ResolvedFileTriggers. + mutator.ResolveJobRunFileTriggers(), + // Reads (dynamic): * (strings) (searches for ${resources.*} references) // Warns (TF engine) or errors (direct engine) when a cross-resource reference // points to a Terraform-only field with no DABs equivalent. diff --git a/bundle/phases/plan.go b/bundle/phases/plan.go index 3db0864c2c4..92ca524cc4f 100644 --- a/bundle/phases/plan.go +++ b/bundle/phases/plan.go @@ -21,7 +21,8 @@ import ( // PreDeployChecks is common set of mutators between "bundle plan" and "bundle deploy". // Note, it is not run in "bundle migrate" so it must not modify the config func PreDeployChecks(ctx context.Context, b *bundle.Bundle, isPlan bool, engine engine.EngineType) { - bundle.ApplySeqContext(ctx, b, + bundle.ApplySeqContext( + ctx, b, terraform.CheckDashboardsModifiedRemotely(isPlan, engine), resourcemutator.SecretScopeFixups(engine), deploy.StatePull(), @@ -30,6 +31,7 @@ func PreDeployChecks(ctx context.Context, b *bundle.Bundle, isPlan bool, engine mutator.ValidateLifecycleStarted(engine), mutator.ValidateCascadeOnDestroy(engine), mutator.ValidateJobRunTriggers(), + mutator.ResolveJobRunFileTriggers(), statemgmt.CheckRunningResource(engine), ) } diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index cc481ee8b0c..0a01fba5022 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -1311,6 +1311,10 @@ "on_bundle_deploy": { "description": "If true, re-fire the run on every bundle deploy. Incompatible with lifecycle.prevent_destroy.", "$ref": "#/$defs/bool" + }, + "on_file_change": { + "description": "Path or glob under the sync root. Re-fire the run when any matched file's content changes. Incompatible with lifecycle.prevent_destroy.", + "$ref": "#/$defs/string" } }, "additionalProperties": false From b4665d0281e9ad9d45984fdfdc19250fec25e470 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Tue, 18 Aug 2026 12:41:58 +0000 Subject: [PATCH 24/75] job_runs: fix exhaustruct and regenerate refschema for on_file_change Satisfy the linter on PrepareState literals after adding OnFileChange, and refresh out.fields.txt so validate-generated stays green. --- acceptance/bundle/refschema/out.fields.txt | 11 +++++++++++ bundle/direct/dresources/job_run.go | 11 +++++++++-- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index 9e1202a2ff8..87d1ac5270d 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -872,8 +872,14 @@ resources.job_runs.*.lifecycle.prevent_destroy bool INPUT resources.job_runs.*.lifecycle.triggers *dresources.JobRunTriggersState STATE resources.job_runs.*.lifecycle.triggers []resources.JobRunTrigger INPUT resources.job_runs.*.lifecycle.triggers.on_bundle_deploy string STATE +resources.job_runs.*.lifecycle.triggers.on_file_change map[string]resources.JobRunFileFingerprint STATE +resources.job_runs.*.lifecycle.triggers.on_file_change.* resources.JobRunFileFingerprint STATE +resources.job_runs.*.lifecycle.triggers.on_file_change.*.hash string STATE +resources.job_runs.*.lifecycle.triggers.on_file_change.*.mtime_ns int64 STATE +resources.job_runs.*.lifecycle.triggers.on_file_change.*.size int64 STATE resources.job_runs.*.lifecycle.triggers[*] resources.JobRunTrigger INPUT resources.job_runs.*.lifecycle.triggers[*].on_bundle_deploy *bool INPUT +resources.job_runs.*.lifecycle.triggers[*].on_file_change *string INPUT resources.job_runs.*.modified_status string INPUT resources.job_runs.*.notebook_params map[string]string ALL resources.job_runs.*.notebook_params.* string ALL @@ -896,6 +902,11 @@ resources.job_runs.*.python_params []string ALL resources.job_runs.*.python_params[*] string ALL resources.job_runs.*.queue *jobs.QueueSettings ALL resources.job_runs.*.queue.enabled bool ALL +resources.job_runs.*.resolved_file_triggers map[string]resources.JobRunFileFingerprint INPUT +resources.job_runs.*.resolved_file_triggers.* resources.JobRunFileFingerprint INPUT +resources.job_runs.*.resolved_file_triggers.*.hash string INPUT +resources.job_runs.*.resolved_file_triggers.*.mtime_ns int64 INPUT +resources.job_runs.*.resolved_file_triggers.*.size int64 INPUT resources.job_runs.*.resolved_job_id int64 INPUT resources.job_runs.*.result_state jobs.RunResultState REMOTE STATE resources.job_runs.*.run_id int64 REMOTE diff --git a/bundle/direct/dresources/job_run.go b/bundle/direct/dresources/job_run.go index 4e57a4bcdf4..f66991f1ca8 100644 --- a/bundle/direct/dresources/job_run.go +++ b/bundle/direct/dresources/job_run.go @@ -107,14 +107,21 @@ func (*ResourceJobRun) PrepareState(input *resources.JobRun) *JobRunState { state := &JobRunState{ RunNow: input.RunNow, ResultState: jobs.RunResultStateSuccess, + Lifecycle: nil, } var triggers *JobRunTriggersState if input.HasOnBundleDeploy() { - triggers = &JobRunTriggersState{OnBundleDeploy: uuid.NewString()} + triggers = &JobRunTriggersState{ + OnBundleDeploy: uuid.NewString(), + OnFileChange: nil, + } } if len(input.ResolvedFileTriggers) > 0 { if triggers == nil { - triggers = &JobRunTriggersState{} + triggers = &JobRunTriggersState{ + OnBundleDeploy: "", + OnFileChange: nil, + } } triggers.OnFileChange = input.ResolvedFileTriggers } From ecf44ef1e6f012be907f8c69b966c73ae888aa5c Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Tue, 18 Aug 2026 13:03:41 +0000 Subject: [PATCH 25/75] job_runs: tighten on_file_change path and directory checks Reject patterns outside the sync root and directory-only matches so the trigger cannot fingerprint escaped paths or silently disarm. --- .../bundles/job-runs-on-file-change.md | 2 +- .../mutator/resolve_job_run_file_triggers.go | 31 ++++++++++++++++--- .../resolve_job_run_file_triggers_test.go | 25 +++++++++++++++ bundle/internal/schema/annotations.yml | 2 +- bundle/schema/jsonschema.json | 2 +- 5 files changed, 55 insertions(+), 7 deletions(-) diff --git a/.nextchanges/bundles/job-runs-on-file-change.md b/.nextchanges/bundles/job-runs-on-file-change.md index 00682403a0d..eadceda6268 100644 --- a/.nextchanges/bundles/job-runs-on-file-change.md +++ b/.nextchanges/bundles/job-runs-on-file-change.md @@ -1 +1 @@ -direct: `resources.job_runs` can set `lifecycle.triggers.on_file_change` to a path or glob to re-fire the run when matched file contents change. Removing the trigger does not recreate the existing run. +direct: `resources.job_runs` can set `lifecycle.triggers.on_file_change` to a path or glob under the sync root to re-fire the run when matched file contents change, or when matches appear or disappear. Removing the trigger does not recreate the existing run. diff --git a/bundle/config/mutator/resolve_job_run_file_triggers.go b/bundle/config/mutator/resolve_job_run_file_triggers.go index a82ed654be3..f0338b2d50a 100644 --- a/bundle/config/mutator/resolve_job_run_file_triggers.go +++ b/bundle/config/mutator/resolve_job_run_file_triggers.go @@ -80,7 +80,17 @@ func resolveFileTriggers(b *bundle.Bundle, name string, patterns []string, prev out := make(map[string]resources.JobRunFileFingerprint) for _, pattern := range patterns { path := fmt.Sprintf("resources.job_runs.%s.lifecycle.triggers", name) - matches, err := filepath.Glob(filepath.Join(b.SyncRootPath, filepath.FromSlash(pattern))) + localPattern := filepath.FromSlash(pattern) + // Keep fingerprints under SyncRoot; same IsLocal gate as translate_paths. + if !filepath.IsLocal(localPattern) { + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: pattern %q is not under the sync root", pattern), + Locations: b.Config.GetLocations(path), + }) + continue + } + matches, err := filepath.Glob(filepath.Join(b.SyncRootPath, localPattern)) if err != nil { diags = diags.Append(diag.Diagnostic{ Severity: diag.Error, @@ -90,7 +100,7 @@ func resolveFileTriggers(b *bundle.Bundle, name string, patterns []string, prev continue } if len(matches) == 0 { - // Distinct state when the path/glob matches nothing (design doc). + // Distinct state when the path/glob matches nothing so appear/disappear recreates. out[filepath.ToSlash(pattern)] = resources.JobRunFileFingerprint{ Size: missingFileSentinelSize, } @@ -101,6 +111,8 @@ func resolveFileTriggers(b *bundle.Bundle, name string, patterns []string, prev }) continue } + regularMatches := 0 + sawNonRegular := false for _, match := range matches { info, err := os.Stat(match) if err != nil { @@ -112,13 +124,15 @@ func resolveFileTriggers(b *bundle.Bundle, name string, patterns []string, prev continue } if !info.Mode().IsRegular() { + sawNonRegular = true continue } + regularMatches++ rel, err := filepath.Rel(b.SyncRootPath, match) - if err != nil { + if err != nil || !filepath.IsLocal(rel) { diags = diags.Append(diag.Diagnostic{ Severity: diag.Error, - Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: relative path for %q: %s", match, err), + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: matched path %q is not under the sync root", match), Locations: b.Config.GetLocations(path), }) continue @@ -135,6 +149,15 @@ func resolveFileTriggers(b *bundle.Bundle, name string, patterns []string, prev } out[key] = fp } + // A directory-only match would otherwise leave ResolvedFileTriggers empty + // and silently disarm the trigger while config still sets on_file_change. + if regularMatches == 0 && sawNonRegular { + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: pattern %q matches no regular files", pattern), + Locations: b.Config.GetLocations(path), + }) + } } return out, diags } diff --git a/bundle/config/mutator/resolve_job_run_file_triggers_test.go b/bundle/config/mutator/resolve_job_run_file_triggers_test.go index 9a9cf77bf53..06266d75894 100644 --- a/bundle/config/mutator/resolve_job_run_file_triggers_test.go +++ b/bundle/config/mutator/resolve_job_run_file_triggers_test.go @@ -113,6 +113,31 @@ func TestResolveJobRunFileTriggers(t *testing.T) { assertFingerprint(t, fps["a.txt"], "aaa") assertFingerprint(t, fps["subdir/x.py"], "bbb") }) + + t.Run("pattern outside sync root is an error", func(t *testing.T) { + dir := t.TempDir() + pattern := "../outside.txt" + b := bundleWithFileTrigger(dir, pattern) + + diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) + require.True(t, diags.HasError()) + require.Len(t, diags, 1) + assert.Contains(t, diags[0].Summary, `not under the sync root`) + assert.Empty(t, b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers) + }) + + t.Run("directory-only match is an error", func(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(dir, "migrations"), 0o755)) + pattern := "migrations" + b := bundleWithFileTrigger(dir, pattern) + + diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) + require.True(t, diags.HasError()) + require.Len(t, diags, 1) + assert.Contains(t, diags[0].Summary, `matches no regular files`) + assert.Empty(t, b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers) + }) } func bundleWithFileTrigger(syncRoot, pattern string) *bundle.Bundle { diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index bfecaef7c00..8f8dc45d066 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -992,7 +992,7 @@ resources: If true, re-fire the run on every bundle deploy. Incompatible with lifecycle.prevent_destroy. "on_file_change": "description": |- - Path or glob under the sync root. Re-fire the run when any matched file's content changes. Incompatible with lifecycle.prevent_destroy. + Path or glob under the sync root (filepath.Glob; `**` is not supported). Re-fire the run when a matched file's content changes, or when the set of matches appears or disappears. Incompatible with lifecycle.prevent_destroy. "python_named_params": "description": |- PLACEHOLDER diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 0a01fba5022..03146b5f1cc 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -1313,7 +1313,7 @@ "$ref": "#/$defs/bool" }, "on_file_change": { - "description": "Path or glob under the sync root. Re-fire the run when any matched file's content changes. Incompatible with lifecycle.prevent_destroy.", + "description": "Path or glob under the sync root (filepath.Glob; `**` is not supported). Re-fire the run when a matched file's content changes, or when the set of matches appears or disappears. Incompatible with lifecycle.prevent_destroy.", "$ref": "#/$defs/string" } }, From 4a0a2b453a56563ee9045cb5cf22f968bc259664 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Tue, 18 Aug 2026 13:16:41 +0000 Subject: [PATCH 26/75] job_runs: store on_file_change as path-to-hash only Drop size/mtime fingerprints and the resolve fast path so content identity alone drives recreate, and touch no longer needs a planner exception. --- .../bundles/job-runs-on-file-change.md | 2 +- acceptance/bundle/refschema/out.fields.txt | 14 +- .../job_runs/on_file_change/output.txt | 11 + .../resources/job_runs/on_file_change/script | 6 + .../mutator/resolve_job_run_file_triggers.go | 204 +++++++----------- ...lve_job_run_file_triggers_internal_test.go | 73 ------- .../resolve_job_run_file_triggers_test.go | 48 +++-- bundle/config/resources/job_run.go | 19 +- bundle/config/resources/lifecycle.go | 6 - bundle/direct/dresources/job_run.go | 4 +- bundle/direct/dresources/job_run_test.go | 42 +--- bundle/direct/dresources/resources.yml | 6 - bundle/internal/schema/annotations.yml | 2 +- bundle/schema/jsonschema.json | 2 +- 14 files changed, 141 insertions(+), 298 deletions(-) delete mode 100644 bundle/config/mutator/resolve_job_run_file_triggers_internal_test.go diff --git a/.nextchanges/bundles/job-runs-on-file-change.md b/.nextchanges/bundles/job-runs-on-file-change.md index eadceda6268..95869a25ccf 100644 --- a/.nextchanges/bundles/job-runs-on-file-change.md +++ b/.nextchanges/bundles/job-runs-on-file-change.md @@ -1 +1 @@ -direct: `resources.job_runs` can set `lifecycle.triggers.on_file_change` to a path or glob under the sync root to re-fire the run when matched file contents change, or when matches appear or disappear. Removing the trigger does not recreate the existing run. +direct: `resources.job_runs` can set `lifecycle.triggers.on_file_change` to a path or glob under the sync root to re-fire the run when matched file contents change (content hash), or when matches appear or disappear. Removing the trigger does not recreate the existing run. diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index 87d1ac5270d..871152e6a9c 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -872,11 +872,8 @@ resources.job_runs.*.lifecycle.prevent_destroy bool INPUT resources.job_runs.*.lifecycle.triggers *dresources.JobRunTriggersState STATE resources.job_runs.*.lifecycle.triggers []resources.JobRunTrigger INPUT resources.job_runs.*.lifecycle.triggers.on_bundle_deploy string STATE -resources.job_runs.*.lifecycle.triggers.on_file_change map[string]resources.JobRunFileFingerprint STATE -resources.job_runs.*.lifecycle.triggers.on_file_change.* resources.JobRunFileFingerprint STATE -resources.job_runs.*.lifecycle.triggers.on_file_change.*.hash string STATE -resources.job_runs.*.lifecycle.triggers.on_file_change.*.mtime_ns int64 STATE -resources.job_runs.*.lifecycle.triggers.on_file_change.*.size int64 STATE +resources.job_runs.*.lifecycle.triggers.on_file_change map[string]string STATE +resources.job_runs.*.lifecycle.triggers.on_file_change.* string STATE resources.job_runs.*.lifecycle.triggers[*] resources.JobRunTrigger INPUT resources.job_runs.*.lifecycle.triggers[*].on_bundle_deploy *bool INPUT resources.job_runs.*.lifecycle.triggers[*].on_file_change *string INPUT @@ -902,11 +899,8 @@ resources.job_runs.*.python_params []string ALL resources.job_runs.*.python_params[*] string ALL resources.job_runs.*.queue *jobs.QueueSettings ALL resources.job_runs.*.queue.enabled bool ALL -resources.job_runs.*.resolved_file_triggers map[string]resources.JobRunFileFingerprint INPUT -resources.job_runs.*.resolved_file_triggers.* resources.JobRunFileFingerprint INPUT -resources.job_runs.*.resolved_file_triggers.*.hash string INPUT -resources.job_runs.*.resolved_file_triggers.*.mtime_ns int64 INPUT -resources.job_runs.*.resolved_file_triggers.*.size int64 INPUT +resources.job_runs.*.resolved_file_triggers map[string]string INPUT +resources.job_runs.*.resolved_file_triggers.* string INPUT resources.job_runs.*.resolved_job_id int64 INPUT resources.job_runs.*.result_state jobs.RunResultState REMOTE STATE resources.job_runs.*.run_id int64 REMOTE diff --git a/acceptance/bundle/resources/job_runs/on_file_change/output.txt b/acceptance/bundle/resources/job_runs/on_file_change/output.txt index bcf5962f8fa..f356fe7fb61 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change/output.txt @@ -33,6 +33,17 @@ Resources: 0 created, 0 changed, 0 deleted, 2 unchanged >>> print_requests.py //jobs/run-now +=== touch without content change plans nothing +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change/default/files... +Files: 3 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 2 unchanged + +>>> print_requests.py //jobs/run-now + === editing the file re-fires >>> [CLI] bundle plan recreate job_runs.my_run diff --git a/acceptance/bundle/resources/job_runs/on_file_change/script b/acceptance/bundle/resources/job_runs/on_file_change/script index ff162e11f0b..0506dd5e106 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change/script +++ b/acceptance/bundle/resources/job_runs/on_file_change/script @@ -16,6 +16,12 @@ trace $CLI bundle plan trace $CLI bundle deploy trace print_requests.py //jobs/run-now +title "touch without content change plans nothing" +python -c "import os; os.utime('seed.txt', None)" +trace $CLI bundle plan +trace $CLI bundle deploy +trace print_requests.py //jobs/run-now + title "editing the file re-fires" update_file.py seed.txt "v1" "v2" trace $CLI bundle plan diff --git a/bundle/config/mutator/resolve_job_run_file_triggers.go b/bundle/config/mutator/resolve_job_run_file_triggers.go index f0338b2d50a..55a55a94b95 100644 --- a/bundle/config/mutator/resolve_job_run_file_triggers.go +++ b/bundle/config/mutator/resolve_job_run_file_triggers.go @@ -4,25 +4,23 @@ import ( "context" "crypto/sha256" "encoding/hex" - "encoding/json" "fmt" "io" "os" "path/filepath" + "strings" "github.com/databricks/cli/bundle" - "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/libs/diag" ) -// missingFileSentinelSize marks a pattern with no matching file so the next -// plan can distinguish "still missing" from "file appeared". -const missingFileSentinelSize = int64(-1) +// missingFileHash marks a pattern with no matching file so appear/disappear recreates. +const missingFileHash = "" type resolveJobRunFileTriggers struct{} // ResolveJobRunFileTriggers expands on_file_change globs and stores per-file -// fingerprints on each job_run for PrepareState to copy into local state. +// content hashes on each job_run for PrepareState to copy into local state. func ResolveJobRunFileTriggers() bundle.Mutator { return &resolveJobRunFileTriggers{} } @@ -34,155 +32,105 @@ func (*resolveJobRunFileTriggers) Name() string { func (*resolveJobRunFileTriggers) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { var diags diag.Diagnostics for name, jr := range b.Config.Resources.JobRuns { - if jr == nil { + if jr == nil || jr.Lifecycle == nil { continue } - patterns := jr.OnFileChangePatterns() - if len(patterns) == 0 { - continue + out := make(map[string]string) + for i, t := range jr.Lifecycle.Triggers { + if t.OnFileChange == nil { + continue + } + path := fmt.Sprintf("resources.job_runs.%s.lifecycle.triggers[%d].on_file_change", name, i) + hashes, d := resolveFileTrigger(b, path, strings.TrimSpace(*t.OnFileChange)) + diags = diags.Extend(d) + for k, v := range hashes { + out[k] = v + } + } + if len(out) == 0 { + jr.ResolvedFileTriggers = nil + } else { + jr.ResolvedFileTriggers = out } - fps, d := resolveFileTriggers(b, name, patterns, previousFileTriggers(b, name)) - diags = diags.Extend(d) - jr.ResolvedFileTriggers = fps } return diags } -// previousFileTriggers reads on_file_change fingerprints from deployment state -// when it is open (plan/deploy after StatePull). Used so unchanged content keeps -// a stable fingerprint across mtime-only updates (e.g. touch). -func previousFileTriggers(b *bundle.Bundle, name string) map[string]resources.JobRunFileFingerprint { - if b.DeploymentBundle.StateDB.Path == "" { - return nil - } - entry, ok := b.DeploymentBundle.StateDB.GetResourceEntry("resources.job_runs." + name) - if !ok || len(entry.State) == 0 { - return nil - } - var state struct { - Lifecycle *struct { - Triggers *struct { - OnFileChange map[string]resources.JobRunFileFingerprint `json:"on_file_change"` - } `json:"triggers"` - } `json:"lifecycle"` +func resolveFileTrigger(b *bundle.Bundle, loc, pattern string) (map[string]string, diag.Diagnostics) { + var diags diag.Diagnostics + out := make(map[string]string) + localPattern := filepath.FromSlash(pattern) + // Keep hashes under SyncRoot; same IsLocal gate as translate_paths. + if !filepath.IsLocal(localPattern) { + return out, diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: pattern %q is not under the sync root", pattern), + Locations: b.Config.GetLocations(loc), + }) } - if err := json.Unmarshal(entry.State, &state); err != nil { - return nil + matches, err := filepath.Glob(filepath.Join(b.SyncRootPath, localPattern)) + if err != nil { + return out, diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: invalid pattern %q: %s", pattern, err), + Locations: b.Config.GetLocations(loc), + }) } - if state.Lifecycle == nil || state.Lifecycle.Triggers == nil { - return nil + if len(matches) == 0 { + out[filepath.ToSlash(pattern)] = missingFileHash + return out, diags.Append(diag.Diagnostic{ + Severity: diag.Warning, + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: no files match %q", pattern), + Locations: b.Config.GetLocations(loc), + }) } - return state.Lifecycle.Triggers.OnFileChange -} - -func resolveFileTriggers(b *bundle.Bundle, name string, patterns []string, prev map[string]resources.JobRunFileFingerprint) (map[string]resources.JobRunFileFingerprint, diag.Diagnostics) { - var diags diag.Diagnostics - out := make(map[string]resources.JobRunFileFingerprint) - for _, pattern := range patterns { - path := fmt.Sprintf("resources.job_runs.%s.lifecycle.triggers", name) - localPattern := filepath.FromSlash(pattern) - // Keep fingerprints under SyncRoot; same IsLocal gate as translate_paths. - if !filepath.IsLocal(localPattern) { + regularMatches := 0 + sawNonRegular := false + for _, match := range matches { + info, err := os.Stat(match) + if err != nil { diags = diags.Append(diag.Diagnostic{ Severity: diag.Error, - Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: pattern %q is not under the sync root", pattern), - Locations: b.Config.GetLocations(path), + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: stat %q: %s", match, err), + Locations: b.Config.GetLocations(loc), }) continue } - matches, err := filepath.Glob(filepath.Join(b.SyncRootPath, localPattern)) - if err != nil { - diags = diags.Append(diag.Diagnostic{ - Severity: diag.Error, - Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: invalid pattern %q: %s", pattern, err), - Locations: b.Config.GetLocations(path), - }) + if !info.Mode().IsRegular() { + sawNonRegular = true continue } - if len(matches) == 0 { - // Distinct state when the path/glob matches nothing so appear/disappear recreates. - out[filepath.ToSlash(pattern)] = resources.JobRunFileFingerprint{ - Size: missingFileSentinelSize, - } + regularMatches++ + rel, err := filepath.Rel(b.SyncRootPath, match) + if err != nil || !filepath.IsLocal(rel) { diags = diags.Append(diag.Diagnostic{ - Severity: diag.Warning, - Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: no files match %q", pattern), - Locations: b.Config.GetLocations(path), + Severity: diag.Error, + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: matched path %q is not under the sync root", match), + Locations: b.Config.GetLocations(loc), }) continue } - regularMatches := 0 - sawNonRegular := false - for _, match := range matches { - info, err := os.Stat(match) - if err != nil { - diags = diags.Append(diag.Diagnostic{ - Severity: diag.Error, - Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: stat %q: %s", match, err), - Locations: b.Config.GetLocations(path), - }) - continue - } - if !info.Mode().IsRegular() { - sawNonRegular = true - continue - } - regularMatches++ - rel, err := filepath.Rel(b.SyncRootPath, match) - if err != nil || !filepath.IsLocal(rel) { - diags = diags.Append(diag.Diagnostic{ - Severity: diag.Error, - Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: matched path %q is not under the sync root", match), - Locations: b.Config.GetLocations(path), - }) - continue - } - key := filepath.ToSlash(rel) - fp, err := fingerprintFile(match, info, prev[key]) - if err != nil { - diags = diags.Append(diag.Diagnostic{ - Severity: diag.Error, - Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: hash %q: %s", match, err), - Locations: b.Config.GetLocations(path), - }) - continue - } - out[key] = fp - } - // A directory-only match would otherwise leave ResolvedFileTriggers empty - // and silently disarm the trigger while config still sets on_file_change. - if regularMatches == 0 && sawNonRegular { + hash, err := hashFile(match) + if err != nil { diags = diags.Append(diag.Diagnostic{ Severity: diag.Error, - Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: pattern %q matches no regular files", pattern), - Locations: b.Config.GetLocations(path), + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: hash %q: %s", match, err), + Locations: b.Config.GetLocations(loc), }) + continue } + out[filepath.ToSlash(rel)] = hash } - return out, diags -} - -// fingerprintFile builds a content fingerprint. If prev has the same size and -// mtime, it is reused without reading the file. If content hash matches prev, -// prev is reused so mtime-only updates (touch) do not change planned state. -func fingerprintFile(path string, info os.FileInfo, prev resources.JobRunFileFingerprint) (resources.JobRunFileFingerprint, error) { - size := info.Size() - mtime := info.ModTime().UnixNano() - if prev.Hash != "" && prev.Size == size && prev.MtimeNs == mtime { - return prev, nil + // A directory-only match would otherwise leave ResolvedFileTriggers empty + // and silently disarm the trigger while config still sets on_file_change. + if regularMatches == 0 && sawNonRegular { + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: pattern %q matches no regular files", pattern), + Locations: b.Config.GetLocations(loc), + }) } - hash, err := hashFile(path) - if err != nil { - return resources.JobRunFileFingerprint{}, err - } - if prev.Hash != "" && prev.Hash == hash { - return prev, nil - } - return resources.JobRunFileFingerprint{ - Hash: hash, - Size: size, - MtimeNs: mtime, - }, nil + return out, diags } func hashFile(path string) (string, error) { diff --git a/bundle/config/mutator/resolve_job_run_file_triggers_internal_test.go b/bundle/config/mutator/resolve_job_run_file_triggers_internal_test.go deleted file mode 100644 index e82bd22945b..00000000000 --- a/bundle/config/mutator/resolve_job_run_file_triggers_internal_test.go +++ /dev/null @@ -1,73 +0,0 @@ -package mutator - -import ( - "os" - "path/filepath" - "testing" - "time" - - "github.com/databricks/cli/bundle/config/resources" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestFingerprintFileContentStableAcrossTouch(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, "seed.txt") - require.NoError(t, os.WriteFile(path, []byte("v1"), 0o644)) - - info, err := os.Stat(path) - require.NoError(t, err) - first, err := fingerprintFile(path, info, resources.JobRunFileFingerprint{}) - require.NoError(t, err) - require.NotEmpty(t, first.Hash) - - // Advance mtime without changing contents (touch). - require.NoError(t, os.Chtimes(path, time.Now().Add(time.Minute), time.Now().Add(time.Minute))) - info, err = os.Stat(path) - require.NoError(t, err) - assert.NotEqual(t, first.MtimeNs, info.ModTime().UnixNano()) - - second, err := fingerprintFile(path, info, first) - require.NoError(t, err) - assert.Equal(t, first, second, "unchanged content must reuse the previous fingerprint") -} - -func TestFingerprintFileFastPathSkipsWhenMtimeAndSizeMatch(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, "seed.txt") - require.NoError(t, os.WriteFile(path, []byte("v1"), 0o644)) - - info, err := os.Stat(path) - require.NoError(t, err) - prev := resources.JobRunFileFingerprint{ - Hash: "not-the-real-hash", - Size: info.Size(), - MtimeNs: info.ModTime().UnixNano(), - } - - got, err := fingerprintFile(path, info, prev) - require.NoError(t, err) - assert.Equal(t, prev, got, "matching size+mtime must reuse prev without re-hashing") -} - -func TestFingerprintFileContentChange(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, "seed.txt") - require.NoError(t, os.WriteFile(path, []byte("v1"), 0o644)) - - info, err := os.Stat(path) - require.NoError(t, err) - first, err := fingerprintFile(path, info, resources.JobRunFileFingerprint{}) - require.NoError(t, err) - - // Different size so the mtime+size fast path cannot reuse prev. - require.NoError(t, os.WriteFile(path, []byte("v2-changed"), 0o644)) - info, err = os.Stat(path) - require.NoError(t, err) - second, err := fingerprintFile(path, info, first) - require.NoError(t, err) - assert.NotEqual(t, first.Hash, second.Hash) - assert.Equal(t, info.Size(), second.Size) - assert.Equal(t, info.ModTime().UnixNano(), second.MtimeNs) -} diff --git a/bundle/config/mutator/resolve_job_run_file_triggers_test.go b/bundle/config/mutator/resolve_job_run_file_triggers_test.go index 06266d75894..3ec663c89bd 100644 --- a/bundle/config/mutator/resolve_job_run_file_triggers_test.go +++ b/bundle/config/mutator/resolve_job_run_file_triggers_test.go @@ -17,7 +17,7 @@ import ( ) func TestResolveJobRunFileTriggers(t *testing.T) { - t.Run("matches files and fills fingerprints", func(t *testing.T) { + t.Run("matches files and fills hashes", func(t *testing.T) { dir := t.TempDir() require.NoError(t, os.WriteFile(filepath.Join(dir, "a.txt"), []byte("hello"), 0o644)) require.NoError(t, os.WriteFile(filepath.Join(dir, "b.txt"), []byte("world"), 0o644)) @@ -28,14 +28,13 @@ func TestResolveJobRunFileTriggers(t *testing.T) { diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) require.False(t, diags.HasError()) - fps := b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers - require.Len(t, fps, 2) - - assertFingerprint(t, fps["a.txt"], "hello") - assertFingerprint(t, fps["b.txt"], "world") + hashes := b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers + require.Len(t, hashes, 2) + assert.Equal(t, contentHash("hello"), hashes["a.txt"]) + assert.Equal(t, contentHash("world"), hashes["b.txt"]) }) - t.Run("no matches warns and stores sentinel", func(t *testing.T) { + t.Run("no matches warns and stores empty hash", func(t *testing.T) { dir := t.TempDir() pattern := "missing.txt" b := bundleWithFileTrigger(dir, pattern) @@ -46,12 +45,9 @@ func TestResolveJobRunFileTriggers(t *testing.T) { assert.Equal(t, diag.Warning, diags[0].Severity) assert.Contains(t, diags[0].Summary, `no files match "missing.txt"`) - fps := b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers - require.Len(t, fps, 1) - fp := fps["missing.txt"] - assert.Empty(t, fp.Hash) - assert.Equal(t, int64(-1), fp.Size) - assert.Zero(t, fp.MtimeNs) + hashes := b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers + require.Len(t, hashes, 1) + assert.Empty(t, hashes["missing.txt"]) }) t.Run("no file triggers is a no-op", func(t *testing.T) { @@ -108,10 +104,10 @@ func TestResolveJobRunFileTriggers(t *testing.T) { diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) require.False(t, diags.HasError()) - fps := b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers - require.Len(t, fps, 2) - assertFingerprint(t, fps["a.txt"], "aaa") - assertFingerprint(t, fps["subdir/x.py"], "bbb") + hashes := b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers + require.Len(t, hashes, 2) + assert.Equal(t, contentHash("aaa"), hashes["a.txt"]) + assert.Equal(t, contentHash("bbb"), hashes["subdir/x.py"]) }) t.Run("pattern outside sync root is an error", func(t *testing.T) { @@ -138,6 +134,17 @@ func TestResolveJobRunFileTriggers(t *testing.T) { assert.Contains(t, diags[0].Summary, `matches no regular files`) assert.Empty(t, b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers) }) + + t.Run("trims pattern whitespace", func(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "seed.txt"), []byte("v1"), 0o644)) + pattern := " seed.txt " + b := bundleWithFileTrigger(dir, pattern) + + diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) + require.False(t, diags.HasError()) + assert.Equal(t, contentHash("v1"), b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers["seed.txt"]) + }) } func bundleWithFileTrigger(syncRoot, pattern string) *bundle.Bundle { @@ -159,10 +166,7 @@ func bundleWithFileTrigger(syncRoot, pattern string) *bundle.Bundle { } } -func assertFingerprint(t *testing.T, fp resources.JobRunFileFingerprint, content string) { - t.Helper() +func contentHash(content string) string { sum := sha256.Sum256([]byte(content)) - assert.Equal(t, hex.EncodeToString(sum[:]), fp.Hash) - assert.Equal(t, int64(len(content)), fp.Size) - assert.NotZero(t, fp.MtimeNs) + return hex.EncodeToString(sum[:]) } diff --git a/bundle/config/resources/job_run.go b/bundle/config/resources/job_run.go index cf66c71f728..f4b16963453 100644 --- a/bundle/config/resources/job_run.go +++ b/bundle/config/resources/job_run.go @@ -28,9 +28,9 @@ type JobRun struct { // reference) lets state loading preserve that reference and its plan dependency. ResolvedJobID int64 `json:"resolved_job_id,omitempty" bundle:"internal"` - // ResolvedFileTriggers is the per-file fingerprint map for on_file_change, - // computed under SyncRoot before plan. bundle:"internal" keeps it out of schema. - ResolvedFileTriggers map[string]JobRunFileFingerprint `json:"resolved_file_triggers,omitempty" bundle:"internal"` + // ResolvedFileTriggers is path → content hash for on_file_change, computed + // under SyncRoot before plan. bundle:"internal" keeps it out of schema. + ResolvedFileTriggers map[string]string `json:"resolved_file_triggers,omitempty" bundle:"internal"` } // HasOnBundleDeploy reports whether any trigger re-fires on every deploy. @@ -58,19 +58,6 @@ func (r *JobRun) HasOnFileChange() bool { return false } -func (r *JobRun) OnFileChangePatterns() []string { - if r.Lifecycle == nil { - return nil - } - var patterns []string - for _, t := range r.Lifecycle.Triggers { - if t.OnFileChange != nil { - patterns = append(patterns, *t.OnFileChange) - } - } - return patterns -} - func (r *JobRun) UnmarshalJSON(b []byte) error { return marshal.Unmarshal(b, r) } diff --git a/bundle/config/resources/lifecycle.go b/bundle/config/resources/lifecycle.go index 59ccb682b81..a4e881d298c 100644 --- a/bundle/config/resources/lifecycle.go +++ b/bundle/config/resources/lifecycle.go @@ -39,9 +39,3 @@ type JobRunTrigger struct { OnBundleDeploy *bool `json:"on_bundle_deploy,omitempty"` OnFileChange *string `json:"on_file_change,omitempty"` // path or glob under sync root } - -type JobRunFileFingerprint struct { - Hash string `json:"hash"` - Size int64 `json:"size"` - MtimeNs int64 `json:"mtime_ns"` -} diff --git a/bundle/direct/dresources/job_run.go b/bundle/direct/dresources/job_run.go index f66991f1ca8..5b97cc9e22b 100644 --- a/bundle/direct/dresources/job_run.go +++ b/bundle/direct/dresources/job_run.go @@ -38,8 +38,8 @@ var jobRunTriggerLocalPaths = []string{ type JobRunTriggersState struct { // Fresh UUID each plan while armed so Old!=New forces recreate. OnBundleDeploy string `json:"on_bundle_deploy,omitempty"` - // Per-file fingerprints from ResolveJobRunFileTriggers; change to recreate. - OnFileChange map[string]resources.JobRunFileFingerprint `json:"on_file_change,omitempty"` + // Path → content hash from ResolveJobRunFileTriggers; change to recreate. + OnFileChange map[string]string `json:"on_file_change,omitempty"` } // JobRunLifecycleState holds local-only lifecycle fields persisted in state. diff --git a/bundle/direct/dresources/job_run_test.go b/bundle/direct/dresources/job_run_test.go index f90517206fa..e80725a7eed 100644 --- a/bundle/direct/dresources/job_run_test.go +++ b/bundle/direct/dresources/job_run_test.go @@ -377,9 +377,7 @@ func TestJobRunPrepareStateOnBundleDeploy(t *testing.T) { } func TestJobRunPrepareStateOnFileChange(t *testing.T) { - fps := map[string]resources.JobRunFileFingerprint{ - "a.txt": {Hash: "abc", Size: 3, MtimeNs: 1}, - } + hashes := map[string]string{"a.txt": "abc"} t.Run("unset", func(t *testing.T) { state := (&ResourceJobRun{}).PrepareState(&resources.JobRun{}) @@ -388,11 +386,11 @@ func TestJobRunPrepareStateOnFileChange(t *testing.T) { t.Run("armed", func(t *testing.T) { state := (&ResourceJobRun{}).PrepareState(&resources.JobRun{ - ResolvedFileTriggers: fps, + ResolvedFileTriggers: hashes, }) require.NotNil(t, state.Lifecycle) require.NotNil(t, state.Lifecycle.Triggers) - assert.Equal(t, fps, state.Lifecycle.Triggers.OnFileChange) + assert.Equal(t, hashes, state.Lifecycle.Triggers.OnFileChange) assert.Empty(t, state.Lifecycle.Triggers.OnBundleDeploy) }) @@ -402,12 +400,12 @@ func TestJobRunPrepareStateOnFileChange(t *testing.T) { Lifecycle: &resources.JobRunLifecycle{ Triggers: []resources.JobRunTrigger{{OnBundleDeploy: &on}}, }, - ResolvedFileTriggers: fps, + ResolvedFileTriggers: hashes, }) require.NotNil(t, state.Lifecycle) require.NotNil(t, state.Lifecycle.Triggers) assert.NotEmpty(t, state.Lifecycle.Triggers.OnBundleDeploy) - assert.Equal(t, fps, state.Lifecycle.Triggers.OnFileChange) + assert.Equal(t, hashes, state.Lifecycle.Triggers.OnFileChange) }) } @@ -439,10 +437,8 @@ func TestJobRunOverrideChangeDescTriggerRemoved(t *testing.T) { t.Run("clearing on_file_change leaf downgrades to update", func(t *testing.T) { change := &ChangeDesc{ Action: deployplan.Recreate, - Old: map[string]resources.JobRunFileFingerprint{ - "a.txt": {Hash: "abc", Size: 3, MtimeNs: 1}, - }, - New: nil, + Old: map[string]string{"a.txt": "abc"}, + New: nil, } require.NoError(t, r.OverrideChangeDesc(t.Context(), structpath.MustParsePath("lifecycle.triggers.on_file_change"), change, nil)) assert.Equal(t, deployplan.Update, change.Action) @@ -459,15 +455,11 @@ func TestJobRunOverrideChangeDescTriggerRemoved(t *testing.T) { assert.Equal(t, deployplan.Recreate, change.Action) }) - t.Run("changed on_file_change fingerprint still recreates", func(t *testing.T) { + t.Run("changed on_file_change hash still recreates", func(t *testing.T) { change := &ChangeDesc{ Action: deployplan.Recreate, - Old: map[string]resources.JobRunFileFingerprint{ - "a.txt": {Hash: "old", Size: 1, MtimeNs: 1}, - }, - New: map[string]resources.JobRunFileFingerprint{ - "a.txt": {Hash: "new", Size: 1, MtimeNs: 2}, - }, + Old: map[string]string{"a.txt": "old"}, + New: map[string]string{"a.txt": "new"}, } require.NoError(t, r.OverrideChangeDesc(t.Context(), structpath.MustParsePath("lifecycle.triggers.on_file_change"), change, nil)) assert.Equal(t, deployplan.Recreate, change.Action) @@ -511,20 +503,6 @@ func TestJobRunIgnoresEveryRequestField(t *testing.T) { assert.False(t, ignoresRemoteChanges(ignored, "result_state"), "result_state must stay comparable against the remote") } -func TestJobRunIgnoresMtimeOnlyFileTriggerDrift(t *testing.T) { - adapters, err := InitAll(nil) - require.NoError(t, err) - ignored := adapters["job_runs"].ResourceConfig().IgnoreLocalChanges - path := structpath.MustParsePath("lifecycle.triggers.on_file_change['seed.txt'].mtime_ns") - assert.True(t, slices.ContainsFunc(ignored, func(r FieldRule) bool { - return path.HasPatternPrefix(r.Field) - }), "mtime-only fingerprint drift must be ignored") - hashPath := structpath.MustParsePath("lifecycle.triggers.on_file_change['seed.txt'].hash") - assert.False(t, slices.ContainsFunc(ignored, func(r FieldRule) bool { - return hashPath.HasPatternPrefix(r.Field) - }), "hash changes must still recreate") -} - // ignoresRemoteChanges reports whether the rules suppress remote drift on field. func ignoresRemoteChanges(rules []FieldRule, field string) bool { path := structpath.MustParsePath(field) diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index 0d49749c15a..b0a9ed6b365 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -182,12 +182,6 @@ resources: recreate_on_changes: - reason: immutable - # mtime is stored for the resolve fast path; content identity is the hash. - # Ignoring mtime-only local drift keeps touch from recreating the run. - ignore_local_changes: - - field: lifecycle.triggers.on_file_change[*].mtime_ns - reason: mtime_only_fingerprint - pipelines: recreate_on_changes: - field: storage diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index 8f8dc45d066..d48a3e2053a 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -992,7 +992,7 @@ resources: If true, re-fire the run on every bundle deploy. Incompatible with lifecycle.prevent_destroy. "on_file_change": "description": |- - Path or glob under the sync root (filepath.Glob; `**` is not supported). Re-fire the run when a matched file's content changes, or when the set of matches appears or disappears. Incompatible with lifecycle.prevent_destroy. + Path or glob under the sync root (filepath.Glob; `**` is not supported). Re-fire the run when a matched file's content hash changes, or when the set of matches appears or disappears. Incompatible with lifecycle.prevent_destroy. "python_named_params": "description": |- PLACEHOLDER diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 03146b5f1cc..78fdaf4a7a8 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -1313,7 +1313,7 @@ "$ref": "#/$defs/bool" }, "on_file_change": { - "description": "Path or glob under the sync root (filepath.Glob; `**` is not supported). Re-fire the run when a matched file's content changes, or when the set of matches appears or disappears. Incompatible with lifecycle.prevent_destroy.", + "description": "Path or glob under the sync root (filepath.Glob; `**` is not supported). Re-fire the run when a matched file's content hash changes, or when the set of matches appears or disappears. Incompatible with lifecycle.prevent_destroy.", "$ref": "#/$defs/string" } }, From 25f95666dc39eac4df509e2b0e169bbbfae4923b Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Tue, 18 Aug 2026 13:23:44 +0000 Subject: [PATCH 27/75] job_runs: use maps.Copy in on_file_change hash merge The modernize linter rejects the manual m[k]=v copy loop, failing CI lint. --- bundle/config/mutator/resolve_job_run_file_triggers.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/bundle/config/mutator/resolve_job_run_file_triggers.go b/bundle/config/mutator/resolve_job_run_file_triggers.go index 55a55a94b95..ead17c59e17 100644 --- a/bundle/config/mutator/resolve_job_run_file_triggers.go +++ b/bundle/config/mutator/resolve_job_run_file_triggers.go @@ -6,6 +6,7 @@ import ( "encoding/hex" "fmt" "io" + "maps" "os" "path/filepath" "strings" @@ -43,9 +44,7 @@ func (*resolveJobRunFileTriggers) Apply(_ context.Context, b *bundle.Bundle) dia path := fmt.Sprintf("resources.job_runs.%s.lifecycle.triggers[%d].on_file_change", name, i) hashes, d := resolveFileTrigger(b, path, strings.TrimSpace(*t.OnFileChange)) diags = diags.Extend(d) - for k, v := range hashes { - out[k] = v - } + maps.Copy(out, hashes) } if len(out) == 0 { jr.ResolvedFileTriggers = nil From 16367e64ddfe7002f3502f074ec89da00591343f Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Wed, 19 Aug 2026 08:39:23 +0000 Subject: [PATCH 28/75] job_runs: drop filepath.Glob jargon from on_file_change schema docs --- bundle/internal/schema/annotations.yml | 2 +- bundle/schema/jsonschema.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index d48a3e2053a..1fd6b452415 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -992,7 +992,7 @@ resources: If true, re-fire the run on every bundle deploy. Incompatible with lifecycle.prevent_destroy. "on_file_change": "description": |- - Path or glob under the sync root (filepath.Glob; `**` is not supported). Re-fire the run when a matched file's content hash changes, or when the set of matches appears or disappears. Incompatible with lifecycle.prevent_destroy. + Path or glob under the sync root. Re-fire the run when a matched file's content hash changes, or when the set of matches appears or disappears. Incompatible with lifecycle.prevent_destroy. "python_named_params": "description": |- PLACEHOLDER diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 78fdaf4a7a8..9afa34882ef 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -1313,7 +1313,7 @@ "$ref": "#/$defs/bool" }, "on_file_change": { - "description": "Path or glob under the sync root (filepath.Glob; `**` is not supported). Re-fire the run when a matched file's content hash changes, or when the set of matches appears or disappears. Incompatible with lifecycle.prevent_destroy.", + "description": "Path or glob under the sync root. Re-fire the run when a matched file's content hash changes, or when the set of matches appears or disappears. Incompatible with lifecycle.prevent_destroy.", "$ref": "#/$defs/string" } }, From 83bbb020d5376676b929f896797a16a6740db844 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Wed, 19 Aug 2026 11:38:20 +0000 Subject: [PATCH 29/75] acc: refresh on_file_change goldens for resource-key progress lines --- .../bundle/resources/job_runs/on_file_change/output.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/acceptance/bundle/resources/job_runs/on_file_change/output.txt b/acceptance/bundle/resources/job_runs/on_file_change/output.txt index f356fe7fb61..a98eb3a6ca7 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change/output.txt @@ -2,8 +2,8 @@ === first deploy triggers a run >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change/default/files... -job run [MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?o=[NUMID] -job run [MY_RUN_ID]: SUCCESS +Output from job_runs.my_run: id=[MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?o=[NUMID] +Output from job_runs.my_run: id=[MY_RUN_ID]: SUCCESS Created job_runs.my_run Created jobs.my_job Files: 5 uploaded, 0 deleted @@ -52,8 +52,8 @@ Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change/default/files... -job run [MY_RUN_ID_2]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_2]?o=[NUMID] -job run [MY_RUN_ID_2]: SUCCESS +Output from job_runs.my_run: id=[MY_RUN_ID_2]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_2]?o=[NUMID] +Output from job_runs.my_run: id=[MY_RUN_ID_2]: SUCCESS Recreated job_runs.my_run Files: 3 uploaded, 0 deleted Resources: 1 created, 0 changed, 1 deleted, 1 unchanged From 1b73b0051d1ea1b55966b0284b91ccd84d2123a6 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Wed, 19 Aug 2026 14:50:14 +0000 Subject: [PATCH 30/75] acc: cover on_file_change globs, errors, and appear/disappear --- .../job_runs/on_file_change/out.test.toml | 1 + .../job_runs/on_file_change/output.txt | 131 +++++++++++++- .../resources/job_runs/on_file_change/script | 26 ++- .../job_runs/on_file_change/test.toml | 14 ++ .../on_file_change_errors/databricks.yml | 18 ++ .../dirs/nested/keep.txt | 1 + .../on_file_change_errors/out.test.toml | 2 + .../job_runs/on_file_change_errors/output.txt | 39 +++++ .../job_runs/on_file_change_errors/script | 12 ++ .../job_runs/on_file_change_errors/test.toml | 2 + .../on_file_change_glob/databricks.yml | 18 ++ .../on_file_change_glob/migrations/001.txt | 1 + .../on_file_change_glob/migrations/002.txt | 1 + .../migrations/archive/003.txt | 1 + .../on_file_change_glob/out.test.toml | 2 + .../job_runs/on_file_change_glob/output.txt | 164 ++++++++++++++++++ .../job_runs/on_file_change_glob/script | 47 +++++ .../job_runs/on_file_change_glob/test.toml | 9 + bundle/phases/plan.go | 4 +- 19 files changed, 486 insertions(+), 7 deletions(-) create mode 100644 acceptance/bundle/resources/job_runs/on_file_change/test.toml create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_errors/databricks.yml create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_errors/dirs/nested/keep.txt create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_errors/out.test.toml create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_errors/script create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_errors/test.toml create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_glob/databricks.yml create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/001.txt create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/002.txt create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/archive/003.txt create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_glob/out.test.toml create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_glob/script create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml diff --git a/acceptance/bundle/resources/job_runs/on_file_change/out.test.toml b/acceptance/bundle/resources/job_runs/on_file_change/out.test.toml index 0938e678987..57b0f616850 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change/out.test.toml +++ b/acceptance/bundle/resources/job_runs/on_file_change/out.test.toml @@ -1,2 +1,3 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/job_runs/on_file_change/output.txt b/acceptance/bundle/resources/job_runs/on_file_change/output.txt index a98eb3a6ca7..d03307b90f2 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change/output.txt @@ -6,7 +6,7 @@ Output from job_runs.my_run: id=[MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[MY_ Output from job_runs.my_run: id=[MY_RUN_ID]: SUCCESS Created job_runs.my_run Created jobs.my_job -Files: 5 uploaded, 0 deleted +Files: 6 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged >>> read_id.py my_job @@ -50,12 +50,46 @@ recreate job_runs.my_run Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged ->>> [CLI] bundle deploy +>>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json +{ + "lifecycle": { + "action": "recreate", + "reason": "immutable", + "old": { + "triggers": { + "on_file_change": { + "seed.txt": "[FILE_HASH][0]" + } + } + }, + "new": { + "triggers": { + "on_file_change": { + "seed.txt": "[FILE_HASH][1]" + } + } + } + }, + "lifecycle.triggers.on_file_change['seed.txt']": { + "action": "recreate", + "reason": "immutable", + "old": "[FILE_HASH][0]", + "new": "[FILE_HASH][1]" + }, + "result_state": { + "action": "skip", + "reason": "remote_already_set", + "new": "SUCCESS", + "remote": "SUCCESS" + } +} + +=== bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change/default/files... Output from job_runs.my_run: id=[MY_RUN_ID_2]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_2]?o=[NUMID] Output from job_runs.my_run: id=[MY_RUN_ID_2]: SUCCESS Recreated job_runs.my_run -Files: 3 uploaded, 0 deleted +Files: 4 uploaded, 0 deleted Resources: 1 created, 0 changed, 1 deleted, 1 unchanged >>> print_requests.py --keep //jobs/runs/delete @@ -77,6 +111,97 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged } } +=== deleting the file warns and re-fires +>>> [CLI] bundle plan +Warning: lifecycle.triggers.on_file_change: no files match "seed.txt" + in databricks.yml:18:29 + +recreate job_runs.my_run + +Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged + +>>> changes +Warning: lifecycle.triggers.on_file_change: no files match "seed.txt" + in databricks.yml:18:29 + +{ + "lifecycle": { + "action": "recreate", + "reason": "immutable", + "old": { + "triggers": { + "on_file_change": { + "seed.txt": "[FILE_HASH][1]" + } + } + }, + "new": { + "triggers": { + "on_file_change": { + "seed.txt": "" + } + } + } + }, + "lifecycle.triggers.on_file_change['seed.txt']": { + "action": "recreate", + "reason": "immutable", + "old": "[FILE_HASH][1]", + "new": "" + }, + "result_state": { + "action": "skip", + "reason": "remote_already_set", + "new": "SUCCESS", + "remote": "SUCCESS" + } +} + +>>> [CLI] bundle deploy +Warning: lifecycle.triggers.on_file_change: no files match "seed.txt" + in databricks.yml:18:29 + +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change/default/files... +Output from job_runs.my_run: id=[MY_RUN_ID_3]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_3]?o=[NUMID] +Output from job_runs.my_run: id=[MY_RUN_ID_3]: SUCCESS +Recreated job_runs.my_run +Files: 3 uploaded, 1 deleted +Resources: 1 created, 0 changed, 1 deleted, 1 unchanged + +>>> print_requests.py //jobs/run-now +{ + "method": "POST", + "path": "/api/2.2/jobs/run-now", + "body": { + "idempotency_token": "[UUID]", + "job_id": [MY_JOB_ID] + } +} + +=== restoring the file re-fires again +>>> [CLI] bundle plan +recreate job_runs.my_run + +Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change/default/files... +Output from job_runs.my_run: id=[MY_RUN_ID_4]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_4]?o=[NUMID] +Output from job_runs.my_run: id=[MY_RUN_ID_4]: SUCCESS +Recreated job_runs.my_run +Files: 3 uploaded, 1 deleted +Resources: 1 created, 0 changed, 1 deleted, 1 unchanged + +>>> print_requests.py //jobs/run-now +{ + "method": "POST", + "path": "/api/2.2/jobs/run-now", + "body": { + "idempotency_token": "[UUID]", + "job_id": [MY_JOB_ID] + } +} + === removing on_file_change rewrites state without a run >>> [CLI] bundle plan update job_runs.my_run diff --git a/acceptance/bundle/resources/job_runs/on_file_change/script b/acceptance/bundle/resources/job_runs/on_file_change/script index 0506dd5e106..fd1e8800c42 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change/script +++ b/acceptance/bundle/resources/job_runs/on_file_change/script @@ -4,6 +4,10 @@ cleanup() { } trap cleanup EXIT +changes() { + $CLI bundle plan -o json | jq '.plan["resources.job_runs.my_run"].changes' +} + title "first deploy triggers a run" trace $CLI bundle deploy trace read_id.py my_job @@ -25,11 +29,31 @@ trace print_requests.py //jobs/run-now title "editing the file re-fires" update_file.py seed.txt "v1" "v2" trace $CLI bundle plan -trace $CLI bundle deploy +# Save the plan so the READPLAN=1 variant deploys the hashes computed here. +# The deploy is not traced: readplanarg makes the command line differ per variant. +$CLI bundle plan -o json > tmp.plan.json +trace jq '.plan["resources.job_runs.my_run"].changes' tmp.plan.json +title "bundle deploy\n" +$CLI bundle deploy $(readplanarg tmp.plan.json) read_id.py my_run > /dev/null trace print_requests.py --keep //jobs/runs/delete trace print_requests.py //jobs/run-now +title "deleting the file warns and re-fires" +mv seed.txt tmp.seed.txt +trace $CLI bundle plan +trace changes +trace $CLI bundle deploy +read_id.py my_run > /dev/null +trace print_requests.py //jobs/run-now + +title "restoring the file re-fires again" +mv tmp.seed.txt seed.txt +trace $CLI bundle plan +trace $CLI bundle deploy +read_id.py my_run > /dev/null +trace print_requests.py //jobs/run-now + title "removing on_file_change rewrites state without a run" update_file.py databricks.yml " lifecycle: triggers: diff --git a/acceptance/bundle/resources/job_runs/on_file_change/test.toml b/acceptance/bundle/resources/job_runs/on_file_change/test.toml new file mode 100644 index 00000000000..7a950e9fd0c --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change/test.toml @@ -0,0 +1,14 @@ +# Deploy both by re-planning and from a plan saved on disk, so the hashes +# computed during planning survive plan serialization. +EnvMatrix.READPLAN = ["", "1"] + +Ignore = ["tmp.plan.json", "tmp.seed.txt"] + +# Content hashes are stable but unreadable, and the parent's `\d{8,}` rule +# mangles any digit run inside them. Order=1 runs before it; Distinct keeps +# different hashes distinguishable. +[[Repls]] +Old = '[0-9a-f]{64}' +New = '[FILE_HASH]' +Order = 1 +Distinct = true diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/databricks.yml b/acceptance/bundle/resources/job_runs/on_file_change_errors/databricks.yml new file mode 100644 index 00000000000..44c5433e52a --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/databricks.yml @@ -0,0 +1,18 @@ +bundle: + name: job-runs-on-file-change-errors + +resources: + jobs: + my_job: + name: my-job + tasks: + - task_key: main + notebook_task: + notebook_path: /Workspace/test + + job_runs: + my_run: + job_id: ${resources.jobs.my_job.id} + lifecycle: + triggers: + - on_file_change: ../shared/*.txt diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/dirs/nested/keep.txt b/acceptance/bundle/resources/job_runs/on_file_change_errors/dirs/nested/keep.txt new file mode 100644 index 00000000000..79c53955ef8 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/dirs/nested/keep.txt @@ -0,0 +1 @@ +nested diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/out.test.toml b/acceptance/bundle/resources/job_runs/on_file_change_errors/out.test.toml new file mode 100644 index 00000000000..0938e678987 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt new file mode 100644 index 00000000000..c7d3a98e1cb --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt @@ -0,0 +1,39 @@ + +=== a pattern outside the sync root is rejected +>>> musterr [CLI] bundle validate +Error: lifecycle.triggers.on_file_change: pattern "../shared/*.txt" is not under the sync root + in databricks.yml:18:29 + +Name: job-runs-on-file-change-errors +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-errors/default + +Found 1 error + +=== a glob that only matches directories is rejected +>>> musterr [CLI] bundle validate +Error: lifecycle.triggers.on_file_change: pattern "dirs/*" matches no regular files + in databricks.yml:18:29 + +Name: job-runs-on-file-change-errors +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-errors/default + +Found 1 error + +=== a pattern that matches nothing warns +>>> [CLI] bundle validate +Warning: lifecycle.triggers.on_file_change: no files match "missing/*.txt" + in databricks.yml:18:29 + +Name: job-runs-on-file-change-errors +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-errors/default + +Found 1 warning diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/script b/acceptance/bundle/resources/job_runs/on_file_change_errors/script new file mode 100644 index 00000000000..39596f0c772 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/script @@ -0,0 +1,12 @@ +title "a pattern outside the sync root is rejected" +trace musterr $CLI bundle validate + +title "a glob that only matches directories is rejected" +# dirs/* matches dirs/nested, a directory: nothing is left to hash, so the +# trigger would silently never fire. +update_file.py databricks.yml "../shared/*.txt" "dirs/*" +trace musterr $CLI bundle validate + +title "a pattern that matches nothing warns" +update_file.py databricks.yml "dirs/*" "missing/*.txt" +trace $CLI bundle validate diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/test.toml b/acceptance/bundle/resources/job_runs/on_file_change_errors/test.toml new file mode 100644 index 00000000000..7b88d613d4c --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/test.toml @@ -0,0 +1,2 @@ +# The bundle is never deployed here, so there are no requests worth recording. +RecordRequests = false diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/databricks.yml b/acceptance/bundle/resources/job_runs/on_file_change_glob/databricks.yml new file mode 100644 index 00000000000..3b148538f38 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/databricks.yml @@ -0,0 +1,18 @@ +bundle: + name: job-runs-on-file-change-glob + +resources: + jobs: + my_job: + name: my-job + tasks: + - task_key: main + notebook_task: + notebook_path: /Workspace/test + + job_runs: + my_run: + job_id: ${resources.jobs.my_job.id} + lifecycle: + triggers: + - on_file_change: migrations/* diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/001.txt b/acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/001.txt new file mode 100644 index 00000000000..5626abf0f72 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/001.txt @@ -0,0 +1 @@ +one diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/002.txt b/acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/002.txt new file mode 100644 index 00000000000..f719efd430d --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/002.txt @@ -0,0 +1 @@ +two diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/archive/003.txt b/acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/archive/003.txt new file mode 100644 index 00000000000..2bdf67abb16 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/archive/003.txt @@ -0,0 +1 @@ +three diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/out.test.toml b/acceptance/bundle/resources/job_runs/on_file_change_glob/out.test.toml new file mode 100644 index 00000000000..0938e678987 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt new file mode 100644 index 00000000000..798b459a9bd --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt @@ -0,0 +1,164 @@ + +=== deploy hashes the matched files +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-glob/default/files... +Output from job_runs.my_run: id=[MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?o=[NUMID] +Output from job_runs.my_run: id=[MY_RUN_ID]: SUCCESS +Created job_runs.my_run +Created jobs.my_job +Files: 8 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +>>> read_id.py my_job +[MY_JOB_ID] + +>>> print_requests.py //jobs/run-now +{ + "method": "POST", + "path": "/api/2.2/jobs/run-now", + "body": { + "idempotency_token": "[UUID]", + "job_id": [MY_JOB_ID] + } +} + +=== editing one matched file re-fires +>>> [CLI] bundle plan +recreate job_runs.my_run + +Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged + +>>> changes +{ + "lifecycle": { + "action": "recreate", + "reason": "immutable", + "old": { + "triggers": { + "on_file_change": { + "migrations/001.txt": "[FILE_HASH][0]", + "migrations/002.txt": "[FILE_HASH][1]" + } + } + }, + "new": { + "triggers": { + "on_file_change": { + "migrations/001.txt": "[FILE_HASH][2]", + "migrations/002.txt": "[FILE_HASH][1]" + } + } + } + }, + "lifecycle.triggers.on_file_change['migrations/001.txt']": { + "action": "recreate", + "reason": "immutable", + "old": "[FILE_HASH][0]", + "new": "[FILE_HASH][2]" + }, + "result_state": { + "action": "skip", + "reason": "remote_already_set", + "new": "SUCCESS", + "remote": "SUCCESS" + } +} + +=== a new matching file re-fires +>>> [CLI] bundle plan +recreate job_runs.my_run + +Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged + +>>> changes +{ + "lifecycle": { + "action": "recreate", + "reason": "immutable", + "old": { + "triggers": { + "on_file_change": { + "migrations/001.txt": "[FILE_HASH][0]", + "migrations/002.txt": "[FILE_HASH][1]" + } + } + }, + "new": { + "triggers": { + "on_file_change": { + "migrations/001.txt": "[FILE_HASH][0]", + "migrations/002.txt": "[FILE_HASH][1]", + "migrations/003.txt": "[FILE_HASH][3]" + } + } + } + }, + "lifecycle.triggers.on_file_change['migrations/003.txt']": { + "action": "recreate", + "reason": "immutable", + "new": "[FILE_HASH][3]" + }, + "result_state": { + "action": "skip", + "reason": "remote_already_set", + "new": "SUCCESS", + "remote": "SUCCESS" + } +} + +=== removing a matched file re-fires +>>> [CLI] bundle plan +recreate job_runs.my_run + +Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged + +>>> changes +{ + "lifecycle": { + "action": "recreate", + "reason": "immutable", + "old": { + "triggers": { + "on_file_change": { + "migrations/001.txt": "[FILE_HASH][0]", + "migrations/002.txt": "[FILE_HASH][1]" + } + } + }, + "new": { + "triggers": { + "on_file_change": { + "migrations/001.txt": "[FILE_HASH][0]" + } + } + } + }, + "lifecycle.triggers.on_file_change['migrations/002.txt']": { + "action": "recreate", + "reason": "immutable", + "old": "[FILE_HASH][1]" + }, + "result_state": { + "action": "skip", + "reason": "remote_already_set", + "new": "SUCCESS", + "remote": "SUCCESS" + } +} + +=== editing a file below the glob does not re-fire +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +=== ** is not recursive: it matches the same files as * +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.job_runs.my_run + delete resources.jobs.my_job + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-glob/default + +Destroy: 2 deleted diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/script b/acceptance/bundle/resources/job_runs/on_file_change_glob/script new file mode 100644 index 00000000000..8feeebcd067 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/script @@ -0,0 +1,47 @@ +cleanup() { + trace $CLI bundle destroy --auto-approve + rm -f out.requests.txt +} +trap cleanup EXIT + +changes() { + $CLI bundle plan -o json | jq '.plan["resources.job_runs.my_run"].changes' +} + +title "deploy hashes the matched files" +trace $CLI bundle deploy +trace read_id.py my_job +read_id.py my_run > /dev/null +trace print_requests.py //jobs/run-now + +# Each phase below plans against the state deployed above and then restores the +# tree, so every plan is compared against the same baseline. + +title "editing one matched file re-fires" +update_file.py migrations/001.txt "one" "one-edited" +trace $CLI bundle plan +trace changes +update_file.py migrations/001.txt "one-edited" "one" + +title "a new matching file re-fires" +printf 'three\n' > migrations/003.txt +trace $CLI bundle plan +trace changes +rm migrations/003.txt + +title "removing a matched file re-fires" +mv migrations/002.txt tmp.002.txt +trace $CLI bundle plan +trace changes +mv tmp.002.txt migrations/002.txt + +title "editing a file below the glob does not re-fire" +# migrations/* matches the archive directory itself, which is skipped: the glob +# does not descend into it. +update_file.py migrations/archive/003.txt "three" "three-edited" +trace $CLI bundle plan +update_file.py migrations/archive/003.txt "three-edited" "three" + +title "** is not recursive: it matches the same files as *" +update_file.py databricks.yml "migrations/*" "migrations/**" +trace $CLI bundle plan diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml b/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml new file mode 100644 index 00000000000..1c349066b94 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml @@ -0,0 +1,9 @@ +Ignore = ["tmp.002.txt"] + +# See on_file_change/test.toml: mask content hashes before the parent's +# `\d{8,}` rule can mangle them, keeping distinct hashes distinguishable. +[[Repls]] +Old = '[0-9a-f]{64}' +New = '[FILE_HASH]' +Order = 1 +Distinct = true diff --git a/bundle/phases/plan.go b/bundle/phases/plan.go index 92ca524cc4f..3db0864c2c4 100644 --- a/bundle/phases/plan.go +++ b/bundle/phases/plan.go @@ -21,8 +21,7 @@ import ( // PreDeployChecks is common set of mutators between "bundle plan" and "bundle deploy". // Note, it is not run in "bundle migrate" so it must not modify the config func PreDeployChecks(ctx context.Context, b *bundle.Bundle, isPlan bool, engine engine.EngineType) { - bundle.ApplySeqContext( - ctx, b, + bundle.ApplySeqContext(ctx, b, terraform.CheckDashboardsModifiedRemotely(isPlan, engine), resourcemutator.SecretScopeFixups(engine), deploy.StatePull(), @@ -31,7 +30,6 @@ func PreDeployChecks(ctx context.Context, b *bundle.Bundle, isPlan bool, engine mutator.ValidateLifecycleStarted(engine), mutator.ValidateCascadeOnDestroy(engine), mutator.ValidateJobRunTriggers(), - mutator.ResolveJobRunFileTriggers(), statemgmt.CheckRunningResource(engine), ) } From 9e7c0fd94d6f936abfcc9810cff6ca4f8dc0801e Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Wed, 19 Aug 2026 19:58:11 +0000 Subject: [PATCH 31/75] acc: expand on_file_change glob coverage and skip ignored files Honor .gitignore and sync.exclude when hashing matches. Cover mysubdir globs, READPLAN, and keep LF on Windows so restored files hash cleanly. --- .../job_runs/on_file_change_glob/.gitignore | 2 + .../on_file_change_glob/databricks.yml | 1 + .../on_file_change_glob/mysubdir/a.txt | 1 + .../on_file_change_glob/out.test.toml | 1 + .../job_runs/on_file_change_glob/output.txt | 183 +++++++++++++++--- .../job_runs/on_file_change_glob/script | 39 ++-- .../job_runs/on_file_change_glob/test.toml | 6 +- .../mutator/resolve_job_run_file_triggers.go | 65 ++++++- .../resolve_job_run_file_triggers_test.go | 40 ++++ 9 files changed, 287 insertions(+), 51 deletions(-) create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_glob/.gitignore create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_glob/mysubdir/a.txt diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/.gitignore b/acceptance/bundle/resources/job_runs/on_file_change_glob/.gitignore new file mode 100644 index 00000000000..57433d43f47 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/.gitignore @@ -0,0 +1,2 @@ +# Matches migrations/* but sync skips it, so a content edit must not re-fire. +migrations/ignored.txt diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/databricks.yml b/acceptance/bundle/resources/job_runs/on_file_change_glob/databricks.yml index 3b148538f38..d47fcf44049 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/databricks.yml +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/databricks.yml @@ -16,3 +16,4 @@ resources: lifecycle: triggers: - on_file_change: migrations/* + - on_file_change: mysubdir/*.txt diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/mysubdir/a.txt b/acceptance/bundle/resources/job_runs/on_file_change_glob/mysubdir/a.txt new file mode 100644 index 00000000000..62e0af52c19 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/mysubdir/a.txt @@ -0,0 +1 @@ +sub diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/out.test.toml b/acceptance/bundle/resources/job_runs/on_file_change_glob/out.test.toml index 0938e678987..57b0f616850 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/out.test.toml +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/out.test.toml @@ -1,2 +1,3 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt index 798b459a9bd..d424482b961 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt @@ -6,7 +6,7 @@ Output from job_runs.my_run: id=[MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[MY_ Output from job_runs.my_run: id=[MY_RUN_ID]: SUCCESS Created job_runs.my_run Created jobs.my_job -Files: 8 uploaded, 0 deleted +Files: 10 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged >>> read_id.py my_job @@ -23,12 +23,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged } === editing one matched file re-fires ->>> [CLI] bundle plan -recreate job_runs.my_run - -Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged - ->>> changes +>>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { "lifecycle": { "action": "recreate", @@ -37,15 +32,17 @@ Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged "triggers": { "on_file_change": { "migrations/001.txt": "[FILE_HASH][0]", - "migrations/002.txt": "[FILE_HASH][1]" + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][2]" } } }, "new": { "triggers": { "on_file_change": { - "migrations/001.txt": "[FILE_HASH][2]", - "migrations/002.txt": "[FILE_HASH][1]" + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][2]" } } } @@ -54,7 +51,7 @@ Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged "action": "recreate", "reason": "immutable", "old": "[FILE_HASH][0]", - "new": "[FILE_HASH][2]" + "new": "[FILE_HASH][3]" }, "result_state": { "action": "skip", @@ -64,13 +61,35 @@ Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged } } -=== a new matching file re-fires ->>> [CLI] bundle plan -recreate job_runs.my_run +=== bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-glob/default/files... +Output from job_runs.my_run: id=[MY_RUN_ID_2]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_2]?o=[NUMID] +Output from job_runs.my_run: id=[MY_RUN_ID_2]: SUCCESS +Recreated job_runs.my_run +Files: 4 uploaded, 0 deleted +Resources: 1 created, 0 changed, 1 deleted, 1 unchanged + +>>> print_requests.py --keep //jobs/runs/delete +{ + "method": "POST", + "path": "/api/2.2/jobs/runs/delete", + "body": { + "run_id": [MY_RUN_ID] + } +} -Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged +>>> print_requests.py //jobs/run-now +{ + "method": "POST", + "path": "/api/2.2/jobs/run-now", + "body": { + "idempotency_token": "[UUID]", + "job_id": [MY_JOB_ID] + } +} ->>> changes +=== editing a file in a subdirectory re-fires +>>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { "lifecycle": { "action": "recreate", @@ -78,17 +97,58 @@ Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged "old": { "triggers": { "on_file_change": { - "migrations/001.txt": "[FILE_HASH][0]", - "migrations/002.txt": "[FILE_HASH][1]" + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][2]" } } }, "new": { "triggers": { "on_file_change": { - "migrations/001.txt": "[FILE_HASH][0]", + "migrations/001.txt": "[FILE_HASH][3]", "migrations/002.txt": "[FILE_HASH][1]", - "migrations/003.txt": "[FILE_HASH][3]" + "mysubdir/a.txt": "[FILE_HASH][4]" + } + } + } + }, + "lifecycle.triggers.on_file_change['mysubdir/a.txt']": { + "action": "recreate", + "reason": "immutable", + "old": "[FILE_HASH][2]", + "new": "[FILE_HASH][4]" + }, + "result_state": { + "action": "skip", + "reason": "remote_already_set", + "new": "SUCCESS", + "remote": "SUCCESS" + } +} + +=== a new matching file re-fires +>>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json +{ + "lifecycle": { + "action": "recreate", + "reason": "immutable", + "old": { + "triggers": { + "on_file_change": { + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][2]" + } + } + }, + "new": { + "triggers": { + "on_file_change": { + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "migrations/003.txt": "[FILE_HASH][5]", + "mysubdir/a.txt": "[FILE_HASH][2]" } } } @@ -96,7 +156,7 @@ Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged "lifecycle.triggers.on_file_change['migrations/003.txt']": { "action": "recreate", "reason": "immutable", - "new": "[FILE_HASH][3]" + "new": "[FILE_HASH][5]" }, "result_state": { "action": "skip", @@ -107,12 +167,7 @@ Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged } === removing a matched file re-fires ->>> [CLI] bundle plan -recreate job_runs.my_run - -Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged - ->>> changes +>>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { "lifecycle": { "action": "recreate", @@ -120,15 +175,17 @@ Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged "old": { "triggers": { "on_file_change": { - "migrations/001.txt": "[FILE_HASH][0]", - "migrations/002.txt": "[FILE_HASH][1]" + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][2]" } } }, "new": { "triggers": { "on_file_change": { - "migrations/001.txt": "[FILE_HASH][0]" + "migrations/001.txt": "[FILE_HASH][3]", + "mysubdir/a.txt": "[FILE_HASH][2]" } } } @@ -147,12 +204,74 @@ Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged } === editing a file below the glob does not re-fire +>>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json +{ + "lifecycle": { + "action": "skip", + "reason": "missing_in_remote", + "old": { + "triggers": { + "on_file_change": { + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][2]" + } + } + }, + "new": { + "triggers": { + "on_file_change": { + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][2]" + } + } + } + }, + "result_state": { + "action": "skip", + "reason": "remote_already_set", + "new": "SUCCESS", + "remote": "SUCCESS" + } +} + +=== editing a gitignored file does not re-fire >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged === ** is not recursive: it matches the same files as * ->>> [CLI] bundle plan -Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged +>>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json +{ + "lifecycle": { + "action": "skip", + "reason": "missing_in_remote", + "old": { + "triggers": { + "on_file_change": { + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][2]" + } + } + }, + "new": { + "triggers": { + "on_file_change": { + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][2]" + } + } + } + }, + "result_state": { + "action": "skip", + "reason": "remote_already_set", + "new": "SUCCESS", + "remote": "SUCCESS" + } +} >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/script b/acceptance/bundle/resources/job_runs/on_file_change_glob/script index 8feeebcd067..983cb9ac401 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/script +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/script @@ -4,8 +4,10 @@ cleanup() { } trap cleanup EXIT +# Plan once, print JSON changes, leave tmp.plan.json for readplanarg. changes() { - $CLI bundle plan -o json | jq '.plan["resources.job_runs.my_run"].changes' + $CLI bundle plan -o json > tmp.plan.json + trace jq '.plan["resources.job_runs.my_run"].changes' tmp.plan.json } title "deploy hashes the matched files" @@ -14,34 +16,45 @@ trace read_id.py my_job read_id.py my_run > /dev/null trace print_requests.py //jobs/run-now -# Each phase below plans against the state deployed above and then restores the -# tree, so every plan is compared against the same baseline. - title "editing one matched file re-fires" update_file.py migrations/001.txt "one" "one-edited" -trace $CLI bundle plan -trace changes -update_file.py migrations/001.txt "one-edited" "one" +changes +# The deploy is not traced: readplanarg makes the command line differ per variant. +title "bundle deploy\n" +$CLI bundle deploy $(readplanarg tmp.plan.json) +read_id.py my_run > /dev/null +trace print_requests.py --keep //jobs/runs/delete +trace print_requests.py //jobs/run-now + +title "editing a file in a subdirectory re-fires" +# printf keeps LF on Windows, where update_file.py's text-mode rewrite would +# restore CRLF and leave the file hashing differently than the deploy above. +printf 'sub-edited\n' > mysubdir/a.txt +changes +printf 'sub\n' > mysubdir/a.txt title "a new matching file re-fires" printf 'three\n' > migrations/003.txt -trace $CLI bundle plan -trace changes +changes rm migrations/003.txt title "removing a matched file re-fires" mv migrations/002.txt tmp.002.txt -trace $CLI bundle plan -trace changes +changes mv tmp.002.txt migrations/002.txt title "editing a file below the glob does not re-fire" # migrations/* matches the archive directory itself, which is skipped: the glob # does not descend into it. update_file.py migrations/archive/003.txt "three" "three-edited" -trace $CLI bundle plan +changes update_file.py migrations/archive/003.txt "three-edited" "three" +title "editing a gitignored file does not re-fire" +update_file.py migrations/ignored.txt "ignored" "ignored-edited" +trace $CLI bundle plan +update_file.py migrations/ignored.txt "ignored-edited" "ignored" + title "** is not recursive: it matches the same files as *" update_file.py databricks.yml "migrations/*" "migrations/**" -trace $CLI bundle plan +changes diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml b/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml index 1c349066b94..3516572564d 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml @@ -1,4 +1,8 @@ -Ignore = ["tmp.002.txt"] +# Deploy both by re-planning and from a plan saved on disk, so the hashes +# computed during planning survive plan serialization. +EnvMatrix.READPLAN = ["", "1"] + +Ignore = ["tmp.plan.json", "tmp.002.txt"] # See on_file_change/test.toml: mask content hashes before the parent's # `\d{8,}` rule can mangle them, keeping distinct hashes distinguishable. diff --git a/bundle/config/mutator/resolve_job_run_file_triggers.go b/bundle/config/mutator/resolve_job_run_file_triggers.go index ead17c59e17..d74b9841c57 100644 --- a/bundle/config/mutator/resolve_job_run_file_triggers.go +++ b/bundle/config/mutator/resolve_job_run_file_triggers.go @@ -13,6 +13,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" + libsync "github.com/databricks/cli/libs/sync" ) // missingFileHash marks a pattern with no matching file so appear/disappear recreates. @@ -30,8 +31,13 @@ func (*resolveJobRunFileTriggers) Name() string { return "ResolveJobRunFileTriggers" } -func (*resolveJobRunFileTriggers) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { +func (*resolveJobRunFileTriggers) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { var diags diag.Diagnostics + syncable, d := syncableRelPaths(ctx, b) + if d.HasError() { + return d + } + diags = diags.Extend(d) for name, jr := range b.Config.Resources.JobRuns { if jr == nil || jr.Lifecycle == nil { continue @@ -42,7 +48,7 @@ func (*resolveJobRunFileTriggers) Apply(_ context.Context, b *bundle.Bundle) dia continue } path := fmt.Sprintf("resources.job_runs.%s.lifecycle.triggers[%d].on_file_change", name, i) - hashes, d := resolveFileTrigger(b, path, strings.TrimSpace(*t.OnFileChange)) + hashes, d := resolveFileTrigger(b, path, strings.TrimSpace(*t.OnFileChange), syncable) diags = diags.Extend(d) maps.Copy(out, hashes) } @@ -55,7 +61,42 @@ func (*resolveJobRunFileTriggers) Apply(_ context.Context, b *bundle.Bundle) dia return diags } -func resolveFileTrigger(b *bundle.Bundle, loc, pattern string) (map[string]string, diag.Diagnostics) { +// syncableRelPaths is the set of relative paths sync would upload. +func syncableRelPaths(ctx context.Context, b *bundle.Bundle) (map[string]struct{}, diag.Diagnostics) { + var diags diag.Diagnostics + needs := false + for _, jr := range b.Config.Resources.JobRuns { + if jr != nil && jr.HasOnFileChange() { + needs = true + break + } + } + if !needs { + return nil, diags + } + + fl, err := libsync.NewFileList(ctx, b.WorktreeRoot, b.SyncRoot, b.Config.Sync.Paths, b.Config.Sync.Include, b.Config.Sync.Exclude) + if err != nil { + return nil, diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: list sync files: %s", err), + }) + } + files, err := fl.Files(ctx) + if err != nil { + return nil, diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: list sync files: %s", err), + }) + } + out := make(map[string]struct{}, len(files)) + for _, f := range files { + out[filepath.ToSlash(f.Relative)] = struct{}{} + } + return out, diags +} + +func resolveFileTrigger(b *bundle.Bundle, loc, pattern string, syncable map[string]struct{}) (map[string]string, diag.Diagnostics) { var diags diag.Diagnostics out := make(map[string]string) localPattern := filepath.FromSlash(pattern) @@ -84,6 +125,7 @@ func resolveFileTrigger(b *bundle.Bundle, loc, pattern string) (map[string]strin }) } regularMatches := 0 + ignoredMatches := 0 sawNonRegular := false for _, match := range matches { info, err := os.Stat(match) @@ -99,7 +141,6 @@ func resolveFileTrigger(b *bundle.Bundle, loc, pattern string) (map[string]strin sawNonRegular = true continue } - regularMatches++ rel, err := filepath.Rel(b.SyncRootPath, match) if err != nil || !filepath.IsLocal(rel) { diags = diags.Append(diag.Diagnostic{ @@ -109,6 +150,12 @@ func resolveFileTrigger(b *bundle.Bundle, loc, pattern string) (map[string]strin }) continue } + // Same membership as sync: .gitignore and sync.exclude drop a glob match. + if _, ok := syncable[filepath.ToSlash(rel)]; !ok { + ignoredMatches++ + continue + } + regularMatches++ hash, err := hashFile(match) if err != nil { diags = diags.Append(diag.Diagnostic{ @@ -122,13 +169,21 @@ func resolveFileTrigger(b *bundle.Bundle, loc, pattern string) (map[string]strin } // A directory-only match would otherwise leave ResolvedFileTriggers empty // and silently disarm the trigger while config still sets on_file_change. - if regularMatches == 0 && sawNonRegular { + if regularMatches == 0 && sawNonRegular && ignoredMatches == 0 { diags = diags.Append(diag.Diagnostic{ Severity: diag.Error, Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: pattern %q matches no regular files", pattern), Locations: b.Config.GetLocations(loc), }) } + if len(out) == 0 && ignoredMatches > 0 { + out[filepath.ToSlash(pattern)] = missingFileHash + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Warning, + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: no files match %q", pattern), + Locations: b.Config.GetLocations(loc), + }) + } return out, diags } diff --git a/bundle/config/mutator/resolve_job_run_file_triggers_test.go b/bundle/config/mutator/resolve_job_run_file_triggers_test.go index 3ec663c89bd..344fed25657 100644 --- a/bundle/config/mutator/resolve_job_run_file_triggers_test.go +++ b/bundle/config/mutator/resolve_job_run_file_triggers_test.go @@ -12,6 +12,7 @@ import ( "github.com/databricks/cli/bundle/config/mutator" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/vfs" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -85,7 +86,10 @@ func TestResolveJobRunFileTriggers(t *testing.T) { patB := "subdir/*.py" b := &bundle.Bundle{ SyncRootPath: dir, + SyncRoot: vfs.MustNew(dir), + WorktreeRoot: vfs.MustNew(dir), Config: config.Root{ + Sync: config.Sync{Paths: []string{"."}}, Resources: config.Resources{ JobRuns: map[string]*resources.JobRun{ "my_run": { @@ -135,6 +139,38 @@ func TestResolveJobRunFileTriggers(t *testing.T) { assert.Empty(t, b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers) }) + t.Run("skips gitignored files", func(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, ".gitignore"), []byte("skip.txt\n"), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "keep.txt"), []byte("keep"), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "skip.txt"), []byte("skip"), 0o644)) + + b := bundleWithFileTrigger(dir, "*.txt") + diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) + require.False(t, diags.HasError()) + + hashes := b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers + require.Len(t, hashes, 1) + assert.Equal(t, contentHash("keep"), hashes["keep.txt"]) + assert.NotContains(t, hashes, "skip.txt") + }) + + t.Run("skips sync.exclude files", func(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "keep.txt"), []byte("keep"), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "skip.txt"), []byte("skip"), 0o644)) + + b := bundleWithFileTrigger(dir, "*.txt") + b.Config.Sync.Exclude = []string{"skip.txt"} + diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) + require.False(t, diags.HasError()) + + hashes := b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers + require.Len(t, hashes, 1) + assert.Equal(t, contentHash("keep"), hashes["keep.txt"]) + assert.NotContains(t, hashes, "skip.txt") + }) + t.Run("trims pattern whitespace", func(t *testing.T) { dir := t.TempDir() require.NoError(t, os.WriteFile(filepath.Join(dir, "seed.txt"), []byte("v1"), 0o644)) @@ -148,9 +184,13 @@ func TestResolveJobRunFileTriggers(t *testing.T) { } func bundleWithFileTrigger(syncRoot, pattern string) *bundle.Bundle { + root := vfs.MustNew(syncRoot) return &bundle.Bundle{ SyncRootPath: syncRoot, + SyncRoot: root, + WorktreeRoot: root, Config: config.Root{ + Sync: config.Sync{Paths: []string{"."}}, Resources: config.Resources{ JobRuns: map[string]*resources.JobRun{ "my_run": { From 4bf37e75b29eaa41bb2d3dc472064bc273067db0 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Wed, 19 Aug 2026 20:10:03 +0000 Subject: [PATCH 32/75] acc: force-add gitignored on_file_change fixture The glob test edits migrations/ignored.txt, but the test .gitignore excluded it from the commit so CI clones would miss the file. --- .../job_runs/on_file_change_glob/migrations/ignored.txt | 1 + 1 file changed, 1 insertion(+) create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/ignored.txt diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/ignored.txt b/acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/ignored.txt new file mode 100644 index 00000000000..ea10ec85c10 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/ignored.txt @@ -0,0 +1 @@ +ignored From 4a05a4c31052f77f8498984a714505d9297418f7 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Thu, 20 Aug 2026 08:11:43 +0000 Subject: [PATCH 33/75] acc: drop READPLAN from the on_file_change glob test Plan serialization is already covered by on_file_change; the glob cases only assert the hash map. --- .../resources/job_runs/on_file_change_glob/out.test.toml | 1 - .../bundle/resources/job_runs/on_file_change_glob/output.txt | 2 +- .../bundle/resources/job_runs/on_file_change_glob/script | 5 +---- .../bundle/resources/job_runs/on_file_change_glob/test.toml | 4 ---- 4 files changed, 2 insertions(+), 10 deletions(-) diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/out.test.toml b/acceptance/bundle/resources/job_runs/on_file_change_glob/out.test.toml index 57b0f616850..0938e678987 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/out.test.toml +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/out.test.toml @@ -1,3 +1,2 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt index d424482b961..bd7d08557f5 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt @@ -61,7 +61,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged } } -=== bundle deploy +>>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-glob/default/files... Output from job_runs.my_run: id=[MY_RUN_ID_2]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_2]?o=[NUMID] Output from job_runs.my_run: id=[MY_RUN_ID_2]: SUCCESS diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/script b/acceptance/bundle/resources/job_runs/on_file_change_glob/script index 983cb9ac401..23390d23a55 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/script +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/script @@ -4,7 +4,6 @@ cleanup() { } trap cleanup EXIT -# Plan once, print JSON changes, leave tmp.plan.json for readplanarg. changes() { $CLI bundle plan -o json > tmp.plan.json trace jq '.plan["resources.job_runs.my_run"].changes' tmp.plan.json @@ -19,9 +18,7 @@ trace print_requests.py //jobs/run-now title "editing one matched file re-fires" update_file.py migrations/001.txt "one" "one-edited" changes -# The deploy is not traced: readplanarg makes the command line differ per variant. -title "bundle deploy\n" -$CLI bundle deploy $(readplanarg tmp.plan.json) +trace $CLI bundle deploy read_id.py my_run > /dev/null trace print_requests.py --keep //jobs/runs/delete trace print_requests.py //jobs/run-now diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml b/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml index 3516572564d..d372fd54eae 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml @@ -1,7 +1,3 @@ -# Deploy both by re-planning and from a plan saved on disk, so the hashes -# computed during planning survive plan serialization. -EnvMatrix.READPLAN = ["", "1"] - Ignore = ["tmp.plan.json", "tmp.002.txt"] # See on_file_change/test.toml: mask content hashes before the parent's From 59d34ca288c307537858a550bcabfdb80dfaa3ff Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Thu, 20 Aug 2026 12:34:44 +0000 Subject: [PATCH 34/75] acc: restore READPLAN on the on_file_change glob test Cover deploying glob hashes from a saved plan, not only from a live re-plan. --- .../resources/job_runs/on_file_change_glob/out.test.toml | 1 + .../bundle/resources/job_runs/on_file_change_glob/output.txt | 2 +- .../bundle/resources/job_runs/on_file_change_glob/script | 5 ++++- .../bundle/resources/job_runs/on_file_change_glob/test.toml | 4 ++++ 4 files changed, 10 insertions(+), 2 deletions(-) diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/out.test.toml b/acceptance/bundle/resources/job_runs/on_file_change_glob/out.test.toml index 0938e678987..57b0f616850 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/out.test.toml +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/out.test.toml @@ -1,2 +1,3 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt index bd7d08557f5..d424482b961 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt @@ -61,7 +61,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged } } ->>> [CLI] bundle deploy +=== bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-glob/default/files... Output from job_runs.my_run: id=[MY_RUN_ID_2]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_2]?o=[NUMID] Output from job_runs.my_run: id=[MY_RUN_ID_2]: SUCCESS diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/script b/acceptance/bundle/resources/job_runs/on_file_change_glob/script index 23390d23a55..983cb9ac401 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/script +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/script @@ -4,6 +4,7 @@ cleanup() { } trap cleanup EXIT +# Plan once, print JSON changes, leave tmp.plan.json for readplanarg. changes() { $CLI bundle plan -o json > tmp.plan.json trace jq '.plan["resources.job_runs.my_run"].changes' tmp.plan.json @@ -18,7 +19,9 @@ trace print_requests.py //jobs/run-now title "editing one matched file re-fires" update_file.py migrations/001.txt "one" "one-edited" changes -trace $CLI bundle deploy +# The deploy is not traced: readplanarg makes the command line differ per variant. +title "bundle deploy\n" +$CLI bundle deploy $(readplanarg tmp.plan.json) read_id.py my_run > /dev/null trace print_requests.py --keep //jobs/runs/delete trace print_requests.py //jobs/run-now diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml b/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml index d372fd54eae..3516572564d 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml @@ -1,3 +1,7 @@ +# Deploy both by re-planning and from a plan saved on disk, so the hashes +# computed during planning survive plan serialization. +EnvMatrix.READPLAN = ["", "1"] + Ignore = ["tmp.plan.json", "tmp.002.txt"] # See on_file_change/test.toml: mask content hashes before the parent's From f344a7e51f76c081c23979912621faaefe2b32ef Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Thu, 20 Aug 2026 14:08:42 +0000 Subject: [PATCH 35/75] job_runs: address on_file_change review comments Keep the gitignore fixture as dot_gitignore so it does not affect the repo while developing, drop unit tests that duplicate acceptance coverage, and allocate trigger state once when a trigger is armed. --- .../{.gitignore => dot_gitignore} | 0 .../job_runs/on_file_change_glob/script | 3 + .../job_runs/on_file_change_glob/test.toml | 2 +- .../resolve_job_run_file_triggers_test.go | 139 +----------------- bundle/direct/dresources/job_run.go | 23 +-- bundle/direct/dresources/job_run_test.go | 36 ++--- 6 files changed, 23 insertions(+), 180 deletions(-) rename acceptance/bundle/resources/job_runs/on_file_change_glob/{.gitignore => dot_gitignore} (100%) diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/.gitignore b/acceptance/bundle/resources/job_runs/on_file_change_glob/dot_gitignore similarity index 100% rename from acceptance/bundle/resources/job_runs/on_file_change_glob/.gitignore rename to acceptance/bundle/resources/job_runs/on_file_change_glob/dot_gitignore diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/script b/acceptance/bundle/resources/job_runs/on_file_change_glob/script index 983cb9ac401..dd737f242b5 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/script +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/script @@ -1,3 +1,6 @@ +# Fixture is named dot_gitignore so a real .gitignore does not affect git while developing. +mv dot_gitignore .gitignore + cleanup() { trace $CLI bundle destroy --auto-approve rm -f out.requests.txt diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml b/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml index 3516572564d..98c98631e5b 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml @@ -2,7 +2,7 @@ # computed during planning survive plan serialization. EnvMatrix.READPLAN = ["", "1"] -Ignore = ["tmp.plan.json", "tmp.002.txt"] +Ignore = ["tmp.plan.json", "tmp.002.txt", ".gitignore"] # See on_file_change/test.toml: mask content hashes before the parent's # `\d{8,}` rule can mangle them, keeping distinct hashes distinguishable. diff --git a/bundle/config/mutator/resolve_job_run_file_triggers_test.go b/bundle/config/mutator/resolve_job_run_file_triggers_test.go index 344fed25657..1cc54eecf4f 100644 --- a/bundle/config/mutator/resolve_job_run_file_triggers_test.go +++ b/bundle/config/mutator/resolve_job_run_file_triggers_test.go @@ -11,14 +11,13 @@ import ( "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/mutator" "github.com/databricks/cli/bundle/config/resources" - "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/vfs" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestResolveJobRunFileTriggers(t *testing.T) { - t.Run("matches files and fills hashes", func(t *testing.T) { + t.Run("hashes file contents with sha256", func(t *testing.T) { dir := t.TempDir() require.NoError(t, os.WriteFile(filepath.Join(dir, "a.txt"), []byte("hello"), 0o644)) require.NoError(t, os.WriteFile(filepath.Join(dir, "b.txt"), []byte("world"), 0o644)) @@ -35,142 +34,6 @@ func TestResolveJobRunFileTriggers(t *testing.T) { assert.Equal(t, contentHash("world"), hashes["b.txt"]) }) - t.Run("no matches warns and stores empty hash", func(t *testing.T) { - dir := t.TempDir() - pattern := "missing.txt" - b := bundleWithFileTrigger(dir, pattern) - - diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) - require.False(t, diags.HasError()) - require.Len(t, diags, 1) - assert.Equal(t, diag.Warning, diags[0].Severity) - assert.Contains(t, diags[0].Summary, `no files match "missing.txt"`) - - hashes := b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers - require.Len(t, hashes, 1) - assert.Empty(t, hashes["missing.txt"]) - }) - - t.Run("no file triggers is a no-op", func(t *testing.T) { - dir := t.TempDir() - on := true - b := &bundle.Bundle{ - SyncRootPath: dir, - Config: config.Root{ - Resources: config.Resources{ - JobRuns: map[string]*resources.JobRun{ - "my_run": { - Lifecycle: &resources.JobRunLifecycle{ - Triggers: []resources.JobRunTrigger{ - {OnBundleDeploy: &on}, - }, - }, - }, - }, - }, - }, - } - - diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) - assert.Empty(t, diags) - assert.Nil(t, b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers) - }) - - t.Run("multiple patterns merge into one map", func(t *testing.T) { - dir := t.TempDir() - require.NoError(t, os.WriteFile(filepath.Join(dir, "a.txt"), []byte("aaa"), 0o644)) - require.NoError(t, os.MkdirAll(filepath.Join(dir, "subdir"), 0o755)) - require.NoError(t, os.WriteFile(filepath.Join(dir, "subdir", "x.py"), []byte("bbb"), 0o644)) - - patA := "a.txt" - patB := "subdir/*.py" - b := &bundle.Bundle{ - SyncRootPath: dir, - SyncRoot: vfs.MustNew(dir), - WorktreeRoot: vfs.MustNew(dir), - Config: config.Root{ - Sync: config.Sync{Paths: []string{"."}}, - Resources: config.Resources{ - JobRuns: map[string]*resources.JobRun{ - "my_run": { - Lifecycle: &resources.JobRunLifecycle{ - Triggers: []resources.JobRunTrigger{ - {OnFileChange: &patA}, - {OnFileChange: &patB}, - }, - }, - }, - }, - }, - }, - } - - diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) - require.False(t, diags.HasError()) - - hashes := b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers - require.Len(t, hashes, 2) - assert.Equal(t, contentHash("aaa"), hashes["a.txt"]) - assert.Equal(t, contentHash("bbb"), hashes["subdir/x.py"]) - }) - - t.Run("pattern outside sync root is an error", func(t *testing.T) { - dir := t.TempDir() - pattern := "../outside.txt" - b := bundleWithFileTrigger(dir, pattern) - - diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) - require.True(t, diags.HasError()) - require.Len(t, diags, 1) - assert.Contains(t, diags[0].Summary, `not under the sync root`) - assert.Empty(t, b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers) - }) - - t.Run("directory-only match is an error", func(t *testing.T) { - dir := t.TempDir() - require.NoError(t, os.MkdirAll(filepath.Join(dir, "migrations"), 0o755)) - pattern := "migrations" - b := bundleWithFileTrigger(dir, pattern) - - diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) - require.True(t, diags.HasError()) - require.Len(t, diags, 1) - assert.Contains(t, diags[0].Summary, `matches no regular files`) - assert.Empty(t, b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers) - }) - - t.Run("skips gitignored files", func(t *testing.T) { - dir := t.TempDir() - require.NoError(t, os.WriteFile(filepath.Join(dir, ".gitignore"), []byte("skip.txt\n"), 0o644)) - require.NoError(t, os.WriteFile(filepath.Join(dir, "keep.txt"), []byte("keep"), 0o644)) - require.NoError(t, os.WriteFile(filepath.Join(dir, "skip.txt"), []byte("skip"), 0o644)) - - b := bundleWithFileTrigger(dir, "*.txt") - diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) - require.False(t, diags.HasError()) - - hashes := b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers - require.Len(t, hashes, 1) - assert.Equal(t, contentHash("keep"), hashes["keep.txt"]) - assert.NotContains(t, hashes, "skip.txt") - }) - - t.Run("skips sync.exclude files", func(t *testing.T) { - dir := t.TempDir() - require.NoError(t, os.WriteFile(filepath.Join(dir, "keep.txt"), []byte("keep"), 0o644)) - require.NoError(t, os.WriteFile(filepath.Join(dir, "skip.txt"), []byte("skip"), 0o644)) - - b := bundleWithFileTrigger(dir, "*.txt") - b.Config.Sync.Exclude = []string{"skip.txt"} - diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) - require.False(t, diags.HasError()) - - hashes := b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers - require.Len(t, hashes, 1) - assert.Equal(t, contentHash("keep"), hashes["keep.txt"]) - assert.NotContains(t, hashes, "skip.txt") - }) - t.Run("trims pattern whitespace", func(t *testing.T) { dir := t.TempDir() require.NoError(t, os.WriteFile(filepath.Join(dir, "seed.txt"), []byte("v1"), 0o644)) diff --git a/bundle/direct/dresources/job_run.go b/bundle/direct/dresources/job_run.go index 5b97cc9e22b..8117af9aa22 100644 --- a/bundle/direct/dresources/job_run.go +++ b/bundle/direct/dresources/job_run.go @@ -109,25 +109,14 @@ func (*ResourceJobRun) PrepareState(input *resources.JobRun) *JobRunState { ResultState: jobs.RunResultStateSuccess, Lifecycle: nil, } - var triggers *JobRunTriggersState - if input.HasOnBundleDeploy() { - triggers = &JobRunTriggersState{ - OnBundleDeploy: uuid.NewString(), - OnFileChange: nil, - } + if !input.HasOnBundleDeploy() && len(input.ResolvedFileTriggers) == 0 { + return state } - if len(input.ResolvedFileTriggers) > 0 { - if triggers == nil { - triggers = &JobRunTriggersState{ - OnBundleDeploy: "", - OnFileChange: nil, - } - } - triggers.OnFileChange = input.ResolvedFileTriggers - } - if triggers != nil { - state.Lifecycle = &JobRunLifecycleState{Triggers: triggers} + triggers := &JobRunTriggersState{OnFileChange: input.ResolvedFileTriggers} + if input.HasOnBundleDeploy() { + triggers.OnBundleDeploy = uuid.NewString() } + state.Lifecycle = &JobRunLifecycleState{Triggers: triggers} return state } diff --git a/bundle/direct/dresources/job_run_test.go b/bundle/direct/dresources/job_run_test.go index e80725a7eed..01c7636d90a 100644 --- a/bundle/direct/dresources/job_run_test.go +++ b/bundle/direct/dresources/job_run_test.go @@ -354,36 +354,24 @@ func TestJobRunPrepareStateRequiresSuccess(t *testing.T) { } func TestJobRunPrepareStateOnBundleDeploy(t *testing.T) { - t.Run("unset", func(t *testing.T) { - state := (&ResourceJobRun{}).PrepareState(&resources.JobRun{}) - assert.Nil(t, state.Lifecycle) - }) - - t.Run("armed", func(t *testing.T) { - on := true - input := &resources.JobRun{ - Lifecycle: &resources.JobRunLifecycle{ - Triggers: []resources.JobRunTrigger{{OnBundleDeploy: &on}}, - }, - } - first := (&ResourceJobRun{}).PrepareState(input) - require.NotNil(t, first.Lifecycle) - require.NotNil(t, first.Lifecycle.Triggers) - assert.NotEmpty(t, first.Lifecycle.Triggers.OnBundleDeploy) + on := true + input := &resources.JobRun{ + Lifecycle: &resources.JobRunLifecycle{ + Triggers: []resources.JobRunTrigger{{OnBundleDeploy: &on}}, + }, + } + first := (&ResourceJobRun{}).PrepareState(input) + require.NotNil(t, first.Lifecycle) + require.NotNil(t, first.Lifecycle.Triggers) + assert.NotEmpty(t, first.Lifecycle.Triggers.OnBundleDeploy) - second := (&ResourceJobRun{}).PrepareState(input) - assert.NotEqual(t, first.Lifecycle.Triggers.OnBundleDeploy, second.Lifecycle.Triggers.OnBundleDeploy) - }) + second := (&ResourceJobRun{}).PrepareState(input) + assert.NotEqual(t, first.Lifecycle.Triggers.OnBundleDeploy, second.Lifecycle.Triggers.OnBundleDeploy) } func TestJobRunPrepareStateOnFileChange(t *testing.T) { hashes := map[string]string{"a.txt": "abc"} - t.Run("unset", func(t *testing.T) { - state := (&ResourceJobRun{}).PrepareState(&resources.JobRun{}) - assert.Nil(t, state.Lifecycle) - }) - t.Run("armed", func(t *testing.T) { state := (&ResourceJobRun{}).PrepareState(&resources.JobRun{ ResolvedFileTriggers: hashes, From 3850eb8acedafd462fd75df81b558671b540f3b4 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Thu, 20 Aug 2026 14:16:35 +0000 Subject: [PATCH 36/75] job_runs: list every trigger field to satisfy exhaustruct The previous simplification omitted OnBundleDeploy from the struct literal, which the linter rejects. --- bundle/direct/dresources/job_run.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/bundle/direct/dresources/job_run.go b/bundle/direct/dresources/job_run.go index 8117af9aa22..ac65e5db697 100644 --- a/bundle/direct/dresources/job_run.go +++ b/bundle/direct/dresources/job_run.go @@ -112,11 +112,16 @@ func (*ResourceJobRun) PrepareState(input *resources.JobRun) *JobRunState { if !input.HasOnBundleDeploy() && len(input.ResolvedFileTriggers) == 0 { return state } - triggers := &JobRunTriggersState{OnFileChange: input.ResolvedFileTriggers} + onBundleDeploy := "" if input.HasOnBundleDeploy() { - triggers.OnBundleDeploy = uuid.NewString() + onBundleDeploy = uuid.NewString() + } + state.Lifecycle = &JobRunLifecycleState{ + Triggers: &JobRunTriggersState{ + OnBundleDeploy: onBundleDeploy, + OnFileChange: input.ResolvedFileTriggers, + }, } - state.Lifecycle = &JobRunLifecycleState{Triggers: triggers} return state } From b27c4ff45c172516b784c14b60e4ea8257bdc631 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Fri, 21 Aug 2026 12:31:38 +0000 Subject: [PATCH 37/75] job_runs: keep empty nested lifecycle on state and remote Always persist the same lifecycle.triggers shape so trigger fingerprints compare at the same paths. Wrap on_file_change hashes in files so dropping the trigger updates instead of recreating. --- acceptance/bundle/refschema/out.fields.txt | 11 +- .../resources/job_runs/failed_run/output.txt | 10 ++ .../job_runs/interrupted_run/output.txt | 5 + .../job_runs/on_bundle_deploy/output.txt | 22 +-- .../job_runs/on_file_change/output.txt | 32 +--- .../job_runs/on_file_change_glob/output.txt | 144 ++++++------------ .../resources/job_runs/redeploy/output.txt | 10 ++ bundle/direct/dresources/job_run.go | 86 +++++++---- bundle/direct/dresources/job_run_test.go | 96 +++++++++--- bundle/direct/dresources/resources.yml | 3 + bundle/direct/dresources/type_test.go | 4 - 11 files changed, 228 insertions(+), 195 deletions(-) diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index 871152e6a9c..3c681d4b199 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -865,15 +865,16 @@ resources.job_runs.*.jar_params[*] string ALL resources.job_runs.*.job_id int64 ALL resources.job_runs.*.job_parameters map[string]string ALL resources.job_runs.*.job_parameters.* string ALL -resources.job_runs.*.lifecycle *dresources.JobRunLifecycleState STATE +resources.job_runs.*.lifecycle *dresources.JobRunLifecycleState REMOTE STATE resources.job_runs.*.lifecycle *resources.JobRunLifecycle INPUT resources.job_runs.*.lifecycle resources.Lifecycle INPUT resources.job_runs.*.lifecycle.prevent_destroy bool INPUT -resources.job_runs.*.lifecycle.triggers *dresources.JobRunTriggersState STATE +resources.job_runs.*.lifecycle.triggers *dresources.JobRunTriggersState REMOTE STATE resources.job_runs.*.lifecycle.triggers []resources.JobRunTrigger INPUT -resources.job_runs.*.lifecycle.triggers.on_bundle_deploy string STATE -resources.job_runs.*.lifecycle.triggers.on_file_change map[string]string STATE -resources.job_runs.*.lifecycle.triggers.on_file_change.* string STATE +resources.job_runs.*.lifecycle.triggers.on_bundle_deploy string REMOTE STATE +resources.job_runs.*.lifecycle.triggers.on_file_change *dresources.JobRunFileTriggerState REMOTE STATE +resources.job_runs.*.lifecycle.triggers.on_file_change.files map[string]string REMOTE STATE +resources.job_runs.*.lifecycle.triggers.on_file_change.files.* string REMOTE STATE resources.job_runs.*.lifecycle.triggers[*] resources.JobRunTrigger INPUT resources.job_runs.*.lifecycle.triggers[*].on_bundle_deploy *bool INPUT resources.job_runs.*.lifecycle.triggers[*].on_file_change *string INPUT diff --git a/acceptance/bundle/resources/job_runs/failed_run/output.txt b/acceptance/bundle/resources/job_runs/failed_run/output.txt index f8b8dbf4398..0c6bddb0d9b 100644 --- a/acceptance/bundle/resources/job_runs/failed_run/output.txt +++ b/acceptance/bundle/resources/job_runs/failed_run/output.txt @@ -27,11 +27,21 @@ Files: 7 uploaded, 0 deleted "new_state": { "value": { "job_id": [MY_JOB_ID], + "lifecycle": { + "triggers": { + "on_file_change": {} + } + }, "result_state": "SUCCESS" } }, "remote_state": { "job_id": [MY_JOB_ID], + "lifecycle": { + "triggers": { + "on_file_change": {} + } + }, "result_state": "FAILED", "run_id": [MY_RUN_ID], "run_name": "test-job-[UNIQUE_NAME]", diff --git a/acceptance/bundle/resources/job_runs/interrupted_run/output.txt b/acceptance/bundle/resources/job_runs/interrupted_run/output.txt index 2e1d9cacd7a..da185c3704d 100644 --- a/acceptance/bundle/resources/job_runs/interrupted_run/output.txt +++ b/acceptance/bundle/resources/job_runs/interrupted_run/output.txt @@ -42,6 +42,11 @@ Exit code: 1 "action": "skip", "remote_state": { "job_id": [MY_JOB_ID], + "lifecycle": { + "triggers": { + "on_file_change": {} + } + }, "run_id": [MY_RUN_ID], "run_name": "my-job", "run_page_url": "[DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?o=[NUMID]", diff --git a/acceptance/bundle/resources/job_runs/on_bundle_deploy/output.txt b/acceptance/bundle/resources/job_runs/on_bundle_deploy/output.txt index 39123f3370f..6cd8d44a418 100644 --- a/acceptance/bundle/resources/job_runs/on_bundle_deploy/output.txt +++ b/acceptance/bundle/resources/job_runs/on_bundle_deploy/output.txt @@ -30,20 +30,6 @@ Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle": { - "action": "recreate", - "reason": "immutable", - "old": { - "triggers": { - "on_bundle_deploy": "[UUID]" - } - }, - "new": { - "triggers": { - "on_bundle_deploy": "[UUID]" - } - } - }, "lifecycle.triggers.on_bundle_deploy": { "action": "recreate", "reason": "immutable", @@ -108,14 +94,10 @@ Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle": { + "lifecycle.triggers.on_bundle_deploy": { "action": "skip", "reason": "trigger removed", - "old": { - "triggers": { - "on_bundle_deploy": "[UUID]" - } - } + "old": "[UUID]" }, "result_state": { "action": "skip", diff --git a/acceptance/bundle/resources/job_runs/on_file_change/output.txt b/acceptance/bundle/resources/job_runs/on_file_change/output.txt index d03307b90f2..9b662e1936d 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change/output.txt @@ -52,25 +52,17 @@ Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle": { + "lifecycle.triggers.on_file_change.files": { "action": "recreate", "reason": "immutable", "old": { - "triggers": { - "on_file_change": { - "seed.txt": "[FILE_HASH][0]" - } - } + "seed.txt": "[FILE_HASH][0]" }, "new": { - "triggers": { - "on_file_change": { - "seed.txt": "[FILE_HASH][1]" - } - } + "seed.txt": "[FILE_HASH][1]" } }, - "lifecycle.triggers.on_file_change['seed.txt']": { + "lifecycle.triggers.on_file_change.files['seed.txt']": { "action": "recreate", "reason": "immutable", "old": "[FILE_HASH][0]", @@ -125,25 +117,17 @@ Warning: lifecycle.triggers.on_file_change: no files match "seed.txt" in databricks.yml:18:29 { - "lifecycle": { + "lifecycle.triggers.on_file_change.files": { "action": "recreate", "reason": "immutable", "old": { - "triggers": { - "on_file_change": { - "seed.txt": "[FILE_HASH][1]" - } - } + "seed.txt": "[FILE_HASH][1]" }, "new": { - "triggers": { - "on_file_change": { - "seed.txt": "" - } - } + "seed.txt": "" } }, - "lifecycle.triggers.on_file_change['seed.txt']": { + "lifecycle.triggers.on_file_change.files['seed.txt']": { "action": "recreate", "reason": "immutable", "old": "[FILE_HASH][1]", diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt index d424482b961..5b4ef23178c 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt @@ -25,29 +25,21 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged === editing one matched file re-fires >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle": { + "lifecycle.triggers.on_file_change.files": { "action": "recreate", "reason": "immutable", "old": { - "triggers": { - "on_file_change": { - "migrations/001.txt": "[FILE_HASH][0]", - "migrations/002.txt": "[FILE_HASH][1]", - "mysubdir/a.txt": "[FILE_HASH][2]" - } - } + "migrations/001.txt": "[FILE_HASH][0]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][2]" }, "new": { - "triggers": { - "on_file_change": { - "migrations/001.txt": "[FILE_HASH][3]", - "migrations/002.txt": "[FILE_HASH][1]", - "mysubdir/a.txt": "[FILE_HASH][2]" - } - } + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][2]" } }, - "lifecycle.triggers.on_file_change['migrations/001.txt']": { + "lifecycle.triggers.on_file_change.files['migrations/001.txt']": { "action": "recreate", "reason": "immutable", "old": "[FILE_HASH][0]", @@ -91,29 +83,21 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged === editing a file in a subdirectory re-fires >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle": { + "lifecycle.triggers.on_file_change.files": { "action": "recreate", "reason": "immutable", "old": { - "triggers": { - "on_file_change": { - "migrations/001.txt": "[FILE_HASH][3]", - "migrations/002.txt": "[FILE_HASH][1]", - "mysubdir/a.txt": "[FILE_HASH][2]" - } - } + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][2]" }, "new": { - "triggers": { - "on_file_change": { - "migrations/001.txt": "[FILE_HASH][3]", - "migrations/002.txt": "[FILE_HASH][1]", - "mysubdir/a.txt": "[FILE_HASH][4]" - } - } + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][4]" } }, - "lifecycle.triggers.on_file_change['mysubdir/a.txt']": { + "lifecycle.triggers.on_file_change.files['mysubdir/a.txt']": { "action": "recreate", "reason": "immutable", "old": "[FILE_HASH][2]", @@ -130,30 +114,22 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged === a new matching file re-fires >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle": { + "lifecycle.triggers.on_file_change.files": { "action": "recreate", "reason": "immutable", "old": { - "triggers": { - "on_file_change": { - "migrations/001.txt": "[FILE_HASH][3]", - "migrations/002.txt": "[FILE_HASH][1]", - "mysubdir/a.txt": "[FILE_HASH][2]" - } - } + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][2]" }, "new": { - "triggers": { - "on_file_change": { - "migrations/001.txt": "[FILE_HASH][3]", - "migrations/002.txt": "[FILE_HASH][1]", - "migrations/003.txt": "[FILE_HASH][5]", - "mysubdir/a.txt": "[FILE_HASH][2]" - } - } + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "migrations/003.txt": "[FILE_HASH][5]", + "mysubdir/a.txt": "[FILE_HASH][2]" } }, - "lifecycle.triggers.on_file_change['migrations/003.txt']": { + "lifecycle.triggers.on_file_change.files['migrations/003.txt']": { "action": "recreate", "reason": "immutable", "new": "[FILE_HASH][5]" @@ -169,28 +145,20 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged === removing a matched file re-fires >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle": { + "lifecycle.triggers.on_file_change.files": { "action": "recreate", "reason": "immutable", "old": { - "triggers": { - "on_file_change": { - "migrations/001.txt": "[FILE_HASH][3]", - "migrations/002.txt": "[FILE_HASH][1]", - "mysubdir/a.txt": "[FILE_HASH][2]" - } - } + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][2]" }, "new": { - "triggers": { - "on_file_change": { - "migrations/001.txt": "[FILE_HASH][3]", - "mysubdir/a.txt": "[FILE_HASH][2]" - } - } + "migrations/001.txt": "[FILE_HASH][3]", + "mysubdir/a.txt": "[FILE_HASH][2]" } }, - "lifecycle.triggers.on_file_change['migrations/002.txt']": { + "lifecycle.triggers.on_file_change.files['migrations/002.txt']": { "action": "recreate", "reason": "immutable", "old": "[FILE_HASH][1]" @@ -206,26 +174,18 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged === editing a file below the glob does not re-fire >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle": { + "lifecycle.triggers.on_file_change.files": { "action": "skip", - "reason": "missing_in_remote", + "reason": "not_returned_by_api", "old": { - "triggers": { - "on_file_change": { - "migrations/001.txt": "[FILE_HASH][3]", - "migrations/002.txt": "[FILE_HASH][1]", - "mysubdir/a.txt": "[FILE_HASH][2]" - } - } + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][2]" }, "new": { - "triggers": { - "on_file_change": { - "migrations/001.txt": "[FILE_HASH][3]", - "migrations/002.txt": "[FILE_HASH][1]", - "mysubdir/a.txt": "[FILE_HASH][2]" - } - } + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][2]" } }, "result_state": { @@ -243,26 +203,18 @@ Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged === ** is not recursive: it matches the same files as * >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle": { + "lifecycle.triggers.on_file_change.files": { "action": "skip", - "reason": "missing_in_remote", + "reason": "not_returned_by_api", "old": { - "triggers": { - "on_file_change": { - "migrations/001.txt": "[FILE_HASH][3]", - "migrations/002.txt": "[FILE_HASH][1]", - "mysubdir/a.txt": "[FILE_HASH][2]" - } - } + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][2]" }, "new": { - "triggers": { - "on_file_change": { - "migrations/001.txt": "[FILE_HASH][3]", - "migrations/002.txt": "[FILE_HASH][1]", - "mysubdir/a.txt": "[FILE_HASH][2]" - } - } + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][2]" } }, "result_state": { diff --git a/acceptance/bundle/resources/job_runs/redeploy/output.txt b/acceptance/bundle/resources/job_runs/redeploy/output.txt index 06129f79373..60111c98a93 100644 --- a/acceptance/bundle/resources/job_runs/redeploy/output.txt +++ b/acceptance/bundle/resources/job_runs/redeploy/output.txt @@ -59,6 +59,11 @@ Resources: "job_parameters": { "env": "prod" }, + "lifecycle": { + "triggers": { + "on_file_change": {} + } + }, "result_state": "SUCCESS" } }, @@ -67,6 +72,11 @@ Resources: "job_parameters": { "env": "dev" }, + "lifecycle": { + "triggers": { + "on_file_change": {} + } + }, "result_state": "SUCCESS", "run_id": [MY_RUN_ID], "run_name": "my-job", diff --git a/bundle/direct/dresources/job_run.go b/bundle/direct/dresources/job_run.go index ac65e5db697..815091a721a 100644 --- a/bundle/direct/dresources/job_run.go +++ b/bundle/direct/dresources/job_run.go @@ -3,6 +3,7 @@ package dresources import ( "cmp" "context" + "encoding/json" "errors" "fmt" "slices" @@ -28,10 +29,12 @@ const jobRunTimeout = 24 * time.Hour // jobRunTriggerLocalPaths are local-only fingerprints: clearing one is skip, not recreate. var jobRunTriggerLocalPaths = []string{ - "lifecycle", - "lifecycle.triggers", "lifecycle.triggers.on_bundle_deploy", - "lifecycle.triggers.on_file_change", + "lifecycle.triggers.on_file_change.files", +} + +func isJobRunTriggerPath(path string) bool { + return slices.Contains(jobRunTriggerLocalPaths, path) } // JobRunTriggersState is the persisted fingerprint of lifecycle.triggers. @@ -39,12 +42,44 @@ type JobRunTriggersState struct { // Fresh UUID each plan while armed so Old!=New forces recreate. OnBundleDeploy string `json:"on_bundle_deploy,omitempty"` // Path → content hash from ResolveJobRunFileTriggers; change to recreate. - OnFileChange map[string]string `json:"on_file_change,omitempty"` + OnFileChange *JobRunFileTriggerState `json:"on_file_change"` +} + +// JobRunFileTriggerState is always present so remote and state share one shape. +// Files is nil when the trigger is off, so dropping the trigger diffs here and +// a file appear/disappear diffs inside the map. +type JobRunFileTriggerState struct { + Files map[string]string `json:"files,omitempty"` +} + +// UnmarshalJSON accepts the wrapped map and the older path-to-hash map. +func (s *JobRunFileTriggerState) UnmarshalJSON(b []byte) error { + var raw map[string]json.RawMessage + if err := json.Unmarshal(b, &raw); err != nil { + return err + } + if files, ok := raw["files"]; ok && len(files) > 0 && files[0] == '{' { + return json.Unmarshal(files, &s.Files) + } + if len(raw) == 0 { + s.Files = nil + return nil + } + return json.Unmarshal(b, &s.Files) } -// JobRunLifecycleState holds local-only lifecycle fields persisted in state. +// JobRunLifecycleState is the local-only trigger fingerprint, also present (empty) on remote. type JobRunLifecycleState struct { - Triggers *JobRunTriggersState `json:"triggers,omitempty"` + Triggers *JobRunTriggersState `json:"triggers"` +} + +func newJobRunLifecycleState() *JobRunLifecycleState { + return &JobRunLifecycleState{ + Triggers: &JobRunTriggersState{ + OnBundleDeploy: "", + OnFileChange: &JobRunFileTriggerState{Files: nil}, + }, + } } // JobRunState is the RunNow request plus the outcome required for planning. @@ -54,12 +89,13 @@ type JobRunState struct { // Always SUCCESS during planning and cleared before persistence. ResultState jobs.RunResultState `json:"result_state,omitempty"` - // Local-only; listed in knownMissingInRemoteType. Nested under lifecycle to - // mirror config and avoid colliding with a future Jobs API field. - Lifecycle *JobRunLifecycleState `json:"lifecycle,omitempty"` + // Local-only. Nested under lifecycle to mirror config and avoid colliding + // with a future Jobs API field. + Lifecycle *JobRunLifecycleState `json:"lifecycle"` } func (s *JobRunState) UnmarshalJSON(b []byte) error { + s.Lifecycle = newJobRunLifecycleState() return marshal.Unmarshal(b, s) } @@ -76,6 +112,9 @@ type JobRunRemote struct { // compare the two. See "RemapState is a dumb copy" in README.md. ResultState jobs.RunResultState `json:"result_state,omitempty"` + // Always the empty fingerprint: GetRun does not return triggers. + Lifecycle *JobRunLifecycleState `json:"lifecycle"` + RunId int64 `json:"run_id,omitempty"` RunName string `json:"run_name,omitempty"` State *jobs.RunState `json:"state,omitempty"` @@ -86,6 +125,7 @@ type JobRunRemote struct { // Custom marshaler needed because embedded RunNow's MarshalJSON would otherwise // take over and drop the additional fields. func (s *JobRunRemote) UnmarshalJSON(b []byte) error { + s.Lifecycle = newJobRunLifecycleState() return marshal.Unmarshal(b, s) } @@ -107,20 +147,13 @@ func (*ResourceJobRun) PrepareState(input *resources.JobRun) *JobRunState { state := &JobRunState{ RunNow: input.RunNow, ResultState: jobs.RunResultStateSuccess, - Lifecycle: nil, - } - if !input.HasOnBundleDeploy() && len(input.ResolvedFileTriggers) == 0 { - return state + Lifecycle: newJobRunLifecycleState(), } - onBundleDeploy := "" if input.HasOnBundleDeploy() { - onBundleDeploy = uuid.NewString() + state.Lifecycle.Triggers.OnBundleDeploy = uuid.NewString() } - state.Lifecycle = &JobRunLifecycleState{ - Triggers: &JobRunTriggersState{ - OnBundleDeploy: onBundleDeploy, - OnFileChange: input.ResolvedFileTriggers, - }, + if len(input.ResolvedFileTriggers) > 0 { + state.Lifecycle.Triggers.OnFileChange.Files = input.ResolvedFileTriggers } return state } @@ -161,6 +194,7 @@ func makeJobRunRemote(run *jobs.Run) *JobRunRemote { ForceSendFields: nil, }, ResultState: run.State.ResultState, + Lifecycle: newJobRunLifecycleState(), RunId: run.RunId, RunName: run.RunName, // Rebuilt, not copied: the SDK records explicitly-sent fields in @@ -201,8 +235,7 @@ func (*ResourceJobRun) RemapState(remote *JobRunRemote) *JobRunState { return &JobRunState{ RunNow: remote.RunNow, ResultState: remote.ResultState, - // Local-only lifecycle fingerprints stay unset on the remapped remote. - Lifecycle: nil, + Lifecycle: remote.Lifecycle, } } @@ -389,10 +422,11 @@ func reportRunLine(ctx context.Context, runID int64, msg string) { // result_state either, so the lifecycle state is what tells the two apart. // Clearing a trigger skips its local-only fingerprint without re-firing the run. func (*ResourceJobRun) OverrideChangeDesc(_ context.Context, path *structpath.PathNode, change *ChangeDesc, remote *JobRunRemote) error { - if slices.Contains(jobRunTriggerLocalPaths, path.String()) { - // A cleared trigger sets New empty; structdiff may report it at lifecycle, - // lifecycle.triggers, or the leaf. - if change.New == nil || change.New == "" { + pathString := path.String() + if isJobRunTriggerPath(pathString) { + removed := pathString == "lifecycle.triggers.on_bundle_deploy" && (change.New == nil || change.New == "") + removed = removed || pathString == "lifecycle.triggers.on_file_change.files" && change.New == nil + if removed { change.Action = deployplan.Skip change.Reason = "trigger removed" } diff --git a/bundle/direct/dresources/job_run_test.go b/bundle/direct/dresources/job_run_test.go index 01c7636d90a..3e9511f3ee5 100644 --- a/bundle/direct/dresources/job_run_test.go +++ b/bundle/direct/dresources/job_run_test.go @@ -351,6 +351,10 @@ func TestJobRunPrepareStateRequiresSuccess(t *testing.T) { state := (&ResourceJobRun{}).PrepareState(&resources.JobRun{RunNow: jobs.RunNow{JobId: 456}}) assert.Equal(t, jobs.RunResultStateSuccess, state.ResultState) + require.NotNil(t, state.Lifecycle) + require.NotNil(t, state.Lifecycle.Triggers) + assert.NotNil(t, state.Lifecycle.Triggers.OnFileChange) + assert.Nil(t, state.Lifecycle.Triggers.OnFileChange.Files) } func TestJobRunPrepareStateOnBundleDeploy(t *testing.T) { @@ -378,7 +382,7 @@ func TestJobRunPrepareStateOnFileChange(t *testing.T) { }) require.NotNil(t, state.Lifecycle) require.NotNil(t, state.Lifecycle.Triggers) - assert.Equal(t, hashes, state.Lifecycle.Triggers.OnFileChange) + assert.Equal(t, hashes, state.Lifecycle.Triggers.OnFileChange.Files) assert.Empty(t, state.Lifecycle.Triggers.OnBundleDeploy) }) @@ -393,24 +397,54 @@ func TestJobRunPrepareStateOnFileChange(t *testing.T) { require.NotNil(t, state.Lifecycle) require.NotNil(t, state.Lifecycle.Triggers) assert.NotEmpty(t, state.Lifecycle.Triggers.OnBundleDeploy) - assert.Equal(t, hashes, state.Lifecycle.Triggers.OnFileChange) + assert.Equal(t, hashes, state.Lifecycle.Triggers.OnFileChange.Files) }) } -func TestJobRunOverrideChangeDescTriggerRemoved(t *testing.T) { - r := &ResourceJobRun{} +func TestJobRunStateUnmarshalLifecycleCompatibility(t *testing.T) { + t.Run("missing lifecycle gets empty shape", func(t *testing.T) { + var state JobRunState + require.NoError(t, json.Unmarshal([]byte(`{}`), &state)) + require.NotNil(t, state.Lifecycle) + require.NotNil(t, state.Lifecycle.Triggers) + require.NotNil(t, state.Lifecycle.Triggers.OnFileChange) + assert.Nil(t, state.Lifecycle.Triggers.OnFileChange.Files) + }) - t.Run("clearing lifecycle downgrades to skip", func(t *testing.T) { - change := &ChangeDesc{ - Action: deployplan.Recreate, - Old: &JobRunLifecycleState{Triggers: &JobRunTriggersState{OnBundleDeploy: "old"}}, - New: nil, - } - require.NoError(t, r.OverrideChangeDesc(t.Context(), structpath.MustParsePath("lifecycle"), change, nil)) - assert.Equal(t, deployplan.Skip, change.Action) - assert.Equal(t, "trigger removed", change.Reason) + t.Run("old path-to-hash map is preserved", func(t *testing.T) { + var state JobRunState + require.NoError(t, json.Unmarshal([]byte(`{ + "lifecycle": { + "triggers": { + "on_file_change": { + "a.txt": "abc" + } + } + } + }`), &state)) + assert.Equal(t, map[string]string{"a.txt": "abc"}, state.Lifecycle.Triggers.OnFileChange.Files) }) + t.Run("new wrapped map is preserved", func(t *testing.T) { + var state JobRunState + require.NoError(t, json.Unmarshal([]byte(`{ + "lifecycle": { + "triggers": { + "on_file_change": { + "files": { + "a.txt": "abc" + } + } + } + } + }`), &state)) + assert.Equal(t, map[string]string{"a.txt": "abc"}, state.Lifecycle.Triggers.OnFileChange.Files) + }) +} + +func TestJobRunOverrideChangeDescTriggerRemoved(t *testing.T) { + r := &ResourceJobRun{} + t.Run("clearing on_bundle_deploy leaf downgrades to skip", func(t *testing.T) { change := &ChangeDesc{ Action: deployplan.Recreate, @@ -422,14 +456,14 @@ func TestJobRunOverrideChangeDescTriggerRemoved(t *testing.T) { assert.Equal(t, "trigger removed", change.Reason) }) - t.Run("clearing on_file_change leaf downgrades to update", func(t *testing.T) { + t.Run("clearing on_file_change files downgrades to skip", func(t *testing.T) { change := &ChangeDesc{ Action: deployplan.Recreate, Old: map[string]string{"a.txt": "abc"}, New: nil, } - require.NoError(t, r.OverrideChangeDesc(t.Context(), structpath.MustParsePath("lifecycle.triggers.on_file_change"), change, nil)) - assert.Equal(t, deployplan.Update, change.Action) + require.NoError(t, r.OverrideChangeDesc(t.Context(), structpath.MustParsePath("lifecycle.triggers.on_file_change.files"), change, nil)) + assert.Equal(t, deployplan.Skip, change.Action) assert.Equal(t, "trigger removed", change.Reason) }) @@ -446,10 +480,30 @@ func TestJobRunOverrideChangeDescTriggerRemoved(t *testing.T) { t.Run("changed on_file_change hash still recreates", func(t *testing.T) { change := &ChangeDesc{ Action: deployplan.Recreate, - Old: map[string]string{"a.txt": "old"}, - New: map[string]string{"a.txt": "new"}, + Old: "old", + New: "new", + } + require.NoError(t, r.OverrideChangeDesc(t.Context(), structpath.MustParsePath("lifecycle.triggers.on_file_change.files['a.txt']"), change, nil)) + assert.Equal(t, deployplan.Recreate, change.Action) + }) + + t.Run("removed matched file still recreates", func(t *testing.T) { + change := &ChangeDesc{ + Action: deployplan.Recreate, + Old: "old", + New: nil, + } + require.NoError(t, r.OverrideChangeDesc(t.Context(), structpath.MustParsePath("lifecycle.triggers.on_file_change.files['a.txt']"), change, nil)) + assert.Equal(t, deployplan.Recreate, change.Action) + }) + + t.Run("missing file fingerprint still recreates", func(t *testing.T) { + change := &ChangeDesc{ + Action: deployplan.Recreate, + Old: "old", + New: "", } - require.NoError(t, r.OverrideChangeDesc(t.Context(), structpath.MustParsePath("lifecycle.triggers.on_file_change"), change, nil)) + require.NoError(t, r.OverrideChangeDesc(t.Context(), structpath.MustParsePath("lifecycle.triggers.on_file_change.files['a.txt']"), change, nil)) assert.Equal(t, deployplan.Recreate, change.Action) }) } @@ -464,11 +518,13 @@ func TestJobRunRemapStateCarriesTheOutcome(t *testing.T) { "", } { t.Run(string(outcome), func(t *testing.T) { - remote := &JobRunRemote{RunId: 123, ResultState: outcome} + lifecycle := newJobRunLifecycleState() + remote := &JobRunRemote{RunId: 123, ResultState: outcome, Lifecycle: lifecycle} state := (&ResourceJobRun{}).RemapState(remote) assert.Equal(t, outcome, state.ResultState) + assert.Same(t, lifecycle, state.Lifecycle) }) } } diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index b0a9ed6b365..67388b30dfe 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -175,6 +175,9 @@ resources: reason: not_returned_by_api - field: queue reason: not_returned_by_api + # Local-only trigger fingerprints; GetRun does not return them. + - field: lifecycle + reason: not_returned_by_api # A run is immutable and fire-once, so any change recreates it. Omitting # `field` matches every field (root; see TestFieldRuleOmittedIsRoot). # `field: ""` would instead match nothing. The one exception is a run that is diff --git a/bundle/direct/dresources/type_test.go b/bundle/direct/dresources/type_test.go index 2d5516d59c7..ca84850c33d 100644 --- a/bundle/direct/dresources/type_test.go +++ b/bundle/direct/dresources/type_test.go @@ -56,10 +56,6 @@ var knownMissingInRemoteType = map[string][]string{ "vector_search_endpoints": { "usage_policy_id", }, - "job_runs": { - // Local-only trigger fingerprints under lifecycle. - "lifecycle", - }, } // commonMissingInStateType lists fields that are commonly missing across all resource types. From 1d77e537571868551dfc632038ecf53109452e01 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Fri, 21 Aug 2026 13:02:42 +0000 Subject: [PATCH 38/75] job_runs: keep the lifecycle fingerprint out of the remote type Nest lifecycle by value so structdiff descends to the leaf that changed instead of reporting the whole subtree, and let RemapState supply the empty shape. GetRun never returns the fingerprints, so the remote type no longer advertises a lifecycle it cannot fill. --- acceptance/bundle/refschema/out.fields.txt | 12 ++--- .../resources/job_runs/failed_run/output.txt | 5 --- .../job_runs/interrupted_run/output.txt | 5 --- .../job_runs/on_file_change_glob/output.txt | 4 +- .../resources/job_runs/redeploy/output.txt | 5 --- bundle/direct/dresources/job_run.go | 45 +++++++++---------- bundle/direct/dresources/job_run_test.go | 17 +------ bundle/direct/dresources/resources.yml | 3 -- bundle/direct/dresources/type_test.go | 5 +++ 9 files changed, 37 insertions(+), 64 deletions(-) diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index 3c681d4b199..cc754a56d00 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -865,16 +865,16 @@ resources.job_runs.*.jar_params[*] string ALL resources.job_runs.*.job_id int64 ALL resources.job_runs.*.job_parameters map[string]string ALL resources.job_runs.*.job_parameters.* string ALL -resources.job_runs.*.lifecycle *dresources.JobRunLifecycleState REMOTE STATE resources.job_runs.*.lifecycle *resources.JobRunLifecycle INPUT +resources.job_runs.*.lifecycle dresources.JobRunLifecycleState STATE resources.job_runs.*.lifecycle resources.Lifecycle INPUT resources.job_runs.*.lifecycle.prevent_destroy bool INPUT -resources.job_runs.*.lifecycle.triggers *dresources.JobRunTriggersState REMOTE STATE resources.job_runs.*.lifecycle.triggers []resources.JobRunTrigger INPUT -resources.job_runs.*.lifecycle.triggers.on_bundle_deploy string REMOTE STATE -resources.job_runs.*.lifecycle.triggers.on_file_change *dresources.JobRunFileTriggerState REMOTE STATE -resources.job_runs.*.lifecycle.triggers.on_file_change.files map[string]string REMOTE STATE -resources.job_runs.*.lifecycle.triggers.on_file_change.files.* string REMOTE STATE +resources.job_runs.*.lifecycle.triggers dresources.JobRunTriggersState STATE +resources.job_runs.*.lifecycle.triggers.on_bundle_deploy string STATE +resources.job_runs.*.lifecycle.triggers.on_file_change dresources.JobRunFileTriggerState STATE +resources.job_runs.*.lifecycle.triggers.on_file_change.files map[string]string STATE +resources.job_runs.*.lifecycle.triggers.on_file_change.files.* string STATE resources.job_runs.*.lifecycle.triggers[*] resources.JobRunTrigger INPUT resources.job_runs.*.lifecycle.triggers[*].on_bundle_deploy *bool INPUT resources.job_runs.*.lifecycle.triggers[*].on_file_change *string INPUT diff --git a/acceptance/bundle/resources/job_runs/failed_run/output.txt b/acceptance/bundle/resources/job_runs/failed_run/output.txt index 0c6bddb0d9b..37f1a831681 100644 --- a/acceptance/bundle/resources/job_runs/failed_run/output.txt +++ b/acceptance/bundle/resources/job_runs/failed_run/output.txt @@ -37,11 +37,6 @@ Files: 7 uploaded, 0 deleted }, "remote_state": { "job_id": [MY_JOB_ID], - "lifecycle": { - "triggers": { - "on_file_change": {} - } - }, "result_state": "FAILED", "run_id": [MY_RUN_ID], "run_name": "test-job-[UNIQUE_NAME]", diff --git a/acceptance/bundle/resources/job_runs/interrupted_run/output.txt b/acceptance/bundle/resources/job_runs/interrupted_run/output.txt index da185c3704d..2e1d9cacd7a 100644 --- a/acceptance/bundle/resources/job_runs/interrupted_run/output.txt +++ b/acceptance/bundle/resources/job_runs/interrupted_run/output.txt @@ -42,11 +42,6 @@ Exit code: 1 "action": "skip", "remote_state": { "job_id": [MY_JOB_ID], - "lifecycle": { - "triggers": { - "on_file_change": {} - } - }, "run_id": [MY_RUN_ID], "run_name": "my-job", "run_page_url": "[DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?o=[NUMID]", diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt index 5b4ef23178c..69a7e5ece94 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt @@ -176,7 +176,7 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged { "lifecycle.triggers.on_file_change.files": { "action": "skip", - "reason": "not_returned_by_api", + "reason": "missing_in_remote", "old": { "migrations/001.txt": "[FILE_HASH][3]", "migrations/002.txt": "[FILE_HASH][1]", @@ -205,7 +205,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged { "lifecycle.triggers.on_file_change.files": { "action": "skip", - "reason": "not_returned_by_api", + "reason": "missing_in_remote", "old": { "migrations/001.txt": "[FILE_HASH][3]", "migrations/002.txt": "[FILE_HASH][1]", diff --git a/acceptance/bundle/resources/job_runs/redeploy/output.txt b/acceptance/bundle/resources/job_runs/redeploy/output.txt index 60111c98a93..a80685733ad 100644 --- a/acceptance/bundle/resources/job_runs/redeploy/output.txt +++ b/acceptance/bundle/resources/job_runs/redeploy/output.txt @@ -72,11 +72,6 @@ Resources: "job_parameters": { "env": "dev" }, - "lifecycle": { - "triggers": { - "on_file_change": {} - } - }, "result_state": "SUCCESS", "run_id": [MY_RUN_ID], "run_name": "my-job", diff --git a/bundle/direct/dresources/job_run.go b/bundle/direct/dresources/job_run.go index 815091a721a..9a937787e02 100644 --- a/bundle/direct/dresources/job_run.go +++ b/bundle/direct/dresources/job_run.go @@ -41,13 +41,12 @@ func isJobRunTriggerPath(path string) bool { type JobRunTriggersState struct { // Fresh UUID each plan while armed so Old!=New forces recreate. OnBundleDeploy string `json:"on_bundle_deploy,omitempty"` - // Path → content hash from ResolveJobRunFileTriggers; change to recreate. - OnFileChange *JobRunFileTriggerState `json:"on_file_change"` + // Content hashes from ResolveJobRunFileTriggers; any change recreates. + OnFileChange JobRunFileTriggerState `json:"on_file_change"` } -// JobRunFileTriggerState is always present so remote and state share one shape. -// Files is nil when the trigger is off, so dropping the trigger diffs here and -// a file appear/disappear diffs inside the map. +// JobRunFileTriggerState wraps the hashes so turning the trigger off (Files nil) +// diffs at one path, distinct from the per-file entries a changed file produces. type JobRunFileTriggerState struct { Files map[string]string `json:"files,omitempty"` } @@ -68,16 +67,21 @@ func (s *JobRunFileTriggerState) UnmarshalJSON(b []byte) error { return json.Unmarshal(b, &s.Files) } -// JobRunLifecycleState is the local-only trigger fingerprint, also present (empty) on remote. +// JobRunLifecycleState is the local-only trigger fingerprint. Nested by value, +// not by pointer: structdiff cannot descend into a nil pointer and would report +// the whole subtree at "lifecycle" instead of the leaf that actually changed. type JobRunLifecycleState struct { - Triggers *JobRunTriggersState `json:"triggers"` + Triggers JobRunTriggersState `json:"triggers"` } -func newJobRunLifecycleState() *JobRunLifecycleState { - return &JobRunLifecycleState{ - Triggers: &JobRunTriggersState{ +// Zero value spelled out field by field, as exhaustruct requires. +func emptyJobRunLifecycleState() JobRunLifecycleState { + return JobRunLifecycleState{ + Triggers: JobRunTriggersState{ OnBundleDeploy: "", - OnFileChange: &JobRunFileTriggerState{Files: nil}, + OnFileChange: JobRunFileTriggerState{ + Files: nil, + }, }, } } @@ -91,11 +95,10 @@ type JobRunState struct { // Local-only. Nested under lifecycle to mirror config and avoid colliding // with a future Jobs API field. - Lifecycle *JobRunLifecycleState `json:"lifecycle"` + Lifecycle JobRunLifecycleState `json:"lifecycle"` } func (s *JobRunState) UnmarshalJSON(b []byte) error { - s.Lifecycle = newJobRunLifecycleState() return marshal.Unmarshal(b, s) } @@ -103,8 +106,8 @@ func (s JobRunState) MarshalJSON() ([]byte, error) { return marshal.Marshal(s) } -// JobRunRemote embeds RunNow so every StateType path is a valid RemoteType path -// (see TestRemoteSuperset), plus the run's output-only fields for a faithful view. +// JobRunRemote is the RunNow request plus the run's output-only fields. It has no +// lifecycle: GetRun never returns the fingerprints (see knownMissingInRemoteType). type JobRunRemote struct { jobs.RunNow @@ -112,9 +115,6 @@ type JobRunRemote struct { // compare the two. See "RemapState is a dumb copy" in README.md. ResultState jobs.RunResultState `json:"result_state,omitempty"` - // Always the empty fingerprint: GetRun does not return triggers. - Lifecycle *JobRunLifecycleState `json:"lifecycle"` - RunId int64 `json:"run_id,omitempty"` RunName string `json:"run_name,omitempty"` State *jobs.RunState `json:"state,omitempty"` @@ -125,7 +125,6 @@ type JobRunRemote struct { // Custom marshaler needed because embedded RunNow's MarshalJSON would otherwise // take over and drop the additional fields. func (s *JobRunRemote) UnmarshalJSON(b []byte) error { - s.Lifecycle = newJobRunLifecycleState() return marshal.Unmarshal(b, s) } @@ -147,7 +146,7 @@ func (*ResourceJobRun) PrepareState(input *resources.JobRun) *JobRunState { state := &JobRunState{ RunNow: input.RunNow, ResultState: jobs.RunResultStateSuccess, - Lifecycle: newJobRunLifecycleState(), + Lifecycle: emptyJobRunLifecycleState(), } if input.HasOnBundleDeploy() { state.Lifecycle.Triggers.OnBundleDeploy = uuid.NewString() @@ -194,7 +193,6 @@ func makeJobRunRemote(run *jobs.Run) *JobRunRemote { ForceSendFields: nil, }, ResultState: run.State.ResultState, - Lifecycle: newJobRunLifecycleState(), RunId: run.RunId, RunName: run.RunName, // Rebuilt, not copied: the SDK records explicitly-sent fields in @@ -230,12 +228,13 @@ func (r *ResourceJobRun) DoRead(ctx context.Context, id string) (*JobRunRemote, } // RemapState extracts the fields used for diffing: the RunNow request and the -// outcome the run reached. +// outcome the run reached. Lifecycle has no remote counterpart, so it stays empty +// and the planner skips it as missing_in_remote. func (*ResourceJobRun) RemapState(remote *JobRunRemote) *JobRunState { return &JobRunState{ RunNow: remote.RunNow, ResultState: remote.ResultState, - Lifecycle: remote.Lifecycle, + Lifecycle: emptyJobRunLifecycleState(), } } diff --git a/bundle/direct/dresources/job_run_test.go b/bundle/direct/dresources/job_run_test.go index 3e9511f3ee5..7d452cd5e0f 100644 --- a/bundle/direct/dresources/job_run_test.go +++ b/bundle/direct/dresources/job_run_test.go @@ -351,9 +351,6 @@ func TestJobRunPrepareStateRequiresSuccess(t *testing.T) { state := (&ResourceJobRun{}).PrepareState(&resources.JobRun{RunNow: jobs.RunNow{JobId: 456}}) assert.Equal(t, jobs.RunResultStateSuccess, state.ResultState) - require.NotNil(t, state.Lifecycle) - require.NotNil(t, state.Lifecycle.Triggers) - assert.NotNil(t, state.Lifecycle.Triggers.OnFileChange) assert.Nil(t, state.Lifecycle.Triggers.OnFileChange.Files) } @@ -365,8 +362,6 @@ func TestJobRunPrepareStateOnBundleDeploy(t *testing.T) { }, } first := (&ResourceJobRun{}).PrepareState(input) - require.NotNil(t, first.Lifecycle) - require.NotNil(t, first.Lifecycle.Triggers) assert.NotEmpty(t, first.Lifecycle.Triggers.OnBundleDeploy) second := (&ResourceJobRun{}).PrepareState(input) @@ -380,8 +375,6 @@ func TestJobRunPrepareStateOnFileChange(t *testing.T) { state := (&ResourceJobRun{}).PrepareState(&resources.JobRun{ ResolvedFileTriggers: hashes, }) - require.NotNil(t, state.Lifecycle) - require.NotNil(t, state.Lifecycle.Triggers) assert.Equal(t, hashes, state.Lifecycle.Triggers.OnFileChange.Files) assert.Empty(t, state.Lifecycle.Triggers.OnBundleDeploy) }) @@ -394,8 +387,6 @@ func TestJobRunPrepareStateOnFileChange(t *testing.T) { }, ResolvedFileTriggers: hashes, }) - require.NotNil(t, state.Lifecycle) - require.NotNil(t, state.Lifecycle.Triggers) assert.NotEmpty(t, state.Lifecycle.Triggers.OnBundleDeploy) assert.Equal(t, hashes, state.Lifecycle.Triggers.OnFileChange.Files) }) @@ -405,9 +396,6 @@ func TestJobRunStateUnmarshalLifecycleCompatibility(t *testing.T) { t.Run("missing lifecycle gets empty shape", func(t *testing.T) { var state JobRunState require.NoError(t, json.Unmarshal([]byte(`{}`), &state)) - require.NotNil(t, state.Lifecycle) - require.NotNil(t, state.Lifecycle.Triggers) - require.NotNil(t, state.Lifecycle.Triggers.OnFileChange) assert.Nil(t, state.Lifecycle.Triggers.OnFileChange.Files) }) @@ -518,13 +506,12 @@ func TestJobRunRemapStateCarriesTheOutcome(t *testing.T) { "", } { t.Run(string(outcome), func(t *testing.T) { - lifecycle := newJobRunLifecycleState() - remote := &JobRunRemote{RunId: 123, ResultState: outcome, Lifecycle: lifecycle} + remote := &JobRunRemote{RunId: 123, ResultState: outcome} state := (&ResourceJobRun{}).RemapState(remote) assert.Equal(t, outcome, state.ResultState) - assert.Same(t, lifecycle, state.Lifecycle) + assert.Equal(t, emptyJobRunLifecycleState(), state.Lifecycle) }) } } diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index 67388b30dfe..b0a9ed6b365 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -175,9 +175,6 @@ resources: reason: not_returned_by_api - field: queue reason: not_returned_by_api - # Local-only trigger fingerprints; GetRun does not return them. - - field: lifecycle - reason: not_returned_by_api # A run is immutable and fire-once, so any change recreates it. Omitting # `field` matches every field (root; see TestFieldRuleOmittedIsRoot). # `field: ""` would instead match nothing. The one exception is a run that is diff --git a/bundle/direct/dresources/type_test.go b/bundle/direct/dresources/type_test.go index ca84850c33d..7ad9872c373 100644 --- a/bundle/direct/dresources/type_test.go +++ b/bundle/direct/dresources/type_test.go @@ -19,6 +19,11 @@ var knownMissingInRemoteType = map[string][]string{ "external_locations": { "skip_validation", }, + "job_runs": { + // Deliberate omission: the trigger fingerprints are client-side only, so + // GetRun has nothing to report them under. + "lifecycle", + }, "model_serving_endpoints": { "rate_limits", }, From 1364bf986a8e804663c95dc7f3a22d7067852185 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Mon, 24 Aug 2026 07:55:05 +0000 Subject: [PATCH 39/75] job_runs: store on_file_change hashes on the trigger itself Drop the files wrapper so plan diffs land on lifecycle.triggers.on_file_change, matching the config path without an extra state-only level. --- acceptance/bundle/refschema/out.fields.txt | 5 +-- .../job_runs/on_file_change/output.txt | 8 ++-- .../job_runs/on_file_change_glob/output.txt | 20 +++++----- bundle/direct/dresources/job_run.go | 35 +++-------------- bundle/direct/dresources/job_run_test.go | 38 ++++++------------- 5 files changed, 32 insertions(+), 74 deletions(-) diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index cc754a56d00..06d260541a5 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -872,9 +872,8 @@ resources.job_runs.*.lifecycle.prevent_destroy bool INPUT resources.job_runs.*.lifecycle.triggers []resources.JobRunTrigger INPUT resources.job_runs.*.lifecycle.triggers dresources.JobRunTriggersState STATE resources.job_runs.*.lifecycle.triggers.on_bundle_deploy string STATE -resources.job_runs.*.lifecycle.triggers.on_file_change dresources.JobRunFileTriggerState STATE -resources.job_runs.*.lifecycle.triggers.on_file_change.files map[string]string STATE -resources.job_runs.*.lifecycle.triggers.on_file_change.files.* string STATE +resources.job_runs.*.lifecycle.triggers.on_file_change map[string]string STATE +resources.job_runs.*.lifecycle.triggers.on_file_change.* string STATE resources.job_runs.*.lifecycle.triggers[*] resources.JobRunTrigger INPUT resources.job_runs.*.lifecycle.triggers[*].on_bundle_deploy *bool INPUT resources.job_runs.*.lifecycle.triggers[*].on_file_change *string INPUT diff --git a/acceptance/bundle/resources/job_runs/on_file_change/output.txt b/acceptance/bundle/resources/job_runs/on_file_change/output.txt index 9b662e1936d..28deeee38a4 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change/output.txt @@ -52,7 +52,7 @@ Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle.triggers.on_file_change.files": { + "lifecycle.triggers.on_file_change": { "action": "recreate", "reason": "immutable", "old": { @@ -62,7 +62,7 @@ Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged "seed.txt": "[FILE_HASH][1]" } }, - "lifecycle.triggers.on_file_change.files['seed.txt']": { + "lifecycle.triggers.on_file_change['seed.txt']": { "action": "recreate", "reason": "immutable", "old": "[FILE_HASH][0]", @@ -117,7 +117,7 @@ Warning: lifecycle.triggers.on_file_change: no files match "seed.txt" in databricks.yml:18:29 { - "lifecycle.triggers.on_file_change.files": { + "lifecycle.triggers.on_file_change": { "action": "recreate", "reason": "immutable", "old": { @@ -127,7 +127,7 @@ Warning: lifecycle.triggers.on_file_change: no files match "seed.txt" "seed.txt": "" } }, - "lifecycle.triggers.on_file_change.files['seed.txt']": { + "lifecycle.triggers.on_file_change['seed.txt']": { "action": "recreate", "reason": "immutable", "old": "[FILE_HASH][1]", diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt index 69a7e5ece94..a04e3710fb4 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt @@ -25,7 +25,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged === editing one matched file re-fires >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle.triggers.on_file_change.files": { + "lifecycle.triggers.on_file_change": { "action": "recreate", "reason": "immutable", "old": { @@ -39,7 +39,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "mysubdir/a.txt": "[FILE_HASH][2]" } }, - "lifecycle.triggers.on_file_change.files['migrations/001.txt']": { + "lifecycle.triggers.on_file_change['migrations/001.txt']": { "action": "recreate", "reason": "immutable", "old": "[FILE_HASH][0]", @@ -83,7 +83,7 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged === editing a file in a subdirectory re-fires >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle.triggers.on_file_change.files": { + "lifecycle.triggers.on_file_change": { "action": "recreate", "reason": "immutable", "old": { @@ -97,7 +97,7 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged "mysubdir/a.txt": "[FILE_HASH][4]" } }, - "lifecycle.triggers.on_file_change.files['mysubdir/a.txt']": { + "lifecycle.triggers.on_file_change['mysubdir/a.txt']": { "action": "recreate", "reason": "immutable", "old": "[FILE_HASH][2]", @@ -114,7 +114,7 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged === a new matching file re-fires >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle.triggers.on_file_change.files": { + "lifecycle.triggers.on_file_change": { "action": "recreate", "reason": "immutable", "old": { @@ -129,7 +129,7 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged "mysubdir/a.txt": "[FILE_HASH][2]" } }, - "lifecycle.triggers.on_file_change.files['migrations/003.txt']": { + "lifecycle.triggers.on_file_change['migrations/003.txt']": { "action": "recreate", "reason": "immutable", "new": "[FILE_HASH][5]" @@ -145,7 +145,7 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged === removing a matched file re-fires >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle.triggers.on_file_change.files": { + "lifecycle.triggers.on_file_change": { "action": "recreate", "reason": "immutable", "old": { @@ -158,7 +158,7 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged "mysubdir/a.txt": "[FILE_HASH][2]" } }, - "lifecycle.triggers.on_file_change.files['migrations/002.txt']": { + "lifecycle.triggers.on_file_change['migrations/002.txt']": { "action": "recreate", "reason": "immutable", "old": "[FILE_HASH][1]" @@ -174,7 +174,7 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged === editing a file below the glob does not re-fire >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle.triggers.on_file_change.files": { + "lifecycle.triggers.on_file_change": { "action": "skip", "reason": "missing_in_remote", "old": { @@ -203,7 +203,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged === ** is not recursive: it matches the same files as * >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle.triggers.on_file_change.files": { + "lifecycle.triggers.on_file_change": { "action": "skip", "reason": "missing_in_remote", "old": { diff --git a/bundle/direct/dresources/job_run.go b/bundle/direct/dresources/job_run.go index 9a937787e02..9d1a272a7a3 100644 --- a/bundle/direct/dresources/job_run.go +++ b/bundle/direct/dresources/job_run.go @@ -3,7 +3,6 @@ package dresources import ( "cmp" "context" - "encoding/json" "errors" "fmt" "slices" @@ -30,7 +29,7 @@ const jobRunTimeout = 24 * time.Hour // jobRunTriggerLocalPaths are local-only fingerprints: clearing one is skip, not recreate. var jobRunTriggerLocalPaths = []string{ "lifecycle.triggers.on_bundle_deploy", - "lifecycle.triggers.on_file_change.files", + "lifecycle.triggers.on_file_change", } func isJobRunTriggerPath(path string) bool { @@ -42,29 +41,7 @@ type JobRunTriggersState struct { // Fresh UUID each plan while armed so Old!=New forces recreate. OnBundleDeploy string `json:"on_bundle_deploy,omitempty"` // Content hashes from ResolveJobRunFileTriggers; any change recreates. - OnFileChange JobRunFileTriggerState `json:"on_file_change"` -} - -// JobRunFileTriggerState wraps the hashes so turning the trigger off (Files nil) -// diffs at one path, distinct from the per-file entries a changed file produces. -type JobRunFileTriggerState struct { - Files map[string]string `json:"files,omitempty"` -} - -// UnmarshalJSON accepts the wrapped map and the older path-to-hash map. -func (s *JobRunFileTriggerState) UnmarshalJSON(b []byte) error { - var raw map[string]json.RawMessage - if err := json.Unmarshal(b, &raw); err != nil { - return err - } - if files, ok := raw["files"]; ok && len(files) > 0 && files[0] == '{' { - return json.Unmarshal(files, &s.Files) - } - if len(raw) == 0 { - s.Files = nil - return nil - } - return json.Unmarshal(b, &s.Files) + OnFileChange map[string]string `json:"on_file_change,omitempty"` } // JobRunLifecycleState is the local-only trigger fingerprint. Nested by value, @@ -79,9 +56,7 @@ func emptyJobRunLifecycleState() JobRunLifecycleState { return JobRunLifecycleState{ Triggers: JobRunTriggersState{ OnBundleDeploy: "", - OnFileChange: JobRunFileTriggerState{ - Files: nil, - }, + OnFileChange: nil, }, } } @@ -152,7 +127,7 @@ func (*ResourceJobRun) PrepareState(input *resources.JobRun) *JobRunState { state.Lifecycle.Triggers.OnBundleDeploy = uuid.NewString() } if len(input.ResolvedFileTriggers) > 0 { - state.Lifecycle.Triggers.OnFileChange.Files = input.ResolvedFileTriggers + state.Lifecycle.Triggers.OnFileChange = input.ResolvedFileTriggers } return state } @@ -424,7 +399,7 @@ func (*ResourceJobRun) OverrideChangeDesc(_ context.Context, path *structpath.Pa pathString := path.String() if isJobRunTriggerPath(pathString) { removed := pathString == "lifecycle.triggers.on_bundle_deploy" && (change.New == nil || change.New == "") - removed = removed || pathString == "lifecycle.triggers.on_file_change.files" && change.New == nil + removed = removed || pathString == "lifecycle.triggers.on_file_change" && change.New == nil if removed { change.Action = deployplan.Skip change.Reason = "trigger removed" diff --git a/bundle/direct/dresources/job_run_test.go b/bundle/direct/dresources/job_run_test.go index 7d452cd5e0f..d173265ca03 100644 --- a/bundle/direct/dresources/job_run_test.go +++ b/bundle/direct/dresources/job_run_test.go @@ -351,7 +351,7 @@ func TestJobRunPrepareStateRequiresSuccess(t *testing.T) { state := (&ResourceJobRun{}).PrepareState(&resources.JobRun{RunNow: jobs.RunNow{JobId: 456}}) assert.Equal(t, jobs.RunResultStateSuccess, state.ResultState) - assert.Nil(t, state.Lifecycle.Triggers.OnFileChange.Files) + assert.Nil(t, state.Lifecycle.Triggers.OnFileChange) } func TestJobRunPrepareStateOnBundleDeploy(t *testing.T) { @@ -375,7 +375,7 @@ func TestJobRunPrepareStateOnFileChange(t *testing.T) { state := (&ResourceJobRun{}).PrepareState(&resources.JobRun{ ResolvedFileTriggers: hashes, }) - assert.Equal(t, hashes, state.Lifecycle.Triggers.OnFileChange.Files) + assert.Equal(t, hashes, state.Lifecycle.Triggers.OnFileChange) assert.Empty(t, state.Lifecycle.Triggers.OnBundleDeploy) }) @@ -388,7 +388,7 @@ func TestJobRunPrepareStateOnFileChange(t *testing.T) { ResolvedFileTriggers: hashes, }) assert.NotEmpty(t, state.Lifecycle.Triggers.OnBundleDeploy) - assert.Equal(t, hashes, state.Lifecycle.Triggers.OnFileChange.Files) + assert.Equal(t, hashes, state.Lifecycle.Triggers.OnFileChange) }) } @@ -396,10 +396,10 @@ func TestJobRunStateUnmarshalLifecycleCompatibility(t *testing.T) { t.Run("missing lifecycle gets empty shape", func(t *testing.T) { var state JobRunState require.NoError(t, json.Unmarshal([]byte(`{}`), &state)) - assert.Nil(t, state.Lifecycle.Triggers.OnFileChange.Files) + assert.Nil(t, state.Lifecycle.Triggers.OnFileChange) }) - t.Run("old path-to-hash map is preserved", func(t *testing.T) { + t.Run("path-to-hash map is preserved", func(t *testing.T) { var state JobRunState require.NoError(t, json.Unmarshal([]byte(`{ "lifecycle": { @@ -410,23 +410,7 @@ func TestJobRunStateUnmarshalLifecycleCompatibility(t *testing.T) { } } }`), &state)) - assert.Equal(t, map[string]string{"a.txt": "abc"}, state.Lifecycle.Triggers.OnFileChange.Files) - }) - - t.Run("new wrapped map is preserved", func(t *testing.T) { - var state JobRunState - require.NoError(t, json.Unmarshal([]byte(`{ - "lifecycle": { - "triggers": { - "on_file_change": { - "files": { - "a.txt": "abc" - } - } - } - } - }`), &state)) - assert.Equal(t, map[string]string{"a.txt": "abc"}, state.Lifecycle.Triggers.OnFileChange.Files) + assert.Equal(t, map[string]string{"a.txt": "abc"}, state.Lifecycle.Triggers.OnFileChange) }) } @@ -444,13 +428,13 @@ func TestJobRunOverrideChangeDescTriggerRemoved(t *testing.T) { assert.Equal(t, "trigger removed", change.Reason) }) - t.Run("clearing on_file_change files downgrades to skip", func(t *testing.T) { + t.Run("clearing on_file_change downgrades to skip", func(t *testing.T) { change := &ChangeDesc{ Action: deployplan.Recreate, Old: map[string]string{"a.txt": "abc"}, New: nil, } - require.NoError(t, r.OverrideChangeDesc(t.Context(), structpath.MustParsePath("lifecycle.triggers.on_file_change.files"), change, nil)) + require.NoError(t, r.OverrideChangeDesc(t.Context(), structpath.MustParsePath("lifecycle.triggers.on_file_change"), change, nil)) assert.Equal(t, deployplan.Skip, change.Action) assert.Equal(t, "trigger removed", change.Reason) }) @@ -471,7 +455,7 @@ func TestJobRunOverrideChangeDescTriggerRemoved(t *testing.T) { Old: "old", New: "new", } - require.NoError(t, r.OverrideChangeDesc(t.Context(), structpath.MustParsePath("lifecycle.triggers.on_file_change.files['a.txt']"), change, nil)) + require.NoError(t, r.OverrideChangeDesc(t.Context(), structpath.MustParsePath("lifecycle.triggers.on_file_change['a.txt']"), change, nil)) assert.Equal(t, deployplan.Recreate, change.Action) }) @@ -481,7 +465,7 @@ func TestJobRunOverrideChangeDescTriggerRemoved(t *testing.T) { Old: "old", New: nil, } - require.NoError(t, r.OverrideChangeDesc(t.Context(), structpath.MustParsePath("lifecycle.triggers.on_file_change.files['a.txt']"), change, nil)) + require.NoError(t, r.OverrideChangeDesc(t.Context(), structpath.MustParsePath("lifecycle.triggers.on_file_change['a.txt']"), change, nil)) assert.Equal(t, deployplan.Recreate, change.Action) }) @@ -491,7 +475,7 @@ func TestJobRunOverrideChangeDescTriggerRemoved(t *testing.T) { Old: "old", New: "", } - require.NoError(t, r.OverrideChangeDesc(t.Context(), structpath.MustParsePath("lifecycle.triggers.on_file_change.files['a.txt']"), change, nil)) + require.NoError(t, r.OverrideChangeDesc(t.Context(), structpath.MustParsePath("lifecycle.triggers.on_file_change['a.txt']"), change, nil)) assert.Equal(t, deployplan.Recreate, change.Action) }) } From bd2b53119b0c2b1367eb8ace3d0575cdebd1e57b Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Mon, 24 Aug 2026 08:17:19 +0000 Subject: [PATCH 40/75] job_runs: drop unit tests already covered by acceptance The trigger, PrepareState and OverrideChangeDesc cases are asserted end to end by on_bundle_deploy and on_file_change, so the unit copies only duplicated goldens. Keep the ones acceptance cannot reach without stubbing GetRun. --- bundle/direct/dresources/job_run_test.go | 212 +---------------------- 1 file changed, 6 insertions(+), 206 deletions(-) diff --git a/bundle/direct/dresources/job_run_test.go b/bundle/direct/dresources/job_run_test.go index d173265ca03..ad23ca605bc 100644 --- a/bundle/direct/dresources/job_run_test.go +++ b/bundle/direct/dresources/job_run_test.go @@ -11,8 +11,6 @@ import ( "time" "github.com/databricks/cli/bundle/config/resources" - "github.com/databricks/cli/bundle/deployplan" - "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/cli/libs/testserver" "github.com/databricks/databricks-sdk-go" @@ -68,28 +66,6 @@ func waitForTestRun(t *testing.T, ctx context.Context, client *databricks.Worksp return r.WaitAfterCreate(ctx, "123", &JobRunState{}) } -func TestJobRunWaitSucceeds(t *testing.T) { - client := jobRunClient(t, &jobs.RunState{ - LifeCycleState: jobs.RunLifeCycleStateTerminated, - ResultState: jobs.RunResultStateSuccess, - }) - - remote, err := waitForTestRun(t, t.Context(), client) - - require.NoError(t, err) - require.NotNil(t, remote.State) - assert.Equal(t, jobs.RunResultStateSuccess, remote.State.ResultState) -} - -func TestReportRunLineIncludesResourceKey(t *testing.T) { - ctx, stderr := cmdio.NewTestContextWithStderr(t.Context()) - ctx = WithResourceKey(ctx, "job_runs.my_run") - - reportRunLine(ctx, 123, "SUCCESS") - - assert.Equal(t, "Output from job_runs.my_run: id=123: SUCCESS\n", stderr.String()) -} - func TestJobRunWaitFailsOnFailedResult(t *testing.T) { client := jobRunClient(t, &jobs.RunState{ LifeCycleState: jobs.RunLifeCycleStateTerminated, @@ -102,36 +78,6 @@ func TestJobRunWaitFailsOnFailedResult(t *testing.T) { require.ErrorContains(t, err, "did not succeed: FAILED: task failed") } -func TestJobRunWaitReportsFailedTask(t *testing.T) { - failed := &jobs.RunState{ - LifeCycleState: jobs.RunLifeCycleStateTerminated, - ResultState: jobs.RunResultStateFailed, - } - server := testserver.New(t) - server.Handle("GET", "/api/2.2/jobs/runs/get", func(req testserver.Request) any { - return jobs.Run{ - RunId: 123, - JobId: 456, - State: failed, - Tasks: []jobs.RunTask{ - {TaskKey: "ok", RunId: 998, State: &jobs.RunState{ - LifeCycleState: jobs.RunLifeCycleStateTerminated, - ResultState: jobs.RunResultStateSuccess, - }}, - {TaskKey: "main", RunId: 999, State: failed}, - }, - } - }) - server.Handle("GET", "/api/2.2/jobs/runs/get-output", func(req testserver.Request) any { - return jobs.RunOutput{Error: "notebook not found"} - }) - - _, err := waitForTestRun(t, t.Context(), jobRunClientFor(t, server)) - - require.ErrorContains(t, err, `task "main": notebook not found`) - assert.NotContains(t, err.Error(), `task "ok"`) -} - // Without the deprecated per-task state, a failed task is told apart from a // skipped one by its termination details. func TestJobRunWaitReportsFailedTaskWithoutDeprecatedState(t *testing.T) { @@ -257,39 +203,6 @@ func TestJobRunWaitFailsOnInternalError(t *testing.T) { require.ErrorContains(t, err, testRunPageLink) } -// A real workspace reports a run whose task failed as INTERNAL_ERROR in the -// deprecated life_cycle_state. The failing task still has to be named. -func TestJobRunWaitReportsFailedTaskOfInternalErrorRun(t *testing.T) { - server := testserver.New(t) - server.Handle("GET", "/api/2.2/jobs/runs/get", func(req testserver.Request) any { - return jobs.Run{ - RunId: 123, - JobId: 456, - RunPageUrl: testRunPageURL, - State: &jobs.RunState{ - LifeCycleState: jobs.RunLifeCycleStateInternalError, - ResultState: jobs.RunResultStateFailed, - StateMessage: "Task main failed with message: Workload failed, see run output for details.", - }, - Tasks: []jobs.RunTask{ - {TaskKey: "main", RunId: 999, State: &jobs.RunState{ - LifeCycleState: jobs.RunLifeCycleStateTerminated, - ResultState: jobs.RunResultStateFailed, - }}, - }, - } - }) - server.Handle("GET", "/api/2.2/jobs/runs/get-output", func(req testserver.Request) any { - return jobs.RunOutput{Error: "RuntimeError: intentional failure"} - }) - - _, err := waitForTestRun(t, t.Context(), jobRunClientFor(t, server)) - - require.ErrorContains(t, err, "run did not succeed: FAILED") - require.ErrorContains(t, err, `task "main": RuntimeError: intentional failure`) - require.ErrorContains(t, err, testRunPageLink) -} - func TestJobRunWaitReportsOnlyTheLastAttemptOfATask(t *testing.T) { failed := &jobs.RunState{ LifeCycleState: jobs.RunLifeCycleStateTerminated, @@ -332,64 +245,17 @@ func TestJobRunWaitAbandonedLinksTheRun(t *testing.T) { require.ErrorContains(t, err, testRunPageLink) } -// An abandoned wait leaves the run going with its id recorded, so the next deploy -// reads an empty outcome, which result_state drift catches. -func TestJobRunReadOfUnfinishedRunReportsNoResult(t *testing.T) { - client := jobRunClient(t, &jobs.RunState{LifeCycleState: jobs.RunLifeCycleStateRunning}) - - remote, err := (&ResourceJobRun{}).New(client).DoRead(t.Context(), "123") - - require.NoError(t, err) - require.NotNil(t, remote.State) - assert.Equal(t, jobs.RunLifeCycleStateRunning, remote.State.LifeCycleState) - assert.Empty(t, remote.ResultState) -} - -// PrepareState records the outcome the run must reach, the same for every run, -// so the planner has something to compare the remote against. -func TestJobRunPrepareStateRequiresSuccess(t *testing.T) { - state := (&ResourceJobRun{}).PrepareState(&resources.JobRun{RunNow: jobs.RunNow{JobId: 456}}) - - assert.Equal(t, jobs.RunResultStateSuccess, state.ResultState) - assert.Nil(t, state.Lifecycle.Triggers.OnFileChange) -} - -func TestJobRunPrepareStateOnBundleDeploy(t *testing.T) { +func TestJobRunPrepareStateBothTriggers(t *testing.T) { on := true - input := &resources.JobRun{ + hashes := map[string]string{"a.txt": "abc"} + state := (&ResourceJobRun{}).PrepareState(&resources.JobRun{ Lifecycle: &resources.JobRunLifecycle{ Triggers: []resources.JobRunTrigger{{OnBundleDeploy: &on}}, }, - } - first := (&ResourceJobRun{}).PrepareState(input) - assert.NotEmpty(t, first.Lifecycle.Triggers.OnBundleDeploy) - - second := (&ResourceJobRun{}).PrepareState(input) - assert.NotEqual(t, first.Lifecycle.Triggers.OnBundleDeploy, second.Lifecycle.Triggers.OnBundleDeploy) -} - -func TestJobRunPrepareStateOnFileChange(t *testing.T) { - hashes := map[string]string{"a.txt": "abc"} - - t.Run("armed", func(t *testing.T) { - state := (&ResourceJobRun{}).PrepareState(&resources.JobRun{ - ResolvedFileTriggers: hashes, - }) - assert.Equal(t, hashes, state.Lifecycle.Triggers.OnFileChange) - assert.Empty(t, state.Lifecycle.Triggers.OnBundleDeploy) - }) - - t.Run("both triggers", func(t *testing.T) { - on := true - state := (&ResourceJobRun{}).PrepareState(&resources.JobRun{ - Lifecycle: &resources.JobRunLifecycle{ - Triggers: []resources.JobRunTrigger{{OnBundleDeploy: &on}}, - }, - ResolvedFileTriggers: hashes, - }) - assert.NotEmpty(t, state.Lifecycle.Triggers.OnBundleDeploy) - assert.Equal(t, hashes, state.Lifecycle.Triggers.OnFileChange) + ResolvedFileTriggers: hashes, }) + assert.NotEmpty(t, state.Lifecycle.Triggers.OnBundleDeploy) + assert.Equal(t, hashes, state.Lifecycle.Triggers.OnFileChange) } func TestJobRunStateUnmarshalLifecycleCompatibility(t *testing.T) { @@ -414,72 +280,6 @@ func TestJobRunStateUnmarshalLifecycleCompatibility(t *testing.T) { }) } -func TestJobRunOverrideChangeDescTriggerRemoved(t *testing.T) { - r := &ResourceJobRun{} - - t.Run("clearing on_bundle_deploy leaf downgrades to skip", func(t *testing.T) { - change := &ChangeDesc{ - Action: deployplan.Recreate, - Old: "old", - New: "", - } - require.NoError(t, r.OverrideChangeDesc(t.Context(), structpath.MustParsePath("lifecycle.triggers.on_bundle_deploy"), change, nil)) - assert.Equal(t, deployplan.Skip, change.Action) - assert.Equal(t, "trigger removed", change.Reason) - }) - - t.Run("clearing on_file_change downgrades to skip", func(t *testing.T) { - change := &ChangeDesc{ - Action: deployplan.Recreate, - Old: map[string]string{"a.txt": "abc"}, - New: nil, - } - require.NoError(t, r.OverrideChangeDesc(t.Context(), structpath.MustParsePath("lifecycle.triggers.on_file_change"), change, nil)) - assert.Equal(t, deployplan.Skip, change.Action) - assert.Equal(t, "trigger removed", change.Reason) - }) - - t.Run("fresh fingerprint still recreates", func(t *testing.T) { - change := &ChangeDesc{ - Action: deployplan.Recreate, - Old: "old", - New: "new", - } - require.NoError(t, r.OverrideChangeDesc(t.Context(), structpath.MustParsePath("lifecycle.triggers.on_bundle_deploy"), change, nil)) - assert.Equal(t, deployplan.Recreate, change.Action) - }) - - t.Run("changed on_file_change hash still recreates", func(t *testing.T) { - change := &ChangeDesc{ - Action: deployplan.Recreate, - Old: "old", - New: "new", - } - require.NoError(t, r.OverrideChangeDesc(t.Context(), structpath.MustParsePath("lifecycle.triggers.on_file_change['a.txt']"), change, nil)) - assert.Equal(t, deployplan.Recreate, change.Action) - }) - - t.Run("removed matched file still recreates", func(t *testing.T) { - change := &ChangeDesc{ - Action: deployplan.Recreate, - Old: "old", - New: nil, - } - require.NoError(t, r.OverrideChangeDesc(t.Context(), structpath.MustParsePath("lifecycle.triggers.on_file_change['a.txt']"), change, nil)) - assert.Equal(t, deployplan.Recreate, change.Action) - }) - - t.Run("missing file fingerprint still recreates", func(t *testing.T) { - change := &ChangeDesc{ - Action: deployplan.Recreate, - Old: "old", - New: "", - } - require.NoError(t, r.OverrideChangeDesc(t.Context(), structpath.MustParsePath("lifecycle.triggers.on_file_change['a.txt']"), change, nil)) - assert.Equal(t, deployplan.Recreate, change.Action) - }) -} - // The planner diffs RemapState(remote) against PrepareState(config), so a run // that did not end in SUCCESS has to surface as a difference on result_state. func TestJobRunRemapStateCarriesTheOutcome(t *testing.T) { From ee9196bd7410ec336c28cf4cf4a7fb091cea1d42 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Mon, 24 Aug 2026 08:27:10 +0000 Subject: [PATCH 41/75] acc: refresh job_runs goldens for the omitted empty trigger Flattening on_file_change to a map made an unarmed trigger drop out of the serialized state, so the three plans that print an unarmed job_run now report an empty triggers object. --- acceptance/bundle/resources/job_runs/failed_run/output.txt | 4 +--- acceptance/bundle/resources/job_runs/redeploy/output.txt | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/acceptance/bundle/resources/job_runs/failed_run/output.txt b/acceptance/bundle/resources/job_runs/failed_run/output.txt index 37f1a831681..5b8b6dbcd5f 100644 --- a/acceptance/bundle/resources/job_runs/failed_run/output.txt +++ b/acceptance/bundle/resources/job_runs/failed_run/output.txt @@ -28,9 +28,7 @@ Files: 7 uploaded, 0 deleted "value": { "job_id": [MY_JOB_ID], "lifecycle": { - "triggers": { - "on_file_change": {} - } + "triggers": {} }, "result_state": "SUCCESS" } diff --git a/acceptance/bundle/resources/job_runs/redeploy/output.txt b/acceptance/bundle/resources/job_runs/redeploy/output.txt index a80685733ad..8532692bdd9 100644 --- a/acceptance/bundle/resources/job_runs/redeploy/output.txt +++ b/acceptance/bundle/resources/job_runs/redeploy/output.txt @@ -60,9 +60,7 @@ Resources: "env": "prod" }, "lifecycle": { - "triggers": { - "on_file_change": {} - } + "triggers": {} }, "result_state": "SUCCESS" } From 9e25ff87d7f7e9dd6a8f9eb134f895f85de50c10 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Mon, 24 Aug 2026 08:42:42 +0000 Subject: [PATCH 42/75] job_runs: cover cancel-before-delete with an acceptance test The unit test staged an asynchronous cancellation by hand; the acceptance test shows the real request order a user's destroy produces. Also drops two unit tests that the on_file_change acceptance tests already cover. --- .../destroy_unfinished_run/databricks.yml | 15 +++ .../destroy_unfinished_run/out.test.toml | 2 + .../destroy_unfinished_run/output.txt | 44 ++++++++ .../job_runs/destroy_unfinished_run/script | 18 ++++ bundle/direct/dresources/job_run_test.go | 100 +++--------------- 5 files changed, 94 insertions(+), 85 deletions(-) create mode 100644 acceptance/bundle/resources/job_runs/destroy_unfinished_run/databricks.yml create mode 100644 acceptance/bundle/resources/job_runs/destroy_unfinished_run/out.test.toml create mode 100644 acceptance/bundle/resources/job_runs/destroy_unfinished_run/output.txt create mode 100644 acceptance/bundle/resources/job_runs/destroy_unfinished_run/script diff --git a/acceptance/bundle/resources/job_runs/destroy_unfinished_run/databricks.yml b/acceptance/bundle/resources/job_runs/destroy_unfinished_run/databricks.yml new file mode 100644 index 00000000000..177a9ca2227 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/destroy_unfinished_run/databricks.yml @@ -0,0 +1,15 @@ +bundle: + name: job-runs-destroy-unfinished-run + +resources: + jobs: + my_job: + name: my-job + tasks: + - task_key: main + notebook_task: + notebook_path: /Workspace/test + + job_runs: + my_run: + job_id: ${resources.jobs.my_job.id} diff --git a/acceptance/bundle/resources/job_runs/destroy_unfinished_run/out.test.toml b/acceptance/bundle/resources/job_runs/destroy_unfinished_run/out.test.toml new file mode 100644 index 00000000000..0938e678987 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/destroy_unfinished_run/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/job_runs/destroy_unfinished_run/output.txt b/acceptance/bundle/resources/job_runs/destroy_unfinished_run/output.txt new file mode 100644 index 00000000000..1b1fd310b0c --- /dev/null +++ b/acceptance/bundle/resources/job_runs/destroy_unfinished_run/output.txt @@ -0,0 +1,44 @@ + +=== the deploy stops waiting before the run finishes +>>> errcode [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-destroy-unfinished-run/default/files... +Error: cannot create resources.job_runs.my_run: waiting after creating id=[MY_RUN_ID]: Fault injected by test. (403 INJECTED) + +Endpoint: GET [DATABRICKS_URL]/api/2.2/jobs/runs/get?run_id=[MY_RUN_ID] +HTTP Status: 403 Forbidden +API error_code: INJECTED +API message: Fault injected by test. + +Files: 4 uploaded, 0 deleted + +Exit code: 1 + +>>> read_id.py my_run +[MY_RUN_ID] + +=== destroy cancels the run before deleting it +>>> [CLI] bundle destroy --auto-approve +Warn: planning resources.job_runs.my_run: reading resources.job_runs.my_run id="[MY_RUN_ID]": Fault injected by test. +The following resources will be deleted: + delete resources.job_runs.my_run + delete resources.jobs.my_job + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/job-runs-destroy-unfinished-run/default + +Destroy: 2 deleted + +>>> print_requests.py //jobs/runs +{ + "method": "POST", + "path": "/api/2.2/jobs/runs/cancel", + "body": { + "run_id": [MY_RUN_ID] + } +} +{ + "method": "POST", + "path": "/api/2.2/jobs/runs/delete", + "body": { + "run_id": [MY_RUN_ID] + } +} diff --git a/acceptance/bundle/resources/job_runs/destroy_unfinished_run/script b/acceptance/bundle/resources/job_runs/destroy_unfinished_run/script new file mode 100644 index 00000000000..3b9914b1037 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/destroy_unfinished_run/script @@ -0,0 +1,18 @@ +cleanup() { + rm -f out.requests.txt +} +trap cleanup EXIT + +# The fake workspace settles a run on its first successful poll, so both the +# deploy's wait and the refresh destroy plans with are faulted: the run is still +# going when the delete reads it, which is what an interrupted deploy leaves behind. +fault.py "GET /api/2.2/jobs/runs/get" 403 0 2 + +title "the deploy stops waiting before the run finishes" +trace errcode $CLI bundle deploy +trace read_id.py my_run + +# jobs/runs/delete rejects an active run, so the run is cancelled first. +title "destroy cancels the run before deleting it" +trace $CLI bundle destroy --auto-approve +trace print_requests.py //jobs/runs diff --git a/bundle/direct/dresources/job_run_test.go b/bundle/direct/dresources/job_run_test.go index ad23ca605bc..804968f7707 100644 --- a/bundle/direct/dresources/job_run_test.go +++ b/bundle/direct/dresources/job_run_test.go @@ -10,7 +10,6 @@ import ( "testing" "time" - "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/cli/libs/testserver" "github.com/databricks/databricks-sdk-go" @@ -245,39 +244,13 @@ func TestJobRunWaitAbandonedLinksTheRun(t *testing.T) { require.ErrorContains(t, err, testRunPageLink) } -func TestJobRunPrepareStateBothTriggers(t *testing.T) { - on := true - hashes := map[string]string{"a.txt": "abc"} - state := (&ResourceJobRun{}).PrepareState(&resources.JobRun{ - Lifecycle: &resources.JobRunLifecycle{ - Triggers: []resources.JobRunTrigger{{OnBundleDeploy: &on}}, - }, - ResolvedFileTriggers: hashes, - }) - assert.NotEmpty(t, state.Lifecycle.Triggers.OnBundleDeploy) - assert.Equal(t, hashes, state.Lifecycle.Triggers.OnFileChange) -} +// State written before lifecycle existed has no such key, and must still load. +func TestJobRunStateUnmarshalWithoutLifecycle(t *testing.T) { + var state JobRunState -func TestJobRunStateUnmarshalLifecycleCompatibility(t *testing.T) { - t.Run("missing lifecycle gets empty shape", func(t *testing.T) { - var state JobRunState - require.NoError(t, json.Unmarshal([]byte(`{}`), &state)) - assert.Nil(t, state.Lifecycle.Triggers.OnFileChange) - }) + require.NoError(t, json.Unmarshal([]byte(`{}`), &state)) - t.Run("path-to-hash map is preserved", func(t *testing.T) { - var state JobRunState - require.NoError(t, json.Unmarshal([]byte(`{ - "lifecycle": { - "triggers": { - "on_file_change": { - "a.txt": "abc" - } - } - } - }`), &state)) - assert.Equal(t, map[string]string{"a.txt": "abc"}, state.Lifecycle.Triggers.OnFileChange) - }) + assert.Nil(t, state.Lifecycle.Triggers.OnFileChange) } // The planner diffs RemapState(remote) against PrepareState(config), so a run @@ -372,68 +345,25 @@ func TestJobRunCreateSendsAFreshIdempotencyToken(t *testing.T) { assert.Empty(t, config.IdempotencyToken) } -// jobRunDeletion records what the fake workspace saw while a run was deleted. -type jobRunDeletion struct { - cancelled atomic.Bool - settled atomic.Bool - settledAtDelete atomic.Bool -} - -// jobRunDeleteClient returns a client for a run in the given state, whose cancel -// settles one poll late the way the API's asynchronous cancellation does. -func jobRunDeleteClient(t *testing.T, state *jobs.RunState) (*databricks.WorkspaceClient, *jobRunDeletion) { - t.Helper() - var deletion jobRunDeletion - cancelled := &jobs.RunState{ - LifeCycleState: jobs.RunLifeCycleStateTerminated, - ResultState: jobs.RunResultStateCanceled, - } - +func TestJobRunDeleteLeavesFinishedRunAlone(t *testing.T) { + var cancelled atomic.Bool server := testserver.New(t) server.Handle("GET", "/api/2.2/jobs/runs/get", func(req testserver.Request) any { - current := state - switch { - case deletion.settled.Load(): - current = cancelled - case deletion.cancelled.Load(): - // Report the run's old state once more, then settle on the next poll. - deletion.settled.Store(true) - } - return jobs.Run{RunId: 123, JobId: 456, State: current} + return jobs.Run{RunId: 123, JobId: 456, State: &jobs.RunState{ + LifeCycleState: jobs.RunLifeCycleStateTerminated, + ResultState: jobs.RunResultStateSuccess, + }} }) server.Handle("POST", "/api/2.2/jobs/runs/cancel", func(req testserver.Request) any { - deletion.cancelled.Store(true) + cancelled.Store(true) return testserver.Response{} }) server.Handle("POST", "/api/2.2/jobs/runs/delete", func(req testserver.Request) any { - deletion.settledAtDelete.Store(deletion.settled.Load()) return testserver.Response{} }) - return jobRunClientFor(t, server), &deletion -} - -func deleteTestRun(t *testing.T, client *databricks.WorkspaceClient) error { - t.Helper() - return (&ResourceJobRun{}).New(client).DoDelete(t.Context(), "123", &JobRunState{}) -} - -func TestJobRunDeleteCancelsUnfinishedRun(t *testing.T) { - // An interrupted wait leaves the run going, and jobs/runs/delete rejects it. - client, deletion := jobRunDeleteClient(t, &jobs.RunState{LifeCycleState: jobs.RunLifeCycleStateRunning}) - - require.NoError(t, deleteTestRun(t, client)) - - assert.True(t, deletion.cancelled.Load(), "expected the run to be cancelled") - assert.True(t, deletion.settledAtDelete.Load(), "expected the delete to wait for the cancellation to settle") -} - -func TestJobRunDeleteLeavesFinishedRunAlone(t *testing.T) { - client, deletion := jobRunDeleteClient(t, &jobs.RunState{ - LifeCycleState: jobs.RunLifeCycleStateTerminated, - ResultState: jobs.RunResultStateSuccess, - }) + r := (&ResourceJobRun{}).New(jobRunClientFor(t, server)) - require.NoError(t, deleteTestRun(t, client)) + require.NoError(t, r.DoDelete(t.Context(), "123", &JobRunState{})) - assert.False(t, deletion.cancelled.Load(), "a run that already finished has nothing to cancel") + assert.False(t, cancelled.Load(), "a run that already finished has nothing to cancel") } From c255530fd59d6a7bf44c63e4ce5f524ba8487b52 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Mon, 24 Aug 2026 09:37:16 +0000 Subject: [PATCH 43/75] job_runs: error when on_file_change hashes nothing Directory-only and sync-excluded matches silently disarmed the trigger. Treat them as errors, warn that ** is not recursive, and cover both in acc. --- .../job_runs/destroy_unfinished_run/script | 5 ++- .../on_file_change_errors/allexcluded/a.txt | 1 + .../on_file_change_errors/databricks.yml | 6 ++++ .../on_file_change_errors/mixed/excluded.txt | 1 + .../mixed/nested/keep.txt | 1 + .../job_runs/on_file_change_errors/output.txt | 34 ++++++++++++++++--- .../job_runs/on_file_change_errors/script | 15 ++++++-- .../job_runs/on_file_change_glob/output.txt | 6 +++- .../job_runs/on_file_change_glob/script | 4 ++- .../mutator/resolve_job_run_file_triggers.go | 28 +++++++-------- 10 files changed, 75 insertions(+), 26 deletions(-) create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_errors/allexcluded/a.txt create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_errors/mixed/excluded.txt create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_errors/mixed/nested/keep.txt diff --git a/acceptance/bundle/resources/job_runs/destroy_unfinished_run/script b/acceptance/bundle/resources/job_runs/destroy_unfinished_run/script index 3b9914b1037..61074d97eea 100644 --- a/acceptance/bundle/resources/job_runs/destroy_unfinished_run/script +++ b/acceptance/bundle/resources/job_runs/destroy_unfinished_run/script @@ -3,9 +3,8 @@ cleanup() { } trap cleanup EXIT -# The fake workspace settles a run on its first successful poll, so both the -# deploy's wait and the refresh destroy plans with are faulted: the run is still -# going when the delete reads it, which is what an interrupted deploy leaves behind. +# Fake workspace settles a run on the first successful poll, so both the deploy +# wait and destroy's refresh are faulted: delete sees an in-progress run. fault.py "GET /api/2.2/jobs/runs/get" 403 0 2 title "the deploy stops waiting before the run finishes" diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/allexcluded/a.txt b/acceptance/bundle/resources/job_runs/on_file_change_errors/allexcluded/a.txt new file mode 100644 index 00000000000..78981922613 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/allexcluded/a.txt @@ -0,0 +1 @@ +a diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/databricks.yml b/acceptance/bundle/resources/job_runs/on_file_change_errors/databricks.yml index 44c5433e52a..0ce9819887c 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_errors/databricks.yml +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/databricks.yml @@ -1,6 +1,12 @@ bundle: name: job-runs-on-file-change-errors +sync: + exclude: + - mixed/excluded.txt + # Literal path: the script rewrites the trigger by substring, and allexcluded/* would hit this line. + - allexcluded/a.txt + resources: jobs: my_job: diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/mixed/excluded.txt b/acceptance/bundle/resources/job_runs/on_file_change_errors/mixed/excluded.txt new file mode 100644 index 00000000000..bbde3dc9de9 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/mixed/excluded.txt @@ -0,0 +1 @@ +excluded diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/mixed/nested/keep.txt b/acceptance/bundle/resources/job_runs/on_file_change_errors/mixed/nested/keep.txt new file mode 100644 index 00000000000..2fa992c0b8b --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/mixed/nested/keep.txt @@ -0,0 +1 @@ +keep diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt index c7d3a98e1cb..381dc3e114f 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt @@ -2,7 +2,7 @@ === a pattern outside the sync root is rejected >>> musterr [CLI] bundle validate Error: lifecycle.triggers.on_file_change: pattern "../shared/*.txt" is not under the sync root - in databricks.yml:18:29 + in databricks.yml:25:29 Name: job-runs-on-file-change-errors Target: default @@ -14,8 +14,34 @@ Found 1 error === a glob that only matches directories is rejected >>> musterr [CLI] bundle validate -Error: lifecycle.triggers.on_file_change: pattern "dirs/*" matches no regular files - in databricks.yml:18:29 +Error: lifecycle.triggers.on_file_change: pattern "dirs/*" matches only directories or files excluded from sync, leaving nothing to hash + in databricks.yml:25:29 + +Name: job-runs-on-file-change-errors +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-errors/default + +Found 1 error + +=== a glob matching only a directory and an excluded file is rejected +>>> musterr [CLI] bundle validate +Error: lifecycle.triggers.on_file_change: pattern "mixed/*" matches only directories or files excluded from sync, leaving nothing to hash + in databricks.yml:25:29 + +Name: job-runs-on-file-change-errors +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-errors/default + +Found 1 error + +=== a glob matching only excluded files is rejected +>>> musterr [CLI] bundle validate +Error: lifecycle.triggers.on_file_change: pattern "allexcluded/*" matches only directories or files excluded from sync, leaving nothing to hash + in databricks.yml:25:29 Name: job-runs-on-file-change-errors Target: default @@ -28,7 +54,7 @@ Found 1 error === a pattern that matches nothing warns >>> [CLI] bundle validate Warning: lifecycle.triggers.on_file_change: no files match "missing/*.txt" - in databricks.yml:18:29 + in databricks.yml:25:29 Name: job-runs-on-file-change-errors Target: default diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/script b/acceptance/bundle/resources/job_runs/on_file_change_errors/script index 39596f0c772..48b9c5c7fc4 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_errors/script +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/script @@ -2,11 +2,20 @@ title "a pattern outside the sync root is rejected" trace musterr $CLI bundle validate title "a glob that only matches directories is rejected" -# dirs/* matches dirs/nested, a directory: nothing is left to hash, so the -# trigger would silently never fire. +# dirs/* matches dirs/nested, a directory, so nothing is left to hash. update_file.py databricks.yml "../shared/*.txt" "dirs/*" trace musterr $CLI bundle validate +title "a glob matching only a directory and an excluded file is rejected" +# mixed/excluded.txt is dropped by sync.exclude; mixed/nested is a directory. +update_file.py databricks.yml "dirs/*" "mixed/*" +trace musterr $CLI bundle validate + +title "a glob matching only excluded files is rejected" +# Every match is dropped by sync.exclude; unlike a missing pattern, this cannot re-arm later. +update_file.py databricks.yml "mixed/*" "allexcluded/*" +trace musterr $CLI bundle validate + title "a pattern that matches nothing warns" -update_file.py databricks.yml "dirs/*" "missing/*.txt" +update_file.py databricks.yml "allexcluded/*" "missing/*.txt" trace $CLI bundle validate diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt index a04e3710fb4..2f66f33e664 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt @@ -200,7 +200,11 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged -=== ** is not recursive: it matches the same files as * +=== ** is not recursive: it warns and matches the same files as * +Warning: lifecycle.triggers.on_file_change: ** in "migrations/**" is not recursive and matches the same files as * + in databricks.yml:18:29 + + >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { "lifecycle.triggers.on_file_change": { diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/script b/acceptance/bundle/resources/job_runs/on_file_change_glob/script index dd737f242b5..0f7107a43db 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/script +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/script @@ -58,6 +58,8 @@ update_file.py migrations/ignored.txt "ignored" "ignored-edited" trace $CLI bundle plan update_file.py migrations/ignored.txt "ignored-edited" "ignored" -title "** is not recursive: it matches the same files as *" +# Warning is on stderr of the untraced plan in changes(); trailing newline keeps it off the title. +title "** is not recursive: it warns and matches the same files as *\n" update_file.py databricks.yml "migrations/*" "migrations/**" changes +update_file.py databricks.yml "migrations/**" "migrations/*" diff --git a/bundle/config/mutator/resolve_job_run_file_triggers.go b/bundle/config/mutator/resolve_job_run_file_triggers.go index d74b9841c57..fbbabb60815 100644 --- a/bundle/config/mutator/resolve_job_run_file_triggers.go +++ b/bundle/config/mutator/resolve_job_run_file_triggers.go @@ -108,6 +108,14 @@ func resolveFileTrigger(b *bundle.Bundle, loc, pattern string, syncable map[stri Locations: b.Config.GetLocations(loc), }) } + // filepath.Glob treats ** as two *, so doublestar-style patterns match less than expected. + if strings.Contains(pattern, "**") { + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Warning, + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: ** in %q is not recursive and matches the same files as *", pattern), + Locations: b.Config.GetLocations(loc), + }) + } matches, err := filepath.Glob(filepath.Join(b.SyncRootPath, localPattern)) if err != nil { return out, diags.Append(diag.Diagnostic{ @@ -150,7 +158,7 @@ func resolveFileTrigger(b *bundle.Bundle, loc, pattern string, syncable map[stri }) continue } - // Same membership as sync: .gitignore and sync.exclude drop a glob match. + // Honor .gitignore and sync.exclude the same way sync does. if _, ok := syncable[filepath.ToSlash(rel)]; !ok { ignoredMatches++ continue @@ -167,20 +175,12 @@ func resolveFileTrigger(b *bundle.Bundle, loc, pattern string, syncable map[stri } out[filepath.ToSlash(rel)] = hash } - // A directory-only match would otherwise leave ResolvedFileTriggers empty - // and silently disarm the trigger while config still sets on_file_change. - if regularMatches == 0 && sawNonRegular && ignoredMatches == 0 { - diags = diags.Append(diag.Diagnostic{ + // Directories or excluded files would leave hashes empty and disarm the trigger. + // Unlike a missing-file warning, this cannot re-arm when a file appears later. + if regularMatches == 0 && (sawNonRegular || ignoredMatches > 0) { + return out, diags.Append(diag.Diagnostic{ Severity: diag.Error, - Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: pattern %q matches no regular files", pattern), - Locations: b.Config.GetLocations(loc), - }) - } - if len(out) == 0 && ignoredMatches > 0 { - out[filepath.ToSlash(pattern)] = missingFileHash - diags = diags.Append(diag.Diagnostic{ - Severity: diag.Warning, - Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: no files match %q", pattern), + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: pattern %q matches only directories or files excluded from sync, leaving nothing to hash", pattern), Locations: b.Config.GetLocations(loc), }) } From 79b269e22894874b72a55c6850a966b1cf26f978 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Mon, 24 Aug 2026 11:58:36 +0000 Subject: [PATCH 44/75] acc: fix on_file_change_errors diagnostic line numbers The goldens were captured against a one-line-taller databricks.yml, so CI failed after the source settled on line 24. --- .../job_runs/on_file_change_errors/output.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt index 381dc3e114f..15934e8eeb6 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt @@ -2,7 +2,7 @@ === a pattern outside the sync root is rejected >>> musterr [CLI] bundle validate Error: lifecycle.triggers.on_file_change: pattern "../shared/*.txt" is not under the sync root - in databricks.yml:25:29 + in databricks.yml:24:29 Name: job-runs-on-file-change-errors Target: default @@ -15,7 +15,7 @@ Found 1 error === a glob that only matches directories is rejected >>> musterr [CLI] bundle validate Error: lifecycle.triggers.on_file_change: pattern "dirs/*" matches only directories or files excluded from sync, leaving nothing to hash - in databricks.yml:25:29 + in databricks.yml:24:29 Name: job-runs-on-file-change-errors Target: default @@ -28,7 +28,7 @@ Found 1 error === a glob matching only a directory and an excluded file is rejected >>> musterr [CLI] bundle validate Error: lifecycle.triggers.on_file_change: pattern "mixed/*" matches only directories or files excluded from sync, leaving nothing to hash - in databricks.yml:25:29 + in databricks.yml:24:29 Name: job-runs-on-file-change-errors Target: default @@ -41,7 +41,7 @@ Found 1 error === a glob matching only excluded files is rejected >>> musterr [CLI] bundle validate Error: lifecycle.triggers.on_file_change: pattern "allexcluded/*" matches only directories or files excluded from sync, leaving nothing to hash - in databricks.yml:25:29 + in databricks.yml:24:29 Name: job-runs-on-file-change-errors Target: default @@ -54,7 +54,7 @@ Found 1 error === a pattern that matches nothing warns >>> [CLI] bundle validate Warning: lifecycle.triggers.on_file_change: no files match "missing/*.txt" - in databricks.yml:25:29 + in databricks.yml:24:29 Name: job-runs-on-file-change-errors Target: default From 4e7b960870d845f08e7e0e128c6213574b2dd7b3 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Mon, 24 Aug 2026 13:11:35 +0000 Subject: [PATCH 45/75] acc: removing on_file_change is unchanged, not an update Clearing a trigger is now a skip, so the plan reports the run as unchanged and the deploy makes no API call. --- .../bundle/resources/job_runs/on_file_change/output.txt | 9 +++------ .../bundle/resources/job_runs/on_file_change/script | 2 +- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/acceptance/bundle/resources/job_runs/on_file_change/output.txt b/acceptance/bundle/resources/job_runs/on_file_change/output.txt index 28deeee38a4..978e4b4394a 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change/output.txt @@ -186,17 +186,14 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged } } -=== removing on_file_change rewrites state without a run +=== removing on_file_change is unchanged and does not run >>> [CLI] bundle plan -update job_runs.my_run - -Plan: 0 to add, 1 to change, 0 to delete, 1 unchanged +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change/default/files... -Updated job_runs.my_run Files: 3 uploaded, 0 deleted -Resources: 0 created, 1 changed, 0 deleted, 1 unchanged +Resources: 0 created, 0 changed, 0 deleted, 2 unchanged >>> print_requests.py //jobs/run-now diff --git a/acceptance/bundle/resources/job_runs/on_file_change/script b/acceptance/bundle/resources/job_runs/on_file_change/script index fd1e8800c42..ef432d76033 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change/script +++ b/acceptance/bundle/resources/job_runs/on_file_change/script @@ -54,7 +54,7 @@ trace $CLI bundle deploy read_id.py my_run > /dev/null trace print_requests.py //jobs/run-now -title "removing on_file_change rewrites state without a run" +title "removing on_file_change is unchanged and does not run" update_file.py databricks.yml " lifecycle: triggers: - on_file_change: seed.txt From 3865b6ba18b08900fe56ea2beecbc6806cf7b285 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Mon, 24 Aug 2026 14:31:27 +0000 Subject: [PATCH 46/75] job_runs: drop on_value_change unit tests covered by acceptance on_value_change_ref already asserts depends_on and persisted resolved ids. --- bundle/direct/dresources/job_run.go | 6 ++-- bundle/direct/dresources/job_run_test.go | 42 ------------------------ 2 files changed, 4 insertions(+), 44 deletions(-) diff --git a/bundle/direct/dresources/job_run.go b/bundle/direct/dresources/job_run.go index ee0fbbfda5b..144190850ef 100644 --- a/bundle/direct/dresources/job_run.go +++ b/bundle/direct/dresources/job_run.go @@ -142,6 +142,8 @@ func (*ResourceJobRun) PrepareState(input *resources.JobRun) *JobRunState { return state } +// PrepareInputConfig puts resource refs on the state path of each watched +// expression so the deploy graph depends on that value, not the config wrapper. func (*ResourceJobRun) PrepareInputConfig(input *resources.JobRun, _ string) (*structvar.StructVar, error) { refs := map[string]string{} for expr := range jobRunValueChangeState(input) { @@ -179,8 +181,8 @@ func jobRunValueChangeState(input *resources.JobRun) map[string]string { } // DropJobRunValueChangeConfigRefs drops lifecycle.triggers[N].on_value_change. -// ExtractReferences treats [0] on the triggers struct as a no-op, so that path -// is the wrapper, which cannot hold a resolved id. +// ExtractReferences treats [0] on a struct as a no-op, so that path is the +// wrapper and cannot hold the resolved id. func DropJobRunValueChangeConfigRefs(refs map[string]string) { for k := range refs { if strings.Contains(k, ".triggers[") && strings.HasSuffix(k, "].on_value_change") { diff --git a/bundle/direct/dresources/job_run_test.go b/bundle/direct/dresources/job_run_test.go index da325b143b9..804968f7707 100644 --- a/bundle/direct/dresources/job_run_test.go +++ b/bundle/direct/dresources/job_run_test.go @@ -10,7 +10,6 @@ import ( "testing" "time" - "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/cli/libs/testserver" "github.com/databricks/databricks-sdk-go" @@ -254,47 +253,6 @@ func TestJobRunStateUnmarshalWithoutLifecycle(t *testing.T) { assert.Nil(t, state.Lifecycle.Triggers.OnFileChange) } -// The ref path is what the deploy graph keys the dependency on, so it has to be -// the state path of the watched expression, not the config path it came from. -func TestJobRunPrepareInputConfigOnValueChange(t *testing.T) { - expr := "${resources.jobs.foo.id}" - literal := "v1" - input := &resources.JobRun{ - Lifecycle: &resources.JobRunLifecycle{ - Triggers: []resources.JobRunTrigger{ - {OnValueChange: &expr}, - {OnValueChange: &literal}, - }, - }, - } - - sv, err := (&ResourceJobRun{}).PrepareInputConfig(input, "resources.job_runs.my_run") - - require.NoError(t, err) - assert.Same(t, input, sv.Value) - // Only the interpolated expression is a ref; the literal has nothing to resolve. - path := structpath.NewStringKey(structpath.MustParsePath("lifecycle.triggers.on_value_change"), expr) - assert.Equal(t, map[string]string{path.String(): expr}, sv.Refs) -} - -func TestDropJobRunValueChangeConfigRefs(t *testing.T) { - expr := "${resources.jobs.other.id}" - statePath := structpath.NewStringKey(structpath.MustParsePath("lifecycle.triggers.on_value_change"), expr).String() - refs := map[string]string{ - "lifecycle.triggers[0].on_value_change": expr, - "job_id": "${resources.jobs.my_job.id}", - statePath: expr, - } - - DropJobRunValueChangeConfigRefs(refs) - - // The config path is dropped; the state path that carries the resolved id stays. - assert.Equal(t, map[string]string{ - "job_id": "${resources.jobs.my_job.id}", - statePath: expr, - }, refs) -} - // The planner diffs RemapState(remote) against PrepareState(config), so a run // that did not end in SUCCESS has to surface as a difference on result_state. func TestJobRunRemapStateCarriesTheOutcome(t *testing.T) { From 74b03cac9dcb693f470251bfa193fcc870fcd499 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Mon, 24 Aug 2026 16:39:23 +0000 Subject: [PATCH 47/75] job_runs: drop unit tests covered by acceptance failed_run, on_bundle_deploy, wait, and interrupted_run already assert the user-visible wait and trigger behavior. Add destroy_unfinished_run so cancel-before-delete is covered the same way. --- .../destroy_unfinished_run/databricks.yml | 15 ++ .../destroy_unfinished_run/out.test.toml | 2 + .../destroy_unfinished_run/output.txt | 44 ++++ .../job_runs/destroy_unfinished_run/script | 17 ++ bundle/direct/dresources/job_run_test.go | 231 +----------------- 5 files changed, 88 insertions(+), 221 deletions(-) create mode 100644 acceptance/bundle/resources/job_runs/destroy_unfinished_run/databricks.yml create mode 100644 acceptance/bundle/resources/job_runs/destroy_unfinished_run/out.test.toml create mode 100644 acceptance/bundle/resources/job_runs/destroy_unfinished_run/output.txt create mode 100755 acceptance/bundle/resources/job_runs/destroy_unfinished_run/script diff --git a/acceptance/bundle/resources/job_runs/destroy_unfinished_run/databricks.yml b/acceptance/bundle/resources/job_runs/destroy_unfinished_run/databricks.yml new file mode 100644 index 00000000000..177a9ca2227 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/destroy_unfinished_run/databricks.yml @@ -0,0 +1,15 @@ +bundle: + name: job-runs-destroy-unfinished-run + +resources: + jobs: + my_job: + name: my-job + tasks: + - task_key: main + notebook_task: + notebook_path: /Workspace/test + + job_runs: + my_run: + job_id: ${resources.jobs.my_job.id} diff --git a/acceptance/bundle/resources/job_runs/destroy_unfinished_run/out.test.toml b/acceptance/bundle/resources/job_runs/destroy_unfinished_run/out.test.toml new file mode 100644 index 00000000000..0938e678987 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/destroy_unfinished_run/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/job_runs/destroy_unfinished_run/output.txt b/acceptance/bundle/resources/job_runs/destroy_unfinished_run/output.txt new file mode 100644 index 00000000000..1b1fd310b0c --- /dev/null +++ b/acceptance/bundle/resources/job_runs/destroy_unfinished_run/output.txt @@ -0,0 +1,44 @@ + +=== the deploy stops waiting before the run finishes +>>> errcode [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-destroy-unfinished-run/default/files... +Error: cannot create resources.job_runs.my_run: waiting after creating id=[MY_RUN_ID]: Fault injected by test. (403 INJECTED) + +Endpoint: GET [DATABRICKS_URL]/api/2.2/jobs/runs/get?run_id=[MY_RUN_ID] +HTTP Status: 403 Forbidden +API error_code: INJECTED +API message: Fault injected by test. + +Files: 4 uploaded, 0 deleted + +Exit code: 1 + +>>> read_id.py my_run +[MY_RUN_ID] + +=== destroy cancels the run before deleting it +>>> [CLI] bundle destroy --auto-approve +Warn: planning resources.job_runs.my_run: reading resources.job_runs.my_run id="[MY_RUN_ID]": Fault injected by test. +The following resources will be deleted: + delete resources.job_runs.my_run + delete resources.jobs.my_job + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/job-runs-destroy-unfinished-run/default + +Destroy: 2 deleted + +>>> print_requests.py //jobs/runs +{ + "method": "POST", + "path": "/api/2.2/jobs/runs/cancel", + "body": { + "run_id": [MY_RUN_ID] + } +} +{ + "method": "POST", + "path": "/api/2.2/jobs/runs/delete", + "body": { + "run_id": [MY_RUN_ID] + } +} diff --git a/acceptance/bundle/resources/job_runs/destroy_unfinished_run/script b/acceptance/bundle/resources/job_runs/destroy_unfinished_run/script new file mode 100755 index 00000000000..61074d97eea --- /dev/null +++ b/acceptance/bundle/resources/job_runs/destroy_unfinished_run/script @@ -0,0 +1,17 @@ +cleanup() { + rm -f out.requests.txt +} +trap cleanup EXIT + +# Fake workspace settles a run on the first successful poll, so both the deploy +# wait and destroy's refresh are faulted: delete sees an in-progress run. +fault.py "GET /api/2.2/jobs/runs/get" 403 0 2 + +title "the deploy stops waiting before the run finishes" +trace errcode $CLI bundle deploy +trace read_id.py my_run + +# jobs/runs/delete rejects an active run, so the run is cancelled first. +title "destroy cancels the run before deleting it" +trace $CLI bundle destroy --auto-approve +trace print_requests.py //jobs/runs diff --git a/bundle/direct/dresources/job_run_test.go b/bundle/direct/dresources/job_run_test.go index cef8cadecce..0ffc1c48040 100644 --- a/bundle/direct/dresources/job_run_test.go +++ b/bundle/direct/dresources/job_run_test.go @@ -10,9 +10,6 @@ import ( "testing" "time" - "github.com/databricks/cli/bundle/config/resources" - "github.com/databricks/cli/bundle/deployplan" - "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/cli/libs/testserver" "github.com/databricks/databricks-sdk-go" @@ -68,28 +65,6 @@ func waitForTestRun(t *testing.T, ctx context.Context, client *databricks.Worksp return r.WaitAfterCreate(ctx, "123", &JobRunState{}) } -func TestJobRunWaitSucceeds(t *testing.T) { - client := jobRunClient(t, &jobs.RunState{ - LifeCycleState: jobs.RunLifeCycleStateTerminated, - ResultState: jobs.RunResultStateSuccess, - }) - - remote, err := waitForTestRun(t, t.Context(), client) - - require.NoError(t, err) - require.NotNil(t, remote.State) - assert.Equal(t, jobs.RunResultStateSuccess, remote.State.ResultState) -} - -func TestReportRunLineIncludesResourceKey(t *testing.T) { - ctx, stderr := cmdio.NewTestContextWithStderr(t.Context()) - ctx = WithResourceKey(ctx, "job_runs.my_run") - - reportRunLine(ctx, 123, "SUCCESS") - - assert.Equal(t, "Output from job_runs.my_run: id=123: SUCCESS\n", stderr.String()) -} - func TestJobRunWaitFailsOnFailedResult(t *testing.T) { client := jobRunClient(t, &jobs.RunState{ LifeCycleState: jobs.RunLifeCycleStateTerminated, @@ -102,36 +77,6 @@ func TestJobRunWaitFailsOnFailedResult(t *testing.T) { require.ErrorContains(t, err, "did not succeed: FAILED: task failed") } -func TestJobRunWaitReportsFailedTask(t *testing.T) { - failed := &jobs.RunState{ - LifeCycleState: jobs.RunLifeCycleStateTerminated, - ResultState: jobs.RunResultStateFailed, - } - server := testserver.New(t) - server.Handle("GET", "/api/2.2/jobs/runs/get", func(req testserver.Request) any { - return jobs.Run{ - RunId: 123, - JobId: 456, - State: failed, - Tasks: []jobs.RunTask{ - {TaskKey: "ok", RunId: 998, State: &jobs.RunState{ - LifeCycleState: jobs.RunLifeCycleStateTerminated, - ResultState: jobs.RunResultStateSuccess, - }}, - {TaskKey: "main", RunId: 999, State: failed}, - }, - } - }) - server.Handle("GET", "/api/2.2/jobs/runs/get-output", func(req testserver.Request) any { - return jobs.RunOutput{Error: "notebook not found"} - }) - - _, err := waitForTestRun(t, t.Context(), jobRunClientFor(t, server)) - - require.ErrorContains(t, err, `task "main": notebook not found`) - assert.NotContains(t, err.Error(), `task "ok"`) -} - // Without the deprecated per-task state, a failed task is told apart from a // skipped one by its termination details. func TestJobRunWaitReportsFailedTaskWithoutDeprecatedState(t *testing.T) { @@ -257,39 +202,6 @@ func TestJobRunWaitFailsOnInternalError(t *testing.T) { require.ErrorContains(t, err, testRunPageLink) } -// A real workspace reports a run whose task failed as INTERNAL_ERROR in the -// deprecated life_cycle_state. The failing task still has to be named. -func TestJobRunWaitReportsFailedTaskOfInternalErrorRun(t *testing.T) { - server := testserver.New(t) - server.Handle("GET", "/api/2.2/jobs/runs/get", func(req testserver.Request) any { - return jobs.Run{ - RunId: 123, - JobId: 456, - RunPageUrl: testRunPageURL, - State: &jobs.RunState{ - LifeCycleState: jobs.RunLifeCycleStateInternalError, - ResultState: jobs.RunResultStateFailed, - StateMessage: "Task main failed with message: Workload failed, see run output for details.", - }, - Tasks: []jobs.RunTask{ - {TaskKey: "main", RunId: 999, State: &jobs.RunState{ - LifeCycleState: jobs.RunLifeCycleStateTerminated, - ResultState: jobs.RunResultStateFailed, - }}, - }, - } - }) - server.Handle("GET", "/api/2.2/jobs/runs/get-output", func(req testserver.Request) any { - return jobs.RunOutput{Error: "RuntimeError: intentional failure"} - }) - - _, err := waitForTestRun(t, t.Context(), jobRunClientFor(t, server)) - - require.ErrorContains(t, err, "run did not succeed: FAILED") - require.ErrorContains(t, err, `task "main": RuntimeError: intentional failure`) - require.ErrorContains(t, err, testRunPageLink) -} - func TestJobRunWaitReportsOnlyTheLastAttemptOfATask(t *testing.T) { failed := &jobs.RunState{ LifeCycleState: jobs.RunLifeCycleStateTerminated, @@ -332,86 +244,6 @@ func TestJobRunWaitAbandonedLinksTheRun(t *testing.T) { require.ErrorContains(t, err, testRunPageLink) } -// An abandoned wait leaves the run going with its id recorded, so the next deploy -// reads an empty outcome, which result_state drift catches. -func TestJobRunReadOfUnfinishedRunReportsNoResult(t *testing.T) { - client := jobRunClient(t, &jobs.RunState{LifeCycleState: jobs.RunLifeCycleStateRunning}) - - remote, err := (&ResourceJobRun{}).New(client).DoRead(t.Context(), "123") - - require.NoError(t, err) - require.NotNil(t, remote.State) - assert.Equal(t, jobs.RunLifeCycleStateRunning, remote.State.LifeCycleState) - assert.Empty(t, remote.ResultState) -} - -// PrepareState records the outcome the run must reach, the same for every run, -// so the planner has something to compare the remote against. -func TestJobRunPrepareStateRequiresSuccess(t *testing.T) { - state := (&ResourceJobRun{}).PrepareState(&resources.JobRun{RunNow: jobs.RunNow{JobId: 456}}) - - assert.Equal(t, jobs.RunResultStateSuccess, state.ResultState) -} - -func TestJobRunPrepareStateOnBundleDeploy(t *testing.T) { - t.Run("unset", func(t *testing.T) { - state := (&ResourceJobRun{}).PrepareState(&resources.JobRun{}) - assert.Nil(t, state.Lifecycle) - }) - - t.Run("armed", func(t *testing.T) { - on := true - input := &resources.JobRun{ - Lifecycle: &resources.JobRunLifecycle{ - Triggers: []resources.JobRunTrigger{{OnBundleDeploy: &on}}, - }, - } - first := (&ResourceJobRun{}).PrepareState(input) - require.NotNil(t, first.Lifecycle) - require.NotNil(t, first.Lifecycle.Triggers) - assert.NotEmpty(t, first.Lifecycle.Triggers.OnBundleDeploy) - - second := (&ResourceJobRun{}).PrepareState(input) - assert.NotEqual(t, first.Lifecycle.Triggers.OnBundleDeploy, second.Lifecycle.Triggers.OnBundleDeploy) - }) -} - -func TestJobRunOverrideChangeDescTriggerRemoved(t *testing.T) { - r := &ResourceJobRun{} - - t.Run("clearing lifecycle downgrades to skip", func(t *testing.T) { - change := &ChangeDesc{ - Action: deployplan.Recreate, - Old: &JobRunLifecycleState{Triggers: &JobRunTriggersState{OnBundleDeploy: "old"}}, - New: nil, - } - require.NoError(t, r.OverrideChangeDesc(t.Context(), structpath.MustParsePath("lifecycle"), change, nil)) - assert.Equal(t, deployplan.Skip, change.Action) - assert.Equal(t, "trigger removed", change.Reason) - }) - - t.Run("clearing on_bundle_deploy leaf downgrades to skip", func(t *testing.T) { - change := &ChangeDesc{ - Action: deployplan.Recreate, - Old: "old", - New: "", - } - require.NoError(t, r.OverrideChangeDesc(t.Context(), structpath.MustParsePath("lifecycle.triggers.on_bundle_deploy"), change, nil)) - assert.Equal(t, deployplan.Skip, change.Action) - assert.Equal(t, "trigger removed", change.Reason) - }) - - t.Run("fresh fingerprint still recreates", func(t *testing.T) { - change := &ChangeDesc{ - Action: deployplan.Recreate, - Old: "old", - New: "new", - } - require.NoError(t, r.OverrideChangeDesc(t.Context(), structpath.MustParsePath("lifecycle.triggers.on_bundle_deploy"), change, nil)) - assert.Equal(t, deployplan.Recreate, change.Action) - }) -} - // The planner diffs RemapState(remote) against PrepareState(config), so a run // that did not end in SUCCESS has to surface as a difference on result_state. func TestJobRunRemapStateCarriesTheOutcome(t *testing.T) { @@ -503,68 +335,25 @@ func TestJobRunCreateSendsAFreshIdempotencyToken(t *testing.T) { assert.Empty(t, config.IdempotencyToken) } -// jobRunDeletion records what the fake workspace saw while a run was deleted. -type jobRunDeletion struct { - cancelled atomic.Bool - settled atomic.Bool - settledAtDelete atomic.Bool -} - -// jobRunDeleteClient returns a client for a run in the given state, whose cancel -// settles one poll late the way the API's asynchronous cancellation does. -func jobRunDeleteClient(t *testing.T, state *jobs.RunState) (*databricks.WorkspaceClient, *jobRunDeletion) { - t.Helper() - var deletion jobRunDeletion - cancelled := &jobs.RunState{ - LifeCycleState: jobs.RunLifeCycleStateTerminated, - ResultState: jobs.RunResultStateCanceled, - } - +func TestJobRunDeleteLeavesFinishedRunAlone(t *testing.T) { + var cancelled atomic.Bool server := testserver.New(t) server.Handle("GET", "/api/2.2/jobs/runs/get", func(req testserver.Request) any { - current := state - switch { - case deletion.settled.Load(): - current = cancelled - case deletion.cancelled.Load(): - // Report the run's old state once more, then settle on the next poll. - deletion.settled.Store(true) - } - return jobs.Run{RunId: 123, JobId: 456, State: current} + return jobs.Run{RunId: 123, JobId: 456, State: &jobs.RunState{ + LifeCycleState: jobs.RunLifeCycleStateTerminated, + ResultState: jobs.RunResultStateSuccess, + }} }) server.Handle("POST", "/api/2.2/jobs/runs/cancel", func(req testserver.Request) any { - deletion.cancelled.Store(true) + cancelled.Store(true) return testserver.Response{} }) server.Handle("POST", "/api/2.2/jobs/runs/delete", func(req testserver.Request) any { - deletion.settledAtDelete.Store(deletion.settled.Load()) return testserver.Response{} }) - return jobRunClientFor(t, server), &deletion -} - -func deleteTestRun(t *testing.T, client *databricks.WorkspaceClient) error { - t.Helper() - return (&ResourceJobRun{}).New(client).DoDelete(t.Context(), "123", &JobRunState{}) -} - -func TestJobRunDeleteCancelsUnfinishedRun(t *testing.T) { - // An interrupted wait leaves the run going, and jobs/runs/delete rejects it. - client, deletion := jobRunDeleteClient(t, &jobs.RunState{LifeCycleState: jobs.RunLifeCycleStateRunning}) - - require.NoError(t, deleteTestRun(t, client)) - - assert.True(t, deletion.cancelled.Load(), "expected the run to be cancelled") - assert.True(t, deletion.settledAtDelete.Load(), "expected the delete to wait for the cancellation to settle") -} - -func TestJobRunDeleteLeavesFinishedRunAlone(t *testing.T) { - client, deletion := jobRunDeleteClient(t, &jobs.RunState{ - LifeCycleState: jobs.RunLifeCycleStateTerminated, - ResultState: jobs.RunResultStateSuccess, - }) + r := (&ResourceJobRun{}).New(jobRunClientFor(t, server)) - require.NoError(t, deleteTestRun(t, client)) + require.NoError(t, r.DoDelete(t.Context(), "123", &JobRunState{})) - assert.False(t, deletion.cancelled.Load(), "a run that already finished has nothing to cancel") + assert.False(t, cancelled.Load(), "a run that already finished has nothing to cancel") } From 32850575e1829d108d4eb10ba97d430e45beead8 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Mon, 24 Aug 2026 16:40:05 +0000 Subject: [PATCH 48/75] acc: match job_runs script file mode --- .../bundle/resources/job_runs/destroy_unfinished_run/script | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 acceptance/bundle/resources/job_runs/destroy_unfinished_run/script diff --git a/acceptance/bundle/resources/job_runs/destroy_unfinished_run/script b/acceptance/bundle/resources/job_runs/destroy_unfinished_run/script old mode 100755 new mode 100644 From 5580fe8e35fed092117cbaf33be37e1c4f3cc126 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Tue, 18 Aug 2026 12:25:26 +0000 Subject: [PATCH 49/75] job_runs: add lifecycle.triggers.on_file_change Re-fire a run when matched file contents change, using stable hash fingerprints so mtime-only updates (touch) do not recreate. --- .../bundles/job-runs-on-file-change.md | 1 + .../job_runs/on_file_change/databricks.yml | 18 ++ .../job_runs/on_file_change/out.test.toml | 2 + .../job_runs/on_file_change/output.txt | 90 +++++++++ .../resources/job_runs/on_file_change/script | 34 ++++ .../job_runs/on_file_change/seed.txt | 1 + .../mutator/resolve_job_run_file_triggers.go | 177 ++++++++++++++++++ ...lve_job_run_file_triggers_internal_test.go | 73 ++++++++ .../resolve_job_run_file_triggers_test.go | 143 ++++++++++++++ .../mutator/validate_job_run_triggers.go | 26 ++- .../mutator/validate_job_run_triggers_test.go | 50 ++++- bundle/config/resources/job_run.go | 29 +++ bundle/config/resources/lifecycle.go | 9 +- bundle/direct/dresources/job_run.go | 28 ++- bundle/direct/dresources/resources.yml | 6 + bundle/internal/schema/annotations.yml | 3 + bundle/phases/initialize.go | 6 +- bundle/phases/plan.go | 4 +- bundle/schema/jsonschema.json | 4 + 19 files changed, 689 insertions(+), 15 deletions(-) create mode 100644 .nextchanges/bundles/job-runs-on-file-change.md create mode 100644 acceptance/bundle/resources/job_runs/on_file_change/databricks.yml create mode 100644 acceptance/bundle/resources/job_runs/on_file_change/out.test.toml create mode 100644 acceptance/bundle/resources/job_runs/on_file_change/output.txt create mode 100644 acceptance/bundle/resources/job_runs/on_file_change/script create mode 100644 acceptance/bundle/resources/job_runs/on_file_change/seed.txt create mode 100644 bundle/config/mutator/resolve_job_run_file_triggers.go create mode 100644 bundle/config/mutator/resolve_job_run_file_triggers_internal_test.go create mode 100644 bundle/config/mutator/resolve_job_run_file_triggers_test.go diff --git a/.nextchanges/bundles/job-runs-on-file-change.md b/.nextchanges/bundles/job-runs-on-file-change.md new file mode 100644 index 00000000000..00682403a0d --- /dev/null +++ b/.nextchanges/bundles/job-runs-on-file-change.md @@ -0,0 +1 @@ +direct: `resources.job_runs` can set `lifecycle.triggers.on_file_change` to a path or glob to re-fire the run when matched file contents change. Removing the trigger does not recreate the existing run. diff --git a/acceptance/bundle/resources/job_runs/on_file_change/databricks.yml b/acceptance/bundle/resources/job_runs/on_file_change/databricks.yml new file mode 100644 index 00000000000..0a6df75047a --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change/databricks.yml @@ -0,0 +1,18 @@ +bundle: + name: job-runs-on-file-change + +resources: + jobs: + my_job: + name: my-job + tasks: + - task_key: main + notebook_task: + notebook_path: /Workspace/test + + job_runs: + my_run: + job_id: ${resources.jobs.my_job.id} + lifecycle: + triggers: + - on_file_change: seed.txt diff --git a/acceptance/bundle/resources/job_runs/on_file_change/out.test.toml b/acceptance/bundle/resources/job_runs/on_file_change/out.test.toml new file mode 100644 index 00000000000..0938e678987 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/job_runs/on_file_change/output.txt b/acceptance/bundle/resources/job_runs/on_file_change/output.txt new file mode 100644 index 00000000000..bcf5962f8fa --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change/output.txt @@ -0,0 +1,90 @@ + +=== first deploy triggers a run +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change/default/files... +job run [MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?o=[NUMID] +job run [MY_RUN_ID]: SUCCESS +Created job_runs.my_run +Created jobs.my_job +Files: 5 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +>>> read_id.py my_job +[MY_JOB_ID] + +>>> print_requests.py //jobs/run-now +{ + "method": "POST", + "path": "/api/2.2/jobs/run-now", + "body": { + "idempotency_token": "[UUID]", + "job_id": [MY_JOB_ID] + } +} + +=== redeploy with unchanged file plans nothing +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change/default/files... +Files: 2 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 2 unchanged + +>>> print_requests.py //jobs/run-now + +=== editing the file re-fires +>>> [CLI] bundle plan +recreate job_runs.my_run + +Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change/default/files... +job run [MY_RUN_ID_2]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_2]?o=[NUMID] +job run [MY_RUN_ID_2]: SUCCESS +Recreated job_runs.my_run +Files: 3 uploaded, 0 deleted +Resources: 1 created, 0 changed, 1 deleted, 1 unchanged + +>>> print_requests.py --keep //jobs/runs/delete +{ + "method": "POST", + "path": "/api/2.2/jobs/runs/delete", + "body": { + "run_id": [MY_RUN_ID] + } +} + +>>> print_requests.py //jobs/run-now +{ + "method": "POST", + "path": "/api/2.2/jobs/run-now", + "body": { + "idempotency_token": "[UUID]", + "job_id": [MY_JOB_ID] + } +} + +=== removing on_file_change rewrites state without a run +>>> [CLI] bundle plan +update job_runs.my_run + +Plan: 0 to add, 1 to change, 0 to delete, 1 unchanged + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change/default/files... +Updated job_runs.my_run +Files: 3 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 1 unchanged + +>>> print_requests.py //jobs/run-now + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.job_runs.my_run + delete resources.jobs.my_job + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change/default + +Destroy: 2 deleted diff --git a/acceptance/bundle/resources/job_runs/on_file_change/script b/acceptance/bundle/resources/job_runs/on_file_change/script new file mode 100644 index 00000000000..ff162e11f0b --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change/script @@ -0,0 +1,34 @@ +cleanup() { + trace $CLI bundle destroy --auto-approve + rm -f out.requests.txt +} +trap cleanup EXIT + +title "first deploy triggers a run" +trace $CLI bundle deploy +trace read_id.py my_job +# Name the first run so the recreated one becomes [MY_RUN_ID_2]. +read_id.py my_run > /dev/null +trace print_requests.py //jobs/run-now + +title "redeploy with unchanged file plans nothing" +trace $CLI bundle plan +trace $CLI bundle deploy +trace print_requests.py //jobs/run-now + +title "editing the file re-fires" +update_file.py seed.txt "v1" "v2" +trace $CLI bundle plan +trace $CLI bundle deploy +read_id.py my_run > /dev/null +trace print_requests.py --keep //jobs/runs/delete +trace print_requests.py //jobs/run-now + +title "removing on_file_change rewrites state without a run" +update_file.py databricks.yml " lifecycle: + triggers: + - on_file_change: seed.txt +" "" +trace $CLI bundle plan +trace $CLI bundle deploy +trace print_requests.py //jobs/run-now diff --git a/acceptance/bundle/resources/job_runs/on_file_change/seed.txt b/acceptance/bundle/resources/job_runs/on_file_change/seed.txt new file mode 100644 index 00000000000..626799f0f85 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change/seed.txt @@ -0,0 +1 @@ +v1 diff --git a/bundle/config/mutator/resolve_job_run_file_triggers.go b/bundle/config/mutator/resolve_job_run_file_triggers.go new file mode 100644 index 00000000000..a82ed654be3 --- /dev/null +++ b/bundle/config/mutator/resolve_job_run_file_triggers.go @@ -0,0 +1,177 @@ +package mutator + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "os" + "path/filepath" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/libs/diag" +) + +// missingFileSentinelSize marks a pattern with no matching file so the next +// plan can distinguish "still missing" from "file appeared". +const missingFileSentinelSize = int64(-1) + +type resolveJobRunFileTriggers struct{} + +// ResolveJobRunFileTriggers expands on_file_change globs and stores per-file +// fingerprints on each job_run for PrepareState to copy into local state. +func ResolveJobRunFileTriggers() bundle.Mutator { + return &resolveJobRunFileTriggers{} +} + +func (*resolveJobRunFileTriggers) Name() string { + return "ResolveJobRunFileTriggers" +} + +func (*resolveJobRunFileTriggers) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { + var diags diag.Diagnostics + for name, jr := range b.Config.Resources.JobRuns { + if jr == nil { + continue + } + patterns := jr.OnFileChangePatterns() + if len(patterns) == 0 { + continue + } + fps, d := resolveFileTriggers(b, name, patterns, previousFileTriggers(b, name)) + diags = diags.Extend(d) + jr.ResolvedFileTriggers = fps + } + return diags +} + +// previousFileTriggers reads on_file_change fingerprints from deployment state +// when it is open (plan/deploy after StatePull). Used so unchanged content keeps +// a stable fingerprint across mtime-only updates (e.g. touch). +func previousFileTriggers(b *bundle.Bundle, name string) map[string]resources.JobRunFileFingerprint { + if b.DeploymentBundle.StateDB.Path == "" { + return nil + } + entry, ok := b.DeploymentBundle.StateDB.GetResourceEntry("resources.job_runs." + name) + if !ok || len(entry.State) == 0 { + return nil + } + var state struct { + Lifecycle *struct { + Triggers *struct { + OnFileChange map[string]resources.JobRunFileFingerprint `json:"on_file_change"` + } `json:"triggers"` + } `json:"lifecycle"` + } + if err := json.Unmarshal(entry.State, &state); err != nil { + return nil + } + if state.Lifecycle == nil || state.Lifecycle.Triggers == nil { + return nil + } + return state.Lifecycle.Triggers.OnFileChange +} + +func resolveFileTriggers(b *bundle.Bundle, name string, patterns []string, prev map[string]resources.JobRunFileFingerprint) (map[string]resources.JobRunFileFingerprint, diag.Diagnostics) { + var diags diag.Diagnostics + out := make(map[string]resources.JobRunFileFingerprint) + for _, pattern := range patterns { + path := fmt.Sprintf("resources.job_runs.%s.lifecycle.triggers", name) + matches, err := filepath.Glob(filepath.Join(b.SyncRootPath, filepath.FromSlash(pattern))) + if err != nil { + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: invalid pattern %q: %s", pattern, err), + Locations: b.Config.GetLocations(path), + }) + continue + } + if len(matches) == 0 { + // Distinct state when the path/glob matches nothing (design doc). + out[filepath.ToSlash(pattern)] = resources.JobRunFileFingerprint{ + Size: missingFileSentinelSize, + } + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Warning, + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: no files match %q", pattern), + Locations: b.Config.GetLocations(path), + }) + continue + } + for _, match := range matches { + info, err := os.Stat(match) + if err != nil { + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: stat %q: %s", match, err), + Locations: b.Config.GetLocations(path), + }) + continue + } + if !info.Mode().IsRegular() { + continue + } + rel, err := filepath.Rel(b.SyncRootPath, match) + if err != nil { + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: relative path for %q: %s", match, err), + Locations: b.Config.GetLocations(path), + }) + continue + } + key := filepath.ToSlash(rel) + fp, err := fingerprintFile(match, info, prev[key]) + if err != nil { + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: hash %q: %s", match, err), + Locations: b.Config.GetLocations(path), + }) + continue + } + out[key] = fp + } + } + return out, diags +} + +// fingerprintFile builds a content fingerprint. If prev has the same size and +// mtime, it is reused without reading the file. If content hash matches prev, +// prev is reused so mtime-only updates (touch) do not change planned state. +func fingerprintFile(path string, info os.FileInfo, prev resources.JobRunFileFingerprint) (resources.JobRunFileFingerprint, error) { + size := info.Size() + mtime := info.ModTime().UnixNano() + if prev.Hash != "" && prev.Size == size && prev.MtimeNs == mtime { + return prev, nil + } + hash, err := hashFile(path) + if err != nil { + return resources.JobRunFileFingerprint{}, err + } + if prev.Hash != "" && prev.Hash == hash { + return prev, nil + } + return resources.JobRunFileFingerprint{ + Hash: hash, + Size: size, + MtimeNs: mtime, + }, nil +} + +func hashFile(path string) (string, error) { + f, err := os.Open(path) + if err != nil { + return "", err + } + defer f.Close() + + h := sha256.New() + if _, err := io.Copy(h, f); err != nil { + return "", err + } + return hex.EncodeToString(h.Sum(nil)), nil +} diff --git a/bundle/config/mutator/resolve_job_run_file_triggers_internal_test.go b/bundle/config/mutator/resolve_job_run_file_triggers_internal_test.go new file mode 100644 index 00000000000..e82bd22945b --- /dev/null +++ b/bundle/config/mutator/resolve_job_run_file_triggers_internal_test.go @@ -0,0 +1,73 @@ +package mutator + +import ( + "os" + "path/filepath" + "testing" + "time" + + "github.com/databricks/cli/bundle/config/resources" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestFingerprintFileContentStableAcrossTouch(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "seed.txt") + require.NoError(t, os.WriteFile(path, []byte("v1"), 0o644)) + + info, err := os.Stat(path) + require.NoError(t, err) + first, err := fingerprintFile(path, info, resources.JobRunFileFingerprint{}) + require.NoError(t, err) + require.NotEmpty(t, first.Hash) + + // Advance mtime without changing contents (touch). + require.NoError(t, os.Chtimes(path, time.Now().Add(time.Minute), time.Now().Add(time.Minute))) + info, err = os.Stat(path) + require.NoError(t, err) + assert.NotEqual(t, first.MtimeNs, info.ModTime().UnixNano()) + + second, err := fingerprintFile(path, info, first) + require.NoError(t, err) + assert.Equal(t, first, second, "unchanged content must reuse the previous fingerprint") +} + +func TestFingerprintFileFastPathSkipsWhenMtimeAndSizeMatch(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "seed.txt") + require.NoError(t, os.WriteFile(path, []byte("v1"), 0o644)) + + info, err := os.Stat(path) + require.NoError(t, err) + prev := resources.JobRunFileFingerprint{ + Hash: "not-the-real-hash", + Size: info.Size(), + MtimeNs: info.ModTime().UnixNano(), + } + + got, err := fingerprintFile(path, info, prev) + require.NoError(t, err) + assert.Equal(t, prev, got, "matching size+mtime must reuse prev without re-hashing") +} + +func TestFingerprintFileContentChange(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "seed.txt") + require.NoError(t, os.WriteFile(path, []byte("v1"), 0o644)) + + info, err := os.Stat(path) + require.NoError(t, err) + first, err := fingerprintFile(path, info, resources.JobRunFileFingerprint{}) + require.NoError(t, err) + + // Different size so the mtime+size fast path cannot reuse prev. + require.NoError(t, os.WriteFile(path, []byte("v2-changed"), 0o644)) + info, err = os.Stat(path) + require.NoError(t, err) + second, err := fingerprintFile(path, info, first) + require.NoError(t, err) + assert.NotEqual(t, first.Hash, second.Hash) + assert.Equal(t, info.Size(), second.Size) + assert.Equal(t, info.ModTime().UnixNano(), second.MtimeNs) +} diff --git a/bundle/config/mutator/resolve_job_run_file_triggers_test.go b/bundle/config/mutator/resolve_job_run_file_triggers_test.go new file mode 100644 index 00000000000..9a9cf77bf53 --- /dev/null +++ b/bundle/config/mutator/resolve_job_run_file_triggers_test.go @@ -0,0 +1,143 @@ +package mutator_test + +import ( + "crypto/sha256" + "encoding/hex" + "os" + "path/filepath" + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/mutator" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/libs/diag" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestResolveJobRunFileTriggers(t *testing.T) { + t.Run("matches files and fills fingerprints", func(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "a.txt"), []byte("hello"), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "b.txt"), []byte("world"), 0o644)) + + pattern := "*.txt" + b := bundleWithFileTrigger(dir, pattern) + + diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) + require.False(t, diags.HasError()) + + fps := b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers + require.Len(t, fps, 2) + + assertFingerprint(t, fps["a.txt"], "hello") + assertFingerprint(t, fps["b.txt"], "world") + }) + + t.Run("no matches warns and stores sentinel", func(t *testing.T) { + dir := t.TempDir() + pattern := "missing.txt" + b := bundleWithFileTrigger(dir, pattern) + + diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) + require.False(t, diags.HasError()) + require.Len(t, diags, 1) + assert.Equal(t, diag.Warning, diags[0].Severity) + assert.Contains(t, diags[0].Summary, `no files match "missing.txt"`) + + fps := b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers + require.Len(t, fps, 1) + fp := fps["missing.txt"] + assert.Empty(t, fp.Hash) + assert.Equal(t, int64(-1), fp.Size) + assert.Zero(t, fp.MtimeNs) + }) + + t.Run("no file triggers is a no-op", func(t *testing.T) { + dir := t.TempDir() + on := true + b := &bundle.Bundle{ + SyncRootPath: dir, + Config: config.Root{ + Resources: config.Resources{ + JobRuns: map[string]*resources.JobRun{ + "my_run": { + Lifecycle: &resources.JobRunLifecycle{ + Triggers: []resources.JobRunTrigger{ + {OnBundleDeploy: &on}, + }, + }, + }, + }, + }, + }, + } + + diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) + assert.Empty(t, diags) + assert.Nil(t, b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers) + }) + + t.Run("multiple patterns merge into one map", func(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "a.txt"), []byte("aaa"), 0o644)) + require.NoError(t, os.MkdirAll(filepath.Join(dir, "subdir"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "subdir", "x.py"), []byte("bbb"), 0o644)) + + patA := "a.txt" + patB := "subdir/*.py" + b := &bundle.Bundle{ + SyncRootPath: dir, + Config: config.Root{ + Resources: config.Resources{ + JobRuns: map[string]*resources.JobRun{ + "my_run": { + Lifecycle: &resources.JobRunLifecycle{ + Triggers: []resources.JobRunTrigger{ + {OnFileChange: &patA}, + {OnFileChange: &patB}, + }, + }, + }, + }, + }, + }, + } + + diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) + require.False(t, diags.HasError()) + + fps := b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers + require.Len(t, fps, 2) + assertFingerprint(t, fps["a.txt"], "aaa") + assertFingerprint(t, fps["subdir/x.py"], "bbb") + }) +} + +func bundleWithFileTrigger(syncRoot, pattern string) *bundle.Bundle { + return &bundle.Bundle{ + SyncRootPath: syncRoot, + Config: config.Root{ + Resources: config.Resources{ + JobRuns: map[string]*resources.JobRun{ + "my_run": { + Lifecycle: &resources.JobRunLifecycle{ + Triggers: []resources.JobRunTrigger{ + {OnFileChange: &pattern}, + }, + }, + }, + }, + }, + }, + } +} + +func assertFingerprint(t *testing.T, fp resources.JobRunFileFingerprint, content string) { + t.Helper() + sum := sha256.Sum256([]byte(content)) + assert.Equal(t, hex.EncodeToString(sum[:]), fp.Hash) + assert.Equal(t, int64(len(content)), fp.Size) + assert.NotZero(t, fp.MtimeNs) +} diff --git a/bundle/config/mutator/validate_job_run_triggers.go b/bundle/config/mutator/validate_job_run_triggers.go index a7c5ada925e..c185d3b5781 100644 --- a/bundle/config/mutator/validate_job_run_triggers.go +++ b/bundle/config/mutator/validate_job_run_triggers.go @@ -3,6 +3,7 @@ package mutator import ( "context" "fmt" + "strings" "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" @@ -26,30 +27,45 @@ func (*validateJobRunTriggers) Apply(_ context.Context, b *bundle.Bundle) diag.D continue } // Recreate-every-deploy cannot coexist with prevent_destroy. - if jr.HasOnBundleDeploy() && jr.Lifecycle.PreventDestroy { + if (jr.HasOnBundleDeploy() || jr.HasOnFileChange()) && jr.Lifecycle.PreventDestroy { diags = diags.Append(diag.Diagnostic{ Severity: diag.Error, - Summary: "lifecycle.triggers.on_bundle_deploy is incompatible with lifecycle.prevent_destroy", + Summary: "lifecycle.triggers.on_bundle_deploy or on_file_change is incompatible with lifecycle.prevent_destroy", Locations: b.Config.GetLocations(fmt.Sprintf("resources.job_runs.%s.lifecycle", name)), }) } for i, t := range jr.Lifecycle.Triggers { path := fmt.Sprintf("resources.job_runs.%s.lifecycle.triggers[%d]", name, i) - if t.OnBundleDeploy == nil { + if t.OnBundleDeploy == nil && t.OnFileChange == nil { diags = diags.Append(diag.Diagnostic{ Severity: diag.Error, - Summary: "lifecycle.triggers entry must set on_bundle_deploy: true", + Summary: "lifecycle.triggers entry must set on_bundle_deploy or on_file_change", Locations: b.Config.GetLocations(path), }) continue } - if !*t.OnBundleDeploy { + if t.OnBundleDeploy != nil && t.OnFileChange != nil { + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: "lifecycle.triggers entry must set only one of on_bundle_deploy or on_file_change", + Locations: b.Config.GetLocations(path), + }) + continue + } + if t.OnBundleDeploy != nil && !*t.OnBundleDeploy { diags = diags.Append(diag.Diagnostic{ Severity: diag.Error, Summary: "lifecycle.triggers.on_bundle_deploy must be true when set", Locations: b.Config.GetLocations(path + ".on_bundle_deploy"), }) } + if t.OnFileChange != nil && strings.TrimSpace(*t.OnFileChange) == "" { + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: "lifecycle.triggers.on_file_change must be non-empty when set", + Locations: b.Config.GetLocations(path + ".on_file_change"), + }) + } } } return diags diff --git a/bundle/config/mutator/validate_job_run_triggers_test.go b/bundle/config/mutator/validate_job_run_triggers_test.go index 9c225457471..7e8772d8d8b 100644 --- a/bundle/config/mutator/validate_job_run_triggers_test.go +++ b/bundle/config/mutator/validate_job_run_triggers_test.go @@ -14,6 +14,10 @@ func TestValidateJobRunTriggers(t *testing.T) { trueVal := true falseVal := false + fileChange := "seed.txt" + emptyFile := "" + whitespaceFile := " \t" + tests := []struct { name string triggers []resources.JobRunTrigger @@ -26,12 +30,32 @@ func TestValidateJobRunTriggers(t *testing.T) { {OnBundleDeploy: &trueVal}, }, }, + { + name: "on_file_change set", + triggers: []resources.JobRunTrigger{ + {OnFileChange: &fileChange}, + }, + }, + { + name: "both triggers as separate entries", + triggers: []resources.JobRunTrigger{ + {OnFileChange: &fileChange}, + {OnBundleDeploy: &trueVal}, + }, + }, { name: "empty entry", triggers: []resources.JobRunTrigger{ {}, }, - summary: "lifecycle.triggers entry must set on_bundle_deploy: true", + summary: "lifecycle.triggers entry must set on_bundle_deploy or on_file_change", + }, + { + name: "both keys on one entry", + triggers: []resources.JobRunTrigger{ + {OnBundleDeploy: &trueVal, OnFileChange: &fileChange}, + }, + summary: "lifecycle.triggers entry must set only one of on_bundle_deploy or on_file_change", }, { name: "on_bundle_deploy false", @@ -40,13 +64,35 @@ func TestValidateJobRunTriggers(t *testing.T) { }, summary: "lifecycle.triggers.on_bundle_deploy must be true when set", }, + { + name: "on_file_change empty", + triggers: []resources.JobRunTrigger{ + {OnFileChange: &emptyFile}, + }, + summary: "lifecycle.triggers.on_file_change must be non-empty when set", + }, + { + name: "on_file_change whitespace", + triggers: []resources.JobRunTrigger{ + {OnFileChange: &whitespaceFile}, + }, + summary: "lifecycle.triggers.on_file_change must be non-empty when set", + }, { name: "on_bundle_deploy with prevent_destroy", triggers: []resources.JobRunTrigger{ {OnBundleDeploy: &trueVal}, }, preventDestroy: true, - summary: "lifecycle.triggers.on_bundle_deploy is incompatible with lifecycle.prevent_destroy", + summary: "lifecycle.triggers.on_bundle_deploy or on_file_change is incompatible with lifecycle.prevent_destroy", + }, + { + name: "on_file_change with prevent_destroy", + triggers: []resources.JobRunTrigger{ + {OnFileChange: &fileChange}, + }, + preventDestroy: true, + summary: "lifecycle.triggers.on_bundle_deploy or on_file_change is incompatible with lifecycle.prevent_destroy", }, { name: "prevent_destroy alone", diff --git a/bundle/config/resources/job_run.go b/bundle/config/resources/job_run.go index 56f3c39a356..cf66c71f728 100644 --- a/bundle/config/resources/job_run.go +++ b/bundle/config/resources/job_run.go @@ -27,6 +27,10 @@ type JobRun struct { // the run URL. Keeping it separate from RunNow.JobId (a ${resources.jobs.*.id} // reference) lets state loading preserve that reference and its plan dependency. ResolvedJobID int64 `json:"resolved_job_id,omitempty" bundle:"internal"` + + // ResolvedFileTriggers is the per-file fingerprint map for on_file_change, + // computed under SyncRoot before plan. bundle:"internal" keeps it out of schema. + ResolvedFileTriggers map[string]JobRunFileFingerprint `json:"resolved_file_triggers,omitempty" bundle:"internal"` } // HasOnBundleDeploy reports whether any trigger re-fires on every deploy. @@ -42,6 +46,31 @@ func (r *JobRun) HasOnBundleDeploy() bool { return false } +func (r *JobRun) HasOnFileChange() bool { + if r.Lifecycle == nil { + return false + } + for _, t := range r.Lifecycle.Triggers { + if t.OnFileChange != nil { + return true + } + } + return false +} + +func (r *JobRun) OnFileChangePatterns() []string { + if r.Lifecycle == nil { + return nil + } + var patterns []string + for _, t := range r.Lifecycle.Triggers { + if t.OnFileChange != nil { + patterns = append(patterns, *t.OnFileChange) + } + } + return patterns +} + func (r *JobRun) UnmarshalJSON(b []byte) error { return marshal.Unmarshal(b, r) } diff --git a/bundle/config/resources/lifecycle.go b/bundle/config/resources/lifecycle.go index 88d130239ee..59ccb682b81 100644 --- a/bundle/config/resources/lifecycle.go +++ b/bundle/config/resources/lifecycle.go @@ -36,5 +36,12 @@ type JobRunLifecycle struct { // JobRunTrigger is one lifecycle.triggers entry. type JobRunTrigger struct { - OnBundleDeploy *bool `json:"on_bundle_deploy,omitempty"` + OnBundleDeploy *bool `json:"on_bundle_deploy,omitempty"` + OnFileChange *string `json:"on_file_change,omitempty"` // path or glob under sync root +} + +type JobRunFileFingerprint struct { + Hash string `json:"hash"` + Size int64 `json:"size"` + MtimeNs int64 `json:"mtime_ns"` } diff --git a/bundle/direct/dresources/job_run.go b/bundle/direct/dresources/job_run.go index 9eb671f430c..4e57a4bcdf4 100644 --- a/bundle/direct/dresources/job_run.go +++ b/bundle/direct/dresources/job_run.go @@ -26,10 +26,20 @@ import ( // jobRunTimeout matches the timeout `bundle run` allows a run (bundle/run/job.go). const jobRunTimeout = 24 * time.Hour +// jobRunTriggerLocalPaths are local-only fingerprints: clearing one is skip, not recreate. +var jobRunTriggerLocalPaths = []string{ + "lifecycle", + "lifecycle.triggers", + "lifecycle.triggers.on_bundle_deploy", + "lifecycle.triggers.on_file_change", +} + // JobRunTriggersState is the persisted fingerprint of lifecycle.triggers. type JobRunTriggersState struct { // Fresh UUID each plan while armed so Old!=New forces recreate. OnBundleDeploy string `json:"on_bundle_deploy,omitempty"` + // Per-file fingerprints from ResolveJobRunFileTriggers; change to recreate. + OnFileChange map[string]resources.JobRunFileFingerprint `json:"on_file_change,omitempty"` } // JobRunLifecycleState holds local-only lifecycle fields persisted in state. @@ -97,12 +107,19 @@ func (*ResourceJobRun) PrepareState(input *resources.JobRun) *JobRunState { state := &JobRunState{ RunNow: input.RunNow, ResultState: jobs.RunResultStateSuccess, - Lifecycle: nil, } + var triggers *JobRunTriggersState if input.HasOnBundleDeploy() { - state.Lifecycle = &JobRunLifecycleState{ - Triggers: &JobRunTriggersState{OnBundleDeploy: uuid.NewString()}, + triggers = &JobRunTriggersState{OnBundleDeploy: uuid.NewString()} + } + if len(input.ResolvedFileTriggers) > 0 { + if triggers == nil { + triggers = &JobRunTriggersState{} } + triggers.OnFileChange = input.ResolvedFileTriggers + } + if triggers != nil { + state.Lifecycle = &JobRunLifecycleState{Triggers: triggers} } return state } @@ -371,8 +388,7 @@ func reportRunLine(ctx context.Context, runID int64, msg string) { // result_state either, so the lifecycle state is what tells the two apart. // Clearing a trigger skips its local-only fingerprint without re-firing the run. func (*ResourceJobRun) OverrideChangeDesc(_ context.Context, path *structpath.PathNode, change *ChangeDesc, remote *JobRunRemote) error { - switch path.String() { - case "lifecycle", "lifecycle.triggers", "lifecycle.triggers.on_bundle_deploy": + if slices.Contains(jobRunTriggerLocalPaths, path.String()) { // A cleared trigger sets New empty; structdiff may report it at lifecycle, // lifecycle.triggers, or the leaf. if change.New == nil || change.New == "" { @@ -380,6 +396,8 @@ func (*ResourceJobRun) OverrideChangeDesc(_ context.Context, path *structpath.Pa change.Reason = "trigger removed" } return nil + } + switch path.String() { case "result_state": // The planner passes no remote state when the run could not be read. if remote == nil || runIsTerminal(remote.State.LifeCycleState) { diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index b0a9ed6b365..0d49749c15a 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -182,6 +182,12 @@ resources: recreate_on_changes: - reason: immutable + # mtime is stored for the resolve fast path; content identity is the hash. + # Ignoring mtime-only local drift keeps touch from recreating the run. + ignore_local_changes: + - field: lifecycle.triggers.on_file_change[*].mtime_ns + reason: mtime_only_fingerprint + pipelines: recreate_on_changes: - field: storage diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index a47e8151681..bfecaef7c00 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -990,6 +990,9 @@ resources: "on_bundle_deploy": "description": |- If true, re-fire the run on every bundle deploy. Incompatible with lifecycle.prevent_destroy. + "on_file_change": + "description": |- + Path or glob under the sync root. Re-fire the run when any matched file's content changes. Incompatible with lifecycle.prevent_destroy. "python_named_params": "description": |- PLACEHOLDER diff --git a/bundle/phases/initialize.go b/bundle/phases/initialize.go index b15e1c30df6..9eba6f466e0 100644 --- a/bundle/phases/initialize.go +++ b/bundle/phases/initialize.go @@ -26,7 +26,8 @@ import ( func Initialize(ctx context.Context, b *bundle.Bundle) { log.Info(ctx, "Phase: initialize") - bundle.ApplySeqContext(ctx, b, + bundle.ApplySeqContext( + ctx, b, // Reads (dynamic): resource.*.* // Checks that none of resources.. is nil. Raises error otherwise. validate.AllResourcesHaveValues(), @@ -191,6 +192,9 @@ func Initialize(ctx context.Context, b *bundle.Bundle) { // Reject invalid job_runs.lifecycle.triggers (empty, false, prevent_destroy). mutator.ValidateJobRunTriggers(), + // Expand on_file_change globs and hash matched files into ResolvedFileTriggers. + mutator.ResolveJobRunFileTriggers(), + // Reads (dynamic): * (strings) (searches for ${resources.*} references) // Warns (TF engine) or errors (direct engine) when a cross-resource reference // points to a Terraform-only field with no DABs equivalent. diff --git a/bundle/phases/plan.go b/bundle/phases/plan.go index 3db0864c2c4..92ca524cc4f 100644 --- a/bundle/phases/plan.go +++ b/bundle/phases/plan.go @@ -21,7 +21,8 @@ import ( // PreDeployChecks is common set of mutators between "bundle plan" and "bundle deploy". // Note, it is not run in "bundle migrate" so it must not modify the config func PreDeployChecks(ctx context.Context, b *bundle.Bundle, isPlan bool, engine engine.EngineType) { - bundle.ApplySeqContext(ctx, b, + bundle.ApplySeqContext( + ctx, b, terraform.CheckDashboardsModifiedRemotely(isPlan, engine), resourcemutator.SecretScopeFixups(engine), deploy.StatePull(), @@ -30,6 +31,7 @@ func PreDeployChecks(ctx context.Context, b *bundle.Bundle, isPlan bool, engine mutator.ValidateLifecycleStarted(engine), mutator.ValidateCascadeOnDestroy(engine), mutator.ValidateJobRunTriggers(), + mutator.ResolveJobRunFileTriggers(), statemgmt.CheckRunningResource(engine), ) } diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index cc481ee8b0c..0a01fba5022 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -1311,6 +1311,10 @@ "on_bundle_deploy": { "description": "If true, re-fire the run on every bundle deploy. Incompatible with lifecycle.prevent_destroy.", "$ref": "#/$defs/bool" + }, + "on_file_change": { + "description": "Path or glob under the sync root. Re-fire the run when any matched file's content changes. Incompatible with lifecycle.prevent_destroy.", + "$ref": "#/$defs/string" } }, "additionalProperties": false From 8d001bdfc5c8325c0daab99e8dfeeff56bf19ce9 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Tue, 18 Aug 2026 12:41:58 +0000 Subject: [PATCH 50/75] job_runs: fix exhaustruct and regenerate refschema for on_file_change Satisfy the linter on PrepareState literals after adding OnFileChange, and refresh out.fields.txt so validate-generated stays green. --- acceptance/bundle/refschema/out.fields.txt | 11 +++++++++++ bundle/direct/dresources/job_run.go | 11 +++++++++-- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index 9e1202a2ff8..87d1ac5270d 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -872,8 +872,14 @@ resources.job_runs.*.lifecycle.prevent_destroy bool INPUT resources.job_runs.*.lifecycle.triggers *dresources.JobRunTriggersState STATE resources.job_runs.*.lifecycle.triggers []resources.JobRunTrigger INPUT resources.job_runs.*.lifecycle.triggers.on_bundle_deploy string STATE +resources.job_runs.*.lifecycle.triggers.on_file_change map[string]resources.JobRunFileFingerprint STATE +resources.job_runs.*.lifecycle.triggers.on_file_change.* resources.JobRunFileFingerprint STATE +resources.job_runs.*.lifecycle.triggers.on_file_change.*.hash string STATE +resources.job_runs.*.lifecycle.triggers.on_file_change.*.mtime_ns int64 STATE +resources.job_runs.*.lifecycle.triggers.on_file_change.*.size int64 STATE resources.job_runs.*.lifecycle.triggers[*] resources.JobRunTrigger INPUT resources.job_runs.*.lifecycle.triggers[*].on_bundle_deploy *bool INPUT +resources.job_runs.*.lifecycle.triggers[*].on_file_change *string INPUT resources.job_runs.*.modified_status string INPUT resources.job_runs.*.notebook_params map[string]string ALL resources.job_runs.*.notebook_params.* string ALL @@ -896,6 +902,11 @@ resources.job_runs.*.python_params []string ALL resources.job_runs.*.python_params[*] string ALL resources.job_runs.*.queue *jobs.QueueSettings ALL resources.job_runs.*.queue.enabled bool ALL +resources.job_runs.*.resolved_file_triggers map[string]resources.JobRunFileFingerprint INPUT +resources.job_runs.*.resolved_file_triggers.* resources.JobRunFileFingerprint INPUT +resources.job_runs.*.resolved_file_triggers.*.hash string INPUT +resources.job_runs.*.resolved_file_triggers.*.mtime_ns int64 INPUT +resources.job_runs.*.resolved_file_triggers.*.size int64 INPUT resources.job_runs.*.resolved_job_id int64 INPUT resources.job_runs.*.result_state jobs.RunResultState REMOTE STATE resources.job_runs.*.run_id int64 REMOTE diff --git a/bundle/direct/dresources/job_run.go b/bundle/direct/dresources/job_run.go index 4e57a4bcdf4..f66991f1ca8 100644 --- a/bundle/direct/dresources/job_run.go +++ b/bundle/direct/dresources/job_run.go @@ -107,14 +107,21 @@ func (*ResourceJobRun) PrepareState(input *resources.JobRun) *JobRunState { state := &JobRunState{ RunNow: input.RunNow, ResultState: jobs.RunResultStateSuccess, + Lifecycle: nil, } var triggers *JobRunTriggersState if input.HasOnBundleDeploy() { - triggers = &JobRunTriggersState{OnBundleDeploy: uuid.NewString()} + triggers = &JobRunTriggersState{ + OnBundleDeploy: uuid.NewString(), + OnFileChange: nil, + } } if len(input.ResolvedFileTriggers) > 0 { if triggers == nil { - triggers = &JobRunTriggersState{} + triggers = &JobRunTriggersState{ + OnBundleDeploy: "", + OnFileChange: nil, + } } triggers.OnFileChange = input.ResolvedFileTriggers } From 6959d1dc3d8fbc4ab800c1eec68110c8d016ad94 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Tue, 18 Aug 2026 13:03:41 +0000 Subject: [PATCH 51/75] job_runs: tighten on_file_change path and directory checks Reject patterns outside the sync root and directory-only matches so the trigger cannot fingerprint escaped paths or silently disarm. --- .../bundles/job-runs-on-file-change.md | 2 +- .../mutator/resolve_job_run_file_triggers.go | 31 ++++++++++++++++--- .../resolve_job_run_file_triggers_test.go | 25 +++++++++++++++ bundle/internal/schema/annotations.yml | 2 +- bundle/schema/jsonschema.json | 2 +- 5 files changed, 55 insertions(+), 7 deletions(-) diff --git a/.nextchanges/bundles/job-runs-on-file-change.md b/.nextchanges/bundles/job-runs-on-file-change.md index 00682403a0d..eadceda6268 100644 --- a/.nextchanges/bundles/job-runs-on-file-change.md +++ b/.nextchanges/bundles/job-runs-on-file-change.md @@ -1 +1 @@ -direct: `resources.job_runs` can set `lifecycle.triggers.on_file_change` to a path or glob to re-fire the run when matched file contents change. Removing the trigger does not recreate the existing run. +direct: `resources.job_runs` can set `lifecycle.triggers.on_file_change` to a path or glob under the sync root to re-fire the run when matched file contents change, or when matches appear or disappear. Removing the trigger does not recreate the existing run. diff --git a/bundle/config/mutator/resolve_job_run_file_triggers.go b/bundle/config/mutator/resolve_job_run_file_triggers.go index a82ed654be3..f0338b2d50a 100644 --- a/bundle/config/mutator/resolve_job_run_file_triggers.go +++ b/bundle/config/mutator/resolve_job_run_file_triggers.go @@ -80,7 +80,17 @@ func resolveFileTriggers(b *bundle.Bundle, name string, patterns []string, prev out := make(map[string]resources.JobRunFileFingerprint) for _, pattern := range patterns { path := fmt.Sprintf("resources.job_runs.%s.lifecycle.triggers", name) - matches, err := filepath.Glob(filepath.Join(b.SyncRootPath, filepath.FromSlash(pattern))) + localPattern := filepath.FromSlash(pattern) + // Keep fingerprints under SyncRoot; same IsLocal gate as translate_paths. + if !filepath.IsLocal(localPattern) { + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: pattern %q is not under the sync root", pattern), + Locations: b.Config.GetLocations(path), + }) + continue + } + matches, err := filepath.Glob(filepath.Join(b.SyncRootPath, localPattern)) if err != nil { diags = diags.Append(diag.Diagnostic{ Severity: diag.Error, @@ -90,7 +100,7 @@ func resolveFileTriggers(b *bundle.Bundle, name string, patterns []string, prev continue } if len(matches) == 0 { - // Distinct state when the path/glob matches nothing (design doc). + // Distinct state when the path/glob matches nothing so appear/disappear recreates. out[filepath.ToSlash(pattern)] = resources.JobRunFileFingerprint{ Size: missingFileSentinelSize, } @@ -101,6 +111,8 @@ func resolveFileTriggers(b *bundle.Bundle, name string, patterns []string, prev }) continue } + regularMatches := 0 + sawNonRegular := false for _, match := range matches { info, err := os.Stat(match) if err != nil { @@ -112,13 +124,15 @@ func resolveFileTriggers(b *bundle.Bundle, name string, patterns []string, prev continue } if !info.Mode().IsRegular() { + sawNonRegular = true continue } + regularMatches++ rel, err := filepath.Rel(b.SyncRootPath, match) - if err != nil { + if err != nil || !filepath.IsLocal(rel) { diags = diags.Append(diag.Diagnostic{ Severity: diag.Error, - Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: relative path for %q: %s", match, err), + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: matched path %q is not under the sync root", match), Locations: b.Config.GetLocations(path), }) continue @@ -135,6 +149,15 @@ func resolveFileTriggers(b *bundle.Bundle, name string, patterns []string, prev } out[key] = fp } + // A directory-only match would otherwise leave ResolvedFileTriggers empty + // and silently disarm the trigger while config still sets on_file_change. + if regularMatches == 0 && sawNonRegular { + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: pattern %q matches no regular files", pattern), + Locations: b.Config.GetLocations(path), + }) + } } return out, diags } diff --git a/bundle/config/mutator/resolve_job_run_file_triggers_test.go b/bundle/config/mutator/resolve_job_run_file_triggers_test.go index 9a9cf77bf53..06266d75894 100644 --- a/bundle/config/mutator/resolve_job_run_file_triggers_test.go +++ b/bundle/config/mutator/resolve_job_run_file_triggers_test.go @@ -113,6 +113,31 @@ func TestResolveJobRunFileTriggers(t *testing.T) { assertFingerprint(t, fps["a.txt"], "aaa") assertFingerprint(t, fps["subdir/x.py"], "bbb") }) + + t.Run("pattern outside sync root is an error", func(t *testing.T) { + dir := t.TempDir() + pattern := "../outside.txt" + b := bundleWithFileTrigger(dir, pattern) + + diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) + require.True(t, diags.HasError()) + require.Len(t, diags, 1) + assert.Contains(t, diags[0].Summary, `not under the sync root`) + assert.Empty(t, b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers) + }) + + t.Run("directory-only match is an error", func(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(dir, "migrations"), 0o755)) + pattern := "migrations" + b := bundleWithFileTrigger(dir, pattern) + + diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) + require.True(t, diags.HasError()) + require.Len(t, diags, 1) + assert.Contains(t, diags[0].Summary, `matches no regular files`) + assert.Empty(t, b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers) + }) } func bundleWithFileTrigger(syncRoot, pattern string) *bundle.Bundle { diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index bfecaef7c00..8f8dc45d066 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -992,7 +992,7 @@ resources: If true, re-fire the run on every bundle deploy. Incompatible with lifecycle.prevent_destroy. "on_file_change": "description": |- - Path or glob under the sync root. Re-fire the run when any matched file's content changes. Incompatible with lifecycle.prevent_destroy. + Path or glob under the sync root (filepath.Glob; `**` is not supported). Re-fire the run when a matched file's content changes, or when the set of matches appears or disappears. Incompatible with lifecycle.prevent_destroy. "python_named_params": "description": |- PLACEHOLDER diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 0a01fba5022..03146b5f1cc 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -1313,7 +1313,7 @@ "$ref": "#/$defs/bool" }, "on_file_change": { - "description": "Path or glob under the sync root. Re-fire the run when any matched file's content changes. Incompatible with lifecycle.prevent_destroy.", + "description": "Path or glob under the sync root (filepath.Glob; `**` is not supported). Re-fire the run when a matched file's content changes, or when the set of matches appears or disappears. Incompatible with lifecycle.prevent_destroy.", "$ref": "#/$defs/string" } }, From 38df5ff2517048317e43c6c236f3a5ccd55bd82d Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Tue, 18 Aug 2026 13:16:41 +0000 Subject: [PATCH 52/75] job_runs: store on_file_change as path-to-hash only Drop size/mtime fingerprints and the resolve fast path so content identity alone drives recreate, and touch no longer needs a planner exception. --- .../bundles/job-runs-on-file-change.md | 2 +- acceptance/bundle/refschema/out.fields.txt | 14 +- .../job_runs/on_file_change/output.txt | 11 + .../resources/job_runs/on_file_change/script | 6 + .../mutator/resolve_job_run_file_triggers.go | 204 +++++++----------- ...lve_job_run_file_triggers_internal_test.go | 73 ------- .../resolve_job_run_file_triggers_test.go | 48 +++-- bundle/config/resources/job_run.go | 19 +- bundle/config/resources/lifecycle.go | 6 - bundle/direct/dresources/job_run.go | 4 +- bundle/direct/dresources/resources.yml | 6 - bundle/internal/schema/annotations.yml | 2 +- bundle/schema/jsonschema.json | 2 +- 13 files changed, 131 insertions(+), 266 deletions(-) delete mode 100644 bundle/config/mutator/resolve_job_run_file_triggers_internal_test.go diff --git a/.nextchanges/bundles/job-runs-on-file-change.md b/.nextchanges/bundles/job-runs-on-file-change.md index eadceda6268..95869a25ccf 100644 --- a/.nextchanges/bundles/job-runs-on-file-change.md +++ b/.nextchanges/bundles/job-runs-on-file-change.md @@ -1 +1 @@ -direct: `resources.job_runs` can set `lifecycle.triggers.on_file_change` to a path or glob under the sync root to re-fire the run when matched file contents change, or when matches appear or disappear. Removing the trigger does not recreate the existing run. +direct: `resources.job_runs` can set `lifecycle.triggers.on_file_change` to a path or glob under the sync root to re-fire the run when matched file contents change (content hash), or when matches appear or disappear. Removing the trigger does not recreate the existing run. diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index 87d1ac5270d..871152e6a9c 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -872,11 +872,8 @@ resources.job_runs.*.lifecycle.prevent_destroy bool INPUT resources.job_runs.*.lifecycle.triggers *dresources.JobRunTriggersState STATE resources.job_runs.*.lifecycle.triggers []resources.JobRunTrigger INPUT resources.job_runs.*.lifecycle.triggers.on_bundle_deploy string STATE -resources.job_runs.*.lifecycle.triggers.on_file_change map[string]resources.JobRunFileFingerprint STATE -resources.job_runs.*.lifecycle.triggers.on_file_change.* resources.JobRunFileFingerprint STATE -resources.job_runs.*.lifecycle.triggers.on_file_change.*.hash string STATE -resources.job_runs.*.lifecycle.triggers.on_file_change.*.mtime_ns int64 STATE -resources.job_runs.*.lifecycle.triggers.on_file_change.*.size int64 STATE +resources.job_runs.*.lifecycle.triggers.on_file_change map[string]string STATE +resources.job_runs.*.lifecycle.triggers.on_file_change.* string STATE resources.job_runs.*.lifecycle.triggers[*] resources.JobRunTrigger INPUT resources.job_runs.*.lifecycle.triggers[*].on_bundle_deploy *bool INPUT resources.job_runs.*.lifecycle.triggers[*].on_file_change *string INPUT @@ -902,11 +899,8 @@ resources.job_runs.*.python_params []string ALL resources.job_runs.*.python_params[*] string ALL resources.job_runs.*.queue *jobs.QueueSettings ALL resources.job_runs.*.queue.enabled bool ALL -resources.job_runs.*.resolved_file_triggers map[string]resources.JobRunFileFingerprint INPUT -resources.job_runs.*.resolved_file_triggers.* resources.JobRunFileFingerprint INPUT -resources.job_runs.*.resolved_file_triggers.*.hash string INPUT -resources.job_runs.*.resolved_file_triggers.*.mtime_ns int64 INPUT -resources.job_runs.*.resolved_file_triggers.*.size int64 INPUT +resources.job_runs.*.resolved_file_triggers map[string]string INPUT +resources.job_runs.*.resolved_file_triggers.* string INPUT resources.job_runs.*.resolved_job_id int64 INPUT resources.job_runs.*.result_state jobs.RunResultState REMOTE STATE resources.job_runs.*.run_id int64 REMOTE diff --git a/acceptance/bundle/resources/job_runs/on_file_change/output.txt b/acceptance/bundle/resources/job_runs/on_file_change/output.txt index bcf5962f8fa..f356fe7fb61 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change/output.txt @@ -33,6 +33,17 @@ Resources: 0 created, 0 changed, 0 deleted, 2 unchanged >>> print_requests.py //jobs/run-now +=== touch without content change plans nothing +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change/default/files... +Files: 3 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 2 unchanged + +>>> print_requests.py //jobs/run-now + === editing the file re-fires >>> [CLI] bundle plan recreate job_runs.my_run diff --git a/acceptance/bundle/resources/job_runs/on_file_change/script b/acceptance/bundle/resources/job_runs/on_file_change/script index ff162e11f0b..0506dd5e106 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change/script +++ b/acceptance/bundle/resources/job_runs/on_file_change/script @@ -16,6 +16,12 @@ trace $CLI bundle plan trace $CLI bundle deploy trace print_requests.py //jobs/run-now +title "touch without content change plans nothing" +python -c "import os; os.utime('seed.txt', None)" +trace $CLI bundle plan +trace $CLI bundle deploy +trace print_requests.py //jobs/run-now + title "editing the file re-fires" update_file.py seed.txt "v1" "v2" trace $CLI bundle plan diff --git a/bundle/config/mutator/resolve_job_run_file_triggers.go b/bundle/config/mutator/resolve_job_run_file_triggers.go index f0338b2d50a..55a55a94b95 100644 --- a/bundle/config/mutator/resolve_job_run_file_triggers.go +++ b/bundle/config/mutator/resolve_job_run_file_triggers.go @@ -4,25 +4,23 @@ import ( "context" "crypto/sha256" "encoding/hex" - "encoding/json" "fmt" "io" "os" "path/filepath" + "strings" "github.com/databricks/cli/bundle" - "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/libs/diag" ) -// missingFileSentinelSize marks a pattern with no matching file so the next -// plan can distinguish "still missing" from "file appeared". -const missingFileSentinelSize = int64(-1) +// missingFileHash marks a pattern with no matching file so appear/disappear recreates. +const missingFileHash = "" type resolveJobRunFileTriggers struct{} // ResolveJobRunFileTriggers expands on_file_change globs and stores per-file -// fingerprints on each job_run for PrepareState to copy into local state. +// content hashes on each job_run for PrepareState to copy into local state. func ResolveJobRunFileTriggers() bundle.Mutator { return &resolveJobRunFileTriggers{} } @@ -34,155 +32,105 @@ func (*resolveJobRunFileTriggers) Name() string { func (*resolveJobRunFileTriggers) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { var diags diag.Diagnostics for name, jr := range b.Config.Resources.JobRuns { - if jr == nil { + if jr == nil || jr.Lifecycle == nil { continue } - patterns := jr.OnFileChangePatterns() - if len(patterns) == 0 { - continue + out := make(map[string]string) + for i, t := range jr.Lifecycle.Triggers { + if t.OnFileChange == nil { + continue + } + path := fmt.Sprintf("resources.job_runs.%s.lifecycle.triggers[%d].on_file_change", name, i) + hashes, d := resolveFileTrigger(b, path, strings.TrimSpace(*t.OnFileChange)) + diags = diags.Extend(d) + for k, v := range hashes { + out[k] = v + } + } + if len(out) == 0 { + jr.ResolvedFileTriggers = nil + } else { + jr.ResolvedFileTriggers = out } - fps, d := resolveFileTriggers(b, name, patterns, previousFileTriggers(b, name)) - diags = diags.Extend(d) - jr.ResolvedFileTriggers = fps } return diags } -// previousFileTriggers reads on_file_change fingerprints from deployment state -// when it is open (plan/deploy after StatePull). Used so unchanged content keeps -// a stable fingerprint across mtime-only updates (e.g. touch). -func previousFileTriggers(b *bundle.Bundle, name string) map[string]resources.JobRunFileFingerprint { - if b.DeploymentBundle.StateDB.Path == "" { - return nil - } - entry, ok := b.DeploymentBundle.StateDB.GetResourceEntry("resources.job_runs." + name) - if !ok || len(entry.State) == 0 { - return nil - } - var state struct { - Lifecycle *struct { - Triggers *struct { - OnFileChange map[string]resources.JobRunFileFingerprint `json:"on_file_change"` - } `json:"triggers"` - } `json:"lifecycle"` +func resolveFileTrigger(b *bundle.Bundle, loc, pattern string) (map[string]string, diag.Diagnostics) { + var diags diag.Diagnostics + out := make(map[string]string) + localPattern := filepath.FromSlash(pattern) + // Keep hashes under SyncRoot; same IsLocal gate as translate_paths. + if !filepath.IsLocal(localPattern) { + return out, diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: pattern %q is not under the sync root", pattern), + Locations: b.Config.GetLocations(loc), + }) } - if err := json.Unmarshal(entry.State, &state); err != nil { - return nil + matches, err := filepath.Glob(filepath.Join(b.SyncRootPath, localPattern)) + if err != nil { + return out, diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: invalid pattern %q: %s", pattern, err), + Locations: b.Config.GetLocations(loc), + }) } - if state.Lifecycle == nil || state.Lifecycle.Triggers == nil { - return nil + if len(matches) == 0 { + out[filepath.ToSlash(pattern)] = missingFileHash + return out, diags.Append(diag.Diagnostic{ + Severity: diag.Warning, + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: no files match %q", pattern), + Locations: b.Config.GetLocations(loc), + }) } - return state.Lifecycle.Triggers.OnFileChange -} - -func resolveFileTriggers(b *bundle.Bundle, name string, patterns []string, prev map[string]resources.JobRunFileFingerprint) (map[string]resources.JobRunFileFingerprint, diag.Diagnostics) { - var diags diag.Diagnostics - out := make(map[string]resources.JobRunFileFingerprint) - for _, pattern := range patterns { - path := fmt.Sprintf("resources.job_runs.%s.lifecycle.triggers", name) - localPattern := filepath.FromSlash(pattern) - // Keep fingerprints under SyncRoot; same IsLocal gate as translate_paths. - if !filepath.IsLocal(localPattern) { + regularMatches := 0 + sawNonRegular := false + for _, match := range matches { + info, err := os.Stat(match) + if err != nil { diags = diags.Append(diag.Diagnostic{ Severity: diag.Error, - Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: pattern %q is not under the sync root", pattern), - Locations: b.Config.GetLocations(path), + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: stat %q: %s", match, err), + Locations: b.Config.GetLocations(loc), }) continue } - matches, err := filepath.Glob(filepath.Join(b.SyncRootPath, localPattern)) - if err != nil { - diags = diags.Append(diag.Diagnostic{ - Severity: diag.Error, - Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: invalid pattern %q: %s", pattern, err), - Locations: b.Config.GetLocations(path), - }) + if !info.Mode().IsRegular() { + sawNonRegular = true continue } - if len(matches) == 0 { - // Distinct state when the path/glob matches nothing so appear/disappear recreates. - out[filepath.ToSlash(pattern)] = resources.JobRunFileFingerprint{ - Size: missingFileSentinelSize, - } + regularMatches++ + rel, err := filepath.Rel(b.SyncRootPath, match) + if err != nil || !filepath.IsLocal(rel) { diags = diags.Append(diag.Diagnostic{ - Severity: diag.Warning, - Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: no files match %q", pattern), - Locations: b.Config.GetLocations(path), + Severity: diag.Error, + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: matched path %q is not under the sync root", match), + Locations: b.Config.GetLocations(loc), }) continue } - regularMatches := 0 - sawNonRegular := false - for _, match := range matches { - info, err := os.Stat(match) - if err != nil { - diags = diags.Append(diag.Diagnostic{ - Severity: diag.Error, - Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: stat %q: %s", match, err), - Locations: b.Config.GetLocations(path), - }) - continue - } - if !info.Mode().IsRegular() { - sawNonRegular = true - continue - } - regularMatches++ - rel, err := filepath.Rel(b.SyncRootPath, match) - if err != nil || !filepath.IsLocal(rel) { - diags = diags.Append(diag.Diagnostic{ - Severity: diag.Error, - Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: matched path %q is not under the sync root", match), - Locations: b.Config.GetLocations(path), - }) - continue - } - key := filepath.ToSlash(rel) - fp, err := fingerprintFile(match, info, prev[key]) - if err != nil { - diags = diags.Append(diag.Diagnostic{ - Severity: diag.Error, - Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: hash %q: %s", match, err), - Locations: b.Config.GetLocations(path), - }) - continue - } - out[key] = fp - } - // A directory-only match would otherwise leave ResolvedFileTriggers empty - // and silently disarm the trigger while config still sets on_file_change. - if regularMatches == 0 && sawNonRegular { + hash, err := hashFile(match) + if err != nil { diags = diags.Append(diag.Diagnostic{ Severity: diag.Error, - Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: pattern %q matches no regular files", pattern), - Locations: b.Config.GetLocations(path), + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: hash %q: %s", match, err), + Locations: b.Config.GetLocations(loc), }) + continue } + out[filepath.ToSlash(rel)] = hash } - return out, diags -} - -// fingerprintFile builds a content fingerprint. If prev has the same size and -// mtime, it is reused without reading the file. If content hash matches prev, -// prev is reused so mtime-only updates (touch) do not change planned state. -func fingerprintFile(path string, info os.FileInfo, prev resources.JobRunFileFingerprint) (resources.JobRunFileFingerprint, error) { - size := info.Size() - mtime := info.ModTime().UnixNano() - if prev.Hash != "" && prev.Size == size && prev.MtimeNs == mtime { - return prev, nil + // A directory-only match would otherwise leave ResolvedFileTriggers empty + // and silently disarm the trigger while config still sets on_file_change. + if regularMatches == 0 && sawNonRegular { + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: pattern %q matches no regular files", pattern), + Locations: b.Config.GetLocations(loc), + }) } - hash, err := hashFile(path) - if err != nil { - return resources.JobRunFileFingerprint{}, err - } - if prev.Hash != "" && prev.Hash == hash { - return prev, nil - } - return resources.JobRunFileFingerprint{ - Hash: hash, - Size: size, - MtimeNs: mtime, - }, nil + return out, diags } func hashFile(path string) (string, error) { diff --git a/bundle/config/mutator/resolve_job_run_file_triggers_internal_test.go b/bundle/config/mutator/resolve_job_run_file_triggers_internal_test.go deleted file mode 100644 index e82bd22945b..00000000000 --- a/bundle/config/mutator/resolve_job_run_file_triggers_internal_test.go +++ /dev/null @@ -1,73 +0,0 @@ -package mutator - -import ( - "os" - "path/filepath" - "testing" - "time" - - "github.com/databricks/cli/bundle/config/resources" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestFingerprintFileContentStableAcrossTouch(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, "seed.txt") - require.NoError(t, os.WriteFile(path, []byte("v1"), 0o644)) - - info, err := os.Stat(path) - require.NoError(t, err) - first, err := fingerprintFile(path, info, resources.JobRunFileFingerprint{}) - require.NoError(t, err) - require.NotEmpty(t, first.Hash) - - // Advance mtime without changing contents (touch). - require.NoError(t, os.Chtimes(path, time.Now().Add(time.Minute), time.Now().Add(time.Minute))) - info, err = os.Stat(path) - require.NoError(t, err) - assert.NotEqual(t, first.MtimeNs, info.ModTime().UnixNano()) - - second, err := fingerprintFile(path, info, first) - require.NoError(t, err) - assert.Equal(t, first, second, "unchanged content must reuse the previous fingerprint") -} - -func TestFingerprintFileFastPathSkipsWhenMtimeAndSizeMatch(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, "seed.txt") - require.NoError(t, os.WriteFile(path, []byte("v1"), 0o644)) - - info, err := os.Stat(path) - require.NoError(t, err) - prev := resources.JobRunFileFingerprint{ - Hash: "not-the-real-hash", - Size: info.Size(), - MtimeNs: info.ModTime().UnixNano(), - } - - got, err := fingerprintFile(path, info, prev) - require.NoError(t, err) - assert.Equal(t, prev, got, "matching size+mtime must reuse prev without re-hashing") -} - -func TestFingerprintFileContentChange(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, "seed.txt") - require.NoError(t, os.WriteFile(path, []byte("v1"), 0o644)) - - info, err := os.Stat(path) - require.NoError(t, err) - first, err := fingerprintFile(path, info, resources.JobRunFileFingerprint{}) - require.NoError(t, err) - - // Different size so the mtime+size fast path cannot reuse prev. - require.NoError(t, os.WriteFile(path, []byte("v2-changed"), 0o644)) - info, err = os.Stat(path) - require.NoError(t, err) - second, err := fingerprintFile(path, info, first) - require.NoError(t, err) - assert.NotEqual(t, first.Hash, second.Hash) - assert.Equal(t, info.Size(), second.Size) - assert.Equal(t, info.ModTime().UnixNano(), second.MtimeNs) -} diff --git a/bundle/config/mutator/resolve_job_run_file_triggers_test.go b/bundle/config/mutator/resolve_job_run_file_triggers_test.go index 06266d75894..3ec663c89bd 100644 --- a/bundle/config/mutator/resolve_job_run_file_triggers_test.go +++ b/bundle/config/mutator/resolve_job_run_file_triggers_test.go @@ -17,7 +17,7 @@ import ( ) func TestResolveJobRunFileTriggers(t *testing.T) { - t.Run("matches files and fills fingerprints", func(t *testing.T) { + t.Run("matches files and fills hashes", func(t *testing.T) { dir := t.TempDir() require.NoError(t, os.WriteFile(filepath.Join(dir, "a.txt"), []byte("hello"), 0o644)) require.NoError(t, os.WriteFile(filepath.Join(dir, "b.txt"), []byte("world"), 0o644)) @@ -28,14 +28,13 @@ func TestResolveJobRunFileTriggers(t *testing.T) { diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) require.False(t, diags.HasError()) - fps := b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers - require.Len(t, fps, 2) - - assertFingerprint(t, fps["a.txt"], "hello") - assertFingerprint(t, fps["b.txt"], "world") + hashes := b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers + require.Len(t, hashes, 2) + assert.Equal(t, contentHash("hello"), hashes["a.txt"]) + assert.Equal(t, contentHash("world"), hashes["b.txt"]) }) - t.Run("no matches warns and stores sentinel", func(t *testing.T) { + t.Run("no matches warns and stores empty hash", func(t *testing.T) { dir := t.TempDir() pattern := "missing.txt" b := bundleWithFileTrigger(dir, pattern) @@ -46,12 +45,9 @@ func TestResolveJobRunFileTriggers(t *testing.T) { assert.Equal(t, diag.Warning, diags[0].Severity) assert.Contains(t, diags[0].Summary, `no files match "missing.txt"`) - fps := b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers - require.Len(t, fps, 1) - fp := fps["missing.txt"] - assert.Empty(t, fp.Hash) - assert.Equal(t, int64(-1), fp.Size) - assert.Zero(t, fp.MtimeNs) + hashes := b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers + require.Len(t, hashes, 1) + assert.Empty(t, hashes["missing.txt"]) }) t.Run("no file triggers is a no-op", func(t *testing.T) { @@ -108,10 +104,10 @@ func TestResolveJobRunFileTriggers(t *testing.T) { diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) require.False(t, diags.HasError()) - fps := b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers - require.Len(t, fps, 2) - assertFingerprint(t, fps["a.txt"], "aaa") - assertFingerprint(t, fps["subdir/x.py"], "bbb") + hashes := b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers + require.Len(t, hashes, 2) + assert.Equal(t, contentHash("aaa"), hashes["a.txt"]) + assert.Equal(t, contentHash("bbb"), hashes["subdir/x.py"]) }) t.Run("pattern outside sync root is an error", func(t *testing.T) { @@ -138,6 +134,17 @@ func TestResolveJobRunFileTriggers(t *testing.T) { assert.Contains(t, diags[0].Summary, `matches no regular files`) assert.Empty(t, b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers) }) + + t.Run("trims pattern whitespace", func(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "seed.txt"), []byte("v1"), 0o644)) + pattern := " seed.txt " + b := bundleWithFileTrigger(dir, pattern) + + diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) + require.False(t, diags.HasError()) + assert.Equal(t, contentHash("v1"), b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers["seed.txt"]) + }) } func bundleWithFileTrigger(syncRoot, pattern string) *bundle.Bundle { @@ -159,10 +166,7 @@ func bundleWithFileTrigger(syncRoot, pattern string) *bundle.Bundle { } } -func assertFingerprint(t *testing.T, fp resources.JobRunFileFingerprint, content string) { - t.Helper() +func contentHash(content string) string { sum := sha256.Sum256([]byte(content)) - assert.Equal(t, hex.EncodeToString(sum[:]), fp.Hash) - assert.Equal(t, int64(len(content)), fp.Size) - assert.NotZero(t, fp.MtimeNs) + return hex.EncodeToString(sum[:]) } diff --git a/bundle/config/resources/job_run.go b/bundle/config/resources/job_run.go index cf66c71f728..f4b16963453 100644 --- a/bundle/config/resources/job_run.go +++ b/bundle/config/resources/job_run.go @@ -28,9 +28,9 @@ type JobRun struct { // reference) lets state loading preserve that reference and its plan dependency. ResolvedJobID int64 `json:"resolved_job_id,omitempty" bundle:"internal"` - // ResolvedFileTriggers is the per-file fingerprint map for on_file_change, - // computed under SyncRoot before plan. bundle:"internal" keeps it out of schema. - ResolvedFileTriggers map[string]JobRunFileFingerprint `json:"resolved_file_triggers,omitempty" bundle:"internal"` + // ResolvedFileTriggers is path → content hash for on_file_change, computed + // under SyncRoot before plan. bundle:"internal" keeps it out of schema. + ResolvedFileTriggers map[string]string `json:"resolved_file_triggers,omitempty" bundle:"internal"` } // HasOnBundleDeploy reports whether any trigger re-fires on every deploy. @@ -58,19 +58,6 @@ func (r *JobRun) HasOnFileChange() bool { return false } -func (r *JobRun) OnFileChangePatterns() []string { - if r.Lifecycle == nil { - return nil - } - var patterns []string - for _, t := range r.Lifecycle.Triggers { - if t.OnFileChange != nil { - patterns = append(patterns, *t.OnFileChange) - } - } - return patterns -} - func (r *JobRun) UnmarshalJSON(b []byte) error { return marshal.Unmarshal(b, r) } diff --git a/bundle/config/resources/lifecycle.go b/bundle/config/resources/lifecycle.go index 59ccb682b81..a4e881d298c 100644 --- a/bundle/config/resources/lifecycle.go +++ b/bundle/config/resources/lifecycle.go @@ -39,9 +39,3 @@ type JobRunTrigger struct { OnBundleDeploy *bool `json:"on_bundle_deploy,omitempty"` OnFileChange *string `json:"on_file_change,omitempty"` // path or glob under sync root } - -type JobRunFileFingerprint struct { - Hash string `json:"hash"` - Size int64 `json:"size"` - MtimeNs int64 `json:"mtime_ns"` -} diff --git a/bundle/direct/dresources/job_run.go b/bundle/direct/dresources/job_run.go index f66991f1ca8..5b97cc9e22b 100644 --- a/bundle/direct/dresources/job_run.go +++ b/bundle/direct/dresources/job_run.go @@ -38,8 +38,8 @@ var jobRunTriggerLocalPaths = []string{ type JobRunTriggersState struct { // Fresh UUID each plan while armed so Old!=New forces recreate. OnBundleDeploy string `json:"on_bundle_deploy,omitempty"` - // Per-file fingerprints from ResolveJobRunFileTriggers; change to recreate. - OnFileChange map[string]resources.JobRunFileFingerprint `json:"on_file_change,omitempty"` + // Path → content hash from ResolveJobRunFileTriggers; change to recreate. + OnFileChange map[string]string `json:"on_file_change,omitempty"` } // JobRunLifecycleState holds local-only lifecycle fields persisted in state. diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index 0d49749c15a..b0a9ed6b365 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -182,12 +182,6 @@ resources: recreate_on_changes: - reason: immutable - # mtime is stored for the resolve fast path; content identity is the hash. - # Ignoring mtime-only local drift keeps touch from recreating the run. - ignore_local_changes: - - field: lifecycle.triggers.on_file_change[*].mtime_ns - reason: mtime_only_fingerprint - pipelines: recreate_on_changes: - field: storage diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index 8f8dc45d066..d48a3e2053a 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -992,7 +992,7 @@ resources: If true, re-fire the run on every bundle deploy. Incompatible with lifecycle.prevent_destroy. "on_file_change": "description": |- - Path or glob under the sync root (filepath.Glob; `**` is not supported). Re-fire the run when a matched file's content changes, or when the set of matches appears or disappears. Incompatible with lifecycle.prevent_destroy. + Path or glob under the sync root (filepath.Glob; `**` is not supported). Re-fire the run when a matched file's content hash changes, or when the set of matches appears or disappears. Incompatible with lifecycle.prevent_destroy. "python_named_params": "description": |- PLACEHOLDER diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 03146b5f1cc..78fdaf4a7a8 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -1313,7 +1313,7 @@ "$ref": "#/$defs/bool" }, "on_file_change": { - "description": "Path or glob under the sync root (filepath.Glob; `**` is not supported). Re-fire the run when a matched file's content changes, or when the set of matches appears or disappears. Incompatible with lifecycle.prevent_destroy.", + "description": "Path or glob under the sync root (filepath.Glob; `**` is not supported). Re-fire the run when a matched file's content hash changes, or when the set of matches appears or disappears. Incompatible with lifecycle.prevent_destroy.", "$ref": "#/$defs/string" } }, From 55e71e0eaeab2a15561ee61f8213e94d7d03118e Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Tue, 18 Aug 2026 13:23:44 +0000 Subject: [PATCH 53/75] job_runs: use maps.Copy in on_file_change hash merge The modernize linter rejects the manual m[k]=v copy loop, failing CI lint. --- bundle/config/mutator/resolve_job_run_file_triggers.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/bundle/config/mutator/resolve_job_run_file_triggers.go b/bundle/config/mutator/resolve_job_run_file_triggers.go index 55a55a94b95..ead17c59e17 100644 --- a/bundle/config/mutator/resolve_job_run_file_triggers.go +++ b/bundle/config/mutator/resolve_job_run_file_triggers.go @@ -6,6 +6,7 @@ import ( "encoding/hex" "fmt" "io" + "maps" "os" "path/filepath" "strings" @@ -43,9 +44,7 @@ func (*resolveJobRunFileTriggers) Apply(_ context.Context, b *bundle.Bundle) dia path := fmt.Sprintf("resources.job_runs.%s.lifecycle.triggers[%d].on_file_change", name, i) hashes, d := resolveFileTrigger(b, path, strings.TrimSpace(*t.OnFileChange)) diags = diags.Extend(d) - for k, v := range hashes { - out[k] = v - } + maps.Copy(out, hashes) } if len(out) == 0 { jr.ResolvedFileTriggers = nil From 6eb2db27d9525bc5e8501f22cd333de32e82b142 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Wed, 19 Aug 2026 08:39:23 +0000 Subject: [PATCH 54/75] job_runs: drop filepath.Glob jargon from on_file_change schema docs --- bundle/internal/schema/annotations.yml | 2 +- bundle/schema/jsonschema.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index d48a3e2053a..1fd6b452415 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -992,7 +992,7 @@ resources: If true, re-fire the run on every bundle deploy. Incompatible with lifecycle.prevent_destroy. "on_file_change": "description": |- - Path or glob under the sync root (filepath.Glob; `**` is not supported). Re-fire the run when a matched file's content hash changes, or when the set of matches appears or disappears. Incompatible with lifecycle.prevent_destroy. + Path or glob under the sync root. Re-fire the run when a matched file's content hash changes, or when the set of matches appears or disappears. Incompatible with lifecycle.prevent_destroy. "python_named_params": "description": |- PLACEHOLDER diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 78fdaf4a7a8..9afa34882ef 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -1313,7 +1313,7 @@ "$ref": "#/$defs/bool" }, "on_file_change": { - "description": "Path or glob under the sync root (filepath.Glob; `**` is not supported). Re-fire the run when a matched file's content hash changes, or when the set of matches appears or disappears. Incompatible with lifecycle.prevent_destroy.", + "description": "Path or glob under the sync root. Re-fire the run when a matched file's content hash changes, or when the set of matches appears or disappears. Incompatible with lifecycle.prevent_destroy.", "$ref": "#/$defs/string" } }, From 0ce58fd2046fd08776dd0717ab8935035ae13b86 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Wed, 19 Aug 2026 11:38:20 +0000 Subject: [PATCH 55/75] acc: refresh on_file_change goldens for resource-key progress lines --- .../bundle/resources/job_runs/on_file_change/output.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/acceptance/bundle/resources/job_runs/on_file_change/output.txt b/acceptance/bundle/resources/job_runs/on_file_change/output.txt index f356fe7fb61..a98eb3a6ca7 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change/output.txt @@ -2,8 +2,8 @@ === first deploy triggers a run >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change/default/files... -job run [MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?o=[NUMID] -job run [MY_RUN_ID]: SUCCESS +Output from job_runs.my_run: id=[MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?o=[NUMID] +Output from job_runs.my_run: id=[MY_RUN_ID]: SUCCESS Created job_runs.my_run Created jobs.my_job Files: 5 uploaded, 0 deleted @@ -52,8 +52,8 @@ Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change/default/files... -job run [MY_RUN_ID_2]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_2]?o=[NUMID] -job run [MY_RUN_ID_2]: SUCCESS +Output from job_runs.my_run: id=[MY_RUN_ID_2]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_2]?o=[NUMID] +Output from job_runs.my_run: id=[MY_RUN_ID_2]: SUCCESS Recreated job_runs.my_run Files: 3 uploaded, 0 deleted Resources: 1 created, 0 changed, 1 deleted, 1 unchanged From 3e384a1fd70bcae82621dc98ae41b7d8a7568148 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Wed, 19 Aug 2026 14:50:14 +0000 Subject: [PATCH 56/75] acc: cover on_file_change globs, errors, and appear/disappear --- .../job_runs/on_file_change/out.test.toml | 1 + .../job_runs/on_file_change/output.txt | 131 +++++++++++++- .../resources/job_runs/on_file_change/script | 26 ++- .../job_runs/on_file_change/test.toml | 14 ++ .../on_file_change_errors/databricks.yml | 18 ++ .../dirs/nested/keep.txt | 1 + .../on_file_change_errors/out.test.toml | 2 + .../job_runs/on_file_change_errors/output.txt | 39 +++++ .../job_runs/on_file_change_errors/script | 12 ++ .../job_runs/on_file_change_errors/test.toml | 2 + .../on_file_change_glob/databricks.yml | 18 ++ .../on_file_change_glob/migrations/001.txt | 1 + .../on_file_change_glob/migrations/002.txt | 1 + .../migrations/archive/003.txt | 1 + .../on_file_change_glob/out.test.toml | 2 + .../job_runs/on_file_change_glob/output.txt | 164 ++++++++++++++++++ .../job_runs/on_file_change_glob/script | 47 +++++ .../job_runs/on_file_change_glob/test.toml | 9 + bundle/phases/plan.go | 4 +- 19 files changed, 486 insertions(+), 7 deletions(-) create mode 100644 acceptance/bundle/resources/job_runs/on_file_change/test.toml create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_errors/databricks.yml create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_errors/dirs/nested/keep.txt create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_errors/out.test.toml create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_errors/script create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_errors/test.toml create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_glob/databricks.yml create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/001.txt create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/002.txt create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/archive/003.txt create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_glob/out.test.toml create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_glob/script create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml diff --git a/acceptance/bundle/resources/job_runs/on_file_change/out.test.toml b/acceptance/bundle/resources/job_runs/on_file_change/out.test.toml index 0938e678987..57b0f616850 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change/out.test.toml +++ b/acceptance/bundle/resources/job_runs/on_file_change/out.test.toml @@ -1,2 +1,3 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/job_runs/on_file_change/output.txt b/acceptance/bundle/resources/job_runs/on_file_change/output.txt index a98eb3a6ca7..d03307b90f2 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change/output.txt @@ -6,7 +6,7 @@ Output from job_runs.my_run: id=[MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[MY_ Output from job_runs.my_run: id=[MY_RUN_ID]: SUCCESS Created job_runs.my_run Created jobs.my_job -Files: 5 uploaded, 0 deleted +Files: 6 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged >>> read_id.py my_job @@ -50,12 +50,46 @@ recreate job_runs.my_run Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged ->>> [CLI] bundle deploy +>>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json +{ + "lifecycle": { + "action": "recreate", + "reason": "immutable", + "old": { + "triggers": { + "on_file_change": { + "seed.txt": "[FILE_HASH][0]" + } + } + }, + "new": { + "triggers": { + "on_file_change": { + "seed.txt": "[FILE_HASH][1]" + } + } + } + }, + "lifecycle.triggers.on_file_change['seed.txt']": { + "action": "recreate", + "reason": "immutable", + "old": "[FILE_HASH][0]", + "new": "[FILE_HASH][1]" + }, + "result_state": { + "action": "skip", + "reason": "remote_already_set", + "new": "SUCCESS", + "remote": "SUCCESS" + } +} + +=== bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change/default/files... Output from job_runs.my_run: id=[MY_RUN_ID_2]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_2]?o=[NUMID] Output from job_runs.my_run: id=[MY_RUN_ID_2]: SUCCESS Recreated job_runs.my_run -Files: 3 uploaded, 0 deleted +Files: 4 uploaded, 0 deleted Resources: 1 created, 0 changed, 1 deleted, 1 unchanged >>> print_requests.py --keep //jobs/runs/delete @@ -77,6 +111,97 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged } } +=== deleting the file warns and re-fires +>>> [CLI] bundle plan +Warning: lifecycle.triggers.on_file_change: no files match "seed.txt" + in databricks.yml:18:29 + +recreate job_runs.my_run + +Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged + +>>> changes +Warning: lifecycle.triggers.on_file_change: no files match "seed.txt" + in databricks.yml:18:29 + +{ + "lifecycle": { + "action": "recreate", + "reason": "immutable", + "old": { + "triggers": { + "on_file_change": { + "seed.txt": "[FILE_HASH][1]" + } + } + }, + "new": { + "triggers": { + "on_file_change": { + "seed.txt": "" + } + } + } + }, + "lifecycle.triggers.on_file_change['seed.txt']": { + "action": "recreate", + "reason": "immutable", + "old": "[FILE_HASH][1]", + "new": "" + }, + "result_state": { + "action": "skip", + "reason": "remote_already_set", + "new": "SUCCESS", + "remote": "SUCCESS" + } +} + +>>> [CLI] bundle deploy +Warning: lifecycle.triggers.on_file_change: no files match "seed.txt" + in databricks.yml:18:29 + +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change/default/files... +Output from job_runs.my_run: id=[MY_RUN_ID_3]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_3]?o=[NUMID] +Output from job_runs.my_run: id=[MY_RUN_ID_3]: SUCCESS +Recreated job_runs.my_run +Files: 3 uploaded, 1 deleted +Resources: 1 created, 0 changed, 1 deleted, 1 unchanged + +>>> print_requests.py //jobs/run-now +{ + "method": "POST", + "path": "/api/2.2/jobs/run-now", + "body": { + "idempotency_token": "[UUID]", + "job_id": [MY_JOB_ID] + } +} + +=== restoring the file re-fires again +>>> [CLI] bundle plan +recreate job_runs.my_run + +Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change/default/files... +Output from job_runs.my_run: id=[MY_RUN_ID_4]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_4]?o=[NUMID] +Output from job_runs.my_run: id=[MY_RUN_ID_4]: SUCCESS +Recreated job_runs.my_run +Files: 3 uploaded, 1 deleted +Resources: 1 created, 0 changed, 1 deleted, 1 unchanged + +>>> print_requests.py //jobs/run-now +{ + "method": "POST", + "path": "/api/2.2/jobs/run-now", + "body": { + "idempotency_token": "[UUID]", + "job_id": [MY_JOB_ID] + } +} + === removing on_file_change rewrites state without a run >>> [CLI] bundle plan update job_runs.my_run diff --git a/acceptance/bundle/resources/job_runs/on_file_change/script b/acceptance/bundle/resources/job_runs/on_file_change/script index 0506dd5e106..fd1e8800c42 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change/script +++ b/acceptance/bundle/resources/job_runs/on_file_change/script @@ -4,6 +4,10 @@ cleanup() { } trap cleanup EXIT +changes() { + $CLI bundle plan -o json | jq '.plan["resources.job_runs.my_run"].changes' +} + title "first deploy triggers a run" trace $CLI bundle deploy trace read_id.py my_job @@ -25,11 +29,31 @@ trace print_requests.py //jobs/run-now title "editing the file re-fires" update_file.py seed.txt "v1" "v2" trace $CLI bundle plan -trace $CLI bundle deploy +# Save the plan so the READPLAN=1 variant deploys the hashes computed here. +# The deploy is not traced: readplanarg makes the command line differ per variant. +$CLI bundle plan -o json > tmp.plan.json +trace jq '.plan["resources.job_runs.my_run"].changes' tmp.plan.json +title "bundle deploy\n" +$CLI bundle deploy $(readplanarg tmp.plan.json) read_id.py my_run > /dev/null trace print_requests.py --keep //jobs/runs/delete trace print_requests.py //jobs/run-now +title "deleting the file warns and re-fires" +mv seed.txt tmp.seed.txt +trace $CLI bundle plan +trace changes +trace $CLI bundle deploy +read_id.py my_run > /dev/null +trace print_requests.py //jobs/run-now + +title "restoring the file re-fires again" +mv tmp.seed.txt seed.txt +trace $CLI bundle plan +trace $CLI bundle deploy +read_id.py my_run > /dev/null +trace print_requests.py //jobs/run-now + title "removing on_file_change rewrites state without a run" update_file.py databricks.yml " lifecycle: triggers: diff --git a/acceptance/bundle/resources/job_runs/on_file_change/test.toml b/acceptance/bundle/resources/job_runs/on_file_change/test.toml new file mode 100644 index 00000000000..7a950e9fd0c --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change/test.toml @@ -0,0 +1,14 @@ +# Deploy both by re-planning and from a plan saved on disk, so the hashes +# computed during planning survive plan serialization. +EnvMatrix.READPLAN = ["", "1"] + +Ignore = ["tmp.plan.json", "tmp.seed.txt"] + +# Content hashes are stable but unreadable, and the parent's `\d{8,}` rule +# mangles any digit run inside them. Order=1 runs before it; Distinct keeps +# different hashes distinguishable. +[[Repls]] +Old = '[0-9a-f]{64}' +New = '[FILE_HASH]' +Order = 1 +Distinct = true diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/databricks.yml b/acceptance/bundle/resources/job_runs/on_file_change_errors/databricks.yml new file mode 100644 index 00000000000..44c5433e52a --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/databricks.yml @@ -0,0 +1,18 @@ +bundle: + name: job-runs-on-file-change-errors + +resources: + jobs: + my_job: + name: my-job + tasks: + - task_key: main + notebook_task: + notebook_path: /Workspace/test + + job_runs: + my_run: + job_id: ${resources.jobs.my_job.id} + lifecycle: + triggers: + - on_file_change: ../shared/*.txt diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/dirs/nested/keep.txt b/acceptance/bundle/resources/job_runs/on_file_change_errors/dirs/nested/keep.txt new file mode 100644 index 00000000000..79c53955ef8 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/dirs/nested/keep.txt @@ -0,0 +1 @@ +nested diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/out.test.toml b/acceptance/bundle/resources/job_runs/on_file_change_errors/out.test.toml new file mode 100644 index 00000000000..0938e678987 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt new file mode 100644 index 00000000000..c7d3a98e1cb --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt @@ -0,0 +1,39 @@ + +=== a pattern outside the sync root is rejected +>>> musterr [CLI] bundle validate +Error: lifecycle.triggers.on_file_change: pattern "../shared/*.txt" is not under the sync root + in databricks.yml:18:29 + +Name: job-runs-on-file-change-errors +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-errors/default + +Found 1 error + +=== a glob that only matches directories is rejected +>>> musterr [CLI] bundle validate +Error: lifecycle.triggers.on_file_change: pattern "dirs/*" matches no regular files + in databricks.yml:18:29 + +Name: job-runs-on-file-change-errors +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-errors/default + +Found 1 error + +=== a pattern that matches nothing warns +>>> [CLI] bundle validate +Warning: lifecycle.triggers.on_file_change: no files match "missing/*.txt" + in databricks.yml:18:29 + +Name: job-runs-on-file-change-errors +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-errors/default + +Found 1 warning diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/script b/acceptance/bundle/resources/job_runs/on_file_change_errors/script new file mode 100644 index 00000000000..39596f0c772 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/script @@ -0,0 +1,12 @@ +title "a pattern outside the sync root is rejected" +trace musterr $CLI bundle validate + +title "a glob that only matches directories is rejected" +# dirs/* matches dirs/nested, a directory: nothing is left to hash, so the +# trigger would silently never fire. +update_file.py databricks.yml "../shared/*.txt" "dirs/*" +trace musterr $CLI bundle validate + +title "a pattern that matches nothing warns" +update_file.py databricks.yml "dirs/*" "missing/*.txt" +trace $CLI bundle validate diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/test.toml b/acceptance/bundle/resources/job_runs/on_file_change_errors/test.toml new file mode 100644 index 00000000000..7b88d613d4c --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/test.toml @@ -0,0 +1,2 @@ +# The bundle is never deployed here, so there are no requests worth recording. +RecordRequests = false diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/databricks.yml b/acceptance/bundle/resources/job_runs/on_file_change_glob/databricks.yml new file mode 100644 index 00000000000..3b148538f38 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/databricks.yml @@ -0,0 +1,18 @@ +bundle: + name: job-runs-on-file-change-glob + +resources: + jobs: + my_job: + name: my-job + tasks: + - task_key: main + notebook_task: + notebook_path: /Workspace/test + + job_runs: + my_run: + job_id: ${resources.jobs.my_job.id} + lifecycle: + triggers: + - on_file_change: migrations/* diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/001.txt b/acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/001.txt new file mode 100644 index 00000000000..5626abf0f72 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/001.txt @@ -0,0 +1 @@ +one diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/002.txt b/acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/002.txt new file mode 100644 index 00000000000..f719efd430d --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/002.txt @@ -0,0 +1 @@ +two diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/archive/003.txt b/acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/archive/003.txt new file mode 100644 index 00000000000..2bdf67abb16 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/archive/003.txt @@ -0,0 +1 @@ +three diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/out.test.toml b/acceptance/bundle/resources/job_runs/on_file_change_glob/out.test.toml new file mode 100644 index 00000000000..0938e678987 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt new file mode 100644 index 00000000000..798b459a9bd --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt @@ -0,0 +1,164 @@ + +=== deploy hashes the matched files +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-glob/default/files... +Output from job_runs.my_run: id=[MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?o=[NUMID] +Output from job_runs.my_run: id=[MY_RUN_ID]: SUCCESS +Created job_runs.my_run +Created jobs.my_job +Files: 8 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +>>> read_id.py my_job +[MY_JOB_ID] + +>>> print_requests.py //jobs/run-now +{ + "method": "POST", + "path": "/api/2.2/jobs/run-now", + "body": { + "idempotency_token": "[UUID]", + "job_id": [MY_JOB_ID] + } +} + +=== editing one matched file re-fires +>>> [CLI] bundle plan +recreate job_runs.my_run + +Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged + +>>> changes +{ + "lifecycle": { + "action": "recreate", + "reason": "immutable", + "old": { + "triggers": { + "on_file_change": { + "migrations/001.txt": "[FILE_HASH][0]", + "migrations/002.txt": "[FILE_HASH][1]" + } + } + }, + "new": { + "triggers": { + "on_file_change": { + "migrations/001.txt": "[FILE_HASH][2]", + "migrations/002.txt": "[FILE_HASH][1]" + } + } + } + }, + "lifecycle.triggers.on_file_change['migrations/001.txt']": { + "action": "recreate", + "reason": "immutable", + "old": "[FILE_HASH][0]", + "new": "[FILE_HASH][2]" + }, + "result_state": { + "action": "skip", + "reason": "remote_already_set", + "new": "SUCCESS", + "remote": "SUCCESS" + } +} + +=== a new matching file re-fires +>>> [CLI] bundle plan +recreate job_runs.my_run + +Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged + +>>> changes +{ + "lifecycle": { + "action": "recreate", + "reason": "immutable", + "old": { + "triggers": { + "on_file_change": { + "migrations/001.txt": "[FILE_HASH][0]", + "migrations/002.txt": "[FILE_HASH][1]" + } + } + }, + "new": { + "triggers": { + "on_file_change": { + "migrations/001.txt": "[FILE_HASH][0]", + "migrations/002.txt": "[FILE_HASH][1]", + "migrations/003.txt": "[FILE_HASH][3]" + } + } + } + }, + "lifecycle.triggers.on_file_change['migrations/003.txt']": { + "action": "recreate", + "reason": "immutable", + "new": "[FILE_HASH][3]" + }, + "result_state": { + "action": "skip", + "reason": "remote_already_set", + "new": "SUCCESS", + "remote": "SUCCESS" + } +} + +=== removing a matched file re-fires +>>> [CLI] bundle plan +recreate job_runs.my_run + +Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged + +>>> changes +{ + "lifecycle": { + "action": "recreate", + "reason": "immutable", + "old": { + "triggers": { + "on_file_change": { + "migrations/001.txt": "[FILE_HASH][0]", + "migrations/002.txt": "[FILE_HASH][1]" + } + } + }, + "new": { + "triggers": { + "on_file_change": { + "migrations/001.txt": "[FILE_HASH][0]" + } + } + } + }, + "lifecycle.triggers.on_file_change['migrations/002.txt']": { + "action": "recreate", + "reason": "immutable", + "old": "[FILE_HASH][1]" + }, + "result_state": { + "action": "skip", + "reason": "remote_already_set", + "new": "SUCCESS", + "remote": "SUCCESS" + } +} + +=== editing a file below the glob does not re-fire +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +=== ** is not recursive: it matches the same files as * +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.job_runs.my_run + delete resources.jobs.my_job + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-glob/default + +Destroy: 2 deleted diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/script b/acceptance/bundle/resources/job_runs/on_file_change_glob/script new file mode 100644 index 00000000000..8feeebcd067 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/script @@ -0,0 +1,47 @@ +cleanup() { + trace $CLI bundle destroy --auto-approve + rm -f out.requests.txt +} +trap cleanup EXIT + +changes() { + $CLI bundle plan -o json | jq '.plan["resources.job_runs.my_run"].changes' +} + +title "deploy hashes the matched files" +trace $CLI bundle deploy +trace read_id.py my_job +read_id.py my_run > /dev/null +trace print_requests.py //jobs/run-now + +# Each phase below plans against the state deployed above and then restores the +# tree, so every plan is compared against the same baseline. + +title "editing one matched file re-fires" +update_file.py migrations/001.txt "one" "one-edited" +trace $CLI bundle plan +trace changes +update_file.py migrations/001.txt "one-edited" "one" + +title "a new matching file re-fires" +printf 'three\n' > migrations/003.txt +trace $CLI bundle plan +trace changes +rm migrations/003.txt + +title "removing a matched file re-fires" +mv migrations/002.txt tmp.002.txt +trace $CLI bundle plan +trace changes +mv tmp.002.txt migrations/002.txt + +title "editing a file below the glob does not re-fire" +# migrations/* matches the archive directory itself, which is skipped: the glob +# does not descend into it. +update_file.py migrations/archive/003.txt "three" "three-edited" +trace $CLI bundle plan +update_file.py migrations/archive/003.txt "three-edited" "three" + +title "** is not recursive: it matches the same files as *" +update_file.py databricks.yml "migrations/*" "migrations/**" +trace $CLI bundle plan diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml b/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml new file mode 100644 index 00000000000..1c349066b94 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml @@ -0,0 +1,9 @@ +Ignore = ["tmp.002.txt"] + +# See on_file_change/test.toml: mask content hashes before the parent's +# `\d{8,}` rule can mangle them, keeping distinct hashes distinguishable. +[[Repls]] +Old = '[0-9a-f]{64}' +New = '[FILE_HASH]' +Order = 1 +Distinct = true diff --git a/bundle/phases/plan.go b/bundle/phases/plan.go index 92ca524cc4f..3db0864c2c4 100644 --- a/bundle/phases/plan.go +++ b/bundle/phases/plan.go @@ -21,8 +21,7 @@ import ( // PreDeployChecks is common set of mutators between "bundle plan" and "bundle deploy". // Note, it is not run in "bundle migrate" so it must not modify the config func PreDeployChecks(ctx context.Context, b *bundle.Bundle, isPlan bool, engine engine.EngineType) { - bundle.ApplySeqContext( - ctx, b, + bundle.ApplySeqContext(ctx, b, terraform.CheckDashboardsModifiedRemotely(isPlan, engine), resourcemutator.SecretScopeFixups(engine), deploy.StatePull(), @@ -31,7 +30,6 @@ func PreDeployChecks(ctx context.Context, b *bundle.Bundle, isPlan bool, engine mutator.ValidateLifecycleStarted(engine), mutator.ValidateCascadeOnDestroy(engine), mutator.ValidateJobRunTriggers(), - mutator.ResolveJobRunFileTriggers(), statemgmt.CheckRunningResource(engine), ) } From fd93d18d1697c3db4776f7f161cb3420513812b4 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Wed, 19 Aug 2026 19:58:11 +0000 Subject: [PATCH 57/75] acc: expand on_file_change glob coverage and skip ignored files Honor .gitignore and sync.exclude when hashing matches. Cover mysubdir globs, READPLAN, and keep LF on Windows so restored files hash cleanly. --- .../job_runs/on_file_change_glob/.gitignore | 2 + .../on_file_change_glob/databricks.yml | 1 + .../on_file_change_glob/mysubdir/a.txt | 1 + .../on_file_change_glob/out.test.toml | 1 + .../job_runs/on_file_change_glob/output.txt | 183 +++++++++++++++--- .../job_runs/on_file_change_glob/script | 39 ++-- .../job_runs/on_file_change_glob/test.toml | 6 +- .../mutator/resolve_job_run_file_triggers.go | 65 ++++++- .../resolve_job_run_file_triggers_test.go | 40 ++++ 9 files changed, 287 insertions(+), 51 deletions(-) create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_glob/.gitignore create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_glob/mysubdir/a.txt diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/.gitignore b/acceptance/bundle/resources/job_runs/on_file_change_glob/.gitignore new file mode 100644 index 00000000000..57433d43f47 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/.gitignore @@ -0,0 +1,2 @@ +# Matches migrations/* but sync skips it, so a content edit must not re-fire. +migrations/ignored.txt diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/databricks.yml b/acceptance/bundle/resources/job_runs/on_file_change_glob/databricks.yml index 3b148538f38..d47fcf44049 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/databricks.yml +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/databricks.yml @@ -16,3 +16,4 @@ resources: lifecycle: triggers: - on_file_change: migrations/* + - on_file_change: mysubdir/*.txt diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/mysubdir/a.txt b/acceptance/bundle/resources/job_runs/on_file_change_glob/mysubdir/a.txt new file mode 100644 index 00000000000..62e0af52c19 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/mysubdir/a.txt @@ -0,0 +1 @@ +sub diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/out.test.toml b/acceptance/bundle/resources/job_runs/on_file_change_glob/out.test.toml index 0938e678987..57b0f616850 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/out.test.toml +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/out.test.toml @@ -1,2 +1,3 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt index 798b459a9bd..d424482b961 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt @@ -6,7 +6,7 @@ Output from job_runs.my_run: id=[MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[MY_ Output from job_runs.my_run: id=[MY_RUN_ID]: SUCCESS Created job_runs.my_run Created jobs.my_job -Files: 8 uploaded, 0 deleted +Files: 10 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged >>> read_id.py my_job @@ -23,12 +23,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged } === editing one matched file re-fires ->>> [CLI] bundle plan -recreate job_runs.my_run - -Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged - ->>> changes +>>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { "lifecycle": { "action": "recreate", @@ -37,15 +32,17 @@ Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged "triggers": { "on_file_change": { "migrations/001.txt": "[FILE_HASH][0]", - "migrations/002.txt": "[FILE_HASH][1]" + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][2]" } } }, "new": { "triggers": { "on_file_change": { - "migrations/001.txt": "[FILE_HASH][2]", - "migrations/002.txt": "[FILE_HASH][1]" + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][2]" } } } @@ -54,7 +51,7 @@ Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged "action": "recreate", "reason": "immutable", "old": "[FILE_HASH][0]", - "new": "[FILE_HASH][2]" + "new": "[FILE_HASH][3]" }, "result_state": { "action": "skip", @@ -64,13 +61,35 @@ Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged } } -=== a new matching file re-fires ->>> [CLI] bundle plan -recreate job_runs.my_run +=== bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-glob/default/files... +Output from job_runs.my_run: id=[MY_RUN_ID_2]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_2]?o=[NUMID] +Output from job_runs.my_run: id=[MY_RUN_ID_2]: SUCCESS +Recreated job_runs.my_run +Files: 4 uploaded, 0 deleted +Resources: 1 created, 0 changed, 1 deleted, 1 unchanged + +>>> print_requests.py --keep //jobs/runs/delete +{ + "method": "POST", + "path": "/api/2.2/jobs/runs/delete", + "body": { + "run_id": [MY_RUN_ID] + } +} -Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged +>>> print_requests.py //jobs/run-now +{ + "method": "POST", + "path": "/api/2.2/jobs/run-now", + "body": { + "idempotency_token": "[UUID]", + "job_id": [MY_JOB_ID] + } +} ->>> changes +=== editing a file in a subdirectory re-fires +>>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { "lifecycle": { "action": "recreate", @@ -78,17 +97,58 @@ Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged "old": { "triggers": { "on_file_change": { - "migrations/001.txt": "[FILE_HASH][0]", - "migrations/002.txt": "[FILE_HASH][1]" + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][2]" } } }, "new": { "triggers": { "on_file_change": { - "migrations/001.txt": "[FILE_HASH][0]", + "migrations/001.txt": "[FILE_HASH][3]", "migrations/002.txt": "[FILE_HASH][1]", - "migrations/003.txt": "[FILE_HASH][3]" + "mysubdir/a.txt": "[FILE_HASH][4]" + } + } + } + }, + "lifecycle.triggers.on_file_change['mysubdir/a.txt']": { + "action": "recreate", + "reason": "immutable", + "old": "[FILE_HASH][2]", + "new": "[FILE_HASH][4]" + }, + "result_state": { + "action": "skip", + "reason": "remote_already_set", + "new": "SUCCESS", + "remote": "SUCCESS" + } +} + +=== a new matching file re-fires +>>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json +{ + "lifecycle": { + "action": "recreate", + "reason": "immutable", + "old": { + "triggers": { + "on_file_change": { + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][2]" + } + } + }, + "new": { + "triggers": { + "on_file_change": { + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "migrations/003.txt": "[FILE_HASH][5]", + "mysubdir/a.txt": "[FILE_HASH][2]" } } } @@ -96,7 +156,7 @@ Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged "lifecycle.triggers.on_file_change['migrations/003.txt']": { "action": "recreate", "reason": "immutable", - "new": "[FILE_HASH][3]" + "new": "[FILE_HASH][5]" }, "result_state": { "action": "skip", @@ -107,12 +167,7 @@ Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged } === removing a matched file re-fires ->>> [CLI] bundle plan -recreate job_runs.my_run - -Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged - ->>> changes +>>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { "lifecycle": { "action": "recreate", @@ -120,15 +175,17 @@ Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged "old": { "triggers": { "on_file_change": { - "migrations/001.txt": "[FILE_HASH][0]", - "migrations/002.txt": "[FILE_HASH][1]" + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][2]" } } }, "new": { "triggers": { "on_file_change": { - "migrations/001.txt": "[FILE_HASH][0]" + "migrations/001.txt": "[FILE_HASH][3]", + "mysubdir/a.txt": "[FILE_HASH][2]" } } } @@ -147,12 +204,74 @@ Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged } === editing a file below the glob does not re-fire +>>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json +{ + "lifecycle": { + "action": "skip", + "reason": "missing_in_remote", + "old": { + "triggers": { + "on_file_change": { + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][2]" + } + } + }, + "new": { + "triggers": { + "on_file_change": { + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][2]" + } + } + } + }, + "result_state": { + "action": "skip", + "reason": "remote_already_set", + "new": "SUCCESS", + "remote": "SUCCESS" + } +} + +=== editing a gitignored file does not re-fire >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged === ** is not recursive: it matches the same files as * ->>> [CLI] bundle plan -Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged +>>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json +{ + "lifecycle": { + "action": "skip", + "reason": "missing_in_remote", + "old": { + "triggers": { + "on_file_change": { + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][2]" + } + } + }, + "new": { + "triggers": { + "on_file_change": { + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][2]" + } + } + } + }, + "result_state": { + "action": "skip", + "reason": "remote_already_set", + "new": "SUCCESS", + "remote": "SUCCESS" + } +} >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/script b/acceptance/bundle/resources/job_runs/on_file_change_glob/script index 8feeebcd067..983cb9ac401 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/script +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/script @@ -4,8 +4,10 @@ cleanup() { } trap cleanup EXIT +# Plan once, print JSON changes, leave tmp.plan.json for readplanarg. changes() { - $CLI bundle plan -o json | jq '.plan["resources.job_runs.my_run"].changes' + $CLI bundle plan -o json > tmp.plan.json + trace jq '.plan["resources.job_runs.my_run"].changes' tmp.plan.json } title "deploy hashes the matched files" @@ -14,34 +16,45 @@ trace read_id.py my_job read_id.py my_run > /dev/null trace print_requests.py //jobs/run-now -# Each phase below plans against the state deployed above and then restores the -# tree, so every plan is compared against the same baseline. - title "editing one matched file re-fires" update_file.py migrations/001.txt "one" "one-edited" -trace $CLI bundle plan -trace changes -update_file.py migrations/001.txt "one-edited" "one" +changes +# The deploy is not traced: readplanarg makes the command line differ per variant. +title "bundle deploy\n" +$CLI bundle deploy $(readplanarg tmp.plan.json) +read_id.py my_run > /dev/null +trace print_requests.py --keep //jobs/runs/delete +trace print_requests.py //jobs/run-now + +title "editing a file in a subdirectory re-fires" +# printf keeps LF on Windows, where update_file.py's text-mode rewrite would +# restore CRLF and leave the file hashing differently than the deploy above. +printf 'sub-edited\n' > mysubdir/a.txt +changes +printf 'sub\n' > mysubdir/a.txt title "a new matching file re-fires" printf 'three\n' > migrations/003.txt -trace $CLI bundle plan -trace changes +changes rm migrations/003.txt title "removing a matched file re-fires" mv migrations/002.txt tmp.002.txt -trace $CLI bundle plan -trace changes +changes mv tmp.002.txt migrations/002.txt title "editing a file below the glob does not re-fire" # migrations/* matches the archive directory itself, which is skipped: the glob # does not descend into it. update_file.py migrations/archive/003.txt "three" "three-edited" -trace $CLI bundle plan +changes update_file.py migrations/archive/003.txt "three-edited" "three" +title "editing a gitignored file does not re-fire" +update_file.py migrations/ignored.txt "ignored" "ignored-edited" +trace $CLI bundle plan +update_file.py migrations/ignored.txt "ignored-edited" "ignored" + title "** is not recursive: it matches the same files as *" update_file.py databricks.yml "migrations/*" "migrations/**" -trace $CLI bundle plan +changes diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml b/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml index 1c349066b94..3516572564d 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml @@ -1,4 +1,8 @@ -Ignore = ["tmp.002.txt"] +# Deploy both by re-planning and from a plan saved on disk, so the hashes +# computed during planning survive plan serialization. +EnvMatrix.READPLAN = ["", "1"] + +Ignore = ["tmp.plan.json", "tmp.002.txt"] # See on_file_change/test.toml: mask content hashes before the parent's # `\d{8,}` rule can mangle them, keeping distinct hashes distinguishable. diff --git a/bundle/config/mutator/resolve_job_run_file_triggers.go b/bundle/config/mutator/resolve_job_run_file_triggers.go index ead17c59e17..d74b9841c57 100644 --- a/bundle/config/mutator/resolve_job_run_file_triggers.go +++ b/bundle/config/mutator/resolve_job_run_file_triggers.go @@ -13,6 +13,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" + libsync "github.com/databricks/cli/libs/sync" ) // missingFileHash marks a pattern with no matching file so appear/disappear recreates. @@ -30,8 +31,13 @@ func (*resolveJobRunFileTriggers) Name() string { return "ResolveJobRunFileTriggers" } -func (*resolveJobRunFileTriggers) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { +func (*resolveJobRunFileTriggers) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { var diags diag.Diagnostics + syncable, d := syncableRelPaths(ctx, b) + if d.HasError() { + return d + } + diags = diags.Extend(d) for name, jr := range b.Config.Resources.JobRuns { if jr == nil || jr.Lifecycle == nil { continue @@ -42,7 +48,7 @@ func (*resolveJobRunFileTriggers) Apply(_ context.Context, b *bundle.Bundle) dia continue } path := fmt.Sprintf("resources.job_runs.%s.lifecycle.triggers[%d].on_file_change", name, i) - hashes, d := resolveFileTrigger(b, path, strings.TrimSpace(*t.OnFileChange)) + hashes, d := resolveFileTrigger(b, path, strings.TrimSpace(*t.OnFileChange), syncable) diags = diags.Extend(d) maps.Copy(out, hashes) } @@ -55,7 +61,42 @@ func (*resolveJobRunFileTriggers) Apply(_ context.Context, b *bundle.Bundle) dia return diags } -func resolveFileTrigger(b *bundle.Bundle, loc, pattern string) (map[string]string, diag.Diagnostics) { +// syncableRelPaths is the set of relative paths sync would upload. +func syncableRelPaths(ctx context.Context, b *bundle.Bundle) (map[string]struct{}, diag.Diagnostics) { + var diags diag.Diagnostics + needs := false + for _, jr := range b.Config.Resources.JobRuns { + if jr != nil && jr.HasOnFileChange() { + needs = true + break + } + } + if !needs { + return nil, diags + } + + fl, err := libsync.NewFileList(ctx, b.WorktreeRoot, b.SyncRoot, b.Config.Sync.Paths, b.Config.Sync.Include, b.Config.Sync.Exclude) + if err != nil { + return nil, diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: list sync files: %s", err), + }) + } + files, err := fl.Files(ctx) + if err != nil { + return nil, diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: list sync files: %s", err), + }) + } + out := make(map[string]struct{}, len(files)) + for _, f := range files { + out[filepath.ToSlash(f.Relative)] = struct{}{} + } + return out, diags +} + +func resolveFileTrigger(b *bundle.Bundle, loc, pattern string, syncable map[string]struct{}) (map[string]string, diag.Diagnostics) { var diags diag.Diagnostics out := make(map[string]string) localPattern := filepath.FromSlash(pattern) @@ -84,6 +125,7 @@ func resolveFileTrigger(b *bundle.Bundle, loc, pattern string) (map[string]strin }) } regularMatches := 0 + ignoredMatches := 0 sawNonRegular := false for _, match := range matches { info, err := os.Stat(match) @@ -99,7 +141,6 @@ func resolveFileTrigger(b *bundle.Bundle, loc, pattern string) (map[string]strin sawNonRegular = true continue } - regularMatches++ rel, err := filepath.Rel(b.SyncRootPath, match) if err != nil || !filepath.IsLocal(rel) { diags = diags.Append(diag.Diagnostic{ @@ -109,6 +150,12 @@ func resolveFileTrigger(b *bundle.Bundle, loc, pattern string) (map[string]strin }) continue } + // Same membership as sync: .gitignore and sync.exclude drop a glob match. + if _, ok := syncable[filepath.ToSlash(rel)]; !ok { + ignoredMatches++ + continue + } + regularMatches++ hash, err := hashFile(match) if err != nil { diags = diags.Append(diag.Diagnostic{ @@ -122,13 +169,21 @@ func resolveFileTrigger(b *bundle.Bundle, loc, pattern string) (map[string]strin } // A directory-only match would otherwise leave ResolvedFileTriggers empty // and silently disarm the trigger while config still sets on_file_change. - if regularMatches == 0 && sawNonRegular { + if regularMatches == 0 && sawNonRegular && ignoredMatches == 0 { diags = diags.Append(diag.Diagnostic{ Severity: diag.Error, Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: pattern %q matches no regular files", pattern), Locations: b.Config.GetLocations(loc), }) } + if len(out) == 0 && ignoredMatches > 0 { + out[filepath.ToSlash(pattern)] = missingFileHash + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Warning, + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: no files match %q", pattern), + Locations: b.Config.GetLocations(loc), + }) + } return out, diags } diff --git a/bundle/config/mutator/resolve_job_run_file_triggers_test.go b/bundle/config/mutator/resolve_job_run_file_triggers_test.go index 3ec663c89bd..344fed25657 100644 --- a/bundle/config/mutator/resolve_job_run_file_triggers_test.go +++ b/bundle/config/mutator/resolve_job_run_file_triggers_test.go @@ -12,6 +12,7 @@ import ( "github.com/databricks/cli/bundle/config/mutator" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/vfs" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -85,7 +86,10 @@ func TestResolveJobRunFileTriggers(t *testing.T) { patB := "subdir/*.py" b := &bundle.Bundle{ SyncRootPath: dir, + SyncRoot: vfs.MustNew(dir), + WorktreeRoot: vfs.MustNew(dir), Config: config.Root{ + Sync: config.Sync{Paths: []string{"."}}, Resources: config.Resources{ JobRuns: map[string]*resources.JobRun{ "my_run": { @@ -135,6 +139,38 @@ func TestResolveJobRunFileTriggers(t *testing.T) { assert.Empty(t, b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers) }) + t.Run("skips gitignored files", func(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, ".gitignore"), []byte("skip.txt\n"), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "keep.txt"), []byte("keep"), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "skip.txt"), []byte("skip"), 0o644)) + + b := bundleWithFileTrigger(dir, "*.txt") + diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) + require.False(t, diags.HasError()) + + hashes := b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers + require.Len(t, hashes, 1) + assert.Equal(t, contentHash("keep"), hashes["keep.txt"]) + assert.NotContains(t, hashes, "skip.txt") + }) + + t.Run("skips sync.exclude files", func(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "keep.txt"), []byte("keep"), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "skip.txt"), []byte("skip"), 0o644)) + + b := bundleWithFileTrigger(dir, "*.txt") + b.Config.Sync.Exclude = []string{"skip.txt"} + diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) + require.False(t, diags.HasError()) + + hashes := b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers + require.Len(t, hashes, 1) + assert.Equal(t, contentHash("keep"), hashes["keep.txt"]) + assert.NotContains(t, hashes, "skip.txt") + }) + t.Run("trims pattern whitespace", func(t *testing.T) { dir := t.TempDir() require.NoError(t, os.WriteFile(filepath.Join(dir, "seed.txt"), []byte("v1"), 0o644)) @@ -148,9 +184,13 @@ func TestResolveJobRunFileTriggers(t *testing.T) { } func bundleWithFileTrigger(syncRoot, pattern string) *bundle.Bundle { + root := vfs.MustNew(syncRoot) return &bundle.Bundle{ SyncRootPath: syncRoot, + SyncRoot: root, + WorktreeRoot: root, Config: config.Root{ + Sync: config.Sync{Paths: []string{"."}}, Resources: config.Resources{ JobRuns: map[string]*resources.JobRun{ "my_run": { From 8763d63ea87bd2c9a1d88d2d5cd6852dcf316266 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Wed, 19 Aug 2026 20:10:03 +0000 Subject: [PATCH 58/75] acc: force-add gitignored on_file_change fixture The glob test edits migrations/ignored.txt, but the test .gitignore excluded it from the commit so CI clones would miss the file. --- .../job_runs/on_file_change_glob/migrations/ignored.txt | 1 + 1 file changed, 1 insertion(+) create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/ignored.txt diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/ignored.txt b/acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/ignored.txt new file mode 100644 index 00000000000..ea10ec85c10 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/ignored.txt @@ -0,0 +1 @@ +ignored From bd2b3f8f0f07d49662f5e60eceb9fb5e0f0da4c4 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Thu, 20 Aug 2026 08:11:43 +0000 Subject: [PATCH 59/75] acc: drop READPLAN from the on_file_change glob test Plan serialization is already covered by on_file_change; the glob cases only assert the hash map. --- .../resources/job_runs/on_file_change_glob/out.test.toml | 1 - .../bundle/resources/job_runs/on_file_change_glob/output.txt | 2 +- .../bundle/resources/job_runs/on_file_change_glob/script | 5 +---- .../bundle/resources/job_runs/on_file_change_glob/test.toml | 4 ---- 4 files changed, 2 insertions(+), 10 deletions(-) diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/out.test.toml b/acceptance/bundle/resources/job_runs/on_file_change_glob/out.test.toml index 57b0f616850..0938e678987 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/out.test.toml +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/out.test.toml @@ -1,3 +1,2 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt index d424482b961..bd7d08557f5 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt @@ -61,7 +61,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged } } -=== bundle deploy +>>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-glob/default/files... Output from job_runs.my_run: id=[MY_RUN_ID_2]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_2]?o=[NUMID] Output from job_runs.my_run: id=[MY_RUN_ID_2]: SUCCESS diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/script b/acceptance/bundle/resources/job_runs/on_file_change_glob/script index 983cb9ac401..23390d23a55 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/script +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/script @@ -4,7 +4,6 @@ cleanup() { } trap cleanup EXIT -# Plan once, print JSON changes, leave tmp.plan.json for readplanarg. changes() { $CLI bundle plan -o json > tmp.plan.json trace jq '.plan["resources.job_runs.my_run"].changes' tmp.plan.json @@ -19,9 +18,7 @@ trace print_requests.py //jobs/run-now title "editing one matched file re-fires" update_file.py migrations/001.txt "one" "one-edited" changes -# The deploy is not traced: readplanarg makes the command line differ per variant. -title "bundle deploy\n" -$CLI bundle deploy $(readplanarg tmp.plan.json) +trace $CLI bundle deploy read_id.py my_run > /dev/null trace print_requests.py --keep //jobs/runs/delete trace print_requests.py //jobs/run-now diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml b/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml index 3516572564d..d372fd54eae 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml @@ -1,7 +1,3 @@ -# Deploy both by re-planning and from a plan saved on disk, so the hashes -# computed during planning survive plan serialization. -EnvMatrix.READPLAN = ["", "1"] - Ignore = ["tmp.plan.json", "tmp.002.txt"] # See on_file_change/test.toml: mask content hashes before the parent's From 8595e98af03375825ff64518c56fd7949d51e193 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Thu, 20 Aug 2026 12:34:44 +0000 Subject: [PATCH 60/75] acc: restore READPLAN on the on_file_change glob test Cover deploying glob hashes from a saved plan, not only from a live re-plan. --- .../resources/job_runs/on_file_change_glob/out.test.toml | 1 + .../bundle/resources/job_runs/on_file_change_glob/output.txt | 2 +- .../bundle/resources/job_runs/on_file_change_glob/script | 5 ++++- .../bundle/resources/job_runs/on_file_change_glob/test.toml | 4 ++++ 4 files changed, 10 insertions(+), 2 deletions(-) diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/out.test.toml b/acceptance/bundle/resources/job_runs/on_file_change_glob/out.test.toml index 0938e678987..57b0f616850 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/out.test.toml +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/out.test.toml @@ -1,2 +1,3 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt index bd7d08557f5..d424482b961 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt @@ -61,7 +61,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged } } ->>> [CLI] bundle deploy +=== bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-glob/default/files... Output from job_runs.my_run: id=[MY_RUN_ID_2]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_2]?o=[NUMID] Output from job_runs.my_run: id=[MY_RUN_ID_2]: SUCCESS diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/script b/acceptance/bundle/resources/job_runs/on_file_change_glob/script index 23390d23a55..983cb9ac401 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/script +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/script @@ -4,6 +4,7 @@ cleanup() { } trap cleanup EXIT +# Plan once, print JSON changes, leave tmp.plan.json for readplanarg. changes() { $CLI bundle plan -o json > tmp.plan.json trace jq '.plan["resources.job_runs.my_run"].changes' tmp.plan.json @@ -18,7 +19,9 @@ trace print_requests.py //jobs/run-now title "editing one matched file re-fires" update_file.py migrations/001.txt "one" "one-edited" changes -trace $CLI bundle deploy +# The deploy is not traced: readplanarg makes the command line differ per variant. +title "bundle deploy\n" +$CLI bundle deploy $(readplanarg tmp.plan.json) read_id.py my_run > /dev/null trace print_requests.py --keep //jobs/runs/delete trace print_requests.py //jobs/run-now diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml b/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml index d372fd54eae..3516572564d 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml @@ -1,3 +1,7 @@ +# Deploy both by re-planning and from a plan saved on disk, so the hashes +# computed during planning survive plan serialization. +EnvMatrix.READPLAN = ["", "1"] + Ignore = ["tmp.plan.json", "tmp.002.txt"] # See on_file_change/test.toml: mask content hashes before the parent's From ab142ed1497f9ccc48437da4151e670f8b804d01 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Thu, 20 Aug 2026 14:08:42 +0000 Subject: [PATCH 61/75] job_runs: address on_file_change review comments Keep the gitignore fixture as dot_gitignore so it does not affect the repo while developing, drop unit tests that duplicate acceptance coverage, and allocate trigger state once when a trigger is armed. --- .../{.gitignore => dot_gitignore} | 0 .../job_runs/on_file_change_glob/script | 3 + .../job_runs/on_file_change_glob/test.toml | 2 +- .../resolve_job_run_file_triggers_test.go | 139 +----------------- bundle/direct/dresources/job_run.go | 23 +-- 5 files changed, 11 insertions(+), 156 deletions(-) rename acceptance/bundle/resources/job_runs/on_file_change_glob/{.gitignore => dot_gitignore} (100%) diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/.gitignore b/acceptance/bundle/resources/job_runs/on_file_change_glob/dot_gitignore similarity index 100% rename from acceptance/bundle/resources/job_runs/on_file_change_glob/.gitignore rename to acceptance/bundle/resources/job_runs/on_file_change_glob/dot_gitignore diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/script b/acceptance/bundle/resources/job_runs/on_file_change_glob/script index 983cb9ac401..dd737f242b5 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/script +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/script @@ -1,3 +1,6 @@ +# Fixture is named dot_gitignore so a real .gitignore does not affect git while developing. +mv dot_gitignore .gitignore + cleanup() { trace $CLI bundle destroy --auto-approve rm -f out.requests.txt diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml b/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml index 3516572564d..98c98631e5b 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml @@ -2,7 +2,7 @@ # computed during planning survive plan serialization. EnvMatrix.READPLAN = ["", "1"] -Ignore = ["tmp.plan.json", "tmp.002.txt"] +Ignore = ["tmp.plan.json", "tmp.002.txt", ".gitignore"] # See on_file_change/test.toml: mask content hashes before the parent's # `\d{8,}` rule can mangle them, keeping distinct hashes distinguishable. diff --git a/bundle/config/mutator/resolve_job_run_file_triggers_test.go b/bundle/config/mutator/resolve_job_run_file_triggers_test.go index 344fed25657..1cc54eecf4f 100644 --- a/bundle/config/mutator/resolve_job_run_file_triggers_test.go +++ b/bundle/config/mutator/resolve_job_run_file_triggers_test.go @@ -11,14 +11,13 @@ import ( "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/mutator" "github.com/databricks/cli/bundle/config/resources" - "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/vfs" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestResolveJobRunFileTriggers(t *testing.T) { - t.Run("matches files and fills hashes", func(t *testing.T) { + t.Run("hashes file contents with sha256", func(t *testing.T) { dir := t.TempDir() require.NoError(t, os.WriteFile(filepath.Join(dir, "a.txt"), []byte("hello"), 0o644)) require.NoError(t, os.WriteFile(filepath.Join(dir, "b.txt"), []byte("world"), 0o644)) @@ -35,142 +34,6 @@ func TestResolveJobRunFileTriggers(t *testing.T) { assert.Equal(t, contentHash("world"), hashes["b.txt"]) }) - t.Run("no matches warns and stores empty hash", func(t *testing.T) { - dir := t.TempDir() - pattern := "missing.txt" - b := bundleWithFileTrigger(dir, pattern) - - diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) - require.False(t, diags.HasError()) - require.Len(t, diags, 1) - assert.Equal(t, diag.Warning, diags[0].Severity) - assert.Contains(t, diags[0].Summary, `no files match "missing.txt"`) - - hashes := b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers - require.Len(t, hashes, 1) - assert.Empty(t, hashes["missing.txt"]) - }) - - t.Run("no file triggers is a no-op", func(t *testing.T) { - dir := t.TempDir() - on := true - b := &bundle.Bundle{ - SyncRootPath: dir, - Config: config.Root{ - Resources: config.Resources{ - JobRuns: map[string]*resources.JobRun{ - "my_run": { - Lifecycle: &resources.JobRunLifecycle{ - Triggers: []resources.JobRunTrigger{ - {OnBundleDeploy: &on}, - }, - }, - }, - }, - }, - }, - } - - diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) - assert.Empty(t, diags) - assert.Nil(t, b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers) - }) - - t.Run("multiple patterns merge into one map", func(t *testing.T) { - dir := t.TempDir() - require.NoError(t, os.WriteFile(filepath.Join(dir, "a.txt"), []byte("aaa"), 0o644)) - require.NoError(t, os.MkdirAll(filepath.Join(dir, "subdir"), 0o755)) - require.NoError(t, os.WriteFile(filepath.Join(dir, "subdir", "x.py"), []byte("bbb"), 0o644)) - - patA := "a.txt" - patB := "subdir/*.py" - b := &bundle.Bundle{ - SyncRootPath: dir, - SyncRoot: vfs.MustNew(dir), - WorktreeRoot: vfs.MustNew(dir), - Config: config.Root{ - Sync: config.Sync{Paths: []string{"."}}, - Resources: config.Resources{ - JobRuns: map[string]*resources.JobRun{ - "my_run": { - Lifecycle: &resources.JobRunLifecycle{ - Triggers: []resources.JobRunTrigger{ - {OnFileChange: &patA}, - {OnFileChange: &patB}, - }, - }, - }, - }, - }, - }, - } - - diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) - require.False(t, diags.HasError()) - - hashes := b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers - require.Len(t, hashes, 2) - assert.Equal(t, contentHash("aaa"), hashes["a.txt"]) - assert.Equal(t, contentHash("bbb"), hashes["subdir/x.py"]) - }) - - t.Run("pattern outside sync root is an error", func(t *testing.T) { - dir := t.TempDir() - pattern := "../outside.txt" - b := bundleWithFileTrigger(dir, pattern) - - diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) - require.True(t, diags.HasError()) - require.Len(t, diags, 1) - assert.Contains(t, diags[0].Summary, `not under the sync root`) - assert.Empty(t, b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers) - }) - - t.Run("directory-only match is an error", func(t *testing.T) { - dir := t.TempDir() - require.NoError(t, os.MkdirAll(filepath.Join(dir, "migrations"), 0o755)) - pattern := "migrations" - b := bundleWithFileTrigger(dir, pattern) - - diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) - require.True(t, diags.HasError()) - require.Len(t, diags, 1) - assert.Contains(t, diags[0].Summary, `matches no regular files`) - assert.Empty(t, b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers) - }) - - t.Run("skips gitignored files", func(t *testing.T) { - dir := t.TempDir() - require.NoError(t, os.WriteFile(filepath.Join(dir, ".gitignore"), []byte("skip.txt\n"), 0o644)) - require.NoError(t, os.WriteFile(filepath.Join(dir, "keep.txt"), []byte("keep"), 0o644)) - require.NoError(t, os.WriteFile(filepath.Join(dir, "skip.txt"), []byte("skip"), 0o644)) - - b := bundleWithFileTrigger(dir, "*.txt") - diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) - require.False(t, diags.HasError()) - - hashes := b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers - require.Len(t, hashes, 1) - assert.Equal(t, contentHash("keep"), hashes["keep.txt"]) - assert.NotContains(t, hashes, "skip.txt") - }) - - t.Run("skips sync.exclude files", func(t *testing.T) { - dir := t.TempDir() - require.NoError(t, os.WriteFile(filepath.Join(dir, "keep.txt"), []byte("keep"), 0o644)) - require.NoError(t, os.WriteFile(filepath.Join(dir, "skip.txt"), []byte("skip"), 0o644)) - - b := bundleWithFileTrigger(dir, "*.txt") - b.Config.Sync.Exclude = []string{"skip.txt"} - diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) - require.False(t, diags.HasError()) - - hashes := b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers - require.Len(t, hashes, 1) - assert.Equal(t, contentHash("keep"), hashes["keep.txt"]) - assert.NotContains(t, hashes, "skip.txt") - }) - t.Run("trims pattern whitespace", func(t *testing.T) { dir := t.TempDir() require.NoError(t, os.WriteFile(filepath.Join(dir, "seed.txt"), []byte("v1"), 0o644)) diff --git a/bundle/direct/dresources/job_run.go b/bundle/direct/dresources/job_run.go index 5b97cc9e22b..8117af9aa22 100644 --- a/bundle/direct/dresources/job_run.go +++ b/bundle/direct/dresources/job_run.go @@ -109,25 +109,14 @@ func (*ResourceJobRun) PrepareState(input *resources.JobRun) *JobRunState { ResultState: jobs.RunResultStateSuccess, Lifecycle: nil, } - var triggers *JobRunTriggersState - if input.HasOnBundleDeploy() { - triggers = &JobRunTriggersState{ - OnBundleDeploy: uuid.NewString(), - OnFileChange: nil, - } + if !input.HasOnBundleDeploy() && len(input.ResolvedFileTriggers) == 0 { + return state } - if len(input.ResolvedFileTriggers) > 0 { - if triggers == nil { - triggers = &JobRunTriggersState{ - OnBundleDeploy: "", - OnFileChange: nil, - } - } - triggers.OnFileChange = input.ResolvedFileTriggers - } - if triggers != nil { - state.Lifecycle = &JobRunLifecycleState{Triggers: triggers} + triggers := &JobRunTriggersState{OnFileChange: input.ResolvedFileTriggers} + if input.HasOnBundleDeploy() { + triggers.OnBundleDeploy = uuid.NewString() } + state.Lifecycle = &JobRunLifecycleState{Triggers: triggers} return state } From cc95d730a4c78ebd9f4466512eee563e61f739ca Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Thu, 20 Aug 2026 14:16:35 +0000 Subject: [PATCH 62/75] job_runs: list every trigger field to satisfy exhaustruct The previous simplification omitted OnBundleDeploy from the struct literal, which the linter rejects. --- bundle/direct/dresources/job_run.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/bundle/direct/dresources/job_run.go b/bundle/direct/dresources/job_run.go index 8117af9aa22..ac65e5db697 100644 --- a/bundle/direct/dresources/job_run.go +++ b/bundle/direct/dresources/job_run.go @@ -112,11 +112,16 @@ func (*ResourceJobRun) PrepareState(input *resources.JobRun) *JobRunState { if !input.HasOnBundleDeploy() && len(input.ResolvedFileTriggers) == 0 { return state } - triggers := &JobRunTriggersState{OnFileChange: input.ResolvedFileTriggers} + onBundleDeploy := "" if input.HasOnBundleDeploy() { - triggers.OnBundleDeploy = uuid.NewString() + onBundleDeploy = uuid.NewString() + } + state.Lifecycle = &JobRunLifecycleState{ + Triggers: &JobRunTriggersState{ + OnBundleDeploy: onBundleDeploy, + OnFileChange: input.ResolvedFileTriggers, + }, } - state.Lifecycle = &JobRunLifecycleState{Triggers: triggers} return state } From d05a78a89b90189e30a93bf9ccb28496325f76dc Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Fri, 21 Aug 2026 12:31:38 +0000 Subject: [PATCH 63/75] job_runs: keep empty nested lifecycle on state and remote Always persist the same lifecycle.triggers shape so trigger fingerprints compare at the same paths. Wrap on_file_change hashes in files so dropping the trigger updates instead of recreating. --- acceptance/bundle/refschema/out.fields.txt | 11 +- .../resources/job_runs/failed_run/output.txt | 10 ++ .../job_runs/interrupted_run/output.txt | 5 + .../job_runs/on_bundle_deploy/output.txt | 22 +-- .../job_runs/on_file_change/output.txt | 32 +--- .../job_runs/on_file_change_glob/output.txt | 144 ++++++------------ .../resources/job_runs/redeploy/output.txt | 10 ++ bundle/direct/dresources/job_run.go | 86 +++++++---- bundle/direct/dresources/job_run_test.go | 16 +- bundle/direct/dresources/resources.yml | 3 + bundle/direct/dresources/type_test.go | 4 - 11 files changed, 167 insertions(+), 176 deletions(-) diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index 871152e6a9c..3c681d4b199 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -865,15 +865,16 @@ resources.job_runs.*.jar_params[*] string ALL resources.job_runs.*.job_id int64 ALL resources.job_runs.*.job_parameters map[string]string ALL resources.job_runs.*.job_parameters.* string ALL -resources.job_runs.*.lifecycle *dresources.JobRunLifecycleState STATE +resources.job_runs.*.lifecycle *dresources.JobRunLifecycleState REMOTE STATE resources.job_runs.*.lifecycle *resources.JobRunLifecycle INPUT resources.job_runs.*.lifecycle resources.Lifecycle INPUT resources.job_runs.*.lifecycle.prevent_destroy bool INPUT -resources.job_runs.*.lifecycle.triggers *dresources.JobRunTriggersState STATE +resources.job_runs.*.lifecycle.triggers *dresources.JobRunTriggersState REMOTE STATE resources.job_runs.*.lifecycle.triggers []resources.JobRunTrigger INPUT -resources.job_runs.*.lifecycle.triggers.on_bundle_deploy string STATE -resources.job_runs.*.lifecycle.triggers.on_file_change map[string]string STATE -resources.job_runs.*.lifecycle.triggers.on_file_change.* string STATE +resources.job_runs.*.lifecycle.triggers.on_bundle_deploy string REMOTE STATE +resources.job_runs.*.lifecycle.triggers.on_file_change *dresources.JobRunFileTriggerState REMOTE STATE +resources.job_runs.*.lifecycle.triggers.on_file_change.files map[string]string REMOTE STATE +resources.job_runs.*.lifecycle.triggers.on_file_change.files.* string REMOTE STATE resources.job_runs.*.lifecycle.triggers[*] resources.JobRunTrigger INPUT resources.job_runs.*.lifecycle.triggers[*].on_bundle_deploy *bool INPUT resources.job_runs.*.lifecycle.triggers[*].on_file_change *string INPUT diff --git a/acceptance/bundle/resources/job_runs/failed_run/output.txt b/acceptance/bundle/resources/job_runs/failed_run/output.txt index f8b8dbf4398..0c6bddb0d9b 100644 --- a/acceptance/bundle/resources/job_runs/failed_run/output.txt +++ b/acceptance/bundle/resources/job_runs/failed_run/output.txt @@ -27,11 +27,21 @@ Files: 7 uploaded, 0 deleted "new_state": { "value": { "job_id": [MY_JOB_ID], + "lifecycle": { + "triggers": { + "on_file_change": {} + } + }, "result_state": "SUCCESS" } }, "remote_state": { "job_id": [MY_JOB_ID], + "lifecycle": { + "triggers": { + "on_file_change": {} + } + }, "result_state": "FAILED", "run_id": [MY_RUN_ID], "run_name": "test-job-[UNIQUE_NAME]", diff --git a/acceptance/bundle/resources/job_runs/interrupted_run/output.txt b/acceptance/bundle/resources/job_runs/interrupted_run/output.txt index 2e1d9cacd7a..da185c3704d 100644 --- a/acceptance/bundle/resources/job_runs/interrupted_run/output.txt +++ b/acceptance/bundle/resources/job_runs/interrupted_run/output.txt @@ -42,6 +42,11 @@ Exit code: 1 "action": "skip", "remote_state": { "job_id": [MY_JOB_ID], + "lifecycle": { + "triggers": { + "on_file_change": {} + } + }, "run_id": [MY_RUN_ID], "run_name": "my-job", "run_page_url": "[DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?o=[NUMID]", diff --git a/acceptance/bundle/resources/job_runs/on_bundle_deploy/output.txt b/acceptance/bundle/resources/job_runs/on_bundle_deploy/output.txt index 39123f3370f..6cd8d44a418 100644 --- a/acceptance/bundle/resources/job_runs/on_bundle_deploy/output.txt +++ b/acceptance/bundle/resources/job_runs/on_bundle_deploy/output.txt @@ -30,20 +30,6 @@ Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle": { - "action": "recreate", - "reason": "immutable", - "old": { - "triggers": { - "on_bundle_deploy": "[UUID]" - } - }, - "new": { - "triggers": { - "on_bundle_deploy": "[UUID]" - } - } - }, "lifecycle.triggers.on_bundle_deploy": { "action": "recreate", "reason": "immutable", @@ -108,14 +94,10 @@ Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle": { + "lifecycle.triggers.on_bundle_deploy": { "action": "skip", "reason": "trigger removed", - "old": { - "triggers": { - "on_bundle_deploy": "[UUID]" - } - } + "old": "[UUID]" }, "result_state": { "action": "skip", diff --git a/acceptance/bundle/resources/job_runs/on_file_change/output.txt b/acceptance/bundle/resources/job_runs/on_file_change/output.txt index d03307b90f2..9b662e1936d 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change/output.txt @@ -52,25 +52,17 @@ Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle": { + "lifecycle.triggers.on_file_change.files": { "action": "recreate", "reason": "immutable", "old": { - "triggers": { - "on_file_change": { - "seed.txt": "[FILE_HASH][0]" - } - } + "seed.txt": "[FILE_HASH][0]" }, "new": { - "triggers": { - "on_file_change": { - "seed.txt": "[FILE_HASH][1]" - } - } + "seed.txt": "[FILE_HASH][1]" } }, - "lifecycle.triggers.on_file_change['seed.txt']": { + "lifecycle.triggers.on_file_change.files['seed.txt']": { "action": "recreate", "reason": "immutable", "old": "[FILE_HASH][0]", @@ -125,25 +117,17 @@ Warning: lifecycle.triggers.on_file_change: no files match "seed.txt" in databricks.yml:18:29 { - "lifecycle": { + "lifecycle.triggers.on_file_change.files": { "action": "recreate", "reason": "immutable", "old": { - "triggers": { - "on_file_change": { - "seed.txt": "[FILE_HASH][1]" - } - } + "seed.txt": "[FILE_HASH][1]" }, "new": { - "triggers": { - "on_file_change": { - "seed.txt": "" - } - } + "seed.txt": "" } }, - "lifecycle.triggers.on_file_change['seed.txt']": { + "lifecycle.triggers.on_file_change.files['seed.txt']": { "action": "recreate", "reason": "immutable", "old": "[FILE_HASH][1]", diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt index d424482b961..5b4ef23178c 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt @@ -25,29 +25,21 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged === editing one matched file re-fires >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle": { + "lifecycle.triggers.on_file_change.files": { "action": "recreate", "reason": "immutable", "old": { - "triggers": { - "on_file_change": { - "migrations/001.txt": "[FILE_HASH][0]", - "migrations/002.txt": "[FILE_HASH][1]", - "mysubdir/a.txt": "[FILE_HASH][2]" - } - } + "migrations/001.txt": "[FILE_HASH][0]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][2]" }, "new": { - "triggers": { - "on_file_change": { - "migrations/001.txt": "[FILE_HASH][3]", - "migrations/002.txt": "[FILE_HASH][1]", - "mysubdir/a.txt": "[FILE_HASH][2]" - } - } + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][2]" } }, - "lifecycle.triggers.on_file_change['migrations/001.txt']": { + "lifecycle.triggers.on_file_change.files['migrations/001.txt']": { "action": "recreate", "reason": "immutable", "old": "[FILE_HASH][0]", @@ -91,29 +83,21 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged === editing a file in a subdirectory re-fires >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle": { + "lifecycle.triggers.on_file_change.files": { "action": "recreate", "reason": "immutable", "old": { - "triggers": { - "on_file_change": { - "migrations/001.txt": "[FILE_HASH][3]", - "migrations/002.txt": "[FILE_HASH][1]", - "mysubdir/a.txt": "[FILE_HASH][2]" - } - } + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][2]" }, "new": { - "triggers": { - "on_file_change": { - "migrations/001.txt": "[FILE_HASH][3]", - "migrations/002.txt": "[FILE_HASH][1]", - "mysubdir/a.txt": "[FILE_HASH][4]" - } - } + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][4]" } }, - "lifecycle.triggers.on_file_change['mysubdir/a.txt']": { + "lifecycle.triggers.on_file_change.files['mysubdir/a.txt']": { "action": "recreate", "reason": "immutable", "old": "[FILE_HASH][2]", @@ -130,30 +114,22 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged === a new matching file re-fires >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle": { + "lifecycle.triggers.on_file_change.files": { "action": "recreate", "reason": "immutable", "old": { - "triggers": { - "on_file_change": { - "migrations/001.txt": "[FILE_HASH][3]", - "migrations/002.txt": "[FILE_HASH][1]", - "mysubdir/a.txt": "[FILE_HASH][2]" - } - } + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][2]" }, "new": { - "triggers": { - "on_file_change": { - "migrations/001.txt": "[FILE_HASH][3]", - "migrations/002.txt": "[FILE_HASH][1]", - "migrations/003.txt": "[FILE_HASH][5]", - "mysubdir/a.txt": "[FILE_HASH][2]" - } - } + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "migrations/003.txt": "[FILE_HASH][5]", + "mysubdir/a.txt": "[FILE_HASH][2]" } }, - "lifecycle.triggers.on_file_change['migrations/003.txt']": { + "lifecycle.triggers.on_file_change.files['migrations/003.txt']": { "action": "recreate", "reason": "immutable", "new": "[FILE_HASH][5]" @@ -169,28 +145,20 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged === removing a matched file re-fires >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle": { + "lifecycle.triggers.on_file_change.files": { "action": "recreate", "reason": "immutable", "old": { - "triggers": { - "on_file_change": { - "migrations/001.txt": "[FILE_HASH][3]", - "migrations/002.txt": "[FILE_HASH][1]", - "mysubdir/a.txt": "[FILE_HASH][2]" - } - } + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][2]" }, "new": { - "triggers": { - "on_file_change": { - "migrations/001.txt": "[FILE_HASH][3]", - "mysubdir/a.txt": "[FILE_HASH][2]" - } - } + "migrations/001.txt": "[FILE_HASH][3]", + "mysubdir/a.txt": "[FILE_HASH][2]" } }, - "lifecycle.triggers.on_file_change['migrations/002.txt']": { + "lifecycle.triggers.on_file_change.files['migrations/002.txt']": { "action": "recreate", "reason": "immutable", "old": "[FILE_HASH][1]" @@ -206,26 +174,18 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged === editing a file below the glob does not re-fire >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle": { + "lifecycle.triggers.on_file_change.files": { "action": "skip", - "reason": "missing_in_remote", + "reason": "not_returned_by_api", "old": { - "triggers": { - "on_file_change": { - "migrations/001.txt": "[FILE_HASH][3]", - "migrations/002.txt": "[FILE_HASH][1]", - "mysubdir/a.txt": "[FILE_HASH][2]" - } - } + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][2]" }, "new": { - "triggers": { - "on_file_change": { - "migrations/001.txt": "[FILE_HASH][3]", - "migrations/002.txt": "[FILE_HASH][1]", - "mysubdir/a.txt": "[FILE_HASH][2]" - } - } + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][2]" } }, "result_state": { @@ -243,26 +203,18 @@ Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged === ** is not recursive: it matches the same files as * >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle": { + "lifecycle.triggers.on_file_change.files": { "action": "skip", - "reason": "missing_in_remote", + "reason": "not_returned_by_api", "old": { - "triggers": { - "on_file_change": { - "migrations/001.txt": "[FILE_HASH][3]", - "migrations/002.txt": "[FILE_HASH][1]", - "mysubdir/a.txt": "[FILE_HASH][2]" - } - } + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][2]" }, "new": { - "triggers": { - "on_file_change": { - "migrations/001.txt": "[FILE_HASH][3]", - "migrations/002.txt": "[FILE_HASH][1]", - "mysubdir/a.txt": "[FILE_HASH][2]" - } - } + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][2]" } }, "result_state": { diff --git a/acceptance/bundle/resources/job_runs/redeploy/output.txt b/acceptance/bundle/resources/job_runs/redeploy/output.txt index 06129f79373..60111c98a93 100644 --- a/acceptance/bundle/resources/job_runs/redeploy/output.txt +++ b/acceptance/bundle/resources/job_runs/redeploy/output.txt @@ -59,6 +59,11 @@ Resources: "job_parameters": { "env": "prod" }, + "lifecycle": { + "triggers": { + "on_file_change": {} + } + }, "result_state": "SUCCESS" } }, @@ -67,6 +72,11 @@ Resources: "job_parameters": { "env": "dev" }, + "lifecycle": { + "triggers": { + "on_file_change": {} + } + }, "result_state": "SUCCESS", "run_id": [MY_RUN_ID], "run_name": "my-job", diff --git a/bundle/direct/dresources/job_run.go b/bundle/direct/dresources/job_run.go index ac65e5db697..815091a721a 100644 --- a/bundle/direct/dresources/job_run.go +++ b/bundle/direct/dresources/job_run.go @@ -3,6 +3,7 @@ package dresources import ( "cmp" "context" + "encoding/json" "errors" "fmt" "slices" @@ -28,10 +29,12 @@ const jobRunTimeout = 24 * time.Hour // jobRunTriggerLocalPaths are local-only fingerprints: clearing one is skip, not recreate. var jobRunTriggerLocalPaths = []string{ - "lifecycle", - "lifecycle.triggers", "lifecycle.triggers.on_bundle_deploy", - "lifecycle.triggers.on_file_change", + "lifecycle.triggers.on_file_change.files", +} + +func isJobRunTriggerPath(path string) bool { + return slices.Contains(jobRunTriggerLocalPaths, path) } // JobRunTriggersState is the persisted fingerprint of lifecycle.triggers. @@ -39,12 +42,44 @@ type JobRunTriggersState struct { // Fresh UUID each plan while armed so Old!=New forces recreate. OnBundleDeploy string `json:"on_bundle_deploy,omitempty"` // Path → content hash from ResolveJobRunFileTriggers; change to recreate. - OnFileChange map[string]string `json:"on_file_change,omitempty"` + OnFileChange *JobRunFileTriggerState `json:"on_file_change"` +} + +// JobRunFileTriggerState is always present so remote and state share one shape. +// Files is nil when the trigger is off, so dropping the trigger diffs here and +// a file appear/disappear diffs inside the map. +type JobRunFileTriggerState struct { + Files map[string]string `json:"files,omitempty"` +} + +// UnmarshalJSON accepts the wrapped map and the older path-to-hash map. +func (s *JobRunFileTriggerState) UnmarshalJSON(b []byte) error { + var raw map[string]json.RawMessage + if err := json.Unmarshal(b, &raw); err != nil { + return err + } + if files, ok := raw["files"]; ok && len(files) > 0 && files[0] == '{' { + return json.Unmarshal(files, &s.Files) + } + if len(raw) == 0 { + s.Files = nil + return nil + } + return json.Unmarshal(b, &s.Files) } -// JobRunLifecycleState holds local-only lifecycle fields persisted in state. +// JobRunLifecycleState is the local-only trigger fingerprint, also present (empty) on remote. type JobRunLifecycleState struct { - Triggers *JobRunTriggersState `json:"triggers,omitempty"` + Triggers *JobRunTriggersState `json:"triggers"` +} + +func newJobRunLifecycleState() *JobRunLifecycleState { + return &JobRunLifecycleState{ + Triggers: &JobRunTriggersState{ + OnBundleDeploy: "", + OnFileChange: &JobRunFileTriggerState{Files: nil}, + }, + } } // JobRunState is the RunNow request plus the outcome required for planning. @@ -54,12 +89,13 @@ type JobRunState struct { // Always SUCCESS during planning and cleared before persistence. ResultState jobs.RunResultState `json:"result_state,omitempty"` - // Local-only; listed in knownMissingInRemoteType. Nested under lifecycle to - // mirror config and avoid colliding with a future Jobs API field. - Lifecycle *JobRunLifecycleState `json:"lifecycle,omitempty"` + // Local-only. Nested under lifecycle to mirror config and avoid colliding + // with a future Jobs API field. + Lifecycle *JobRunLifecycleState `json:"lifecycle"` } func (s *JobRunState) UnmarshalJSON(b []byte) error { + s.Lifecycle = newJobRunLifecycleState() return marshal.Unmarshal(b, s) } @@ -76,6 +112,9 @@ type JobRunRemote struct { // compare the two. See "RemapState is a dumb copy" in README.md. ResultState jobs.RunResultState `json:"result_state,omitempty"` + // Always the empty fingerprint: GetRun does not return triggers. + Lifecycle *JobRunLifecycleState `json:"lifecycle"` + RunId int64 `json:"run_id,omitempty"` RunName string `json:"run_name,omitempty"` State *jobs.RunState `json:"state,omitempty"` @@ -86,6 +125,7 @@ type JobRunRemote struct { // Custom marshaler needed because embedded RunNow's MarshalJSON would otherwise // take over and drop the additional fields. func (s *JobRunRemote) UnmarshalJSON(b []byte) error { + s.Lifecycle = newJobRunLifecycleState() return marshal.Unmarshal(b, s) } @@ -107,20 +147,13 @@ func (*ResourceJobRun) PrepareState(input *resources.JobRun) *JobRunState { state := &JobRunState{ RunNow: input.RunNow, ResultState: jobs.RunResultStateSuccess, - Lifecycle: nil, - } - if !input.HasOnBundleDeploy() && len(input.ResolvedFileTriggers) == 0 { - return state + Lifecycle: newJobRunLifecycleState(), } - onBundleDeploy := "" if input.HasOnBundleDeploy() { - onBundleDeploy = uuid.NewString() + state.Lifecycle.Triggers.OnBundleDeploy = uuid.NewString() } - state.Lifecycle = &JobRunLifecycleState{ - Triggers: &JobRunTriggersState{ - OnBundleDeploy: onBundleDeploy, - OnFileChange: input.ResolvedFileTriggers, - }, + if len(input.ResolvedFileTriggers) > 0 { + state.Lifecycle.Triggers.OnFileChange.Files = input.ResolvedFileTriggers } return state } @@ -161,6 +194,7 @@ func makeJobRunRemote(run *jobs.Run) *JobRunRemote { ForceSendFields: nil, }, ResultState: run.State.ResultState, + Lifecycle: newJobRunLifecycleState(), RunId: run.RunId, RunName: run.RunName, // Rebuilt, not copied: the SDK records explicitly-sent fields in @@ -201,8 +235,7 @@ func (*ResourceJobRun) RemapState(remote *JobRunRemote) *JobRunState { return &JobRunState{ RunNow: remote.RunNow, ResultState: remote.ResultState, - // Local-only lifecycle fingerprints stay unset on the remapped remote. - Lifecycle: nil, + Lifecycle: remote.Lifecycle, } } @@ -389,10 +422,11 @@ func reportRunLine(ctx context.Context, runID int64, msg string) { // result_state either, so the lifecycle state is what tells the two apart. // Clearing a trigger skips its local-only fingerprint without re-firing the run. func (*ResourceJobRun) OverrideChangeDesc(_ context.Context, path *structpath.PathNode, change *ChangeDesc, remote *JobRunRemote) error { - if slices.Contains(jobRunTriggerLocalPaths, path.String()) { - // A cleared trigger sets New empty; structdiff may report it at lifecycle, - // lifecycle.triggers, or the leaf. - if change.New == nil || change.New == "" { + pathString := path.String() + if isJobRunTriggerPath(pathString) { + removed := pathString == "lifecycle.triggers.on_bundle_deploy" && (change.New == nil || change.New == "") + removed = removed || pathString == "lifecycle.triggers.on_file_change.files" && change.New == nil + if removed { change.Action = deployplan.Skip change.Reason = "trigger removed" } diff --git a/bundle/direct/dresources/job_run_test.go b/bundle/direct/dresources/job_run_test.go index 0ffc1c48040..66f61646116 100644 --- a/bundle/direct/dresources/job_run_test.go +++ b/bundle/direct/dresources/job_run_test.go @@ -244,6 +244,18 @@ func TestJobRunWaitAbandonedLinksTheRun(t *testing.T) { require.ErrorContains(t, err, testRunPageLink) } +// State written before lifecycle existed has no such key, and must still load. +func TestJobRunStateUnmarshalWithoutLifecycle(t *testing.T) { + var state JobRunState + + require.NoError(t, json.Unmarshal([]byte(`{}`), &state)) + + require.NotNil(t, state.Lifecycle) + require.NotNil(t, state.Lifecycle.Triggers) + require.NotNil(t, state.Lifecycle.Triggers.OnFileChange) + assert.Nil(t, state.Lifecycle.Triggers.OnFileChange.Files) +} + // The planner diffs RemapState(remote) against PrepareState(config), so a run // that did not end in SUCCESS has to surface as a difference on result_state. func TestJobRunRemapStateCarriesTheOutcome(t *testing.T) { @@ -254,11 +266,13 @@ func TestJobRunRemapStateCarriesTheOutcome(t *testing.T) { "", } { t.Run(string(outcome), func(t *testing.T) { - remote := &JobRunRemote{RunId: 123, ResultState: outcome} + lifecycle := newJobRunLifecycleState() + remote := &JobRunRemote{RunId: 123, ResultState: outcome, Lifecycle: lifecycle} state := (&ResourceJobRun{}).RemapState(remote) assert.Equal(t, outcome, state.ResultState) + assert.Same(t, lifecycle, state.Lifecycle) }) } } diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index b0a9ed6b365..67388b30dfe 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -175,6 +175,9 @@ resources: reason: not_returned_by_api - field: queue reason: not_returned_by_api + # Local-only trigger fingerprints; GetRun does not return them. + - field: lifecycle + reason: not_returned_by_api # A run is immutable and fire-once, so any change recreates it. Omitting # `field` matches every field (root; see TestFieldRuleOmittedIsRoot). # `field: ""` would instead match nothing. The one exception is a run that is diff --git a/bundle/direct/dresources/type_test.go b/bundle/direct/dresources/type_test.go index 2d5516d59c7..ca84850c33d 100644 --- a/bundle/direct/dresources/type_test.go +++ b/bundle/direct/dresources/type_test.go @@ -56,10 +56,6 @@ var knownMissingInRemoteType = map[string][]string{ "vector_search_endpoints": { "usage_policy_id", }, - "job_runs": { - // Local-only trigger fingerprints under lifecycle. - "lifecycle", - }, } // commonMissingInStateType lists fields that are commonly missing across all resource types. From 55a78ca7e84f5807d52dfb945e880e814dae26fc Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Fri, 21 Aug 2026 13:02:42 +0000 Subject: [PATCH 64/75] job_runs: keep the lifecycle fingerprint out of the remote type Nest lifecycle by value so structdiff descends to the leaf that changed instead of reporting the whole subtree, and let RemapState supply the empty shape. GetRun never returns the fingerprints, so the remote type no longer advertises a lifecycle it cannot fill. --- acceptance/bundle/refschema/out.fields.txt | 12 ++--- .../resources/job_runs/failed_run/output.txt | 5 --- .../job_runs/interrupted_run/output.txt | 5 --- .../job_runs/on_file_change_glob/output.txt | 4 +- .../resources/job_runs/redeploy/output.txt | 5 --- bundle/direct/dresources/job_run.go | 45 +++++++++---------- bundle/direct/dresources/job_run_test.go | 8 +--- bundle/direct/dresources/resources.yml | 3 -- bundle/direct/dresources/type_test.go | 5 +++ 9 files changed, 37 insertions(+), 55 deletions(-) diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index 3c681d4b199..cc754a56d00 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -865,16 +865,16 @@ resources.job_runs.*.jar_params[*] string ALL resources.job_runs.*.job_id int64 ALL resources.job_runs.*.job_parameters map[string]string ALL resources.job_runs.*.job_parameters.* string ALL -resources.job_runs.*.lifecycle *dresources.JobRunLifecycleState REMOTE STATE resources.job_runs.*.lifecycle *resources.JobRunLifecycle INPUT +resources.job_runs.*.lifecycle dresources.JobRunLifecycleState STATE resources.job_runs.*.lifecycle resources.Lifecycle INPUT resources.job_runs.*.lifecycle.prevent_destroy bool INPUT -resources.job_runs.*.lifecycle.triggers *dresources.JobRunTriggersState REMOTE STATE resources.job_runs.*.lifecycle.triggers []resources.JobRunTrigger INPUT -resources.job_runs.*.lifecycle.triggers.on_bundle_deploy string REMOTE STATE -resources.job_runs.*.lifecycle.triggers.on_file_change *dresources.JobRunFileTriggerState REMOTE STATE -resources.job_runs.*.lifecycle.triggers.on_file_change.files map[string]string REMOTE STATE -resources.job_runs.*.lifecycle.triggers.on_file_change.files.* string REMOTE STATE +resources.job_runs.*.lifecycle.triggers dresources.JobRunTriggersState STATE +resources.job_runs.*.lifecycle.triggers.on_bundle_deploy string STATE +resources.job_runs.*.lifecycle.triggers.on_file_change dresources.JobRunFileTriggerState STATE +resources.job_runs.*.lifecycle.triggers.on_file_change.files map[string]string STATE +resources.job_runs.*.lifecycle.triggers.on_file_change.files.* string STATE resources.job_runs.*.lifecycle.triggers[*] resources.JobRunTrigger INPUT resources.job_runs.*.lifecycle.triggers[*].on_bundle_deploy *bool INPUT resources.job_runs.*.lifecycle.triggers[*].on_file_change *string INPUT diff --git a/acceptance/bundle/resources/job_runs/failed_run/output.txt b/acceptance/bundle/resources/job_runs/failed_run/output.txt index 0c6bddb0d9b..37f1a831681 100644 --- a/acceptance/bundle/resources/job_runs/failed_run/output.txt +++ b/acceptance/bundle/resources/job_runs/failed_run/output.txt @@ -37,11 +37,6 @@ Files: 7 uploaded, 0 deleted }, "remote_state": { "job_id": [MY_JOB_ID], - "lifecycle": { - "triggers": { - "on_file_change": {} - } - }, "result_state": "FAILED", "run_id": [MY_RUN_ID], "run_name": "test-job-[UNIQUE_NAME]", diff --git a/acceptance/bundle/resources/job_runs/interrupted_run/output.txt b/acceptance/bundle/resources/job_runs/interrupted_run/output.txt index da185c3704d..2e1d9cacd7a 100644 --- a/acceptance/bundle/resources/job_runs/interrupted_run/output.txt +++ b/acceptance/bundle/resources/job_runs/interrupted_run/output.txt @@ -42,11 +42,6 @@ Exit code: 1 "action": "skip", "remote_state": { "job_id": [MY_JOB_ID], - "lifecycle": { - "triggers": { - "on_file_change": {} - } - }, "run_id": [MY_RUN_ID], "run_name": "my-job", "run_page_url": "[DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?o=[NUMID]", diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt index 5b4ef23178c..69a7e5ece94 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt @@ -176,7 +176,7 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged { "lifecycle.triggers.on_file_change.files": { "action": "skip", - "reason": "not_returned_by_api", + "reason": "missing_in_remote", "old": { "migrations/001.txt": "[FILE_HASH][3]", "migrations/002.txt": "[FILE_HASH][1]", @@ -205,7 +205,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged { "lifecycle.triggers.on_file_change.files": { "action": "skip", - "reason": "not_returned_by_api", + "reason": "missing_in_remote", "old": { "migrations/001.txt": "[FILE_HASH][3]", "migrations/002.txt": "[FILE_HASH][1]", diff --git a/acceptance/bundle/resources/job_runs/redeploy/output.txt b/acceptance/bundle/resources/job_runs/redeploy/output.txt index 60111c98a93..a80685733ad 100644 --- a/acceptance/bundle/resources/job_runs/redeploy/output.txt +++ b/acceptance/bundle/resources/job_runs/redeploy/output.txt @@ -72,11 +72,6 @@ Resources: "job_parameters": { "env": "dev" }, - "lifecycle": { - "triggers": { - "on_file_change": {} - } - }, "result_state": "SUCCESS", "run_id": [MY_RUN_ID], "run_name": "my-job", diff --git a/bundle/direct/dresources/job_run.go b/bundle/direct/dresources/job_run.go index 815091a721a..9a937787e02 100644 --- a/bundle/direct/dresources/job_run.go +++ b/bundle/direct/dresources/job_run.go @@ -41,13 +41,12 @@ func isJobRunTriggerPath(path string) bool { type JobRunTriggersState struct { // Fresh UUID each plan while armed so Old!=New forces recreate. OnBundleDeploy string `json:"on_bundle_deploy,omitempty"` - // Path → content hash from ResolveJobRunFileTriggers; change to recreate. - OnFileChange *JobRunFileTriggerState `json:"on_file_change"` + // Content hashes from ResolveJobRunFileTriggers; any change recreates. + OnFileChange JobRunFileTriggerState `json:"on_file_change"` } -// JobRunFileTriggerState is always present so remote and state share one shape. -// Files is nil when the trigger is off, so dropping the trigger diffs here and -// a file appear/disappear diffs inside the map. +// JobRunFileTriggerState wraps the hashes so turning the trigger off (Files nil) +// diffs at one path, distinct from the per-file entries a changed file produces. type JobRunFileTriggerState struct { Files map[string]string `json:"files,omitempty"` } @@ -68,16 +67,21 @@ func (s *JobRunFileTriggerState) UnmarshalJSON(b []byte) error { return json.Unmarshal(b, &s.Files) } -// JobRunLifecycleState is the local-only trigger fingerprint, also present (empty) on remote. +// JobRunLifecycleState is the local-only trigger fingerprint. Nested by value, +// not by pointer: structdiff cannot descend into a nil pointer and would report +// the whole subtree at "lifecycle" instead of the leaf that actually changed. type JobRunLifecycleState struct { - Triggers *JobRunTriggersState `json:"triggers"` + Triggers JobRunTriggersState `json:"triggers"` } -func newJobRunLifecycleState() *JobRunLifecycleState { - return &JobRunLifecycleState{ - Triggers: &JobRunTriggersState{ +// Zero value spelled out field by field, as exhaustruct requires. +func emptyJobRunLifecycleState() JobRunLifecycleState { + return JobRunLifecycleState{ + Triggers: JobRunTriggersState{ OnBundleDeploy: "", - OnFileChange: &JobRunFileTriggerState{Files: nil}, + OnFileChange: JobRunFileTriggerState{ + Files: nil, + }, }, } } @@ -91,11 +95,10 @@ type JobRunState struct { // Local-only. Nested under lifecycle to mirror config and avoid colliding // with a future Jobs API field. - Lifecycle *JobRunLifecycleState `json:"lifecycle"` + Lifecycle JobRunLifecycleState `json:"lifecycle"` } func (s *JobRunState) UnmarshalJSON(b []byte) error { - s.Lifecycle = newJobRunLifecycleState() return marshal.Unmarshal(b, s) } @@ -103,8 +106,8 @@ func (s JobRunState) MarshalJSON() ([]byte, error) { return marshal.Marshal(s) } -// JobRunRemote embeds RunNow so every StateType path is a valid RemoteType path -// (see TestRemoteSuperset), plus the run's output-only fields for a faithful view. +// JobRunRemote is the RunNow request plus the run's output-only fields. It has no +// lifecycle: GetRun never returns the fingerprints (see knownMissingInRemoteType). type JobRunRemote struct { jobs.RunNow @@ -112,9 +115,6 @@ type JobRunRemote struct { // compare the two. See "RemapState is a dumb copy" in README.md. ResultState jobs.RunResultState `json:"result_state,omitempty"` - // Always the empty fingerprint: GetRun does not return triggers. - Lifecycle *JobRunLifecycleState `json:"lifecycle"` - RunId int64 `json:"run_id,omitempty"` RunName string `json:"run_name,omitempty"` State *jobs.RunState `json:"state,omitempty"` @@ -125,7 +125,6 @@ type JobRunRemote struct { // Custom marshaler needed because embedded RunNow's MarshalJSON would otherwise // take over and drop the additional fields. func (s *JobRunRemote) UnmarshalJSON(b []byte) error { - s.Lifecycle = newJobRunLifecycleState() return marshal.Unmarshal(b, s) } @@ -147,7 +146,7 @@ func (*ResourceJobRun) PrepareState(input *resources.JobRun) *JobRunState { state := &JobRunState{ RunNow: input.RunNow, ResultState: jobs.RunResultStateSuccess, - Lifecycle: newJobRunLifecycleState(), + Lifecycle: emptyJobRunLifecycleState(), } if input.HasOnBundleDeploy() { state.Lifecycle.Triggers.OnBundleDeploy = uuid.NewString() @@ -194,7 +193,6 @@ func makeJobRunRemote(run *jobs.Run) *JobRunRemote { ForceSendFields: nil, }, ResultState: run.State.ResultState, - Lifecycle: newJobRunLifecycleState(), RunId: run.RunId, RunName: run.RunName, // Rebuilt, not copied: the SDK records explicitly-sent fields in @@ -230,12 +228,13 @@ func (r *ResourceJobRun) DoRead(ctx context.Context, id string) (*JobRunRemote, } // RemapState extracts the fields used for diffing: the RunNow request and the -// outcome the run reached. +// outcome the run reached. Lifecycle has no remote counterpart, so it stays empty +// and the planner skips it as missing_in_remote. func (*ResourceJobRun) RemapState(remote *JobRunRemote) *JobRunState { return &JobRunState{ RunNow: remote.RunNow, ResultState: remote.ResultState, - Lifecycle: remote.Lifecycle, + Lifecycle: emptyJobRunLifecycleState(), } } diff --git a/bundle/direct/dresources/job_run_test.go b/bundle/direct/dresources/job_run_test.go index 66f61646116..34174454ab7 100644 --- a/bundle/direct/dresources/job_run_test.go +++ b/bundle/direct/dresources/job_run_test.go @@ -250,9 +250,6 @@ func TestJobRunStateUnmarshalWithoutLifecycle(t *testing.T) { require.NoError(t, json.Unmarshal([]byte(`{}`), &state)) - require.NotNil(t, state.Lifecycle) - require.NotNil(t, state.Lifecycle.Triggers) - require.NotNil(t, state.Lifecycle.Triggers.OnFileChange) assert.Nil(t, state.Lifecycle.Triggers.OnFileChange.Files) } @@ -266,13 +263,12 @@ func TestJobRunRemapStateCarriesTheOutcome(t *testing.T) { "", } { t.Run(string(outcome), func(t *testing.T) { - lifecycle := newJobRunLifecycleState() - remote := &JobRunRemote{RunId: 123, ResultState: outcome, Lifecycle: lifecycle} + remote := &JobRunRemote{RunId: 123, ResultState: outcome} state := (&ResourceJobRun{}).RemapState(remote) assert.Equal(t, outcome, state.ResultState) - assert.Same(t, lifecycle, state.Lifecycle) + assert.Equal(t, emptyJobRunLifecycleState(), state.Lifecycle) }) } } diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index 67388b30dfe..b0a9ed6b365 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -175,9 +175,6 @@ resources: reason: not_returned_by_api - field: queue reason: not_returned_by_api - # Local-only trigger fingerprints; GetRun does not return them. - - field: lifecycle - reason: not_returned_by_api # A run is immutable and fire-once, so any change recreates it. Omitting # `field` matches every field (root; see TestFieldRuleOmittedIsRoot). # `field: ""` would instead match nothing. The one exception is a run that is diff --git a/bundle/direct/dresources/type_test.go b/bundle/direct/dresources/type_test.go index ca84850c33d..7ad9872c373 100644 --- a/bundle/direct/dresources/type_test.go +++ b/bundle/direct/dresources/type_test.go @@ -19,6 +19,11 @@ var knownMissingInRemoteType = map[string][]string{ "external_locations": { "skip_validation", }, + "job_runs": { + // Deliberate omission: the trigger fingerprints are client-side only, so + // GetRun has nothing to report them under. + "lifecycle", + }, "model_serving_endpoints": { "rate_limits", }, From 196a6c6f544c8ebc872474479ec41bbbd8f6ae73 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Mon, 24 Aug 2026 07:55:05 +0000 Subject: [PATCH 65/75] job_runs: store on_file_change hashes on the trigger itself Drop the files wrapper so plan diffs land on lifecycle.triggers.on_file_change, matching the config path without an extra state-only level. --- acceptance/bundle/refschema/out.fields.txt | 5 ++- .../job_runs/on_file_change/output.txt | 8 ++--- .../job_runs/on_file_change_glob/output.txt | 20 +++++------ bundle/direct/dresources/job_run.go | 35 +++---------------- bundle/direct/dresources/job_run_test.go | 2 +- 5 files changed, 22 insertions(+), 48 deletions(-) diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index cc754a56d00..06d260541a5 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -872,9 +872,8 @@ resources.job_runs.*.lifecycle.prevent_destroy bool INPUT resources.job_runs.*.lifecycle.triggers []resources.JobRunTrigger INPUT resources.job_runs.*.lifecycle.triggers dresources.JobRunTriggersState STATE resources.job_runs.*.lifecycle.triggers.on_bundle_deploy string STATE -resources.job_runs.*.lifecycle.triggers.on_file_change dresources.JobRunFileTriggerState STATE -resources.job_runs.*.lifecycle.triggers.on_file_change.files map[string]string STATE -resources.job_runs.*.lifecycle.triggers.on_file_change.files.* string STATE +resources.job_runs.*.lifecycle.triggers.on_file_change map[string]string STATE +resources.job_runs.*.lifecycle.triggers.on_file_change.* string STATE resources.job_runs.*.lifecycle.triggers[*] resources.JobRunTrigger INPUT resources.job_runs.*.lifecycle.triggers[*].on_bundle_deploy *bool INPUT resources.job_runs.*.lifecycle.triggers[*].on_file_change *string INPUT diff --git a/acceptance/bundle/resources/job_runs/on_file_change/output.txt b/acceptance/bundle/resources/job_runs/on_file_change/output.txt index 9b662e1936d..28deeee38a4 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change/output.txt @@ -52,7 +52,7 @@ Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle.triggers.on_file_change.files": { + "lifecycle.triggers.on_file_change": { "action": "recreate", "reason": "immutable", "old": { @@ -62,7 +62,7 @@ Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged "seed.txt": "[FILE_HASH][1]" } }, - "lifecycle.triggers.on_file_change.files['seed.txt']": { + "lifecycle.triggers.on_file_change['seed.txt']": { "action": "recreate", "reason": "immutable", "old": "[FILE_HASH][0]", @@ -117,7 +117,7 @@ Warning: lifecycle.triggers.on_file_change: no files match "seed.txt" in databricks.yml:18:29 { - "lifecycle.triggers.on_file_change.files": { + "lifecycle.triggers.on_file_change": { "action": "recreate", "reason": "immutable", "old": { @@ -127,7 +127,7 @@ Warning: lifecycle.triggers.on_file_change: no files match "seed.txt" "seed.txt": "" } }, - "lifecycle.triggers.on_file_change.files['seed.txt']": { + "lifecycle.triggers.on_file_change['seed.txt']": { "action": "recreate", "reason": "immutable", "old": "[FILE_HASH][1]", diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt index 69a7e5ece94..a04e3710fb4 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt @@ -25,7 +25,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged === editing one matched file re-fires >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle.triggers.on_file_change.files": { + "lifecycle.triggers.on_file_change": { "action": "recreate", "reason": "immutable", "old": { @@ -39,7 +39,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "mysubdir/a.txt": "[FILE_HASH][2]" } }, - "lifecycle.triggers.on_file_change.files['migrations/001.txt']": { + "lifecycle.triggers.on_file_change['migrations/001.txt']": { "action": "recreate", "reason": "immutable", "old": "[FILE_HASH][0]", @@ -83,7 +83,7 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged === editing a file in a subdirectory re-fires >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle.triggers.on_file_change.files": { + "lifecycle.triggers.on_file_change": { "action": "recreate", "reason": "immutable", "old": { @@ -97,7 +97,7 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged "mysubdir/a.txt": "[FILE_HASH][4]" } }, - "lifecycle.triggers.on_file_change.files['mysubdir/a.txt']": { + "lifecycle.triggers.on_file_change['mysubdir/a.txt']": { "action": "recreate", "reason": "immutable", "old": "[FILE_HASH][2]", @@ -114,7 +114,7 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged === a new matching file re-fires >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle.triggers.on_file_change.files": { + "lifecycle.triggers.on_file_change": { "action": "recreate", "reason": "immutable", "old": { @@ -129,7 +129,7 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged "mysubdir/a.txt": "[FILE_HASH][2]" } }, - "lifecycle.triggers.on_file_change.files['migrations/003.txt']": { + "lifecycle.triggers.on_file_change['migrations/003.txt']": { "action": "recreate", "reason": "immutable", "new": "[FILE_HASH][5]" @@ -145,7 +145,7 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged === removing a matched file re-fires >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle.triggers.on_file_change.files": { + "lifecycle.triggers.on_file_change": { "action": "recreate", "reason": "immutable", "old": { @@ -158,7 +158,7 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged "mysubdir/a.txt": "[FILE_HASH][2]" } }, - "lifecycle.triggers.on_file_change.files['migrations/002.txt']": { + "lifecycle.triggers.on_file_change['migrations/002.txt']": { "action": "recreate", "reason": "immutable", "old": "[FILE_HASH][1]" @@ -174,7 +174,7 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged === editing a file below the glob does not re-fire >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle.triggers.on_file_change.files": { + "lifecycle.triggers.on_file_change": { "action": "skip", "reason": "missing_in_remote", "old": { @@ -203,7 +203,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged === ** is not recursive: it matches the same files as * >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle.triggers.on_file_change.files": { + "lifecycle.triggers.on_file_change": { "action": "skip", "reason": "missing_in_remote", "old": { diff --git a/bundle/direct/dresources/job_run.go b/bundle/direct/dresources/job_run.go index 9a937787e02..9d1a272a7a3 100644 --- a/bundle/direct/dresources/job_run.go +++ b/bundle/direct/dresources/job_run.go @@ -3,7 +3,6 @@ package dresources import ( "cmp" "context" - "encoding/json" "errors" "fmt" "slices" @@ -30,7 +29,7 @@ const jobRunTimeout = 24 * time.Hour // jobRunTriggerLocalPaths are local-only fingerprints: clearing one is skip, not recreate. var jobRunTriggerLocalPaths = []string{ "lifecycle.triggers.on_bundle_deploy", - "lifecycle.triggers.on_file_change.files", + "lifecycle.triggers.on_file_change", } func isJobRunTriggerPath(path string) bool { @@ -42,29 +41,7 @@ type JobRunTriggersState struct { // Fresh UUID each plan while armed so Old!=New forces recreate. OnBundleDeploy string `json:"on_bundle_deploy,omitempty"` // Content hashes from ResolveJobRunFileTriggers; any change recreates. - OnFileChange JobRunFileTriggerState `json:"on_file_change"` -} - -// JobRunFileTriggerState wraps the hashes so turning the trigger off (Files nil) -// diffs at one path, distinct from the per-file entries a changed file produces. -type JobRunFileTriggerState struct { - Files map[string]string `json:"files,omitempty"` -} - -// UnmarshalJSON accepts the wrapped map and the older path-to-hash map. -func (s *JobRunFileTriggerState) UnmarshalJSON(b []byte) error { - var raw map[string]json.RawMessage - if err := json.Unmarshal(b, &raw); err != nil { - return err - } - if files, ok := raw["files"]; ok && len(files) > 0 && files[0] == '{' { - return json.Unmarshal(files, &s.Files) - } - if len(raw) == 0 { - s.Files = nil - return nil - } - return json.Unmarshal(b, &s.Files) + OnFileChange map[string]string `json:"on_file_change,omitempty"` } // JobRunLifecycleState is the local-only trigger fingerprint. Nested by value, @@ -79,9 +56,7 @@ func emptyJobRunLifecycleState() JobRunLifecycleState { return JobRunLifecycleState{ Triggers: JobRunTriggersState{ OnBundleDeploy: "", - OnFileChange: JobRunFileTriggerState{ - Files: nil, - }, + OnFileChange: nil, }, } } @@ -152,7 +127,7 @@ func (*ResourceJobRun) PrepareState(input *resources.JobRun) *JobRunState { state.Lifecycle.Triggers.OnBundleDeploy = uuid.NewString() } if len(input.ResolvedFileTriggers) > 0 { - state.Lifecycle.Triggers.OnFileChange.Files = input.ResolvedFileTriggers + state.Lifecycle.Triggers.OnFileChange = input.ResolvedFileTriggers } return state } @@ -424,7 +399,7 @@ func (*ResourceJobRun) OverrideChangeDesc(_ context.Context, path *structpath.Pa pathString := path.String() if isJobRunTriggerPath(pathString) { removed := pathString == "lifecycle.triggers.on_bundle_deploy" && (change.New == nil || change.New == "") - removed = removed || pathString == "lifecycle.triggers.on_file_change.files" && change.New == nil + removed = removed || pathString == "lifecycle.triggers.on_file_change" && change.New == nil if removed { change.Action = deployplan.Skip change.Reason = "trigger removed" diff --git a/bundle/direct/dresources/job_run_test.go b/bundle/direct/dresources/job_run_test.go index 34174454ab7..804968f7707 100644 --- a/bundle/direct/dresources/job_run_test.go +++ b/bundle/direct/dresources/job_run_test.go @@ -250,7 +250,7 @@ func TestJobRunStateUnmarshalWithoutLifecycle(t *testing.T) { require.NoError(t, json.Unmarshal([]byte(`{}`), &state)) - assert.Nil(t, state.Lifecycle.Triggers.OnFileChange.Files) + assert.Nil(t, state.Lifecycle.Triggers.OnFileChange) } // The planner diffs RemapState(remote) against PrepareState(config), so a run From b00042c83d629e65e3f50ccc793806645a36fb21 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Mon, 24 Aug 2026 08:27:10 +0000 Subject: [PATCH 66/75] acc: refresh job_runs goldens for the omitted empty trigger Flattening on_file_change to a map made an unarmed trigger drop out of the serialized state, so the three plans that print an unarmed job_run now report an empty triggers object. --- acceptance/bundle/resources/job_runs/failed_run/output.txt | 4 +--- acceptance/bundle/resources/job_runs/redeploy/output.txt | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/acceptance/bundle/resources/job_runs/failed_run/output.txt b/acceptance/bundle/resources/job_runs/failed_run/output.txt index 37f1a831681..5b8b6dbcd5f 100644 --- a/acceptance/bundle/resources/job_runs/failed_run/output.txt +++ b/acceptance/bundle/resources/job_runs/failed_run/output.txt @@ -28,9 +28,7 @@ Files: 7 uploaded, 0 deleted "value": { "job_id": [MY_JOB_ID], "lifecycle": { - "triggers": { - "on_file_change": {} - } + "triggers": {} }, "result_state": "SUCCESS" } diff --git a/acceptance/bundle/resources/job_runs/redeploy/output.txt b/acceptance/bundle/resources/job_runs/redeploy/output.txt index a80685733ad..8532692bdd9 100644 --- a/acceptance/bundle/resources/job_runs/redeploy/output.txt +++ b/acceptance/bundle/resources/job_runs/redeploy/output.txt @@ -60,9 +60,7 @@ Resources: "env": "prod" }, "lifecycle": { - "triggers": { - "on_file_change": {} - } + "triggers": {} }, "result_state": "SUCCESS" } From 2006c6ce376b8f7a4a7405b614a5916e430360d1 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Mon, 24 Aug 2026 09:37:16 +0000 Subject: [PATCH 67/75] job_runs: error when on_file_change hashes nothing Directory-only and sync-excluded matches silently disarmed the trigger. Treat them as errors, warn that ** is not recursive, and cover both in acc. --- .../on_file_change_errors/allexcluded/a.txt | 1 + .../on_file_change_errors/databricks.yml | 6 ++++ .../on_file_change_errors/mixed/excluded.txt | 1 + .../mixed/nested/keep.txt | 1 + .../job_runs/on_file_change_errors/output.txt | 34 ++++++++++++++++--- .../job_runs/on_file_change_errors/script | 15 ++++++-- .../job_runs/on_file_change_glob/output.txt | 6 +++- .../job_runs/on_file_change_glob/script | 4 ++- .../mutator/resolve_job_run_file_triggers.go | 28 +++++++-------- 9 files changed, 73 insertions(+), 23 deletions(-) create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_errors/allexcluded/a.txt create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_errors/mixed/excluded.txt create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_errors/mixed/nested/keep.txt diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/allexcluded/a.txt b/acceptance/bundle/resources/job_runs/on_file_change_errors/allexcluded/a.txt new file mode 100644 index 00000000000..78981922613 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/allexcluded/a.txt @@ -0,0 +1 @@ +a diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/databricks.yml b/acceptance/bundle/resources/job_runs/on_file_change_errors/databricks.yml index 44c5433e52a..0ce9819887c 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_errors/databricks.yml +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/databricks.yml @@ -1,6 +1,12 @@ bundle: name: job-runs-on-file-change-errors +sync: + exclude: + - mixed/excluded.txt + # Literal path: the script rewrites the trigger by substring, and allexcluded/* would hit this line. + - allexcluded/a.txt + resources: jobs: my_job: diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/mixed/excluded.txt b/acceptance/bundle/resources/job_runs/on_file_change_errors/mixed/excluded.txt new file mode 100644 index 00000000000..bbde3dc9de9 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/mixed/excluded.txt @@ -0,0 +1 @@ +excluded diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/mixed/nested/keep.txt b/acceptance/bundle/resources/job_runs/on_file_change_errors/mixed/nested/keep.txt new file mode 100644 index 00000000000..2fa992c0b8b --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/mixed/nested/keep.txt @@ -0,0 +1 @@ +keep diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt index c7d3a98e1cb..381dc3e114f 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt @@ -2,7 +2,7 @@ === a pattern outside the sync root is rejected >>> musterr [CLI] bundle validate Error: lifecycle.triggers.on_file_change: pattern "../shared/*.txt" is not under the sync root - in databricks.yml:18:29 + in databricks.yml:25:29 Name: job-runs-on-file-change-errors Target: default @@ -14,8 +14,34 @@ Found 1 error === a glob that only matches directories is rejected >>> musterr [CLI] bundle validate -Error: lifecycle.triggers.on_file_change: pattern "dirs/*" matches no regular files - in databricks.yml:18:29 +Error: lifecycle.triggers.on_file_change: pattern "dirs/*" matches only directories or files excluded from sync, leaving nothing to hash + in databricks.yml:25:29 + +Name: job-runs-on-file-change-errors +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-errors/default + +Found 1 error + +=== a glob matching only a directory and an excluded file is rejected +>>> musterr [CLI] bundle validate +Error: lifecycle.triggers.on_file_change: pattern "mixed/*" matches only directories or files excluded from sync, leaving nothing to hash + in databricks.yml:25:29 + +Name: job-runs-on-file-change-errors +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-errors/default + +Found 1 error + +=== a glob matching only excluded files is rejected +>>> musterr [CLI] bundle validate +Error: lifecycle.triggers.on_file_change: pattern "allexcluded/*" matches only directories or files excluded from sync, leaving nothing to hash + in databricks.yml:25:29 Name: job-runs-on-file-change-errors Target: default @@ -28,7 +54,7 @@ Found 1 error === a pattern that matches nothing warns >>> [CLI] bundle validate Warning: lifecycle.triggers.on_file_change: no files match "missing/*.txt" - in databricks.yml:18:29 + in databricks.yml:25:29 Name: job-runs-on-file-change-errors Target: default diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/script b/acceptance/bundle/resources/job_runs/on_file_change_errors/script index 39596f0c772..48b9c5c7fc4 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_errors/script +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/script @@ -2,11 +2,20 @@ title "a pattern outside the sync root is rejected" trace musterr $CLI bundle validate title "a glob that only matches directories is rejected" -# dirs/* matches dirs/nested, a directory: nothing is left to hash, so the -# trigger would silently never fire. +# dirs/* matches dirs/nested, a directory, so nothing is left to hash. update_file.py databricks.yml "../shared/*.txt" "dirs/*" trace musterr $CLI bundle validate +title "a glob matching only a directory and an excluded file is rejected" +# mixed/excluded.txt is dropped by sync.exclude; mixed/nested is a directory. +update_file.py databricks.yml "dirs/*" "mixed/*" +trace musterr $CLI bundle validate + +title "a glob matching only excluded files is rejected" +# Every match is dropped by sync.exclude; unlike a missing pattern, this cannot re-arm later. +update_file.py databricks.yml "mixed/*" "allexcluded/*" +trace musterr $CLI bundle validate + title "a pattern that matches nothing warns" -update_file.py databricks.yml "dirs/*" "missing/*.txt" +update_file.py databricks.yml "allexcluded/*" "missing/*.txt" trace $CLI bundle validate diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt index a04e3710fb4..2f66f33e664 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt @@ -200,7 +200,11 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged -=== ** is not recursive: it matches the same files as * +=== ** is not recursive: it warns and matches the same files as * +Warning: lifecycle.triggers.on_file_change: ** in "migrations/**" is not recursive and matches the same files as * + in databricks.yml:18:29 + + >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { "lifecycle.triggers.on_file_change": { diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/script b/acceptance/bundle/resources/job_runs/on_file_change_glob/script index dd737f242b5..0f7107a43db 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/script +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/script @@ -58,6 +58,8 @@ update_file.py migrations/ignored.txt "ignored" "ignored-edited" trace $CLI bundle plan update_file.py migrations/ignored.txt "ignored-edited" "ignored" -title "** is not recursive: it matches the same files as *" +# Warning is on stderr of the untraced plan in changes(); trailing newline keeps it off the title. +title "** is not recursive: it warns and matches the same files as *\n" update_file.py databricks.yml "migrations/*" "migrations/**" changes +update_file.py databricks.yml "migrations/**" "migrations/*" diff --git a/bundle/config/mutator/resolve_job_run_file_triggers.go b/bundle/config/mutator/resolve_job_run_file_triggers.go index d74b9841c57..fbbabb60815 100644 --- a/bundle/config/mutator/resolve_job_run_file_triggers.go +++ b/bundle/config/mutator/resolve_job_run_file_triggers.go @@ -108,6 +108,14 @@ func resolveFileTrigger(b *bundle.Bundle, loc, pattern string, syncable map[stri Locations: b.Config.GetLocations(loc), }) } + // filepath.Glob treats ** as two *, so doublestar-style patterns match less than expected. + if strings.Contains(pattern, "**") { + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Warning, + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: ** in %q is not recursive and matches the same files as *", pattern), + Locations: b.Config.GetLocations(loc), + }) + } matches, err := filepath.Glob(filepath.Join(b.SyncRootPath, localPattern)) if err != nil { return out, diags.Append(diag.Diagnostic{ @@ -150,7 +158,7 @@ func resolveFileTrigger(b *bundle.Bundle, loc, pattern string, syncable map[stri }) continue } - // Same membership as sync: .gitignore and sync.exclude drop a glob match. + // Honor .gitignore and sync.exclude the same way sync does. if _, ok := syncable[filepath.ToSlash(rel)]; !ok { ignoredMatches++ continue @@ -167,20 +175,12 @@ func resolveFileTrigger(b *bundle.Bundle, loc, pattern string, syncable map[stri } out[filepath.ToSlash(rel)] = hash } - // A directory-only match would otherwise leave ResolvedFileTriggers empty - // and silently disarm the trigger while config still sets on_file_change. - if regularMatches == 0 && sawNonRegular && ignoredMatches == 0 { - diags = diags.Append(diag.Diagnostic{ + // Directories or excluded files would leave hashes empty and disarm the trigger. + // Unlike a missing-file warning, this cannot re-arm when a file appears later. + if regularMatches == 0 && (sawNonRegular || ignoredMatches > 0) { + return out, diags.Append(diag.Diagnostic{ Severity: diag.Error, - Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: pattern %q matches no regular files", pattern), - Locations: b.Config.GetLocations(loc), - }) - } - if len(out) == 0 && ignoredMatches > 0 { - out[filepath.ToSlash(pattern)] = missingFileHash - diags = diags.Append(diag.Diagnostic{ - Severity: diag.Warning, - Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: no files match %q", pattern), + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: pattern %q matches only directories or files excluded from sync, leaving nothing to hash", pattern), Locations: b.Config.GetLocations(loc), }) } From 7014ae44032d72d84e721113bf9ab4840a11d13f Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Mon, 24 Aug 2026 11:58:36 +0000 Subject: [PATCH 68/75] acc: fix on_file_change_errors diagnostic line numbers The goldens were captured against a one-line-taller databricks.yml, so CI failed after the source settled on line 24. --- .../job_runs/on_file_change_errors/output.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt index 381dc3e114f..15934e8eeb6 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt @@ -2,7 +2,7 @@ === a pattern outside the sync root is rejected >>> musterr [CLI] bundle validate Error: lifecycle.triggers.on_file_change: pattern "../shared/*.txt" is not under the sync root - in databricks.yml:25:29 + in databricks.yml:24:29 Name: job-runs-on-file-change-errors Target: default @@ -15,7 +15,7 @@ Found 1 error === a glob that only matches directories is rejected >>> musterr [CLI] bundle validate Error: lifecycle.triggers.on_file_change: pattern "dirs/*" matches only directories or files excluded from sync, leaving nothing to hash - in databricks.yml:25:29 + in databricks.yml:24:29 Name: job-runs-on-file-change-errors Target: default @@ -28,7 +28,7 @@ Found 1 error === a glob matching only a directory and an excluded file is rejected >>> musterr [CLI] bundle validate Error: lifecycle.triggers.on_file_change: pattern "mixed/*" matches only directories or files excluded from sync, leaving nothing to hash - in databricks.yml:25:29 + in databricks.yml:24:29 Name: job-runs-on-file-change-errors Target: default @@ -41,7 +41,7 @@ Found 1 error === a glob matching only excluded files is rejected >>> musterr [CLI] bundle validate Error: lifecycle.triggers.on_file_change: pattern "allexcluded/*" matches only directories or files excluded from sync, leaving nothing to hash - in databricks.yml:25:29 + in databricks.yml:24:29 Name: job-runs-on-file-change-errors Target: default @@ -54,7 +54,7 @@ Found 1 error === a pattern that matches nothing warns >>> [CLI] bundle validate Warning: lifecycle.triggers.on_file_change: no files match "missing/*.txt" - in databricks.yml:25:29 + in databricks.yml:24:29 Name: job-runs-on-file-change-errors Target: default From 2dda9e3c2f7a931d8b89d4b66e405ab88e40ec20 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Mon, 24 Aug 2026 13:11:35 +0000 Subject: [PATCH 69/75] acc: removing on_file_change is unchanged, not an update Clearing a trigger is now a skip, so the plan reports the run as unchanged and the deploy makes no API call. --- .../bundle/resources/job_runs/on_file_change/output.txt | 9 +++------ .../bundle/resources/job_runs/on_file_change/script | 2 +- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/acceptance/bundle/resources/job_runs/on_file_change/output.txt b/acceptance/bundle/resources/job_runs/on_file_change/output.txt index 28deeee38a4..978e4b4394a 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change/output.txt @@ -186,17 +186,14 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged } } -=== removing on_file_change rewrites state without a run +=== removing on_file_change is unchanged and does not run >>> [CLI] bundle plan -update job_runs.my_run - -Plan: 0 to add, 1 to change, 0 to delete, 1 unchanged +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change/default/files... -Updated job_runs.my_run Files: 3 uploaded, 0 deleted -Resources: 0 created, 1 changed, 0 deleted, 1 unchanged +Resources: 0 created, 0 changed, 0 deleted, 2 unchanged >>> print_requests.py //jobs/run-now diff --git a/acceptance/bundle/resources/job_runs/on_file_change/script b/acceptance/bundle/resources/job_runs/on_file_change/script index fd1e8800c42..ef432d76033 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change/script +++ b/acceptance/bundle/resources/job_runs/on_file_change/script @@ -54,7 +54,7 @@ trace $CLI bundle deploy read_id.py my_run > /dev/null trace print_requests.py //jobs/run-now -title "removing on_file_change rewrites state without a run" +title "removing on_file_change is unchanged and does not run" update_file.py databricks.yml " lifecycle: triggers: - on_file_change: seed.txt From 45c4ccd765d3f66b8d2ba6746daef54806018ecd Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Mon, 24 Aug 2026 16:56:03 +0000 Subject: [PATCH 70/75] job_runs: resolve on_file_change relative to the defining YAML file Patterns must still land under the sync root. Reject **, warn when a glob matches directories, and skip only when the trigger itself is cleared. --- .../job_runs/on_file_change_errors/output.txt | 29 ++++++--- .../job_runs/on_file_change_errors/script | 12 ++-- .../job_runs/on_file_change_glob/output.txt | 59 ++++++++++--------- .../job_runs/on_file_change_glob/script | 16 ++--- .../on_file_change_include/databricks.yml | 14 +++++ .../on_file_change_include/out.test.toml | 2 + .../on_file_change_include/output.txt | 29 +++++++++ .../resources/job_run.yml | 7 +++ .../resources/watched.txt | 1 + .../job_runs/on_file_change_include/script | 17 ++++++ .../job_runs/on_file_change_include/test.toml | 3 + .../on_file_change_include/watched.txt | 1 + .../bundle/databricks.yml | 19 ++++++ .../bundle/resources/job_run.yml | 7 +++ .../bundle/resources/watched.txt | 1 + .../bundle/watched.txt | 1 + .../out.test.toml | 2 + .../output.txt | 33 +++++++++++ .../on_file_change_include_sync_root/script | 22 +++++++ .../shared.txt | 1 + .../test.toml | 2 + .../bundle/databricks.yml | 23 ++++++++ .../bundle/shared.txt | 1 + .../on_file_change_sync_root/out.test.toml | 2 + .../on_file_change_sync_root/output.txt | 29 +++++++++ .../job_runs/on_file_change_sync_root/script | 17 ++++++ .../on_file_change_sync_root/shared.txt | 1 + .../on_file_change_sync_root/test.toml | 2 + bundle/config/mutator/normalize_paths_test.go | 38 ++++++++++++ .../mutator/paths/job_run_paths_visitor.go | 24 ++++++++ .../paths/job_run_paths_visitor_test.go | 34 +++++++++++ bundle/config/mutator/paths/visitor.go | 1 + .../mutator/resolve_job_run_file_triggers.go | 46 +++++++++------ .../resolve_job_run_file_triggers_test.go | 22 ++++++- bundle/config/resources/lifecycle.go | 2 +- bundle/direct/dresources/job_run.go | 34 +++++++---- bundle/direct/dresources/job_run_test.go | 27 +++++++++ bundle/internal/schema/annotations.yml | 2 +- bundle/schema/jsonschema.json | 2 +- 39 files changed, 499 insertions(+), 86 deletions(-) create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_include/databricks.yml create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_include/out.test.toml create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_include/output.txt create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_include/resources/job_run.yml create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_include/resources/watched.txt create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_include/script create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_include/test.toml create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_include/watched.txt create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/bundle/databricks.yml create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/bundle/resources/job_run.yml create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/bundle/resources/watched.txt create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/bundle/watched.txt create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/out.test.toml create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/output.txt create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/script create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/shared.txt create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/test.toml create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_sync_root/bundle/databricks.yml create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_sync_root/bundle/shared.txt create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_sync_root/out.test.toml create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_sync_root/output.txt create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_sync_root/script create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_sync_root/shared.txt create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_sync_root/test.toml create mode 100644 bundle/config/mutator/paths/job_run_paths_visitor.go create mode 100644 bundle/config/mutator/paths/job_run_paths_visitor_test.go diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt index 15934e8eeb6..12e4730d8d9 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt @@ -1,5 +1,5 @@ -=== a pattern outside the sync root is rejected +=== a pattern that escapes the sync root is rejected >>> musterr [CLI] bundle validate Error: lifecycle.triggers.on_file_change: pattern "../shared/*.txt" is not under the sync root in databricks.yml:24:29 @@ -12,9 +12,9 @@ Workspace: Found 1 error -=== a glob that only matches directories is rejected ->>> musterr [CLI] bundle validate -Error: lifecycle.triggers.on_file_change: pattern "dirs/*" matches only directories or files excluded from sync, leaving nothing to hash +=== a glob that only matches directories warns +>>> [CLI] bundle validate +Warning: lifecycle.triggers.on_file_change: pattern "dirs/*" matches only directories in databricks.yml:24:29 Name: job-runs-on-file-change-errors @@ -23,11 +23,11 @@ Workspace: User: [USERNAME] Path: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-errors/default -Found 1 error +Found 1 warning -=== a glob matching only a directory and an excluded file is rejected +=== a glob matching a directory and an excluded file is rejected >>> musterr [CLI] bundle validate -Error: lifecycle.triggers.on_file_change: pattern "mixed/*" matches only directories or files excluded from sync, leaving nothing to hash +Error: lifecycle.triggers.on_file_change: pattern "mixed/*" matches only files excluded from sync, leaving nothing to hash in databricks.yml:24:29 Name: job-runs-on-file-change-errors @@ -40,7 +40,7 @@ Found 1 error === a glob matching only excluded files is rejected >>> musterr [CLI] bundle validate -Error: lifecycle.triggers.on_file_change: pattern "allexcluded/*" matches only directories or files excluded from sync, leaving nothing to hash +Error: lifecycle.triggers.on_file_change: pattern "allexcluded/*" matches only files excluded from sync, leaving nothing to hash in databricks.yml:24:29 Name: job-runs-on-file-change-errors @@ -63,3 +63,16 @@ Workspace: Path: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-errors/default Found 1 warning + +=== ** is not supported +>>> musterr [CLI] bundle validate +Error: lifecycle.triggers.on_file_change: ** in "src/**" is not supported; use * for a single directory level + in databricks.yml:24:29 + +Name: job-runs-on-file-change-errors +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-errors/default + +Found 1 error diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/script b/acceptance/bundle/resources/job_runs/on_file_change_errors/script index 48b9c5c7fc4..4b6e4ead73b 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_errors/script +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/script @@ -1,12 +1,12 @@ -title "a pattern outside the sync root is rejected" +title "a pattern that escapes the sync root is rejected" trace musterr $CLI bundle validate -title "a glob that only matches directories is rejected" +title "a glob that only matches directories warns" # dirs/* matches dirs/nested, a directory, so nothing is left to hash. update_file.py databricks.yml "../shared/*.txt" "dirs/*" -trace musterr $CLI bundle validate +trace $CLI bundle validate -title "a glob matching only a directory and an excluded file is rejected" +title "a glob matching a directory and an excluded file is rejected" # mixed/excluded.txt is dropped by sync.exclude; mixed/nested is a directory. update_file.py databricks.yml "dirs/*" "mixed/*" trace musterr $CLI bundle validate @@ -19,3 +19,7 @@ trace musterr $CLI bundle validate title "a pattern that matches nothing warns" update_file.py databricks.yml "allexcluded/*" "missing/*.txt" trace $CLI bundle validate + +title "** is not supported" +update_file.py databricks.yml "missing/*.txt" "src/**" +trace musterr $CLI bundle validate diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt index 2f66f33e664..390dda9f513 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt @@ -1,6 +1,9 @@ === deploy hashes the matched files >>> [CLI] bundle deploy +Warning: lifecycle.triggers.on_file_change: pattern "migrations/*" also matched directories, which are not hashed + in databricks.yml:18:29 + Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-glob/default/files... Output from job_runs.my_run: id=[MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?o=[NUMID] Output from job_runs.my_run: id=[MY_RUN_ID]: SUCCESS @@ -23,6 +26,10 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged } === editing one matched file re-fires +Warning: lifecycle.triggers.on_file_change: pattern "migrations/*" also matched directories, which are not hashed + in databricks.yml:18:29 + + >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { "lifecycle.triggers.on_file_change": { @@ -54,6 +61,9 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged } === bundle deploy +Warning: lifecycle.triggers.on_file_change: pattern "migrations/*" also matched directories, which are not hashed + in databricks.yml:18:29 + Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-glob/default/files... Output from job_runs.my_run: id=[MY_RUN_ID_2]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_2]?o=[NUMID] Output from job_runs.my_run: id=[MY_RUN_ID_2]: SUCCESS @@ -81,6 +91,10 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged } === editing a file in a subdirectory re-fires +Warning: lifecycle.triggers.on_file_change: pattern "migrations/*" also matched directories, which are not hashed + in databricks.yml:18:29 + + >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { "lifecycle.triggers.on_file_change": { @@ -112,6 +126,10 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged } === a new matching file re-fires +Warning: lifecycle.triggers.on_file_change: pattern "migrations/*" also matched directories, which are not hashed + in databricks.yml:18:29 + + >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { "lifecycle.triggers.on_file_change": { @@ -143,6 +161,10 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged } === removing a matched file re-fires +Warning: lifecycle.triggers.on_file_change: pattern "migrations/*" also matched directories, which are not hashed + in databricks.yml:18:29 + + >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { "lifecycle.triggers.on_file_change": { @@ -172,6 +194,10 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged } === editing a file below the glob does not re-fire +Warning: lifecycle.triggers.on_file_change: pattern "migrations/*" also matched directories, which are not hashed + in databricks.yml:18:29 + + >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { "lifecycle.triggers.on_file_change": { @@ -198,38 +224,15 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged === editing a gitignored file does not re-fire >>> [CLI] bundle plan -Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged - -=== ** is not recursive: it warns and matches the same files as * -Warning: lifecycle.triggers.on_file_change: ** in "migrations/**" is not recursive and matches the same files as * +Warning: lifecycle.triggers.on_file_change: pattern "migrations/*" also matched directories, which are not hashed in databricks.yml:18:29 - ->>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json -{ - "lifecycle.triggers.on_file_change": { - "action": "skip", - "reason": "missing_in_remote", - "old": { - "migrations/001.txt": "[FILE_HASH][3]", - "migrations/002.txt": "[FILE_HASH][1]", - "mysubdir/a.txt": "[FILE_HASH][2]" - }, - "new": { - "migrations/001.txt": "[FILE_HASH][3]", - "migrations/002.txt": "[FILE_HASH][1]", - "mysubdir/a.txt": "[FILE_HASH][2]" - } - }, - "result_state": { - "action": "skip", - "reason": "remote_already_set", - "new": "SUCCESS", - "remote": "SUCCESS" - } -} +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged >>> [CLI] bundle destroy --auto-approve +Warning: lifecycle.triggers.on_file_change: pattern "migrations/*" also matched directories, which are not hashed + in databricks.yml:18:29 + The following resources will be deleted: delete resources.job_runs.my_run delete resources.jobs.my_job diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/script b/acceptance/bundle/resources/job_runs/on_file_change_glob/script index 0f7107a43db..f0130f2637b 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/script +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/script @@ -19,7 +19,7 @@ trace read_id.py my_job read_id.py my_run > /dev/null trace print_requests.py //jobs/run-now -title "editing one matched file re-fires" +title "editing one matched file re-fires\n" update_file.py migrations/001.txt "one" "one-edited" changes # The deploy is not traced: readplanarg makes the command line differ per variant. @@ -29,24 +29,24 @@ read_id.py my_run > /dev/null trace print_requests.py --keep //jobs/runs/delete trace print_requests.py //jobs/run-now -title "editing a file in a subdirectory re-fires" +title "editing a file in a subdirectory re-fires\n" # printf keeps LF on Windows, where update_file.py's text-mode rewrite would # restore CRLF and leave the file hashing differently than the deploy above. printf 'sub-edited\n' > mysubdir/a.txt changes printf 'sub\n' > mysubdir/a.txt -title "a new matching file re-fires" +title "a new matching file re-fires\n" printf 'three\n' > migrations/003.txt changes rm migrations/003.txt -title "removing a matched file re-fires" +title "removing a matched file re-fires\n" mv migrations/002.txt tmp.002.txt changes mv tmp.002.txt migrations/002.txt -title "editing a file below the glob does not re-fire" +title "editing a file below the glob does not re-fire\n" # migrations/* matches the archive directory itself, which is skipped: the glob # does not descend into it. update_file.py migrations/archive/003.txt "three" "three-edited" @@ -57,9 +57,3 @@ title "editing a gitignored file does not re-fire" update_file.py migrations/ignored.txt "ignored" "ignored-edited" trace $CLI bundle plan update_file.py migrations/ignored.txt "ignored-edited" "ignored" - -# Warning is on stderr of the untraced plan in changes(); trailing newline keeps it off the title. -title "** is not recursive: it warns and matches the same files as *\n" -update_file.py databricks.yml "migrations/*" "migrations/**" -changes -update_file.py databricks.yml "migrations/**" "migrations/*" diff --git a/acceptance/bundle/resources/job_runs/on_file_change_include/databricks.yml b/acceptance/bundle/resources/job_runs/on_file_change_include/databricks.yml new file mode 100644 index 00000000000..3d8b74d28f8 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_include/databricks.yml @@ -0,0 +1,14 @@ +bundle: + name: job-runs-on-file-change-include + +include: + - resources/job_run.yml + +resources: + jobs: + my_job: + name: my-job + tasks: + - task_key: main + notebook_task: + notebook_path: /Workspace/test diff --git a/acceptance/bundle/resources/job_runs/on_file_change_include/out.test.toml b/acceptance/bundle/resources/job_runs/on_file_change_include/out.test.toml new file mode 100644 index 00000000000..0938e678987 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_include/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/job_runs/on_file_change_include/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_include/output.txt new file mode 100644 index 00000000000..52de054e734 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_include/output.txt @@ -0,0 +1,29 @@ + +=== first deploy +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-include/default/files... +Output from job_runs.my_run: id=[NUMID]: Run URL: [DATABRICKS_URL]/jobs/[NUMID]/runs/[NUMID]?o=[NUMID] +Output from job_runs.my_run: id=[NUMID]: SUCCESS +Created job_runs.my_run +Created jobs.my_job +Files: 7 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +=== editing a same-named file at the bundle root does not re-fire +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +=== editing the file next to the included YAML re-fires +>>> [CLI] bundle plan +recreate job_runs.my_run + +Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.job_runs.my_run + delete resources.jobs.my_job + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-include/default + +Destroy: 2 deleted diff --git a/acceptance/bundle/resources/job_runs/on_file_change_include/resources/job_run.yml b/acceptance/bundle/resources/job_runs/on_file_change_include/resources/job_run.yml new file mode 100644 index 00000000000..091489142f5 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_include/resources/job_run.yml @@ -0,0 +1,7 @@ +resources: + job_runs: + my_run: + job_id: ${resources.jobs.my_job.id} + lifecycle: + triggers: + - on_file_change: watched.txt diff --git a/acceptance/bundle/resources/job_runs/on_file_change_include/resources/watched.txt b/acceptance/bundle/resources/job_runs/on_file_change_include/resources/watched.txt new file mode 100644 index 00000000000..8b4e79d3c47 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_include/resources/watched.txt @@ -0,0 +1 @@ +include-v1 diff --git a/acceptance/bundle/resources/job_runs/on_file_change_include/script b/acceptance/bundle/resources/job_runs/on_file_change_include/script new file mode 100644 index 00000000000..65a8cea7a5b --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_include/script @@ -0,0 +1,17 @@ +cleanup() { + trace $CLI bundle destroy --auto-approve +} +trap cleanup EXIT + +title "first deploy" +trace $CLI bundle deploy + +title "editing a same-named file at the bundle root does not re-fire" +update_file.py watched.txt "root-v1" "root-v2" +trace $CLI bundle plan | contains.py "2 unchanged" "!recreate" +update_file.py watched.txt "root-v2" "root-v1" + +title "editing the file next to the included YAML re-fires" +update_file.py resources/watched.txt "include-v1" "include-v2" +trace $CLI bundle plan | contains.py "recreate job_runs.my_run" +update_file.py resources/watched.txt "include-v2" "include-v1" diff --git a/acceptance/bundle/resources/job_runs/on_file_change_include/test.toml b/acceptance/bundle/resources/job_runs/on_file_change_include/test.toml new file mode 100644 index 00000000000..c81bf4f943a --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_include/test.toml @@ -0,0 +1,3 @@ +# Path resolution only; the HTTP traffic is the same as on_file_change. +RecordRequests = false +Ignore = [".databricks"] diff --git a/acceptance/bundle/resources/job_runs/on_file_change_include/watched.txt b/acceptance/bundle/resources/job_runs/on_file_change_include/watched.txt new file mode 100644 index 00000000000..d1688530992 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_include/watched.txt @@ -0,0 +1 @@ +root-v1 diff --git a/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/bundle/databricks.yml b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/bundle/databricks.yml new file mode 100644 index 00000000000..2bdd1c06d13 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/bundle/databricks.yml @@ -0,0 +1,19 @@ +bundle: + name: job-runs-on-file-change-include-sync-root + +include: + - resources/job_run.yml + +sync: + paths: + - . + - ../shared.txt + +resources: + jobs: + my_job: + name: my-job + tasks: + - task_key: main + notebook_task: + notebook_path: /Workspace/test diff --git a/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/bundle/resources/job_run.yml b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/bundle/resources/job_run.yml new file mode 100644 index 00000000000..091489142f5 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/bundle/resources/job_run.yml @@ -0,0 +1,7 @@ +resources: + job_runs: + my_run: + job_id: ${resources.jobs.my_job.id} + lifecycle: + triggers: + - on_file_change: watched.txt diff --git a/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/bundle/resources/watched.txt b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/bundle/resources/watched.txt new file mode 100644 index 00000000000..8b4e79d3c47 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/bundle/resources/watched.txt @@ -0,0 +1 @@ +include-v1 diff --git a/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/bundle/watched.txt b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/bundle/watched.txt new file mode 100644 index 00000000000..6f22941eacf --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/bundle/watched.txt @@ -0,0 +1 @@ +bundle-v1 diff --git a/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/out.test.toml b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/out.test.toml new file mode 100644 index 00000000000..0938e678987 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/output.txt new file mode 100644 index 00000000000..6c475c0dbfc --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/output.txt @@ -0,0 +1,33 @@ + +=== first deploy +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-include-sync-root/default/files... +Output from job_runs.my_run: id=[NUMID]: Run URL: [DATABRICKS_URL]/jobs/[NUMID]/runs/[NUMID]?o=[NUMID] +Output from job_runs.my_run: id=[NUMID]: SUCCESS +Created job_runs.my_run +Created jobs.my_job +Files: 5 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +=== editing a same-named file in the bundle does not re-fire +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +=== editing a file at the sync root does not re-fire +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +=== editing the file next to the included YAML re-fires +>>> [CLI] bundle plan +recreate job_runs.my_run + +Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.job_runs.my_run + delete resources.jobs.my_job + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-include-sync-root/default + +Destroy: 2 deleted diff --git a/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/script b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/script new file mode 100644 index 00000000000..5a608ea9188 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/script @@ -0,0 +1,22 @@ +cleanup() { + withdir bundle trace $CLI bundle destroy --auto-approve +} +trap cleanup EXIT + +title "first deploy" +withdir bundle trace $CLI bundle deploy + +title "editing a same-named file in the bundle does not re-fire" +update_file.py bundle/watched.txt "bundle-v1" "bundle-v2" +withdir bundle trace $CLI bundle plan | contains.py "2 unchanged" "!recreate" +update_file.py bundle/watched.txt "bundle-v2" "bundle-v1" + +title "editing a file at the sync root does not re-fire" +update_file.py shared.txt "parent-v1" "parent-v2" +withdir bundle trace $CLI bundle plan | contains.py "2 unchanged" "!recreate" +update_file.py shared.txt "parent-v2" "parent-v1" + +title "editing the file next to the included YAML re-fires" +update_file.py bundle/resources/watched.txt "include-v1" "include-v2" +withdir bundle trace $CLI bundle plan | contains.py "recreate job_runs.my_run" +update_file.py bundle/resources/watched.txt "include-v2" "include-v1" diff --git a/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/shared.txt b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/shared.txt new file mode 100644 index 00000000000..4ddcc71c374 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/shared.txt @@ -0,0 +1 @@ +parent-v1 diff --git a/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/test.toml b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/test.toml new file mode 100644 index 00000000000..e1290bad9f4 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/test.toml @@ -0,0 +1,2 @@ +RecordRequests = false +Ignore = [".databricks"] diff --git a/acceptance/bundle/resources/job_runs/on_file_change_sync_root/bundle/databricks.yml b/acceptance/bundle/resources/job_runs/on_file_change_sync_root/bundle/databricks.yml new file mode 100644 index 00000000000..772a7ef92e7 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_sync_root/bundle/databricks.yml @@ -0,0 +1,23 @@ +bundle: + name: job-runs-on-file-change-sync-root + +sync: + paths: + - . + - ../shared.txt + +resources: + jobs: + my_job: + name: my-job + tasks: + - task_key: main + notebook_task: + notebook_path: /Workspace/test + + job_runs: + my_run: + job_id: ${resources.jobs.my_job.id} + lifecycle: + triggers: + - on_file_change: ../shared.txt diff --git a/acceptance/bundle/resources/job_runs/on_file_change_sync_root/bundle/shared.txt b/acceptance/bundle/resources/job_runs/on_file_change_sync_root/bundle/shared.txt new file mode 100644 index 00000000000..6f22941eacf --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_sync_root/bundle/shared.txt @@ -0,0 +1 @@ +bundle-v1 diff --git a/acceptance/bundle/resources/job_runs/on_file_change_sync_root/out.test.toml b/acceptance/bundle/resources/job_runs/on_file_change_sync_root/out.test.toml new file mode 100644 index 00000000000..0938e678987 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_sync_root/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/job_runs/on_file_change_sync_root/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_sync_root/output.txt new file mode 100644 index 00000000000..6a52f127ce8 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_sync_root/output.txt @@ -0,0 +1,29 @@ + +=== first deploy +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-sync-root/default/files... +Output from job_runs.my_run: id=[NUMID]: Run URL: [DATABRICKS_URL]/jobs/[NUMID]/runs/[NUMID]?o=[NUMID] +Output from job_runs.my_run: id=[NUMID]: SUCCESS +Created job_runs.my_run +Created jobs.my_job +Files: 3 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +=== editing a same-named file in the bundle does not re-fire +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +=== editing the file at the sync root re-fires +>>> [CLI] bundle plan +recreate job_runs.my_run + +Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.job_runs.my_run + delete resources.jobs.my_job + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-sync-root/default + +Destroy: 2 deleted diff --git a/acceptance/bundle/resources/job_runs/on_file_change_sync_root/script b/acceptance/bundle/resources/job_runs/on_file_change_sync_root/script new file mode 100644 index 00000000000..dcbc475840b --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_sync_root/script @@ -0,0 +1,17 @@ +cleanup() { + withdir bundle trace $CLI bundle destroy --auto-approve +} +trap cleanup EXIT + +title "first deploy" +withdir bundle trace $CLI bundle deploy + +title "editing a same-named file in the bundle does not re-fire" +update_file.py bundle/shared.txt "bundle-v1" "bundle-v2" +withdir bundle trace $CLI bundle plan | contains.py "2 unchanged" "!recreate" +update_file.py bundle/shared.txt "bundle-v2" "bundle-v1" + +title "editing the file at the sync root re-fires" +update_file.py shared.txt "sync-v1" "sync-v2" +withdir bundle trace $CLI bundle plan | contains.py "recreate job_runs.my_run" +update_file.py shared.txt "sync-v2" "sync-v1" diff --git a/acceptance/bundle/resources/job_runs/on_file_change_sync_root/shared.txt b/acceptance/bundle/resources/job_runs/on_file_change_sync_root/shared.txt new file mode 100644 index 00000000000..04f175f2a1c --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_sync_root/shared.txt @@ -0,0 +1 @@ +sync-v1 diff --git a/acceptance/bundle/resources/job_runs/on_file_change_sync_root/test.toml b/acceptance/bundle/resources/job_runs/on_file_change_sync_root/test.toml new file mode 100644 index 00000000000..e1290bad9f4 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_sync_root/test.toml @@ -0,0 +1,2 @@ +RecordRequests = false +Ignore = [".databricks"] diff --git a/bundle/config/mutator/normalize_paths_test.go b/bundle/config/mutator/normalize_paths_test.go index 8bee6c5064b..2399a03aa83 100644 --- a/bundle/config/mutator/normalize_paths_test.go +++ b/bundle/config/mutator/normalize_paths_test.go @@ -53,6 +53,44 @@ func TestNormalizePaths(t *testing.T) { require.Equal(t, "src/notebook.py", newValue.MustString()) } +func TestNormalizePaths_jobRunOnFileChange(t *testing.T) { + tmpDir := t.TempDir() + pattern := "../data/*.txt" + m := NormalizePaths() + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + JobRuns: map[string]*resources.JobRun{ + "run1": { + Lifecycle: &resources.JobRunLifecycle{ + Triggers: []resources.JobRunTrigger{ + {OnFileChange: &pattern}, + }, + }, + }, + }, + }, + }, + BundleRootPath: tmpDir, + } + + location := dyn.Location{File: filepath.Join(tmpDir, "resources", "run.yml")} + path := dyn.MustPathFromString("resources.job_runs.run1.lifecycle.triggers[0].on_file_change") + err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { + return dyn.MapByPath(v, path, func(path dyn.Path, value dyn.Value) (dyn.Value, error) { + return dyn.NewValue(value.MustString(), []dyn.Location{location}), nil + }) + }) + require.NoError(t, err) + + diags := bundle.Apply(t.Context(), b, m) + require.NoError(t, diags.Error()) + + newValue, err := dyn.GetByPath(b.Config.Value(), path) + require.NoError(t, err) + require.Equal(t, "data/*.txt", newValue.MustString()) +} + func TestNormalizePath_absolutePath(t *testing.T) { value, err := normalizePath("/notebook.py", dyn.Location{}, "/tmp") assert.NoError(t, err) diff --git a/bundle/config/mutator/paths/job_run_paths_visitor.go b/bundle/config/mutator/paths/job_run_paths_visitor.go new file mode 100644 index 00000000000..3bc56bacf72 --- /dev/null +++ b/bundle/config/mutator/paths/job_run_paths_visitor.go @@ -0,0 +1,24 @@ +package paths + +import ( + "github.com/databricks/cli/libs/dyn" +) + +// VisitJobRunPaths visits local paths on job_runs so NormalizePaths can rewrite +// them relative to the bundle root. Not used by TranslatePaths: hashing still +// needs a local glob, not a workspace path. +func VisitJobRunPaths(value dyn.Value, fn VisitFunc) (dyn.Value, error) { + pattern := dyn.NewPattern( + dyn.Key("resources"), + dyn.Key("job_runs"), + dyn.AnyKey(), + dyn.Key("lifecycle"), + dyn.Key("triggers"), + dyn.AnyIndex(), + dyn.Key("on_file_change"), + ) + + return dyn.MapByPattern(value, pattern, func(path dyn.Path, value dyn.Value) (dyn.Value, error) { + return fn(path, TranslateModeLocalRelative, value) + }) +} diff --git a/bundle/config/mutator/paths/job_run_paths_visitor_test.go b/bundle/config/mutator/paths/job_run_paths_visitor_test.go new file mode 100644 index 00000000000..0a006fd9df1 --- /dev/null +++ b/bundle/config/mutator/paths/job_run_paths_visitor_test.go @@ -0,0 +1,34 @@ +package paths + +import ( + "testing" + + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/libs/dyn" + "github.com/stretchr/testify/assert" +) + +func TestVisitJobRunPaths(t *testing.T) { + watched := "watched.txt" + root := config.Root{ + Resources: config.Resources{ + JobRuns: map[string]*resources.JobRun{ + "run0": { + Lifecycle: &resources.JobRunLifecycle{ + Triggers: []resources.JobRunTrigger{ + {OnFileChange: &watched}, + }, + }, + }, + }, + }, + } + + actual := collectVisitedPaths(t, root, VisitJobRunPaths) + expected := []dyn.Path{ + dyn.MustPathFromString("resources.job_runs.run0.lifecycle.triggers[0].on_file_change"), + } + + assert.ElementsMatch(t, expected, actual) +} diff --git a/bundle/config/mutator/paths/visitor.go b/bundle/config/mutator/paths/visitor.go index bdf42188fde..cadc52b75fe 100644 --- a/bundle/config/mutator/paths/visitor.go +++ b/bundle/config/mutator/paths/visitor.go @@ -10,6 +10,7 @@ type VisitFunc func(path dyn.Path, mode TranslateMode, value dyn.Value) (dyn.Val func VisitPaths(root dyn.Value, fn VisitFunc) (dyn.Value, error) { visitors := []func(dyn.Value, VisitFunc) (dyn.Value, error){ VisitJobPaths, + VisitJobRunPaths, VisitJobLibrariesPaths, VisitAppPaths, VisitArtifactPaths, diff --git a/bundle/config/mutator/resolve_job_run_file_triggers.go b/bundle/config/mutator/resolve_job_run_file_triggers.go index fbbabb60815..2b3f11a1a88 100644 --- a/bundle/config/mutator/resolve_job_run_file_triggers.go +++ b/bundle/config/mutator/resolve_job_run_file_triggers.go @@ -52,11 +52,7 @@ func (*resolveJobRunFileTriggers) Apply(ctx context.Context, b *bundle.Bundle) d diags = diags.Extend(d) maps.Copy(out, hashes) } - if len(out) == 0 { - jr.ResolvedFileTriggers = nil - } else { - jr.ResolvedFileTriggers = out - } + jr.ResolvedFileTriggers = out } return diags } @@ -100,23 +96,27 @@ func resolveFileTrigger(b *bundle.Bundle, loc, pattern string, syncable map[stri var diags diag.Diagnostics out := make(map[string]string) localPattern := filepath.FromSlash(pattern) - // Keep hashes under SyncRoot; same IsLocal gate as translate_paths. - if !filepath.IsLocal(localPattern) { + // filepath.Glob treats ** as two *, so doublestar-style patterns match less than expected. + if strings.Contains(pattern, "**") { return out, diags.Append(diag.Diagnostic{ Severity: diag.Error, - Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: pattern %q is not under the sync root", pattern), + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: ** in %q is not supported; use * for a single directory level", pattern), Locations: b.Config.GetLocations(loc), }) } - // filepath.Glob treats ** as two *, so doublestar-style patterns match less than expected. - if strings.Contains(pattern, "**") { - diags = diags.Append(diag.Diagnostic{ - Severity: diag.Warning, - Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: ** in %q is not recursive and matches the same files as *", pattern), + // NormalizePaths has already rewritten YAML-relative globs to be bundle-root + // relative. Join that onto the bundle root, then require the result stay + // under the sync root (an ancestor of the bundle when sync.paths uses ..). + joined := filepath.Join(b.BundleRootPath, localPattern) + relPattern, err := filepath.Rel(b.SyncRootPath, joined) + if err != nil || !filepath.IsLocal(relPattern) { + return out, diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: pattern %q is not under the sync root", pattern), Locations: b.Config.GetLocations(loc), }) } - matches, err := filepath.Glob(filepath.Join(b.SyncRootPath, localPattern)) + matches, err := filepath.Glob(joined) if err != nil { return out, diags.Append(diag.Diagnostic{ Severity: diag.Error, @@ -175,12 +175,22 @@ func resolveFileTrigger(b *bundle.Bundle, loc, pattern string, syncable map[stri } out[filepath.ToSlash(rel)] = hash } - // Directories or excluded files would leave hashes empty and disarm the trigger. - // Unlike a missing-file warning, this cannot re-arm when a file appears later. - if regularMatches == 0 && (sawNonRegular || ignoredMatches > 0) { + if regularMatches == 0 && ignoredMatches > 0 { return out, diags.Append(diag.Diagnostic{ Severity: diag.Error, - Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: pattern %q matches only directories or files excluded from sync, leaving nothing to hash", pattern), + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: pattern %q matches only files excluded from sync, leaving nothing to hash", pattern), + Locations: b.Config.GetLocations(loc), + }) + } + if sawNonRegular { + msg := fmt.Sprintf("lifecycle.triggers.on_file_change: pattern %q also matched directories, which are not hashed", pattern) + if regularMatches == 0 { + out[filepath.ToSlash(pattern)] = missingFileHash + msg = fmt.Sprintf("lifecycle.triggers.on_file_change: pattern %q matches only directories", pattern) + } + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Warning, + Summary: msg, Locations: b.Config.GetLocations(loc), }) } diff --git a/bundle/config/mutator/resolve_job_run_file_triggers_test.go b/bundle/config/mutator/resolve_job_run_file_triggers_test.go index 1cc54eecf4f..0630a4b35f6 100644 --- a/bundle/config/mutator/resolve_job_run_file_triggers_test.go +++ b/bundle/config/mutator/resolve_job_run_file_triggers_test.go @@ -44,14 +44,30 @@ func TestResolveJobRunFileTriggers(t *testing.T) { require.False(t, diags.HasError()) assert.Equal(t, contentHash("v1"), b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers["seed.txt"]) }) + + t.Run("globs from the bundle root when the sync root is an ancestor", func(t *testing.T) { + parent := t.TempDir() + bundleDir := filepath.Join(parent, "bundle") + require.NoError(t, os.Mkdir(bundleDir, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(parent, "shared.txt"), []byte("from-sync-root"), 0o644)) + + pattern := "../shared.txt" + b := bundleWithFileTrigger(parent, pattern) + b.BundleRootPath = bundleDir + + diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) + require.False(t, diags.HasError()) + assert.Equal(t, contentHash("from-sync-root"), b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers["shared.txt"]) + }) } func bundleWithFileTrigger(syncRoot, pattern string) *bundle.Bundle { root := vfs.MustNew(syncRoot) return &bundle.Bundle{ - SyncRootPath: syncRoot, - SyncRoot: root, - WorktreeRoot: root, + BundleRootPath: syncRoot, + SyncRootPath: syncRoot, + SyncRoot: root, + WorktreeRoot: root, Config: config.Root{ Sync: config.Sync{Paths: []string{"."}}, Resources: config.Resources{ diff --git a/bundle/config/resources/lifecycle.go b/bundle/config/resources/lifecycle.go index a4e881d298c..1f5d576d835 100644 --- a/bundle/config/resources/lifecycle.go +++ b/bundle/config/resources/lifecycle.go @@ -37,5 +37,5 @@ type JobRunLifecycle struct { // JobRunTrigger is one lifecycle.triggers entry. type JobRunTrigger struct { OnBundleDeploy *bool `json:"on_bundle_deploy,omitempty"` - OnFileChange *string `json:"on_file_change,omitempty"` // path or glob under sync root + OnFileChange *string `json:"on_file_change,omitempty"` // path or glob relative to the defining YAML file; must resolve under the sync root } diff --git a/bundle/direct/dresources/job_run.go b/bundle/direct/dresources/job_run.go index 9d1a272a7a3..872720ffcd4 100644 --- a/bundle/direct/dresources/job_run.go +++ b/bundle/direct/dresources/job_run.go @@ -26,14 +26,29 @@ import ( // jobRunTimeout matches the timeout `bundle run` allows a run (bundle/run/job.go). const jobRunTimeout = 24 * time.Hour -// jobRunTriggerLocalPaths are local-only fingerprints: clearing one is skip, not recreate. -var jobRunTriggerLocalPaths = []string{ - "lifecycle.triggers.on_bundle_deploy", - "lifecycle.triggers.on_file_change", +// Parsed so HasPrefix catches parent and child paths; skip is only when the +// trigger itself is cleared, not when a file hash under on_file_change changes. +var ( + jobRunTriggersPath = structpath.MustParsePath("lifecycle.triggers") + jobRunOnBundleDeployPath = structpath.MustParsePath("lifecycle.triggers.on_bundle_deploy") + jobRunOnFileChangePath = structpath.MustParsePath("lifecycle.triggers.on_file_change") +) + +func samePath(a, b *structpath.PathNode) bool { + return a.HasPrefix(b) && b.HasPrefix(a) } -func isJobRunTriggerPath(path string) bool { - return slices.Contains(jobRunTriggerLocalPaths, path) +func jobRunTriggerRemoved(path *structpath.PathNode, change *ChangeDesc) bool { + switch { + case samePath(path, jobRunOnBundleDeployPath): + return change.New == nil || change.New == "" + case samePath(path, jobRunOnFileChangePath): + return change.New == nil + case samePath(path, jobRunTriggersPath): + return change.New == nil + default: + return false + } } // JobRunTriggersState is the persisted fingerprint of lifecycle.triggers. @@ -396,11 +411,8 @@ func reportRunLine(ctx context.Context, runID int64, msg string) { // result_state either, so the lifecycle state is what tells the two apart. // Clearing a trigger skips its local-only fingerprint without re-firing the run. func (*ResourceJobRun) OverrideChangeDesc(_ context.Context, path *structpath.PathNode, change *ChangeDesc, remote *JobRunRemote) error { - pathString := path.String() - if isJobRunTriggerPath(pathString) { - removed := pathString == "lifecycle.triggers.on_bundle_deploy" && (change.New == nil || change.New == "") - removed = removed || pathString == "lifecycle.triggers.on_file_change" && change.New == nil - if removed { + if path.HasPrefix(jobRunTriggersPath) { + if jobRunTriggerRemoved(path, change) { change.Action = deployplan.Skip change.Reason = "trigger removed" } diff --git a/bundle/direct/dresources/job_run_test.go b/bundle/direct/dresources/job_run_test.go index 804968f7707..68738b93e18 100644 --- a/bundle/direct/dresources/job_run_test.go +++ b/bundle/direct/dresources/job_run_test.go @@ -10,6 +10,7 @@ import ( "testing" "time" + "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/cli/libs/testserver" "github.com/databricks/databricks-sdk-go" @@ -367,3 +368,29 @@ func TestJobRunDeleteLeavesFinishedRunAlone(t *testing.T) { assert.False(t, cancelled.Load(), "a run that already finished has nothing to cancel") } + +func TestJobRunOverrideChangeDescTriggerRemoved(t *testing.T) { + r := &ResourceJobRun{} + for _, tt := range []struct { + name string + path string + new any + action deployplan.ActionType + }{ + {"cleared on_bundle_deploy string", "lifecycle.triggers.on_bundle_deploy", "", deployplan.Skip}, + {"nil on_bundle_deploy", "lifecycle.triggers.on_bundle_deploy", nil, deployplan.Skip}, + {"rotated on_bundle_deploy", "lifecycle.triggers.on_bundle_deploy", "uuid", deployplan.Recreate}, + {"cleared on_file_change", "lifecycle.triggers.on_file_change", nil, deployplan.Skip}, + {"changed on_file_change map", "lifecycle.triggers.on_file_change", map[string]string{"a.txt": "h"}, deployplan.Recreate}, + {"cleared on_file_change child", "lifecycle.triggers.on_file_change['a.txt']", nil, deployplan.Recreate}, + {"cleared triggers parent", "lifecycle.triggers", nil, deployplan.Skip}, + {"changed triggers parent", "lifecycle.triggers", JobRunTriggersState{OnBundleDeploy: "uuid"}, deployplan.Recreate}, + {"result_state unchanged", "result_state", nil, deployplan.Recreate}, + } { + t.Run(tt.name, func(t *testing.T) { + change := &ChangeDesc{Action: deployplan.Recreate, New: tt.new} + require.NoError(t, r.OverrideChangeDesc(t.Context(), structpath.MustParsePath(tt.path), change, nil)) + assert.Equal(t, tt.action, change.Action) + }) + } +} diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index 1fd6b452415..6409d280bba 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -992,7 +992,7 @@ resources: If true, re-fire the run on every bundle deploy. Incompatible with lifecycle.prevent_destroy. "on_file_change": "description": |- - Path or glob under the sync root. Re-fire the run when a matched file's content hash changes, or when the set of matches appears or disappears. Incompatible with lifecycle.prevent_destroy. + Path or glob relative to the defining YAML file. It must resolve under the sync root. Re-fire the run when a matched file's content hash changes, or when the set of matches appears or disappears. Incompatible with lifecycle.prevent_destroy. "python_named_params": "description": |- PLACEHOLDER diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 9afa34882ef..b5856769778 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -1313,7 +1313,7 @@ "$ref": "#/$defs/bool" }, "on_file_change": { - "description": "Path or glob under the sync root. Re-fire the run when a matched file's content hash changes, or when the set of matches appears or disappears. Incompatible with lifecycle.prevent_destroy.", + "description": "Path or glob relative to the defining YAML file. It must resolve under the sync root. Re-fire the run when a matched file's content hash changes, or when the set of matches appears or disappears. Incompatible with lifecycle.prevent_destroy.", "$ref": "#/$defs/string" } }, From 7ecdacd8def079fe46c56278907fdbe75e51cf54 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Tue, 25 Aug 2026 08:30:03 +0000 Subject: [PATCH 71/75] job_runs: reject absolute on_file_change patterns filepath.Join rebases an absolute glob under the bundle root, so reject those up front. Key missing-file placeholders to the sync root, and skip a cleared trigger from path.String() instead of prefix matching. --- .../job_runs/on_file_change_errors/output.txt | 13 ++++++ .../job_runs/on_file_change_errors/script | 4 ++ .../mutator/resolve_job_run_file_triggers.go | 20 ++++++--- .../resolve_job_run_file_triggers_test.go | 22 ++++++++-- .../mutator/validate_job_run_triggers.go | 28 +++++++++---- .../mutator/validate_job_run_triggers_test.go | 13 +++++- bundle/config/resources/job_run.go | 1 + bundle/direct/dresources/job_run.go | 42 ++++--------------- bundle/phases/initialize.go | 3 +- 9 files changed, 93 insertions(+), 53 deletions(-) diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt index 12e4730d8d9..37ea391136b 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt @@ -76,3 +76,16 @@ Workspace: Path: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-errors/default Found 1 error + +=== an absolute pattern is rejected +>>> musterr [CLI] bundle validate +Error: lifecycle.triggers.on_file_change: pattern "/tmp/seed.txt" must be relative to the defining YAML file + in databricks.yml:24:29 + +Name: job-runs-on-file-change-errors +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-errors/default + +Found 1 error diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/script b/acceptance/bundle/resources/job_runs/on_file_change_errors/script index 4b6e4ead73b..86b10b70bf0 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_errors/script +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/script @@ -23,3 +23,7 @@ trace $CLI bundle validate title "** is not supported" update_file.py databricks.yml "missing/*.txt" "src/**" trace musterr $CLI bundle validate + +title "an absolute pattern is rejected" +update_file.py databricks.yml "src/**" "/tmp/seed.txt" +trace musterr $CLI bundle validate diff --git a/bundle/config/mutator/resolve_job_run_file_triggers.go b/bundle/config/mutator/resolve_job_run_file_triggers.go index 2b3f11a1a88..dc615f030c8 100644 --- a/bundle/config/mutator/resolve_job_run_file_triggers.go +++ b/bundle/config/mutator/resolve_job_run_file_triggers.go @@ -8,6 +8,7 @@ import ( "io" "maps" "os" + pathlib "path" "path/filepath" "strings" @@ -48,7 +49,7 @@ func (*resolveJobRunFileTriggers) Apply(ctx context.Context, b *bundle.Bundle) d continue } path := fmt.Sprintf("resources.job_runs.%s.lifecycle.triggers[%d].on_file_change", name, i) - hashes, d := resolveFileTrigger(b, path, strings.TrimSpace(*t.OnFileChange), syncable) + hashes, d := resolveFileTrigger(b, path, *t.OnFileChange, syncable) diags = diags.Extend(d) maps.Copy(out, hashes) } @@ -95,7 +96,6 @@ func syncableRelPaths(ctx context.Context, b *bundle.Bundle) (map[string]struct{ func resolveFileTrigger(b *bundle.Bundle, loc, pattern string, syncable map[string]struct{}) (map[string]string, diag.Diagnostics) { var diags diag.Diagnostics out := make(map[string]string) - localPattern := filepath.FromSlash(pattern) // filepath.Glob treats ** as two *, so doublestar-style patterns match less than expected. if strings.Contains(pattern, "**") { return out, diags.Append(diag.Diagnostic{ @@ -104,10 +104,19 @@ func resolveFileTrigger(b *bundle.Bundle, loc, pattern string, syncable map[stri Locations: b.Config.GetLocations(loc), }) } + // filepath.Join would otherwise rebase an absolute pattern under the bundle + // root (Join("/bundle", "/etc/passwd") is "/bundle/etc/passwd"). + if filepath.IsAbs(filepath.FromSlash(pattern)) || pathlib.IsAbs(pattern) { + return out, diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: pattern %q must be relative to the defining YAML file", pattern), + Locations: b.Config.GetLocations(loc), + }) + } // NormalizePaths has already rewritten YAML-relative globs to be bundle-root // relative. Join that onto the bundle root, then require the result stay // under the sync root (an ancestor of the bundle when sync.paths uses ..). - joined := filepath.Join(b.BundleRootPath, localPattern) + joined := filepath.Join(b.BundleRootPath, filepath.FromSlash(pattern)) relPattern, err := filepath.Rel(b.SyncRootPath, joined) if err != nil || !filepath.IsLocal(relPattern) { return out, diags.Append(diag.Diagnostic{ @@ -116,6 +125,7 @@ func resolveFileTrigger(b *bundle.Bundle, loc, pattern string, syncable map[stri Locations: b.Config.GetLocations(loc), }) } + key := filepath.ToSlash(relPattern) matches, err := filepath.Glob(joined) if err != nil { return out, diags.Append(diag.Diagnostic{ @@ -125,7 +135,7 @@ func resolveFileTrigger(b *bundle.Bundle, loc, pattern string, syncable map[stri }) } if len(matches) == 0 { - out[filepath.ToSlash(pattern)] = missingFileHash + out[key] = missingFileHash return out, diags.Append(diag.Diagnostic{ Severity: diag.Warning, Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: no files match %q", pattern), @@ -185,7 +195,7 @@ func resolveFileTrigger(b *bundle.Bundle, loc, pattern string, syncable map[stri if sawNonRegular { msg := fmt.Sprintf("lifecycle.triggers.on_file_change: pattern %q also matched directories, which are not hashed", pattern) if regularMatches == 0 { - out[filepath.ToSlash(pattern)] = missingFileHash + out[key] = missingFileHash msg = fmt.Sprintf("lifecycle.triggers.on_file_change: pattern %q matches only directories", pattern) } diags = diags.Append(diag.Diagnostic{ diff --git a/bundle/config/mutator/resolve_job_run_file_triggers_test.go b/bundle/config/mutator/resolve_job_run_file_triggers_test.go index 0630a4b35f6..9271589bb48 100644 --- a/bundle/config/mutator/resolve_job_run_file_triggers_test.go +++ b/bundle/config/mutator/resolve_job_run_file_triggers_test.go @@ -34,15 +34,29 @@ func TestResolveJobRunFileTriggers(t *testing.T) { assert.Equal(t, contentHash("world"), hashes["b.txt"]) }) - t.Run("trims pattern whitespace", func(t *testing.T) { + t.Run("rejects an absolute pattern", func(t *testing.T) { dir := t.TempDir() - require.NoError(t, os.WriteFile(filepath.Join(dir, "seed.txt"), []byte("v1"), 0o644)) - pattern := " seed.txt " + pattern := "/etc/passwd" b := bundleWithFileTrigger(dir, pattern) + diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) + require.True(t, diags.HasError()) + require.Equal(t, `lifecycle.triggers.on_file_change: pattern "/etc/passwd" must be relative to the defining YAML file`, diags[0].Summary) + assert.Empty(t, b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers) + }) + + t.Run("missing pattern is keyed relative to the sync root", func(t *testing.T) { + parent := t.TempDir() + bundleDir := filepath.Join(parent, "bundle") + require.NoError(t, os.Mkdir(bundleDir, 0o755)) + + pattern := "../missing.txt" + b := bundleWithFileTrigger(parent, pattern) + b.BundleRootPath = bundleDir + diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) require.False(t, diags.HasError()) - assert.Equal(t, contentHash("v1"), b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers["seed.txt"]) + assert.Equal(t, map[string]string{"missing.txt": ""}, b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers) }) t.Run("globs from the bundle root when the sync root is an ancestor", func(t *testing.T) { diff --git a/bundle/config/mutator/validate_job_run_triggers.go b/bundle/config/mutator/validate_job_run_triggers.go index c185d3b5781..fa31032c689 100644 --- a/bundle/config/mutator/validate_job_run_triggers.go +++ b/bundle/config/mutator/validate_job_run_triggers.go @@ -26,13 +26,14 @@ func (*validateJobRunTriggers) Apply(_ context.Context, b *bundle.Bundle) diag.D if jr == nil || jr.Lifecycle == nil { continue } - // Recreate-every-deploy cannot coexist with prevent_destroy. - if (jr.HasOnBundleDeploy() || jr.HasOnFileChange()) && jr.Lifecycle.PreventDestroy { - diags = diags.Append(diag.Diagnostic{ - Severity: diag.Error, - Summary: "lifecycle.triggers.on_bundle_deploy or on_file_change is incompatible with lifecycle.prevent_destroy", - Locations: b.Config.GetLocations(fmt.Sprintf("resources.job_runs.%s.lifecycle", name)), - }) + if jr.Lifecycle.PreventDestroy { + if summary := preventDestroyError(jr.HasOnBundleDeploy(), jr.HasOnFileChange()); summary != "" { + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: summary, + Locations: b.Config.GetLocations(fmt.Sprintf("resources.job_runs.%s.lifecycle", name)), + }) + } } for i, t := range jr.Lifecycle.Triggers { path := fmt.Sprintf("resources.job_runs.%s.lifecycle.triggers[%d]", name, i) @@ -70,3 +71,16 @@ func (*validateJobRunTriggers) Apply(_ context.Context, b *bundle.Bundle) diag.D } return diags } + +func preventDestroyError(onBundleDeploy, onFileChange bool) string { + switch { + case onBundleDeploy && onFileChange: + return "lifecycle.triggers.on_bundle_deploy and on_file_change are incompatible with lifecycle.prevent_destroy" + case onBundleDeploy: + return "lifecycle.triggers.on_bundle_deploy is incompatible with lifecycle.prevent_destroy" + case onFileChange: + return "lifecycle.triggers.on_file_change is incompatible with lifecycle.prevent_destroy" + default: + return "" + } +} diff --git a/bundle/config/mutator/validate_job_run_triggers_test.go b/bundle/config/mutator/validate_job_run_triggers_test.go index 7e8772d8d8b..07a7276a41c 100644 --- a/bundle/config/mutator/validate_job_run_triggers_test.go +++ b/bundle/config/mutator/validate_job_run_triggers_test.go @@ -84,7 +84,7 @@ func TestValidateJobRunTriggers(t *testing.T) { {OnBundleDeploy: &trueVal}, }, preventDestroy: true, - summary: "lifecycle.triggers.on_bundle_deploy or on_file_change is incompatible with lifecycle.prevent_destroy", + summary: "lifecycle.triggers.on_bundle_deploy is incompatible with lifecycle.prevent_destroy", }, { name: "on_file_change with prevent_destroy", @@ -92,7 +92,16 @@ func TestValidateJobRunTriggers(t *testing.T) { {OnFileChange: &fileChange}, }, preventDestroy: true, - summary: "lifecycle.triggers.on_bundle_deploy or on_file_change is incompatible with lifecycle.prevent_destroy", + summary: "lifecycle.triggers.on_file_change is incompatible with lifecycle.prevent_destroy", + }, + { + name: "both triggers with prevent_destroy", + triggers: []resources.JobRunTrigger{ + {OnFileChange: &fileChange}, + {OnBundleDeploy: &trueVal}, + }, + preventDestroy: true, + summary: "lifecycle.triggers.on_bundle_deploy and on_file_change are incompatible with lifecycle.prevent_destroy", }, { name: "prevent_destroy alone", diff --git a/bundle/config/resources/job_run.go b/bundle/config/resources/job_run.go index f4b16963453..512077e2738 100644 --- a/bundle/config/resources/job_run.go +++ b/bundle/config/resources/job_run.go @@ -46,6 +46,7 @@ func (r *JobRun) HasOnBundleDeploy() bool { return false } +// HasOnFileChange reports whether any trigger re-fires when matched files change. func (r *JobRun) HasOnFileChange() bool { if r.Lifecycle == nil { return false diff --git a/bundle/direct/dresources/job_run.go b/bundle/direct/dresources/job_run.go index 872720ffcd4..d7574bdc2ac 100644 --- a/bundle/direct/dresources/job_run.go +++ b/bundle/direct/dresources/job_run.go @@ -26,31 +26,6 @@ import ( // jobRunTimeout matches the timeout `bundle run` allows a run (bundle/run/job.go). const jobRunTimeout = 24 * time.Hour -// Parsed so HasPrefix catches parent and child paths; skip is only when the -// trigger itself is cleared, not when a file hash under on_file_change changes. -var ( - jobRunTriggersPath = structpath.MustParsePath("lifecycle.triggers") - jobRunOnBundleDeployPath = structpath.MustParsePath("lifecycle.triggers.on_bundle_deploy") - jobRunOnFileChangePath = structpath.MustParsePath("lifecycle.triggers.on_file_change") -) - -func samePath(a, b *structpath.PathNode) bool { - return a.HasPrefix(b) && b.HasPrefix(a) -} - -func jobRunTriggerRemoved(path *structpath.PathNode, change *ChangeDesc) bool { - switch { - case samePath(path, jobRunOnBundleDeployPath): - return change.New == nil || change.New == "" - case samePath(path, jobRunOnFileChangePath): - return change.New == nil - case samePath(path, jobRunTriggersPath): - return change.New == nil - default: - return false - } -} - // JobRunTriggersState is the persisted fingerprint of lifecycle.triggers. type JobRunTriggersState struct { // Fresh UUID each plan while armed so Old!=New forces recreate. @@ -411,14 +386,17 @@ func reportRunLine(ctx context.Context, runID int64, msg string) { // result_state either, so the lifecycle state is what tells the two apart. // Clearing a trigger skips its local-only fingerprint without re-firing the run. func (*ResourceJobRun) OverrideChangeDesc(_ context.Context, path *structpath.PathNode, change *ChangeDesc, remote *JobRunRemote) error { - if path.HasPrefix(jobRunTriggersPath) { - if jobRunTriggerRemoved(path, change) { + switch path.String() { + case "lifecycle.triggers.on_bundle_deploy": + if change.New == nil || change.New == "" { + change.Action = deployplan.Skip + change.Reason = "trigger removed" + } + case "lifecycle.triggers.on_file_change", "lifecycle.triggers": + if change.New == nil { change.Action = deployplan.Skip change.Reason = "trigger removed" } - return nil - } - switch path.String() { case "result_state": // The planner passes no remote state when the run could not be read. if remote == nil || runIsTerminal(remote.State.LifeCycleState) { @@ -426,10 +404,8 @@ func (*ResourceJobRun) OverrideChangeDesc(_ context.Context, path *structpath.Pa } change.Action = deployplan.Skip change.Reason = "run in progress" - return nil - default: - return nil } + return nil } // DoDelete deletes the run via jobs/runs/delete, on both destroy and the diff --git a/bundle/phases/initialize.go b/bundle/phases/initialize.go index 9eba6f466e0..d2ba4e76efb 100644 --- a/bundle/phases/initialize.go +++ b/bundle/phases/initialize.go @@ -26,8 +26,7 @@ import ( func Initialize(ctx context.Context, b *bundle.Bundle) { log.Info(ctx, "Phase: initialize") - bundle.ApplySeqContext( - ctx, b, + bundle.ApplySeqContext(ctx, b, // Reads (dynamic): resource.*.* // Checks that none of resources.. is nil. Raises error otherwise. validate.AllResourcesHaveValues(), From f3a44795bd89c6378e08e8c153044aa2d2c00398 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Tue, 25 Aug 2026 11:55:50 +0000 Subject: [PATCH 72/75] job_runs: collect the armed trigger names in one place preventDestroyError took one bool per trigger, so every new trigger widened the signature and risked a swapped argument. Have JobRun report the armed names instead. --- .../mutator/validate_job_run_triggers.go | 15 +++-------- bundle/config/resources/job_run.go | 26 ++++++++++++++----- bundle/config/resources/lifecycle.go | 1 + 3 files changed, 23 insertions(+), 19 deletions(-) diff --git a/bundle/config/mutator/validate_job_run_triggers.go b/bundle/config/mutator/validate_job_run_triggers.go index 7d9286f4a57..2c397583127 100644 --- a/bundle/config/mutator/validate_job_run_triggers.go +++ b/bundle/config/mutator/validate_job_run_triggers.go @@ -27,7 +27,7 @@ func (*validateJobRunTriggers) Apply(_ context.Context, b *bundle.Bundle) diag.D continue } if jr.Lifecycle.PreventDestroy { - if summary := preventDestroyError(jr.HasOnBundleDeploy(), jr.HasOnFileChange(), jr.HasOnValueChange()); summary != "" { + if summary := preventDestroyError(jr.ArmedTriggerNames()); summary != "" { diags = diags.Append(diag.Diagnostic{ Severity: diag.Error, Summary: summary, @@ -46,6 +46,7 @@ func (*validateJobRunTriggers) Apply(_ context.Context, b *bundle.Bundle) diag.D }) continue case 1: + // Exactly one key: valid. default: diags = diags.Append(diag.Diagnostic{ Severity: diag.Error, @@ -82,17 +83,7 @@ func (*validateJobRunTriggers) Apply(_ context.Context, b *bundle.Bundle) diag.D // preventDestroyError names the armed triggers that conflict with prevent_destroy, // or returns an empty string when none are armed. -func preventDestroyError(onBundleDeploy, onFileChange, onValueChange bool) string { - var names []string - if onBundleDeploy { - names = append(names, "on_bundle_deploy") - } - if onFileChange { - names = append(names, "on_file_change") - } - if onValueChange { - names = append(names, "on_value_change") - } +func preventDestroyError(names []string) string { switch len(names) { case 0: return "" diff --git a/bundle/config/resources/job_run.go b/bundle/config/resources/job_run.go index f1a7a55e6d3..c36347146c6 100644 --- a/bundle/config/resources/job_run.go +++ b/bundle/config/resources/job_run.go @@ -59,17 +59,29 @@ func (r *JobRun) HasOnFileChange() bool { return false } -// HasOnValueChange reports whether any trigger re-fires when a resolved value changes. -func (r *JobRun) HasOnValueChange() bool { +// ArmedTriggerNames returns the names of the trigger fields any entry arms, in +// schema order so that diagnostics naming them are stable. +func (r *JobRun) ArmedTriggerNames() []string { if r.Lifecycle == nil { - return false + return nil } + var onBundleDeploy, onFileChange, onValueChange bool for _, t := range r.Lifecycle.Triggers { - if t.OnValueChange != nil { - return true - } + onBundleDeploy = onBundleDeploy || (t.OnBundleDeploy != nil && *t.OnBundleDeploy) + onFileChange = onFileChange || t.OnFileChange != nil + onValueChange = onValueChange || t.OnValueChange != nil } - return false + var names []string + if onBundleDeploy { + names = append(names, "on_bundle_deploy") + } + if onFileChange { + names = append(names, "on_file_change") + } + if onValueChange { + names = append(names, "on_value_change") + } + return names } func (r *JobRun) UnmarshalJSON(b []byte) error { diff --git a/bundle/config/resources/lifecycle.go b/bundle/config/resources/lifecycle.go index 6153f9399ab..af0b98ab754 100644 --- a/bundle/config/resources/lifecycle.go +++ b/bundle/config/resources/lifecycle.go @@ -41,6 +41,7 @@ type JobRunTrigger struct { OnValueChange *string `json:"on_value_change,omitempty"` // interpolated expr; re-fire when the resolved value changes } +// ArmedCount returns the number of trigger fields set on this entry. func (t JobRunTrigger) ArmedCount() int { n := 0 if t.OnBundleDeploy != nil { From 2c1d501961827a1d829ad671e86c3f9ffdab2d0a Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Tue, 25 Aug 2026 11:56:35 +0000 Subject: [PATCH 73/75] structaccess: validate config reference paths strictly extractReferences validated a config path against the state type with the Terraform rule that treats [0] on a struct as a no-op. That let lifecycle.triggers[0].on_value_change match the flattened state map and register a reference that cannot resolve. ValidatePathStrict drops the tolerance for these native DABs paths. --- bundle/direct/bundle_plan.go | 6 ++++-- bundle/direct/bundle_plan_test.go | 28 ++++++++++++++++++++++++++ libs/structs/structaccess/typecheck.go | 18 +++++++++++++---- 3 files changed, 46 insertions(+), 6 deletions(-) diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index 11c34d5cdee..2206ac180a0 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -1007,7 +1007,6 @@ func (b *DeploymentBundle) makePlan(ctx context.Context, configRoot *config.Root } maps.Copy(refs, inputStructVar.Refs) - dresources.DropJobRunValueChangeConfigRefs(refs) var dependsOn []deployplan.DependsOnEntry for _, reference := range refs { @@ -1154,7 +1153,10 @@ func extractReferences(root dyn.Value, node string, stateType reflect.Type) (map // bundle:"readonly" field like volumes' volume_path — is dropped before deploy, so a // reference it carries cannot resolve into state and is not a dependency here. Such // references are still resolved earlier during initialize. - if structaccess.ValidatePath(stateType, fieldPath) == nil { + // Strict: config paths are native DABs paths, so tolerating Terraform's + // [0]-on-struct would let a config list path resolve onto a state field of + // a different shape (e.g. job_runs' lifecycle.triggers). + if structaccess.ValidatePathStrict(stateType, fieldPath) == nil { // Store the original string that contains references, not individual references. refs[fieldPath.String()] = ref.Str } diff --git a/bundle/direct/bundle_plan_test.go b/bundle/direct/bundle_plan_test.go index bf875680c2b..a50b8512248 100644 --- a/bundle/direct/bundle_plan_test.go +++ b/bundle/direct/bundle_plan_test.go @@ -2,6 +2,7 @@ package direct import ( "bytes" + "reflect" "testing" "github.com/databricks/cli/bundle/config/resources" @@ -77,6 +78,33 @@ resources: }, refs) } +func TestExtractReferences_DoesNotTreatConfigSliceAsStateStruct(t *testing.T) { + type triggersState struct { + OnValueChange map[string]string `json:"on_value_change,omitempty"` + } + type lifecycleState struct { + Triggers triggersState `json:"triggers"` + } + type state struct { + Lifecycle lifecycleState `json:"lifecycle"` + } + + const yml = ` +resources: + job_runs: + run: + lifecycle: + triggers: + - on_value_change: "${resources.jobs.watched.id}" +` + root, err := yamlloader.LoadYAML("test", bytes.NewBufferString(yml)) + require.NoError(t, err) + + refs, err := extractReferences(root, "resources.job_runs.run", reflect.TypeFor[*state]()) + require.NoError(t, err) + assert.Empty(t, refs) +} + func TestShouldSkipBackendDefault_ManagedPropertiesOnly(t *testing.T) { // Rules mirror the schemas backend_defaults in resources.yml, but the test is // deliberately self-contained so that edits to resources.yml don't break it. diff --git a/libs/structs/structaccess/typecheck.go b/libs/structs/structaccess/typecheck.go index 7147fa0f435..d4bf129b71e 100644 --- a/libs/structs/structaccess/typecheck.go +++ b/libs/structs/structaccess/typecheck.go @@ -34,6 +34,16 @@ func ValidatePath(t reflect.Type, path *structpath.PathNode) error { return ValidatePattern(t, (*structpath.PatternNode)(path)) } +// ValidatePathStrict is ValidatePath without the Terraform compatibility that +// treats [0] on a struct as a no-op. Use it for paths that are native to DABs, +// where indexing a struct is always a mismatch. +func ValidatePathStrict(t reflect.Type, path *structpath.PathNode) error { + if path.IsRoot() { + return nil + } + return validateNodeSlice(t, (*structpath.PatternNode)(path).AsSlice(), false) +} + // ValidatePattern reports whether the given pattern path is valid for the provided type. // It returns nil if the path resolves fully, or an error indicating where resolution failed. // Patterns may include wildcards ([*] and .*). @@ -41,11 +51,11 @@ func ValidatePattern(t reflect.Type, path *structpath.PatternNode) error { if path.IsRoot() { return nil } - return validateNodeSlice(t, path.AsSlice()) + return validateNodeSlice(t, path.AsSlice(), true) } -// validateNodeSlice is the implementation for ValidatePattern. -func validateNodeSlice(t reflect.Type, nodes []*structpath.PatternNode) error { +// validateNodeSlice is the implementation for ValidatePattern and ValidatePathStrict. +func validateNodeSlice(t reflect.Type, nodes []*structpath.PatternNode, allowStructIndex bool) error { cur := t for _, node := range nodes { // Always dereference pointers at the type level. @@ -64,7 +74,7 @@ func validateNodeSlice(t reflect.Type, nodes []*structpath.PatternNode) error { if kind != reflect.Slice && kind != reflect.Array { // Terraform represents single-block fields as lists and uses [0] to access them. // Treat [0] on a struct as a no-op so TF-style paths work against DABs structs. - if idx == 0 && kind == reflect.Struct { + if allowStructIndex && idx == 0 && kind == reflect.Struct { continue } return fmt.Errorf("%s: cannot index %s", node.String(), kind) From c12313ff73813b37602ff7b83dfff0a0009594d4 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Tue, 25 Aug 2026 11:57:03 +0000 Subject: [PATCH 74/75] job_runs: fingerprint the values watched by on_value_change State kept each watched value verbatim and keyed it by the expression, so a long or sensitive resolved value ended up in the plan and in state. Key the entries by the value itself and hash it once it grows past the digest, normalizing through structvar as soon as the last reference resolves. With the expression gone from the key, dropping an entry can only mean the config stopped watching that value, so it skips instead of re-firing the run. DropJobRunValueChangeConfigRefs goes away with the strict path validation. --- .../bundles/job-runs-on-value-change.md | 2 +- .../job_runs/on_value_change/databricks.yml | 3 + .../job_runs/on_value_change/output.txt | 27 +++++- .../resources/job_runs/on_value_change/script | 14 ++- .../job_runs/on_value_change_ref/output.txt | 33 ++++++- .../job_runs/on_value_change_ref/script | 9 ++ bundle/direct/dresources/job_run.go | 85 +++++++++++++++---- bundle/direct/dresources/job_run_test.go | 53 ++++++++++-- bundle/internal/schema/annotations.yml | 2 +- bundle/schema/jsonschema.json | 2 +- libs/structs/structvar/structvar.go | 10 +++ 11 files changed, 207 insertions(+), 33 deletions(-) diff --git a/.nextchanges/bundles/job-runs-on-value-change.md b/.nextchanges/bundles/job-runs-on-value-change.md index 94d0b06290f..53ec0227df0 100644 --- a/.nextchanges/bundles/job-runs-on-value-change.md +++ b/.nextchanges/bundles/job-runs-on-value-change.md @@ -1 +1 @@ -direct: `resources.job_runs` can set `lifecycle.triggers.on_value_change` to an interpolated expression (for example `${resources.jobs.foo.id}`) to re-fire the run when that value changes. Removing the trigger does not recreate the existing run. +direct: `resources.job_runs` can set `lifecycle.triggers.on_value_change` to an interpolated expression (for example `${resources.jobs.foo.id}`) to re-fire the run when that value changes. Long resolved values are recorded as a SHA-256 hash. Removing the trigger does not recreate the existing run. diff --git a/acceptance/bundle/resources/job_runs/on_value_change/databricks.yml b/acceptance/bundle/resources/job_runs/on_value_change/databricks.yml index b0ed0322b9c..cbb68d9354a 100644 --- a/acceptance/bundle/resources/job_runs/on_value_change/databricks.yml +++ b/acceptance/bundle/resources/job_runs/on_value_change/databricks.yml @@ -6,6 +6,8 @@ variables: default: v1 other: default: a1 + long: + default: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa resources: jobs: @@ -23,3 +25,4 @@ resources: triggers: - on_value_change: ${var.stamp} - on_value_change: ${var.other} + - on_value_change: ${var.long} diff --git a/acceptance/bundle/resources/job_runs/on_value_change/output.txt b/acceptance/bundle/resources/job_runs/on_value_change/output.txt index 99683d5ef57..d09d9736114 100644 --- a/acceptance/bundle/resources/job_runs/on_value_change/output.txt +++ b/acceptance/bundle/resources/job_runs/on_value_change/output.txt @@ -22,6 +22,14 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged } } +=== long values are stored as hashes +>>> print_state.py +{ + "a1": "a1", + "sha256:d66304b6180365e47c858f6c84d3da065caf4b3350c9f45277a1af82e3dbb055": "sha256:d66304b6180365e47c858f6c84d3da065caf4b3350c9f45277a1af82e3dbb055", + "v1": "v1" +} + === redeploy with unchanged values plans nothing >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged @@ -46,16 +54,18 @@ Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged "reason": "immutable", "old": { "a1": "a1", + "sha256:d66304b6180365e47c858f6c84d3da065caf4b3350c9f45277a1af82e3dbb055": "sha256:d66304b6180365e47c858f6c84d3da065caf4b3350c9f45277a1af82e3dbb055", "v1": "v1" }, "new": { "a1": "a1", + "sha256:d66304b6180365e47c858f6c84d3da065caf4b3350c9f45277a1af82e3dbb055": "sha256:d66304b6180365e47c858f6c84d3da065caf4b3350c9f45277a1af82e3dbb055", "v2": "v2" } }, "lifecycle.triggers.on_value_change['v1']": { - "action": "recreate", - "reason": "immutable", + "action": "skip", + "reason": "trigger removed", "old": "v1" }, "lifecycle.triggers.on_value_change['v2']": { @@ -98,7 +108,18 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged } } -=== removing on_value_change is unchanged and does not run +=== removing one on_value_change is unchanged and does not run +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-value-change/default/files... +Files: 3 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 2 unchanged + +>>> print_requests.py //jobs/run-now + +=== removing the last on_value_change is unchanged and does not run >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged diff --git a/acceptance/bundle/resources/job_runs/on_value_change/script b/acceptance/bundle/resources/job_runs/on_value_change/script index df445ffa1d5..39caf00182d 100644 --- a/acceptance/bundle/resources/job_runs/on_value_change/script +++ b/acceptance/bundle/resources/job_runs/on_value_change/script @@ -14,6 +14,9 @@ trace read_id.py my_job read_id.py my_run > /dev/null trace print_requests.py //jobs/run-now +title "long values are stored as hashes" +trace print_state.py | jq '.state["resources.job_runs.my_run"].state.lifecycle.triggers.on_value_change' + title "redeploy with unchanged values plans nothing" trace $CLI bundle plan trace $CLI bundle deploy @@ -28,11 +31,18 @@ read_id.py my_run > /dev/null trace print_requests.py --keep //jobs/runs/delete trace print_requests.py //jobs/run-now -title "removing on_value_change is unchanged and does not run" +title "removing one on_value_change is unchanged and does not run" +update_file.py databricks.yml ' - on_value_change: ${var.other} +' '' +trace $CLI bundle plan +trace $CLI bundle deploy +trace print_requests.py //jobs/run-now + +title "removing the last on_value_change is unchanged and does not run" update_file.py databricks.yml ' lifecycle: triggers: - on_value_change: ${var.stamp} - - on_value_change: ${var.other} + - on_value_change: ${var.long} ' '' trace $CLI bundle plan trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/job_runs/on_value_change_ref/output.txt b/acceptance/bundle/resources/job_runs/on_value_change_ref/output.txt index f1521167db3..825f880c987 100644 --- a/acceptance/bundle/resources/job_runs/on_value_change_ref/output.txt +++ b/acceptance/bundle/resources/job_runs/on_value_change_ref/output.txt @@ -50,7 +50,7 @@ Resources: 4 created, 0 changed, 0 deleted, 0 unchanged === state records the concatenated resolved ids >>> print_state.py { - "${resources.jobs.other.id}-${resources.jobs.extra.id}": "[OTHER_ID]-[EXTRA_ID]" + "[OTHER_ID]-[EXTRA_ID]": "[OTHER_ID]-[EXTRA_ID]" } === redeploy with unchanged ids plans nothing @@ -64,6 +64,37 @@ Resources: 0 created, 0 changed, 0 deleted, 4 unchanged >>> print_requests.py //jobs/run-now +=== replacing a referenced job re-fires the run +>>> [CLI] jobs delete [OTHER_ID] + +>>> [CLI] bundle plan +recreate job_runs.my_run +create jobs.other + +Plan: 2 to add, 0 to change, 1 to delete, 2 unchanged + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-value-change-ref/default/files... +Output from job_runs.my_run: id=[MY_RUN_ID_2]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_2]?o=[NUMID] +Output from job_runs.my_run: id=[MY_RUN_ID_2]: SUCCESS +Recreated job_runs.my_run +Created jobs.other +Files: 2 uploaded, 0 deleted +Resources: 2 created, 0 changed, 1 deleted, 2 unchanged + +>>> read_id.py other +[OTHER_ID_3] + +>>> print_requests.py //jobs/run-now +{ + "method": "POST", + "path": "/api/2.2/jobs/run-now", + "body": { + "idempotency_token": "[UUID]", + "job_id": [MY_JOB_ID] + } +} + >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: delete resources.job_runs.my_run diff --git a/acceptance/bundle/resources/job_runs/on_value_change_ref/script b/acceptance/bundle/resources/job_runs/on_value_change_ref/script index 7b0c981b10f..707e8d8e839 100644 --- a/acceptance/bundle/resources/job_runs/on_value_change_ref/script +++ b/acceptance/bundle/resources/job_runs/on_value_change_ref/script @@ -27,3 +27,12 @@ title "redeploy with unchanged ids plans nothing" trace $CLI bundle plan trace $CLI bundle deploy trace print_requests.py //jobs/run-now + +title "replacing a referenced job re-fires the run" +other_id=$(read_id.py other) +trace $CLI jobs delete "$other_id" +trace $CLI bundle plan +trace $CLI bundle deploy +trace read_id.py other +read_id.py my_run > /dev/null +trace print_requests.py //jobs/run-now diff --git a/bundle/direct/dresources/job_run.go b/bundle/direct/dresources/job_run.go index ea58353e525..373e9323ac6 100644 --- a/bundle/direct/dresources/job_run.go +++ b/bundle/direct/dresources/job_run.go @@ -3,6 +3,8 @@ package dresources import ( "cmp" "context" + "crypto/sha256" + "encoding/hex" "errors" "fmt" "slices" @@ -29,13 +31,18 @@ import ( // jobRunTimeout matches the timeout `bundle run` allows a run (bundle/run/job.go). const jobRunTimeout = 24 * time.Hour +const ( + jobRunValueHashPrefix = "sha256:" + jobRunValueHashLength = len(jobRunValueHashPrefix) + sha256.Size*2 +) + // JobRunTriggersState is the persisted fingerprint of lifecycle.triggers. type JobRunTriggersState struct { // Fresh UUID each plan while armed so Old!=New forces recreate. OnBundleDeploy string `json:"on_bundle_deploy,omitempty"` // Content hashes from ResolveJobRunFileTriggers; any change recreates. OnFileChange map[string]string `json:"on_file_change,omitempty"` - // Resolved expression per watched value; any change recreates. + // Resolved value fingerprints; any change recreates. OnValueChange map[string]string `json:"on_value_change,omitempty"` } @@ -122,6 +129,7 @@ func (*ResourceJobRun) PrepareState(input *resources.JobRun) *JobRunState { if values := jobRunValueChangeState(input); len(values) > 0 { state.Lifecycle.Triggers.OnValueChange = values } + state.NormalizeAfterResolve() return state } @@ -136,9 +144,6 @@ func (*ResourceJobRun) PrepareInputConfig(input *resources.JobRun, _ string) (*s path := structpath.NewStringKey(structpath.MustParsePath("lifecycle.triggers.on_value_change"), expr) refs[path.String()] = expr } - if len(refs) == 0 { - refs = nil - } return &structvar.StructVar{Value: input, Refs: refs}, nil } @@ -152,9 +157,6 @@ func jobRunValueChangeState(input *resources.JobRun) map[string]string { continue } expr := strings.TrimSpace(*t.OnValueChange) - if expr == "" { - continue - } out[expr] = expr } if len(out) == 0 { @@ -163,15 +165,33 @@ func jobRunValueChangeState(input *resources.JobRun) map[string]string { return out } -// DropJobRunValueChangeConfigRefs drops lifecycle.triggers[N].on_value_change. -// ExtractReferences treats [0] on a struct as a no-op, so that path is the -// wrapper and cannot hold the resolved id. -func DropJobRunValueChangeConfigRefs(refs map[string]string) { - for k := range refs { - if strings.Contains(k, ".triggers[") && strings.HasSuffix(k, "].on_value_change") { - delete(refs, k) +// NormalizeAfterResolve rekeys watched values by their fingerprint, so that the +// identity of a trigger is the value it resolved to and not the expression text. +// An entry still holding a reference is left alone until it fully resolves. +func (s *JobRunState) NormalizeAfterResolve() { + values := s.Lifecycle.Triggers.OnValueChange + if len(values) == 0 { + return + } + normalized := make(map[string]string, len(values)) + for key, value := range values { + if dynvar.ContainsVariableReference(value) { + normalized[key] = value + continue } + fingerprint := compactJobRunValue(value) + normalized[fingerprint] = fingerprint + } + s.Lifecycle.Triggers.OnValueChange = normalized +} + +// compactJobRunValue hashes a value only when the digest is shorter than it. +func compactJobRunValue(value string) string { + if len(value) <= jobRunValueHashLength { + return value } + sum := sha256.Sum256([]byte(value)) + return jobRunValueHashPrefix + hex.EncodeToString(sum[:]) } // makeJobRunRemote maps the GetRun response into the RunNow-shaped remote: GET @@ -444,11 +464,20 @@ func (*ResourceJobRun) OverrideChangeDesc(_ context.Context, path *structpath.Pa change.Action = deployplan.Skip change.Reason = "trigger removed" } - case "lifecycle.triggers.on_file_change", "lifecycle.triggers.on_value_change": + case "lifecycle.triggers.on_file_change": + // Only a cleared trigger skips: a file dropping out of the map means the + // match disappeared, which is a real change. if change.New == nil { change.Action = deployplan.Skip change.Reason = "trigger removed" } + case "lifecycle.triggers.on_value_change": + // A watched value always resolves to something, so entries disappear only + // when the config stops watching them. + if valueTriggersOnlyRemoved(change.Old, change.New) { + change.Action = deployplan.Skip + change.Reason = "trigger removed" + } case "result_state": // The planner passes no remote state when the run could not be read. if remote == nil || runIsTerminal(remote.State.LifeCycleState) { @@ -456,10 +485,36 @@ func (*ResourceJobRun) OverrideChangeDesc(_ context.Context, path *structpath.Pa } change.Action = deployplan.Skip change.Reason = "run in progress" + default: + // The per-entry change that accompanies the map above. + parent := path.Parent() + if change.New == nil && parent != nil && parent.String() == "lifecycle.triggers.on_value_change" { + change.Action = deployplan.Skip + change.Reason = "trigger removed" + } } return nil } +// valueTriggersOnlyRemoved reports whether the remaining fingerprints are a +// strict subset of the old ones, i.e. entries were dropped and none added. +func valueTriggersOnlyRemoved(oldValue, newValue any) bool { + if newValue == nil { + return true + } + oldMap, okOld := oldValue.(map[string]string) + newMap, okNew := newValue.(map[string]string) + if !okOld || !okNew { + return false + } + for key, value := range newMap { + if oldMap[key] != value { + return false + } + } + return len(newMap) < len(oldMap) +} + // DoDelete deletes the run via jobs/runs/delete, on both destroy and the // recreate path. The API rejects a still-active run, which an interrupted wait // leaves behind, so cancel it first. diff --git a/bundle/direct/dresources/job_run_test.go b/bundle/direct/dresources/job_run_test.go index f9f21999316..9ee8eab1aa7 100644 --- a/bundle/direct/dresources/job_run_test.go +++ b/bundle/direct/dresources/job_run_test.go @@ -10,8 +10,10 @@ import ( "testing" "time" + "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" "github.com/databricks/cli/libs/testserver" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/service/jobs" @@ -346,6 +348,33 @@ func TestJobRunCreateSendsAFreshIdempotencyToken(t *testing.T) { assert.Empty(t, config.IdempotencyToken) } +func TestCompactJobRunValue(t *testing.T) { + assert.Equal(t, strings.Repeat("a", jobRunValueHashLength), compactJobRunValue(strings.Repeat("a", jobRunValueHashLength))) + assert.Equal( + t, + "sha256:d66304b6180365e47c858f6c84d3da065caf4b3350c9f45277a1af82e3dbb055", + compactJobRunValue(strings.Repeat("a", jobRunValueHashLength+1)), + ) +} + +func TestJobRunValueChangeStateNormalizesAfterAllReferencesResolve(t *testing.T) { + expr := "${resources.jobs.other.id}-${resources.jobs.extra.id}" + var trigger resources.JobRunTrigger + trigger.OnValueChange = &expr + var input resources.JobRun + input.Lifecycle = &resources.JobRunLifecycle{} + input.Lifecycle.Triggers = []resources.JobRunTrigger{trigger} + state := (&ResourceJobRun{}).PrepareState(&input) + path := structpath.NewStringKey(structpath.MustParsePath("lifecycle.triggers.on_value_change"), expr) + sv := structvar.NewStructVar(state, map[string]string{path.String(): expr}) + + require.NoError(t, sv.ResolveRef("${resources.jobs.other.id}", int64(123))) + assert.Equal(t, map[string]string{expr: "123-${resources.jobs.extra.id}"}, state.Lifecycle.Triggers.OnValueChange) + + require.NoError(t, sv.ResolveRef("${resources.jobs.extra.id}", int64(456))) + assert.Equal(t, map[string]string{"123-456": "123-456"}, state.Lifecycle.Triggers.OnValueChange) +} + func TestJobRunDeleteLeavesFinishedRunAlone(t *testing.T) { var cancelled atomic.Bool server := testserver.New(t) @@ -374,21 +403,27 @@ func TestJobRunOverrideChangeDescTriggerRemoved(t *testing.T) { for _, tt := range []struct { name string path string + old any new any action deployplan.ActionType }{ - {"cleared on_bundle_deploy string", "lifecycle.triggers.on_bundle_deploy", "", deployplan.Skip}, - {"nil on_bundle_deploy", "lifecycle.triggers.on_bundle_deploy", nil, deployplan.Skip}, - {"rotated on_bundle_deploy", "lifecycle.triggers.on_bundle_deploy", "uuid", deployplan.Recreate}, - {"cleared on_file_change", "lifecycle.triggers.on_file_change", nil, deployplan.Skip}, - {"changed on_file_change map", "lifecycle.triggers.on_file_change", map[string]string{"a.txt": "h"}, deployplan.Recreate}, + {"cleared on_bundle_deploy string", "lifecycle.triggers.on_bundle_deploy", "old", "", deployplan.Skip}, + {"nil on_bundle_deploy", "lifecycle.triggers.on_bundle_deploy", "old", nil, deployplan.Skip}, + {"rotated on_bundle_deploy", "lifecycle.triggers.on_bundle_deploy", "old", "uuid", deployplan.Recreate}, + {"cleared on_file_change", "lifecycle.triggers.on_file_change", map[string]string{"a.txt": "h"}, nil, deployplan.Skip}, + {"changed on_file_change map", "lifecycle.triggers.on_file_change", map[string]string{"a.txt": "h"}, map[string]string{"a.txt": "new"}, deployplan.Recreate}, // A file dropping out of the map is a real change, so the skip must not - // extend to paths below on_file_change. - {"cleared on_file_change child", "lifecycle.triggers.on_file_change['a.txt']", nil, deployplan.Recreate}, - {"result_state with unreadable remote", "result_state", nil, deployplan.Recreate}, + // extend to on_file_change entries. + {"removed one on_file_change", "lifecycle.triggers.on_file_change", map[string]string{"a.txt": "h", "b.txt": "h"}, map[string]string{"a.txt": "h"}, deployplan.Recreate}, + {"cleared on_file_change child", "lifecycle.triggers.on_file_change['a.txt']", "h", nil, deployplan.Recreate}, + {"cleared on_value_change", "lifecycle.triggers.on_value_change", map[string]string{"a": "a"}, nil, deployplan.Skip}, + {"removed one on_value_change", "lifecycle.triggers.on_value_change", map[string]string{"a": "a", "b": "b"}, map[string]string{"b": "b"}, deployplan.Skip}, + {"changed on_value_change", "lifecycle.triggers.on_value_change", map[string]string{"a": "a"}, map[string]string{"b": "b"}, deployplan.Recreate}, + {"cleared on_value_change child", "lifecycle.triggers.on_value_change['b']", "b", nil, deployplan.Skip}, + {"result_state with unreadable remote", "result_state", "", nil, deployplan.Recreate}, } { t.Run(tt.name, func(t *testing.T) { - change := &ChangeDesc{Action: deployplan.Recreate, New: tt.new} + change := &ChangeDesc{Action: deployplan.Recreate, Old: tt.old, New: tt.new} require.NoError(t, r.OverrideChangeDesc(t.Context(), structpath.MustParsePath(tt.path), change, nil)) assert.Equal(t, tt.action, change.Action) }) diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index ff84a1387b8..8f788ac35ad 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -1022,7 +1022,7 @@ resources: Path or glob relative to the defining YAML file. It must resolve under the sync root. Re-fire the run when a matched file's content hash changes, or when the set of matches appears or disappears. Only files the bundle syncs are hashed, so .gitignore and sync.exclude apply. Use * to match a single directory level; ** is not supported. Incompatible with lifecycle.prevent_destroy. "on_value_change": "description": |- - Interpolated expression. Re-fire the run when the resolved value changes. Resource references in the expression become deployment dependencies. Incompatible with lifecycle.prevent_destroy. + Interpolated expression. Re-fire the run when the resolved value changes. Resource references in the expression become deployment dependencies. Long resolved values are recorded as a SHA-256 hash instead of the value itself. Incompatible with lifecycle.prevent_destroy. "python_named_params": "description": |- PLACEHOLDER diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 22c6180c8ad..2feec7b2142 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -1400,7 +1400,7 @@ "$ref": "#/$defs/string" }, "on_value_change": { - "description": "Interpolated expression. Re-fire the run when the resolved value changes. Resource references in the expression become deployment dependencies. Incompatible with lifecycle.prevent_destroy.", + "description": "Interpolated expression. Re-fire the run when the resolved value changes. Resource references in the expression become deployment dependencies. Long resolved values are recorded as a SHA-256 hash instead of the value itself. Incompatible with lifecycle.prevent_destroy.", "$ref": "#/$defs/string" } }, diff --git a/libs/structs/structvar/structvar.go b/libs/structs/structvar/structvar.go index b60f8c15ee7..d6c42d4e740 100644 --- a/libs/structs/structvar/structvar.go +++ b/libs/structs/structvar/structvar.go @@ -22,6 +22,12 @@ type StructVar struct { Refs map[string]string `json:"vars,omitempty"` } +// ReferenceNormalizer lets a value re-derive fields that depend on resolved +// references. ResolveRef calls it after each substitution. +type ReferenceNormalizer interface { + NormalizeAfterResolve() +} + // StructVarJSON is the serialized form of StructVar for persisting in plan files. type StructVarJSON struct { Value json.RawMessage `json:"value"` @@ -123,6 +129,10 @@ func (sv *StructVar) ResolveRef(reference string, value any) error { return ErrNotFound } + if normalizer, ok := sv.Value.(ReferenceNormalizer); ok { + normalizer.NormalizeAfterResolve() + } + return nil } From 4f50a398c4bc821525b84823ac29311327dacf94 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Tue, 25 Aug 2026 12:49:30 +0000 Subject: [PATCH 75/75] job_runs: store on_value_change fingerprints as a list Keying state by value treated two watches converging as a removal, so the run did not re-fire. A list in trigger order distinguishes a changed fingerprint from a dropped watch. --- acceptance/bundle/refschema/out.fields.txt | 4 +- .../job_runs/on_value_change/output.txt | 95 ++++++++++++++----- .../resources/job_runs/on_value_change/script | 9 ++ .../job_runs/on_value_change_ref/output.txt | 6 +- bundle/direct/bundle_plan_test.go | 2 +- bundle/direct/dresources/job_run.go | 69 +++++++------- bundle/direct/dresources/job_run_test.go | 17 ++-- 7 files changed, 128 insertions(+), 74 deletions(-) diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index 6d665469c6f..dd5033ab682 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -914,8 +914,8 @@ resources.job_runs.*.lifecycle.triggers dresources.JobRunTriggersState STATE resources.job_runs.*.lifecycle.triggers.on_bundle_deploy string STATE resources.job_runs.*.lifecycle.triggers.on_file_change map[string]string STATE resources.job_runs.*.lifecycle.triggers.on_file_change.* string STATE -resources.job_runs.*.lifecycle.triggers.on_value_change map[string]string STATE -resources.job_runs.*.lifecycle.triggers.on_value_change.* string STATE +resources.job_runs.*.lifecycle.triggers.on_value_change []string STATE +resources.job_runs.*.lifecycle.triggers.on_value_change[*] string STATE resources.job_runs.*.lifecycle.triggers[*] resources.JobRunTrigger INPUT resources.job_runs.*.lifecycle.triggers[*].on_bundle_deploy *bool INPUT resources.job_runs.*.lifecycle.triggers[*].on_file_change *string INPUT diff --git a/acceptance/bundle/resources/job_runs/on_value_change/output.txt b/acceptance/bundle/resources/job_runs/on_value_change/output.txt index d09d9736114..fba1f6081f3 100644 --- a/acceptance/bundle/resources/job_runs/on_value_change/output.txt +++ b/acceptance/bundle/resources/job_runs/on_value_change/output.txt @@ -24,11 +24,11 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged === long values are stored as hashes >>> print_state.py -{ - "a1": "a1", - "sha256:d66304b6180365e47c858f6c84d3da065caf4b3350c9f45277a1af82e3dbb055": "sha256:d66304b6180365e47c858f6c84d3da065caf4b3350c9f45277a1af82e3dbb055", - "v1": "v1" -} +[ + "v1", + "a1", + "sha256:d66304b6180365e47c858f6c84d3da065caf4b3350c9f45277a1af82e3dbb055" +] === redeploy with unchanged values plans nothing >>> [CLI] bundle plan @@ -52,26 +52,73 @@ Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged "lifecycle.triggers.on_value_change": { "action": "recreate", "reason": "immutable", - "old": { - "a1": "a1", - "sha256:d66304b6180365e47c858f6c84d3da065caf4b3350c9f45277a1af82e3dbb055": "sha256:d66304b6180365e47c858f6c84d3da065caf4b3350c9f45277a1af82e3dbb055", - "v1": "v1" - }, - "new": { - "a1": "a1", - "sha256:d66304b6180365e47c858f6c84d3da065caf4b3350c9f45277a1af82e3dbb055": "sha256:d66304b6180365e47c858f6c84d3da065caf4b3350c9f45277a1af82e3dbb055", - "v2": "v2" - } + "old": [ + "v1", + "a1", + "sha256:d66304b6180365e47c858f6c84d3da065caf4b3350c9f45277a1af82e3dbb055" + ], + "new": [ + "v2", + "a1", + "sha256:d66304b6180365e47c858f6c84d3da065caf4b3350c9f45277a1af82e3dbb055" + ] }, - "lifecycle.triggers.on_value_change['v1']": { + "result_state": { "action": "skip", - "reason": "trigger removed", - "old": "v1" - }, - "lifecycle.triggers.on_value_change['v2']": { + "reason": "remote_already_set", + "new": "SUCCESS", + "remote": "SUCCESS" + } +} + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-value-change/default/files... +Output from job_runs.my_run: id=[MY_RUN_ID_2]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_2]?o=[NUMID] +Output from job_runs.my_run: id=[MY_RUN_ID_2]: SUCCESS +Recreated job_runs.my_run +Files: 3 uploaded, 0 deleted +Resources: 1 created, 0 changed, 1 deleted, 1 unchanged + +>>> print_requests.py --keep //jobs/runs/delete +{ + "method": "POST", + "path": "/api/2.2/jobs/runs/delete", + "body": { + "run_id": [MY_RUN_ID] + } +} + +>>> print_requests.py //jobs/run-now +{ + "method": "POST", + "path": "/api/2.2/jobs/run-now", + "body": { + "idempotency_token": "[UUID]", + "job_id": [MY_JOB_ID] + } +} + +=== converging two values re-fires +>>> [CLI] bundle plan +recreate job_runs.my_run + +Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged + +>>> changes +{ + "lifecycle.triggers.on_value_change": { "action": "recreate", "reason": "immutable", - "new": "v2" + "old": [ + "v2", + "a1", + "sha256:d66304b6180365e47c858f6c84d3da065caf4b3350c9f45277a1af82e3dbb055" + ], + "new": [ + "v2", + "v2", + "sha256:d66304b6180365e47c858f6c84d3da065caf4b3350c9f45277a1af82e3dbb055" + ] }, "result_state": { "action": "skip", @@ -83,8 +130,8 @@ Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-value-change/default/files... -Output from job_runs.my_run: id=[MY_RUN_ID_2]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_2]?o=[NUMID] -Output from job_runs.my_run: id=[MY_RUN_ID_2]: SUCCESS +Output from job_runs.my_run: id=[MY_RUN_ID_3]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_3]?o=[NUMID] +Output from job_runs.my_run: id=[MY_RUN_ID_3]: SUCCESS Recreated job_runs.my_run Files: 3 uploaded, 0 deleted Resources: 1 created, 0 changed, 1 deleted, 1 unchanged @@ -94,7 +141,7 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged "method": "POST", "path": "/api/2.2/jobs/runs/delete", "body": { - "run_id": [MY_RUN_ID] + "run_id": [MY_RUN_ID_2] } } diff --git a/acceptance/bundle/resources/job_runs/on_value_change/script b/acceptance/bundle/resources/job_runs/on_value_change/script index 39caf00182d..f60be7c9f08 100644 --- a/acceptance/bundle/resources/job_runs/on_value_change/script +++ b/acceptance/bundle/resources/job_runs/on_value_change/script @@ -31,6 +31,15 @@ read_id.py my_run > /dev/null trace print_requests.py --keep //jobs/runs/delete trace print_requests.py //jobs/run-now +title "converging two values re-fires" +update_file.py databricks.yml "default: a1" "default: v2" +trace $CLI bundle plan +trace changes +trace $CLI bundle deploy +read_id.py my_run > /dev/null +trace print_requests.py --keep //jobs/runs/delete +trace print_requests.py //jobs/run-now + title "removing one on_value_change is unchanged and does not run" update_file.py databricks.yml ' - on_value_change: ${var.other} ' '' diff --git a/acceptance/bundle/resources/job_runs/on_value_change_ref/output.txt b/acceptance/bundle/resources/job_runs/on_value_change_ref/output.txt index 825f880c987..70349876476 100644 --- a/acceptance/bundle/resources/job_runs/on_value_change_ref/output.txt +++ b/acceptance/bundle/resources/job_runs/on_value_change_ref/output.txt @@ -49,9 +49,9 @@ Resources: 4 created, 0 changed, 0 deleted, 0 unchanged === state records the concatenated resolved ids >>> print_state.py -{ - "[OTHER_ID]-[EXTRA_ID]": "[OTHER_ID]-[EXTRA_ID]" -} +[ + "[OTHER_ID]-[EXTRA_ID]" +] === redeploy with unchanged ids plans nothing >>> [CLI] bundle plan diff --git a/bundle/direct/bundle_plan_test.go b/bundle/direct/bundle_plan_test.go index a50b8512248..623a29376da 100644 --- a/bundle/direct/bundle_plan_test.go +++ b/bundle/direct/bundle_plan_test.go @@ -80,7 +80,7 @@ resources: func TestExtractReferences_DoesNotTreatConfigSliceAsStateStruct(t *testing.T) { type triggersState struct { - OnValueChange map[string]string `json:"on_value_change,omitempty"` + OnValueChange []string `json:"on_value_change,omitempty"` } type lifecycleState struct { Triggers triggersState `json:"triggers"` diff --git a/bundle/direct/dresources/job_run.go b/bundle/direct/dresources/job_run.go index 373e9323ac6..42e20f8967b 100644 --- a/bundle/direct/dresources/job_run.go +++ b/bundle/direct/dresources/job_run.go @@ -42,8 +42,9 @@ type JobRunTriggersState struct { OnBundleDeploy string `json:"on_bundle_deploy,omitempty"` // Content hashes from ResolveJobRunFileTriggers; any change recreates. OnFileChange map[string]string `json:"on_file_change,omitempty"` - // Resolved value fingerprints; any change recreates. - OnValueChange map[string]string `json:"on_value_change,omitempty"` + // Fingerprints in trigger order. A list so two watches that resolve to the + // same value stay distinct (a map would treat converging them as a removal). + OnValueChange []string `json:"on_value_change,omitempty"` } // JobRunLifecycleState is the local-only trigger fingerprint. Nested by value, @@ -137,52 +138,39 @@ func (*ResourceJobRun) PrepareState(input *resources.JobRun) *JobRunState { // expression so the deploy graph depends on that value, not the config wrapper. func (*ResourceJobRun) PrepareInputConfig(input *resources.JobRun, _ string) (*structvar.StructVar, error) { refs := map[string]string{} - for expr := range jobRunValueChangeState(input) { + parent := structpath.MustParsePath("lifecycle.triggers.on_value_change") + for i, expr := range jobRunValueChangeState(input) { if _, ok := dynvar.NewRef(dyn.V(expr)); !ok { continue } - path := structpath.NewStringKey(structpath.MustParsePath("lifecycle.triggers.on_value_change"), expr) - refs[path.String()] = expr + refs[structpath.NewIndex(parent, i).String()] = expr } return &structvar.StructVar{Value: input, Refs: refs}, nil } -func jobRunValueChangeState(input *resources.JobRun) map[string]string { +func jobRunValueChangeState(input *resources.JobRun) []string { if input.Lifecycle == nil { return nil } - out := make(map[string]string) + var out []string for _, t := range input.Lifecycle.Triggers { if t.OnValueChange == nil { continue } - expr := strings.TrimSpace(*t.OnValueChange) - out[expr] = expr - } - if len(out) == 0 { - return nil + out = append(out, strings.TrimSpace(*t.OnValueChange)) } return out } -// NormalizeAfterResolve rekeys watched values by their fingerprint, so that the -// identity of a trigger is the value it resolved to and not the expression text. -// An entry still holding a reference is left alone until it fully resolves. +// NormalizeAfterResolve hashes a watch once it no longer contains a reference. func (s *JobRunState) NormalizeAfterResolve() { values := s.Lifecycle.Triggers.OnValueChange - if len(values) == 0 { - return - } - normalized := make(map[string]string, len(values)) - for key, value := range values { + for i, value := range values { if dynvar.ContainsVariableReference(value) { - normalized[key] = value continue } - fingerprint := compactJobRunValue(value) - normalized[fingerprint] = fingerprint + values[i] = compactJobRunValue(value) } - s.Lifecycle.Triggers.OnValueChange = normalized } // compactJobRunValue hashes a value only when the digest is shorter than it. @@ -472,8 +460,6 @@ func (*ResourceJobRun) OverrideChangeDesc(_ context.Context, path *structpath.Pa change.Reason = "trigger removed" } case "lifecycle.triggers.on_value_change": - // A watched value always resolves to something, so entries disappear only - // when the config stops watching them. if valueTriggersOnlyRemoved(change.Old, change.New) { change.Action = deployplan.Skip change.Reason = "trigger removed" @@ -486,33 +472,42 @@ func (*ResourceJobRun) OverrideChangeDesc(_ context.Context, path *structpath.Pa change.Action = deployplan.Skip change.Reason = "run in progress" default: - // The per-entry change that accompanies the map above. parent := path.Parent() - if change.New == nil && parent != nil && parent.String() == "lifecycle.triggers.on_value_change" { + if parent != nil && parent.String() == "lifecycle.triggers.on_value_change" { + // Classified on the list; a deletion that shifts later entries is not + // a change to those fingerprints. change.Action = deployplan.Skip - change.Reason = "trigger removed" + change.Reason = deployplan.ReasonDrop } } return nil } -// valueTriggersOnlyRemoved reports whether the remaining fingerprints are a -// strict subset of the old ones, i.e. entries were dropped and none added. +// valueTriggersOnlyRemoved is true when new is old minus some watches (order kept). +// Converging two watches replaces a fingerprint, so it is not a removal. func valueTriggersOnlyRemoved(oldValue, newValue any) bool { if newValue == nil { return true } - oldMap, okOld := oldValue.(map[string]string) - newMap, okNew := newValue.(map[string]string) + oldList, okOld := oldValue.([]string) + newList, okNew := newValue.([]string) if !okOld || !okNew { return false } - for key, value := range newMap { - if oldMap[key] != value { - return false + if len(newList) >= len(oldList) { + return false + } + return isStringSubsequence(oldList, newList) +} + +func isStringSubsequence(oldList, newList []string) bool { + i := 0 + for _, value := range oldList { + if i < len(newList) && value == newList[i] { + i++ } } - return len(newMap) < len(oldMap) + return i == len(newList) } // DoDelete deletes the run via jobs/runs/delete, on both destroy and the diff --git a/bundle/direct/dresources/job_run_test.go b/bundle/direct/dresources/job_run_test.go index 9ee8eab1aa7..f977f79d8b6 100644 --- a/bundle/direct/dresources/job_run_test.go +++ b/bundle/direct/dresources/job_run_test.go @@ -365,14 +365,14 @@ func TestJobRunValueChangeStateNormalizesAfterAllReferencesResolve(t *testing.T) input.Lifecycle = &resources.JobRunLifecycle{} input.Lifecycle.Triggers = []resources.JobRunTrigger{trigger} state := (&ResourceJobRun{}).PrepareState(&input) - path := structpath.NewStringKey(structpath.MustParsePath("lifecycle.triggers.on_value_change"), expr) + path := structpath.NewIndex(structpath.MustParsePath("lifecycle.triggers.on_value_change"), 0) sv := structvar.NewStructVar(state, map[string]string{path.String(): expr}) require.NoError(t, sv.ResolveRef("${resources.jobs.other.id}", int64(123))) - assert.Equal(t, map[string]string{expr: "123-${resources.jobs.extra.id}"}, state.Lifecycle.Triggers.OnValueChange) + assert.Equal(t, []string{"123-${resources.jobs.extra.id}"}, state.Lifecycle.Triggers.OnValueChange) require.NoError(t, sv.ResolveRef("${resources.jobs.extra.id}", int64(456))) - assert.Equal(t, map[string]string{"123-456": "123-456"}, state.Lifecycle.Triggers.OnValueChange) + assert.Equal(t, []string{"123-456"}, state.Lifecycle.Triggers.OnValueChange) } func TestJobRunDeleteLeavesFinishedRunAlone(t *testing.T) { @@ -416,10 +416,13 @@ func TestJobRunOverrideChangeDescTriggerRemoved(t *testing.T) { // extend to on_file_change entries. {"removed one on_file_change", "lifecycle.triggers.on_file_change", map[string]string{"a.txt": "h", "b.txt": "h"}, map[string]string{"a.txt": "h"}, deployplan.Recreate}, {"cleared on_file_change child", "lifecycle.triggers.on_file_change['a.txt']", "h", nil, deployplan.Recreate}, - {"cleared on_value_change", "lifecycle.triggers.on_value_change", map[string]string{"a": "a"}, nil, deployplan.Skip}, - {"removed one on_value_change", "lifecycle.triggers.on_value_change", map[string]string{"a": "a", "b": "b"}, map[string]string{"b": "b"}, deployplan.Skip}, - {"changed on_value_change", "lifecycle.triggers.on_value_change", map[string]string{"a": "a"}, map[string]string{"b": "b"}, deployplan.Recreate}, - {"cleared on_value_change child", "lifecycle.triggers.on_value_change['b']", "b", nil, deployplan.Skip}, + {"cleared on_value_change", "lifecycle.triggers.on_value_change", []string{"a"}, nil, deployplan.Skip}, + {"removed last on_value_change", "lifecycle.triggers.on_value_change", []string{"a", "b"}, []string{"a"}, deployplan.Skip}, + {"removed first on_value_change", "lifecycle.triggers.on_value_change", []string{"a", "b"}, []string{"b"}, deployplan.Skip}, + {"changed on_value_change", "lifecycle.triggers.on_value_change", []string{"a"}, []string{"b"}, deployplan.Recreate}, + {"converged on_value_change", "lifecycle.triggers.on_value_change", []string{"a", "b"}, []string{"a", "a"}, deployplan.Recreate}, + {"cleared on_value_change child", "lifecycle.triggers.on_value_change[1]", "b", nil, deployplan.Skip}, + {"shifted on_value_change child", "lifecycle.triggers.on_value_change[0]", "a", "b", deployplan.Skip}, {"result_state with unreadable remote", "result_state", "", nil, deployplan.Recreate}, } { t.Run(tt.name, func(t *testing.T) {