diff --git a/acceptance/bundle/python/telemetry/databricks.yml b/acceptance/bundle/python/telemetry/databricks.yml new file mode 100644 index 00000000000..ba73d97729e --- /dev/null +++ b/acceptance/bundle/python/telemetry/databricks.yml @@ -0,0 +1,16 @@ +bundle: + name: my_project + +sync: {paths: []} # don't need to copy files + +# Single line so the script can switch it to 'experimental/python'. +python: {resources: ["resources:load_resources"], mutators: ["mutators:update_job"]} + +resources: + jobs: + yaml_job: + name: "YAML Job" + schemas: + yaml_schema: + catalog_name: main + name: yaml_schema diff --git a/acceptance/bundle/python/telemetry/mutators.py b/acceptance/bundle/python/telemetry/mutators.py new file mode 100644 index 00000000000..9335dd63f19 --- /dev/null +++ b/acceptance/bundle/python/telemetry/mutators.py @@ -0,0 +1,11 @@ +from dataclasses import replace + +from databricks.bundles.core import job_mutator +from databricks.bundles.jobs import Job + + +@job_mutator +def update_job(job: Job) -> Job: + assert isinstance(job.name, str) + + return replace(job, name=f"{job.name} (updated)") diff --git a/acceptance/bundle/python/telemetry/out.test.toml b/acceptance/bundle/python/telemetry/out.test.toml new file mode 100644 index 00000000000..d2d80ecfd50 --- /dev/null +++ b/acceptance/bundle/python/telemetry/out.test.toml @@ -0,0 +1,3 @@ +Cloud = false +EnvMatrix.DMS = [""] +EnvMatrix.PYDAB_VERSION = ["current"] diff --git a/acceptance/bundle/python/telemetry/output.txt b/acceptance/bundle/python/telemetry/output.txt new file mode 100644 index 00000000000..1b6ba81e813 --- /dev/null +++ b/acceptance/bundle/python/telemetry/output.txt @@ -0,0 +1,70 @@ + +=== Deploy with 'python' +>>> uv run [UV_ARGS] -q [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/my_project/default/files... +Created jobs.python_job +Created jobs.yaml_job +Created pipelines.python_pipeline +Created schemas.yaml_schema +Files: 0 uploaded, 0 deleted +Resources: 4 created, 0 changed, 0 deleted, 0 unchanged +{ + "pydabs": { + "resource_loaders_count": 1, + "resource_mutators_count": 1, + "added_resources_count": 2, + "updated_resources_count": 2, + "resources": [ + { + "resource_type": "jobs", + "added_count": 1, + "updated_count": 2 + }, + { + "resource_type": "pipelines", + "added_count": 1, + "updated_count": 0 + } + ], + "config_section": "PYTHON" + }, + "all_resource_types": [ + "jobs", + "pipelines", + "schemas" + ] +} + +=== Deploy with 'experimental/python' +>>> update_file.py databricks.yml python: {resources: ["resources:load_resources"], mutators: ["mutators:update_job"]} experimental: {python: {resources: ["resources:load_resources"], mutators: ["mutators:update_job"]}} + +>>> uv run [UV_ARGS] -q [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/my_project/default/files... +Files: 0 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 4 unchanged +{ + "pydabs": { + "resource_loaders_count": 1, + "resource_mutators_count": 1, + "added_resources_count": 2, + "updated_resources_count": 2, + "resources": [ + { + "resource_type": "jobs", + "added_count": 1, + "updated_count": 2 + }, + { + "resource_type": "pipelines", + "added_count": 1, + "updated_count": 0 + } + ], + "config_section": "EXPERIMENTAL_PYTHON" + }, + "all_resource_types": [ + "jobs", + "pipelines", + "schemas" + ] +} diff --git a/acceptance/bundle/python/telemetry/resources.py b/acceptance/bundle/python/telemetry/resources.py new file mode 100644 index 00000000000..9f9555ae9e0 --- /dev/null +++ b/acceptance/bundle/python/telemetry/resources.py @@ -0,0 +1,9 @@ +from databricks.bundles.core import Resources + + +def load_resources() -> Resources: + resources = Resources() + resources.add_job("python_job", {"name": "Python Job"}) + resources.add_pipeline("python_pipeline", {"name": "Python Pipeline"}) + + return resources diff --git a/acceptance/bundle/python/telemetry/script b/acceptance/bundle/python/telemetry/script new file mode 100644 index 00000000000..311998c3e9c --- /dev/null +++ b/acceptance/bundle/python/telemetry/script @@ -0,0 +1,20 @@ +# pydabs covers only resources defined or modified in Python (jobs, pipelines), +# while resources_metadata covers every resource in the bundle (including schemas). +print_pydabs_telemetry() { + jq 'select(has("path") and .path == "/telemetry-ext") | .body.protoLogs[] | fromjson | .entry.databricks_cli_log.bundle_deploy_event | { + pydabs, + all_resource_types: [.resources_metadata.resources[].resource_type] + }' "$OUT_REQUESTS" + rm "$OUT_REQUESTS" +} + +title "Deploy with 'python'" +trace uv run $UV_ARGS -q $CLI bundle deploy +print_pydabs_telemetry + +title "Deploy with 'experimental/python'" +trace update_file.py databricks.yml 'python: {resources: ["resources:load_resources"], mutators: ["mutators:update_job"]}' 'experimental: {python: {resources: ["resources:load_resources"], mutators: ["mutators:update_job"]}}' +trace uv run $UV_ARGS -q $CLI bundle deploy +print_pydabs_telemetry + +rm -fr .databricks __pycache__ diff --git a/acceptance/bundle/python/telemetry/test.toml b/acceptance/bundle/python/telemetry/test.toml new file mode 100644 index 00000000000..355ba35fa71 --- /dev/null +++ b/acceptance/bundle/python/telemetry/test.toml @@ -0,0 +1,5 @@ +RecordRequests = true + +EnvMatrix.DMS = [""] +# Telemetry doesn't depend on the databricks-bundles version. +EnvMatrix.PYDAB_VERSION = ["current"] diff --git a/bundle/bundle.go b/bundle/bundle.go index 6c80882134a..253ba9001b4 100644 --- a/bundle/bundle.go +++ b/bundle/bundle.go @@ -67,14 +67,19 @@ const terraformStateFilename = "terraform.tfstate" // This struct is used as a communication channel to collect metrics // from all over the bundle codebase to finally be emitted as telemetry. type Metrics struct { - ConfigurationFileCount int64 - TargetCount int64 - DeploymentId uuid.UUID - BoolValues []protos.BoolMapEntry - PythonAddedResourcesCount int64 - PythonUpdatedResourcesCount int64 - ExecutionTimes []protos.IntMapEntry - LocalCacheMeasurementsMs []protos.IntMapEntry // Local cache measurements stored as milliseconds + ConfigurationFileCount int64 + TargetCount int64 + DeploymentId uuid.UUID + BoolValues []protos.BoolMapEntry + ExecutionTimes []protos.IntMapEntry + LocalCacheMeasurementsMs []protos.IntMapEntry // Local cache measurements stored as milliseconds + + // Number of resources added and updated by the Python mutator, keyed by resource type. + PythonAddedResources map[string]int64 + PythonUpdatedResources map[string]int64 + + // Configuration section Python support is declared in. + PythonConfigSection protos.PydabsConfigSection // StateEngine is the engine that ran (or would have run) the deploy. Set to the // requested engine as soon as it is resolved, then refined to the state's engine diff --git a/bundle/config/mutator/python/python_mutator.go b/bundle/config/mutator/python/python_mutator.go index 5d6e8b0d801..a6dbdfb8fd4 100644 --- a/bundle/config/mutator/python/python_mutator.go +++ b/bundle/config/mutator/python/python_mutator.go @@ -32,6 +32,7 @@ import ( "github.com/databricks/cli/libs/dyn/convert" "github.com/databricks/cli/libs/dyn/yamlloader" "github.com/databricks/cli/libs/process" + "github.com/databricks/cli/libs/telemetry/protos" ) type phase string @@ -107,6 +108,17 @@ type runPythonMutatorOpts struct { authEnv map[string]string } +// addResourceCounts adds the number of resources in set to counts, per resource type. +func addResourceCounts(counts map[string]int64, set resourcemutator.ResourceKeySet) map[string]int64 { + for resourceType, names := range set { + if counts == nil { + counts = make(map[string]int64) + } + counts[resourceType] += int64(len(names)) + } + return counts +} + // getOpts adapts deprecated PyDABs and upcoming Python configuration // into a common structure. func getOpts(b *bundle.Bundle, phase phase) (opts, error) { @@ -151,6 +163,23 @@ func getOpts(b *bundle.Bundle, phase phase) (opts, error) { } } +// pythonConfigSection reports which section the user declared Python support in. +func pythonConfigSection(b *bundle.Bundle) protos.PydabsConfigSection { + pythonEnabled := !reflect.DeepEqual(b.Config.Python, config.Python{}) + experimentalPythonEnabled := b.Config.Experimental != nil && !reflect.DeepEqual(b.Config.Experimental.Python, config.Python{}) + + switch { + case pythonEnabled && experimentalPythonEnabled: + return protos.PydabsConfigSectionBoth + case pythonEnabled: + return protos.PydabsConfigSectionPython + case experimentalPythonEnabled: + return protos.PydabsConfigSectionExperimentalPython + default: + return protos.PydabsConfigSectionUnspecified + } +} + // applyBackwardsCompatibilityFixes applies fixes to bundle configuration // so that older version of databricks-bundles Python library continue to see // 'experimental.python' section even if bundle configuration uses 'python' section. @@ -199,6 +228,12 @@ func applyBackwardsCompatibilityFixes(b *bundle.Bundle) error { } func (m *pythonMutator) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + // Record before applyBackwardsCompatibilityFixes copies 'python' into 'experimental/python', + // which makes the sections indistinguishable. The load_resources phase runs first. + if m.phase == PythonMutatorPhaseLoadResources { + b.Metrics.PythonConfigSection = pythonConfigSection(b) + } + err := applyBackwardsCompatibilityFixes(b) if err != nil { return diag.FromErr(fmt.Errorf("failed to apply backwards compatibility fixes: %w", err)) @@ -290,8 +325,8 @@ func (m *pythonMutator) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagno }) // we can precisely track resources that are added/updated, so sum doesn't double-count - b.Metrics.PythonUpdatedResourcesCount += int64(result.UpdatedResources.Size()) - b.Metrics.PythonAddedResourcesCount += int64(result.AddedResources.Size()) + b.Metrics.PythonUpdatedResources = addResourceCounts(b.Metrics.PythonUpdatedResources, result.UpdatedResources) + b.Metrics.PythonAddedResources = addResourceCounts(b.Metrics.PythonAddedResources, result.AddedResources) if errors.Is(err, mutateDiagsHasError) { if !mutateDiags.HasError() { diff --git a/bundle/config/mutator/python/python_mutator_test.go b/bundle/config/mutator/python/python_mutator_test.go index f2c91749e87..f3378b1bd51 100644 --- a/bundle/config/mutator/python/python_mutator_test.go +++ b/bundle/config/mutator/python/python_mutator_test.go @@ -24,6 +24,7 @@ import ( "github.com/databricks/cli/internal/testutil" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/process" + "github.com/databricks/cli/libs/telemetry/protos" "github.com/stretchr/testify/assert" ) @@ -157,9 +158,6 @@ workspace: { current_user: { userName: test }}`) }) assert.NoError(t, err) - assert.Equal(t, int64(2), b.Metrics.PythonAddedResourcesCount) - assert.Equal(t, int64(0), b.Metrics.PythonUpdatedResourcesCount) - assert.Len(t, diags, 1) assert.Equal(t, "job doesn't have any tasks", diags[0].Summary) assert.Equal(t, []diag.Location{ @@ -237,9 +235,24 @@ resources: return v, nil }) assert.NoError(t, err) +} - assert.Equal(t, int64(0), b.Metrics.PythonAddedResourcesCount) - assert.Equal(t, int64(1), b.Metrics.PythonUpdatedResourcesCount) +func TestPythonConfigSection(t *testing.T) { + tests := []struct { + name string + yaml string + expected protos.PydabsConfigSection + }{ + {"none", `bundle: {name: test}`, protos.PydabsConfigSectionUnspecified}, + {"both", ` +python: {resources: ["resources:load"]} +experimental: {python: {resources: ["resources:load"]}}`, protos.PydabsConfigSectionBoth}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.expected, pythonConfigSection(loadYaml("databricks.yml", tc.yaml))) + }) + } } func TestPythonMutator_badOutput(t *testing.T) { diff --git a/bundle/phases/telemetry.go b/bundle/phases/telemetry.go index 99464f3ef24..c4f6caf5473 100644 --- a/bundle/phases/telemetry.go +++ b/bundle/phases/telemetry.go @@ -3,6 +3,7 @@ package phases import ( "cmp" "context" + "maps" "math" "reflect" "slices" @@ -195,6 +196,39 @@ func aiRuntimeTaskMetrics(jobs map[string]*resources.Job) (present, scheduled, m return present, scheduled, multitask } +// pydabsMetrics reports usage of Python support. Loaders and mutators are read from +// 'experimental/python' only: the Python mutator copies 'python' there. +func pydabsMetrics(b *bundle.Bundle) *protos.BundleDeployPydabs { + experimentalConfig := b.Config.Experimental + if experimentalConfig == nil { + experimentalConfig = &config.Experimental{} + } + + out := &protos.BundleDeployPydabs{ + ResourceLoadersCount: int64(len(experimentalConfig.Python.Resources)), + ResourceMutatorsCount: int64(len(experimentalConfig.Python.Mutators)), + ConfigSection: b.Metrics.PythonConfigSection, + } + + byType := make(map[string]*protos.PydabsResourceTypeCount) + for t, n := range b.Metrics.PythonAddedResources { + byType[t] = &protos.PydabsResourceTypeCount{ResourceType: t, AddedCount: n} + out.AddedResourcesCount += n + } + for t, n := range b.Metrics.PythonUpdatedResources { + if byType[t] == nil { + byType[t] = &protos.PydabsResourceTypeCount{ResourceType: t} + } + byType[t].UpdatedCount = n + out.UpdatedResourcesCount += n + } + + for _, t := range slices.Sorted(maps.Keys(byType)) { + out.Resources = append(out.Resources, *byType[t]) + } + return out +} + // LogDeployTelemetry logs a telemetry event for a bundle deploy command. func LogDeployTelemetry(ctx context.Context, b *bundle.Bundle, errMsg string) { errMsg = telemetry.ScrubErrorMessage(errMsg) @@ -349,9 +383,12 @@ func LogDeployTelemetry(ctx context.Context, b *bundle.Bundle, errMsg string) { mode = protos.BundleModeProduction } - experimentalConfig := b.Config.Experimental - if experimentalConfig == nil { - experimentalConfig = &config.Experimental{} + pydabs := pydabsMetrics(b) + + // Omit the pydabs section for bundles that don't use Python support. + var pydabsSection *protos.BundleDeployPydabs + if pydabs.ResourceLoadersCount > 0 || pydabs.ResourceMutatorsCount > 0 { + pydabsSection = pydabs } uploadFileSizers := make([]sizer, len(b.Files)) @@ -386,6 +423,8 @@ func LogDeployTelemetry(ctx context.Context, b *bundle.Bundle, errMsg string) { ResourcesMetadata: collectResourcesMetadata(ctx, b), + Pydabs: pydabsSection, + Experimental: &protos.BundleDeployExperimental{ BundleMode: mode, ConfigurationFileCount: b.Metrics.ConfigurationFileCount, @@ -395,10 +434,6 @@ func LogDeployTelemetry(ctx context.Context, b *bundle.Bundle, errMsg string) { WorkspaceArtifactPathType: artifactPathType, BoolValues: b.Metrics.BoolValues, LocalCacheMeasurementsMs: b.Metrics.LocalCacheMeasurementsMs, - PythonAddedResourcesCount: b.Metrics.PythonAddedResourcesCount, - PythonUpdatedResourcesCount: b.Metrics.PythonUpdatedResourcesCount, - PythonResourceLoadersCount: int64(len(experimentalConfig.Python.Resources)), - PythonResourceMutatorsCount: int64(len(experimentalConfig.Python.Mutators)), VariableCount: int64(variableCount), ComplexVariableCount: complexVariableCount, LookupVariableCount: lookupVariableCount, diff --git a/bundle/phases/telemetry_test.go b/bundle/phases/telemetry_test.go index d6badfaa792..1436ecb7e84 100644 --- a/bundle/phases/telemetry_test.go +++ b/bundle/phases/telemetry_test.go @@ -3,7 +3,10 @@ package phases import ( "testing" + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/libs/telemetry/protos" "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/stretchr/testify/assert" ) @@ -174,3 +177,30 @@ func TestAiRuntimeTaskMetrics(t *testing.T) { }) } } + +func TestPydabsMetrics(t *testing.T) { + pythonConfig := config.Python{ + Resources: []string{"resources:load_resources"}, + Mutators: []string{"mutators:m1", "mutators:m2"}, + } + metrics := bundle.Metrics{ + PythonAddedResources: map[string]int64{"jobs": 2, "pipelines": 1}, + PythonUpdatedResources: map[string]int64{"jobs": 3, "apps": 1}, + PythonConfigSection: protos.PydabsConfigSectionPython, + } + expected := &protos.BundleDeployPydabs{ + ResourceLoadersCount: 1, + ResourceMutatorsCount: 2, + AddedResourcesCount: 3, + UpdatedResourcesCount: 4, + Resources: []protos.PydabsResourceTypeCount{ + {ResourceType: "apps", AddedCount: 0, UpdatedCount: 1}, + {ResourceType: "jobs", AddedCount: 2, UpdatedCount: 3}, + {ResourceType: "pipelines", AddedCount: 1, UpdatedCount: 0}, + }, + ConfigSection: protos.PydabsConfigSectionPython, + } + + b := &bundle.Bundle{Config: config.Root{Experimental: &config.Experimental{Python: pythonConfig}}, Metrics: metrics} + assert.Equal(t, expected, pydabsMetrics(b)) +} diff --git a/libs/telemetry/protos/bundle_deploy.go b/libs/telemetry/protos/bundle_deploy.go index ed5a79690a3..4464f36bca4 100644 --- a/libs/telemetry/protos/bundle_deploy.go +++ b/libs/telemetry/protos/bundle_deploy.go @@ -38,6 +38,10 @@ type BundleDeployEvent struct { // Per-resource-type metadata (counts and state-size statistics). ResourcesMetadata *BundleResourcesMetadata `json:"resources_metadata,omitempty"` + // Usage of Python support (PyDABs). Set only when the bundle declares + // Python resource loaders or mutators. + Pydabs *BundleDeployPydabs `json:"pydabs,omitempty"` + Experimental *BundleDeployExperimental `json:"experimental,omitempty"` } @@ -75,18 +79,6 @@ type BundleDeployExperimental struct { // Execution time per mutator for a selected subset of mutators. BundleMutatorExecutionTimeMs []IntMapEntry `json:"bundle_mutator_execution_time_ms,omitempty"` - // Number of resources added by PythonMutator - PythonAddedResourcesCount int64 `json:"python_added_resources_count,omitempty"` - - // Number of resources updated by PythonMutator - PythonUpdatedResourcesCount int64 `json:"python_updated_resources_count,omitempty"` - - // Number of resource loaders declared at 'python/resources' in databricks.yml - PythonResourceLoadersCount int64 `json:"python_resource_loaders_count,omitempty"` - - // Number of resource mutators declared at 'python/mutators' in databricks.yml - PythonResourceMutatorsCount int64 `json:"python_resource_mutators_count,omitempty"` - // Number of files in the synced source tree considered for upload: the full // set, not the incremental delta actually transferred (an incremental deploy // reports the full count even when nothing changed). The set is whatever the @@ -150,6 +142,42 @@ type ResourceMetadata struct { StateSizeMedianBytes int64 `json:"state_size_median_bytes,omitempty"` } +// BundleDeployPydabs mirrors the universe proto. Usage of Python support +// (PyDABs) in one bundle deployment. +type BundleDeployPydabs struct { + // Number of resource loaders declared at 'python/resources' (or 'experimental/python/resources'). + ResourceLoadersCount int64 `json:"resource_loaders_count"` + + // Number of resource mutators declared at 'python/mutators' (or 'experimental/python/mutators'). + ResourceMutatorsCount int64 `json:"resource_mutators_count"` + + // Number of resources defined in Python. + AddedResourcesCount int64 `json:"added_resources_count"` + + // Number of resources modified by Python mutators, including resources + // defined in Python, so this can overlap with AddedResourcesCount. + UpdatedResourcesCount int64 `json:"updated_resources_count"` + + // One entry per resource type added or updated by Python, ordered by type name. + Resources []PydabsResourceTypeCount `json:"resources,omitempty"` + + // Configuration section the user declared Python support in: 'python', + // 'experimental/python', or both. + ConfigSection PydabsConfigSection `json:"config_section,omitempty"` +} + +// PydabsResourceTypeCount holds PyDABs counts for a single resource type. +type PydabsResourceTypeCount struct { + // Resource type name, same values as ResourceMetadata.ResourceType. + ResourceType string `json:"resource_type,omitempty"` + + // Number of resources of this type defined in Python. + AddedCount int64 `json:"added_count"` + + // Number of resources of this type modified by Python mutators. + UpdatedCount int64 `json:"updated_count"` +} + type BoolMapEntry struct { Key string `json:"key,omitempty"` Value bool `json:"value"` diff --git a/libs/telemetry/protos/enum.go b/libs/telemetry/protos/enum.go index 7f6780cb657..a0c873063fa 100644 --- a/libs/telemetry/protos/enum.go +++ b/libs/telemetry/protos/enum.go @@ -24,3 +24,12 @@ const ( BundleDeployArtifactPathTypeWorkspace BundleDeployArtifactPathType = "WORKSPACE_FILE_SYSTEM" BundleDeployArtifactPathTypeVolume BundleDeployArtifactPathType = "UC_VOLUME" ) + +type PydabsConfigSection string + +const ( + PydabsConfigSectionUnspecified PydabsConfigSection = "TYPE_UNSPECIFIED" + PydabsConfigSectionPython PydabsConfigSection = "PYTHON" + PydabsConfigSectionExperimentalPython PydabsConfigSection = "EXPERIMENTAL_PYTHON" + PydabsConfigSectionBoth PydabsConfigSection = "BOTH" +)