From cee23e4006dcd11b6ab9a57d4bc888860380e504 Mon Sep 17 00:00:00 2001 From: Peter Ebden Date: Thu, 8 Oct 2026 13:18:42 +0100 Subject: [PATCH] Download runtime data for targets recursively --- .circleci/rex_test.sh | 4 ++++ VERSION | 2 +- src/core/state.go | 35 ++++++++++++++++++++++++++--------- test/remote_data/BUILD | 21 ++++++++++++++++++++- 4 files changed, 51 insertions(+), 11 deletions(-) diff --git a/.circleci/rex_test.sh b/.circleci/rex_test.sh index e36f7b353..b17d5634f 100755 --- a/.circleci/rex_test.sh +++ b/.circleci/rex_test.sh @@ -47,3 +47,7 @@ if [ ! -f plz-out/gen/test/remote_data/slow_data.txt ]; then echo "Runtime data for //test/remote_data:needs_data wasn't downloaded" exit 1 fi +if [ ! -f plz-out/gen/test/remote_data/nested_data.txt ]; then + echo "Transitive runtime data for //test/remote_data:needs_data wasn't downloaded" + exit 1 +fi diff --git a/VERSION b/VERSION index 665666191..a9374e34b 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -17.34.0-beta.2 +17.34.0-beta.3 diff --git a/src/core/state.go b/src/core/state.go index 73157223a..82286c703 100644 --- a/src/core/state.go +++ b/src/core/state.go @@ -772,15 +772,32 @@ func (state *BuildState) GetPreloadedSubincludes() []BuildLabel { // DownloadInputsIfNeeded downloads all the inputs (or runtime files) for a target if we are building remotely. func (state *BuildState) DownloadInputsIfNeeded(target *BuildTarget, runtime bool) error { - if state.RemoteClient != nil { - for input := range state.IterInputs(target, runtime) { - if l, ok := input.Label(); ok { - dep := state.Graph.TargetOrDie(l) - if s := dep.State(); s == BuiltRemotely || s == ReusedRemotely { - if err := state.RemoteClient.Download(dep); err != nil { - return err - } - } + if state.RemoteClient == nil { + return nil + } + return state.downloadInputs(target, runtime, map[BuildLabel]struct{}{}) +} + +// downloadInputs implements DownloadInputsIfNeeded. At runtime it recurses into each dependency, since +// data can have data of its own (e.g. an sh_cmd whose data is another sh_cmd) which is needed to run it. +func (state *BuildState) downloadInputs(target *BuildTarget, runtime bool, seen map[BuildLabel]struct{}) error { + for input := range state.IterInputs(target, runtime) { + l, ok := input.Label() + if !ok { + continue + } else if _, present := seen[l]; present { + continue + } + seen[l] = struct{}{} + dep := state.Graph.TargetOrDie(l) + if s := dep.State(); s == BuiltRemotely || s == ReusedRemotely { + if err := state.RemoteClient.Download(dep); err != nil { + return err + } + } + if runtime && dep != target { + if err := state.downloadInputs(dep, runtime, seen); err != nil { + return err } } } diff --git a/test/remote_data/BUILD b/test/remote_data/BUILD index e51382562..ad0c63b95 100644 --- a/test/remote_data/BUILD +++ b/test/remote_data/BUILD @@ -13,10 +13,29 @@ genrule( labels = ["manual"], ) +# Data of data must be downloaded too (e.g. an sh_cmd whose data is another sh_cmd). +genrule( + name = "nested_data", + outs = ["nested_data.txt"], + cmd = "echo hello > $OUT", + labels = ["manual"], +) + +genrule( + name = "middle_data", + outs = ["middle_data.txt"], + cmd = "echo hello > $OUT", + data = [":nested_data"], + labels = ["manual"], +) + genrule( name = "needs_data", outs = ["needs_data.txt"], cmd = "echo hello > $OUT", - data = [":slow_data"], + data = [ + ":middle_data", + ":slow_data", + ], labels = ["manual"], )