Download runtime data for targets recursively

.circleci/rex_test.sh

@@ -47,3 +47,7 @@ if [ ! -f plz-out/gen/test/remote_data/slow_data.txt ]; then
     echo "Runtime data for //test/remote_data:needs_data wasn't downloaded"
     exit 1
 fi
+if [ ! -f plz-out/gen/test/remote_data/nested_data.txt ]; then
+    echo "Transitive runtime data for //test/remote_data:needs_data wasn't downloaded"
+    exit 1
+fi

VERSION

@@ -1 +1 @@
-17.34.0-beta.2
+17.34.0-beta.3

src/core/state.go

@@ -772,15 +772,32 @@ func (state *BuildState) GetPreloadedSubincludes() []BuildLabel {
 
 // DownloadInputsIfNeeded downloads all the inputs (or runtime files) for a target if we are building remotely.
 func (state *BuildState) DownloadInputsIfNeeded(target *BuildTarget, runtime bool) error {
-	if state.RemoteClient != nil {
-		for input := range state.IterInputs(target, runtime) {
-			if l, ok := input.Label(); ok {
-				dep := state.Graph.TargetOrDie(l)
-				if s := dep.State(); s == BuiltRemotely || s == ReusedRemotely {
-					if err := state.RemoteClient.Download(dep); err != nil {
-						return err
-					}
-				}
+	if state.RemoteClient == nil {
+		return nil
+	}
+	return state.downloadInputs(target, runtime, map[BuildLabel]struct{}{})
+}
+
+// downloadInputs implements DownloadInputsIfNeeded. At runtime it recurses into each dependency, since
+// data can have data of its own (e.g. an sh_cmd whose data is another sh_cmd) which is needed to run it.
+func (state *BuildState) downloadInputs(target *BuildTarget, runtime bool, seen map[BuildLabel]struct{}) error {
+	for input := range state.IterInputs(target, runtime) {
+		l, ok := input.Label()
+		if !ok {
+			continue
+		} else if _, present := seen[l]; present {
+			continue
+		}
+		seen[l] = struct{}{}
+		dep := state.Graph.TargetOrDie(l)
+		if s := dep.State(); s == BuiltRemotely || s == ReusedRemotely {
+			if err := state.RemoteClient.Download(dep); err != nil {
+				return err
+			}
+		}
+		if runtime && dep != target {
+			if err := state.downloadInputs(dep, runtime, seen); err != nil {
+				return err
 			}
 		}
 	}

test/remote_data/BUILD

@@ -13,10 +13,29 @@ genrule(
     labels = ["manual"],
 )
 
+# Data of data must be downloaded too (e.g. an sh_cmd whose data is another sh_cmd).
+genrule(
+    name = "nested_data",
+    outs = ["nested_data.txt"],
+    cmd = "echo hello > $OUT",
+    labels = ["manual"],
+)
+
+genrule(
+    name = "middle_data",
+    outs = ["middle_data.txt"],
+    cmd = "echo hello > $OUT",
+    data = [":nested_data"],
+    labels = ["manual"],
+)
+
 genrule(
     name = "needs_data",
     outs = ["needs_data.txt"],
     cmd = "echo hello > $OUT",
-    data = [":slow_data"],
+    data = [
+        ":middle_data",
+        ":slow_data",
+    ],
     labels = ["manual"],
 )