fix(cluster): run the join wait from the image's filesystem

roles/github_runner_cluster/files/k3s/node-entrypoint.sh

@@ -100,10 +100,13 @@ wait_for_mesh_host() {
   fi
 }
 
-# Writes a `tailscale` into $2, which k3s is started with first on its PATH (tailscale_up_wait_dir below), that runs the real one ($3) for every command unchanged and, after `tailscale up`, waits as wait_for_mesh_host does for $1, the host this node joins. k3s's own --vpn-auth start (StartVPN in k3s's pkg/vpn/vpn.go) runs `tailscale up --authkey ... --timeout=30s --reset` on every start, which restarts the Tailscale backend from its cached network map, and k3s dials the join host as soon as that command returns. Confirmed live on Tailscale's own control plane with Tailscale v1.102.4: the backend reloaded a cached map with no peers ("Start: loaded netmap from disk cache; 0 peers"), so the join host was neither a peer nor a MagicDNS name for about a second, k3s exited with "no such host", and every restart of the container repeated it, so the node never joined. The wait above, before k3s starts, cannot cover it, since the reset comes after it. $2 is under /run, a tmpfs in docker-compose.yml, so the wrapper is written afresh at every start of the container and never stands in front of the entrypoint's own tailscale commands; it follows the entrypoint, so it needs no image rebuild, and a node that joins nothing gets none.
-tailscale_up_wait_dir=/run/github-runner-node/bin
+# Writes a `tailscale` into $2, which k3s is started with first on its PATH (tailscale_up_wait_dir below), that runs the real one ($3) for every command unchanged and, after `tailscale up`, waits as wait_for_mesh_host does for $1, the host this node joins, then says so in the container's log. k3s's own --vpn-auth start (StartVPN in k3s's pkg/vpn/vpn.go) runs `tailscale up --authkey ... --timeout=30s --reset` on every start, which restarts the Tailscale backend from its cached network map, and k3s dials the join host as soon as that command returns. Confirmed live on Tailscale's own control plane with Tailscale v1.102.4: the backend reloaded a cached map with no peers ("Start: loaded netmap from disk cache; 0 peers"), so the join host was neither a peer nor a MagicDNS name for about a second, k3s exited with "no such host", and every restart of the container repeated it, so the node never joined. The wait above, before k3s starts, cannot cover it, since the reset comes after it.
+#
+# $2 is on the image's own filesystem, never under the service's tmpfs mounts: Docker mounts a tmpfs noexec unless told otherwise, and k3s finds the command with Go's exec.LookPath, which passes over a file it may not execute, so a wrapper under /run was never run (confirmed live: the node kept failing as before, with the wrapper in place). The container's filesystem outlives a restart, so the wrapper is written afresh at every start, and removed when the node joins nothing, so a wrapper from an earlier configuration never waits for a host this node no longer joins. Only k3s has $2 on its PATH, so the entrypoint's own tailscale commands run the real one.
+tailscale_up_wait_dir=/usr/local/lib/github-runner-node/bin
 install_tailscale_up_wait() {
   if [ -z "$1" ]; then
+    rm -f "$2/tailscale"
     return 0
   fi
   mkdir -p "$2"
@@ -114,6 +117,8 @@ install_tailscale_up_wait() {
     # shellcheck disable=SC2016 # Written into the wrapper unexpanded, for the wrapper to expand.
     echo 'if [ "${1:-}" = up ]; then'
     printf "  until '%s' ip -4 '%s' >/dev/null 2>&1 && nslookup '%s' 100.100.100.100 >/dev/null 2>&1; do sleep 0.5; done\n" "$3" "$1" "$1"
+    # The container's log is PID 1's standard error, k3s's own once the entrypoint has exec'd it; k3s keeps this command's output to itself. Best effort: the line is for whoever reads the log (and the mesh integration test), and failing to write it must not fail k3s's tailscale up.
+    printf "  echo 'github-runner-node: %s is a peer and resolves after tailscale up' 2>/dev/null > /proc/1/fd/2 || true\n" "$1"
     echo 'fi'
   } > "$2/tailscale"
   chmod 0755 "$2/tailscale"

tests/mesh/run.sh

@@ -201,6 +201,14 @@ assert_cluster() {
   done
   kubectl_on 1 get nodes -o wide
 
+  # The entrypoint's wrapper around k3s's own `tailscale up` logs this line after it has waited for the join host; its absence means k3s never ran the wrapper (as when it sat on a noexec tmpfs, where exec.LookPath passes over it), whether or not this control server needed the wait.
+  log "Checking each joining server's k3s ran the wait after its own tailscale up"
+  for index in $(seq 2 "$node_count"); do
+    # grep without -q reads the whole log: with -q it would stop at the first match and, under pipefail, the docker logs it cut off would fail the check.
+    docker logs "$(container "$index")" 2>&1 | grep -F "github-runner-node: $(node_name 1).mesh.grtest.internal is a peer and resolves after tailscale up" >/dev/null \
+      || fail "$(container "$index") never logged the wait after k3s's tailscale up"
+  done
+
   log "Checking the servers were chosen automatically (all three) and etcd has three members"
   nodes=$(kubectl_on 1 get nodes -l node-role.kubernetes.io/etcd=true --no-headers | wc -l | tr -d ' ')
   [ "$nodes" = "$node_count" ] || fail "expected ${node_count} etcd nodes, found ${nodes}"

tests/unit/test_cluster_join_wait.py

@@ -58,7 +58,7 @@ def setUp(self) -> None:
         nslookup.write_text(NSLOOKUP_STUB)
         nslookup.chmod(0o755)
         (self.stub / "peer-after").write_text("3")
-        self.wrapper_dir = self.work / "run" / "github-runner-node" / "bin"
+        self.wrapper_dir = self.work / "usr" / "local" / "lib" / "github-runner-node" / "bin"
 
     def install(self, host: str = HOST) -> subprocess.CompletedProcess[str]:
         script = "\n".join(["set -u", function_source("install_tailscale_up_wait"), f'install_tailscale_up_wait "{host}" "{self.wrapper_dir}" "{self.real}"'])
@@ -80,6 +80,13 @@ def test_up_runs_unchanged_then_waits_until_the_join_host_is_a_peer_and_resolves
             ["up --authkey example --timeout=30s --reset", f"ip -4 {HOST}", f"ip -4 {HOST}", f"ip -4 {HOST}", f"nslookup {HOST} 100.100.100.100"],
         )
 
+    def test_the_wait_is_written_to_the_container_s_log_without_failing_when_it_cannot_be(self) -> None:
+        # Outside a container /proc/1/fd/2 is not this test's to write, which is the case the wrapper must survive; inside one it is the container's log, where the mesh integration test finds the line.
+        self.install()
+        self.assertIn(f"echo 'github-runner-node: {HOST} is a peer and resolves after tailscale up' 2>/dev/null > /proc/1/fd/2 || true", (self.wrapper_dir / "tailscale").read_text())
+        result = self.run_wrapper("up")
+        self.assertEqual(result.returncode, 0, result.stderr)
+
     def test_any_other_command_runs_unchanged_and_waits_for_nothing(self) -> None:
         self.install()
         result = self.run_wrapper("status", "--json")
@@ -97,6 +104,12 @@ def test_a_node_that_joins_nothing_gets_no_wrapper(self) -> None:
         self.assertEqual(self.install(host="").returncode, 0)
         self.assertFalse(self.wrapper_dir.exists())
 
+    def test_a_wrapper_left_by_an_earlier_start_is_removed_when_the_node_joins_nothing_now(self) -> None:
+        # The container's filesystem outlives a restart, so without this a node whose configuration no longer joins anything would wait for its old join host.
+        self.install()
+        self.assertEqual(self.install(host="").returncode, 0)
+        self.assertFalse((self.wrapper_dir / "tailscale").exists())
+
 
 class JoinHostTest(unittest.TestCase):
     def join_host(self, url: str) -> str:
@@ -127,13 +140,16 @@ def test_an_agent_waits_for_the_server_it_dials(self) -> None:
         self.assertIn('install_tailscale_up_wait "$agent_join_host" "$tailscale_up_wait_dir" /usr/local/bin/tailscale', body)
         self.assertIn('PATH="$tailscale_up_wait_dir:$PATH" exec /bin/k3s agent', body)
 
-    def test_the_wrapper_lives_on_a_tmpfs_of_both_services_so_every_start_writes_it_afresh(self) -> None:
+    def test_the_wrapper_is_on_the_image_s_filesystem_where_it_may_be_executed(self) -> None:
+        # Docker mounts a tmpfs noexec, and Go's exec.LookPath passes over a file it may not execute, so a wrapper under a tmpfs or a mount is never run. A mount's options are not visible here, so the directory is kept clear of every mount either service has.
         directory = re.search(r"^tailscale_up_wait_dir=(\S+)$", ENTRYPOINT.read_text(), re.MULTILINE)
         assert directory is not None
         services = yaml.safe_load(COMPOSE.read_text())["services"]
         for name in ("k3s", "k3s-agent"):
-            with self.subTest(service=name):
-                self.assertTrue(any(directory.group(1).startswith(mount.rstrip("/") + "/") for mount in services[name]["tmpfs"]))
+            mounts = [*services[name]["tmpfs"], *(volume.split(":")[1] for volume in services[name]["volumes"])]
+            for mount in mounts:
+                with self.subTest(service=name, mount=mount):
+                    self.assertFalse((directory.group(1) + "/").startswith(mount.rstrip("/") + "/"))
 
 
 if __name__ == "__main__":