test(mesh): check a server stepped down to etcd-only gives the Lease back

tests/mesh/run.sh

@@ -173,6 +173,8 @@ EOF
     echo "        github_runner_cluster_etcd_heartbeat_interval: ${etcd_heartbeat_ms}"
     echo "        github_runner_cluster_etcd_election_timeout: ${etcd_election_ms}"
     echo "        github_runner_cluster_regrade: ${regrade}"
+    # The test cluster is throwaway: the role must not install a login item that starts the host's Colima profile.
+    echo "        github_runner_cluster_runtime_autostart: false"
     if [ "$pin_context" = 1 ]; then
       echo "        github_runner_cluster_docker_context: ${grtest_pinned_context}"
     fi
@@ -575,6 +577,15 @@ peer_refusal() {
   ! regrade_on 3 fleet-regrade down >"${work}/peer.log" 2>&1 && grep -F "etcd-voting nodes not ready: $(node_name 2)" "${work}/peer.log" >/dev/null
 }
 at_level() { [ "$(level_of "$1")" = "$2" ]; }
+# Whether the regrade Lease exists and nobody holds it, as a release leaves it, read through node 1's API. A Lease left to expire keeps its holder, so this tells a release from an expiry at any time.
+lease_free() { [ "$(kubectl_on 1 get lease fleet-regrade --namespace kube-system -o 'jsonpath={.spec.holderIdentity}' 2>/dev/null; echo "/$?")" = /0 ]; }
+# Whether the regrade service of node $1 has logged, in the file $2 inside it, that it reached the level $3.
+logged_level() { regrade_on "$1" cat "$2" | grep -Fx "now at $3" >/dev/null; }
+# A step down to etcd-only gives the Lease back from another server's API, since the stepped-down server has none: once the step has finished the Lease is free, and the service never logged that it could not release it.
+assert_released() {
+  wait_for "the Lease to be released after $(node_name "$1")'s step down to etcd-only" 120 lease_free
+  if regrade_on "$1" cat "$2" | grep -F 'lease could not be released' >/dev/null; then regrade_on "$1" cat "$2"; fail "$(node_name "$1") could not release the Lease"; fi
+}
 node_is() { [ "$(node_status "$1" "$2")" = "$3" ]; }
 
 # The numbers of the nodes whose regrade service reports a level below full. A read that gives no level, or unknown, is not a step: a test node's k3s container can restart by itself under the strain, and its status then reads unknown or fails for a moment, which says nothing about whether the server stepped.
@@ -636,6 +647,8 @@ EOF
     [ "$SECONDS" -lt "$deadline" ] || { regrade_on 2 cat /tmp/watch.log || true; fail "$(node_name 2) did not reach etcd-only"; }
     sleep 2
   done
+  wait_for "the watcher on $(node_name 2) to finish its step to etcd-only" 300 logged_level 2 /tmp/watch.log etcd-only
+  assert_released 2 /tmp/watch.log
   regrade_on 2 cat /tmp/watch.log
   regrade_on 2 cat /tmp/watch.log | grep -F 'step down full -> control-plane' >/dev/null || fail "the watcher did not step down through control-plane first"
 
@@ -699,8 +712,10 @@ EOF
   wait_for "$(node_name 3) to read control-plane after a step down by hand" 180 at_level 3 control-plane
   wait_for "$(node_name 3) to be cordoned" 60 node_is 1 3 Ready,SchedulingDisabled
   [ "$(level_label 3)" = control-plane ] || fail "the Node does not carry the level label at control-plane"
-  regrade_on 3 fleet-regrade down
+  regrade_on 3 sh -c 'fleet-regrade down >/tmp/down.log 2>&1' || { regrade_on 3 cat /tmp/down.log; fail "the second step down by hand on $(node_name 3) failed"; }
+  regrade_on 3 cat /tmp/down.log
   wait_for "$(node_name 3) to read etcd-only after a second step down by hand" 180 at_level 3 etcd-only
+  assert_released 3 /tmp/down.log
   # The level is a file on the data volume that the entrypoint reads at every start, so neither a restart by hand nor a recreate, which is what Compose does when the role changes the container's configuration, brings the server back at full.
   docker restart "$(container 3)" >/dev/null
   wait_for "$(node_name 3) to be at etcd-only after a restart by hand" 300 at_level 3 etcd-only