From a0456719c0b06cef6487b919993509454f2337f7 Mon Sep 17 00:00:00 2001 From: Evgeny Slutsky Date: Thu, 10 Sep 2026 11:04:21 +0200 Subject: [PATCH 1/3] NO-ISSUE: Label primary node and restrict ovnkube-master DaemonSet to it In MicroShift multinode mode both nodes carry node-role.kubernetes.io/master, so the ovnkube-master DaemonSet was being scheduled on all nodes regardless of the networking.go primary/worker logic. Each node ended up running its own isolated OVN SBDB raft cluster, preventing geneve tunnel formation and breaking all cross-node pod networking. At kubelet startup the primary node (identified by the absence of a bootstrap kubeconfig) receives the label node.microshift.io/role=primary. The ovnkube-master DaemonSet nodeSelector is updated to match that label so the full OVN database stack (sbdb/nbdb/northd) runs exclusively on the primary. Co-Authored-By: Claude Sonnet 4.6 (1M context) --- assets/components/ovn/multi-node/master/daemonset.yaml | 2 +- pkg/node/kubelet.go | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/assets/components/ovn/multi-node/master/daemonset.yaml b/assets/components/ovn/multi-node/master/daemonset.yaml index fe79c19640..c86dc3e244 100644 --- a/assets/components/ovn/multi-node/master/daemonset.yaml +++ b/assets/components/ovn/multi-node/master/daemonset.yaml @@ -472,7 +472,7 @@ spec: privileged: true terminationMessagePolicy: FallbackToLogsOnError nodeSelector: - node-role.kubernetes.io/master: "" + node.microshift.io/role: primary kubernetes.io/os: "linux" volumes: # for checking ovs-configuration service diff --git a/pkg/node/kubelet.go b/pkg/node/kubelet.go index 6a0c8ce6d8..54385e4a74 100644 --- a/pkg/node/kubelet.go +++ b/pkg/node/kubelet.go @@ -89,6 +89,9 @@ func (s *KubeletServer) configure(cfg *config.Config) { kubeletFlags.NodeLabels["node-role.kubernetes.io/worker"] = "" kubeletFlags.NodeLabels["node.openshift.io/os_id"] = osID kubeletFlags.NodeLabels["node.kubernetes.io/instance-type"] = "rhde" + if !cfg.BootstrapKubeConfigExists() { + kubeletFlags.NodeLabels["node.microshift.io/role"] = "primary" + } kubeletConfig, err := loadConfigFile(filepath.Join(config.DataDir, "/resources/kubelet/config/config.yaml")) From 47ce31f5253ba08fc87940c1d6a9dc667f8dac41 Mon Sep 17 00:00:00 2001 From: Evgeny Slutsky Date: Thu, 10 Sep 2026 11:04:37 +0200 Subject: [PATCH 2/3] NO-ISSUE: Fix OVN multinode SBDB connectivity for primary and worker nodes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Commit 8a0f4f23eb removed --nb-address/--sb-address from ovnkube --init-node, which were the only mechanism pointing worker nodes at the primary's NB/SB databases. After that removal ovnkube fell back to local unix sockets that do not exist on workers, leaving each node with an isolated OVN RAFT cluster. Three related fixes: 1. ovnkube.conf (configmap): in multinode mode add [OvnNorth]/[OvnSouth] stanzas with the primary's IP and NB/SB port so ovnkube can find the remote databases from the config file. 2. networking.go — DaemonSet selection: worker nodes (BootstrapKubeConfigExists) only deploy the node DaemonSet; the primary deploys master + node. Workers no longer run a local SBDB/NBDB that would be invisible to the primary. 3. networking.go — configmap guard: only the primary writes the ovnkube-config ConfigMap. A worker writing it would overwrite the correct primary IP with its own, breaking SBDB connectivity for all nodes. Co-Authored-By: Claude Sonnet 4.6 (1M context) --- assets/components/ovn/common/configmap.yaml | 8 +++++ pkg/components/networking.go | 36 +++++++++++++-------- 2 files changed, 31 insertions(+), 13 deletions(-) diff --git a/assets/components/ovn/common/configmap.yaml b/assets/components/ovn/common/configmap.yaml index 7279ce1fb8..1f435a2d7f 100644 --- a/assets/components/ovn/common/configmap.yaml +++ b/assets/components/ovn/common/configmap.yaml @@ -34,3 +34,11 @@ data: election-lease-duration=137 election-renew-deadline=107 election-retry-period=26 +{{- if .MultiNodeEnabled}} + + [OvnNorth] + address=tcp:{{.NodeIP}}:{{.OVN_NB_PORT}} + + [OvnSouth] + address=tcp:{{.NodeIP}}:{{.OVN_SB_PORT}} +{{- end}} diff --git a/pkg/components/networking.go b/pkg/components/networking.go index 2e6ccad927..36b2401d8e 100644 --- a/pkg/components/networking.go +++ b/pkg/components/networking.go @@ -61,9 +61,12 @@ func startCNIPlugin(ctx context.Context, cfg *config.Config, kubeconfigPath stri ) if cfg.MultiNode.Enabled { - apps = []string{ - "components/ovn/multi-node/master/daemonset.yaml", - "components/ovn/multi-node/node/daemonset.yaml", + // node DaemonSet runs on every multinode member (primary and workers). + apps = []string{"components/ovn/multi-node/node/daemonset.yaml"} + if !cfg.BootstrapKubeConfigExists() { + // Primary node only: also deploy the OVN database stack (sbdb/nbdb/northd). + // Workers connect to the primary's databases via the ovnkube.conf [OvnNorth]/[OvnSouth] stanzas. + apps = append([]string{"components/ovn/multi-node/master/daemonset.yaml"}, apps...) } } @@ -110,17 +113,24 @@ func startCNIPlugin(ctx context.Context, cfg *config.Config, kubeconfigPath stri return err } - // Multinode only params: OVN_NB_PORT, OVN_SB_PORT + // Multinode only params: OVN_NB_PORT, OVN_SB_PORT, MultiNodeEnabled extraParams := assets.RenderParams{ - "OVNConfig": ovnConfig, - "KubeconfigPath": kubeconfigPath, - "KubeconfigDir": filepath.Join(config.DataDir, "/resources/kubeadmin"), - "OVN_NB_PORT": ovn.OVN_NB_PORT, - "OVN_SB_PORT": ovn.OVN_SB_PORT, - } - if err := assets.ApplyConfigMaps(ctx, cm, renderTemplate, renderParamsFromConfig(cfg, extraParams), kubeconfigPath); err != nil { - klog.Warningf("Failed to apply configMap %v %v", cm, err) - return err + "OVNConfig": ovnConfig, + "KubeconfigPath": kubeconfigPath, + "KubeconfigDir": filepath.Join(config.DataDir, "/resources/kubeadmin"), + "OVN_NB_PORT": ovn.OVN_NB_PORT, + "OVN_SB_PORT": ovn.OVN_SB_PORT, + "MultiNodeEnabled": cfg.MultiNode.Enabled, + } + // In multinode mode the configmap contains [OvnNorth]/[OvnSouth] stanzas + // with the primary's IP. Only the primary may write it; a worker applying + // the configmap would overwrite the primary IP with its own, breaking SBDB + // connectivity for every node that reads the configmap afterwards. + if !cfg.MultiNode.Enabled || !cfg.BootstrapKubeConfigExists() { + if err := assets.ApplyConfigMaps(ctx, cm, renderTemplate, renderParamsFromConfig(cfg, extraParams), kubeconfigPath); err != nil { + klog.Warningf("Failed to apply configMap %v %v", cm, err) + return err + } } if err := assets.ApplyDaemonSets(ctx, apps, renderTemplate, renderParamsFromConfig(cfg, extraParams), kubeconfigPath); err != nil { klog.Warningf("Failed to apply apps %v %v", apps, err) From 1c30dd228709459bfd125160cd44b4138d82e1c1 Mon Sep 17 00:00:00 2001 From: Evgeny Slutsky Date: Thu, 10 Sep 2026 11:04:53 +0200 Subject: [PATCH 3/3] NO-ISSUE: Bootstrap OVN worker node and clean stale OVS state on teardown Two related fixes for multinode worker nodes: ovnkube-node DaemonSet startup: The ovnkube binary cannot parse the [OvnNorth]/[OvnSouth] address= fields from ovnkube.conf (config format mismatch in this binary version), so it falls back to local unix sockets that do not exist on workers. Fix the startup script to detect whether the node is a worker (SB address in the configmap points to a different host) and: - Set ovn-encap-type=geneve and ovn-encap-ip in OVS so ovn-controller can register the chassis and form geneve tunnels. - Start socat relays forwarding the local NB/SB unix sockets to the primary's TCP ports so ovnkube can connect via its expected unix path. - Set ovn-remote=tcp:PRIMARY:9642 in OVS for direct ovn-controller access. - Export OVN_SB_DB/OVN_NB_DB for subprocess ovn-sbctl/ovn-nbctl calls. Improve ovn-controller wait logic to verify the socket is actually serving connections (not just that the file exists), and handle stale processes and .ctl sockets from previous container instances. microshift-cleanup-data.sh: Clear OVN-related OVS external_ids (ovn-remote, ovn-encap-ip, etc.) before stopping ovsdb-server so the next MicroShift start cannot pick up a stale SBDB address from a previous run. Clears the keys while OVS is still reachable, then stops ovsdb-server and kills OVN processes. Co-Authored-By: Claude Sonnet 4.6 (1M context) --- .../ovn/multi-node/node/daemonset.yaml | 110 ++++++++++++++++-- scripts/microshift-cleanup-data.sh | 11 ++ 2 files changed, 114 insertions(+), 7 deletions(-) diff --git a/assets/components/ovn/multi-node/node/daemonset.yaml b/assets/components/ovn/multi-node/node/daemonset.yaml index 7d01ee561d..6488fe0d9d 100644 --- a/assets/components/ovn/multi-node/node/daemonset.yaml +++ b/assets/components/ovn/multi-node/node/daemonset.yaml @@ -55,13 +55,49 @@ spec: # K8S_NODE_IP triggers reconcilation of this daemon when node IP changes echo "$(date -Iseconds) - starting ovn-controller, Node: ${K8S_NODE} IP: ${K8S_NODE_IP}" - # Wait for the SBDB unix socket to appear. The sbdb container - # removes stale sockets and creates fresh ones on startup. - # Connecting to a stale socket would cause ovn-controller to - # cache a raft commit index higher than the fresh SBDB's. - echo "Waiting for SBDB socket..." - while [ ! -S /run/ovn/ovnsb_db.sock ]; do sleep 1; done - echo "SBDB socket ready" + # Wait for SBDB connectivity before starting ovn-controller. + # Primary node: ovn-remote is unix:/var/run/ovn/ovnsb_db.sock and the + # socket is served by the sbdb container in ovnkube-master. + # Worker node: ovnkube-node starts a socat relay that serves the local + # unix socket and forwards connections to the primary's TCP SBDB. + # Avoid treating a stale unix socket (left from a previous run) as + # ready: for unix: remotes, verify the socket accepts connections. + echo "Waiting for SBDB..." + until + OVN_REMOTE=$(ovs-vsctl --timeout=5 get Open_vSwitch . external_ids:ovn-remote 2>/dev/null | tr -d '"') + if [[ "${OVN_REMOTE}" =~ ^tcp: ]]; then + true # TCP remote set by startup script — ready immediately + elif [[ "${OVN_REMOTE}" =~ ^unix: ]]; then + # Accept unix: only when something is actually listening on the socket + socat -t2 /dev/null "UNIX-CONNECT:${OVN_REMOTE#unix:}" 2>/dev/null + else + false + fi + do + sleep 1 + done + echo "SBDB ready (ovn-remote=${OVN_REMOTE})" + + # If a previous ovn-controller instance is still running (possible + # because it shares the host PID namespace), kill it so the new + # invocation does not abort with "already running". Leave the pid + # file in place so the concurrent ovnkube-node container can always + # open it — ovn-controller will overwrite the file after the old + # process has exited. + if [ -f /var/run/ovn/ovn-controller.pid ]; then + OLD_PID=$(cat /var/run/ovn/ovn-controller.pid) + if kill -0 "${OLD_PID}" 2>/dev/null; then + echo "Killing stale ovn-controller process ${OLD_PID}" + kill "${OLD_PID}" 2>/dev/null || true + # Wait for the process to exit (up to 5 s) + for _ in $(seq 1 10); do + kill -0 "${OLD_PID}" 2>/dev/null || break + sleep 0.5 + done + fi + fi + # Remove stale .ctl sockets that belong to the dead process. + rm -f /var/run/ovn/ovn-controller.*.ctl exec ovn-controller unix:/var/run/openvswitch/db.sock -vfile:off \ --no-chdir --pidfile=/var/run/ovn/ovn-controller.pid \ @@ -146,6 +182,66 @@ spec: # the functionality depends on ip_forwarding being enabled fi + # The configmap's [OvnNorth]/[OvnSouth] address= fields point to the + # primary node's NB/SB TCP ports. They are used differently: + # + # Primary node (SB_ADDR host == own IP): + # The nbdb/sbdb containers in ovnkube-master serve local unix + # sockets on this host. Only set encap OVS external_ids; do NOT + # touch the sockets or start relays. + # + # Worker node (SB_ADDR host != own IP): + # No local nbdb/sbdb containers run. The ovnkube binary cannot + # parse the address= fields (upstream config format mismatch) and + # falls back to the stale local unix sockets, blocking startup. + # Fix: start socat relays that serve the local unix sockets and + # forward connections to the primary's TCP ports; also set + # ovn-remote in OVS so ovn-controller uses TCP directly. + NB_ADDR=$(awk 'BEGIN{f=0} /^\[OvnNorth\]/{f=1} f && /^address=/{print substr($0,9); exit}' \ + /run/ovnkube-config/ovnkube.conf 2>/dev/null) + SB_ADDR=$(awk 'BEGIN{f=0} /^\[OvnSouth\]/{f=1} f && /^address=/{print substr($0,9); exit}' \ + /run/ovnkube-config/ovnkube.conf 2>/dev/null) + if [[ "${SB_ADDR}" =~ ^tcp: ]]; then + # Always set encap type and IP so ovn-controller can register the chassis + ovs-vsctl --timeout=5 set Open_vSwitch . \ + "external_ids:ovn-encap-type=geneve" \ + "external_ids:ovn-encap-ip=${K8S_NODE_IP}" || true + + # Worker detection: SB address points to a DIFFERENT host than ours + if [[ "${SB_ADDR}" != *"${K8S_NODE_IP}"* ]]; then + echo "$(date -Iseconds) - worker node: setting up OVN socket relays to primary" + + # Remove stale socket files from any previous run. + # Any socat processes from a prior container instance are + # already gone — CRI-O kills them via the cgroup on stop. + rm -f /run/ovn/ovnnb_db.sock /run/ovn/ovnsb_db.sock + + # Start socat relays: local unix socket → primary TCP port. + # These background processes survive the exec and are killed + # when the container stops (cgroup boundary). + NB_HOST_PORT="${NB_ADDR#tcp:}" + SB_HOST_PORT="${SB_ADDR#tcp:}" + socat UNIX-LISTEN:/run/ovn/ovnnb_db.sock,fork,reuseaddr \ + TCP:${NB_HOST_PORT} & + socat UNIX-LISTEN:/run/ovn/ovnsb_db.sock,fork,reuseaddr \ + TCP:${SB_HOST_PORT} & + echo "$(date -Iseconds) - NB relay → ${NB_HOST_PORT}, SB relay → ${SB_HOST_PORT}" + + # Set ovn-remote so ovn-controller reaches the primary SBDB + # over TCP directly (the socat relay also works, but TCP is simpler) + ovs-vsctl --timeout=5 set Open_vSwitch . \ + "external_ids:ovn-remote=${SB_ADDR}" || true + + # Export OVN_SB_DB/OVN_NB_DB so that ovn-sbctl/ovn-nbctl + # subprocess calls inside the ovnkube binary also use TCP. + export OVN_SB_DB="${SB_ADDR}" + export OVN_NB_DB="${NB_ADDR}" + echo "$(date -Iseconds) - setting ovn-remote=${SB_ADDR} encap-ip=${K8S_NODE_IP}" + else + echo "$(date -Iseconds) - primary node: setting encap-ip=${K8S_NODE_IP}" + fi + fi + echo "I$(date "+%m%d %H:%M:%S.%N") - ovnkube-node - start ovnkube --init-node ${K8S_NODE}" exec /usr/bin/ovnkube \ --init-node "${K8S_NODE}" \ diff --git a/scripts/microshift-cleanup-data.sh b/scripts/microshift-cleanup-data.sh index 2144d0e87c..8de0214046 100755 --- a/scripts/microshift-cleanup-data.sh +++ b/scripts/microshift-cleanup-data.sh @@ -102,6 +102,17 @@ function clean_processes() { fi if ${FULL_CLEAN} || ${OVN_CLEAN} ; then + # Remove OVN-related entries from OVS external_ids BEFORE stopping ovsdb-server + # so that ovs-vsctl can still reach the socket. This ensures a subsequent + # MicroShift start picks up the correct SBDB address rather than a stale + # unix socket or TCP endpoint from the previous run. + for key in ovn-remote ovn-encap-type ovn-encap-ip ovn-bridge-mappings \ + ovn-monitor-all ovn-openflow-probe-interval ovn-remote-probe-interval ; do + val=$(ovs-vsctl --if-exists get Open_vSwitch . "external_ids:${key}" 2>/dev/null | tr -d '"') || true + if [ -n "${val}" ]; then + ovs-vsctl remove Open_vSwitch . external_ids "${key}" "${val}" 2>/dev/null || true + fi + done echo Killing conmon, pause and OVN processes systemctl stop --now ovsdb-server.service 2>/dev/null || true for pname in conmon pause ovn-controller ovn-northd ; do