USHIFT-7432: Fix OVN multinode SBDB connectivity regression

assets/components/ovn/common/configmap.yaml

@@ -34,3 +34,11 @@ data:
     election-lease-duration=137
     election-renew-deadline=107
     election-retry-period=26
+{{- if .MultiNodeEnabled}}
+
+    [OvnNorth]
+    address=tcp:{{.NodeIP}}:{{.OVN_NB_PORT}}
+
+    [OvnSouth]
+    address=tcp:{{.NodeIP}}:{{.OVN_SB_PORT}}
+{{- end}}

assets/components/ovn/multi-node/master/daemonset.yaml

@@ -472,7 +472,7 @@ spec:
           privileged: true
         terminationMessagePolicy: FallbackToLogsOnError
       nodeSelector:
-        node-role.kubernetes.io/master: ""
+        node.microshift.io/role: primary
         kubernetes.io/os: "linux"
       volumes:
       # for checking ovs-configuration service

assets/components/ovn/multi-node/node/daemonset.yaml

@@ -55,13 +55,49 @@ spec:
           # K8S_NODE_IP triggers reconcilation of this daemon when node IP changes
           echo "$(date -Iseconds) - starting ovn-controller, Node: ${K8S_NODE} IP: ${K8S_NODE_IP}"
 
-          # Wait for the SBDB unix socket to appear. The sbdb container
-          # removes stale sockets and creates fresh ones on startup.
-          # Connecting to a stale socket would cause ovn-controller to
-          # cache a raft commit index higher than the fresh SBDB's.
-          echo "Waiting for SBDB socket..."
-          while [ ! -S /run/ovn/ovnsb_db.sock ]; do sleep 1; done
-          echo "SBDB socket ready"
+          # Wait for SBDB connectivity before starting ovn-controller.
+          # Primary node: ovn-remote is unix:/var/run/ovn/ovnsb_db.sock and the
+          #   socket is served by the sbdb container in ovnkube-master.
+          # Worker node: ovnkube-node starts a socat relay that serves the local
+          #   unix socket and forwards connections to the primary's TCP SBDB.
+          # Avoid treating a stale unix socket (left from a previous run) as
+          # ready: for unix: remotes, verify the socket accepts connections.
+          echo "Waiting for SBDB..."
+          until
+            OVN_REMOTE=$(ovs-vsctl --timeout=5 get Open_vSwitch . external_ids:ovn-remote 2>/dev/null | tr -d '"')
+            if [[ "${OVN_REMOTE}" =~ ^tcp: ]]; then
+              true   # TCP remote set by startup script — ready immediately
+            elif [[ "${OVN_REMOTE}" =~ ^unix: ]]; then
+              # Accept unix: only when something is actually listening on the socket
+              socat -t2 /dev/null "UNIX-CONNECT:${OVN_REMOTE#unix:}" 2>/dev/null
+            else
+              false
+            fi
+          do
+            sleep 1
+          done
+          echo "SBDB ready (ovn-remote=${OVN_REMOTE})"
+
+          # If a previous ovn-controller instance is still running (possible
+          # because it shares the host PID namespace), kill it so the new
+          # invocation does not abort with "already running".  Leave the pid
+          # file in place so the concurrent ovnkube-node container can always
+          # open it — ovn-controller will overwrite the file after the old
+          # process has exited.
+          if [ -f /var/run/ovn/ovn-controller.pid ]; then
+            OLD_PID=$(cat /var/run/ovn/ovn-controller.pid)
+            if kill -0 "${OLD_PID}" 2>/dev/null; then
+              echo "Killing stale ovn-controller process ${OLD_PID}"
+              kill "${OLD_PID}" 2>/dev/null || true
+              # Wait for the process to exit (up to 5 s)
+              for _ in $(seq 1 10); do
+                kill -0 "${OLD_PID}" 2>/dev/null || break
+                sleep 0.5
+              done
+            fi
+          fi
+          # Remove stale .ctl sockets that belong to the dead process.
+          rm -f /var/run/ovn/ovn-controller.*.ctl
 
           exec ovn-controller unix:/var/run/openvswitch/db.sock -vfile:off \
             --no-chdir --pidfile=/var/run/ovn/ovn-controller.pid \
@@ -146,6 +182,66 @@ spec:
             # the functionality depends on ip_forwarding being enabled
           fi
 
+          # The configmap's [OvnNorth]/[OvnSouth] address= fields point to the
+          # primary node's NB/SB TCP ports.  They are used differently:
+          #
+          # Primary node (SB_ADDR host == own IP):
+          #   The nbdb/sbdb containers in ovnkube-master serve local unix
+          #   sockets on this host.  Only set encap OVS external_ids; do NOT
+          #   touch the sockets or start relays.
+          #
+          # Worker node (SB_ADDR host != own IP):
+          #   No local nbdb/sbdb containers run.  The ovnkube binary cannot
+          #   parse the address= fields (upstream config format mismatch) and
+          #   falls back to the stale local unix sockets, blocking startup.
+          #   Fix: start socat relays that serve the local unix sockets and
+          #   forward connections to the primary's TCP ports; also set
+          #   ovn-remote in OVS so ovn-controller uses TCP directly.
+          NB_ADDR=$(awk 'BEGIN{f=0} /^\[OvnNorth\]/{f=1} f && /^address=/{print substr($0,9); exit}' \
+              /run/ovnkube-config/ovnkube.conf 2>/dev/null)
+          SB_ADDR=$(awk 'BEGIN{f=0} /^\[OvnSouth\]/{f=1} f && /^address=/{print substr($0,9); exit}' \
+              /run/ovnkube-config/ovnkube.conf 2>/dev/null)
+          if [[ "${SB_ADDR}" =~ ^tcp: ]]; then
+              # Always set encap type and IP so ovn-controller can register the chassis
+              ovs-vsctl --timeout=5 set Open_vSwitch . \
+                  "external_ids:ovn-encap-type=geneve" \
+                  "external_ids:ovn-encap-ip=${K8S_NODE_IP}" || true
+
+              # Worker detection: SB address points to a DIFFERENT host than ours
+              if [[ "${SB_ADDR}" != *"${K8S_NODE_IP}"* ]]; then
+                  echo "$(date -Iseconds) - worker node: setting up OVN socket relays to primary"
+
+                  # Remove stale socket files from any previous run.
+                  # Any socat processes from a prior container instance are
+                  # already gone — CRI-O kills them via the cgroup on stop.
+                  rm -f /run/ovn/ovnnb_db.sock /run/ovn/ovnsb_db.sock
+
+                  # Start socat relays: local unix socket → primary TCP port.
+                  # These background processes survive the exec and are killed
+                  # when the container stops (cgroup boundary).
+                  NB_HOST_PORT="${NB_ADDR#tcp:}"
+                  SB_HOST_PORT="${SB_ADDR#tcp:}"
+                  socat UNIX-LISTEN:/run/ovn/ovnnb_db.sock,fork,reuseaddr \
+                      TCP:${NB_HOST_PORT} &
+                  socat UNIX-LISTEN:/run/ovn/ovnsb_db.sock,fork,reuseaddr \
+                      TCP:${SB_HOST_PORT} &
+                  echo "$(date -Iseconds) - NB relay → ${NB_HOST_PORT}, SB relay → ${SB_HOST_PORT}"
+
+                  # Set ovn-remote so ovn-controller reaches the primary SBDB
+                  # over TCP directly (the socat relay also works, but TCP is simpler)
+                  ovs-vsctl --timeout=5 set Open_vSwitch . \
+                      "external_ids:ovn-remote=${SB_ADDR}" || true
+
+                  # Export OVN_SB_DB/OVN_NB_DB so that ovn-sbctl/ovn-nbctl
+                  # subprocess calls inside the ovnkube binary also use TCP.
+                  export OVN_SB_DB="${SB_ADDR}"
+                  export OVN_NB_DB="${NB_ADDR}"
+                  echo "$(date -Iseconds) - setting ovn-remote=${SB_ADDR} encap-ip=${K8S_NODE_IP}"
+              else
+                  echo "$(date -Iseconds) - primary node: setting encap-ip=${K8S_NODE_IP}"
+              fi
+          fi
+
           echo "I$(date "+%m%d %H:%M:%S.%N") - ovnkube-node - start ovnkube --init-node ${K8S_NODE}"
           exec /usr/bin/ovnkube \
             --init-node "${K8S_NODE}" \

pkg/components/networking.go

@@ -61,9 +61,12 @@ func startCNIPlugin(ctx context.Context, cfg *config.Config, kubeconfigPath stri
 	)
 
 	if cfg.MultiNode.Enabled {
-		apps = []string{
-			"components/ovn/multi-node/master/daemonset.yaml",
-			"components/ovn/multi-node/node/daemonset.yaml",
+		// node DaemonSet runs on every multinode member (primary and workers).
+		apps = []string{"components/ovn/multi-node/node/daemonset.yaml"}
+		if !cfg.BootstrapKubeConfigExists() {
+			// Primary node only: also deploy the OVN database stack (sbdb/nbdb/northd).
+			// Workers connect to the primary's databases via the ovnkube.conf [OvnNorth]/[OvnSouth] stanzas.
+			apps = append([]string{"components/ovn/multi-node/master/daemonset.yaml"}, apps...)
 		}
 	}
 
@@ -110,17 +113,24 @@ func startCNIPlugin(ctx context.Context, cfg *config.Config, kubeconfigPath stri
 		return err
 	}
 
-	// Multinode only params: OVN_NB_PORT, OVN_SB_PORT
+	// Multinode only params: OVN_NB_PORT, OVN_SB_PORT, MultiNodeEnabled
 	extraParams := assets.RenderParams{
-		"OVNConfig":      ovnConfig,
-		"KubeconfigPath": kubeconfigPath,
-		"KubeconfigDir":  filepath.Join(config.DataDir, "/resources/kubeadmin"),
-		"OVN_NB_PORT":    ovn.OVN_NB_PORT,
-		"OVN_SB_PORT":    ovn.OVN_SB_PORT,
-	}
-	if err := assets.ApplyConfigMaps(ctx, cm, renderTemplate, renderParamsFromConfig(cfg, extraParams), kubeconfigPath); err != nil {
-		klog.Warningf("Failed to apply configMap %v %v", cm, err)
-		return err
+		"OVNConfig":        ovnConfig,
+		"KubeconfigPath":   kubeconfigPath,
+		"KubeconfigDir":    filepath.Join(config.DataDir, "/resources/kubeadmin"),
+		"OVN_NB_PORT":      ovn.OVN_NB_PORT,
+		"OVN_SB_PORT":      ovn.OVN_SB_PORT,
+		"MultiNodeEnabled": cfg.MultiNode.Enabled,
+	}
+	// In multinode mode the configmap contains [OvnNorth]/[OvnSouth] stanzas
+	// with the primary's IP. Only the primary may write it; a worker applying
+	// the configmap would overwrite the primary IP with its own, breaking SBDB
+	// connectivity for every node that reads the configmap afterwards.
+	if !cfg.MultiNode.Enabled || !cfg.BootstrapKubeConfigExists() {
+		if err := assets.ApplyConfigMaps(ctx, cm, renderTemplate, renderParamsFromConfig(cfg, extraParams), kubeconfigPath); err != nil {
+			klog.Warningf("Failed to apply configMap %v %v", cm, err)
+			return err
+		}
 	}
 	if err := assets.ApplyDaemonSets(ctx, apps, renderTemplate, renderParamsFromConfig(cfg, extraParams), kubeconfigPath); err != nil {
 		klog.Warningf("Failed to apply apps %v %v", apps, err)

pkg/node/kubelet.go

@@ -89,6 +89,9 @@ func (s *KubeletServer) configure(cfg *config.Config) {
 	kubeletFlags.NodeLabels["node-role.kubernetes.io/worker"] = ""
 	kubeletFlags.NodeLabels["node.openshift.io/os_id"] = osID
 	kubeletFlags.NodeLabels["node.kubernetes.io/instance-type"] = "rhde"
+	if !cfg.BootstrapKubeConfigExists() {
+		kubeletFlags.NodeLabels["node.microshift.io/role"] = "primary"
+	}
 
 	kubeletConfig, err := loadConfigFile(filepath.Join(config.DataDir, "/resources/kubelet/config/config.yaml"))
 

scripts/microshift-cleanup-data.sh

@@ -102,6 +102,17 @@ function clean_processes() {
     fi
 
     if ${FULL_CLEAN} || ${OVN_CLEAN} ; then
+        # Remove OVN-related entries from OVS external_ids BEFORE stopping ovsdb-server
+        # so that ovs-vsctl can still reach the socket. This ensures a subsequent
+        # MicroShift start picks up the correct SBDB address rather than a stale
+        # unix socket or TCP endpoint from the previous run.
+        for key in ovn-remote ovn-encap-type ovn-encap-ip ovn-bridge-mappings \
+                   ovn-monitor-all ovn-openflow-probe-interval ovn-remote-probe-interval ; do
+            val=$(ovs-vsctl --if-exists get Open_vSwitch . "external_ids:${key}" 2>/dev/null | tr -d '"') || true
+            if [ -n "${val}" ]; then
+                ovs-vsctl remove Open_vSwitch . external_ids "${key}" "${val}" 2>/dev/null || true
+            fi
+        done
         echo Killing conmon, pause and OVN processes
         systemctl stop --now ovsdb-server.service 2>/dev/null || true
         for pname in conmon pause ovn-controller ovn-northd ; do

test/bin/ci_phase_boot_and_test.sh

@@ -22,9 +22,12 @@ prepare_scenario_sources() {
     rm -rf "${SCENARIOS_TO_RUN}"
     mkdir -p "${SCENARIOS_TO_RUN}"
     cp "${SCENARIO_SOURCES}"/*.sh "${SCENARIOS_TO_RUN}"/
-    if ${EXCLUDE_CNCF_CONFORMANCE}; then
-        find "${SCENARIOS_TO_RUN}" -name "*cncf-conformance.sh" -delete
-    fi
+    # TODO: Temporarily disabled so that the CNCF conformance scenario runs
+    # unconditionally while the multinode OVN SBDB fix (PR #7344) is validated
+    # in CI. Revert once the job is confirmed green.
+    # if ${EXCLUDE_CNCF_CONFORMANCE}; then
+    #     find "${SCENARIOS_TO_RUN}" -name "*cncf-conformance.sh" -delete
+    # fi
 }
 
 # Log output automatically