diff --git a/assets/components/ovn/common/configmap.yaml b/assets/components/ovn/common/configmap.yaml index 7279ce1fb8..1f435a2d7f 100644 --- a/assets/components/ovn/common/configmap.yaml +++ b/assets/components/ovn/common/configmap.yaml @@ -34,3 +34,11 @@ data: election-lease-duration=137 election-renew-deadline=107 election-retry-period=26 +{{- if .MultiNodeEnabled}} + + [OvnNorth] + address=tcp:{{.NodeIP}}:{{.OVN_NB_PORT}} + + [OvnSouth] + address=tcp:{{.NodeIP}}:{{.OVN_SB_PORT}} +{{- end}} diff --git a/assets/components/ovn/multi-node/master/daemonset.yaml b/assets/components/ovn/multi-node/master/daemonset.yaml index fe79c19640..c86dc3e244 100644 --- a/assets/components/ovn/multi-node/master/daemonset.yaml +++ b/assets/components/ovn/multi-node/master/daemonset.yaml @@ -472,7 +472,7 @@ spec: privileged: true terminationMessagePolicy: FallbackToLogsOnError nodeSelector: - node-role.kubernetes.io/master: "" + node.microshift.io/role: primary kubernetes.io/os: "linux" volumes: # for checking ovs-configuration service diff --git a/assets/components/ovn/multi-node/node/daemonset.yaml b/assets/components/ovn/multi-node/node/daemonset.yaml index 7d01ee561d..6488fe0d9d 100644 --- a/assets/components/ovn/multi-node/node/daemonset.yaml +++ b/assets/components/ovn/multi-node/node/daemonset.yaml @@ -55,13 +55,49 @@ spec: # K8S_NODE_IP triggers reconcilation of this daemon when node IP changes echo "$(date -Iseconds) - starting ovn-controller, Node: ${K8S_NODE} IP: ${K8S_NODE_IP}" - # Wait for the SBDB unix socket to appear. The sbdb container - # removes stale sockets and creates fresh ones on startup. - # Connecting to a stale socket would cause ovn-controller to - # cache a raft commit index higher than the fresh SBDB's. - echo "Waiting for SBDB socket..." - while [ ! -S /run/ovn/ovnsb_db.sock ]; do sleep 1; done - echo "SBDB socket ready" + # Wait for SBDB connectivity before starting ovn-controller. + # Primary node: ovn-remote is unix:/var/run/ovn/ovnsb_db.sock and the + # socket is served by the sbdb container in ovnkube-master. + # Worker node: ovnkube-node starts a socat relay that serves the local + # unix socket and forwards connections to the primary's TCP SBDB. + # Avoid treating a stale unix socket (left from a previous run) as + # ready: for unix: remotes, verify the socket accepts connections. + echo "Waiting for SBDB..." + until + OVN_REMOTE=$(ovs-vsctl --timeout=5 get Open_vSwitch . external_ids:ovn-remote 2>/dev/null | tr -d '"') + if [[ "${OVN_REMOTE}" =~ ^tcp: ]]; then + true # TCP remote set by startup script — ready immediately + elif [[ "${OVN_REMOTE}" =~ ^unix: ]]; then + # Accept unix: only when something is actually listening on the socket + socat -t2 /dev/null "UNIX-CONNECT:${OVN_REMOTE#unix:}" 2>/dev/null + else + false + fi + do + sleep 1 + done + echo "SBDB ready (ovn-remote=${OVN_REMOTE})" + + # If a previous ovn-controller instance is still running (possible + # because it shares the host PID namespace), kill it so the new + # invocation does not abort with "already running". Leave the pid + # file in place so the concurrent ovnkube-node container can always + # open it — ovn-controller will overwrite the file after the old + # process has exited. + if [ -f /var/run/ovn/ovn-controller.pid ]; then + OLD_PID=$(cat /var/run/ovn/ovn-controller.pid) + if kill -0 "${OLD_PID}" 2>/dev/null; then + echo "Killing stale ovn-controller process ${OLD_PID}" + kill "${OLD_PID}" 2>/dev/null || true + # Wait for the process to exit (up to 5 s) + for _ in $(seq 1 10); do + kill -0 "${OLD_PID}" 2>/dev/null || break + sleep 0.5 + done + fi + fi + # Remove stale .ctl sockets that belong to the dead process. + rm -f /var/run/ovn/ovn-controller.*.ctl exec ovn-controller unix:/var/run/openvswitch/db.sock -vfile:off \ --no-chdir --pidfile=/var/run/ovn/ovn-controller.pid \ @@ -146,6 +182,66 @@ spec: # the functionality depends on ip_forwarding being enabled fi + # The configmap's [OvnNorth]/[OvnSouth] address= fields point to the + # primary node's NB/SB TCP ports. They are used differently: + # + # Primary node (SB_ADDR host == own IP): + # The nbdb/sbdb containers in ovnkube-master serve local unix + # sockets on this host. Only set encap OVS external_ids; do NOT + # touch the sockets or start relays. + # + # Worker node (SB_ADDR host != own IP): + # No local nbdb/sbdb containers run. The ovnkube binary cannot + # parse the address= fields (upstream config format mismatch) and + # falls back to the stale local unix sockets, blocking startup. + # Fix: start socat relays that serve the local unix sockets and + # forward connections to the primary's TCP ports; also set + # ovn-remote in OVS so ovn-controller uses TCP directly. + NB_ADDR=$(awk 'BEGIN{f=0} /^\[OvnNorth\]/{f=1} f && /^address=/{print substr($0,9); exit}' \ + /run/ovnkube-config/ovnkube.conf 2>/dev/null) + SB_ADDR=$(awk 'BEGIN{f=0} /^\[OvnSouth\]/{f=1} f && /^address=/{print substr($0,9); exit}' \ + /run/ovnkube-config/ovnkube.conf 2>/dev/null) + if [[ "${SB_ADDR}" =~ ^tcp: ]]; then + # Always set encap type and IP so ovn-controller can register the chassis + ovs-vsctl --timeout=5 set Open_vSwitch . \ + "external_ids:ovn-encap-type=geneve" \ + "external_ids:ovn-encap-ip=${K8S_NODE_IP}" || true + + # Worker detection: SB address points to a DIFFERENT host than ours + if [[ "${SB_ADDR}" != *"${K8S_NODE_IP}"* ]]; then + echo "$(date -Iseconds) - worker node: setting up OVN socket relays to primary" + + # Remove stale socket files from any previous run. + # Any socat processes from a prior container instance are + # already gone — CRI-O kills them via the cgroup on stop. + rm -f /run/ovn/ovnnb_db.sock /run/ovn/ovnsb_db.sock + + # Start socat relays: local unix socket → primary TCP port. + # These background processes survive the exec and are killed + # when the container stops (cgroup boundary). + NB_HOST_PORT="${NB_ADDR#tcp:}" + SB_HOST_PORT="${SB_ADDR#tcp:}" + socat UNIX-LISTEN:/run/ovn/ovnnb_db.sock,fork,reuseaddr \ + TCP:${NB_HOST_PORT} & + socat UNIX-LISTEN:/run/ovn/ovnsb_db.sock,fork,reuseaddr \ + TCP:${SB_HOST_PORT} & + echo "$(date -Iseconds) - NB relay → ${NB_HOST_PORT}, SB relay → ${SB_HOST_PORT}" + + # Set ovn-remote so ovn-controller reaches the primary SBDB + # over TCP directly (the socat relay also works, but TCP is simpler) + ovs-vsctl --timeout=5 set Open_vSwitch . \ + "external_ids:ovn-remote=${SB_ADDR}" || true + + # Export OVN_SB_DB/OVN_NB_DB so that ovn-sbctl/ovn-nbctl + # subprocess calls inside the ovnkube binary also use TCP. + export OVN_SB_DB="${SB_ADDR}" + export OVN_NB_DB="${NB_ADDR}" + echo "$(date -Iseconds) - setting ovn-remote=${SB_ADDR} encap-ip=${K8S_NODE_IP}" + else + echo "$(date -Iseconds) - primary node: setting encap-ip=${K8S_NODE_IP}" + fi + fi + echo "I$(date "+%m%d %H:%M:%S.%N") - ovnkube-node - start ovnkube --init-node ${K8S_NODE}" exec /usr/bin/ovnkube \ --init-node "${K8S_NODE}" \ diff --git a/pkg/components/networking.go b/pkg/components/networking.go index 2e6ccad927..36b2401d8e 100644 --- a/pkg/components/networking.go +++ b/pkg/components/networking.go @@ -61,9 +61,12 @@ func startCNIPlugin(ctx context.Context, cfg *config.Config, kubeconfigPath stri ) if cfg.MultiNode.Enabled { - apps = []string{ - "components/ovn/multi-node/master/daemonset.yaml", - "components/ovn/multi-node/node/daemonset.yaml", + // node DaemonSet runs on every multinode member (primary and workers). + apps = []string{"components/ovn/multi-node/node/daemonset.yaml"} + if !cfg.BootstrapKubeConfigExists() { + // Primary node only: also deploy the OVN database stack (sbdb/nbdb/northd). + // Workers connect to the primary's databases via the ovnkube.conf [OvnNorth]/[OvnSouth] stanzas. + apps = append([]string{"components/ovn/multi-node/master/daemonset.yaml"}, apps...) } } @@ -110,17 +113,24 @@ func startCNIPlugin(ctx context.Context, cfg *config.Config, kubeconfigPath stri return err } - // Multinode only params: OVN_NB_PORT, OVN_SB_PORT + // Multinode only params: OVN_NB_PORT, OVN_SB_PORT, MultiNodeEnabled extraParams := assets.RenderParams{ - "OVNConfig": ovnConfig, - "KubeconfigPath": kubeconfigPath, - "KubeconfigDir": filepath.Join(config.DataDir, "/resources/kubeadmin"), - "OVN_NB_PORT": ovn.OVN_NB_PORT, - "OVN_SB_PORT": ovn.OVN_SB_PORT, - } - if err := assets.ApplyConfigMaps(ctx, cm, renderTemplate, renderParamsFromConfig(cfg, extraParams), kubeconfigPath); err != nil { - klog.Warningf("Failed to apply configMap %v %v", cm, err) - return err + "OVNConfig": ovnConfig, + "KubeconfigPath": kubeconfigPath, + "KubeconfigDir": filepath.Join(config.DataDir, "/resources/kubeadmin"), + "OVN_NB_PORT": ovn.OVN_NB_PORT, + "OVN_SB_PORT": ovn.OVN_SB_PORT, + "MultiNodeEnabled": cfg.MultiNode.Enabled, + } + // In multinode mode the configmap contains [OvnNorth]/[OvnSouth] stanzas + // with the primary's IP. Only the primary may write it; a worker applying + // the configmap would overwrite the primary IP with its own, breaking SBDB + // connectivity for every node that reads the configmap afterwards. + if !cfg.MultiNode.Enabled || !cfg.BootstrapKubeConfigExists() { + if err := assets.ApplyConfigMaps(ctx, cm, renderTemplate, renderParamsFromConfig(cfg, extraParams), kubeconfigPath); err != nil { + klog.Warningf("Failed to apply configMap %v %v", cm, err) + return err + } } if err := assets.ApplyDaemonSets(ctx, apps, renderTemplate, renderParamsFromConfig(cfg, extraParams), kubeconfigPath); err != nil { klog.Warningf("Failed to apply apps %v %v", apps, err) diff --git a/pkg/node/kubelet.go b/pkg/node/kubelet.go index 6a0c8ce6d8..54385e4a74 100644 --- a/pkg/node/kubelet.go +++ b/pkg/node/kubelet.go @@ -89,6 +89,9 @@ func (s *KubeletServer) configure(cfg *config.Config) { kubeletFlags.NodeLabels["node-role.kubernetes.io/worker"] = "" kubeletFlags.NodeLabels["node.openshift.io/os_id"] = osID kubeletFlags.NodeLabels["node.kubernetes.io/instance-type"] = "rhde" + if !cfg.BootstrapKubeConfigExists() { + kubeletFlags.NodeLabels["node.microshift.io/role"] = "primary" + } kubeletConfig, err := loadConfigFile(filepath.Join(config.DataDir, "/resources/kubelet/config/config.yaml")) diff --git a/scripts/microshift-cleanup-data.sh b/scripts/microshift-cleanup-data.sh index 2144d0e87c..8de0214046 100755 --- a/scripts/microshift-cleanup-data.sh +++ b/scripts/microshift-cleanup-data.sh @@ -102,6 +102,17 @@ function clean_processes() { fi if ${FULL_CLEAN} || ${OVN_CLEAN} ; then + # Remove OVN-related entries from OVS external_ids BEFORE stopping ovsdb-server + # so that ovs-vsctl can still reach the socket. This ensures a subsequent + # MicroShift start picks up the correct SBDB address rather than a stale + # unix socket or TCP endpoint from the previous run. + for key in ovn-remote ovn-encap-type ovn-encap-ip ovn-bridge-mappings \ + ovn-monitor-all ovn-openflow-probe-interval ovn-remote-probe-interval ; do + val=$(ovs-vsctl --if-exists get Open_vSwitch . "external_ids:${key}" 2>/dev/null | tr -d '"') || true + if [ -n "${val}" ]; then + ovs-vsctl remove Open_vSwitch . external_ids "${key}" "${val}" 2>/dev/null || true + fi + done echo Killing conmon, pause and OVN processes systemctl stop --now ovsdb-server.service 2>/dev/null || true for pname in conmon pause ovn-controller ovn-northd ; do diff --git a/test/bin/ci_phase_boot_and_test.sh b/test/bin/ci_phase_boot_and_test.sh index 56aef1cb30..4b3d60631a 100755 --- a/test/bin/ci_phase_boot_and_test.sh +++ b/test/bin/ci_phase_boot_and_test.sh @@ -22,9 +22,12 @@ prepare_scenario_sources() { rm -rf "${SCENARIOS_TO_RUN}" mkdir -p "${SCENARIOS_TO_RUN}" cp "${SCENARIO_SOURCES}"/*.sh "${SCENARIOS_TO_RUN}"/ - if ${EXCLUDE_CNCF_CONFORMANCE}; then - find "${SCENARIOS_TO_RUN}" -name "*cncf-conformance.sh" -delete - fi + # TODO: Temporarily disabled so that the CNCF conformance scenario runs + # unconditionally while the multinode OVN SBDB fix (PR #7344) is validated + # in CI. Revert once the job is confirmed green. + # if ${EXCLUDE_CNCF_CONFORMANCE}; then + # find "${SCENARIOS_TO_RUN}" -name "*cncf-conformance.sh" -delete + # fi } # Log output automatically