diff --git a/assets/components/ovn/common/configmap.yaml b/assets/components/ovn/common/configmap.yaml index 7279ce1fb8..1f435a2d7f 100644 --- a/assets/components/ovn/common/configmap.yaml +++ b/assets/components/ovn/common/configmap.yaml @@ -34,3 +34,11 @@ data: election-lease-duration=137 election-renew-deadline=107 election-retry-period=26 +{{- if .MultiNodeEnabled}} + + [OvnNorth] + address=tcp:{{.NodeIP}}:{{.OVN_NB_PORT}} + + [OvnSouth] + address=tcp:{{.NodeIP}}:{{.OVN_SB_PORT}} +{{- end}} diff --git a/assets/components/ovn/multi-node/master/daemonset.yaml b/assets/components/ovn/multi-node/master/daemonset.yaml index fe79c19640..94ca5cf491 100644 --- a/assets/components/ovn/multi-node/master/daemonset.yaml +++ b/assets/components/ovn/multi-node/master/daemonset.yaml @@ -141,6 +141,19 @@ spec: --db-nb-cluster-local-proto=tcp \ --no-monitor" + # On worker nodes the configmap contains [OvnNorth] address= pointing to + # the primary. Join the primary's NBDB RAFT cluster so this node gets a + # local unix socket connected to the shared database. + NB_ADDR=$(awk 'BEGIN{f=0} /^\[OvnNorth\]/{f=1} f && /^address=/{print substr($0,9); exit}' \ + /run/ovnkube-config/ovnkube.conf 2>/dev/null) + if [[ "${NB_ADDR}" =~ ^tcp: ]] && [[ "${NB_ADDR}" != *"${K8S_NODE_IP}"* ]]; then + PRIMARY_IP="${NB_ADDR#tcp:}"; PRIMARY_IP="${PRIMARY_IP%%:*}" + OVN_ARGS="${OVN_ARGS} \ + --db-nb-cluster-remote-addr=$(bracketify ${PRIMARY_IP}) \ + --db-nb-cluster-remote-port=9643 \ + --db-nb-cluster-remote-proto=tcp" + fi + rm -f /run/ovn/ovnnb_db.sock echo "$(date -Iseconds) - starting nbdb" @@ -234,6 +247,8 @@ spec: name: run-openvswitch - mountPath: /run/ovn/ name: run-ovn + - mountPath: /run/ovnkube-config/ + name: ovnkube-config - mountPath: /env name: env-overrides resources: @@ -288,6 +303,19 @@ spec: --db-sb-cluster-local-proto=tcp \ --no-monitor" + # On worker nodes the configmap contains [OvnSouth] address= pointing to + # the primary. Join the primary's SBDB RAFT cluster so this node gets a + # local unix socket connected to the shared database. + SB_ADDR=$(awk 'BEGIN{f=0} /^\[OvnSouth\]/{f=1} f && /^address=/{print substr($0,9); exit}' \ + /run/ovnkube-config/ovnkube.conf 2>/dev/null) + if [[ "${SB_ADDR}" =~ ^tcp: ]] && [[ "${SB_ADDR}" != *"${K8S_NODE_IP}"* ]]; then + PRIMARY_IP="${SB_ADDR#tcp:}"; PRIMARY_IP="${PRIMARY_IP%%:*}" + OVN_ARGS="${OVN_ARGS} \ + --db-sb-cluster-remote-addr=$(bracketify ${PRIMARY_IP}) \ + --db-sb-cluster-remote-port=9644 \ + --db-sb-cluster-remote-proto=tcp" + fi + rm -f /run/ovn/ovnsb_db.sock echo "$(date -Iseconds) - starting sbdb " @@ -345,6 +373,8 @@ spec: name: run-openvswitch - mountPath: /run/ovn/ name: run-ovn + - mountPath: /run/ovnkube-config/ + name: ovnkube-config - mountPath: /env name: env-overrides resources: diff --git a/pkg/components/networking.go b/pkg/components/networking.go index 2e6ccad927..31606affb4 100644 --- a/pkg/components/networking.go +++ b/pkg/components/networking.go @@ -110,17 +110,24 @@ func startCNIPlugin(ctx context.Context, cfg *config.Config, kubeconfigPath stri return err } - // Multinode only params: OVN_NB_PORT, OVN_SB_PORT + // Multinode only params: OVN_NB_PORT, OVN_SB_PORT, MultiNodeEnabled extraParams := assets.RenderParams{ - "OVNConfig": ovnConfig, - "KubeconfigPath": kubeconfigPath, - "KubeconfigDir": filepath.Join(config.DataDir, "/resources/kubeadmin"), - "OVN_NB_PORT": ovn.OVN_NB_PORT, - "OVN_SB_PORT": ovn.OVN_SB_PORT, - } - if err := assets.ApplyConfigMaps(ctx, cm, renderTemplate, renderParamsFromConfig(cfg, extraParams), kubeconfigPath); err != nil { - klog.Warningf("Failed to apply configMap %v %v", cm, err) - return err + "OVNConfig": ovnConfig, + "KubeconfigPath": kubeconfigPath, + "KubeconfigDir": filepath.Join(config.DataDir, "/resources/kubeadmin"), + "OVN_NB_PORT": ovn.OVN_NB_PORT, + "OVN_SB_PORT": ovn.OVN_SB_PORT, + "MultiNodeEnabled": cfg.MultiNode.Enabled, + } + // In multinode mode the configmap contains [OvnNorth]/[OvnSouth] stanzas + // with the primary's IP. Only the primary may write it; a worker applying + // the configmap would overwrite the primary IP with its own, breaking SBDB + // connectivity for every node that reads the configmap afterwards. + if !cfg.MultiNode.Enabled || !cfg.BootstrapKubeConfigExists() { + if err := assets.ApplyConfigMaps(ctx, cm, renderTemplate, renderParamsFromConfig(cfg, extraParams), kubeconfigPath); err != nil { + klog.Warningf("Failed to apply configMap %v %v", cm, err) + return err + } } if err := assets.ApplyDaemonSets(ctx, apps, renderTemplate, renderParamsFromConfig(cfg, extraParams), kubeconfigPath); err != nil { klog.Warningf("Failed to apply apps %v %v", apps, err) diff --git a/pkg/node/kubelet.go b/pkg/node/kubelet.go index 6a0c8ce6d8..54385e4a74 100644 --- a/pkg/node/kubelet.go +++ b/pkg/node/kubelet.go @@ -89,6 +89,9 @@ func (s *KubeletServer) configure(cfg *config.Config) { kubeletFlags.NodeLabels["node-role.kubernetes.io/worker"] = "" kubeletFlags.NodeLabels["node.openshift.io/os_id"] = osID kubeletFlags.NodeLabels["node.kubernetes.io/instance-type"] = "rhde" + if !cfg.BootstrapKubeConfigExists() { + kubeletFlags.NodeLabels["node.microshift.io/role"] = "primary" + } kubeletConfig, err := loadConfigFile(filepath.Join(config.DataDir, "/resources/kubelet/config/config.yaml")) diff --git a/test/bin/ci_phase_boot_and_test.sh b/test/bin/ci_phase_boot_and_test.sh index 79bd22e8c6..8cfb4dcf51 100755 --- a/test/bin/ci_phase_boot_and_test.sh +++ b/test/bin/ci_phase_boot_and_test.sh @@ -22,9 +22,12 @@ prepare_scenario_sources() { rm -rf "${SCENARIOS_TO_RUN}" mkdir -p "${SCENARIOS_TO_RUN}" cp "${SCENARIO_SOURCES}"/*.sh "${SCENARIOS_TO_RUN}"/ - if ${EXCLUDE_CNCF_CONFORMANCE}; then - find "${SCENARIOS_TO_RUN}" -name "*cncf-conformance.sh" -delete - fi + # TODO: Temporarily disabled so that the CNCF conformance scenario runs + # unconditionally while the multinode OVN SBDB fix (PR #7344) is validated + # in CI. Revert once the job is confirmed green. + # if ${EXCLUDE_CNCF_CONFORMANCE}; then + # find "${SCENARIOS_TO_RUN}" -name "*cncf-conformance.sh" -delete + # fi } # Log output automatically