<fix>[lb]: reject tcp ipvs on old zvr

plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerApiInterceptor.java

@@ -815,6 +815,18 @@ private boolean hasHttpHealthCheckParameters(APIChangeLoadBalancerListenerMsg ms
                 msg.getHealthCheckHttpCode() != null;
     }
 
+    private boolean hasEnabledTcpProxyProtocol(String tcpProxyProtocol) {
+        return !StringUtils.isEmpty(tcpProxyProtocol) &&
+                !DisableLbSupportTcpProxyProtocol.equals(tcpProxyProtocol);
+    }
+
+    private void validateTcpIpvsDoesNotUseTcpProxyProtocol(String protocol, String dataPlane, String tcpProxyProtocol) {
+        if (isTcpIpvsListener(protocol, dataPlane) && hasEnabledTcpProxyProtocol(tcpProxyProtocol)) {
+            throw new ApiMessageInterceptionException(
+                    operr(ORG_ZSTACK_NETWORK_SERVICE_LB_10191, "tcp ipvs listener doesn't support tcpProxyProtocol"));
+        }
+    }
+
     private String getHealthCheckProtocolFromTarget(String healthCheckTarget) {
         if (healthCheckTarget == null) {
             return null;
@@ -1381,6 +1393,7 @@ private void validate(APICreateLoadBalancerListenerMsg msg) {
                 throw new ApiMessageInterceptionException(
                         argerr(ORG_ZSTACK_NETWORK_SERVICE_LB_10093, "cloud not create the loadbalancer listener, because only support tcp proxy protocol %s", LbSupportTcpProxyProtocol));
             }
+            validateTcpIpvsDoesNotUseTcpProxyProtocol(msg.getProtocol(), dataPlane, msg.getTcpProxyProtocol());
 
             if (!msg.getTcpProxyProtocol().equals(DisableLbSupportTcpProxyProtocol)) {
                 insertTagIfNotExisting(
@@ -1697,6 +1710,7 @@ private void validate(APIChangeLoadBalancerListenerMsg msg) {
                 throw new ApiMessageInterceptionException(
                         argerr(ORG_ZSTACK_NETWORK_SERVICE_LB_10125, "cloud not change the loadbalancer listener, because only support tcp proxy protocol %s", LbSupportTcpProxyProtocol));
             }
+            validateTcpIpvsDoesNotUseTcpProxyProtocol(listenerVO.getProtocol(), dataPlane, msg.getTcpProxyProtocol());
         }
 
         if (!CollectionUtils.isEmpty(msg.getHttpCompressAlgos())) {

plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerBackend.java

@@ -21,6 +21,10 @@ public interface LoadBalancerBackend {
 
     void addListener(LoadBalancerStruct struct,  LoadBalancerListenerInventory listener, Completion completion);
 
+    default void validateBeforeCreateListener(LoadBalancerVO lbVO, APICreateLoadBalancerListenerMsg msg, Completion completion) {
+        completion.success();
+    }
+
     void removeListener(LoadBalancerStruct struct,  LoadBalancerListenerInventory listener, Completion completion);
 
     void destroyLoadBalancer(LoadBalancerStruct struct, Completion completion);

plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerBase.java

@@ -1811,9 +1811,33 @@ public void run(final SyncTaskChain chain) {
                     return;
                 }
 
-                createListener(msg, new NoErrorCompletion(chain) {
+                LoadBalancerBackend backend = getBackend();
+                if (backend == null) {
+                    createListener(msg, new NoErrorCompletion(chain) {
+                        @Override
+                        public void done() {
+                            chain.next();
+                        }
+                    });
+                    return;
+                }
+
+                backend.validateBeforeCreateListener(self, msg, new Completion(chain) {
                     @Override
-                    public void done() {
+                    public void success() {
+                        createListener(msg, new NoErrorCompletion(chain) {
+                            @Override
+                            public void done() {
+                                chain.next();
+                            }
+                        });
+                    }
+
+                    @Override
+                    public void fail(ErrorCode errorCode) {
+                        APICreateLoadBalancerListenerEvent evt = new APICreateLoadBalancerListenerEvent(msg.getId());
+                        evt.setError(errorCode);
+                        bus.publish(evt);
                         chain.next();
                     }
                 });

plugin/virtualRouterProvider/src/main/java/org/zstack/network/service/virtualrouter/lb/VirtualRouterLoadBalancerBackend.java

@@ -47,6 +47,7 @@
 import org.zstack.network.service.lb.*;
 import org.zstack.network.service.vip.*;
 import org.zstack.network.service.virtualrouter.*;
+import org.zstack.network.service.virtualrouter.vyos.VyosConstants;
 import org.zstack.network.service.virtualrouter.vyos.VyosGlobalConfig;
 import org.zstack.network.service.virtualrouter.VirtualRouterCommands.AgentCommand;
 import org.zstack.network.service.virtualrouter.VirtualRouterCommands.AgentResponse;
@@ -1303,6 +1304,46 @@ public void handle(ErrorCode errCode, Map data) {
         }).start();
     }
 
+    @Override
+    public void validateBeforeCreateListener(LoadBalancerVO lbVO, APICreateLoadBalancerListenerMsg msg, Completion completion) {
+        if (!isTcpIpvsListener(msg)) {
+            completion.success();
+            return;
+        }
+
+        VirtualRouterVmInventory vr = findVirtualRouterVm(lbVO.getUuid());
+        if (vr == null) {
+            completion.success();
+            return;
+        }
+
+        ErrorCode errorCode = validateTcpIpvsZvrVersion(vr.getUuid());
+        if (errorCode != null) {
+            completion.fail(errorCode);
+            return;
+        }
+
+        completion.success();
+    }
+
+    private boolean isTcpIpvsListener(APICreateLoadBalancerListenerMsg msg) {
+        return LoadBalancerConstants.LB_PROTOCOL_TCP.equals(msg.getProtocol()) &&
+                LoadBalancerConstants.DATA_PLANE_IPVS.equals(msg.getDataPlane());
+    }
+
+    private ErrorCode validateTcpIpvsZvrVersion(String vmUuid) {
+        VirtualRouterMetadataVO metadataVO = dbf.findByUuid(vmUuid, VirtualRouterMetadataVO.class);
+        String zvrVersion = metadataVO == null ? null : metadataVO.getZvrVersion();
+        if (!VirtualRouterMetadataOperator.zvrVersionCheck(zvrVersion) ||
+                new VersionComparator(zvrVersion).compare(VyosConstants.TCP_IPVS_MIN_ZVR_VERSION) < 0) {
+            return operr(ORG_ZSTACK_NETWORK_SERVICE_LB_10190,
+                    "target appliance vm[uuid:%s] zvr version [%s] does not support tcp ipvs listener, required >= %s",
+                    vmUuid, zvrVersion == null ? "unknown" : zvrVersion, VyosConstants.TCP_IPVS_MIN_ZVR_VERSION);
+        }
+
+        return null;
+    }
+
     public void refreshCertsAndListeners(VirtualRouterVmInventory vr,
                                          Map<String, String> certs,
                                  List<VirtualRouterLoadBalancerBackend.LbTO> listeners,

plugin/virtualRouterProvider/src/main/java/org/zstack/network/service/virtualrouter/vyos/VyosConstants.java

@@ -23,6 +23,7 @@ enum BootstrapInfoKey {
 
     String VYOS_VERSION_PATH = "ansible/zvr/version";
     int VYOS_VERSION_LENGTH = 4;
+    String TCP_IPVS_MIN_ZVR_VERSION = "5.5.38.0";
 
     String REPLACE_FIREWALL_WITH_IPTBALES = "SkipVyosIptables";
     String HA_STATUS = "haStatus";

test/src/test/groovy/org/zstack/test/integration/networkservice/provider/virtualrouter/loadbalancer/TcpIpvsLoadBalancerListenerApiCase.groovy

@@ -14,6 +14,8 @@ import org.zstack.network.service.lb.LoadBalancerSystemTags
 import org.zstack.network.service.lb.LoadBalancerVO
 import org.zstack.network.service.lb.LoadBalancerVO_
 import org.zstack.network.service.portforwarding.PortForwardingConstant
+import org.zstack.network.service.virtualrouter.VirtualRouterCommands
+import org.zstack.network.service.virtualrouter.VirtualRouterConstant
 import org.zstack.network.service.virtualrouter.lb.VirtualRouterLoadBalancerBackend
 import org.zstack.network.service.virtualrouter.vyos.VyosConstants
 import org.zstack.network.service.virtualrouter.vyos.VyosGlobalConfig
@@ -201,6 +203,7 @@ class TcpIpvsLoadBalancerListenerApiCase extends SubCase {
     @Override
     void test() {
         env.create {
+            installSupportedZvrVersionSimulator()
             prepareDedicatedLoadBalancer()
             installRefreshLbCapture()
             testTcpIpvsIpv6Validation()
@@ -211,13 +214,21 @@ class TcpIpvsLoadBalancerListenerApiCase extends SubCase {
             testTcpIpvsDefaultForwardMode()
             testTcpIpvsForwardModeCannotBeChanged()
             testTcpIpvsCreateValidation()
+            testTcpIpvsTcpProxyProtocolValidation()
             testTcpIpvsHealthCheckParameterValidation()
             testTcpHaproxyBackendRefreshPayload()
             testUdpHaproxyBackendRefreshPayload()
             testTcpIpvsDedicatedListenerBackendRefreshPayload()
         }
     }
 
+    void installSupportedZvrVersionSimulator() {
+        env.afterSimulator(VirtualRouterConstant.VR_INIT) { VirtualRouterCommands.InitRsp rsp, HttpEntity<String> e ->
+            rsp.zvrVersion = VyosConstants.TCP_IPVS_MIN_ZVR_VERSION
+            return rsp
+        }
+    }
+
     void prepareDedicatedLoadBalancer() {
         L3NetworkInventory publicL3 = env.inventoryByName("publicL3") as L3NetworkInventory
         VipInventory vip = createVip {
@@ -541,6 +552,47 @@ class TcpIpvsLoadBalancerListenerApiCase extends SubCase {
         assert result.error.details.contains("forwardMode is only supported when dataPlane is ipvs")
     }
 
+    void testTcpIpvsTcpProxyProtocolValidation() {
+        ["v1", "v2"].eachWithIndex { String protocol, int index ->
+            int port = 11260 + index
+            CreateLoadBalancerListenerAction.Result result = assertCreateTcpIpvsListenerError(port) { CreateLoadBalancerListenerAction action ->
+                action.tcpProxyProtocol = protocol
+            }
+            assertUnsupportedTcpProxyProtocolError(result.error)
+            assert !Q.New(LoadBalancerListenerVO.class)
+                    .eq(LoadBalancerListenerVO_.name, "tcp-ipvs-error-${port}".toString())
+                    .isExists()
+        }
+
+        LoadBalancerListenerInventory disabledOnCreate = createLoadBalancerListener {
+            delegate.name = "tcp-ipvs-disabled-tcp-proxy-protocol"
+            delegate.loadBalancerUuid = lb.uuid
+            delegate.protocol = LoadBalancerConstants.LB_PROTOCOL_TCP
+            delegate.loadBalancerPort = 11262
+            delegate.instancePort = 8080
+            delegate.dataPlane = LoadBalancerConstants.DATA_PLANE_IPVS
+            delegate.forwardMode = LoadBalancerConstants.FORWARD_MODE_FULL_NAT
+            delegate.tcpProxyProtocol = LoadBalancerConstants.DisableLbSupportTcpProxyProtocol
+        }
+        assert disabledOnCreate.uuid != null
+        assert !LoadBalancerSystemTags.TCP_PROXYPROTOCOL.hasTag(disabledOnCreate.uuid, LoadBalancerListenerVO.class)
+
+        LoadBalancerListenerInventory listener = createTcpIpvsListener(
+                "tcp-ipvs-change-tcp-proxy-protocol", 11263, LoadBalancerConstants.BALANCE_ALGORITHM_ROUND_ROBIN)
+        ["v1", "v2"].each { String protocol ->
+            ChangeLoadBalancerListenerAction.Result result = assertChangeListenerError(listener.uuid) { ChangeLoadBalancerListenerAction action ->
+                action.tcpProxyProtocol = protocol
+            }
+            assertUnsupportedTcpProxyProtocolError(result.error)
+        }
+        assert !LoadBalancerSystemTags.TCP_PROXYPROTOCOL.hasTag(listener.uuid, LoadBalancerListenerVO.class)
+
+        assertChangeListenerSuccess(listener.uuid) { ChangeLoadBalancerListenerAction action ->
+            action.tcpProxyProtocol = LoadBalancerConstants.DisableLbSupportTcpProxyProtocol
+        }
+        assert !LoadBalancerSystemTags.TCP_PROXYPROTOCOL.hasTag(listener.uuid, LoadBalancerListenerVO.class)
+    }
+
     void testTcpIpvsHealthCheckParameterValidation() {
         [
                 { CreateLoadBalancerListenerAction action -> action.healthCheckMethod = "GET" },
@@ -1153,6 +1205,11 @@ class TcpIpvsLoadBalancerListenerApiCase extends SubCase {
         assert error.details.contains("tcp ipvs listener doesn't support healthCheckTimeout")
     }
 
+    void assertUnsupportedTcpProxyProtocolError(def error) {
+        assert error.globalErrorCode == "ORG_ZSTACK_NETWORK_SERVICE_LB_10191"
+        assert error.details.contains("tcp ipvs listener doesn't support tcpProxyProtocol")
+    }
+
     String getApiResultString(ApiResult result) {
         def field = ApiResult.class.getDeclaredField("resultString")
         field.setAccessible(true)