diff --git a/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerApiInterceptor.java b/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerApiInterceptor.java index ff0f7c34896..f862872a713 100755 --- a/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerApiInterceptor.java +++ b/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerApiInterceptor.java @@ -815,6 +815,18 @@ private boolean hasHttpHealthCheckParameters(APIChangeLoadBalancerListenerMsg ms msg.getHealthCheckHttpCode() != null; } + private boolean hasEnabledTcpProxyProtocol(String tcpProxyProtocol) { + return !StringUtils.isEmpty(tcpProxyProtocol) && + !DisableLbSupportTcpProxyProtocol.equals(tcpProxyProtocol); + } + + private void validateTcpIpvsDoesNotUseTcpProxyProtocol(String protocol, String dataPlane, String tcpProxyProtocol) { + if (isTcpIpvsListener(protocol, dataPlane) && hasEnabledTcpProxyProtocol(tcpProxyProtocol)) { + throw new ApiMessageInterceptionException( + operr(ORG_ZSTACK_NETWORK_SERVICE_LB_10191, "tcp ipvs listener doesn't support tcpProxyProtocol")); + } + } + private String getHealthCheckProtocolFromTarget(String healthCheckTarget) { if (healthCheckTarget == null) { return null; @@ -1381,6 +1393,7 @@ private void validate(APICreateLoadBalancerListenerMsg msg) { throw new ApiMessageInterceptionException( argerr(ORG_ZSTACK_NETWORK_SERVICE_LB_10093, "cloud not create the loadbalancer listener, because only support tcp proxy protocol %s", LbSupportTcpProxyProtocol)); } + validateTcpIpvsDoesNotUseTcpProxyProtocol(msg.getProtocol(), dataPlane, msg.getTcpProxyProtocol()); if (!msg.getTcpProxyProtocol().equals(DisableLbSupportTcpProxyProtocol)) { insertTagIfNotExisting( @@ -1697,6 +1710,7 @@ private void validate(APIChangeLoadBalancerListenerMsg msg) { throw new ApiMessageInterceptionException( argerr(ORG_ZSTACK_NETWORK_SERVICE_LB_10125, "cloud not change the loadbalancer listener, because only support tcp proxy protocol %s", LbSupportTcpProxyProtocol)); } + validateTcpIpvsDoesNotUseTcpProxyProtocol(listenerVO.getProtocol(), dataPlane, msg.getTcpProxyProtocol()); } if (!CollectionUtils.isEmpty(msg.getHttpCompressAlgos())) { diff --git a/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerBackend.java b/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerBackend.java index 561226ca769..1df195bedfb 100755 --- a/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerBackend.java +++ b/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerBackend.java @@ -21,6 +21,10 @@ public interface LoadBalancerBackend { void addListener(LoadBalancerStruct struct, LoadBalancerListenerInventory listener, Completion completion); + default void validateBeforeCreateListener(LoadBalancerVO lbVO, APICreateLoadBalancerListenerMsg msg, Completion completion) { + completion.success(); + } + void removeListener(LoadBalancerStruct struct, LoadBalancerListenerInventory listener, Completion completion); void destroyLoadBalancer(LoadBalancerStruct struct, Completion completion); diff --git a/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerBase.java b/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerBase.java index 0e78b167c0f..94afa88e6d5 100755 --- a/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerBase.java +++ b/plugin/loadBalancer/src/main/java/org/zstack/network/service/lb/LoadBalancerBase.java @@ -1811,9 +1811,33 @@ public void run(final SyncTaskChain chain) { return; } - createListener(msg, new NoErrorCompletion(chain) { + LoadBalancerBackend backend = getBackend(); + if (backend == null) { + createListener(msg, new NoErrorCompletion(chain) { + @Override + public void done() { + chain.next(); + } + }); + return; + } + + backend.validateBeforeCreateListener(self, msg, new Completion(chain) { @Override - public void done() { + public void success() { + createListener(msg, new NoErrorCompletion(chain) { + @Override + public void done() { + chain.next(); + } + }); + } + + @Override + public void fail(ErrorCode errorCode) { + APICreateLoadBalancerListenerEvent evt = new APICreateLoadBalancerListenerEvent(msg.getId()); + evt.setError(errorCode); + bus.publish(evt); chain.next(); } }); diff --git a/plugin/virtualRouterProvider/src/main/java/org/zstack/network/service/virtualrouter/lb/VirtualRouterLoadBalancerBackend.java b/plugin/virtualRouterProvider/src/main/java/org/zstack/network/service/virtualrouter/lb/VirtualRouterLoadBalancerBackend.java index 72faff543a0..9e088712254 100755 --- a/plugin/virtualRouterProvider/src/main/java/org/zstack/network/service/virtualrouter/lb/VirtualRouterLoadBalancerBackend.java +++ b/plugin/virtualRouterProvider/src/main/java/org/zstack/network/service/virtualrouter/lb/VirtualRouterLoadBalancerBackend.java @@ -47,6 +47,7 @@ import org.zstack.network.service.lb.*; import org.zstack.network.service.vip.*; import org.zstack.network.service.virtualrouter.*; +import org.zstack.network.service.virtualrouter.vyos.VyosConstants; import org.zstack.network.service.virtualrouter.vyos.VyosGlobalConfig; import org.zstack.network.service.virtualrouter.VirtualRouterCommands.AgentCommand; import org.zstack.network.service.virtualrouter.VirtualRouterCommands.AgentResponse; @@ -1303,6 +1304,46 @@ public void handle(ErrorCode errCode, Map data) { }).start(); } + @Override + public void validateBeforeCreateListener(LoadBalancerVO lbVO, APICreateLoadBalancerListenerMsg msg, Completion completion) { + if (!isTcpIpvsListener(msg)) { + completion.success(); + return; + } + + VirtualRouterVmInventory vr = findVirtualRouterVm(lbVO.getUuid()); + if (vr == null) { + completion.success(); + return; + } + + ErrorCode errorCode = validateTcpIpvsZvrVersion(vr.getUuid()); + if (errorCode != null) { + completion.fail(errorCode); + return; + } + + completion.success(); + } + + private boolean isTcpIpvsListener(APICreateLoadBalancerListenerMsg msg) { + return LoadBalancerConstants.LB_PROTOCOL_TCP.equals(msg.getProtocol()) && + LoadBalancerConstants.DATA_PLANE_IPVS.equals(msg.getDataPlane()); + } + + private ErrorCode validateTcpIpvsZvrVersion(String vmUuid) { + VirtualRouterMetadataVO metadataVO = dbf.findByUuid(vmUuid, VirtualRouterMetadataVO.class); + String zvrVersion = metadataVO == null ? null : metadataVO.getZvrVersion(); + if (!VirtualRouterMetadataOperator.zvrVersionCheck(zvrVersion) || + new VersionComparator(zvrVersion).compare(VyosConstants.TCP_IPVS_MIN_ZVR_VERSION) < 0) { + return operr(ORG_ZSTACK_NETWORK_SERVICE_LB_10190, + "target appliance vm[uuid:%s] zvr version [%s] does not support tcp ipvs listener, required >= %s", + vmUuid, zvrVersion == null ? "unknown" : zvrVersion, VyosConstants.TCP_IPVS_MIN_ZVR_VERSION); + } + + return null; + } + public void refreshCertsAndListeners(VirtualRouterVmInventory vr, Map certs, List listeners, diff --git a/plugin/virtualRouterProvider/src/main/java/org/zstack/network/service/virtualrouter/vyos/VyosConstants.java b/plugin/virtualRouterProvider/src/main/java/org/zstack/network/service/virtualrouter/vyos/VyosConstants.java index 93941c5dacc..8b88d644e2f 100755 --- a/plugin/virtualRouterProvider/src/main/java/org/zstack/network/service/virtualrouter/vyos/VyosConstants.java +++ b/plugin/virtualRouterProvider/src/main/java/org/zstack/network/service/virtualrouter/vyos/VyosConstants.java @@ -23,6 +23,7 @@ enum BootstrapInfoKey { String VYOS_VERSION_PATH = "ansible/zvr/version"; int VYOS_VERSION_LENGTH = 4; + String TCP_IPVS_MIN_ZVR_VERSION = "5.5.38.0"; String REPLACE_FIREWALL_WITH_IPTBALES = "SkipVyosIptables"; String HA_STATUS = "haStatus"; diff --git a/test/src/test/groovy/org/zstack/test/integration/networkservice/provider/virtualrouter/loadbalancer/TcpIpvsLoadBalancerListenerApiCase.groovy b/test/src/test/groovy/org/zstack/test/integration/networkservice/provider/virtualrouter/loadbalancer/TcpIpvsLoadBalancerListenerApiCase.groovy index 70631a2af93..80944f7929d 100644 --- a/test/src/test/groovy/org/zstack/test/integration/networkservice/provider/virtualrouter/loadbalancer/TcpIpvsLoadBalancerListenerApiCase.groovy +++ b/test/src/test/groovy/org/zstack/test/integration/networkservice/provider/virtualrouter/loadbalancer/TcpIpvsLoadBalancerListenerApiCase.groovy @@ -14,6 +14,8 @@ import org.zstack.network.service.lb.LoadBalancerSystemTags import org.zstack.network.service.lb.LoadBalancerVO import org.zstack.network.service.lb.LoadBalancerVO_ import org.zstack.network.service.portforwarding.PortForwardingConstant +import org.zstack.network.service.virtualrouter.VirtualRouterCommands +import org.zstack.network.service.virtualrouter.VirtualRouterConstant import org.zstack.network.service.virtualrouter.lb.VirtualRouterLoadBalancerBackend import org.zstack.network.service.virtualrouter.vyos.VyosConstants import org.zstack.network.service.virtualrouter.vyos.VyosGlobalConfig @@ -201,6 +203,7 @@ class TcpIpvsLoadBalancerListenerApiCase extends SubCase { @Override void test() { env.create { + installSupportedZvrVersionSimulator() prepareDedicatedLoadBalancer() installRefreshLbCapture() testTcpIpvsIpv6Validation() @@ -211,6 +214,7 @@ class TcpIpvsLoadBalancerListenerApiCase extends SubCase { testTcpIpvsDefaultForwardMode() testTcpIpvsForwardModeCannotBeChanged() testTcpIpvsCreateValidation() + testTcpIpvsTcpProxyProtocolValidation() testTcpIpvsHealthCheckParameterValidation() testTcpHaproxyBackendRefreshPayload() testUdpHaproxyBackendRefreshPayload() @@ -218,6 +222,13 @@ class TcpIpvsLoadBalancerListenerApiCase extends SubCase { } } + void installSupportedZvrVersionSimulator() { + env.afterSimulator(VirtualRouterConstant.VR_INIT) { VirtualRouterCommands.InitRsp rsp, HttpEntity e -> + rsp.zvrVersion = VyosConstants.TCP_IPVS_MIN_ZVR_VERSION + return rsp + } + } + void prepareDedicatedLoadBalancer() { L3NetworkInventory publicL3 = env.inventoryByName("publicL3") as L3NetworkInventory VipInventory vip = createVip { @@ -541,6 +552,47 @@ class TcpIpvsLoadBalancerListenerApiCase extends SubCase { assert result.error.details.contains("forwardMode is only supported when dataPlane is ipvs") } + void testTcpIpvsTcpProxyProtocolValidation() { + ["v1", "v2"].eachWithIndex { String protocol, int index -> + int port = 11260 + index + CreateLoadBalancerListenerAction.Result result = assertCreateTcpIpvsListenerError(port) { CreateLoadBalancerListenerAction action -> + action.tcpProxyProtocol = protocol + } + assertUnsupportedTcpProxyProtocolError(result.error) + assert !Q.New(LoadBalancerListenerVO.class) + .eq(LoadBalancerListenerVO_.name, "tcp-ipvs-error-${port}".toString()) + .isExists() + } + + LoadBalancerListenerInventory disabledOnCreate = createLoadBalancerListener { + delegate.name = "tcp-ipvs-disabled-tcp-proxy-protocol" + delegate.loadBalancerUuid = lb.uuid + delegate.protocol = LoadBalancerConstants.LB_PROTOCOL_TCP + delegate.loadBalancerPort = 11262 + delegate.instancePort = 8080 + delegate.dataPlane = LoadBalancerConstants.DATA_PLANE_IPVS + delegate.forwardMode = LoadBalancerConstants.FORWARD_MODE_FULL_NAT + delegate.tcpProxyProtocol = LoadBalancerConstants.DisableLbSupportTcpProxyProtocol + } + assert disabledOnCreate.uuid != null + assert !LoadBalancerSystemTags.TCP_PROXYPROTOCOL.hasTag(disabledOnCreate.uuid, LoadBalancerListenerVO.class) + + LoadBalancerListenerInventory listener = createTcpIpvsListener( + "tcp-ipvs-change-tcp-proxy-protocol", 11263, LoadBalancerConstants.BALANCE_ALGORITHM_ROUND_ROBIN) + ["v1", "v2"].each { String protocol -> + ChangeLoadBalancerListenerAction.Result result = assertChangeListenerError(listener.uuid) { ChangeLoadBalancerListenerAction action -> + action.tcpProxyProtocol = protocol + } + assertUnsupportedTcpProxyProtocolError(result.error) + } + assert !LoadBalancerSystemTags.TCP_PROXYPROTOCOL.hasTag(listener.uuid, LoadBalancerListenerVO.class) + + assertChangeListenerSuccess(listener.uuid) { ChangeLoadBalancerListenerAction action -> + action.tcpProxyProtocol = LoadBalancerConstants.DisableLbSupportTcpProxyProtocol + } + assert !LoadBalancerSystemTags.TCP_PROXYPROTOCOL.hasTag(listener.uuid, LoadBalancerListenerVO.class) + } + void testTcpIpvsHealthCheckParameterValidation() { [ { CreateLoadBalancerListenerAction action -> action.healthCheckMethod = "GET" }, @@ -1153,6 +1205,11 @@ class TcpIpvsLoadBalancerListenerApiCase extends SubCase { assert error.details.contains("tcp ipvs listener doesn't support healthCheckTimeout") } + void assertUnsupportedTcpProxyProtocolError(def error) { + assert error.globalErrorCode == "ORG_ZSTACK_NETWORK_SERVICE_LB_10191" + assert error.details.contains("tcp ipvs listener doesn't support tcpProxyProtocol") + } + String getApiResultString(ApiResult result) { def field = ApiResult.class.getDeclaredField("resultString") field.setAccessible(true) diff --git a/test/src/test/groovy/org/zstack/test/integration/networkservice/provider/virtualrouter/loadbalancer/TcpIpvsZvrVersionValidationCase.groovy b/test/src/test/groovy/org/zstack/test/integration/networkservice/provider/virtualrouter/loadbalancer/TcpIpvsZvrVersionValidationCase.groovy new file mode 100644 index 00000000000..d2f532059db --- /dev/null +++ b/test/src/test/groovy/org/zstack/test/integration/networkservice/provider/virtualrouter/loadbalancer/TcpIpvsZvrVersionValidationCase.groovy @@ -0,0 +1,202 @@ +package org.zstack.test.integration.networkservice.provider.virtualrouter.loadbalancer + +import org.zstack.appliancevm.ApplianceVmConstant +import org.zstack.appliancevm.ApplianceVmHaStatus +import org.zstack.appliancevm.ApplianceVmStatus +import org.zstack.core.Platform +import org.zstack.core.db.Q +import org.zstack.core.db.DatabaseFacade +import org.zstack.header.identity.AccountConstant +import org.zstack.header.image.ImagePlatform +import org.zstack.header.network.service.NetworkServiceType +import org.zstack.header.vm.VmInstanceState +import org.zstack.network.service.lb.LoadBalancerConstants +import org.zstack.network.service.lb.LoadBalancerListenerVO +import org.zstack.network.service.lb.LoadBalancerListenerVO_ +import org.zstack.network.service.lb.LoadBalancerVO +import org.zstack.network.service.virtualrouter.VirtualRouterConstant +import org.zstack.network.service.virtualrouter.VirtualRouterMetadataVO +import org.zstack.network.service.virtualrouter.VirtualRouterVmVO +import org.zstack.network.service.virtualrouter.lb.VirtualRouterLoadBalancerRefVO +import org.zstack.network.service.virtualrouter.lb.VirtualRouterLoadBalancerRefVO_ +import org.zstack.network.service.virtualrouter.vyos.VyosConstants +import org.zstack.sdk.CreateLoadBalancerListenerAction +import org.zstack.sdk.L3NetworkInventory +import org.zstack.sdk.LoadBalancerInventory +import org.zstack.sdk.VipInventory +import org.zstack.test.integration.networkservice.provider.NetworkServiceProviderTest +import org.zstack.testlib.EnvSpec +import org.zstack.testlib.SubCase + +class TcpIpvsZvrVersionValidationCase extends SubCase { + EnvSpec env + DatabaseFacade dbf + String syntheticVrUuid + + @Override + void setup() { + useSpring(NetworkServiceProviderTest.springSpec) + } + + @Override + void environment() { + env = env { + zone { + name = "zone" + + l2NoVlanNetwork { + name = "l2" + physicalInterface = "eth0" + + l3Network { + name = "guestL3" + service { + provider = VyosConstants.VYOS_ROUTER_PROVIDER_TYPE + types = [NetworkServiceType.DHCP.toString(), + NetworkServiceType.DNS.toString(), + NetworkServiceType.SNAT.toString(), + LoadBalancerConstants.LB_NETWORK_SERVICE_TYPE_STRING] + } + ip { + startIp = "10.2.226.10" + endIp = "10.2.226.200" + gateway = "10.2.226.1" + netmask = "255.255.255.0" + } + } + + l3Network { + name = "publicL3" + ip { + startIp = "172.24.3.10" + endIp = "172.24.3.200" + gateway = "172.24.3.1" + netmask = "255.255.255.0" + } + } + } + } + } + } + + @Override + void test() { + env.create { + dbf = bean(DatabaseFacade.class) + testTcpIpvsCreateRejectedByOldZvrVersion() + } + } + + @Override + void clean() { + if (syntheticVrUuid != null) { + VirtualRouterLoadBalancerRefVO ref = Q.New(VirtualRouterLoadBalancerRefVO.class) + .eq(VirtualRouterLoadBalancerRefVO_.virtualRouterVmUuid, syntheticVrUuid) + .find() + if (ref != null) { + dbf.remove(ref) + } + + VirtualRouterMetadataVO metadata = dbf.findByUuid(syntheticVrUuid, VirtualRouterMetadataVO.class) + if (metadata != null) { + dbf.remove(metadata) + } + + VirtualRouterVmVO vr = dbf.findByUuid(syntheticVrUuid, VirtualRouterVmVO.class) + if (vr != null) { + dbf.remove(vr) + } + } + env.delete() + } + + void testTcpIpvsCreateRejectedByOldZvrVersion() { + L3NetworkInventory publicL3 = env.inventoryByName("publicL3") as L3NetworkInventory + + VipInventory vip = createVip { + name = "tcp-ipvs-zvr-version-vip" + l3NetworkUuid = publicL3.uuid + } + + LoadBalancerInventory lb = createLoadBalancer { + name = "tcp-ipvs-zvr-version-lb" + vipUuid = vip.uuid + } + setLbProviderType(lb.uuid) + + String vrUuid = createVirtualRouterRef(lb.uuid) + setZvrVersion(vrUuid, "5.5.28.0") + + CreateLoadBalancerListenerAction action = createTcpIpvsListenerAction(lb.uuid, "tcp-ipvs-old-zvr-version", 19095) + CreateLoadBalancerListenerAction.Result result = action.call() + + assert result.error != null + assert result.error.globalErrorCode == "ORG_ZSTACK_NETWORK_SERVICE_LB_10190" + assert result.error.details.contains("does not support tcp ipvs listener") + assert !Q.New(LoadBalancerListenerVO.class) + .eq(LoadBalancerListenerVO_.name, action.name) + .isExists() + + setZvrVersion(vrUuid, VyosConstants.TCP_IPVS_MIN_ZVR_VERSION) + result = createTcpIpvsListenerAction(lb.uuid, "tcp-ipvs-supported-zvr-version", 19096).call() + assert result.error == null + } + + String createVirtualRouterRef(String lbUuid) { + String vrUuid = Platform.getUuid() + VirtualRouterVmVO vr = new VirtualRouterVmVO() + vr.uuid = vrUuid + vr.name = "tcp-ipvs-zvr-version-vr" + vr.type = ApplianceVmConstant.APPLIANCE_VM_TYPE + vr.applianceVmType = VirtualRouterConstant.VIRTUAL_ROUTER_VM_TYPE + vr.hypervisorType = "KVM" + vr.platform = ImagePlatform.Linux.toString() + vr.state = VmInstanceState.Running + vr.status = ApplianceVmStatus.Connected + vr.haStatus = ApplianceVmHaStatus.NoHa + vr.agentPort = 7272 + vr.internalId = 1L + vr.accountUuid = AccountConstant.INITIAL_SYSTEM_ADMIN_UUID + dbf.persist(vr) + syntheticVrUuid = vrUuid + + VirtualRouterLoadBalancerRefVO ref = new VirtualRouterLoadBalancerRefVO() + ref.loadBalancerUuid = lbUuid + ref.virtualRouterVmUuid = vrUuid + dbf.persist(ref) + + return vrUuid + } + + void setLbProviderType(String lbUuid) { + LoadBalancerVO lbVO = dbf.findByUuid(lbUuid, LoadBalancerVO.class) + lbVO.providerType = VyosConstants.VYOS_ROUTER_PROVIDER_TYPE + dbf.update(lbVO) + } + + void setZvrVersion(String vrUuid, String version) { + VirtualRouterMetadataVO metadata = dbf.findByUuid(vrUuid, VirtualRouterMetadataVO.class) + if (metadata == null) { + metadata = new VirtualRouterMetadataVO() + metadata.uuid = vrUuid + metadata.zvrVersion = version + dbf.persist(metadata) + } else { + metadata.zvrVersion = version + dbf.update(metadata) + } + } + + CreateLoadBalancerListenerAction createTcpIpvsListenerAction(String lbUuid, String name, int port) { + CreateLoadBalancerListenerAction action = new CreateLoadBalancerListenerAction() + action.name = name + action.loadBalancerUuid = lbUuid + action.protocol = LoadBalancerConstants.LB_PROTOCOL_TCP + action.loadBalancerPort = port + action.instancePort = 8080 + action.dataPlane = LoadBalancerConstants.DATA_PLANE_IPVS + action.forwardMode = LoadBalancerConstants.FORWARD_MODE_FULL_NAT + action.sessionId = adminSession() + return action + } +} diff --git a/utils/src/main/java/org/zstack/utils/clouderrorcode/CloudOperationsErrorCode.java b/utils/src/main/java/org/zstack/utils/clouderrorcode/CloudOperationsErrorCode.java index 68ab7087c6c..c9e26d0456d 100644 --- a/utils/src/main/java/org/zstack/utils/clouderrorcode/CloudOperationsErrorCode.java +++ b/utils/src/main/java/org/zstack/utils/clouderrorcode/CloudOperationsErrorCode.java @@ -13917,6 +13917,10 @@ public class CloudOperationsErrorCode { public static final String ORG_ZSTACK_NETWORK_SERVICE_LB_10189 = "ORG_ZSTACK_NETWORK_SERVICE_LB_10189"; + public static final String ORG_ZSTACK_NETWORK_SERVICE_LB_10190 = "ORG_ZSTACK_NETWORK_SERVICE_LB_10190"; + + public static final String ORG_ZSTACK_NETWORK_SERVICE_LB_10191 = "ORG_ZSTACK_NETWORK_SERVICE_LB_10191"; + public static final String ORG_ZSTACK_IPSEC_10000 = "ORG_ZSTACK_IPSEC_10000"; public static final String ORG_ZSTACK_IPSEC_10001 = "ORG_ZSTACK_IPSEC_10001";