diff --git a/opendj-server-legacy/src/main/java/org/opends/server/replication/server/ReplicationServer.java b/opendj-server-legacy/src/main/java/org/opends/server/replication/server/ReplicationServer.java index 05232fe9a5..cc4594912d 100644 --- a/opendj-server-legacy/src/main/java/org/opends/server/replication/server/ReplicationServer.java +++ b/opendj-server-legacy/src/main/java/org/opends/server/replication/server/ReplicationServer.java @@ -607,11 +607,9 @@ void runConnect() * cannot guarantee this since the configuration may not contain this * RS. */ - final Set connectedRSAddresses = - getConnectedRSAddresses(domain); for (HostPort rsAddress : configuredRSAddresses) { - if (connectedRSAddresses.contains(rsAddress)) + if (domain.isConnectedToServerAt(rsAddress)) { // Skip: already connected. The connection may be the one that peer made to // this server, which connect() never sees, so this is where a failure @@ -671,16 +669,6 @@ void runConnect() } } - private Set getConnectedRSAddresses(ReplicationServerDomain domain) - { - Set results = new HashSet<>(); - for (ReplicationServerHandler rsHandler : domain.getConnectedRSs().values()) - { - results.add(HostPort.valueOf(rsHandler.getServerAddressURL())); - } - return results; - } - /** * Establish a connection to the server with the address and port. *

@@ -767,13 +755,15 @@ boolean connect(HostPort remoteServerAddress, DN baseDN) * ReplicationServerHandler.connect() registers only above V1, the FIXME there being * older than this, so such a peer is connected and never registered. * - * the address, and the session left open with it, miss a peer which dials out from an - * address other than the one it is configured under -- multi homing, NAT. Its inbound - * handler is registered under the source address of its own connection, - * ServerHandler.toServerAddressURL() reading the host from the session, so the already - * connected branch of runConnect() compares the configured address against one it never - * matches, and the handshake this server offers that same peer aborts on a duplicate - * server id: abortStart() closes the session, and an open session is never seen here + * the address, and the session left open with it, miss a peer which names an address + * this configuration does not use -- the host name of its machine, which setServerURL() + * falls back to when none of the addresses it is configured with is local to it -- and + * dials out from another one this configuration does not use either: multi homing, + * NAT. Its inbound handler is known by the address it named and by the source address + * of its own connection, and the already connected branch of runConnect() finds the + * configured address under neither, so this server offers that same peer a handshake + * which either end resolves as a cross connect, on the address both handlers of the + * peer name: abortStart() closes the session, and an open session is never seen here * again. * * An outage left open is not a line too few but a peer gone silent: recordFailure() @@ -795,11 +785,15 @@ boolean connect(HostPort remoteServerAddress, DN baseDN) * with the same silent abortStart(null), one line up in startFromRemoteRS(). * * The cross connect this server resolves is the one abort of the three where a session - * for the domain does exist: it is the connection the peer made, which the already - * connected branch of runConnect() reports on its next pass. Reaching this line with - * one open needs that registration to land between the snapshot that branch reads and - * the dial below it, so what it costs is one warning, and the pass after it says what - * is true. + * for the domain does exist: it is the connection the peer made. For a peer which names + * an address this configuration uses, that is the session the already connected branch + * of runConnect() reports on its next pass; reaching this line with it open needs the + * registration to land between the check that branch makes and the dial below it, so + * what it costs is one warning, and the pass after it says what is true. For the peer + * of the second reading above no pass matches it, so what was last said of it stays + * said -- nothing where no outage was reported, the warning where one was -- and the + * blacklist runConnect() keeps for an answer without a session is what bounds the + * dialling behind it. */ reportConnectionRestored(remoteServerAddress, baseDN, handshakeCompleted); return handshakeCompleted; @@ -2040,10 +2034,19 @@ private void disconnectRemovedReplicationServers(Set oldRSAddresses) { final Collection serversToDisconnect = new ArrayList<>(); + /* + * The entry of this server is never one to disconnect, and is skipped here as + * runConnect() skips it before it dials: this server holds no session with itself, and + * a peer which names a loopback address of its own machine on this port does answer to + * that entry, because HostPort folds every address local to this machine to localhost. + * Removing the entry of this server -- a list without it is supported, see + * runConnect() -- would otherwise stop the session of such a peer once. + */ + final HostPort localAddress = HostPort.localAddress(getReplicationPort()); final Set newRSAddresses = getConfiguredRSAddresses(); for (HostPort oldRSAddress : oldRSAddresses) { - if (!newRSAddresses.contains(oldRSAddress)) + if (!newRSAddresses.contains(oldRSAddress) && !oldRSAddress.equals(localAddress)) { serversToDisconnect.add(oldRSAddress); } diff --git a/opendj-server-legacy/src/main/java/org/opends/server/replication/server/ReplicationServerDomain.java b/opendj-server-legacy/src/main/java/org/opends/server/replication/server/ReplicationServerDomain.java index c402c5b8fc..6cccf85439 100644 --- a/opendj-server-legacy/src/main/java/org/opends/server/replication/server/ReplicationServerDomain.java +++ b/opendj-server-legacy/src/main/java/org/opends/server/replication/server/ReplicationServerDomain.java @@ -1043,6 +1043,40 @@ else if (origServer.isDataServer()) } } + /** + * Returns whether this domain already holds a session with the replication + * server configured at the provided address. + *

+ * A connected server answers to the address its session came from and to the address it + * named in its start message, and the second of the two is what that server says of + * itself: a peer which names an address this configuration gives to another replication + * server hides that one from the connect thread, which then dials nothing for it and + * reports it connected while it is down. A replication server names itself from its own + * configuration, or, when none of the addresses it is configured with is local to it, + * from the host name of its machine ({@code ReplicationServer.setServerURL()}), so what + * it takes is a named address which resolves here to one this configuration gives to + * another replication server: two of them configured at one address -- a configuration + * copied whole, or two sites whose private ranges overlap -- or that fall back host name + * mapped there by the resolver of this server, which a clone host name, split DNS or a + * stale hosts file gives. The peer it hides is still the one which dials this server. + * + * @param address + * the configured address of a replication server + * @return {@code true} if a connected replication server answers to that + * address, {@code false} otherwise + */ + public boolean isConnectedToServerAt(HostPort address) + { + for (ReplicationServerHandler rsHandler : connectedRSs.values()) + { + if (rsHandler.isServerAt(address)) + { + return true; + } + } + return false; + } + /** * Stop operations with a list of replication servers. * @@ -1054,10 +1088,13 @@ public void stopReplicationServers(Collection serversToDisconnect) { for (ReplicationServerHandler rsHandler : connectedRSs.values()) { - if (serversToDisconnect.contains( - HostPort.valueOf(rsHandler.getServerAddressURL()))) + for (HostPort serverToDisconnect : serversToDisconnect) { - stopServer(rsHandler, false); + if (rsHandler.isServerAt(serverToDisconnect)) + { + stopServer(rsHandler, false); + break; + } } } } @@ -1404,12 +1441,13 @@ public boolean isAlreadyConnectedToRS(ReplicationServerHandler rsHandler) return false; } - if (oldRsHandler.getServerAddressURL().equals( - rsHandler.getServerAddressURL())) + if (oldRsHandler.isSameServerAs(rsHandler)) { // this is the same server, this means that our ServerStart messages // have been sent at about the same time and 2 connections - // have been established. + // have been established -- or that this server reached the same peer at + // an address other than the one that peer connected from, which is what + // a multi homed or NATed peer gives. // Silently drop this connection. return true; } diff --git a/opendj-server-legacy/src/main/java/org/opends/server/replication/server/ReplicationServerHandler.java b/opendj-server-legacy/src/main/java/org/opends/server/replication/server/ReplicationServerHandler.java index 25c4dfb9a6..85f2b76de0 100644 --- a/opendj-server-legacy/src/main/java/org/opends/server/replication/server/ReplicationServerHandler.java +++ b/opendj-server-legacy/src/main/java/org/opends/server/replication/server/ReplicationServerHandler.java @@ -21,6 +21,8 @@ import static org.opends.server.replication.protocol.ProtocolVersion.*; import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Set; @@ -54,6 +56,16 @@ public class ReplicationServerHandler extends ServerHandler /** Properties filled only if remote server is a RS. */ private String serverAddressURL; + /** + * The addresses the remote replication server is known by, built once, when its start + * message names it: the connect thread compares them on every one of its passes, and + * {@link HostPort} logs a name it cannot resolve each time it is built from one -- which + * the fall back of {@code ReplicationServer.setServerURL()} to the host name of the + * machine makes an ordinary thing for a peer to name. What the comparison of two of them + * resolves is reported nowhere above trace, so building them once is what keeps that name + * out of the error log rather than what keeps it out of the resolver. + */ + private List addresses = Collections.emptyList(); /** * This collection will contain as many elements as there are * LDAP servers connected to the remote replication server. @@ -78,7 +90,7 @@ private boolean processStartFromRemote( generationId = inReplServerStartMsg.getGenerationId(); serverId = inReplServerStartMsg.getServerId(); serverURL = inReplServerStartMsg.getServerURL(); - serverAddressURL = toServerAddressURL(serverURL); + setServerAddresses(serverURL); setBaseDNAndDomain(inReplServerStartMsg.getBaseDN(), false); setInitialServerState(inReplServerStartMsg.getServerState()); setSendWindowSize(inReplServerStartMsg.getWindowSize()); @@ -97,11 +109,20 @@ private boolean processStartFromRemote( return inReplServerStartMsg.getSSLEncryption(); } - private String toServerAddressURL(String serverURL) + /** + * Takes the addresses the remote replication server is known by from the URL its start + * message named: that URL, which is the address it is configured under, and the address + * of the connection this session is held on, which is the interface that connection + * happened to use rather than an identity. + */ + private void setServerAddresses(String serverURL) { - final int port = HostPort.valueOf(serverURL).getPort(); + final HostPort namedAddress = HostPort.valueOf(serverURL); // Ensure correct formatting of IPv6 addresses by using a HostPort instance. - return new HostPort(session.getRemoteAddress().getHost(), port).toString(); + final HostPort connectedAddress = + new HostPort(session.getRemoteAddress().getHost(), namedAddress.getPort()); + serverAddressURL = connectedAddress.toString(); + addresses = Arrays.asList(namedAddress, connectedAddress); } /** @@ -715,6 +736,78 @@ public String getServerAddressURL() return serverAddressURL; } + /** + * Returns whether the remote replication server of this handler is the one the provided + * handler holds a session with. + *

+ * Either of the two addresses a remote server is known by identifies it, and the one + * which does depends on where its connection came from: a server reachable at more than + * one address -- a multi homed host, or a NAT where the address a peer connects + * from is not the address it is configured as -- is registered under the + * address of whichever interface the session used, so two sessions with one such server + * carry two different addresses. What both of them do carry is the address that server + * names in its start messages, which is the address it is configured under. + * + * @param other + * the handler to compare the remote server of this one with + * @return {@code true} if both handlers hold a session with the same replication server + */ + boolean isSameServerAs(ReplicationServerHandler other) + { + for (HostPort address : other.addresses) + { + if (isServerAt(address)) + { + return true; + } + } + return false; + } + + /** + * Returns whether the remote replication server of this handler is the one configured at + * the provided address. + *

+ * Both addresses it is known by are compared, because either of them may be the + * configured one: the address the remote server names is the address it is configured + * under in its own configuration, which is the one the rest of the topology configures it + * at as well, while the address its session came from is the only one known of a server + * which names an address this configuration does not use. + *

+ * A name neither end can resolve is compared as the name it is, which is what + * {@link HostPort#equals(Object)} does with it: {@link HostPort#isEquivalentTo(HostPort)} + * resolves both hosts and answers {@code false} for a name it cannot resolve, even + * against that same name. The peer which names one is the peer of the fall back of + * {@code ReplicationServer.setServerURL()}, whose own host name the rest of the topology + * has no reason to resolve, and it is the peer the addresses are there for. What that + * gives up is two servers which share a server id and both name one unresolvable name on + * one port: they are read as one, as two servers which name the same resolvable address + * are under the other arm. + *

+ * A pair the names alone do not answer is resolved on every call, and the handshake road + * makes those calls under the domain lock: both {@code startFromRemoteRS()} and the + * connect road hold that lock across + * {@link ReplicationServerDomain#isAlreadyConnectedToRS(ReplicationServerHandler)}, so a + * resolver which does not answer holds the domain for its own timeout on the road where + * two handlers of one server id name different hosts. Only replication servers reach it: + * the handshake of a data server makes no such comparison. + * + * @param address + * a configured address of a replication server + * @return {@code true} if the remote server of this handler answers to that address + */ + boolean isServerAt(HostPort address) + { + for (HostPort known : addresses) + { + if (address.equals(known) || address.isEquivalentTo(known)) + { + return true; + } + } + return false; + } + /** * Receives a topology msg. * @param topoMsg The message received. diff --git a/opendj-server-legacy/src/test/java/org/opends/server/replication/server/MultiHomedPeerTest.java b/opendj-server-legacy/src/test/java/org/opends/server/replication/server/MultiHomedPeerTest.java new file mode 100644 index 0000000000..0ef74bbad1 --- /dev/null +++ b/opendj-server-legacy/src/test/java/org/opends/server/replication/server/MultiHomedPeerTest.java @@ -0,0 +1,879 @@ +/* + * The contents of this file are subject to the terms of the Common Development and + * Distribution License (the License). You may not use this file except in compliance with the + * License. + * + * You can obtain a copy of the License at legal/CDDLv1.0.txt. See the License for the + * specific language governing permission and limitations under the License. + * + * When distributing Covered Software, include this CDDL Header Notice in each file and include + * the License file at legal/CDDLv1.0.txt. If applicable, add the following below the CDDL + * Header, with the fields enclosed by brackets [] replaced by your own identifying + * information: "Portions copyright [year] [name of copyright owner]". + * + * Copyright 2026 3A Systems, LLC. + */ +package org.opends.server.replication.server; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.opends.messages.ReplicationMessages.*; +import static org.opends.server.TestCaseUtils.*; +import static org.opends.server.util.CollectionUtils.newArrayList; +import static org.opends.server.util.CollectionUtils.newTreeSet; +import static org.opends.server.util.StaticUtils.close; + +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.ServerSocket; +import java.net.Socket; +import java.util.Collections; +import java.util.List; +import java.util.SortedSet; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; + +import org.forgerock.opendj.ldap.DN; +import org.opends.server.TestCaseUtils; +import org.opends.server.replication.ReplicationTestCase; +import org.opends.server.replication.common.DSInfo; +import org.opends.server.replication.common.RSInfo; +import org.opends.server.replication.common.ServerState; +import org.opends.server.replication.protocol.ReplServerStartMsg; +import org.opends.server.replication.protocol.ReplSessionSecurity; +import org.opends.server.replication.protocol.Session; +import org.opends.server.replication.protocol.TopologyMsg; +import org.opends.server.types.HostPort; +import org.testng.annotations.Test; + +/** + * Reproducer for issue #1017: a replication server which is reachable at more than one + * address -- a multi homed host, or a NAT where the address a peer connects from is + * not the address it is configured as -- used to be recognised by the address the + * socket of one of its sessions happened to carry. + *

+ * {@code ReplicationServerHandler.toServerAddressURL()}, which this change removes, took + * the host of a handler from {@code session.getRemoteAddress()} and its port from the start + * message that handler received, so the address a peer was registered under was an artefact + * of which interface its connection used, not an identity. Two things followed, and the + * tests below drive both: + * the already connected test of {@code runConnect()} compared a configured address against + * one which never matches it, so the peer was dialled again on every pass, about once a + * second, for as long as it stayed where it was; and the handshake offered to that same + * peer ran into a handler holding its server id under another address URL, which is + * {@code ERR_DUPLICATE_REPLICATION_SERVER_ID} -- an error logged with no throttle for a + * topology which is merely multi homed. + *

+ * The fixture is the mechanism itself rather than a stand in for it: the peer dials the + * server under test over the loopback interface and names another address in its start + * message, which is what a peer behind a NAT looks like to the server it dials, and the + * session the server dials that other address on reports the address it dialled, which is + * what reaching the same peer at its configured address gives. The addresses named are + * documentation addresses (RFC 5737): nothing answers at them, which is what the connect + * thread of the server under test finds when it dials one -- a host stack sends that SYN to + * its default route, and the connect fails, once per six passes -- a failed dial is + * blacklisted for the five passes after it -- until the peer has registered. + */ +@SuppressWarnings("javadoc") +public class MultiHomedPeerTest extends ReplicationTestCase +{ + private static final int SOCKET_TIMEOUT_MS = 30000; + /** How long the registration of the fake peer is waited for, in milliseconds. */ + private static final long REGISTRATION_TIMEOUT_MS = 30000; + + private static final int RS_ID = 8251; + private static final int PEER_RS_ID = 8252; + /** The server id of the second fake peer, which only the removal case registers. */ + private static final int SECOND_PEER_RS_ID = 8253; + + /** TEST-NET-1: the address the peer is configured under and answers on. */ + private static final byte[] PEER_ADDRESS = { (byte) 192, 0, 2, 1 }; + /** TEST-NET-2: the address of a second, genuinely different server. */ + private static final byte[] OTHER_ADDRESS = { (byte) 198, 51, 100, 1 }; + /** + * A name under the .invalid top level domain (RFC 6761), which resolvers answer does not + * exist: the host name of a peer which this server has no way to resolve, and what + * {@code ReplicationServer.setServerURL()} falls back to naming. + */ + private static final String UNRESOLVABLE_HOST = "nonexistent.invalid"; + + /** + * Tests that a peer which dialled this server from an address it is not configured under + * is still found by the address it is configured under. + *

+ * This is what stops the connect thread from dialling it on every pass: the address the + * handler is registered under is the loopback address the peer's own connection came + * from, and the only address which can match the configured one is the address the peer + * named in its start message. + *

+ * Both halves are asserted, because the predicate and the call site which reads it fail + * apart: what the connect thread does with a peer it finds connected is reported by the + * outage it closes, and an outage is recorded here for it to close. + */ + @Test + public void aPeerRegisteredUnderTheAddressItDialledFromIsFoundByItsConfiguredAddress() + throws Exception + { + TestCaseUtils.startServer(); + + final DN baseDN = DN.valueOf(TEST_ROOT_DN_STRING); + final int[] ports = TestCaseUtils.findFreePorts(2); + final HostPort peerAddress = documentationAddress(PEER_ADDRESS, ports[1]); + + ReplicationServer rs = null; + Session inbound = null; + try + { + rs = startServerWithPeerConfiguredAt(ports[0], "multiHomedPeerSkipDb", peerAddress); + final ReplicationServerDomain domain = rs.getReplicationServerDomain(baseDN, true); + + /* + * An outage for the configured address, which nothing but the already connected + * branch of runConnect() closes: a peer that branch skips is dialled by no one, so a + * connection is reported for it nowhere else. Recorded from this thread rather than + * waited for, because whether the connect thread has dialled that address before the + * peer registers is a race and what is asserted below is not. + */ + assertThat(rs.connect(peerAddress, baseDN)) + .as("nothing answers at " + peerAddress).isFalse(); + + inbound = registerPeerFrom(ports[0], peerAddress, baseDN); + final ReplicationServerHandler registered = waitForRegistration(domain); + + // The precondition of the case rather than an assumption: a peer registered under the + // address it is configured under is a peer this test says nothing about. + assertThat(registered.getServerAddressURL()) + .as("the peer should be registered under the loopback address it dialled from") + .isEqualTo(loopbackAt(ports[1]).toString()); + assertThat(registered.getServerURL()) + .as("the peer should name the address it is configured under in its start message") + .isEqualTo(peerAddress.toString()); + + assertThat(domain.isConnectedToServerAt(peerAddress)) + .as("the peer is connected, so its configured address must not be dialled again") + .isTrue(); + assertThat(waitForConnectRestored(rs, peerAddress, baseDN)) + .as("the connect thread must find the peer at its configured address and close the" + + " outage reported for it, rather than dial it again") + .isTrue(); + } + finally + { + close(inbound); + removeQuietly(rs); + } + } + + /** + * Tests that a second session with a peer already connected under another address is + * dropped as the duplicate connection it is, and not reported as two servers sharing a + * server id. + *

+ * The server under test dials the peer at its configured address while holding the + * session that same peer dialled it on. Both sessions are with one server, which names + * one address in both of its start messages; only the addresses the two sockets carry + * differ, which is what being reachable at more than one address means. + */ + @Test + public void aPeerReachedAtItsConfiguredAddressIsNotReportedAsADuplicateServerId() + throws Exception + { + TestCaseUtils.startServer(); + + final DN baseDN = DN.valueOf(TEST_ROOT_DN_STRING); + final int[] ports = TestCaseUtils.findFreePorts(2); + final HostPort peerAddress = documentationAddress(PEER_ADDRESS, ports[1]); + + ReplicationServer rs = null; + Session inbound = null; + try + { + rs = startServerWithPeerConfiguredAt(ports[0], "multiHomedPeerDuplicateDb", peerAddress); + final ReplicationServerDomain domain = rs.getReplicationServerDomain(baseDN, true); + inbound = registerPeerFrom(ports[0], peerAddress, baseDN); + final ReplicationServerHandler registered = waitForRegistration(domain); + + final List records = + errorLogRecordsOfHandshakeWith(rs, baseDN, peerAddress, peerAddress); + + assertThat(duplicateServerIdRecords(records, rs, loopbackAt(ports[1]), peerAddress)) + .as("a peer reachable at more than one address is not two servers sharing a" + + " server id, and every attempt to reach it would log this again") + .isEmpty(); + assertThat(domain.getConnectedRSs().get(PEER_RS_ID)) + .as("the session the peer dialled must outlive the duplicate connection") + .isSameAs(registered); + } + finally + { + close(inbound); + removeQuietly(rs); + } + } + + /** + * Tests that a peer which names an address this configuration does not use is found by + * the address its own connection came from, which is the only one known of it. + *

+ * The peer of the first case names the address it is configured under, and is found by + * it. This one is the other way round: it is configured at the address it dials this + * server from, and names one this topology does not configure it at -- the host name of + * its machine under another of its addresses, which {@code setServerURL()} falls back to. + * The address it names matches nothing here, so the address its session came from is what + * has to match. + */ + @Test + public void aPeerWhichNamesAnAddressThisConfigurationDoesNotUseIsFoundByTheOneItDialledFrom() + throws Exception + { + TestCaseUtils.startServer(); + + final DN baseDN = DN.valueOf(TEST_ROOT_DN_STRING); + final int[] ports = TestCaseUtils.findFreePorts(2); + final HostPort peerAddress = loopbackAt(ports[1]); + final HostPort namedAddress = documentationAddress(OTHER_ADDRESS, ports[1]); + + ReplicationServer rs = null; + Session inbound = null; + try + { + rs = startServerWithPeerConfiguredAt(ports[0], "multiHomedPeerNamedDb", peerAddress); + final ReplicationServerDomain domain = rs.getReplicationServerDomain(baseDN, true); + inbound = registerPeerFrom(ports[0], namedAddress, baseDN); + final ReplicationServerHandler registered = waitForRegistration(domain); + + // The precondition of the case rather than an assumption, as in the first case: a + // peer which names the address it is configured under is the peer of that one. + assertThat(registered.getServerURL()) + .as("the peer should name the address this configuration does not use") + .isEqualTo(namedAddress.toString()); + assertThat(registered.getServerAddressURL()) + .as("the peer should be registered under the loopback address it dialled from") + .isEqualTo(peerAddress.toString()); + + assertThat(domain.isConnectedToServerAt(peerAddress)) + .as("the address the connection of such a peer came from is the only one which" + + " can match the address it is configured at") + .isTrue(); + } + finally + { + close(inbound); + removeQuietly(rs); + } + } + + /** + * Tests that a peer which names a host this server cannot resolve is recognised by that + * name all the same, rather than reported as two servers sharing a server id. + *

+ * {@code setServerURL()} falls back to the host name of the machine when none of the + * addresses a peer is configured with is local to it, and the rest of the topology has no + * reason to resolve that name. Both handlers of such a peer name it, so it is what tells + * this server they are one -- but a comparison which resolves both sides answers that a + * name it cannot resolve is equivalent to nothing at all, not even to itself, and the + * handshake this server offers such a peer would abort on a duplicate server id on every + * pass which reaches it. + */ + @Test + public void aPeerWhichNamesAnUnresolvableHostIsNotReportedAsADuplicateServerId() + throws Exception + { + TestCaseUtils.startServer(); + + final DN baseDN = DN.valueOf(TEST_ROOT_DN_STRING); + final int[] ports = TestCaseUtils.findFreePorts(2); + final HostPort peerAddress = documentationAddress(PEER_ADDRESS, ports[1]); + final HostPort namedAddress = new HostPort(UNRESOLVABLE_HOST, ports[1]); + + ReplicationServer rs = null; + Session inbound = null; + try + { + rs = startServerWithPeerConfiguredAt(ports[0], "multiHomedPeerUnresolvedDb", peerAddress); + final ReplicationServerDomain domain = rs.getReplicationServerDomain(baseDN, true); + inbound = registerPeerFrom(ports[0], namedAddress, baseDN); + final ReplicationServerHandler registered = waitForRegistration(domain); + + /* + * The premise of this case rather than an assumption about the network it runs on: + * HostPort reports a name it cannot resolve each time it builds an address from one, + * so those records are what says this machine answers that the name below does not + * exist. A resolver which synthesises an address for a name which does not -- consumer + * ISPs, captive portals, some corporate DNS -- would have isEquivalentTo() answer this + * case on its own, which leaves the arm the case is here for unread, the count below + * vacuous and nothing red to say so. + */ + assertThat(recordsContaining(TestCaseUtils.ERROR_TEXT_WRITER.getMessages(), + ERR_COULD_NOT_SOLVE_HOSTNAME.get(UNRESOLVABLE_HOST).toString())) + .as("the resolver of this machine must answer that " + UNRESOLVABLE_HOST + + " does not exist, which is the premise of this case") + .isNotEmpty(); + + /* + * What the addresses are built from is read once, when the start message names them: + * a name which cannot be resolved is reported by HostPort each time one is built from + * it, and the connect thread compares them on every pass. The comparison below is the + * one that thread makes. + */ + TestCaseUtils.ERROR_TEXT_WRITER.clear(); + for (int pass = 0; pass < 3; pass++) + { + domain.isConnectedToServerAt(peerAddress); + } + assertThat(recordsContaining(TestCaseUtils.ERROR_TEXT_WRITER.getMessages(), + ERR_COULD_NOT_SOLVE_HOSTNAME.get(UNRESOLVABLE_HOST).toString())) + .as("comparing the addresses of a handler must not build them again") + .isEmpty(); + + final List records = + errorLogRecordsOfHandshakeWith(rs, baseDN, peerAddress, namedAddress); + + assertThat(duplicateServerIdRecords(records, rs, loopbackAt(ports[1]), peerAddress)) + .as("a peer whose name this server cannot resolve is one server all the same, and" + + " every attempt to reach it would log this again") + .isEmpty(); + assertThat(domain.getConnectedRSs().get(PEER_RS_ID)) + .as("the session the peer dialled must outlive the duplicate connection") + .isSameAs(registered); + } + finally + { + close(inbound); + removeQuietly(rs); + } + } + + /** + * Tests that two genuinely different replication servers sharing a server id are still + * reported, which is the misconfiguration the address comparison is there to catch. + *

+ * Nothing is shared here: the connected peer names one address and dialled from the + * loopback interface, and the server answering the handshake names, and is reached at, + * another address on the same port. The port is the same one on purpose: it is the host + * which tells the two servers apart, and a second port would answer the comparison before + * any host of it is read. + */ + @Test + public void twoServersSharingAServerIdAreStillReported() throws Exception + { + TestCaseUtils.startServer(); + + final DN baseDN = DN.valueOf(TEST_ROOT_DN_STRING); + final int[] ports = TestCaseUtils.findFreePorts(2); + final HostPort peerAddress = documentationAddress(PEER_ADDRESS, ports[1]); + final HostPort otherAddress = documentationAddress(OTHER_ADDRESS, ports[1]); + + ReplicationServer rs = null; + Session inbound = null; + try + { + rs = startServerWithPeerConfiguredAt(ports[0], "multiHomedPeerConflictDb", peerAddress); + final ReplicationServerDomain domain = rs.getReplicationServerDomain(baseDN, true); + inbound = registerPeerFrom(ports[0], peerAddress, baseDN); + waitForRegistration(domain); + + final List records = + errorLogRecordsOfHandshakeWith(rs, baseDN, otherAddress, otherAddress); + + assertThat(duplicateServerIdRecords(records, rs, loopbackAt(ports[1]), otherAddress)) + .as("two servers which share nothing but a server id must still be reported") + .isNotEmpty(); + } + finally + { + close(inbound); + removeQuietly(rs); + } + } + + /** + * Tests that a second session which comes from the address a peer is already connected on + * is read as that peer, whatever that session names. + *

+ * This is the other arm of the comparison the handshake makes, and what it gives up: + * a session from the address a peer is connected on is that peer, so two replication + * servers which share a server id and reach this one from behind a single gateway are read + * as one and the second of them is dropped in silence. The base did the same with them -- + * it compared the addresses the two sockets carried, which for such a pair is one string + * twice -- and this is the arm which recognises the peer of the case above, whose named + * address matches nothing in this configuration. + */ + @Test + public void aSecondSessionFromTheAddressAPeerIsConnectedOnIsReadAsThatPeer() throws Exception + { + TestCaseUtils.startServer(); + + final DN baseDN = DN.valueOf(TEST_ROOT_DN_STRING); + final int[] ports = TestCaseUtils.findFreePorts(2); + final HostPort peerAddress = documentationAddress(PEER_ADDRESS, ports[1]); + final HostPort otherAddress = documentationAddress(OTHER_ADDRESS, ports[1]); + + ReplicationServer rs = null; + Session inbound = null; + try + { + rs = startServerWithPeerConfiguredAt(ports[0], "multiHomedPeerGatewayDb", peerAddress); + final ReplicationServerDomain domain = rs.getReplicationServerDomain(baseDN, true); + inbound = registerPeerFrom(ports[0], peerAddress, baseDN); + final ReplicationServerHandler registered = waitForRegistration(domain); + + /* + * The handshake is answered from the loopback address the session of the registered + * peer came from, and what answers it names an address neither handler carries + * otherwise: the named addresses of the two differ, so the address the two sessions + * have in common is the only thing which can match them. + */ + final List records = + errorLogRecordsOfHandshakeWith(rs, baseDN, loopbackAt(ports[1]), otherAddress); + + assertThat( + duplicateServerIdRecords(records, rs, loopbackAt(ports[1]), loopbackAt(ports[1]))) + .as("a session from the address a peer is connected on is that peer, whatever that" + + " session names") + .isEmpty(); + assertThat(domain.getConnectedRSs().get(PEER_RS_ID)) + .as("the session the peer dialled must outlive the duplicate connection") + .isSameAs(registered); + } + finally + { + close(inbound); + removeQuietly(rs); + } + } + + /** + * Tests that a peer taken out of the configuration is disconnected even though the + * session it is connected on came from another address. + *

+ * {@code disconnectRemovedReplicationServers()} hands the addresses which were removed + * from {@code ds-cfg-replication-server} to this domain, and a handler which does not + * answer to any of them stays connected: the peer an administrator took out of the + * topology keeps replicating with this server until one of the two is restarted. + */ + @Test + public void aPeerRemovedFromTheConfigurationIsDisconnectedByItsConfiguredAddress() + throws Exception + { + TestCaseUtils.startServer(); + + final DN baseDN = DN.valueOf(TEST_ROOT_DN_STRING); + final int[] ports = TestCaseUtils.findFreePorts(2); + final HostPort peerAddress = documentationAddress(PEER_ADDRESS, ports[1]); + + ReplicationServer rs = null; + Session inbound = null; + try + { + rs = startServerWithPeerConfiguredAt(ports[0], "multiHomedPeerRemovedDb", peerAddress); + final ReplicationServerDomain domain = rs.getReplicationServerDomain(baseDN, true); + inbound = registerPeerFrom(ports[0], peerAddress, baseDN); + waitForRegistration(domain); + + domain.stopReplicationServers(Collections.singletonList(peerAddress)); + + assertThat(domain.getConnectedRSs().keySet()) + .as("the peer removed from the configuration must be disconnected") + .doesNotContain(PEER_RS_ID); + } + finally + { + close(inbound); + removeQuietly(rs); + } + } + + /** + * Tests that taking the entry of this server out of {@code ds-cfg-replication-server} + * stops no peer, while taking the entry of a peer out still stops that peer. + *

+ * {@code HostPort} normalises every address local to this machine to {@code localhost}, so + * the entry of this server and the loopback address a peer names for itself are one + * address on this port: a peer whose own configuration lists {@code localhost:P} for + * itself names exactly that in its start messages -- {@code setServerURL()} takes the + * first configured entry which is local to it -- and on this server that name normalises + * to what its own entry does. The connect thread skips the entry of this server before it + * dials; the road which disconnects removed replication servers has to skip it as well, or + * removing it -- a configuration which does not list this server is supported, see + * {@code runConnect()} -- costs such a peer its session, which it then has to dial again. + *

+ * Both entries go in one change, so what tells the two peers apart is that guard alone: + * the peer which names the entry of this server stays, the peer whose own entry was + * removed goes. What the in JVM fixture cannot give is the fold itself, which needs one + * port on two machines; what it names is the address of this server, which normalises the + * same way. + */ + @Test + public void removingTheEntryOfThisServerStopsNoPeer() throws Exception + { + TestCaseUtils.startServer(); + + final DN baseDN = DN.valueOf(TEST_ROOT_DN_STRING); + final int[] ports = TestCaseUtils.findFreePorts(2); + final HostPort ownAddress = loopbackAt(ports[0]); + final HostPort peerAddress = loopbackAt(ports[1]); + final String dbDirName = "multiHomedPeerOwnEntryDb"; + + ReplicationServer rs = null; + Session namesThisServer = null; + Session namesItsOwnEntry = null; + try + { + rs = startServerWithPeersConfiguredAt(ports[0], dbDirName, ownAddress, peerAddress); + final ReplicationServerDomain domain = rs.getReplicationServerDomain(baseDN, true); + namesThisServer = registerPeerFrom(ports[0], PEER_RS_ID, ownAddress, baseDN); + final ReplicationServerHandler namedThisServer = waitForRegistration(domain, PEER_RS_ID); + namesItsOwnEntry = registerPeerFrom(ports[0], SECOND_PEER_RS_ID, peerAddress, baseDN); + waitForRegistration(domain, SECOND_PEER_RS_ID); + + // The precondition of the case rather than an assumption: a peer which does not answer + // to the entry of this server is a peer this case says nothing about. + assertThat(namedThisServer.isServerAt(HostPort.localAddress(ports[0]))) + .as("the peer which names a loopback address on the port of this server should" + + " answer to the entry of this server") + .isTrue(); + + // Both entries removed at once, which is what an administrator emptying the list does. + rs.applyConfigurationChange(configurationWithPeersAt(ports[0], dbDirName)); + + assertThat(domain.getConnectedRSs().keySet()) + .as("a peer which names the address this server listens on must outlive the removal" + + " of the entry of this server") + .contains(PEER_RS_ID); + assertThat(domain.getConnectedRSs().keySet()) + .as("the peer whose own entry was removed must still be disconnected") + .doesNotContain(SECOND_PEER_RS_ID); + } + finally + { + close(namesThisServer); + close(namesItsOwnEntry); + removeQuietly(rs); + } + } + + /** + * Starts a replication server whose only configured peer is at the provided address. + *

+ * Nothing ever answers there -- the address is a documentation address -- which is what + * the connect thread of the server does with it: it dials it, fails, and comes back to it + * six passes later, the failed dial being blacklisted for the five in between. Whether it + * dials it at all is what the first case is about. + */ + private ReplicationServer startServerWithPeerConfiguredAt(int port, String dbDirName, + HostPort peerAddress) throws Exception + { + return startServerWithPeersConfiguredAt(port, dbDirName, peerAddress); + } + + /** Starts a replication server whose configured peers are the provided addresses. */ + private ReplicationServer startServerWithPeersConfiguredAt(int port, String dbDirName, + HostPort... peerAddresses) throws Exception + { + return new ReplicationServer(configurationWithPeersAt(port, dbDirName, peerAddresses)); + } + + /** + * Returns the configuration of the server under test, listing the provided addresses as + * the replication servers of its topology. + */ + private ReplServerFakeConfiguration configurationWithPeersAt(int port, String dbDirName, + HostPort... peerAddresses) + { + final SortedSet configured = newTreeSet(); + for (HostPort peerAddress : peerAddresses) + { + configured.add(peerAddress.toString()); + } + return new ReplServerFakeConfiguration(port, dbDirName, 0, RS_ID, 0, 100, configured); + } + + /** + * Has the fake peer dial the server under test over the loopback interface and complete + * the handshake, which registers it under the address that connection came from while its + * start message names the address it is configured under. + * + * @return the session the registration hangs on, closed by the caller + */ + private Session registerPeerFrom(int port, HostPort registeredAs, DN baseDN) throws Exception + { + return registerPeerFrom(port, PEER_RS_ID, registeredAs, baseDN); + } + + /** + * Has a fake peer of the provided server id dial the server under test, as + * {@link #registerPeerFrom(int, HostPort, DN)} does for the peer of the cases which need + * one peer only. + * + * @return the session the registration hangs on, closed by the caller + */ + private Session registerPeerFrom(int port, int peerServerId, HostPort registeredAs, DN baseDN) + throws Exception + { + final ReplSessionSecurity security = getReplSessionSecurity(); + final Socket socket = new Socket(); + Session session = null; + try + { + socket.setTcpNoDelay(true); + socket.connect(new InetSocketAddress("127.0.0.1", port), SOCKET_TIMEOUT_MS); + session = security.createClientSession(socket, SOCKET_TIMEOUT_MS); + session.publish(peerStartMsg(peerServerId, registeredAs, baseDN)); + session.receive(); + // The initiator of a session decides whether it is encrypted, and the start message + // above asked for it not to be: both ends leave the SSL session together, right after + // the start messages have been exchanged. + session.stopEncryption(); + /* + * The second phase: the server reads this one before it sends its own, and registers + * the handler once it has sent it. The list holds this peer and nothing else -- + * waitAndProcessTopoFromRemoteRS() reads rsInfos.get(0) above protocol version 4, so + * an empty one ends the handshake on an IndexOutOfBoundsException instead, which is + * an abort like any other and would leave the peer unregistered. + */ + final RSInfo peerInfo = new RSInfo(peerServerId, registeredAs.toString(), -1, (byte) 1, 1); + session.publish(new TopologyMsg(Collections. emptyList(), newArrayList(peerInfo))); + session.receive(); + return session; + } + catch (Exception e) + { + close(session); + close(socket); + throw e; + } + } + + /** + * Runs the handshake the server under test offers a peer it dials at the provided address + * and returns the error log records that handshake wrote. + *

+ * The session is dialled over the loopback interface and reports {@code dialledAt} as the + * address it reaches the peer at, which is what dialling a peer at its configured address + * gives whatever interface the peer's own connection to this server used. The answer is + * a well formed start message naming {@code answersAs}: what ends the handshake is the + * handler this server already holds for that server id, not what the peer answers here. + * + * @param listenPort + * the port the fake peer answers the handshake on + * @param dialledAt + * the address the session reports the peer at + * @param answersAs + * the address the peer names in the start message it answers with + */ + private List errorLogRecordsOfHandshakeWith(final ReplicationServer rs, + final DN baseDN, final HostPort dialledAt, final HostPort answersAs) + throws Exception + { + final ExecutorService peerThread = Executors.newSingleThreadExecutor(); + try (ServerSocket peerListen = TestCaseUtils.bindFreePort()) + { + peerListen.setSoTimeout(SOCKET_TIMEOUT_MS); + final ReplSessionSecurity security = getReplSessionSecurity(); + final int listenPort = peerListen.getLocalPort(); + + final Future dialled = peerThread.submit(new Callable() + { + @Override + public Session call() throws Exception + { + return dialPeerAt(security, dialledAt, listenPort); + } + }); + + try (Session peerEnd = + security.createServerSession(peerListen.accept(), SOCKET_TIMEOUT_MS); + Session session = dialled.get(SOCKET_TIMEOUT_MS, TimeUnit.MILLISECONDS)) + { + final Future answer = peerThread.submit(new Callable() + { + @Override + public Void call() throws Exception + { + peerEnd.receive(); + peerEnd.publish(peerStartMsg(answersAs, baseDN)); + return null; + } + }); + + TestCaseUtils.ERROR_TEXT_WRITER.clear(); + new ReplicationServerHandler(session, 100, rs, 100).connect(baseDN, false); + answer.get(SOCKET_TIMEOUT_MS, TimeUnit.MILLISECONDS); + return TestCaseUtils.ERROR_TEXT_WRITER.getMessages(); + } + } + finally + { + peerThread.shutdownNow(); + } + } + + /** + * Connects to the provided local port with a socket which reports the provided address as + * the one it is connected to, as dialling a peer at that address does. + */ + private Session dialPeerAt(ReplSessionSecurity security, HostPort dialledAt, int port) + throws Exception + { + // Named, so that the SSL socket factory asking the socket for its host name does not + // send a reverse lookup of an address no name server knows anything about. + final InetAddress reported = InetAddress.getByAddress("peer.example.com", + InetAddress.getByName(dialledAt.getHost()).getAddress()); + final Socket socket = new Socket() + { + @Override + public InetAddress getInetAddress() + { + return reported; + } + }; + try + { + socket.setTcpNoDelay(true); + socket.connect(new InetSocketAddress("127.0.0.1", port), SOCKET_TIMEOUT_MS); + return security.createClientSession(socket, SOCKET_TIMEOUT_MS); + } + catch (Exception e) + { + close(socket); + throw e; + } + } + + /** Returns the start message of the fake peer, naming the provided address. */ + private ReplServerStartMsg peerStartMsg(HostPort address, DN baseDN) + { + return peerStartMsg(PEER_RS_ID, address, baseDN); + } + + /** Returns the start message of a fake peer of the provided server id. */ + private ReplServerStartMsg peerStartMsg(int peerServerId, HostPort address, DN baseDN) + { + return new ReplServerStartMsg(peerServerId, address.toString(), baseDN, 100, + new ServerState(), -1, false, (byte) 1, 5000); + } + + /** Returns an address of the documentation ranges, which nothing routes. */ + private HostPort documentationAddress(byte[] address, int port) throws Exception + { + return new HostPort(InetAddress.getByAddress(address).getHostAddress(), port); + } + + /** Returns the loopback address a connection of these tests comes from. */ + private HostPort loopbackAt(int port) + { + return new HostPort("127.0.0.1", port); + } + + /** + * Waits for the domain to hold the handler of the fake peer, which the server registers + * after it has sent the topology message the handshake above reads: the registration + * lands just behind the thread which drove it. + */ + private ReplicationServerHandler waitForRegistration(ReplicationServerDomain domain) + throws Exception + { + return waitForRegistration(domain, PEER_RS_ID); + } + + /** + * Waits for the domain to hold the handler of the fake peer of the provided server id, as + * {@link #waitForRegistration(ReplicationServerDomain)} does for the peer of the cases + * which need one peer only. + */ + private ReplicationServerHandler waitForRegistration(ReplicationServerDomain domain, + int peerServerId) throws Exception + { + final long deadline = System.currentTimeMillis() + REGISTRATION_TIMEOUT_MS; + ReplicationServerHandler registered; + while (true) + { + registered = domain.getConnectedRSs().get(peerServerId); + if (registered != null || System.currentTimeMillis() > deadline) + { + break; + } + Thread.sleep(50); + } + assertThat(registered) + .as("the fake peer of server id " + peerServerId + " should have registered with the" + + " domain") + .isNotNull(); + return registered; + } + + /** + * Waits for the connect thread to report the peer at the provided address connected, + * driving a pass rather than waiting one out. + *

+ * That report is what the already connected branch of {@code runConnect()} does with a + * peer it finds registered, and the only thing this server does with one: the branch + * which dials a peer reports a connection only for a handshake it completed, which the + * documentation address of these tests never gives. + */ + private boolean waitForConnectRestored(ReplicationServer rs, HostPort peer, DN baseDN) + throws Exception + { + final String message = + NOTE_REPLICATION_SERVER_CONNECT_RESTORED.get(RS_ID, peer, baseDN).toString(); + final long deadline = System.currentTimeMillis() + REGISTRATION_TIMEOUT_MS; + while (true) + { + if (!recordsContaining(TestCaseUtils.ERROR_TEXT_WRITER.getMessages(), message).isEmpty()) + { + return true; + } + if (System.currentTimeMillis() > deadline) + { + return false; + } + rs.waitConnections(); + } + } + + /** + * Returns the records of the provided log which report the two provided address URLs as + * two replication servers sharing a server id. + *

+ * The addresses are the ones the message names, so a record of the handshake next door + * cannot be read as one of the handshake under test. + */ + private List duplicateServerIdRecords(List records, + ReplicationServer rs, HostPort connectedAs, HostPort reachedAt) + { + return recordsContaining(records, ERR_DUPLICATE_REPLICATION_SERVER_ID.get( + rs.getMonitorInstanceName(), connectedAs, reachedAt, PEER_RS_ID).toString()); + } + + /** Returns the records of the provided log which carry the provided message. */ + private List recordsContaining(List records, String message) + { + final List results = newArrayList(); + for (String record : records) + { + if (record.contains(message)) + { + results.add(record); + } + } + return results; + } + + /** Teardown must never mask the primary assertion failure. */ + private void removeQuietly(ReplicationServer replicationServer) + { + try + { + if (replicationServer != null) + { + remove(replicationServer); + } + } + catch (Exception ignored) + { + } + } +} diff --git a/opendj-server-legacy/src/test/java/org/opends/server/replication/server/ReplicationServerConnectFailureTest.java b/opendj-server-legacy/src/test/java/org/opends/server/replication/server/ReplicationServerConnectFailureTest.java index 3c43b5c6c0..103da8906d 100644 --- a/opendj-server-legacy/src/test/java/org/opends/server/replication/server/ReplicationServerConnectFailureTest.java +++ b/opendj-server-legacy/src/test/java/org/opends/server/replication/server/ReplicationServerConnectFailureTest.java @@ -159,7 +159,7 @@ public void aPeerWhichIsDownIsReportedOnceAndItsReturnIsReported() throws Except * {@code WARN_REPLICATION_SERVER_CONNECT_ERROR} is reported for the socket and for the * session built on it, the handshake throwing nothing of its own. Holding the outage open * across an abort silences the peer this server never sees connected under the address it - * dialled -- the multi homed peer of + * dialled -- the peer registered under an address this configuration does not use, of * {@link #aPeerRegisteredUnderAnotherAddressStillClosesItsOutage}, and one protocol * version down a peer which negotiates V1, connected and never registered. *

@@ -313,17 +313,18 @@ public void aSessionEstablishedAfterAnAnswerWithoutOneIsReported() throws Except * Tests that a peer already registered under an address other than the one it is * configured under still closes the outage reported for it. *

- * This is the multi homed peer, and the reason the recovery can be read neither from the - * address nor from the session. {@code ServerHandler.toServerAddressURL()} takes the host - * of a handler from {@code session.getRemoteAddress()} and its port from the start message - * that handler received, so a peer which dials this server from an address it is not - * configured under is registered under that other address. Two things follow, and this - * test drives both: the already connected branch of {@code runConnect()} compares the - * configured address against one which never matches it, so it can close nothing; and the - * handshake this server offers that same peer runs into a handler holding its server id - * under another address URL, which is {@code ERR_DUPLICATE_REPLICATION_SERVER_ID}, an - * abort of this server rather than of the peer, and a session closed at the end of - * {@code connect()} for as long as the peer stays where it is. + * This is the peer {@code runConnect()} matches by none of the addresses its handler is + * known by, and the reason the recovery can be read neither from the address nor from the + * session. A handler is known by the address its start message names and by the address + * its session came from ({@code ReplicationServerHandler.setServerAddresses()}), and the + * peer below names a port this configuration does not use, which both of those carry. + * Two things follow, and this test drives both: the already connected branch of + * {@code runConnect()} compares the configured address against two which never match it, + * so it can close nothing; and the handshake this server offers that same peer runs into + * a handler holding its server id at another address, which is + * {@code ERR_DUPLICATE_REPLICATION_SERVER_ID}, an abort of this server rather than of the + * peer, and a session closed at the end of {@code connect()} for as long as the peer + * stays where it is. *

* Gating the recovery on either leaves the record of such a peer uncleared for good, and * {@code recordFailure()} returns false from then on: the next real outage of it -- the @@ -337,9 +338,10 @@ public void aPeerRegisteredUnderAnotherAddressStillClosesItsOutage() throws Exce final DN baseDN = DN.valueOf(TEST_ROOT_DN_STRING); /* * Three ports: the server under test, the address the peer is configured under and - * answers on, and the address it registers itself under. The last is never bound -- - * what a multi homed peer costs is that the two addresses are not compared equal, and - * a port nothing listens on is that, without a second address to bind. + * answers on, and the address it names and is therefore registered under. The last is + * never bound -- what such a peer costs is that the configured address matches neither + * of the two addresses its handler is known by, and a port nothing listens on is that, + * without a second address to bind. */ final int[] ports = TestCaseUtils.findFreePorts(3); final HostPort peerAddress = HostPort.valueOf("127.0.0.1:" + ports[1]);