From f18cb11cb976d22b77c38752db8c3683487e51ee Mon Sep 17 00:00:00 2001 From: Valera V Harseko Date: Thu, 10 Sep 2026 14:55:42 +0300 Subject: [PATCH 1/4] [#1017] Recognise a replication server by the address it names, not by the one its session came from A remote replication server was identified by the address the socket of one of its sessions happened to carry, which `ServerHandler.toServerAddressURL()` takes from `session.getRemoteAddress()`. That address is the interface a connection used rather than an identity, so a peer reachable at more than one address -- a multi homed host, or a NAT where the address a peer connects from is not the address it is configured as -- was never recognised as the peer configured at its own address. A remote server is now known by both of the addresses it comes with, the one it names in its start message and the one its session came from, and either identifies it. They are resolved once, when the start message names them, because the connect thread compares them on every one of its passes and `HostPort` logs a name it cannot resolve each time one is built from it. Three places read the address, and all three were wrong for such a peer: * `runConnect()` skipped a configured peer only when a handler was registered under that exact address, so it dialled a peer it was already connected to about once a second, and every one of those handshakes aborted and logged `ERR_DUPLICATE_REPLICATION_SERVER_ID` or `ERR_RS_DISCONNECTED_DURING_HANDSHAKE` with no throttle, for as long as the peer stayed where it was; * `isAlreadyConnectedToRS()` compared the address URLs of the two handlers as strings, so a second session with a peer already connected read as two replication servers sharing a server id, which is a misconfiguration this topology does not have; * `stopReplicationServers()` compared the addresses removed from `ds-cfg-replication-server` against the registered one, so a peer an administrator took out of the topology kept its session. The comparison is `HostPort.isEquivalentTo()`, which is what the data server side already uses for the same question in `ReplicationBroker`: `HostPort.equals()` resolves only the first address a name maps to, and the multi homing that leaves it blind to is noted in a FIXME of its own in `normalizeHost()`. What this gives up is the peer whose configuration was copied whole: two servers which share a server id and name the same address are now read as one and the second session is dropped in silence, where the source addresses used to tell them apart. Two servers which name different addresses are still reported, which is every duplicate id an administrator can act on. --- .../replication/server/ReplicationServer.java | 14 +- .../server/ReplicationServerDomain.java | 37 +- .../server/ReplicationServerHandler.java | 81 ++- .../server/MultiHomedPeerTest.java | 499 ++++++++++++++++++ 4 files changed, 608 insertions(+), 23 deletions(-) create mode 100644 opendj-server-legacy/src/test/java/org/opends/server/replication/server/MultiHomedPeerTest.java diff --git a/opendj-server-legacy/src/main/java/org/opends/server/replication/server/ReplicationServer.java b/opendj-server-legacy/src/main/java/org/opends/server/replication/server/ReplicationServer.java index 05232fe9a5..c2ecc4f5e7 100644 --- a/opendj-server-legacy/src/main/java/org/opends/server/replication/server/ReplicationServer.java +++ b/opendj-server-legacy/src/main/java/org/opends/server/replication/server/ReplicationServer.java @@ -607,11 +607,9 @@ void runConnect() * cannot guarantee this since the configuration may not contain this * RS. */ - final Set connectedRSAddresses = - getConnectedRSAddresses(domain); for (HostPort rsAddress : configuredRSAddresses) { - if (connectedRSAddresses.contains(rsAddress)) + if (domain.isConnectedToServerAt(rsAddress)) { // Skip: already connected. The connection may be the one that peer made to // this server, which connect() never sees, so this is where a failure @@ -671,16 +669,6 @@ void runConnect() } } - private Set getConnectedRSAddresses(ReplicationServerDomain domain) - { - Set results = new HashSet<>(); - for (ReplicationServerHandler rsHandler : domain.getConnectedRSs().values()) - { - results.add(HostPort.valueOf(rsHandler.getServerAddressURL())); - } - return results; - } - /** * Establish a connection to the server with the address and port. *

diff --git a/opendj-server-legacy/src/main/java/org/opends/server/replication/server/ReplicationServerDomain.java b/opendj-server-legacy/src/main/java/org/opends/server/replication/server/ReplicationServerDomain.java index c402c5b8fc..67886770af 100644 --- a/opendj-server-legacy/src/main/java/org/opends/server/replication/server/ReplicationServerDomain.java +++ b/opendj-server-legacy/src/main/java/org/opends/server/replication/server/ReplicationServerDomain.java @@ -1043,6 +1043,27 @@ else if (origServer.isDataServer()) } } + /** + * Returns whether this domain already holds a session with the replication + * server configured at the provided address. + * + * @param address + * the configured address of a replication server + * @return {@code true} if a connected replication server answers to that + * address, {@code false} otherwise + */ + public boolean isConnectedToServerAt(HostPort address) + { + for (ReplicationServerHandler rsHandler : connectedRSs.values()) + { + if (rsHandler.isServerAt(address)) + { + return true; + } + } + return false; + } + /** * Stop operations with a list of replication servers. * @@ -1054,10 +1075,13 @@ public void stopReplicationServers(Collection serversToDisconnect) { for (ReplicationServerHandler rsHandler : connectedRSs.values()) { - if (serversToDisconnect.contains( - HostPort.valueOf(rsHandler.getServerAddressURL()))) + for (HostPort serverToDisconnect : serversToDisconnect) { - stopServer(rsHandler, false); + if (rsHandler.isServerAt(serverToDisconnect)) + { + stopServer(rsHandler, false); + break; + } } } } @@ -1404,12 +1428,13 @@ public boolean isAlreadyConnectedToRS(ReplicationServerHandler rsHandler) return false; } - if (oldRsHandler.getServerAddressURL().equals( - rsHandler.getServerAddressURL())) + if (oldRsHandler.isSameServerAs(rsHandler)) { // this is the same server, this means that our ServerStart messages // have been sent at about the same time and 2 connections - // have been established. + // have been established -- or that this server reached the same peer at + // an address other than the one that peer connected from, which is what + // a multi homed or NATed peer gives. // Silently drop this connection. return true; } diff --git a/opendj-server-legacy/src/main/java/org/opends/server/replication/server/ReplicationServerHandler.java b/opendj-server-legacy/src/main/java/org/opends/server/replication/server/ReplicationServerHandler.java index 25c4dfb9a6..f39b45a810 100644 --- a/opendj-server-legacy/src/main/java/org/opends/server/replication/server/ReplicationServerHandler.java +++ b/opendj-server-legacy/src/main/java/org/opends/server/replication/server/ReplicationServerHandler.java @@ -21,6 +21,8 @@ import static org.opends.server.replication.protocol.ProtocolVersion.*; import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Set; @@ -54,6 +56,14 @@ public class ReplicationServerHandler extends ServerHandler /** Properties filled only if remote server is a RS. */ private String serverAddressURL; + /** + * The addresses the remote replication server is known by, resolved once, when its start + * message names it: the connect thread compares them on every one of its passes, and + * {@link HostPort} logs a name it cannot resolve each time it is built from one -- which + * the fall back of {@code ReplicationServer.setServerURL()} to the host name of the + * machine makes an ordinary thing for a peer to name. + */ + private List addresses = Collections.emptyList(); /** * This collection will contain as many elements as there are * LDAP servers connected to the remote replication server. @@ -78,7 +88,7 @@ private boolean processStartFromRemote( generationId = inReplServerStartMsg.getGenerationId(); serverId = inReplServerStartMsg.getServerId(); serverURL = inReplServerStartMsg.getServerURL(); - serverAddressURL = toServerAddressURL(serverURL); + setServerAddresses(serverURL); setBaseDNAndDomain(inReplServerStartMsg.getBaseDN(), false); setInitialServerState(inReplServerStartMsg.getServerState()); setSendWindowSize(inReplServerStartMsg.getWindowSize()); @@ -97,11 +107,20 @@ private boolean processStartFromRemote( return inReplServerStartMsg.getSSLEncryption(); } - private String toServerAddressURL(String serverURL) + /** + * Takes the addresses the remote replication server is known by from the URL its start + * message named: that URL, which is the address it is configured under, and the address + * of the connection this session is held on, which is the interface that connection + * happened to use rather than an identity. + */ + private void setServerAddresses(String serverURL) { - final int port = HostPort.valueOf(serverURL).getPort(); + final HostPort namedAddress = HostPort.valueOf(serverURL); // Ensure correct formatting of IPv6 addresses by using a HostPort instance. - return new HostPort(session.getRemoteAddress().getHost(), port).toString(); + final HostPort connectedAddress = + new HostPort(session.getRemoteAddress().getHost(), namedAddress.getPort()); + serverAddressURL = connectedAddress.toString(); + addresses = Arrays.asList(namedAddress, connectedAddress); } /** @@ -715,6 +734,60 @@ public String getServerAddressURL() return serverAddressURL; } + /** + * Returns whether the remote replication server of this handler is the one the provided + * handler holds a session with. + *

+ * Either of the two addresses a remote server is known by identifies it, and the one + * which does depends on where its connection came from: a server reachable at more than + * one address -- a multi homed host, or a NAT where the address a peer connects + * from is not the address it is configured as -- is registered under the + * address of whichever interface the session used, so two sessions with one such server + * carry two different addresses. What both of them do carry is the address that server + * names in its start messages, which is the address it is configured under. + * + * @param other + * the handler to compare the remote server of this one with + * @return {@code true} if both handlers hold a session with the same replication server + */ + boolean isSameServerAs(ReplicationServerHandler other) + { + for (HostPort address : other.addresses) + { + if (isServerAt(address)) + { + return true; + } + } + return false; + } + + /** + * Returns whether the remote replication server of this handler is the one configured at + * the provided address. + *

+ * Both addresses it is known by are compared, because either of them may be the + * configured one: the address the remote server names is the address it is configured + * under in its own configuration, which is the one the rest of the topology configures it + * at as well, while the address its session came from is the only one known of a server + * which names an address this configuration does not use. + * + * @param address + * a configured address of a replication server + * @return {@code true} if the remote server of this handler answers to that address + */ + boolean isServerAt(HostPort address) + { + for (HostPort known : addresses) + { + if (address.isEquivalentTo(known)) + { + return true; + } + } + return false; + } + /** * Receives a topology msg. * @param topoMsg The message received. diff --git a/opendj-server-legacy/src/test/java/org/opends/server/replication/server/MultiHomedPeerTest.java b/opendj-server-legacy/src/test/java/org/opends/server/replication/server/MultiHomedPeerTest.java new file mode 100644 index 0000000000..8ce559acba --- /dev/null +++ b/opendj-server-legacy/src/test/java/org/opends/server/replication/server/MultiHomedPeerTest.java @@ -0,0 +1,499 @@ +/* + * The contents of this file are subject to the terms of the Common Development and + * Distribution License (the License). You may not use this file except in compliance with the + * License. + * + * You can obtain a copy of the License at legal/CDDLv1.0.txt. See the License for the + * specific language governing permission and limitations under the License. + * + * When distributing Covered Software, include this CDDL Header Notice in each file and include + * the License file at legal/CDDLv1.0.txt. If applicable, add the following below the CDDL + * Header, with the fields enclosed by brackets [] replaced by your own identifying + * information: "Portions copyright [year] [name of copyright owner]". + * + * Copyright 2026 3A Systems, LLC. + */ +package org.opends.server.replication.server; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.opends.messages.ReplicationMessages.*; +import static org.opends.server.TestCaseUtils.*; +import static org.opends.server.util.CollectionUtils.newArrayList; +import static org.opends.server.util.CollectionUtils.newTreeSet; +import static org.opends.server.util.StaticUtils.close; + +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.ServerSocket; +import java.net.Socket; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; + +import org.forgerock.opendj.ldap.DN; +import org.opends.server.TestCaseUtils; +import org.opends.server.replication.ReplicationTestCase; +import org.opends.server.replication.common.DSInfo; +import org.opends.server.replication.common.RSInfo; +import org.opends.server.replication.common.ServerState; +import org.opends.server.replication.protocol.ReplServerStartMsg; +import org.opends.server.replication.protocol.ReplSessionSecurity; +import org.opends.server.replication.protocol.Session; +import org.opends.server.replication.protocol.TopologyMsg; +import org.opends.server.types.HostPort; +import org.testng.annotations.Test; + +/** + * Reproducer for issue #1017: a replication server which is reachable at more than one + * address -- a multi homed host, or a NAT where the address a peer connects from is + * not the address it is configured as -- used to be recognised by the address the + * socket of one of its sessions happened to carry. + *

+ * {@code ServerHandler.toServerAddressURL()} takes the host of a handler from + * {@code session.getRemoteAddress()} and its port from the start message that handler + * received, so the address a peer is registered under is an artefact of which interface its + * connection used, not an identity. Two things followed, and the tests below drive both: + * the already connected test of {@code runConnect()} compared a configured address against + * one which never matches it, so the peer was dialled again on every pass, about once a + * second, for as long as it stayed where it was; and the handshake offered to that same + * peer ran into a handler holding its server id under another address URL, which is + * {@code ERR_DUPLICATE_REPLICATION_SERVER_ID} -- an error logged with no throttle for a + * topology which is merely multi homed. + *

+ * The fixture is the mechanism itself rather than a stand in for it: the peer dials the + * server under test over the loopback interface and names another address in its start + * message, which is what a peer behind a NAT looks like to the server it dials, and the + * session the server dials that other address on reports the address it dialled, which is + * what reaching the same peer at its configured address gives. The addresses named are + * documentation addresses (RFC 5737), so nothing the tests do can leave the machine. + */ +@SuppressWarnings("javadoc") +public class MultiHomedPeerTest extends ReplicationTestCase +{ + private static final int SOCKET_TIMEOUT_MS = 30000; + /** How long the registration of the fake peer is waited for, in milliseconds. */ + private static final long REGISTRATION_TIMEOUT_MS = 30000; + + private static final int RS_ID = 8251; + private static final int PEER_RS_ID = 8252; + + /** TEST-NET-1: the address the peer is configured under and answers on. */ + private static final byte[] PEER_ADDRESS = { (byte) 192, 0, 2, 1 }; + /** TEST-NET-2: the address of a second, genuinely different server. */ + private static final byte[] OTHER_ADDRESS = { (byte) 198, 51, 100, 1 }; + + /** + * Tests that a peer which dialled this server from an address it is not configured under + * is still found by the address it is configured under. + *

+ * This is what stops the connect thread from dialling it on every pass: the address the + * handler is registered under is the loopback address the peer's own connection came + * from, and the only address which can match the configured one is the address the peer + * named in its start message. + */ + @Test + public void aPeerRegisteredUnderTheAddressItDialledFromIsFoundByItsConfiguredAddress() + throws Exception + { + TestCaseUtils.startServer(); + + final DN baseDN = DN.valueOf(TEST_ROOT_DN_STRING); + final int[] ports = TestCaseUtils.findFreePorts(2); + final HostPort peerAddress = documentationAddress(PEER_ADDRESS, ports[1]); + + ReplicationServer rs = null; + Session inbound = null; + try + { + rs = startServerWithPeerConfiguredAt(ports[0], "multiHomedPeerSkipDb", peerAddress); + final ReplicationServerDomain domain = rs.getReplicationServerDomain(baseDN, true); + inbound = registerPeerFrom(ports[0], peerAddress, baseDN); + final ReplicationServerHandler registered = waitForRegistration(domain); + + // The precondition of the case rather than an assumption: a peer registered under the + // address it is configured under is a peer this test says nothing about. + assertThat(registered.getServerAddressURL()) + .as("the peer should be registered under the loopback address it dialled from") + .isEqualTo(loopbackAt(ports[1]).toString()); + assertThat(registered.getServerURL()) + .as("the peer should name the address it is configured under in its start message") + .isEqualTo(peerAddress.toString()); + + assertThat(domain.isConnectedToServerAt(peerAddress)) + .as("the peer is connected, so its configured address must not be dialled again") + .isTrue(); + } + finally + { + close(inbound); + removeQuietly(rs); + } + } + + /** + * Tests that a second session with a peer already connected under another address is + * dropped as the duplicate connection it is, and not reported as two servers sharing a + * server id. + *

+ * The server under test dials the peer at its configured address while holding the + * session that same peer dialled it on. Both sessions are with one server, which names + * one address in both of its start messages; only the addresses the two sockets carry + * differ, which is what being reachable at more than one address means. + */ + @Test + public void aPeerReachedAtItsConfiguredAddressIsNotReportedAsADuplicateServerId() + throws Exception + { + TestCaseUtils.startServer(); + + final DN baseDN = DN.valueOf(TEST_ROOT_DN_STRING); + final int[] ports = TestCaseUtils.findFreePorts(2); + final HostPort peerAddress = documentationAddress(PEER_ADDRESS, ports[1]); + + ReplicationServer rs = null; + Session inbound = null; + try + { + rs = startServerWithPeerConfiguredAt(ports[0], "multiHomedPeerDuplicateDb", peerAddress); + final ReplicationServerDomain domain = rs.getReplicationServerDomain(baseDN, true); + inbound = registerPeerFrom(ports[0], peerAddress, baseDN); + final ReplicationServerHandler registered = waitForRegistration(domain); + + final List records = + errorLogRecordsOfHandshakeWith(rs, baseDN, peerAddress, peerAddress); + + assertThat(duplicateServerIdRecords(records, rs, loopbackAt(ports[1]), peerAddress)) + .as("a peer reachable at more than one address is not two servers sharing a" + + " server id, and every attempt to reach it would log this again") + .isEmpty(); + assertThat(domain.getConnectedRSs().get(PEER_RS_ID)) + .as("the session the peer dialled must outlive the duplicate connection") + .isSameAs(registered); + } + finally + { + close(inbound); + removeQuietly(rs); + } + } + + /** + * Tests that two genuinely different replication servers sharing a server id are still + * reported, which is the misconfiguration the address comparison is there to catch. + *

+ * Nothing is shared here: the connected peer names one address and dialled from the + * loopback interface, and the server answering the handshake names, and is reached at, + * another address altogether. The two are the same server only by their server id, which + * is exactly what the message says. + */ + @Test + public void twoServersSharingAServerIdAreStillReported() throws Exception + { + TestCaseUtils.startServer(); + + final DN baseDN = DN.valueOf(TEST_ROOT_DN_STRING); + final int[] ports = TestCaseUtils.findFreePorts(3); + final HostPort peerAddress = documentationAddress(PEER_ADDRESS, ports[1]); + final HostPort otherAddress = documentationAddress(OTHER_ADDRESS, ports[2]); + + ReplicationServer rs = null; + Session inbound = null; + try + { + rs = startServerWithPeerConfiguredAt(ports[0], "multiHomedPeerConflictDb", peerAddress); + final ReplicationServerDomain domain = rs.getReplicationServerDomain(baseDN, true); + inbound = registerPeerFrom(ports[0], peerAddress, baseDN); + waitForRegistration(domain); + + final List records = + errorLogRecordsOfHandshakeWith(rs, baseDN, otherAddress, otherAddress); + + assertThat(duplicateServerIdRecords(records, rs, loopbackAt(ports[1]), otherAddress)) + .as("two servers which share nothing but a server id must still be reported") + .isNotEmpty(); + } + finally + { + close(inbound); + removeQuietly(rs); + } + } + + /** + * Tests that a peer taken out of the configuration is disconnected even though the + * session it is connected on came from another address. + *

+ * {@code disconnectRemovedReplicationServers()} hands the addresses which were removed + * from {@code ds-cfg-replication-server} to this domain, and a handler which does not + * answer to any of them stays connected: the peer an administrator took out of the + * topology keeps replicating with this server until one of the two is restarted. + */ + @Test + public void aPeerRemovedFromTheConfigurationIsDisconnectedByItsConfiguredAddress() + throws Exception + { + TestCaseUtils.startServer(); + + final DN baseDN = DN.valueOf(TEST_ROOT_DN_STRING); + final int[] ports = TestCaseUtils.findFreePorts(2); + final HostPort peerAddress = documentationAddress(PEER_ADDRESS, ports[1]); + + ReplicationServer rs = null; + Session inbound = null; + try + { + rs = startServerWithPeerConfiguredAt(ports[0], "multiHomedPeerRemovedDb", peerAddress); + final ReplicationServerDomain domain = rs.getReplicationServerDomain(baseDN, true); + inbound = registerPeerFrom(ports[0], peerAddress, baseDN); + waitForRegistration(domain); + + domain.stopReplicationServers(Collections.singletonList(peerAddress)); + + assertThat(domain.getConnectedRSs().keySet()) + .as("the peer removed from the configuration must be disconnected") + .doesNotContain(PEER_RS_ID); + } + finally + { + close(inbound); + removeQuietly(rs); + } + } + + /** + * Starts a replication server whose only configured peer is at the provided address. + *

+ * Nothing ever answers there -- the address is a documentation address -- which is what + * the connect thread of the server does with it: it dials it, fails, and comes back to it + * on the next pass. Whether it dials it at all is what the first case is about. + */ + private ReplicationServer startServerWithPeerConfiguredAt(int port, String dbDirName, + HostPort peerAddress) throws Exception + { + return new ReplicationServer(new ReplServerFakeConfiguration( + port, dbDirName, 0, RS_ID, 0, 100, newTreeSet(peerAddress.toString()))); + } + + /** + * Has the fake peer dial the server under test over the loopback interface and complete + * the handshake, which registers it under the address that connection came from while its + * start message names the address it is configured under. + * + * @return the session the registration hangs on, closed by the caller + */ + private Session registerPeerFrom(int port, HostPort registeredAs, DN baseDN) throws Exception + { + final ReplSessionSecurity security = getReplSessionSecurity(); + final Socket socket = new Socket(); + Session session = null; + try + { + socket.setTcpNoDelay(true); + socket.connect(new InetSocketAddress("127.0.0.1", port), SOCKET_TIMEOUT_MS); + session = security.createClientSession(socket, SOCKET_TIMEOUT_MS); + session.publish(peerStartMsg(registeredAs, baseDN)); + session.receive(); + // The initiator of a session decides whether it is encrypted, and the start message + // above asked for it not to be: both ends leave the SSL session together, right after + // the start messages have been exchanged. + session.stopEncryption(); + /* + * The second phase: the server reads this one before it sends its own, and registers + * the handler once it has sent it. The list holds this peer and nothing else -- + * waitAndProcessTopoFromRemoteRS() reads rsInfos.get(0) above protocol version 4, so + * an empty one ends the handshake on an IndexOutOfBoundsException instead, which is + * an abort like any other and would leave the peer unregistered. + */ + final RSInfo peerInfo = new RSInfo(PEER_RS_ID, registeredAs.toString(), -1, (byte) 1, 1); + session.publish(new TopologyMsg(Collections. emptyList(), newArrayList(peerInfo))); + session.receive(); + return session; + } + catch (Exception e) + { + close(session); + close(socket); + throw e; + } + } + + /** + * Runs the handshake the server under test offers a peer it dials at the provided address + * and returns the error log records that handshake wrote. + *

+ * The session is dialled over the loopback interface and reports {@code dialledAt} as the + * address it reaches the peer at, which is what dialling a peer at its configured address + * gives whatever interface the peer's own connection to this server used. The answer is + * a well formed start message naming {@code answersAs}: what ends the handshake is the + * handler this server already holds for that server id, not what the peer answers here. + * + * @param listenPort + * the port the fake peer answers the handshake on + * @param dialledAt + * the address the session reports the peer at + * @param answersAs + * the address the peer names in the start message it answers with + */ + private List errorLogRecordsOfHandshakeWith(final ReplicationServer rs, + final DN baseDN, final HostPort dialledAt, final HostPort answersAs) + throws Exception + { + final ExecutorService peerThread = Executors.newSingleThreadExecutor(); + try (ServerSocket peerListen = TestCaseUtils.bindFreePort()) + { + peerListen.setSoTimeout(SOCKET_TIMEOUT_MS); + final ReplSessionSecurity security = getReplSessionSecurity(); + final int listenPort = peerListen.getLocalPort(); + + final Future dialled = peerThread.submit(new Callable() + { + @Override + public Session call() throws Exception + { + return dialPeerAt(security, dialledAt, listenPort); + } + }); + + try (Session peerEnd = + security.createServerSession(peerListen.accept(), SOCKET_TIMEOUT_MS); + Session session = dialled.get(SOCKET_TIMEOUT_MS, TimeUnit.MILLISECONDS)) + { + final Future answer = peerThread.submit(new Callable() + { + @Override + public Void call() throws Exception + { + peerEnd.receive(); + peerEnd.publish(peerStartMsg(answersAs, baseDN)); + return null; + } + }); + + TestCaseUtils.ERROR_TEXT_WRITER.clear(); + new ReplicationServerHandler(session, 100, rs, 100).connect(baseDN, false); + answer.get(SOCKET_TIMEOUT_MS, TimeUnit.MILLISECONDS); + return TestCaseUtils.ERROR_TEXT_WRITER.getMessages(); + } + } + finally + { + peerThread.shutdownNow(); + } + } + + /** + * Connects to the provided local port with a socket which reports the provided address as + * the one it is connected to, as dialling a peer at that address does. + */ + private Session dialPeerAt(ReplSessionSecurity security, HostPort dialledAt, int port) + throws Exception + { + // Named, so that the SSL socket factory asking the socket for its host name does not + // send a reverse lookup of an address no name server knows anything about. + final InetAddress reported = InetAddress.getByAddress("peer.example.com", + InetAddress.getByName(dialledAt.getHost()).getAddress()); + final Socket socket = new Socket() + { + @Override + public InetAddress getInetAddress() + { + return reported; + } + }; + try + { + socket.setTcpNoDelay(true); + socket.connect(new InetSocketAddress("127.0.0.1", port), SOCKET_TIMEOUT_MS); + return security.createClientSession(socket, SOCKET_TIMEOUT_MS); + } + catch (Exception e) + { + close(socket); + throw e; + } + } + + /** Returns the start message of the fake peer, naming the provided address. */ + private ReplServerStartMsg peerStartMsg(HostPort address, DN baseDN) + { + return new ReplServerStartMsg(PEER_RS_ID, address.toString(), baseDN, 100, + new ServerState(), -1, false, (byte) 1, 5000); + } + + /** Returns an address of the documentation ranges, which nothing routes. */ + private HostPort documentationAddress(byte[] address, int port) throws Exception + { + return new HostPort(InetAddress.getByAddress(address).getHostAddress(), port); + } + + /** Returns the loopback address a connection of these tests comes from. */ + private HostPort loopbackAt(int port) + { + return new HostPort("127.0.0.1", port); + } + + /** + * Waits for the domain to hold the handler of the fake peer, which the server registers + * after it has sent the topology message the handshake above reads: the registration + * lands just behind the thread which drove it. + */ + private ReplicationServerHandler waitForRegistration(ReplicationServerDomain domain) + throws Exception + { + final long deadline = System.currentTimeMillis() + REGISTRATION_TIMEOUT_MS; + ReplicationServerHandler registered; + while (true) + { + registered = domain.getConnectedRSs().get(PEER_RS_ID); + if (registered != null || System.currentTimeMillis() > deadline) + { + break; + } + Thread.sleep(50); + } + assertThat(registered).as("the fake peer should have registered with the domain").isNotNull(); + return registered; + } + + /** + * Returns the records of the provided log which report the two provided address URLs as + * two replication servers sharing a server id. + *

+ * The addresses are the ones the message names, so a record of the handshake next door + * cannot be read as one of the handshake under test. + */ + private List duplicateServerIdRecords(List records, + ReplicationServer rs, HostPort connectedAs, HostPort reachedAt) + { + final String message = ERR_DUPLICATE_REPLICATION_SERVER_ID.get( + rs.getMonitorInstanceName(), connectedAs, reachedAt, PEER_RS_ID).toString(); + final List results = newArrayList(); + for (String record : records) + { + if (record.contains(message)) + { + results.add(record); + } + } + return results; + } + + /** Teardown must never mask the primary assertion failure. */ + private void removeQuietly(ReplicationServer replicationServer) + { + try + { + if (replicationServer != null) + { + remove(replicationServer); + } + } + catch (Exception ignored) + { + } + } +} From 6629bc9410f4e584284ba45785857a00da0d4623 Mon Sep 17 00:00:00 2001 From: Valera V Harseko Date: Fri, 11 Sep 2026 21:50:55 +0300 Subject: [PATCH 2/4] [#1017] Say in connect() what a peer this server never sees connected now costs The comment #935 left in ReplicationServer.connect() justified closing an outage on the connection alone with the multi homed peer runConnect() could never match, and described that peer by what the old comparison did with it: registered under the source address of its session, never matched, its handshake aborted on a duplicate server id. Both addresses now identify it, so what is left of that reading is the peer which names an address this configuration does not use, and its handshake is resolved as a cross connect rather than reported as a duplicate. The last paragraph said the pass after a cross connect says what is true, which holds for the peer runConnect() can match and not for this one. --- .../replication/server/ReplicationServer.java | 30 +++++++++++-------- 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/opendj-server-legacy/src/main/java/org/opends/server/replication/server/ReplicationServer.java b/opendj-server-legacy/src/main/java/org/opends/server/replication/server/ReplicationServer.java index c2ecc4f5e7..935302576d 100644 --- a/opendj-server-legacy/src/main/java/org/opends/server/replication/server/ReplicationServer.java +++ b/opendj-server-legacy/src/main/java/org/opends/server/replication/server/ReplicationServer.java @@ -755,13 +755,15 @@ boolean connect(HostPort remoteServerAddress, DN baseDN) * ReplicationServerHandler.connect() registers only above V1, the FIXME there being * older than this, so such a peer is connected and never registered. * - * the address, and the session left open with it, miss a peer which dials out from an - * address other than the one it is configured under -- multi homing, NAT. Its inbound - * handler is registered under the source address of its own connection, - * ServerHandler.toServerAddressURL() reading the host from the session, so the already - * connected branch of runConnect() compares the configured address against one it never - * matches, and the handshake this server offers that same peer aborts on a duplicate - * server id: abortStart() closes the session, and an open session is never seen here + * the address, and the session left open with it, miss a peer which names an address + * this configuration does not use -- the host name of its machine, which setServerURL() + * falls back to when none of the addresses it is configured with is local to it -- and + * dials out from another one this configuration does not use either: multi homing, + * NAT. Its inbound handler is known by the address it named and by the source address + * of its own connection, and the already connected branch of runConnect() finds the + * configured address under neither, so this server offers that same peer a handshake + * which either end resolves as a cross connect, on the address both handlers of the + * peer name: abortStart() closes the session, and an open session is never seen here * again. * * An outage left open is not a line too few but a peer gone silent: recordFailure() @@ -783,11 +785,15 @@ boolean connect(HostPort remoteServerAddress, DN baseDN) * with the same silent abortStart(null), one line up in startFromRemoteRS(). * * The cross connect this server resolves is the one abort of the three where a session - * for the domain does exist: it is the connection the peer made, which the already - * connected branch of runConnect() reports on its next pass. Reaching this line with - * one open needs that registration to land between the snapshot that branch reads and - * the dial below it, so what it costs is one warning, and the pass after it says what - * is true. + * for the domain does exist: it is the connection the peer made. For a peer which names + * an address this configuration uses, that is the session the already connected branch + * of runConnect() reports on its next pass; reaching this line with it open needs the + * registration to land between the check that branch makes and the dial below it, so + * what it costs is one warning, and the pass after it says what is true. For the peer + * of the second reading above no pass matches it, so what was last said of it stays + * said -- nothing where no outage was reported, the warning where one was -- and the + * blacklist runConnect() keeps for an answer without a session is what bounds the + * dialling behind it. */ reportConnectionRestored(remoteServerAddress, baseDN, handshakeCompleted); return handshakeCompleted; From 28333a1da164f59ddf274922fb8c93f950e901eb Mon Sep 17 00:00:00 2001 From: Valera V Harseko Date: Mon, 21 Sep 2026 23:10:02 +0300 Subject: [PATCH 3/4] [#1017] Recognise a peer by the name neither end can resolve, and pin each call site The comparison a remote replication server is recognised by answers false for a host name this server cannot resolve: InetAddress.getAllByName() throws for it and HostPort.isEquivalentTo() reads that as "not the same server", even against the same name. So the peer of the rewritten connect() comment -- the one which falls back to naming the host name of its machine, which the rest of the topology has no reason to resolve -- was still reported as two servers sharing a server id, once per dial. It is now compared as the name it is, which is what HostPort.equals() does with it. What that gives up is two servers which share a server id and both name one unresolvable name on one port: they are read as one, as two which name the same resolvable address already were. Each of the three call sites is now pinned by a case of its own: * runConnect(): the first case records an outage for the configured address and asserts NOTE_REPLICATION_SERVER_CONNECT_RESTORED, which the already connected branch writes and nothing else does -- a second dial writes nothing at all, ConnectFailureReporter recording a peer as DOWN once and connect() having no debug branch under it; * the host half of isServerAt(): the second server of the negative case moved to the port of the first, so the host is what tells the two apart; * the connected arm of the addresses: a case whose peer names an address this configuration does not use and is found by the one it dialled from. Also the javadoc: "built once" rather than "resolved once", and what the comparison resolves on every pass; what the connect thread does with a documentation address, which is a SYN to the default route rather than nothing leaving the machine; the fixture of ReplicationServerConnectFailureTest, which is the peer whose two addresses the configured one matches neither of rather than the multi homed peer; the name of ReplicationServerHandler.toServerAddressURL(); and what trusting the address a peer names costs, in ReplicationServerDomain.isConnectedToServerAt(). --- .../server/ReplicationServerDomain.java | 9 + .../server/ReplicationServerHandler.java | 18 +- .../server/MultiHomedPeerTest.java | 196 +++++++++++++++++- .../ReplicationServerConnectFailureTest.java | 32 +-- 4 files changed, 229 insertions(+), 26 deletions(-) diff --git a/opendj-server-legacy/src/main/java/org/opends/server/replication/server/ReplicationServerDomain.java b/opendj-server-legacy/src/main/java/org/opends/server/replication/server/ReplicationServerDomain.java index 67886770af..2103c0694f 100644 --- a/opendj-server-legacy/src/main/java/org/opends/server/replication/server/ReplicationServerDomain.java +++ b/opendj-server-legacy/src/main/java/org/opends/server/replication/server/ReplicationServerDomain.java @@ -1046,6 +1046,15 @@ else if (origServer.isDataServer()) /** * Returns whether this domain already holds a session with the replication * server configured at the provided address. + *

+ * A connected server answers to the address its session came from and to the address it + * named in its start message, and the second of the two is what that server says of + * itself: a peer which names an address this configuration gives to another replication + * server hides that one from the connect thread, which then dials nothing for it and + * reports it connected while it is down. A replication server names itself from its own + * configuration ({@code ReplicationServer.setServerURL()}), so that takes two of them + * configured at one address -- a configuration copied whole, or two sites whose private + * ranges overlap -- and the peer it hides is still the one which dials this server. * * @param address * the configured address of a replication server diff --git a/opendj-server-legacy/src/main/java/org/opends/server/replication/server/ReplicationServerHandler.java b/opendj-server-legacy/src/main/java/org/opends/server/replication/server/ReplicationServerHandler.java index f39b45a810..c503ecac2c 100644 --- a/opendj-server-legacy/src/main/java/org/opends/server/replication/server/ReplicationServerHandler.java +++ b/opendj-server-legacy/src/main/java/org/opends/server/replication/server/ReplicationServerHandler.java @@ -57,11 +57,13 @@ public class ReplicationServerHandler extends ServerHandler /** Properties filled only if remote server is a RS. */ private String serverAddressURL; /** - * The addresses the remote replication server is known by, resolved once, when its start + * The addresses the remote replication server is known by, built once, when its start * message names it: the connect thread compares them on every one of its passes, and * {@link HostPort} logs a name it cannot resolve each time it is built from one -- which * the fall back of {@code ReplicationServer.setServerURL()} to the host name of the - * machine makes an ordinary thing for a peer to name. + * machine makes an ordinary thing for a peer to name. What the comparison of two of them + * resolves is reported nowhere above trace, so building them once is what keeps that name + * out of the error log rather than what keeps it out of the resolver. */ private List addresses = Collections.emptyList(); /** @@ -771,6 +773,16 @@ boolean isSameServerAs(ReplicationServerHandler other) * under in its own configuration, which is the one the rest of the topology configures it * at as well, while the address its session came from is the only one known of a server * which names an address this configuration does not use. + *

+ * A name neither end can resolve is compared as the name it is, which is what + * {@link HostPort#equals(Object)} does with it: {@link HostPort#isEquivalentTo(HostPort)} + * resolves both hosts and answers {@code false} for a name it cannot resolve, even + * against that same name. The peer which names one is the peer of the fall back of + * {@code ReplicationServer.setServerURL()}, whose own host name the rest of the topology + * has no reason to resolve, and it is the peer the addresses are there for. What that + * gives up is two servers which share a server id and both name one unresolvable name on + * one port: they are read as one, as two servers which name the same resolvable address + * already are. * * @param address * a configured address of a replication server @@ -780,7 +792,7 @@ boolean isServerAt(HostPort address) { for (HostPort known : addresses) { - if (address.isEquivalentTo(known)) + if (address.equals(known) || address.isEquivalentTo(known)) { return true; } diff --git a/opendj-server-legacy/src/test/java/org/opends/server/replication/server/MultiHomedPeerTest.java b/opendj-server-legacy/src/test/java/org/opends/server/replication/server/MultiHomedPeerTest.java index 8ce559acba..41593d0070 100644 --- a/opendj-server-legacy/src/test/java/org/opends/server/replication/server/MultiHomedPeerTest.java +++ b/opendj-server-legacy/src/test/java/org/opends/server/replication/server/MultiHomedPeerTest.java @@ -53,7 +53,7 @@ * not the address it is configured as -- used to be recognised by the address the * socket of one of its sessions happened to carry. *

- * {@code ServerHandler.toServerAddressURL()} takes the host of a handler from + * {@code ReplicationServerHandler.toServerAddressURL()} takes the host of a handler from * {@code session.getRemoteAddress()} and its port from the start message that handler * received, so the address a peer is registered under is an artefact of which interface its * connection used, not an identity. Two things followed, and the tests below drive both: @@ -69,7 +69,9 @@ * message, which is what a peer behind a NAT looks like to the server it dials, and the * session the server dials that other address on reports the address it dialled, which is * what reaching the same peer at its configured address gives. The addresses named are - * documentation addresses (RFC 5737), so nothing the tests do can leave the machine. + * documentation addresses (RFC 5737): nothing answers at them, which is what the connect + * thread of the server under test finds when it dials one -- a host stack sends that SYN to + * its default route, and the connect fails, once per pass until the peer has registered. */ @SuppressWarnings("javadoc") public class MultiHomedPeerTest extends ReplicationTestCase @@ -85,6 +87,12 @@ public class MultiHomedPeerTest extends ReplicationTestCase private static final byte[] PEER_ADDRESS = { (byte) 192, 0, 2, 1 }; /** TEST-NET-2: the address of a second, genuinely different server. */ private static final byte[] OTHER_ADDRESS = { (byte) 198, 51, 100, 1 }; + /** + * A name under the .invalid top level domain (RFC 6761), which resolvers answer does not + * exist: the host name of a peer which this server has no way to resolve, and what + * {@code ReplicationServer.setServerURL()} falls back to naming. + */ + private static final String UNRESOLVABLE_HOST = "nonexistent.invalid"; /** * Tests that a peer which dialled this server from an address it is not configured under @@ -94,6 +102,10 @@ public class MultiHomedPeerTest extends ReplicationTestCase * handler is registered under is the loopback address the peer's own connection came * from, and the only address which can match the configured one is the address the peer * named in its start message. + *

+ * Both halves are asserted, because the predicate and the call site which reads it fail + * apart: what the connect thread does with a peer it finds connected is reported by the + * outage it closes, and an outage is recorded here for it to close. */ @Test public void aPeerRegisteredUnderTheAddressItDialledFromIsFoundByItsConfiguredAddress() @@ -111,6 +123,17 @@ public void aPeerRegisteredUnderTheAddressItDialledFromIsFoundByItsConfiguredAdd { rs = startServerWithPeerConfiguredAt(ports[0], "multiHomedPeerSkipDb", peerAddress); final ReplicationServerDomain domain = rs.getReplicationServerDomain(baseDN, true); + + /* + * An outage for the configured address, which nothing but the already connected + * branch of runConnect() closes: a peer that branch skips is dialled by no one, so a + * connection is reported for it nowhere else. Recorded from this thread rather than + * waited for, because whether the connect thread has dialled that address before the + * peer registers is a race and what is asserted below is not. + */ + assertThat(rs.connect(peerAddress, baseDN)) + .as("nothing answers at " + peerAddress).isFalse(); + inbound = registerPeerFrom(ports[0], peerAddress, baseDN); final ReplicationServerHandler registered = waitForRegistration(domain); @@ -126,6 +149,10 @@ public void aPeerRegisteredUnderTheAddressItDialledFromIsFoundByItsConfiguredAdd assertThat(domain.isConnectedToServerAt(peerAddress)) .as("the peer is connected, so its configured address must not be dialled again") .isTrue(); + assertThat(waitForConnectRestored(rs, peerAddress, baseDN)) + .as("the connect thread must find the peer at its configured address and close the" + + " outage reported for it, rather than dial it again") + .isTrue(); } finally { @@ -181,14 +208,133 @@ public void aPeerReachedAtItsConfiguredAddressIsNotReportedAsADuplicateServerId( } } + /** + * Tests that a peer which names an address this configuration does not use is found by + * the address its own connection came from, which is the only one known of it. + *

+ * The peer of the first case names the address it is configured under, and is found by + * it. This one is the other way round: it is configured at the address it dials this + * server from, and names one this topology does not configure it at -- the host name of + * its machine under another of its addresses, which {@code setServerURL()} falls back to. + * The address it names matches nothing here, so the address its session came from is what + * has to match. + */ + @Test + public void aPeerWhichNamesAnAddressThisConfigurationDoesNotUseIsFoundByTheOneItDialledFrom() + throws Exception + { + TestCaseUtils.startServer(); + + final DN baseDN = DN.valueOf(TEST_ROOT_DN_STRING); + final int[] ports = TestCaseUtils.findFreePorts(2); + final HostPort peerAddress = loopbackAt(ports[1]); + final HostPort namedAddress = documentationAddress(OTHER_ADDRESS, ports[1]); + + ReplicationServer rs = null; + Session inbound = null; + try + { + rs = startServerWithPeerConfiguredAt(ports[0], "multiHomedPeerNamedDb", peerAddress); + final ReplicationServerDomain domain = rs.getReplicationServerDomain(baseDN, true); + inbound = registerPeerFrom(ports[0], namedAddress, baseDN); + final ReplicationServerHandler registered = waitForRegistration(domain); + + // The precondition of the case rather than an assumption, as in the first case: a + // peer which names the address it is configured under is the peer of that one. + assertThat(registered.getServerURL()) + .as("the peer should name the address this configuration does not use") + .isEqualTo(namedAddress.toString()); + assertThat(registered.getServerAddressURL()) + .as("the peer should be registered under the loopback address it dialled from") + .isEqualTo(peerAddress.toString()); + + assertThat(domain.isConnectedToServerAt(peerAddress)) + .as("the address the connection of such a peer came from is the only one which" + + " can match the address it is configured at") + .isTrue(); + } + finally + { + close(inbound); + removeQuietly(rs); + } + } + + /** + * Tests that a peer which names a host this server cannot resolve is recognised by that + * name all the same, rather than reported as two servers sharing a server id. + *

+ * {@code setServerURL()} falls back to the host name of the machine when none of the + * addresses a peer is configured with is local to it, and the rest of the topology has no + * reason to resolve that name. Both handlers of such a peer name it, so it is what tells + * this server they are one -- but a comparison which resolves both sides answers that a + * name it cannot resolve is equivalent to nothing at all, not even to itself, and the + * handshake this server offers such a peer would abort on a duplicate server id on every + * pass which reaches it. + */ + @Test + public void aPeerWhichNamesAnUnresolvableHostIsNotReportedAsADuplicateServerId() + throws Exception + { + TestCaseUtils.startServer(); + + final DN baseDN = DN.valueOf(TEST_ROOT_DN_STRING); + final int[] ports = TestCaseUtils.findFreePorts(2); + final HostPort peerAddress = documentationAddress(PEER_ADDRESS, ports[1]); + final HostPort namedAddress = new HostPort(UNRESOLVABLE_HOST, ports[1]); + + ReplicationServer rs = null; + Session inbound = null; + try + { + rs = startServerWithPeerConfiguredAt(ports[0], "multiHomedPeerUnresolvedDb", peerAddress); + final ReplicationServerDomain domain = rs.getReplicationServerDomain(baseDN, true); + inbound = registerPeerFrom(ports[0], namedAddress, baseDN); + final ReplicationServerHandler registered = waitForRegistration(domain); + + /* + * What the addresses are built from is read once, when the start message names them: + * a name which cannot be resolved is reported by HostPort each time one is built from + * it, and the connect thread compares them on every pass. The comparison below is the + * one that thread makes. + */ + TestCaseUtils.ERROR_TEXT_WRITER.clear(); + for (int pass = 0; pass < 3; pass++) + { + domain.isConnectedToServerAt(peerAddress); + } + assertThat(recordsContaining(TestCaseUtils.ERROR_TEXT_WRITER.getMessages(), + ERR_COULD_NOT_SOLVE_HOSTNAME.get(UNRESOLVABLE_HOST).toString())) + .as("comparing the addresses of a handler must not build them again") + .isEmpty(); + + final List records = + errorLogRecordsOfHandshakeWith(rs, baseDN, peerAddress, namedAddress); + + assertThat(duplicateServerIdRecords(records, rs, loopbackAt(ports[1]), peerAddress)) + .as("a peer whose name this server cannot resolve is one server all the same, and" + + " every attempt to reach it would log this again") + .isEmpty(); + assertThat(domain.getConnectedRSs().get(PEER_RS_ID)) + .as("the session the peer dialled must outlive the duplicate connection") + .isSameAs(registered); + } + finally + { + close(inbound); + removeQuietly(rs); + } + } + /** * Tests that two genuinely different replication servers sharing a server id are still * reported, which is the misconfiguration the address comparison is there to catch. *

* Nothing is shared here: the connected peer names one address and dialled from the * loopback interface, and the server answering the handshake names, and is reached at, - * another address altogether. The two are the same server only by their server id, which - * is exactly what the message says. + * another address on the same port. The port is the same one on purpose: it is the host + * which tells the two servers apart, and a second port would answer the comparison before + * any host of it is read. */ @Test public void twoServersSharingAServerIdAreStillReported() throws Exception @@ -196,9 +342,9 @@ public void twoServersSharingAServerIdAreStillReported() throws Exception TestCaseUtils.startServer(); final DN baseDN = DN.valueOf(TEST_ROOT_DN_STRING); - final int[] ports = TestCaseUtils.findFreePorts(3); + final int[] ports = TestCaseUtils.findFreePorts(2); final HostPort peerAddress = documentationAddress(PEER_ADDRESS, ports[1]); - final HostPort otherAddress = documentationAddress(OTHER_ADDRESS, ports[2]); + final HostPort otherAddress = documentationAddress(OTHER_ADDRESS, ports[1]); ReplicationServer rs = null; Session inbound = null; @@ -459,6 +605,35 @@ private ReplicationServerHandler waitForRegistration(ReplicationServerDomain dom return registered; } + /** + * Waits for the connect thread to report the peer at the provided address connected, + * driving a pass rather than waiting one out. + *

+ * That report is what the already connected branch of {@code runConnect()} does with a + * peer it finds registered, and the only thing this server does with one: the branch + * which dials a peer reports a connection only for a handshake it completed, which the + * documentation address of these tests never gives. + */ + private boolean waitForConnectRestored(ReplicationServer rs, HostPort peer, DN baseDN) + throws Exception + { + final String message = + NOTE_REPLICATION_SERVER_CONNECT_RESTORED.get(RS_ID, peer, baseDN).toString(); + final long deadline = System.currentTimeMillis() + REGISTRATION_TIMEOUT_MS; + while (true) + { + if (!recordsContaining(TestCaseUtils.ERROR_TEXT_WRITER.getMessages(), message).isEmpty()) + { + return true; + } + if (System.currentTimeMillis() > deadline) + { + return false; + } + rs.waitConnections(); + } + } + /** * Returns the records of the provided log which report the two provided address URLs as * two replication servers sharing a server id. @@ -469,8 +644,13 @@ private ReplicationServerHandler waitForRegistration(ReplicationServerDomain dom private List duplicateServerIdRecords(List records, ReplicationServer rs, HostPort connectedAs, HostPort reachedAt) { - final String message = ERR_DUPLICATE_REPLICATION_SERVER_ID.get( - rs.getMonitorInstanceName(), connectedAs, reachedAt, PEER_RS_ID).toString(); + return recordsContaining(records, ERR_DUPLICATE_REPLICATION_SERVER_ID.get( + rs.getMonitorInstanceName(), connectedAs, reachedAt, PEER_RS_ID).toString()); + } + + /** Returns the records of the provided log which carry the provided message. */ + private List recordsContaining(List records, String message) + { final List results = newArrayList(); for (String record : records) { diff --git a/opendj-server-legacy/src/test/java/org/opends/server/replication/server/ReplicationServerConnectFailureTest.java b/opendj-server-legacy/src/test/java/org/opends/server/replication/server/ReplicationServerConnectFailureTest.java index 3c43b5c6c0..103da8906d 100644 --- a/opendj-server-legacy/src/test/java/org/opends/server/replication/server/ReplicationServerConnectFailureTest.java +++ b/opendj-server-legacy/src/test/java/org/opends/server/replication/server/ReplicationServerConnectFailureTest.java @@ -159,7 +159,7 @@ public void aPeerWhichIsDownIsReportedOnceAndItsReturnIsReported() throws Except * {@code WARN_REPLICATION_SERVER_CONNECT_ERROR} is reported for the socket and for the * session built on it, the handshake throwing nothing of its own. Holding the outage open * across an abort silences the peer this server never sees connected under the address it - * dialled -- the multi homed peer of + * dialled -- the peer registered under an address this configuration does not use, of * {@link #aPeerRegisteredUnderAnotherAddressStillClosesItsOutage}, and one protocol * version down a peer which negotiates V1, connected and never registered. *

@@ -313,17 +313,18 @@ public void aSessionEstablishedAfterAnAnswerWithoutOneIsReported() throws Except * Tests that a peer already registered under an address other than the one it is * configured under still closes the outage reported for it. *

- * This is the multi homed peer, and the reason the recovery can be read neither from the - * address nor from the session. {@code ServerHandler.toServerAddressURL()} takes the host - * of a handler from {@code session.getRemoteAddress()} and its port from the start message - * that handler received, so a peer which dials this server from an address it is not - * configured under is registered under that other address. Two things follow, and this - * test drives both: the already connected branch of {@code runConnect()} compares the - * configured address against one which never matches it, so it can close nothing; and the - * handshake this server offers that same peer runs into a handler holding its server id - * under another address URL, which is {@code ERR_DUPLICATE_REPLICATION_SERVER_ID}, an - * abort of this server rather than of the peer, and a session closed at the end of - * {@code connect()} for as long as the peer stays where it is. + * This is the peer {@code runConnect()} matches by none of the addresses its handler is + * known by, and the reason the recovery can be read neither from the address nor from the + * session. A handler is known by the address its start message names and by the address + * its session came from ({@code ReplicationServerHandler.setServerAddresses()}), and the + * peer below names a port this configuration does not use, which both of those carry. + * Two things follow, and this test drives both: the already connected branch of + * {@code runConnect()} compares the configured address against two which never match it, + * so it can close nothing; and the handshake this server offers that same peer runs into + * a handler holding its server id at another address, which is + * {@code ERR_DUPLICATE_REPLICATION_SERVER_ID}, an abort of this server rather than of the + * peer, and a session closed at the end of {@code connect()} for as long as the peer + * stays where it is. *

* Gating the recovery on either leaves the record of such a peer uncleared for good, and * {@code recordFailure()} returns false from then on: the next real outage of it -- the @@ -337,9 +338,10 @@ public void aPeerRegisteredUnderAnotherAddressStillClosesItsOutage() throws Exce final DN baseDN = DN.valueOf(TEST_ROOT_DN_STRING); /* * Three ports: the server under test, the address the peer is configured under and - * answers on, and the address it registers itself under. The last is never bound -- - * what a multi homed peer costs is that the two addresses are not compared equal, and - * a port nothing listens on is that, without a second address to bind. + * answers on, and the address it names and is therefore registered under. The last is + * never bound -- what such a peer costs is that the configured address matches neither + * of the two addresses its handler is known by, and a port nothing listens on is that, + * without a second address to bind. */ final int[] ports = TestCaseUtils.findFreePorts(3); final HostPort peerAddress = HostPort.valueOf("127.0.0.1:" + ports[1]); From cde2f9690c7c05fe2e341e3dc99df6dad83fda73 Mon Sep 17 00:00:00 2001 From: Valera V Harseko Date: Wed, 23 Sep 2026 12:30:18 +0300 Subject: [PATCH 4/4] [#1017] Keep the entry of this server out of the disconnect list, and pin the arms no case read HostPort normalises every address local to this machine to localhost, so the entry of this server and the loopback address a peer names for itself are one address on one port: a peer whose own configuration lists localhost:P for itself names exactly that in its start messages, and this server reads that name as its own entry. disconnectRemovedReplicationServers() handed that entry to the domain like any other, so removing it -- a configuration which does not list this server is supported -- stopped the session of a healthy peer, which then had to dial again. At the base the comparison read the connected address, the real address of the peer, which never normalises that way. The entry of this server is now skipped there, as runConnect() skips it before it dials. Three roads the cases reached without reading are pinned: * removingTheEntryOfThisServerStopsNoPeer registers two peers, one naming the address this server listens on and one naming its own entry, and removes both entries in a single applyConfigurationChange(): the guard alone tells the two apart, so the case is red without it and the peer whose own entry went is the positive control of the road. * aSecondSessionFromTheAddressAPeerIsConnectedOnIsReadAsThatPeer answers a handshake from the address the registered peer is connected on while naming another address, which only the connected arm of isSameServerAs() can match. That arm carries what this change gives up -- two servers behind one gateway read as one -- and no case stated it. * The unresolvable case now asserts the ERR_COULD_NOT_SOLVE_HOSTNAME record its premise rests on before it clears the log. A resolver which answers a name which does not exist would have isEquivalentTo() carry that case on its own, leaving the equals() arm unread and the count of that record vacuous, with nothing red to say so. The javadoc says three things it did not: isServerAt() resolves under the domain lock on the handshake road, whose cost is the domain rather than one comparison; isConnectedToServerAt() is hidden from by any named address which resolves here to another replication server, the fall back host name of setServerURL() included, not only by two servers configured at one address; and the pair the equals() arm reads as one is read as one under the other arm too, which is this change rather than the base. In the test, toServerAddressURL() is named in the past tense, and a dial of an address nothing answers at is one per six passes, the blacklist of #935 holding the five in between. --- .../replication/server/ReplicationServer.java | 11 +- .../server/ReplicationServerDomain.java | 10 +- .../server/ReplicationServerHandler.java | 10 +- .../server/MultiHomedPeerTest.java | 226 +++++++++++++++++- 4 files changed, 239 insertions(+), 18 deletions(-) diff --git a/opendj-server-legacy/src/main/java/org/opends/server/replication/server/ReplicationServer.java b/opendj-server-legacy/src/main/java/org/opends/server/replication/server/ReplicationServer.java index 935302576d..cc4594912d 100644 --- a/opendj-server-legacy/src/main/java/org/opends/server/replication/server/ReplicationServer.java +++ b/opendj-server-legacy/src/main/java/org/opends/server/replication/server/ReplicationServer.java @@ -2034,10 +2034,19 @@ private void disconnectRemovedReplicationServers(Set oldRSAddresses) { final Collection serversToDisconnect = new ArrayList<>(); + /* + * The entry of this server is never one to disconnect, and is skipped here as + * runConnect() skips it before it dials: this server holds no session with itself, and + * a peer which names a loopback address of its own machine on this port does answer to + * that entry, because HostPort folds every address local to this machine to localhost. + * Removing the entry of this server -- a list without it is supported, see + * runConnect() -- would otherwise stop the session of such a peer once. + */ + final HostPort localAddress = HostPort.localAddress(getReplicationPort()); final Set newRSAddresses = getConfiguredRSAddresses(); for (HostPort oldRSAddress : oldRSAddresses) { - if (!newRSAddresses.contains(oldRSAddress)) + if (!newRSAddresses.contains(oldRSAddress) && !oldRSAddress.equals(localAddress)) { serversToDisconnect.add(oldRSAddress); } diff --git a/opendj-server-legacy/src/main/java/org/opends/server/replication/server/ReplicationServerDomain.java b/opendj-server-legacy/src/main/java/org/opends/server/replication/server/ReplicationServerDomain.java index 2103c0694f..6cccf85439 100644 --- a/opendj-server-legacy/src/main/java/org/opends/server/replication/server/ReplicationServerDomain.java +++ b/opendj-server-legacy/src/main/java/org/opends/server/replication/server/ReplicationServerDomain.java @@ -1052,9 +1052,13 @@ else if (origServer.isDataServer()) * itself: a peer which names an address this configuration gives to another replication * server hides that one from the connect thread, which then dials nothing for it and * reports it connected while it is down. A replication server names itself from its own - * configuration ({@code ReplicationServer.setServerURL()}), so that takes two of them - * configured at one address -- a configuration copied whole, or two sites whose private - * ranges overlap -- and the peer it hides is still the one which dials this server. + * configuration, or, when none of the addresses it is configured with is local to it, + * from the host name of its machine ({@code ReplicationServer.setServerURL()}), so what + * it takes is a named address which resolves here to one this configuration gives to + * another replication server: two of them configured at one address -- a configuration + * copied whole, or two sites whose private ranges overlap -- or that fall back host name + * mapped there by the resolver of this server, which a clone host name, split DNS or a + * stale hosts file gives. The peer it hides is still the one which dials this server. * * @param address * the configured address of a replication server diff --git a/opendj-server-legacy/src/main/java/org/opends/server/replication/server/ReplicationServerHandler.java b/opendj-server-legacy/src/main/java/org/opends/server/replication/server/ReplicationServerHandler.java index c503ecac2c..85f2b76de0 100644 --- a/opendj-server-legacy/src/main/java/org/opends/server/replication/server/ReplicationServerHandler.java +++ b/opendj-server-legacy/src/main/java/org/opends/server/replication/server/ReplicationServerHandler.java @@ -782,7 +782,15 @@ boolean isSameServerAs(ReplicationServerHandler other) * has no reason to resolve, and it is the peer the addresses are there for. What that * gives up is two servers which share a server id and both name one unresolvable name on * one port: they are read as one, as two servers which name the same resolvable address - * already are. + * are under the other arm. + *

+ * A pair the names alone do not answer is resolved on every call, and the handshake road + * makes those calls under the domain lock: both {@code startFromRemoteRS()} and the + * connect road hold that lock across + * {@link ReplicationServerDomain#isAlreadyConnectedToRS(ReplicationServerHandler)}, so a + * resolver which does not answer holds the domain for its own timeout on the road where + * two handlers of one server id name different hosts. Only replication servers reach it: + * the handshake of a data server makes no such comparison. * * @param address * a configured address of a replication server diff --git a/opendj-server-legacy/src/test/java/org/opends/server/replication/server/MultiHomedPeerTest.java b/opendj-server-legacy/src/test/java/org/opends/server/replication/server/MultiHomedPeerTest.java index 41593d0070..0ef74bbad1 100644 --- a/opendj-server-legacy/src/test/java/org/opends/server/replication/server/MultiHomedPeerTest.java +++ b/opendj-server-legacy/src/test/java/org/opends/server/replication/server/MultiHomedPeerTest.java @@ -28,6 +28,7 @@ import java.net.Socket; import java.util.Collections; import java.util.List; +import java.util.SortedSet; import java.util.concurrent.Callable; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -53,10 +54,11 @@ * not the address it is configured as -- used to be recognised by the address the * socket of one of its sessions happened to carry. *

- * {@code ReplicationServerHandler.toServerAddressURL()} takes the host of a handler from - * {@code session.getRemoteAddress()} and its port from the start message that handler - * received, so the address a peer is registered under is an artefact of which interface its - * connection used, not an identity. Two things followed, and the tests below drive both: + * {@code ReplicationServerHandler.toServerAddressURL()}, which this change removes, took + * the host of a handler from {@code session.getRemoteAddress()} and its port from the start + * message that handler received, so the address a peer was registered under was an artefact + * of which interface its connection used, not an identity. Two things followed, and the + * tests below drive both: * the already connected test of {@code runConnect()} compared a configured address against * one which never matches it, so the peer was dialled again on every pass, about once a * second, for as long as it stayed where it was; and the handshake offered to that same @@ -71,7 +73,8 @@ * what reaching the same peer at its configured address gives. The addresses named are * documentation addresses (RFC 5737): nothing answers at them, which is what the connect * thread of the server under test finds when it dials one -- a host stack sends that SYN to - * its default route, and the connect fails, once per pass until the peer has registered. + * its default route, and the connect fails, once per six passes -- a failed dial is + * blacklisted for the five passes after it -- until the peer has registered. */ @SuppressWarnings("javadoc") public class MultiHomedPeerTest extends ReplicationTestCase @@ -82,6 +85,8 @@ public class MultiHomedPeerTest extends ReplicationTestCase private static final int RS_ID = 8251; private static final int PEER_RS_ID = 8252; + /** The server id of the second fake peer, which only the removal case registers. */ + private static final int SECOND_PEER_RS_ID = 8253; /** TEST-NET-1: the address the peer is configured under and answers on. */ private static final byte[] PEER_ADDRESS = { (byte) 192, 0, 2, 1 }; @@ -292,6 +297,21 @@ public void aPeerWhichNamesAnUnresolvableHostIsNotReportedAsADuplicateServerId() inbound = registerPeerFrom(ports[0], namedAddress, baseDN); final ReplicationServerHandler registered = waitForRegistration(domain); + /* + * The premise of this case rather than an assumption about the network it runs on: + * HostPort reports a name it cannot resolve each time it builds an address from one, + * so those records are what says this machine answers that the name below does not + * exist. A resolver which synthesises an address for a name which does not -- consumer + * ISPs, captive portals, some corporate DNS -- would have isEquivalentTo() answer this + * case on its own, which leaves the arm the case is here for unread, the count below + * vacuous and nothing red to say so. + */ + assertThat(recordsContaining(TestCaseUtils.ERROR_TEXT_WRITER.getMessages(), + ERR_COULD_NOT_SOLVE_HOSTNAME.get(UNRESOLVABLE_HOST).toString())) + .as("the resolver of this machine must answer that " + UNRESOLVABLE_HOST + + " does not exist, which is the premise of this case") + .isNotEmpty(); + /* * What the addresses are built from is read once, when the start message names them: * a name which cannot be resolved is reported by HostPort each time one is built from @@ -369,6 +389,62 @@ public void twoServersSharingAServerIdAreStillReported() throws Exception } } + /** + * Tests that a second session which comes from the address a peer is already connected on + * is read as that peer, whatever that session names. + *

+ * This is the other arm of the comparison the handshake makes, and what it gives up: + * a session from the address a peer is connected on is that peer, so two replication + * servers which share a server id and reach this one from behind a single gateway are read + * as one and the second of them is dropped in silence. The base did the same with them -- + * it compared the addresses the two sockets carried, which for such a pair is one string + * twice -- and this is the arm which recognises the peer of the case above, whose named + * address matches nothing in this configuration. + */ + @Test + public void aSecondSessionFromTheAddressAPeerIsConnectedOnIsReadAsThatPeer() throws Exception + { + TestCaseUtils.startServer(); + + final DN baseDN = DN.valueOf(TEST_ROOT_DN_STRING); + final int[] ports = TestCaseUtils.findFreePorts(2); + final HostPort peerAddress = documentationAddress(PEER_ADDRESS, ports[1]); + final HostPort otherAddress = documentationAddress(OTHER_ADDRESS, ports[1]); + + ReplicationServer rs = null; + Session inbound = null; + try + { + rs = startServerWithPeerConfiguredAt(ports[0], "multiHomedPeerGatewayDb", peerAddress); + final ReplicationServerDomain domain = rs.getReplicationServerDomain(baseDN, true); + inbound = registerPeerFrom(ports[0], peerAddress, baseDN); + final ReplicationServerHandler registered = waitForRegistration(domain); + + /* + * The handshake is answered from the loopback address the session of the registered + * peer came from, and what answers it names an address neither handler carries + * otherwise: the named addresses of the two differ, so the address the two sessions + * have in common is the only thing which can match them. + */ + final List records = + errorLogRecordsOfHandshakeWith(rs, baseDN, loopbackAt(ports[1]), otherAddress); + + assertThat( + duplicateServerIdRecords(records, rs, loopbackAt(ports[1]), loopbackAt(ports[1]))) + .as("a session from the address a peer is connected on is that peer, whatever that" + + " session names") + .isEmpty(); + assertThat(domain.getConnectedRSs().get(PEER_RS_ID)) + .as("the session the peer dialled must outlive the duplicate connection") + .isSameAs(registered); + } + finally + { + close(inbound); + removeQuietly(rs); + } + } + /** * Tests that a peer taken out of the configuration is disconnected even though the * session it is connected on came from another address. @@ -410,18 +486,109 @@ public void aPeerRemovedFromTheConfigurationIsDisconnectedByItsConfiguredAddress } } + /** + * Tests that taking the entry of this server out of {@code ds-cfg-replication-server} + * stops no peer, while taking the entry of a peer out still stops that peer. + *

+ * {@code HostPort} normalises every address local to this machine to {@code localhost}, so + * the entry of this server and the loopback address a peer names for itself are one + * address on this port: a peer whose own configuration lists {@code localhost:P} for + * itself names exactly that in its start messages -- {@code setServerURL()} takes the + * first configured entry which is local to it -- and on this server that name normalises + * to what its own entry does. The connect thread skips the entry of this server before it + * dials; the road which disconnects removed replication servers has to skip it as well, or + * removing it -- a configuration which does not list this server is supported, see + * {@code runConnect()} -- costs such a peer its session, which it then has to dial again. + *

+ * Both entries go in one change, so what tells the two peers apart is that guard alone: + * the peer which names the entry of this server stays, the peer whose own entry was + * removed goes. What the in JVM fixture cannot give is the fold itself, which needs one + * port on two machines; what it names is the address of this server, which normalises the + * same way. + */ + @Test + public void removingTheEntryOfThisServerStopsNoPeer() throws Exception + { + TestCaseUtils.startServer(); + + final DN baseDN = DN.valueOf(TEST_ROOT_DN_STRING); + final int[] ports = TestCaseUtils.findFreePorts(2); + final HostPort ownAddress = loopbackAt(ports[0]); + final HostPort peerAddress = loopbackAt(ports[1]); + final String dbDirName = "multiHomedPeerOwnEntryDb"; + + ReplicationServer rs = null; + Session namesThisServer = null; + Session namesItsOwnEntry = null; + try + { + rs = startServerWithPeersConfiguredAt(ports[0], dbDirName, ownAddress, peerAddress); + final ReplicationServerDomain domain = rs.getReplicationServerDomain(baseDN, true); + namesThisServer = registerPeerFrom(ports[0], PEER_RS_ID, ownAddress, baseDN); + final ReplicationServerHandler namedThisServer = waitForRegistration(domain, PEER_RS_ID); + namesItsOwnEntry = registerPeerFrom(ports[0], SECOND_PEER_RS_ID, peerAddress, baseDN); + waitForRegistration(domain, SECOND_PEER_RS_ID); + + // The precondition of the case rather than an assumption: a peer which does not answer + // to the entry of this server is a peer this case says nothing about. + assertThat(namedThisServer.isServerAt(HostPort.localAddress(ports[0]))) + .as("the peer which names a loopback address on the port of this server should" + + " answer to the entry of this server") + .isTrue(); + + // Both entries removed at once, which is what an administrator emptying the list does. + rs.applyConfigurationChange(configurationWithPeersAt(ports[0], dbDirName)); + + assertThat(domain.getConnectedRSs().keySet()) + .as("a peer which names the address this server listens on must outlive the removal" + + " of the entry of this server") + .contains(PEER_RS_ID); + assertThat(domain.getConnectedRSs().keySet()) + .as("the peer whose own entry was removed must still be disconnected") + .doesNotContain(SECOND_PEER_RS_ID); + } + finally + { + close(namesThisServer); + close(namesItsOwnEntry); + removeQuietly(rs); + } + } + /** * Starts a replication server whose only configured peer is at the provided address. *

* Nothing ever answers there -- the address is a documentation address -- which is what * the connect thread of the server does with it: it dials it, fails, and comes back to it - * on the next pass. Whether it dials it at all is what the first case is about. + * six passes later, the failed dial being blacklisted for the five in between. Whether it + * dials it at all is what the first case is about. */ private ReplicationServer startServerWithPeerConfiguredAt(int port, String dbDirName, HostPort peerAddress) throws Exception { - return new ReplicationServer(new ReplServerFakeConfiguration( - port, dbDirName, 0, RS_ID, 0, 100, newTreeSet(peerAddress.toString()))); + return startServerWithPeersConfiguredAt(port, dbDirName, peerAddress); + } + + /** Starts a replication server whose configured peers are the provided addresses. */ + private ReplicationServer startServerWithPeersConfiguredAt(int port, String dbDirName, + HostPort... peerAddresses) throws Exception + { + return new ReplicationServer(configurationWithPeersAt(port, dbDirName, peerAddresses)); + } + + /** + * Returns the configuration of the server under test, listing the provided addresses as + * the replication servers of its topology. + */ + private ReplServerFakeConfiguration configurationWithPeersAt(int port, String dbDirName, + HostPort... peerAddresses) + { + final SortedSet configured = newTreeSet(); + for (HostPort peerAddress : peerAddresses) + { + configured.add(peerAddress.toString()); + } + return new ReplServerFakeConfiguration(port, dbDirName, 0, RS_ID, 0, 100, configured); } /** @@ -432,6 +599,19 @@ private ReplicationServer startServerWithPeerConfiguredAt(int port, String dbDir * @return the session the registration hangs on, closed by the caller */ private Session registerPeerFrom(int port, HostPort registeredAs, DN baseDN) throws Exception + { + return registerPeerFrom(port, PEER_RS_ID, registeredAs, baseDN); + } + + /** + * Has a fake peer of the provided server id dial the server under test, as + * {@link #registerPeerFrom(int, HostPort, DN)} does for the peer of the cases which need + * one peer only. + * + * @return the session the registration hangs on, closed by the caller + */ + private Session registerPeerFrom(int port, int peerServerId, HostPort registeredAs, DN baseDN) + throws Exception { final ReplSessionSecurity security = getReplSessionSecurity(); final Socket socket = new Socket(); @@ -441,7 +621,7 @@ private Session registerPeerFrom(int port, HostPort registeredAs, DN baseDN) thr socket.setTcpNoDelay(true); socket.connect(new InetSocketAddress("127.0.0.1", port), SOCKET_TIMEOUT_MS); session = security.createClientSession(socket, SOCKET_TIMEOUT_MS); - session.publish(peerStartMsg(registeredAs, baseDN)); + session.publish(peerStartMsg(peerServerId, registeredAs, baseDN)); session.receive(); // The initiator of a session decides whether it is encrypted, and the start message // above asked for it not to be: both ends leave the SSL session together, right after @@ -454,7 +634,7 @@ private Session registerPeerFrom(int port, HostPort registeredAs, DN baseDN) thr * an empty one ends the handshake on an IndexOutOfBoundsException instead, which is * an abort like any other and would leave the peer unregistered. */ - final RSInfo peerInfo = new RSInfo(PEER_RS_ID, registeredAs.toString(), -1, (byte) 1, 1); + final RSInfo peerInfo = new RSInfo(peerServerId, registeredAs.toString(), -1, (byte) 1, 1); session.publish(new TopologyMsg(Collections. emptyList(), newArrayList(peerInfo))); session.receive(); return session; @@ -566,7 +746,13 @@ public InetAddress getInetAddress() /** Returns the start message of the fake peer, naming the provided address. */ private ReplServerStartMsg peerStartMsg(HostPort address, DN baseDN) { - return new ReplServerStartMsg(PEER_RS_ID, address.toString(), baseDN, 100, + return peerStartMsg(PEER_RS_ID, address, baseDN); + } + + /** Returns the start message of a fake peer of the provided server id. */ + private ReplServerStartMsg peerStartMsg(int peerServerId, HostPort address, DN baseDN) + { + return new ReplServerStartMsg(peerServerId, address.toString(), baseDN, 100, new ServerState(), -1, false, (byte) 1, 5000); } @@ -589,19 +775,33 @@ private HostPort loopbackAt(int port) */ private ReplicationServerHandler waitForRegistration(ReplicationServerDomain domain) throws Exception + { + return waitForRegistration(domain, PEER_RS_ID); + } + + /** + * Waits for the domain to hold the handler of the fake peer of the provided server id, as + * {@link #waitForRegistration(ReplicationServerDomain)} does for the peer of the cases + * which need one peer only. + */ + private ReplicationServerHandler waitForRegistration(ReplicationServerDomain domain, + int peerServerId) throws Exception { final long deadline = System.currentTimeMillis() + REGISTRATION_TIMEOUT_MS; ReplicationServerHandler registered; while (true) { - registered = domain.getConnectedRSs().get(PEER_RS_ID); + registered = domain.getConnectedRSs().get(peerServerId); if (registered != null || System.currentTimeMillis() > deadline) { break; } Thread.sleep(50); } - assertThat(registered).as("the fake peer should have registered with the domain").isNotNull(); + assertThat(registered) + .as("the fake peer of server id " + peerServerId + " should have registered with the" + + " domain") + .isNotNull(); return registered; }