diff --git a/.gitignore b/.gitignore index eb5a6fc47..a427c779b 100644 --- a/.gitignore +++ b/.gitignore @@ -33,8 +33,6 @@ ipch/ .DS_Store src/EventStore/EventStore.Common/Properties/AssemblyVersion.cs -src/EventStore/EventStore.ClientAPI/Properties/AssemblyVersion.cs - *.o *.ii *.s diff --git a/Dockerfile b/Dockerfile index 064281c15..edbfe3228 100644 --- a/Dockerfile +++ b/Dockerfile @@ -42,7 +42,6 @@ FROM mcr.microsoft.com/dotnet/sdk:10.0-${CONTAINER_RUNTIME} AS test WORKDIR /build COPY --from=build ./build/published-tests ./published-tests COPY --from=build ./build/ci ./ci -COPY --from=build ./build/src/EventStore.Core.Tests/Services/Transport/Tcp/test_certificates/ca/ca.crt /usr/local/share/ca-certificates/ca_eventstore_test.crt COPY ./scripts/test.sh /build/test.sh RUN mkdir ./test-results RUN chmod +x /build/test.sh @@ -89,7 +88,7 @@ ReplicationIp: 0.0.0.0" >> /etc/eventstore/eventstore.conf VOLUME /var/lib/eventstore /var/log/eventstore -EXPOSE 1112/tcp 1113/tcp 2113/tcp +EXPOSE 1112/tcp 2113/tcp HEALTHCHECK --interval=5s --timeout=5s --retries=24 \ CMD curl --fail --insecure https://localhost:2113/-/liveness || curl --fail http://localhost:2113/-/liveness || exit 1 diff --git a/docker-compose.yml b/docker-compose.yml index 0896f1114..b63bee223 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -17,6 +17,7 @@ services: - shared.env environment: - EVENTSTORE_GOSSIP_SEED=172.30.240.12:2113,172.30.240.13:2113 + - EVENTSTORE_NODE_IP=172.30.240.11 - EVENTSTORE_REPLICATION_IP=172.30.240.11 - EVENTSTORE_CERTIFICATE_FILE=/etc/eventstore/certs/node/node.crt - EVENTSTORE_CERTIFICATE_PRIVATE_KEY_FILE=/etc/eventstore/certs/node/node.key @@ -44,6 +45,7 @@ services: - shared.env environment: - EVENTSTORE_GOSSIP_SEED=172.30.240.11:2113,172.30.240.13:2113 + - EVENTSTORE_NODE_IP=172.30.240.12 - EVENTSTORE_REPLICATION_IP=172.30.240.12 - EVENTSTORE_CERTIFICATE_FILE=/etc/eventstore/certs/node/node.crt - EVENTSTORE_CERTIFICATE_PRIVATE_KEY_FILE=/etc/eventstore/certs/node/node.key @@ -71,6 +73,7 @@ services: - shared.env environment: - EVENTSTORE_GOSSIP_SEED=172.30.240.11:2113,172.30.240.12:2113 + - EVENTSTORE_NODE_IP=172.30.240.13 - EVENTSTORE_REPLICATION_IP=172.30.240.13 - EVENTSTORE_CERTIFICATE_FILE=/etc/eventstore/certs/node/node.crt - EVENTSTORE_CERTIFICATE_PRIVATE_KEY_FILE=/etc/eventstore/certs/node/node.key diff --git a/docs/README.md b/docs/README.md index b2063b2c6..d73349733 100644 --- a/docs/README.md +++ b/docs/README.md @@ -10,9 +10,13 @@ surfaces for running a node or cluster. TrogonEventStore keeps the database node focused on the durable event log: -- Application event access is gRPC-first. -- HTTP is reserved for the Admin UI, health probes, metrics, and other - infrastructure-level concerns. +- Database client APIs, cluster coordination, and follower-to-leader forwarding + use gRPC over the node HTTP(S) endpoint. Database replication uses gRPC over + a dedicated replication HTTP(S) endpoint. +- Regular HTTP routes are reserved for the Admin UI, health probes, metrics, + and other infrastructure-level concerns. +- The server does not open a separate legacy EventStore TCP protocol listener + or support its TCP transport configuration. - The project is FOSS-only. The documentation does not describe unsupported proprietary server features. - Rich read models, user-defined query engines, connector runtimes, and @@ -37,7 +41,7 @@ For a production node, review: ## Protocols and clients -The supported application protocol is gRPC. Existing TrogonEventStore-compatible +The supported database protocol is gRPC. Existing TrogonEventStore-compatible gRPC clients can be useful while the TrogonDB client libraries continue to evolve, but the server documentation should be treated as authoritative for this repository. diff --git a/docs/admin-ui.md b/docs/admin-ui.md index f92600a96..d569a03ab 100644 --- a/docs/admin-ui.md +++ b/docs/admin-ui.md @@ -10,7 +10,7 @@ The TrogonEventStore Admin UI is available at `http://SERVER_IP:2113/ui` and hel The dashboard opens at `/ui` and combines the daily operational view in one place: -- _Cluster status_: live gossip membership, node state, checkpoints, TCP and HTTP endpoints, replica status, and a copy-friendly snapshot. +- _Cluster status_: live gossip membership, node state, checkpoints, HTTP(S) endpoints, replica status, and a copy-friendly snapshot. - _Queue pressure_: live queue length, throughput, processing time, and currently processed messages. - _Node probes_: inline Ping, Node info, and Gossip checks rendered inside the UI. @@ -24,7 +24,8 @@ The _Observability_ page focuses on runtime diagnostics: - queue groups and individual queue rows - current and last processed messages -- TCP connection statistics +- active connections on the node and replication HTTP/gRPC endpoints, including client identity, protocol, security, traffic rates, totals, and pending bytes +- live gRPC replication sessions, byte totals, pending bytes, and send queue depth - snapshot output for copy-paste debugging ## Configuration diff --git a/docs/architecture.md b/docs/architecture.md index 2b1ac2fc7..885a7960c 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -22,6 +22,17 @@ and sharding work. Features that need their own compute model, query model, or serving model should be separate components that consume the database through subscriptions or reads. +## Network protocol boundary + +gRPC carries database client APIs, cluster coordination, and follower-to-leader +forwarding over the node HTTP(S) endpoint. Ordinary HTTP routes on that endpoint +serve the Admin UI, health probes, metrics, and other operator workflows. +Database replication uses gRPC over a dedicated replication HTTP(S) endpoint so +operators can isolate high-volume replication traffic from clients. + +Both endpoints use the same TLS and node identity configuration. The node has no +separate legacy EventStore TCP protocol listener or configuration surface. + ## Projection execution Projection execution is future external component work by default. diff --git a/docs/cluster.md b/docs/cluster.md index 9c326de56..8563417b0 100644 --- a/docs/cluster.md +++ b/docs/cluster.md @@ -55,15 +55,22 @@ The multi-address DNS name cluster discovery only works for clusters that use ce ### Internal communication -When setting up a cluster, the nodes must be able to reach each other over both the HTTP channel, and the internal TCP channel. You should ensure that these ports are open on firewalls on the machines and between the machines. +Cluster nodes use gRPC over each node's dedicated replication HTTP(S) endpoint +for database replication. Gossip, elections, and follower-to-leader request +forwarding use the node HTTP(S) endpoint. Ensure every node can reach both +advertised endpoints on every other node. -Learn more about [internal TCP configuration](networking.md#replication-protocol) and [HTTP configuration](networking.md#http-configuration) to set up the cluster properly. +Learn more about the [node](networking.md#http-configuration) and +[replication](networking.md#internal-cluster-traffic) endpoints before +configuring cluster firewall or network policy rules. ## Cluster with DNS When you tell TrogonEventStore to use DNS for its gossip, the server will resolve the DNS name to a list of IP addresses and connect to each of those addresses to find other nodes. This method is very flexible because you can change the list of nodes on your DNS server without changing the cluster configuration. The DNS method is also useful in automated deployment scenarios when you control both the cluster deployment and the DNS server from your infrastructure-as-code scripts. -To use DNS discovery, you need to set the `ClusterDns` option to the DNS name that allows making an HTTP call to it. When the server starts, it will attempt to make a gRPC call using the `https://:` URL (`http` if the cluster is insecure). +To use DNS discovery, set the `ClusterDns` option to a DNS name that resolves to +the cluster nodes. When the server starts, it attempts a gRPC call over +`https://:` (`http` if the cluster is insecure). When using a certificate signed by a publicly trusted CA, you'd normally use the wildcard certificate. Ensure that the cluster DNS name fits the wildcard, otherwise the request will fail on SSL check. @@ -107,7 +114,8 @@ The setting accepts a comma-separated list of IP addresses or host names with th TrogonEventStore uses a quorum-based replication model. When working normally, a cluster has one node known as a leader, and the remaining nodes are followers. The leader node is responsible for coordinating writes while it is the leader. Cluster nodes use a consensus algorithm to determine which node should be the leader and which should be followers. TrogonEventStore bases the decision as to which node should be the leader on a number of factors. -For a cluster node to have this information available to them, the nodes gossip with other nodes in the cluster. Gossip runs over HTTP interfaces of cluster nodes. +For a cluster node to have this information available to them, the nodes gossip +with other nodes in the cluster over the node HTTP(S) endpoint. The gossip protocol configuration can be changed using the settings listed below. Pay attention to the settings related to time, like intervals and timeouts, when running in a cloud environment. @@ -229,7 +237,7 @@ candidate. ### Follower -A cluster assigns the follower role based on an election process. A cluster uses one or more nodes with the follower role to form the quorum, or the majority of nodes necessary to confirm that the write is persisted. +A cluster assigns the follower role based on an election process. A cluster uses one or more nodes with the follower role to form the quorum, or the majority of nodes necessary to confirm that the write is persisted. When a follower accepts a request that must run on the leader, it forwards the request to the leader over gRPC on the leader's HTTP(S) endpoint. ### Read-only replica diff --git a/docs/diagnostics/README.md b/docs/diagnostics/README.md index adc52604b..8845b38fa 100644 --- a/docs/diagnostics/README.md +++ b/docs/diagnostics/README.md @@ -6,6 +6,7 @@ TrogonEventStore provides several ways to diagnose and troubleshoot issues. - [Metrics](metrics.md): collect standard metrics using Prometheus or OpenTelemetry. - [Monitoring and alerting](monitoring.md): turn health, metrics, and logs into operational signals. - [Stats](#statistics): runtime statistics exposed through the monitoring gRPC service. +- [Connection statistics](#connection-statistics): active HTTP and gRPC connections on both listeners. You can also use external tools to measure the performance of TrogonEventStore and monitor the cluster health. Learn more on the [Integrations](./integrations.md) page. @@ -19,6 +20,21 @@ cluster-wide state. `Monitoring.Stats` collects fresh node-local statistics and memoizes the result for up to one second. It does not read persisted statistics events. +## Connection statistics + +Use the `Monitoring.ConnectionStats` RPC to inspect active connections on the +node and replication HTTP(S) listeners. Each result identifies the local and +remote endpoints, connection ID, observed client name, protocol, application, +TLS state, connection time, total bytes, and pending bytes. + +Use `Monitoring.ReplicationStats` when the diagnostic question is specifically +about live database replication sessions. Its results include the subscription +and connection IDs, peer endpoint, byte totals, pending bytes, and send queue +depth. + +Both RPCs report node-local snapshots. Query each cluster member when diagnosing +a cluster-wide connection or replication problem. + When statistics persistence is enabled, each node writes events to a reserved `$stats-` stream. For example, a single local node writes to `$stats-127.0.0.1:2113`. @@ -49,17 +65,6 @@ type `$statsCollected`. "proc-gc-largeHeapSize": 0, "proc-gc-timeInGc": 0.0, "proc-gc-totalBytesInHeaps": 0, - "proc-tcp-connections": 0, - "proc-tcp-receivingSpeed": 0.0, - "proc-tcp-sendingSpeed": 0.0, - "proc-tcp-inSend": 0, - "proc-tcp-measureTime": "00:00:19.0534210", - "proc-tcp-pendingReceived": 0, - "proc-tcp-pendingSend": 0, - "proc-tcp-receivedBytesSinceLastRun": 0, - "proc-tcp-receivedBytesTotal": 0, - "proc-tcp-sentBytesSinceLastRun": 0, - "proc-tcp-sentBytesTotal": 0, "es-checksum": 1613144, "es-checksumNonFlushed": 1613144, "sys-drive-/System/Volumes/Data-availableBytes": 545628151808, @@ -104,7 +109,7 @@ type `$statsCollected`. "es-queue-MonitoringQueue-lengthLifetimePeak": 0, "es-queue-MonitoringQueue-totalItemsProcessed": 14, "es-queue-MonitoringQueue-inProgressMessage": "", - "es-queue-MonitoringQueue-lastProcessedMessage": "GetFreshTcpConnectionStats", + "es-queue-MonitoringQueue-lastProcessedMessage": "GetFreshStats", "es-queue-PersistentSubscriptions-queueName": "PersistentSubscriptions", "es-queue-PersistentSubscriptions-groupName": "", "es-queue-PersistentSubscriptions-avgItemsPerSecond": 1, diff --git a/docs/installation.md b/docs/installation.md index 8e4a2a916..2ba67e71a 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -76,6 +76,9 @@ Before running a durable node or cluster: - Store data, index, and logs on durable volumes. - Expose `/-/liveness`, `/-/readiness`, and `/-/metrics` to the platform. - Use gRPC clients for application reads and writes. +- Expose the node HTTP(S) endpoint to clients and operators. Allow peer nodes to + reach the dedicated replication HTTP(S) endpoint on the private cluster + network. No legacy EventStore TCP protocol listener is required or supported. ## Linux service notes diff --git a/docs/networking.md b/docs/networking.md index 84b141649..c5d7a346c 100644 --- a/docs/networking.md +++ b/docs/networking.md @@ -4,22 +4,22 @@ title: Networking ## Network configuration -TrogonEventStore provides two interfaces: -- HTTP(S) for gRPC communication, the Admin UI, and operational endpoints such as health checks and metrics -- TCP for cluster replication (internal) +TrogonEventStore provides two HTTP(S) interfaces: -Nodes in the cluster replicate with each other using the TCP protocol, but use gRPC for [discovering other cluster nodes](cluster.md#discovering-cluster-members). +- The node endpoint carries database client APIs, cluster coordination, follower-to-leader request forwarding, the Admin UI, health probes, metrics, and other operator endpoints. +- The replication endpoint carries node-to-node database replication over gRPC. -Server nodes use a single HTTP binding for gRPC, the Admin UI, health, metrics, -and supported diagnostics. Replication between cluster nodes is internal and can -be placed on a private network interface. Keep public client access on gRPC and -avoid exposing replication ports outside the cluster network. - -For gRPC and HTTP, there's no internal vs external separation of traffic. +The node does not open a separate legacy EventStore TCP protocol listener or +accept the legacy TCP client protocol. gRPC traffic on both interfaces uses +HTTP/2 over TCP at the operating-system transport layer. Keep the replication +endpoint on the private cluster network and expose the node endpoint according +to client and operator requirements. ## HTTP configuration -HTTP is the primary protocol for TrogonEventStore. It carries gRPC communication, the Admin UI, health checks, metrics, and supported diagnostics endpoints. +HTTP/2 on the node endpoint carries gRPC traffic for database clients and +internal cluster coordination, while regular HTTP routes serve the Admin UI, +health checks, metrics, and supported diagnostics. The HTTP endpoint always binds to the IP address configured in the `NodeIp` setting (previously referred to as `ExtIp`). | Format | Syntax | @@ -145,15 +145,13 @@ Value must be greater than or equal to 65,535 and less than 2^31. See the docs [ This is configured with `Kestrel.Limits.Http2.InitialStreamWindowSize` in the settings file. -## Replication protocol - -Replication between cluster nodes uses an internal TCP-based protocol. Options for configuring the internal replication protocol are described below. - -### Interface and port +## Internal cluster traffic -Internal TCP binds to the IP address specified in the `ReplicationIp` setting (previously `IntIp`). It must be configured if you run a multi-node cluster. +Cluster replication uses a dedicated HTTP/2 listener so it can remain isolated +from client and operator traffic while using gRPC. Follower-to-leader request +forwarding and cluster coordination continue to use the node endpoint. -By default, TrogonEventStore binds its internal networking on the loopback interface only (`127.0.0.1`). You can change this behaviour and tell TrogonEventStore to listen on a specific internal IP address. To do that set the `ReplicationIp` to `0.0.0.0` or the IP address of the network interface. +The replication listener binds to `ReplicationIp`: | Format | Syntax | |:---------------------|:----------------------------| @@ -163,13 +161,10 @@ By default, TrogonEventStore binds its internal networking on the loopback inter **Default**: `127.0.0.1` (loopback). -If you keep this setting to its default value, cluster nodes won't be able to talk to each other. +For a multi-node cluster, bind this setting to an interface reachable by the +other database nodes. `0.0.0.0` binds all IPv4 interfaces. -::: warning -Please note that the `IntIp` parameter has been deprecated as of version 23.10.0 and will be removed in future versions. It is recommended to use the `ReplicationIp` parameter instead. -::: - -By default, TrogonEventStore uses port `1112` for internal TCP. You can change this by specifying the `ReplicationPort` setting (previously `IntTcpPort` setting). +The replication listener uses `ReplicationPort`: | Format | Syntax | |:---------------------|:------------------------------| @@ -179,17 +174,24 @@ By default, TrogonEventStore uses port `1112` for internal TCP. You can change t **Default**: `1112` -::: warning -Please note that the `IntTcpPort` parameter has been deprecated as of version 23.10.0 and will be removed in future versions. It is recommended to use the `ReplicationPort` parameter instead. -::: +When TLS is enabled, internal gRPC connections use the configured node +certificate. `DisableTls` disables encryption for both HTTP(S) listeners while +preserving authentication and authorization. + +Replication connections use HTTP/2 keepalive pings for failure detection. The +existing replication heartbeat settings configure the client-side ping interval +and acknowledgement timeout: -### Security +| Format | Interval | Timeout | +|:---------------------|:-------------------------------------------|:------------------------------------------| +| Command line | `--replication-heartbeat-interval` | `--replication-heartbeat-timeout` | +| YAML | `ReplicationHeartbeatInterval` | `ReplicationHeartbeatTimeout` | +| Environment variable | `EVENTSTORE_REPLICATION_HEARTBEAT_INTERVAL` | `EVENTSTORE_REPLICATION_HEARTBEAT_TIMEOUT` | -When TLS is enabled, replication uses the configured node certificate. Replication TLS cannot be disabled -independently. `DisableTls` disables TLS for both HTTP and replication while preserving authentication and -authorization. +**Default**: `700` ms for both settings. -If your network setup requires any kind of IP address, DNS name and port translation for internal communication, you can use available [address translation](#network-address-translation) settings. +Values below `1000` ms remain valid for configuration compatibility and use the +HTTP/2 transport minimum of `1000` ms. ## Network address translation @@ -197,13 +199,14 @@ Due to NAT (network address translation), or other reasons a node may not be bou Options described below allow you to tell the node that even though it is bound to a given address it should not gossip that address. When returning links over HTTP, TrogonEventStore will also use the specified addresses instead of physical addresses, so the clients that use HTTP can follow those links. -Another case when you might want to specify the advertised address although there's no address translation involved. When you configure TrogonEventStore to bind to `0.0.0.0`, it will use the first non-loopback address for gossip. It might or might not be the address you want it to use. Whilst the best way to avoid such a situation is to configure the binding properly using the `NodeIp` and `ReplicationIp` settings, you can also use address translation setting with the correct IP address or DNS name. +Another case when you might want to specify the advertised address although there's no address translation involved. When you configure TrogonEventStore to bind to `0.0.0.0`, it will use the first non-loopback address for gossip. It might or might not be the address you want it to use. Configure `NodeHostAdvertiseAs` for the node endpoint and `ReplicationHostAdvertiseAs` for the replication endpoint when other nodes must connect using specific IP addresses or hostnames. -Also, even if you specified the `NodeIp` and `ReplicationIp` settings in the configuration, you might still want to override the advertised address if you want to use hostnames and not IP addresses. That might be needed when running a secure cluster with certificates that only contain DNS names of the nodes. +You might also override the advertised address when secure cluster certificates +contain DNS names rather than IP addresses. The only place where these settings make any effect is the [gossip](cluster.md#gossip-protocol) endpoint response. -## HTTP translations +## HTTP(S) endpoint advertisement By default, a cluster node will advertise itself using `NodeIp` and `NodePort`. You can override the advertised HTTP port using the `NodePortAdvertiseAs` setting (previously `HttpPortAdvertiseAs` setting). @@ -229,30 +232,21 @@ If you want the node to advertise itself using the hostname rather than its IP a Please note that the `ExtHostAdvertiseAs` parameter has been deprecated as of version 23.10.0 and will be removed in future versions. It is recommended to use the `NodeHostAdvertiseAs` parameter instead. ::: -### TCP translations +## Replication endpoint advertisement -TCP ports used for replication can be advertised using custom values: +If the bound replication address or port is not reachable as-is from the other +nodes, override the endpoint advertised through internal gossip. -| Format | Syntax | -|:---------------------|:-----------------------------------------------| -| Command line | `--replication-tcp-port-advertise-as` | -| YAML | `ReplicationTcpPortAdvertiseAs` | -| Environment variable | `EVENTSTORE_REPLICATION_TCP_PORT_ADVERTISE_AS` | +| Format | Host syntax | Port syntax | +|:---------------------|:---------------------------------------------|:---------------------------------------------| +| Command line | `--replication-host-advertise-as` | `--replication-port-advertise-as` | +| YAML | `ReplicationHostAdvertiseAs` | `ReplicationPortAdvertiseAs` | +| Environment variable | `EVENTSTORE_REPLICATION_HOST_ADVERTISE_AS` | `EVENTSTORE_REPLICATION_PORT_ADVERTISE_AS` | ::: warning -Please note that the `IntTcpPortAdvertiseAs` parameter has been deprecated as of version 23.10.0 and will be removed in future versions. It is recommended to use the `ReplicationTcpPortAdvertiseAs` and `NodeTcpPortAdvertiseAs` parameters instead, respectively. -::: - -If you want to change how the node TCP address is advertised internally, use the `ReplicationHostAdvertiseAs` setting (previously `IntHostAdvertiseAs` setting). You can use an IP address or a hostname. - -| Format | Syntax | -|:---------------------|:-------------------------------------------| -| Command line | `--replication-host-advertise-as` | -| YAML | `ReplicationHostAdvertiseAs` | -| Environment variable | `EVENTSTORE_REPLICATION_HOST_ADVERTISE_AS` | - -::: warning -Please note that the `IntHostAdvertiseAs` parameter has been deprecated as of version 23.10.0 and will be removed in future versions. It is recommended to use the `ReplicationHostAdvertiseAs` parameter instead. +`ReplicationTcpPortAdvertiseAs` is deprecated. Use +`ReplicationPortAdvertiseAs` for the gRPC replication endpoint. If both are +configured, `ReplicationPortAdvertiseAs` takes precedence. ::: ### Advertise to clients @@ -281,49 +275,6 @@ Specify the advertised HTTP(S) port (previously `AdvertiseHttpPortToClientAs` se Please note that the `AdvertiseHttpPortToClientAs` parameter has been deprecated as of version 23.10.0 and will be removed in future versions. It is recommended to use the `AdvertiseNodePortToClientAs` parameter instead. ::: -## Heartbeat timeouts - -TrogonEventStore uses heartbeats over all TCP connections to discover dead clients and nodes. Heartbeat timeouts should not be too short, as short timeouts will produce false positives. At the same time, setting too long timeouts will prevent discovering dead nodes and clients in time. - -Each heartbeat has two points of configuration. The first is the _interval;_ this represents how often the system should consider a heartbeat. TrogonEventStore doesn't send a heartbeat for every interval, but only if it has not heard from a node within the configured interval. In a busy cluster, you may never see any heartbeats. - -The second point of configuration is the _timeout_. This determines how long TrogonEventStore server waits for a client or node to respond to a heartbeat request. - -Different environments need different values for these settings. The defaults are likely fine on a LAN. If you experience frequent elections in your environment, you can try to increase both interval and timeout, for example: - -- An interval of 5000ms. -- A timeout of 1000ms. - -::: tip -If in doubt, choose higher numbers. This adds a small period of time to discover a dead client or node and is better than the alternative, which is false positives. -::: - -Replication/Internal TCP heartbeat (between cluster nodes): - -| Format | Syntax | -|:---------------------|:--------------------------------------------| -| Command line | `--replication-heartbeat-interval` | -| YAML | `ReplicationHeartbeatInterval` | -| Environment variable | `EVENTSTORE_REPLICATION_HEARTBEAT_INTERVAL` | - -**Default**: `700` (ms) - -| Format | Syntax | -|:---------------------|:-------------------------------------------| -| Command line | `--replication-heartbeat-timeout` | -| YAML | `ReplicationHeartbeatTimeout` | -| Environment variable | `EVENTSTORE_REPLICATION_HEARTBEAT_TIMEOUT` | - -**Default**: `700` (ms) - -::: warning -Please note that the `IntTcpHeartbeatInterval` and `IntTcpHeartbeatTimeout` parameters have been deprecated as of version 23.10.0 and will be removed in future versions. It is recommended to use the `ReplicationHeartbeatInterval` and `ReplicationHeartbeatTimeout` parameters instead, respectively. -::: - -### gRPC heartbeats - -For the gRPC heartbeats, TrogonEventStore and its gRPC clients use the protocol feature called _Keepalive ping_. Read more about it on the [HTTP configuration page](#keep-alive-pings). - ## Exposing endpoints If you need to reduce the HTTP surface, you can disable the browser-facing Admin UI and the Prometheus metrics endpoint. Health probes and gRPC remain part of the supported HTTP listener. @@ -348,11 +299,10 @@ You can disable the Prometheus metrics endpoint by setting `DisableStatsOnHttp` **Default**: `false`, the Prometheus metrics endpoint is enabled on `/-/metrics`. -## Application protocol boundary - -TrogonEventStore does not document an external TCP client protocol. Application -reads and writes should use gRPC clients. +## Protocol boundary -Internal replication can still use node-to-node transport that is not part of -the public client API. Treat those settings as cluster internals, not as a -client integration surface. +Database client APIs, replication, and follower-to-leader forwarding use gRPC. +The remaining HTTP routes are operator surfaces, not an application event API. +The server has no legacy EventStore TCP protocol listener, TCP client protocol, +or TCP replication protocol. The separately configurable replication listener +is HTTP/2-only and accepts only the replication gRPC service. diff --git a/docs/security.md b/docs/security.md index d33e3a7a4..1b6f8f35d 100644 --- a/docs/security.md +++ b/docs/security.md @@ -15,14 +15,15 @@ Security features of TrogonEventStore include: ### Protocol security -TrogonEventStore supports gRPC for client communication and internal TCP for -cluster replication. It also has HTTP endpoints for the Admin UI, health, -metrics, and supported operator workflows. -TrogonEventStore also uses HTTP for the gossip seed endpoint, both internally for the cluster gossip, and -internally for clients that connect to the cluster using discovery mode. +TrogonEventStore uses two HTTP(S) endpoints. The node endpoint carries database +client APIs, follower-to-leader forwarding, and cluster discovery over gRPC, as +well as regular HTTP routes for the Admin UI, health, metrics, and supported +operator workflows. The dedicated replication endpoint carries database +replication over gRPC. -All those protocols support encryption with TLS and SSL. Each protocol has its own security configuration, but -you can only use one set of certificates for both TLS and HTTPS. +The same TLS configuration and node certificate protect both listeners. The +server does not open a separate legacy EventStore TCP protocol listener or +support separate TCP security settings. The protocol security configuration depends a lot on the deployment topology and platform. We have created an interactive [configuration tool](installation.md), which also has instructions on how to generate and install @@ -162,7 +163,8 @@ If the domains are `node1.esdb.mycompany.org`, `node2.esdb.mycompany.org` and `n | Environment variable | `EVENTSTORE_CERTIFICATE_RESERVED_NODE_COMMON_NAME` | ::: warning -Server certificates **must** have the internal and external IP addresses (`ReplicationIp` and `NodeIp` respectively) or DNS names as subject alternative names. +Server certificates **must** contain every IP address or DNS name that clients +or peer nodes use to reach the node as a subject alternative name. ::: #### Node certificate Client Authentication usage @@ -188,8 +190,7 @@ validity period, common-name policy, subject alternative names, key usage, and S required. This option does not relax user certificate authentication. ::: -This setting controls node identity authentication for HTTPS cluster traffic. It does not change legacy secure TCP -replication certificate handling. +This setting controls node identity authentication for HTTPS cluster traffic. #### Trusted root certificates @@ -512,11 +513,13 @@ Import-Certificate -FilePath .\ca.crt -CertStoreLocation Cert:\LocalMachine\CA ::: :::: -### Replication protocol security +### Internal gRPC security -When TLS is enabled, cluster replication uses the configured node certificate. Replication TLS cannot be -disabled independently. Use [`DisableTls`](#disable-tls) to disable transport encryption for both HTTP and -replication while preserving authentication and authorization. +When TLS is enabled, cluster replication and follower-to-leader forwarding use +the configured node certificate on the replication and node HTTPS endpoints. +Internal gRPC TLS cannot be disabled independently. Use +[`DisableTls`](#disable-tls) to disable transport encryption for both listeners +while preserving authentication and authorization. ## Authentication @@ -530,7 +533,7 @@ content, and redirects. ### Password authentication admission limits Built-in password authentication shares a node-local admission budget across UI sign-in, HTTP and gRPC -credentials, TCP authentication, and forwarded credentials. Cached credentials still require password +credentials, and forwarded credentials. Cached credentials still require password verification and use the same budget. Attempts are admitted before account reads, including requests for nonexistent accounts, so changing usernames cannot bypass the node-wide limit. @@ -544,7 +547,7 @@ All values must be positive, and `BurstSize` must be at least `AttemptsPerSecond hold a full second's replenishment. Invalid limits prevent the password provider from starting. There is no waiting queue. When either budget is exhausted, requests receive the existing authentication-not-ready response: HTTP returns `503` with `Retry-After`, gRPC -returns `Unavailable`, TCP returns `NotReady`, and browser sign-in reports that the provider is not +returns `Unavailable`, and browser sign-in reports that the provider is not ready. Clients should use bounded retries with backoff and jitter. Limits are shared by all password users on a node, not per account or per IP address. They do not lock diff --git a/docs/upgrade-guide.md b/docs/upgrade-guide.md index 316c1fffe..8af026258 100644 --- a/docs/upgrade-guide.md +++ b/docs/upgrade-guide.md @@ -13,6 +13,9 @@ from this repository. - Record the current server version, configuration file, container image, and command-line arguments. - Check that clients are using gRPC. +- Remove legacy EventStore TCP client-listener settings. Keep the replication + endpoint, advertisement, and heartbeat settings because they now configure + the dedicated gRPC replication listener. - Review changed configuration keys before restarting a durable node. - Verify that health probes use `/-/liveness` and `/-/readiness`. - Verify that metrics scraping uses `/-/metrics`. @@ -49,11 +52,18 @@ During the rollout: Before upgrading, search for obsolete client and HTTP-management settings. The current product direction is: -- gRPC for application event access. +- gRPC for database client APIs. +- gRPC over the dedicated replication HTTP(S) endpoint for database replication. +- gRPC over the node HTTP(S) endpoint for follower-to-leader forwarding and + cluster coordination. - HTTP for Admin UI, health, metrics, and infrastructure concerns. -- No external TCP client protocol. +- No legacy EventStore TCP client protocol listener. - No proprietary plugin configuration. +Use `ReplicationPortAdvertiseAs` for replication endpoint translation. The +previous `ReplicationTcpPortAdvertiseAs` name remains accepted for configuration +compatibility and produces a deprecation warning. + If a setting is no longer documented, remove it rather than carrying it forward silently. @@ -68,6 +78,19 @@ methods. Avoid depending on undocumented authentication plugins. Use [OpenTelemetry integration](diagnostics/integrations.md) for explicit OTLP export and [Metrics](diagnostics/metrics.md) for Prometheus scraping. +The Admin UI continues to show active connections after the legacy protocol is +removed. The connection table reports the node and replication HTTP/gRPC listeners, while +the replication table reports the database replication sessions and their byte +and queue statistics. The connection table keeps the live paging and +per-second traffic view while identifying HTTP, gRPC, TLS, and the observed +client. These replace the legacy listener-specific TCP table. + +Monitoring clients can use `Monitoring.ConnectionStats` for the same +transport-neutral connection inventory and `Monitoring.ReplicationStats` for +replication-session details. The removed `Monitoring.TcpStats` RPC has no place +in the supported contract because its names and fields described the retired +listener rather than the current HTTP/gRPC surfaces. + ## TestClient review The TestClient keeps command names only when their observable behavior can be @@ -87,6 +110,19 @@ The following commands are intentionally retired: - `CHKTCP`, because it validates a retired frame protocol. Use `CHKGRPC` to validate malformed gRPC-frame handling. +The historical `RT` suite is not an application API. It was an in-process +developer harness coupled to the legacy TCP client, direct TCP packages, local +node process control, and the old projections client. It has no equivalent in +this release. Reintroducing those projection and node-failure scenarios requires +a dedicated gRPC and HTTP implementation rather than routing the `RT` name to a +different workload. + +`ClientMessageDtos.proto` is not a gRPC application contract. It encoded the +payloads placed inside legacy TCP packages, so it is removed with that wire +protocol. The supported replacements are the service contracts under +`src/Protos/Grpc`, including streams, persistent subscriptions, operations, +monitoring, gossip, replication, and request forwarding. + Legacy usage telemetry is separate from OTLP observability. See [Usage telemetry](usage-telemetry.md) before running a node in an environment that should not make outbound telemetry calls. diff --git a/docs/whatsnew.md b/docs/whatsnew.md index 0957f7561..10d6f5df8 100644 --- a/docs/whatsnew.md +++ b/docs/whatsnew.md @@ -6,14 +6,20 @@ not a release-note mirror for another distribution. ## Current baseline - TrogonEventStore is documented as a FOSS-only server distribution. -- Application event access is documented as gRPC-first. +- Database client APIs, cluster replication, and follower-to-leader forwarding + are documented as gRPC-only. - HTTP documentation is limited to browser UI, health probes, metrics, and infrastructure concerns. +- The server has no documented legacy EventStore TCP protocol listener or TCP + transport configuration. - Health probes are exposed on `/-/liveness` and `/-/readiness`. - Prometheus scraping is exposed on `/-/metrics`. - OpenTelemetry documentation covers explicit OTLP export for logs, metrics, and traces. - The Admin UI is the embedded Blazor UI served by the node. +- The Admin UI reports shared-endpoint connections and gRPC replication sessions + without depending on the removed legacy listener. +- EventStore.TestClient operational commands exercise the supported gRPC APIs. - User-defined projection execution, connector runtimes, SQL-like query surfaces, and rich read models are documented as external component work by default. diff --git a/samples/server/docker-compose-cluster.yaml b/samples/server/docker-compose-cluster.yaml index 4b32fc779..249a130c5 100644 --- a/samples/server/docker-compose-cluster.yaml +++ b/samples/server/docker-compose-cluster.yaml @@ -1,6 +1,6 @@ # Before starting, provide certs/ca/ca.crt and node.crt/node.key under each node directory. -# Each node certificate must include localhost, its service DNS name, and its configured replication IP in its -# subject alternative names. The replication IPs are 172.30.240.11, 172.30.240.12, and 172.30.240.13. +# Each node certificate must include localhost, its service DNS name, and its configured node and replication IP in +# its subject alternative names. The node IPs are 172.30.240.11, 172.30.240.12, and 172.30.240.13. # Keep each node private key readable only by the account running the container. services: node1.eventstore: &template @@ -10,6 +10,7 @@ services: env_file: - vars.env environment: + - EVENTSTORE_NODE_IP=172.30.240.11 - EVENTSTORE_REPLICATION_IP=172.30.240.11 - EVENTSTORE_ADVERTISE_NODE_PORT_TO_CLIENT_AS=2111 - EVENTSTORE_GOSSIP_SEED=172.30.240.12:2113,172.30.240.13:2113 @@ -43,6 +44,7 @@ services: <<: *template container_name: node2.eventstore environment: + - EVENTSTORE_NODE_IP=172.30.240.12 - EVENTSTORE_REPLICATION_IP=172.30.240.12 - EVENTSTORE_ADVERTISE_NODE_PORT_TO_CLIENT_AS=2112 - EVENTSTORE_GOSSIP_SEED=172.30.240.11:2113,172.30.240.13:2113 @@ -75,6 +77,7 @@ services: <<: *template container_name: node3.eventstore environment: + - EVENTSTORE_NODE_IP=172.30.240.13 - EVENTSTORE_REPLICATION_IP=172.30.240.13 - EVENTSTORE_ADVERTISE_NODE_PORT_TO_CLIENT_AS=2113 - EVENTSTORE_GOSSIP_SEED=172.30.240.11:2113,172.30.240.12:2113 diff --git a/scripts/test.sh b/scripts/test.sh index 925f020b9..61a454629 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -202,7 +202,7 @@ project_filter() { printf '%s\n' "(FullyQualifiedName~EventStore.Core.Tests.Http|FullyQualifiedName~EventStore.Core.Tests.Services.Transport.Http)" ;; core-services:EventStore.Core.Tests) - printf '%s\n' "((FullyQualifiedName~EventStore.Core.Tests.Services&FullyQualifiedName!~EventStore.Core.Tests.Services.Storage&FullyQualifiedName!~EventStore.Core.Tests.Services.Transport.Http&FullyQualifiedName!~EventStore.Core.Tests.Services.Transport.Grpc.Security&FullyQualifiedName!~EventStore.Core.Tests.Services.Transport.Grpc.ServerFeaturesTests&FullyQualifiedName!~EventStore.Core.Tests.Services.ElectionsService)|FullyQualifiedName~EventStore.Core.Tests.Bus|FullyQualifiedName~EventStore.Core.Tests.Helpers|FullyQualifiedName~EventStore.Core.Tests.ClientOperations|FullyQualifiedName~EventStore.Core.Tests.Authentication|FullyQualifiedName~EventStore.Core.Tests.Authorization|FullyQualifiedName~EventStore.Core.Tests.Certificates|FullyQualifiedName~EventStore.Core.Tests.AwakeService|FullyQualifiedName~EventStore.Core.Tests.Settings|FullyQualifiedName~EventStore.Core.Tests.TcpApiTestPlugin)" + printf '%s\n' "((FullyQualifiedName~EventStore.Core.Tests.Services&FullyQualifiedName!~EventStore.Core.Tests.Services.Storage&FullyQualifiedName!~EventStore.Core.Tests.Services.Transport.Http&FullyQualifiedName!~EventStore.Core.Tests.Services.Transport.Grpc.Security&FullyQualifiedName!~EventStore.Core.Tests.Services.Transport.Grpc.ServerFeaturesTests&FullyQualifiedName!~EventStore.Core.Tests.Services.ElectionsService)|FullyQualifiedName~EventStore.Core.Tests.Bus|FullyQualifiedName~EventStore.Core.Tests.Helpers|FullyQualifiedName~EventStore.Core.Tests.ClientOperations|FullyQualifiedName~EventStore.Core.Tests.Authentication|FullyQualifiedName~EventStore.Core.Tests.Authorization|FullyQualifiedName~EventStore.Core.Tests.Certificates|FullyQualifiedName~EventStore.Core.Tests.AwakeService|FullyQualifiedName~EventStore.Core.Tests.Settings)" ;; core-cluster-services:EventStore.Core.Tests) printf '%s\n' "(FullyQualifiedName~EventStore.Core.Tests.Integration|FullyQualifiedName~EventStore.Core.Tests.Cluster|FullyQualifiedName~EventStore.Core.Tests.Replication|FullyQualifiedName~EventStore.Core.Tests.Synchronization|FullyQualifiedName~EventStore.Core.Tests.Services.ElectionsService|FullyQualifiedName~EventStore.Core.Tests.Services.Transport.Grpc.ServerFeaturesTests)" @@ -217,7 +217,7 @@ project_filter() { printf '%s\n' "FullyQualifiedName~EventStore.Core.Tests.Transforms" ;; core-rest:EventStore.Core.Tests) - printf '%s\n' "FullyQualifiedName!~EventStore.Core.Tests.Http&FullyQualifiedName!~EventStore.Core.Tests.Services&FullyQualifiedName!~EventStore.Core.Tests.Integration&FullyQualifiedName!~EventStore.Core.Tests.Cluster&FullyQualifiedName!~EventStore.Core.Tests.Bus&FullyQualifiedName!~EventStore.Core.Tests.Helpers&FullyQualifiedName!~EventStore.Core.Tests.ClientOperations&FullyQualifiedName!~EventStore.Core.Tests.Authentication&FullyQualifiedName!~EventStore.Core.Tests.Authorization&FullyQualifiedName!~EventStore.Core.Tests.Certificates&FullyQualifiedName!~EventStore.Core.Tests.AwakeService&FullyQualifiedName!~EventStore.Core.Tests.Replication&FullyQualifiedName!~EventStore.Core.Tests.Settings&FullyQualifiedName!~EventStore.Core.Tests.Synchronization&FullyQualifiedName!~EventStore.Core.Tests.TcpApiTestPlugin&FullyQualifiedName!~EventStore.Core.Tests.Index&FullyQualifiedName!~EventStore.Core.Tests.TransactionLog&FullyQualifiedName!~EventStore.Core.Tests.Caching&FullyQualifiedName!~EventStore.Core.Tests.DataStructures&FullyQualifiedName!~EventStore.Core.Tests.Transforms&FullyQualifiedName!~EventStore.Core.Tests.Hashes" + printf '%s\n' "FullyQualifiedName!~EventStore.Core.Tests.Http&FullyQualifiedName!~EventStore.Core.Tests.Services&FullyQualifiedName!~EventStore.Core.Tests.Integration&FullyQualifiedName!~EventStore.Core.Tests.Cluster&FullyQualifiedName!~EventStore.Core.Tests.Bus&FullyQualifiedName!~EventStore.Core.Tests.Helpers&FullyQualifiedName!~EventStore.Core.Tests.ClientOperations&FullyQualifiedName!~EventStore.Core.Tests.Authentication&FullyQualifiedName!~EventStore.Core.Tests.Authorization&FullyQualifiedName!~EventStore.Core.Tests.Certificates&FullyQualifiedName!~EventStore.Core.Tests.AwakeService&FullyQualifiedName!~EventStore.Core.Tests.Replication&FullyQualifiedName!~EventStore.Core.Tests.Settings&FullyQualifiedName!~EventStore.Core.Tests.Synchronization&FullyQualifiedName!~EventStore.Core.Tests.Index&FullyQualifiedName!~EventStore.Core.Tests.TransactionLog&FullyQualifiedName!~EventStore.Core.Tests.Caching&FullyQualifiedName!~EventStore.Core.Tests.DataStructures&FullyQualifiedName!~EventStore.Core.Tests.Transforms&FullyQualifiedName!~EventStore.Core.Tests.Hashes" ;; esac } @@ -233,7 +233,7 @@ project_timeout() { printf '%s\n' "15m" ;; core-services:EventStore.Core.Tests) - printf '%s\n' "15m" + printf '%s\n' "20m" ;; core-cluster-services:EventStore.Core.Tests) printf '%s\n' "25m"