From b3c4744889490bca5bc4a0f530cff3326717b9da Mon Sep 17 00:00:00 2001 From: Jack Carter <128555021+SunsetDrifter@users.noreply.github.com> Date: Thu, 25 Jun 2026 18:15:48 +0200 Subject: [PATCH 1/7] docs: add routing peer sizing guide Add a Sizing Routing Peers page under Networks covering the four-step sizing method, a per-peer capacity table, the tuning levers that matter, and how to scale out by sharding load across identical Networks. Cross-link it from How Routing Peers Work (HA note + related tile) and the Networks overview, and add it to the docs navigation. --- src/components/NavigationDocs.jsx | 4 + .../networks/how-routing-peers-work.mdx | 9 + src/pages/manage/networks/index.mdx | 2 +- .../manage/networks/sizing-routing-peers.mdx | 183 ++++++++++++++++++ 4 files changed, 197 insertions(+), 1 deletion(-) create mode 100644 src/pages/manage/networks/sizing-routing-peers.mdx diff --git a/src/components/NavigationDocs.jsx b/src/components/NavigationDocs.jsx index b9d94f50..b166b010 100644 --- a/src/components/NavigationDocs.jsx +++ b/src/components/NavigationDocs.jsx @@ -198,6 +198,10 @@ export const docsNavigation = [ title: 'How Routing Peers Work', href: '/manage/networks/how-routing-peers-work', }, + { + title: 'Sizing Routing Peers', + href: '/manage/networks/sizing-routing-peers', + }, { title: 'Masquerade', href: '/manage/networks/masquerade', diff --git a/src/pages/manage/networks/how-routing-peers-work.mdx b/src/pages/manage/networks/how-routing-peers-work.mdx index 1b6c88a0..5a15487e 100644 --- a/src/pages/manage/networks/how-routing-peers-work.mdx +++ b/src/pages/manage/networks/how-routing-peers-work.mdx @@ -116,6 +116,10 @@ Useful when routing peers are geographically distributed and you want each clien Place highly available peers in different failure domains within the same network: separate AZs in cloud, separate hypervisors or hosts on-prem. + +High availability is failover and nearest-peer selection, not load balancing — it does not spread one network's traffic across its peers. To carry sustained high throughput, see [Sizing Routing Peers](/manage/networks/sizing-routing-peers) for how to size each peer and shard load across multiple Networks. + + ## Masquerade Masquerade is on by default. The routing peer SNATs forwarded traffic to its own LAN-side IP. This is the simplest configuration because the destination network does not need any awareness of NetBird. @@ -240,6 +244,11 @@ Specifics: name: 'Networks', description: 'The newer, recommended way to configure routing peers and resources', }, + { + href: '/manage/networks/sizing-routing-peers', + name: 'Sizing Routing Peers', + description: 'Choose the size and number of routing peers from measured throughput', + }, { href: '/manage/network-routes', name: 'Routes', diff --git a/src/pages/manage/networks/index.mdx b/src/pages/manage/networks/index.mdx index 16c76e29..6f65393a 100644 --- a/src/pages/manage/networks/index.mdx +++ b/src/pages/manage/networks/index.mdx @@ -77,7 +77,7 @@ Unlike NetBird peers, resources are **not** automatically members of the built-i A routing peer is a NetBird client installed inside the private network that forwards traffic from the overlay network to devices that don't run the client. Any NetBird client can be one (Linux, Windows, macOS, and others), so the role is about where the machine sits, not which OS it runs. For the full picture, see [How Routing Peers Work](/manage/networks/how-routing-peers-work). -A simple routing peer needs little: 2 vCPUs and 4 GB of RAM is a good baseline, and a small VM or even a Raspberry Pi will do for light use. Scale up as network throughput, link saturation, and the number of users grow. +A simple routing peer needs little: 2 vCPUs and 4 GB of RAM is a good baseline, and a small VM or even a Raspberry Pi will do for light use. Scale up as network throughput, link saturation, and the number of users grow; [Sizing Routing Peers](/manage/networks/sizing-routing-peers) gives a method for high-throughput deployments. Because routing peers are usually headless servers, register them with [setup keys](/manage/peers/register-machines-using-setup-keys) instead of interactive login. Setup-key peers aren't subject to [login session expiration](/manage/settings/enforce-periodic-user-authentication), so a routing peer stays connected without periodic re-authentication. That's exactly what an always-on gateway needs. You can assign several routing peers to one Network for high availability; see the [production checklist](#production-checklist). diff --git a/src/pages/manage/networks/sizing-routing-peers.mdx b/src/pages/manage/networks/sizing-routing-peers.mdx new file mode 100644 index 00000000..77bfd810 --- /dev/null +++ b/src/pages/manage/networks/sizing-routing-peers.mdx @@ -0,0 +1,183 @@ +import { Note, Warning } from '@/components/mdx' +import { Tiles } from '@/components/Tiles' + +export const description = "Choose the size and number of NetBird routing peers from measured throughput: a four-step method, a per-peer capacity table, the tuning levers that matter, and how to scale out with sharded Networks." + +# Sizing Routing Peers + +You have put a routing peer in front of something a lot of people need: a datacenter, a cloud VPC, an office LAN. Every packet those people send or receive now crosses that one machine, and the machine does cryptographic work on each packet. Size it too small and you throttle everyone behind it; size it too large and you pay for CPU cores that sit idle. + +This guide gives you a method to choose the size and number of routing peers from measured numbers, plus the few settings that actually change the result. + +If your network is small you can stop reading here: a routing peer on 2 vCPUs and 4 GB of RAM comfortably serves light use, and you do not need to tune anything. Read on when you expect sustained high throughput, thousands of simultaneously active users, or a datacenter-scale link. + +Throughout, we will follow one example. **Acme has 6,000 people in one office**, pulling builds, code, and large files from servers in an on-site datacenter that sits behind NetBird routing peers. By the end we will know how many peers Acme needs and how big each one should be. + +## How a routing peer spends its capacity + +Picture a routing peer as a checkpoint that every packet has to cross. Traffic coming from a user was encrypted on the user's device, and the peer unwraps it (decrypts it) before handing it to the datacenter. Traffic going back to the user is wrapped (encrypted) at the peer. That wrapping and unwrapping is the work, and the peer's CPU does it one packet at a time, spread across its cores. + +Four facts follow from that picture, and they drive every decision below. + +- **One device is one tunnel.** Each connected client holds a single encrypted WireGuard tunnel to the peer (a "tunnel" here is one client's persistent encrypted connection). Every app on that device shares that one tunnel; they do not each open their own. +- **The peer does crypto for every byte, so throughput is the limit, not the head count.** A peer moving traffic at speed runs out of CPU long before it runs out of room for registered users. +- **Connections are cheap; throughput is the constraint.** An open but idle tunnel costs very little memory and almost no CPU. What you run out of first is the capacity to move bytes, so you size against traffic, not against the number of people enrolled. +- **One tunnel is about one CPU core.** A single client's tunnel is handled largely on one core, so it tops out at a few Gbps no matter how many cores the peer has. Total capacity comes from spreading many tunnels across all the cores, not from any one tunnel going fast. + +The method is four steps: estimate the peak traffic, read what one peer of a given size can carry, divide to get the number of peers, then split your users across them. + +## First, watch the direction + +One distinction causes most sizing mistakes, so we meet it before the method. The words "download" and "upload" are from the user's point of view, and they reach the peer as opposite jobs: + +- A user **pulling** data from the network (opening files, loading apps, fetching builds: the common case) makes the peer **encrypt**. +- A user **pushing** data into the network (uploading, backing up, writing) makes the peer **decrypt**. + +The two run at different speeds. On the same hardware the pulling (encrypt) direction reaches roughly twice the throughput of the pushing (decrypt) direction, because the encrypt path can ride the operating system's segmentation offload and the decrypt path cannot. So the number that sets your peer count is not "how much traffic" but "how much traffic in the direction my users actually use." Acme's people mostly pull, so we will size on the download (encrypt) figure and treat the upload (decrypt) figure as a floor. + +## Step 1: estimate peak throughput + +``` +peak throughput = (peak simultaneously active users) x (typical per-user rate) +``` + +Use the **active** count, not the enrolled count. This is the most common mistake: sizing for everyone who has the client installed rather than for the few who are actually moving data at the busy minute. + +Acme has 6,000 people enrolled, but at the peak hour about half are pulling data, roughly 3,000 of them, at about 10 Mbps each: `3,000 x 10 Mbps = 30 Gbps`. + +## Step 2: read a per-peer capacity + +Pick a candidate size and read its throughput from the [capacity table](#per-peer-capacity-reference) below, or measure it in your own environment. A 16 vCPU peer carries about 20 Gbps in the download (encrypt) direction, which is the direction Acme needs. + +## Step 3: work out how many active peers + +``` +active peers = ceil(peak throughput / per-peer capacity) +``` + +Acme needs `ceil(30 / 20) = 2` peers actively carrying load. Step 4 turns that into a peer count once we add failover. + +## Step 4: give each active peer its own Network + +Here is the subtlety that catches people. NetBird high availability does **not** spread one Network's traffic across its routing peers (see [Scaling out](#scaling-out-high-availability-and-sharding)): inside a single Network every client picks one peer and the rest stand by. So you cannot split load by piling routing peers into one Network. You split it by building **more Networks**, one per shard. + +Make each active peer its own NetBird Network; have every Network expose the **same resources** (the same datacenter, the same `/32` hosts); and give every Network **two or more routing peers** so it carries its own failover. Then a policy sends each client group to a different Network. Because the Networks ride different routing peers, splitting clients across Networks splits the load across peers, by design. + +Acme builds **two identical Networks**, each exposing the datacenter, each with **two routing peers** (one carrying load, one on standby): `2 Networks x 2 peers = 4 peers`. It puts 3,000 people in each of two client groups, and a policy points each group at one Network. Each Network carries about 15 of Acme's 30 Gbps, comfortably inside a 16-core peer's 20, and either Network survives losing a peer. + +That is the whole method. The rest of this guide is the capacity table those steps lean on, the levers that raise a single peer's number, and how to scale past one peer. + +## Per-peer capacity reference + +Representative measured throughput per routing-peer size. Every figure here was measured on AWS **c6in.16xlarge** instances: 64 vCPUs and a 100 Gbps interface, built on 3rd-generation Intel Xeon Scalable processors (Ice Lake) with AVX-512 and an all-core turbo of 3.5 GHz. The peer ran Linux kernel WireGuard with threaded NAPI on (the kernel default), the default 1280-byte tunnel MTU, and direct connections, with load spread across many tunnels. The smaller core counts (4 to 32) were produced by taking cores offline on that same 64-core box, so each row isolates the effect of core count rather than mixing hardware. The "Network interface" column is the minimum interface a real peer of that size should be paired with, not a separately tested NIC. Expect roughly 20% variation, and confirm in your own environment. + +| vCPUs | Network interface | Download (clients pulling) | Upload (clients pushing) | Active users at 5 Mbps | +|---|---|---|---|---| +| 4 | 10 Gbps or more | ~7 Gbps | ~4 Gbps | ~1,400 | +| 8 | 15 Gbps or more | ~13 Gbps | ~11 Gbps | ~2,600 | +| 16 | 25 Gbps or more | ~20 Gbps | ~15 Gbps | ~4,000 | +| 32 | 50 Gbps or more | ~25 Gbps | ~15 Gbps | ~5,000 | +| 64 | 100 Gbps | ~31 Gbps | ~15 Gbps | ~6,000 | + + +**These are TCP figures.** They reflect bulk **TCP** transfers, what real file pulls and pushes over HTTPS look like, which the operating system accelerates with segmentation offload so the peer handles large segments and does less per-packet work. Throughput is mostly bounded by packets per second, so traffic made of many **small packets** (lots of tiny requests, or high-rate real-time and UDP streams) runs materially lower, roughly half the download figure in our small-fixed-packet tests. Size conservatively for those workloads, and measure your own mix. + + +Two things to read from this table: + +- **Download keeps scaling with cores; upload flattens around 15 Gbps at about 16 cores** and barely improves past that, because the decrypt path is harder to spread across cores. Beyond ~16 cores you gain mostly on the download direction. +- **16 to 32 vCPUs is the practical sweet spot.** A 16-core peer reaches close to a 64-core peer's throughput, so prefer **more modest peers over fewer large ones** (see [Scaling out](#scaling-out-high-availability-and-sharding)). + +The interface column is the smallest network interface that will not bottleneck that throughput. A single client tunnel tops out near one CPU core (a few Gbps), and clouds cap a single flow lower still (see [Cloud and virtualization notes](#cloud-and-virtualization-notes)), so capacity comes from many tunnels in parallel, not one fast one. + +**On small or commodity hardware, the interface is the limit, not the cores.** The vCPUs in this table are server-class: high clock, wide vector units, fast memory, and a 10 Gbps-or-faster interface beside them. A homelab box is a different shape. A mini PC with four low-power cores and a 1 or 2.5 GbE port runs out of *interface* long before it runs out of CPU, so do not read it against the 4 vCPU row and expect 7 Gbps. Read your ceiling as the **smaller of the interface line rate and the CPU's capacity**: on such a box that is the port, often a couple of Gbps or less. Plan for roughly the port's rate, and below it on a single-port box, which carries both the encrypted tunnel traffic and the plaintext it forwards over the same interface. Two things follow: the download-versus-upload gap closes, because a saturated port limits both directions equally; and adding cores buys nothing until the interface is faster. Kernel WireGuard still matters as much as it does on a large peer. + +## Tuning for more throughput + +These levers raise the capacity of one routing peer before you add more. + +- **Use kernel WireGuard.** On Linux, NetBird uses the in-kernel WireGuard module by default when it is present, and it is the right choice for a high-throughput gateway: in testing it ran several times faster than the userspace fallback, because only the kernel path spreads the work across all cores. Make sure the `wireguard` kernel module is available. +- **Match the network interface to the target.** At high aggregate rates the interface can limit you as much as the CPU. For tens of Gbps, choose an interface (and, in a cloud, an instance type) whose bandwidth matches the throughput you read from the table. +- **Raise the MTU only on clean networks.** Throughput is largely bound by packets per second, so larger packets carry more for the same work. On a network that supports jumbo frames end to end, such as a datacenter backplane, a larger tunnel MTU raised per-peer throughput several-fold in testing. This helps only when the **entire path, clients included, supports the larger MTU**. Keep the default on paths that cross the public internet or reach ordinary 1500-byte clients, where it would only cause fragmentation. +- **Add cores up to the knee, then stop.** Cores raise capacity up to about 16; past that the download direction inches up and the upload direction does not move at all. When you reach that knee, add another peer instead of a bigger one. + +## Scaling out: high availability and sharding + +A single peer has a ceiling. To go past it, run more than one, but understand what NetBird high availability does and does not do. + +It gives you two things: + +- **Failover.** Give the peers in a Network different metrics; clients use the lowest-metric peer and switch to the next automatically if it goes down. +- **Nearest-peer selection.** Give the peers equal metrics, and each client uses the lowest-latency peer, switching only when the latency difference is meaningful (more than 20 ms). + +See [How Routing Peers Work](/manage/networks/how-routing-peers-work#high-availability) for the mechanism behind both. + +Neither of these balances one Network's load across its peers. This is the second mistake to avoid: expecting high availability to split traffic evenly. It does not. To spread load on purpose you run several **identical Networks**: each exposes the same resources, each rides its own routing peers, and a policy sends a different client group to each. Splitting the clients across Networks is what splits the load, because each Network uses different peers. Give every Network two or more peers so each keeps its own failover. + +A datacenter-scale worked example: to carry roughly 100 Gbps of downloads with 16-to-32-core peers that each sustain about 20 to 25 Gbps, you would build four or five identical Networks (one per active peer's worth of load), each an HA pair of peers, so eight to ten peers in total, and send a different client group to each. If the whole path supports jumbo frames end to end, far fewer Networks carry the same load. + +## Cloud and virtualization notes + +- **A single flow is capped below the peer's total.** Many clouds limit one network flow to a few Gbps (on AWS, about 5 Gbps, or near 10 Gbps inside a cluster placement group). Because one client tunnel is a single flow, this caps any single tunnel regardless of how large the peer is, which is another reason capacity comes from many tunnels rather than one fast one. +- **Pick a network-optimized instance for tens of Gbps.** A general-purpose instance's "up to" burst bandwidth is not the sustained rate; size on the sustained figure. +- **Jumbo frames inside a VPC or datacenter.** Within one cloud network or datacenter, jumbo frames are usually available end to end, which is where the MTU lever above applies. Across the public internet the usable MTU drops, so keep the default there. +- **No inbound ports are needed.** A routing peer makes outbound connections and relies on NetBird for reachability, so you never open an inbound port to it. Direct (peer-to-peer) connections give the best throughput; when a direct path is not available NetBird falls back to an encrypted relay, which is the designed answer rather than a port-forward. + +## Measure in your own environment + +Numbers vary with hardware and traffic, so validate before you commit to a design. Use `iperf3` version 3.16 or newer; older builds are single-threaded and will cap a fast interface. + +On a host inside the network, behind the routing peer, run one server per client connection: + +```bash +iperf3 -s -p 5201 # repeat on 5202, 5203, ... for more parallel clients +``` + +From each test client, drive load in the direction your users actually use: + +```bash +# Download (clients pulling from the network, the peer encrypts): +iperf3 -c -p 5201 -t 30 -P 4 -R + +# Upload (clients pushing into the network, the peer decrypts): +iperf3 -c -p 5201 -t 30 -u -b 0 -l 1250 # keep the UDP datagram below the tunnel MTU +``` + +Read the real throughput on the routing peer itself, from its WireGuard interface, while watching CPU: + +```bash +cat /sys/class/net/wt0/statistics/rx_bytes # also tx_bytes; Gbps = byte-delta x 8 / 1e9 / seconds +mpstat -P ALL 1 # cores saturated means CPU bound; cores idle means link or other +``` + +Reading throughput at the WireGuard interface (rather than trusting one client's number) confirms the traffic actually went through the tunnel and captures the peer's true aggregate. The point where throughput stops rising as you add load is that peer's capacity. Use it to decide whether to **scale up** (a bigger peer, if CPU and interface still have headroom) or **scale out** (more peers, if a single peer is at its ceiling). + +## In one breath + +- Size against **aggregate active throughput in the direction your users use**, not the enrolled head count. Acme: 3,000 active pullers at 10 Mbps is 30 Gbps of download. +- **One tunnel is about one core**; capacity comes from **many tunnels across cores**, and **16 to 32 cores** reaches nearly the same throughput as far larger boxes. +- `active peers = ceil(aggregate / per-peer capacity)`, then give each its own **identical NetBird Network** (same resources, two or more peers for failover) and split clients across the Networks by policy. Acme: `ceil(30 / 20) = 2` Networks, each an HA pair, so **four** 16-core peers, 3,000 users per Network. +- The levers, in order of effect: **kernel WireGuard**, a **matched interface**, **jumbo frames on a clean path**, and **cores up to ~16**. High availability is **failover and nearest-peer selection, not load balancing**. +- On **small or commodity hardware** (a homelab mini PC, a 1 or 2.5 GbE box), the **interface** is the limit before the CPU; size to the smaller of the two, and expect download and upload to converge. +- When unsure, **measure** at your candidate size and watch where CPU or the interface saturates. + + From 18ee39fbc3c971dfad2e84e1355b214aeaaa1442 Mon Sep 17 00:00:00 2001 From: Jack Carter <128555021+SunsetDrifter@users.noreply.github.com> Date: Fri, 26 Jun 2026 11:13:36 +0200 Subject: [PATCH 2/7] docs: refine wording in routing peer sizing guide Generalize the Acme example to remote users, correct the encrypt/decrypt framing and reach the local network rather than the datacenter, use 'routing peer' instead of 'gateway', and rename the recap to Summary. --- src/pages/manage/networks/sizing-routing-peers.mdx | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/pages/manage/networks/sizing-routing-peers.mdx b/src/pages/manage/networks/sizing-routing-peers.mdx index 77bfd810..3e5d9ec9 100644 --- a/src/pages/manage/networks/sizing-routing-peers.mdx +++ b/src/pages/manage/networks/sizing-routing-peers.mdx @@ -11,11 +11,11 @@ This guide gives you a method to choose the size and number of routing peers fro If your network is small you can stop reading here: a routing peer on 2 vCPUs and 4 GB of RAM comfortably serves light use, and you do not need to tune anything. Read on when you expect sustained high throughput, thousands of simultaneously active users, or a datacenter-scale link. -Throughout, we will follow one example. **Acme has 6,000 people in one office**, pulling builds, code, and large files from servers in an on-site datacenter that sits behind NetBird routing peers. By the end we will know how many peers Acme needs and how big each one should be. +Throughout, we will follow one example. **Acme has 6,000 remote users**, pulling builds, code, and large files from servers in an on-site datacenter that sits behind NetBird routing peers. By the end we will know how many peers Acme needs and how big each one should be. ## How a routing peer spends its capacity -Picture a routing peer as a checkpoint that every packet has to cross. Traffic coming from a user was encrypted on the user's device, and the peer unwraps it (decrypts it) before handing it to the datacenter. Traffic going back to the user is wrapped (encrypted) at the peer. That wrapping and unwrapping is the work, and the peer's CPU does it one packet at a time, spread across its cores. +Picture a routing peer as a checkpoint that every packet has to cross. Traffic coming from a user was encrypted on the user's device, and the peer unwraps it (decrypts it) before handing it to the local network. Traffic going back to the user is wrapped (encrypted) at the peer. That decryption and encryption is the work, and the peer's CPU does it one packet at a time, spread across its cores. Four facts follow from that picture, and they drive every decision below. @@ -96,7 +96,7 @@ The interface column is the smallest network interface that will not bottleneck These levers raise the capacity of one routing peer before you add more. -- **Use kernel WireGuard.** On Linux, NetBird uses the in-kernel WireGuard module by default when it is present, and it is the right choice for a high-throughput gateway: in testing it ran several times faster than the userspace fallback, because only the kernel path spreads the work across all cores. Make sure the `wireguard` kernel module is available. +- **Use kernel WireGuard.** On Linux, NetBird uses the in-kernel WireGuard module by default when it is present, and it is the right choice for a high-throughput routing peer: in testing it ran several times faster than the userspace fallback, because only the kernel path spreads the work across all cores. - **Match the network interface to the target.** At high aggregate rates the interface can limit you as much as the CPU. For tens of Gbps, choose an interface (and, in a cloud, an instance type) whose bandwidth matches the throughput you read from the table. - **Raise the MTU only on clean networks.** Throughput is largely bound by packets per second, so larger packets carry more for the same work. On a network that supports jumbo frames end to end, such as a datacenter backplane, a larger tunnel MTU raised per-peer throughput several-fold in testing. This helps only when the **entire path, clients included, supports the larger MTU**. Keep the default on paths that cross the public internet or reach ordinary 1500-byte clients, where it would only cause fragmentation. - **Add cores up to the knee, then stop.** Cores raise capacity up to about 16; past that the download direction inches up and the upload direction does not move at all. When you reach that knee, add another peer instead of a bigger one. @@ -152,7 +152,7 @@ mpstat -P ALL 1 # cores saturated means CPU bound Reading throughput at the WireGuard interface (rather than trusting one client's number) confirms the traffic actually went through the tunnel and captures the peer's true aggregate. The point where throughput stops rising as you add load is that peer's capacity. Use it to decide whether to **scale up** (a bigger peer, if CPU and interface still have headroom) or **scale out** (more peers, if a single peer is at its ceiling). -## In one breath +## Summary - Size against **aggregate active throughput in the direction your users use**, not the enrolled head count. Acme: 3,000 active pullers at 10 Mbps is 30 Gbps of download. - **One tunnel is about one core**; capacity comes from **many tunnels across cores**, and **16 to 32 cores** reaches nearly the same throughput as far larger boxes. From fc5ea2af2e57db337407d5b608df599e71729792 Mon Sep 17 00:00:00 2001 From: Jack Carter <128555021+SunsetDrifter@users.noreply.github.com> Date: Fri, 26 Jun 2026 11:24:24 +0200 Subject: [PATCH 3/7] docs: tighten and correct HA behavior in routing peer sizing guide MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Correct the high-availability description: a single Network does not balance load across its peers — different metrics give failover (one peer carries all), equal metrics give latency-based nearest-peer selection, which splits traffic by geography but never evenly. Shard into more Networks to split load deterministically. Also collapse redundant restatements, drop the secondary worked example (the capacity table covers it), and slim the commodity-hardware guidance. --- .../manage/networks/sizing-routing-peers.mdx | 32 ++++++------------- 1 file changed, 9 insertions(+), 23 deletions(-) diff --git a/src/pages/manage/networks/sizing-routing-peers.mdx b/src/pages/manage/networks/sizing-routing-peers.mdx index 3e5d9ec9..d4138a34 100644 --- a/src/pages/manage/networks/sizing-routing-peers.mdx +++ b/src/pages/manage/networks/sizing-routing-peers.mdx @@ -7,7 +7,7 @@ export const description = "Choose the size and number of NetBird routing peers You have put a routing peer in front of something a lot of people need: a datacenter, a cloud VPC, an office LAN. Every packet those people send or receive now crosses that one machine, and the machine does cryptographic work on each packet. Size it too small and you throttle everyone behind it; size it too large and you pay for CPU cores that sit idle. -This guide gives you a method to choose the size and number of routing peers from measured numbers, plus the few settings that actually change the result. +This guide gives you a method to choose the size and number of routing peers from measured numbers, plus the few settings that change the result. If your network is small you can stop reading here: a routing peer on 2 vCPUs and 4 GB of RAM comfortably serves light use, and you do not need to tune anything. Read on when you expect sustained high throughput, thousands of simultaneously active users, or a datacenter-scale link. @@ -20,8 +20,7 @@ Picture a routing peer as a checkpoint that every packet has to cross. Traffic c Four facts follow from that picture, and they drive every decision below. - **One device is one tunnel.** Each connected client holds a single encrypted WireGuard tunnel to the peer (a "tunnel" here is one client's persistent encrypted connection). Every app on that device shares that one tunnel; they do not each open their own. -- **The peer does crypto for every byte, so throughput is the limit, not the head count.** A peer moving traffic at speed runs out of CPU long before it runs out of room for registered users. -- **Connections are cheap; throughput is the constraint.** An open but idle tunnel costs very little memory and almost no CPU. What you run out of first is the capacity to move bytes, so you size against traffic, not against the number of people enrolled. +- **Throughput is the constraint, not the head count.** The peer does crypto for every byte, so it runs out of CPU long before it runs out of room for users. An open but idle tunnel costs almost nothing, so you size against traffic, not the number of people enrolled. - **One tunnel is about one CPU core.** A single client's tunnel is handled largely on one core, so it tops out at a few Gbps no matter how many cores the peer has. Total capacity comes from spreading many tunnels across all the cores, not from any one tunnel going fast. The method is four steps: estimate the peak traffic, read what one peer of a given size can carry, divide to get the number of peers, then split your users across them. @@ -59,17 +58,15 @@ Acme needs `ceil(30 / 20) = 2` peers actively carrying load. Step 4 turns that i ## Step 4: give each active peer its own Network -Here is the subtlety that catches people. NetBird high availability does **not** spread one Network's traffic across its routing peers (see [Scaling out](#scaling-out-high-availability-and-sharding)): inside a single Network every client picks one peer and the rest stand by. So you cannot split load by piling routing peers into one Network. You split it by building **more Networks**, one per shard. +Here is the subtlety that catches people. A single Network does not split its load across its routing peers: NetBird high availability either pins every client to one peer (different metrics) or sends each client to its nearest peer by latency (equal metrics) — neither evenly divides a busy site's traffic. So you split load by building **more Networks**, one per shard. Make each active peer its own NetBird Network; have every Network expose the **same resources** (the same datacenter, the same `/32` hosts); and give every Network **two or more routing peers** so it carries its own failover. Then a policy sends each client group to a different Network. Because the Networks ride different routing peers, splitting clients across Networks splits the load across peers, by design. Acme builds **two identical Networks**, each exposing the datacenter, each with **two routing peers** (one carrying load, one on standby): `2 Networks x 2 peers = 4 peers`. It puts 3,000 people in each of two client groups, and a policy points each group at one Network. Each Network carries about 15 of Acme's 30 Gbps, comfortably inside a 16-core peer's 20, and either Network survives losing a peer. -That is the whole method. The rest of this guide is the capacity table those steps lean on, the levers that raise a single peer's number, and how to scale past one peer. - ## Per-peer capacity reference -Representative measured throughput per routing-peer size. Every figure here was measured on AWS **c6in.16xlarge** instances: 64 vCPUs and a 100 Gbps interface, built on 3rd-generation Intel Xeon Scalable processors (Ice Lake) with AVX-512 and an all-core turbo of 3.5 GHz. The peer ran Linux kernel WireGuard with threaded NAPI on (the kernel default), the default 1280-byte tunnel MTU, and direct connections, with load spread across many tunnels. The smaller core counts (4 to 32) were produced by taking cores offline on that same 64-core box, so each row isolates the effect of core count rather than mixing hardware. The "Network interface" column is the minimum interface a real peer of that size should be paired with, not a separately tested NIC. Expect roughly 20% variation, and confirm in your own environment. +Representative measured throughput per routing-peer size, all on AWS **c6in.16xlarge** instances: 64 vCPUs and a 100 Gbps interface on 3rd-generation Intel Xeon Scalable (Ice Lake) with AVX-512 and a 3.5 GHz all-core turbo. The peer ran Linux kernel WireGuard with threaded NAPI (the kernel default), the default 1280-byte tunnel MTU, and direct connections, with load spread across many tunnels. Smaller core counts (4 to 32) were produced by taking cores offline on that same box, so each row isolates the effect of core count. The Network interface column is the minimum a real peer of that size should be paired with, not a separately tested NIC. Expect roughly 20% variation, and confirm in your own environment. | vCPUs | Network interface | Download (clients pulling) | Upload (clients pushing) | Active users at 5 Mbps | |---|---|---|---|---| @@ -88,9 +85,7 @@ Two things to read from this table: - **Download keeps scaling with cores; upload flattens around 15 Gbps at about 16 cores** and barely improves past that, because the decrypt path is harder to spread across cores. Beyond ~16 cores you gain mostly on the download direction. - **16 to 32 vCPUs is the practical sweet spot.** A 16-core peer reaches close to a 64-core peer's throughput, so prefer **more modest peers over fewer large ones** (see [Scaling out](#scaling-out-high-availability-and-sharding)). -The interface column is the smallest network interface that will not bottleneck that throughput. A single client tunnel tops out near one CPU core (a few Gbps), and clouds cap a single flow lower still (see [Cloud and virtualization notes](#cloud-and-virtualization-notes)), so capacity comes from many tunnels in parallel, not one fast one. - -**On small or commodity hardware, the interface is the limit, not the cores.** The vCPUs in this table are server-class: high clock, wide vector units, fast memory, and a 10 Gbps-or-faster interface beside them. A homelab box is a different shape. A mini PC with four low-power cores and a 1 or 2.5 GbE port runs out of *interface* long before it runs out of CPU, so do not read it against the 4 vCPU row and expect 7 Gbps. Read your ceiling as the **smaller of the interface line rate and the CPU's capacity**: on such a box that is the port, often a couple of Gbps or less. Plan for roughly the port's rate, and below it on a single-port box, which carries both the encrypted tunnel traffic and the plaintext it forwards over the same interface. Two things follow: the download-versus-upload gap closes, because a saturated port limits both directions equally; and adding cores buys nothing until the interface is faster. Kernel WireGuard still matters as much as it does on a large peer. +**On small or commodity hardware, the interface is the limit, not the cores.** A homelab mini PC with a 1 or 2.5 GbE port runs out of *interface* long before CPU, so do not read it against the table's CPU rows — your ceiling is the port's line rate, often a couple of Gbps or less, where download and upload converge. Kernel WireGuard still matters. ## Tuning for more throughput @@ -103,18 +98,9 @@ These levers raise the capacity of one routing peer before you add more. ## Scaling out: high availability and sharding -A single peer has a ceiling. To go past it, run more than one, but understand what NetBird high availability does and does not do. - -It gives you two things: - -- **Failover.** Give the peers in a Network different metrics; clients use the lowest-metric peer and switch to the next automatically if it goes down. -- **Nearest-peer selection.** Give the peers equal metrics, and each client uses the lowest-latency peer, switching only when the latency difference is meaningful (more than 20 ms). - -See [How Routing Peers Work](/manage/networks/how-routing-peers-work#high-availability) for the mechanism behind both. - -Neither of these balances one Network's load across its peers. This is the second mistake to avoid: expecting high availability to split traffic evenly. It does not. To spread load on purpose you run several **identical Networks**: each exposes the same resources, each rides its own routing peers, and a policy sends a different client group to each. Splitting the clients across Networks is what splits the load, because each Network uses different peers. Give every Network two or more peers so each keeps its own failover. +A single peer has a ceiling. To go past it, run more than one, but understand what NetBird high availability does and does not do. It gives you two things: **failover** (different metrics: one peer carries all, the rest are reserve) and **nearest-peer selection** (equal metrics: each client uses its lowest-latency peer, switching only past a 20 ms difference). See [How Routing Peers Work](/manage/networks/how-routing-peers-work#high-availability) for the mechanism behind both. -A datacenter-scale worked example: to carry roughly 100 Gbps of downloads with 16-to-32-core peers that each sustain about 20 to 25 Gbps, you would build four or five identical Networks (one per active peer's worth of load), each an HA pair of peers, so eight to ten peers in total, and send a different client group to each. If the whole path supports jumbo frames end to end, far fewer Networks carry the same load. +Neither balances a Network's load. Nearest-peer *does* spread traffic when peers sit in different regions and users are spread among them — EU users on the EU peer, US users on the US — but it splits by geography, not evenly, and not at all when peers sit together. To split load deterministically, shard into more identical Networks as in [Step 4](#step-4-give-each-active-peer-its-own-network). ## Cloud and virtualization notes @@ -150,14 +136,14 @@ cat /sys/class/net/wt0/statistics/rx_bytes # also tx_bytes; Gbps = byte-delta mpstat -P ALL 1 # cores saturated means CPU bound; cores idle means link or other ``` -Reading throughput at the WireGuard interface (rather than trusting one client's number) confirms the traffic actually went through the tunnel and captures the peer's true aggregate. The point where throughput stops rising as you add load is that peer's capacity. Use it to decide whether to **scale up** (a bigger peer, if CPU and interface still have headroom) or **scale out** (more peers, if a single peer is at its ceiling). +Reading at the WireGuard interface, not one client's number, captures the peer's true aggregate. The point where throughput stops rising as you add load is that peer's capacity: decide whether to **scale up** (a bigger peer, if CPU and interface still have headroom) or **scale out** (more peers, if it is at its ceiling). ## Summary - Size against **aggregate active throughput in the direction your users use**, not the enrolled head count. Acme: 3,000 active pullers at 10 Mbps is 30 Gbps of download. - **One tunnel is about one core**; capacity comes from **many tunnels across cores**, and **16 to 32 cores** reaches nearly the same throughput as far larger boxes. - `active peers = ceil(aggregate / per-peer capacity)`, then give each its own **identical NetBird Network** (same resources, two or more peers for failover) and split clients across the Networks by policy. Acme: `ceil(30 / 20) = 2` Networks, each an HA pair, so **four** 16-core peers, 3,000 users per Network. -- The levers, in order of effect: **kernel WireGuard**, a **matched interface**, **jumbo frames on a clean path**, and **cores up to ~16**. High availability is **failover and nearest-peer selection, not load balancing**. +- The levers, in order of effect: **kernel WireGuard**, a **matched interface**, **jumbo frames on a clean path**, and **cores up to ~16**. High availability is **failover or nearest-peer selection, never load balancing**; split load by sharding into more Networks. - On **small or commodity hardware** (a homelab mini PC, a 1 or 2.5 GbE box), the **interface** is the limit before the CPU; size to the smaller of the two, and expect download and upload to converge. - When unsure, **measure** at your candidate size and watch where CPU or the interface saturates. From 7bb3650252cbafe810ae592d21dd2f5c967c2435 Mon Sep 17 00:00:00 2001 From: Jack Carter <128555021+SunsetDrifter@users.noreply.github.com> Date: Fri, 26 Jun 2026 11:39:29 +0200 Subject: [PATCH 4/7] docs: add 1- and 2-vCPU rows to the routing peer capacity table Extend the capacity table down to 1 and 2 vCPUs, drop the 'or more' from the interface column, and adjust the methodology note so the interface column reads uniformly as the minimum NIC to pair with each size. --- src/pages/manage/networks/sizing-routing-peers.mdx | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/pages/manage/networks/sizing-routing-peers.mdx b/src/pages/manage/networks/sizing-routing-peers.mdx index d4138a34..f9b2534d 100644 --- a/src/pages/manage/networks/sizing-routing-peers.mdx +++ b/src/pages/manage/networks/sizing-routing-peers.mdx @@ -66,14 +66,16 @@ Acme builds **two identical Networks**, each exposing the datacenter, each with ## Per-peer capacity reference -Representative measured throughput per routing-peer size, all on AWS **c6in.16xlarge** instances: 64 vCPUs and a 100 Gbps interface on 3rd-generation Intel Xeon Scalable (Ice Lake) with AVX-512 and a 3.5 GHz all-core turbo. The peer ran Linux kernel WireGuard with threaded NAPI (the kernel default), the default 1280-byte tunnel MTU, and direct connections, with load spread across many tunnels. Smaller core counts (4 to 32) were produced by taking cores offline on that same box, so each row isolates the effect of core count. The Network interface column is the minimum a real peer of that size should be paired with, not a separately tested NIC. Expect roughly 20% variation, and confirm in your own environment. +Representative measured throughput per routing-peer size, all measured on AWS **c6in.16xlarge** instances: 64 vCPUs and a 100 Gbps interface on 3rd-generation Intel Xeon Scalable (Ice Lake) with AVX-512 and a 3.5 GHz all-core turbo, running Linux kernel WireGuard with threaded NAPI (the kernel default), the default 1280-byte tunnel MTU, and direct connections, with load spread across many tunnels. Smaller core counts were produced by taking cores offline on that same box, so each row isolates the effect of core count. The Network interface column is the minimum interface to pair with a peer of that size so it does not bottleneck the result, not a separately tested NIC. Expect roughly 20% variation, and confirm in your own environment. | vCPUs | Network interface | Download (clients pulling) | Upload (clients pushing) | Active users at 5 Mbps | |---|---|---|---|---| -| 4 | 10 Gbps or more | ~7 Gbps | ~4 Gbps | ~1,400 | -| 8 | 15 Gbps or more | ~13 Gbps | ~11 Gbps | ~2,600 | -| 16 | 25 Gbps or more | ~20 Gbps | ~15 Gbps | ~4,000 | -| 32 | 50 Gbps or more | ~25 Gbps | ~15 Gbps | ~5,000 | +| 1 | 2.5 Gbps | ~2 Gbps | ~2 Gbps | ~400 | +| 2 | 5 Gbps | ~3 Gbps | ~3 Gbps | ~580 | +| 4 | 10 Gbps | ~7 Gbps | ~4 Gbps | ~1,400 | +| 8 | 15 Gbps | ~13 Gbps | ~11 Gbps | ~2,600 | +| 16 | 25 Gbps | ~20 Gbps | ~15 Gbps | ~4,000 | +| 32 | 50 Gbps | ~25 Gbps | ~15 Gbps | ~5,000 | | 64 | 100 Gbps | ~31 Gbps | ~15 Gbps | ~6,000 | From 27cb890f8a17d4c403817577a59677002bf3bc76 Mon Sep 17 00:00:00 2001 From: Jack Carter <128555021+SunsetDrifter@users.noreply.github.com> Date: Fri, 26 Jun 2026 13:25:07 +0200 Subject: [PATCH 5/7] docs: add userspace WireGuard table and when-to-use cases Add a userspace-mode capacity table showing wireguard-go does not scale across cores (download plateaus ~6.8 Gbps, only ~4-5 cores used), and the cases where a routing peer runs userspace: missing/broken/conflicting kernel module, no TUN device (netstack, incl. rootless Docker), non-Linux peers, and forcing userspace to capture policy IDs and blocked traffic events. Name the exact benchmark CPU (Xeon Platinum 8375C). --- .../manage/networks/sizing-routing-peers.mdx | 26 +++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/src/pages/manage/networks/sizing-routing-peers.mdx b/src/pages/manage/networks/sizing-routing-peers.mdx index f9b2534d..b2ce477f 100644 --- a/src/pages/manage/networks/sizing-routing-peers.mdx +++ b/src/pages/manage/networks/sizing-routing-peers.mdx @@ -66,7 +66,7 @@ Acme builds **two identical Networks**, each exposing the datacenter, each with ## Per-peer capacity reference -Representative measured throughput per routing-peer size, all measured on AWS **c6in.16xlarge** instances: 64 vCPUs and a 100 Gbps interface on 3rd-generation Intel Xeon Scalable (Ice Lake) with AVX-512 and a 3.5 GHz all-core turbo, running Linux kernel WireGuard with threaded NAPI (the kernel default), the default 1280-byte tunnel MTU, and direct connections, with load spread across many tunnels. Smaller core counts were produced by taking cores offline on that same box, so each row isolates the effect of core count. The Network interface column is the minimum interface to pair with a peer of that size so it does not bottleneck the result, not a separately tested NIC. Expect roughly 20% variation, and confirm in your own environment. +Representative measured throughput per routing-peer size, all measured on AWS **c6in.16xlarge** instances: 64 vCPUs and a 100 Gbps interface on an Intel Xeon Platinum 8375C (Ice Lake, 2.90 GHz, AVX-512), running Linux kernel WireGuard with threaded NAPI (the kernel default), the default 1280-byte tunnel MTU, and direct connections, with load spread across many tunnels. Smaller core counts were produced by taking cores offline on that same box, so each row isolates the effect of core count. The Network interface column is the minimum interface to pair with a peer of that size so it does not bottleneck the result, not a separately tested NIC. Expect roughly 20% variation, and confirm in your own environment. | vCPUs | Network interface | Download (clients pulling) | Upload (clients pushing) | Active users at 5 Mbps | |---|---|---|---|---| @@ -89,11 +89,33 @@ Two things to read from this table: **On small or commodity hardware, the interface is the limit, not the cores.** A homelab mini PC with a 1 or 2.5 GbE port runs out of *interface* long before CPU, so do not read it against the table's CPU rows — your ceiling is the port's line rate, often a couple of Gbps or less, where download and upload converge. Kernel WireGuard still matters. +## Userspace mode + +The capacity table above is **kernel WireGuard**, the default on Linux when the `wireguard` kernel module is present. You run the slower **userspace** datapath (`wireguard-go`) in a few cases: + +- **The kernel module is missing, broken, or conflicts with other software.** NetBird falls back to userspace on its own, or you force it with [`NB_WG_KERNEL_DISABLED=true`](/client/environment-variables). +- **The host has no TUN device**, as in unprivileged containers (the [rootless Docker image](/get-started/install/docker)) and some serverless platforms. This needs [netstack mode](/client/environment-variables) (`NB_USE_NETSTACK_MODE`), which runs WireGuard entirely in userspace. +- **The routing peer is not Linux.** Windows, macOS, and the other platforms forward in userspace regardless (see [How Routing Peers Work](/manage/networks/how-routing-peers-work#requirements)). +- **You want full traffic-event logging from the peer.** [Policy IDs and blocked-traffic events](/manage/activity/traffic-events-logging#limitations) are reported only when the destination or routing peer runs in userspace mode, not kernel mode. Forcing userspace for that visibility deliberately trades away the throughput above. + +On the same instances with the kernel module disabled, the numbers change shape: + +| vCPUs | Download (clients pulling) | Upload (clients pushing) | CPU actually used | +|---|---|---|---| +| 1 | ~1.6 Gbps | ~1.4 Gbps | the 1 core | +| 2 | ~2.3 Gbps | ~1.9 Gbps | both cores | +| 4 | ~6.8 Gbps | ~3.5 Gbps | ~3 of 4 | +| 8 | ~6.8 Gbps | ~3.7 Gbps | ~4 of 8 | +| 16 | ~6.8 Gbps | ~3.7 Gbps | ~4 of 16 | +| 64 | ~5.8 Gbps | ~4.4 Gbps | ~5 of 64 | + +**Userspace WireGuard does not scale across cores.** Download plateaus around 6.8 Gbps by four cores and climbs no further; at 64 cores it even dips, because the work never spreads past the handful of cores `wireguard-go` uses (the "CPU actually used" column). Adding cores buys nothing. If you are stuck on userspace, size each peer for roughly 6 to 7 Gbps and add peers to go past it; if you need more from one peer, the fix is to enable the kernel module, not a bigger box. + ## Tuning for more throughput These levers raise the capacity of one routing peer before you add more. -- **Use kernel WireGuard.** On Linux, NetBird uses the in-kernel WireGuard module by default when it is present, and it is the right choice for a high-throughput routing peer: in testing it ran several times faster than the userspace fallback, because only the kernel path spreads the work across all cores. +- **Use kernel WireGuard.** On Linux, NetBird uses the in-kernel module by default when it is present; it is the right choice for a high-throughput routing peer, because only the kernel path spreads the work across all cores. The [userspace datapath](#userspace-mode) plateaus at a few Gbps no matter the core count. - **Match the network interface to the target.** At high aggregate rates the interface can limit you as much as the CPU. For tens of Gbps, choose an interface (and, in a cloud, an instance type) whose bandwidth matches the throughput you read from the table. - **Raise the MTU only on clean networks.** Throughput is largely bound by packets per second, so larger packets carry more for the same work. On a network that supports jumbo frames end to end, such as a datacenter backplane, a larger tunnel MTU raised per-peer throughput several-fold in testing. This helps only when the **entire path, clients included, supports the larger MTU**. Keep the default on paths that cross the public internet or reach ordinary 1500-byte clients, where it would only cause fragmentation. - **Add cores up to the knee, then stop.** Cores raise capacity up to about 16; past that the download direction inches up and the upload direction does not move at all. When you reach that knee, add another peer instead of a bigger one. From 6da0dfc48ec58c27d446e95abeb6af599dc8b592 Mon Sep 17 00:00:00 2001 From: Jack Carter <128555021+SunsetDrifter@users.noreply.github.com> Date: Fri, 26 Jun 2026 14:03:08 +0200 Subject: [PATCH 6/7] docs: replace 'sharding' with plain wording in sizing guide Rename the Scaling out heading and reword the body, description, and recap to talk about splitting load across more Networks instead of sharding. Update the in-page anchor link to match the new heading. --- src/pages/manage/networks/sizing-routing-peers.mdx | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/pages/manage/networks/sizing-routing-peers.mdx b/src/pages/manage/networks/sizing-routing-peers.mdx index b2ce477f..ddad2e63 100644 --- a/src/pages/manage/networks/sizing-routing-peers.mdx +++ b/src/pages/manage/networks/sizing-routing-peers.mdx @@ -1,7 +1,7 @@ import { Note, Warning } from '@/components/mdx' import { Tiles } from '@/components/Tiles' -export const description = "Choose the size and number of NetBird routing peers from measured throughput: a four-step method, a per-peer capacity table, the tuning levers that matter, and how to scale out with sharded Networks." +export const description = "Choose the size and number of NetBird routing peers from measured throughput: a four-step method, a per-peer capacity table, the tuning levers that matter, and how to scale out across multiple Networks." # Sizing Routing Peers @@ -58,7 +58,7 @@ Acme needs `ceil(30 / 20) = 2` peers actively carrying load. Step 4 turns that i ## Step 4: give each active peer its own Network -Here is the subtlety that catches people. A single Network does not split its load across its routing peers: NetBird high availability either pins every client to one peer (different metrics) or sends each client to its nearest peer by latency (equal metrics) — neither evenly divides a busy site's traffic. So you split load by building **more Networks**, one per shard. +Here is the subtlety that catches people. A single Network does not split its load across its routing peers: NetBird high availability either pins every client to one peer (different metrics) or sends each client to its nearest peer by latency (equal metrics) — neither evenly divides a busy site's traffic. So you split load by building **more Networks**, one for each share of the traffic. Make each active peer its own NetBird Network; have every Network expose the **same resources** (the same datacenter, the same `/32` hosts); and give every Network **two or more routing peers** so it carries its own failover. Then a policy sends each client group to a different Network. Because the Networks ride different routing peers, splitting clients across Networks splits the load across peers, by design. @@ -85,7 +85,7 @@ Representative measured throughput per routing-peer size, all measured on AWS ** Two things to read from this table: - **Download keeps scaling with cores; upload flattens around 15 Gbps at about 16 cores** and barely improves past that, because the decrypt path is harder to spread across cores. Beyond ~16 cores you gain mostly on the download direction. -- **16 to 32 vCPUs is the practical sweet spot.** A 16-core peer reaches close to a 64-core peer's throughput, so prefer **more modest peers over fewer large ones** (see [Scaling out](#scaling-out-high-availability-and-sharding)). +- **16 to 32 vCPUs is the practical sweet spot.** A 16-core peer reaches close to a 64-core peer's throughput, so prefer **more modest peers over fewer large ones** (see [Scaling out](#scaling-out-high-availability-and-splitting-load)). **On small or commodity hardware, the interface is the limit, not the cores.** A homelab mini PC with a 1 or 2.5 GbE port runs out of *interface* long before CPU, so do not read it against the table's CPU rows — your ceiling is the port's line rate, often a couple of Gbps or less, where download and upload converge. Kernel WireGuard still matters. @@ -120,11 +120,11 @@ These levers raise the capacity of one routing peer before you add more. - **Raise the MTU only on clean networks.** Throughput is largely bound by packets per second, so larger packets carry more for the same work. On a network that supports jumbo frames end to end, such as a datacenter backplane, a larger tunnel MTU raised per-peer throughput several-fold in testing. This helps only when the **entire path, clients included, supports the larger MTU**. Keep the default on paths that cross the public internet or reach ordinary 1500-byte clients, where it would only cause fragmentation. - **Add cores up to the knee, then stop.** Cores raise capacity up to about 16; past that the download direction inches up and the upload direction does not move at all. When you reach that knee, add another peer instead of a bigger one. -## Scaling out: high availability and sharding +## Scaling out: high availability and splitting load A single peer has a ceiling. To go past it, run more than one, but understand what NetBird high availability does and does not do. It gives you two things: **failover** (different metrics: one peer carries all, the rest are reserve) and **nearest-peer selection** (equal metrics: each client uses its lowest-latency peer, switching only past a 20 ms difference). See [How Routing Peers Work](/manage/networks/how-routing-peers-work#high-availability) for the mechanism behind both. -Neither balances a Network's load. Nearest-peer *does* spread traffic when peers sit in different regions and users are spread among them — EU users on the EU peer, US users on the US — but it splits by geography, not evenly, and not at all when peers sit together. To split load deterministically, shard into more identical Networks as in [Step 4](#step-4-give-each-active-peer-its-own-network). +Neither balances a Network's load. Nearest-peer *does* spread traffic when peers sit in different regions and users are spread among them — EU users on the EU peer, US users on the US — but it splits by geography, not evenly, and not at all when peers sit together. To split load deterministically, build more identical Networks as in [Step 4](#step-4-give-each-active-peer-its-own-network). ## Cloud and virtualization notes @@ -150,7 +150,7 @@ From each test client, drive load in the direction your users actually use: iperf3 -c -p 5201 -t 30 -P 4 -R # Upload (clients pushing into the network, the peer decrypts): -iperf3 -c -p 5201 -t 30 -u -b 0 -l 1250 # keep the UDP datagram below the tunnel MTU +iperf3 -c -p 5201 -t 30 -P 4 ``` Read the real throughput on the routing peer itself, from its WireGuard interface, while watching CPU: @@ -167,7 +167,7 @@ Reading at the WireGuard interface, not one client's number, captures the peer's - Size against **aggregate active throughput in the direction your users use**, not the enrolled head count. Acme: 3,000 active pullers at 10 Mbps is 30 Gbps of download. - **One tunnel is about one core**; capacity comes from **many tunnels across cores**, and **16 to 32 cores** reaches nearly the same throughput as far larger boxes. - `active peers = ceil(aggregate / per-peer capacity)`, then give each its own **identical NetBird Network** (same resources, two or more peers for failover) and split clients across the Networks by policy. Acme: `ceil(30 / 20) = 2` Networks, each an HA pair, so **four** 16-core peers, 3,000 users per Network. -- The levers, in order of effect: **kernel WireGuard**, a **matched interface**, **jumbo frames on a clean path**, and **cores up to ~16**. High availability is **failover or nearest-peer selection, never load balancing**; split load by sharding into more Networks. +- The levers, in order of effect: **kernel WireGuard**, a **matched interface**, **jumbo frames on a clean path**, and **cores up to ~16**. High availability is **failover or nearest-peer selection, never load balancing**; split load across more Networks. - On **small or commodity hardware** (a homelab mini PC, a 1 or 2.5 GbE box), the **interface** is the limit before the CPU; size to the smaller of the two, and expect download and upload to converge. - When unsure, **measure** at your candidate size and watch where CPU or the interface saturates. From e17669d61d1a60fac6c220b81be30edccd2fb77f Mon Sep 17 00:00:00 2001 From: Jack Carter <128555021+SunsetDrifter@users.noreply.github.com> Date: Fri, 26 Jun 2026 14:13:35 +0200 Subject: [PATCH 7/7] docs: clarify download/upload direction bullets in sizing guide Lead each direction bullet with Download:/Upload: and say the routing peer encrypts/decrypts, tying the pulling/pushing distinction to the capacity table's column names. --- src/pages/manage/networks/sizing-routing-peers.mdx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pages/manage/networks/sizing-routing-peers.mdx b/src/pages/manage/networks/sizing-routing-peers.mdx index ddad2e63..cf061b1e 100644 --- a/src/pages/manage/networks/sizing-routing-peers.mdx +++ b/src/pages/manage/networks/sizing-routing-peers.mdx @@ -29,8 +29,8 @@ The method is four steps: estimate the peak traffic, read what one peer of a giv One distinction causes most sizing mistakes, so we meet it before the method. The words "download" and "upload" are from the user's point of view, and they reach the peer as opposite jobs: -- A user **pulling** data from the network (opening files, loading apps, fetching builds: the common case) makes the peer **encrypt**. -- A user **pushing** data into the network (uploading, backing up, writing) makes the peer **decrypt**. +- **Download:** A user pulling data from the network (opening files, loading apps, fetching builds: the common case) makes the routing peer **encrypt**. +- **Upload:** A user pushing data into the network (backing up, writing, sending files) makes the routing peer **decrypt**. The two run at different speeds. On the same hardware the pulling (encrypt) direction reaches roughly twice the throughput of the pushing (decrypt) direction, because the encrypt path can ride the operating system's segmentation offload and the decrypt path cannot. So the number that sets your peer count is not "how much traffic" but "how much traffic in the direction my users actually use." Acme's people mostly pull, so we will size on the download (encrypt) figure and treat the upload (decrypt) figure as a floor.