Skip to content

Backend triggers: resolve dependencies by their real container name #124

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
antoncxx wants to merge 2 commits into
base: main
Choose a base branch
from
Open

Backend triggers: resolve dependencies by their real container name #124

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
be37274
backend triggers: resolve dependencies by their real container name
antoncxx Aug 6, 2026
bb3eff6
Merge remote-tracking branch 'origin/main' into feature/backend-trigg…
antoncxx Aug 7, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 38 additions & 9 deletions members/nullnet-client/src/commands/dnat.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,23 +28,38 @@ pub(crate) fn init() {

/// Install a DNAT for `port → overlay_ip:port`. When `container_ip` is a
/// real address, the rule is scoped to that source via `-s` so co-located
/// replicas hit independent chains. `Ipv4Addr::UNSPECIFIED` (0.0.0.0) means
/// "no source filter" — used by legacy callers that don't know the source.
/// replicas hit independent chains. When `dest_ip` is a real address, the
/// rule is additionally scoped to that destination via `-d` — so two
/// backend-trigger dependencies sharing a port from the same initiator (each
/// with its own placeholder address, see `placeholder.rs`) get independent
/// rules instead of one clobbering the other. `Ipv4Addr::UNSPECIFIED`
/// (0.0.0.0) means "no filter" for either — used by legacy callers that
/// don't know the source/destination.
/// Returns `false` if any of the per-proto `iptables` rules failed to apply.
pub(crate) fn install(port: u16, overlay_ip: Ipv4Addr, container_ip: Ipv4Addr) -> bool {
pub(crate) fn install(
port: u16,
overlay_ip: Ipv4Addr,
container_ip: Ipv4Addr,
dest_ip: Ipv4Addr,
) -> bool {
let mut ok = true;
for proto in PROTOS {
ok &= run_iptables("-A", proto, port, overlay_ip, container_ip);
ok &= run_iptables("-A", proto, port, overlay_ip, container_ip, dest_ip);
}
flush_conntrack(port, container_ip);
ok
}

/// Returns `false` if any of the per-proto `iptables` rules failed to delete.
pub(crate) fn remove(port: u16, overlay_ip: Ipv4Addr, container_ip: Ipv4Addr) -> bool {
pub(crate) fn remove(
port: u16,
overlay_ip: Ipv4Addr,
container_ip: Ipv4Addr,
dest_ip: Ipv4Addr,
) -> bool {
let mut ok = true;
for proto in PROTOS {
ok &= run_iptables("-D", proto, port, overlay_ip, container_ip);
ok &= run_iptables("-D", proto, port, overlay_ip, container_ip, dest_ip);
}
flush_conntrack(port, container_ip);
ok
Expand All @@ -57,14 +72,19 @@ fn run_iptables(
port: u16,
overlay_ip: Ipv4Addr,
container_ip: Ipv4Addr,
dest_ip: Ipv4Addr,
) -> bool {
let port_s = port.to_string();
let target = format!("{overlay_ip}:{port}");
let container_ip_s = container_ip.to_string();
let dest_ip_s = dest_ip.to_string();
let mut args: Vec<&str> = vec!["iptables", "-t", "nat", action, CHAIN, "-p", proto];
if !container_ip.is_unspecified() {
args.extend_from_slice(&["-s", &container_ip_s]);
}
if !dest_ip.is_unspecified() {
args.extend_from_slice(&["-d", &dest_ip_s]);
}
args.extend_from_slice(&[
"--dport",
&port_s,
Expand All @@ -79,19 +99,28 @@ fn run_iptables(
} else {
container_ip_s.clone()
};
let dst = if dest_ip.is_unspecified() {
"any".to_string()
} else {
dest_ip_s.clone()
};
match status {
Ok(s) if s.success() => {
println!("[dnat] iptables {action} {CHAIN} {proto}/{port} -s {src} -> {target}");
println!(
"[dnat] iptables {action} {CHAIN} {proto}/{port} -s {src} -d {dst} -> {target}"
);
true
}
Ok(s) => {
eprintln!(
"[dnat] iptables {action} {CHAIN} {proto}/{port} -s {src} -> {target} exited {s}"
"[dnat] iptables {action} {CHAIN} {proto}/{port} -s {src} -d {dst} -> {target} exited {s}"
);
false
}
Err(e) => {
eprintln!("[dnat] iptables {action} {CHAIN} {proto}/{port} -s {src} -> {target}: {e}");
eprintln!(
"[dnat] iptables {action} {CHAIN} {proto}/{port} -s {src} -d {dst} -> {target}: {e}"
);
false
}
}
Expand Down
34 changes: 34 additions & 0 deletions members/nullnet-client/src/commands/egress.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,19 @@ pub(crate) fn init() {
&["ipset", "add", "-exist", INTERNAL_SET, range],
);
}
// The backend-trigger placeholder block (see `placeholder::placeholder_cidr`)
// is synthetic — never real internet traffic — but isn't one of the
// static private/special ranges above. Without this, a trigger dial to
// a port `nullnet_watched_ports` hasn't caught up on yet (a startup race
// against the gRPC round trip that populates it, see `nfqueue::apply_ports_diff`)
// falls through to this rule instead and gets misclassified as an egress
// candidate — held for the egress-trigger timeout, then dropped, well
// past most callers' own request timeout.
let placeholder_cidr = crate::placeholder::placeholder_cidr();
sudo_ok(
"ipset add internal placeholder",
&["ipset", "add", "-exist", INTERNAL_SET, &placeholder_cidr],
);

// NFQUEUE trigger rule: NEW flows to non-internal destinations → queue 1.
// The listener filters by container (registered services only). --queue-bypass
Expand Down Expand Up @@ -213,6 +226,27 @@ pub(crate) fn install_steer(
],
);
}
// Same bypass for the backend-trigger placeholder block — see the
// matching comment in `init()`. Lands at base+9, still clear of the
// catch-all at base+15.
let placeholder_cidr = crate::placeholder::placeholder_cidr();
let placeholder_prio = (base + INTERNAL_RANGES.len() as u32).to_string();
ok &= sudo_ok(
"ip rule add placeholder bypass",
&[
"ip",
"rule",
"add",
"from",
&cip,
"to",
&placeholder_cidr,
"lookup",
"main",
"priority",
&placeholder_prio,
],
);
// Catch-all: everything else from this container → the egress table.
let catch_all = (base + 15).to_string();
ok &= sudo_ok(
Expand Down
135 changes: 67 additions & 68 deletions members/nullnet-client/src/control_channel.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,7 @@ use crate::commands::{RtNetLinkHandle, configure_access_port, dnat, egress, remo
use crate::ebpf::{FirewallPeers, FirewallVxlanPorts, NetId};
use crate::egress_policy::{PolicyVerdicts, flush_container_conntrack};
use crate::egress_state::{EgressRecord, EgressState};
use crate::host_mappings::{
HOSTS_MARKER, HostMappingsState, edit_container_hosts, hosts_file_lock,
};
use crate::host_mappings::{self, HOSTS_MARKER, HostMappingsState, hosts_file_lock};
use crate::nfqueue::BridgeIpCache;
use crate::peers::peer::{Peers, VethKey};
use crate::triggers::TriggersState;
Expand Down Expand Up @@ -531,6 +529,7 @@ async fn handle_vxlan_setup(
if let Some(container) = message.docker_container.as_deref() {
triggers_state.mark_active(
container,
crate::triggers::EGRESS_DST_IP,
crate::triggers::EGRESS_TRIGGER_PORT,
vxlan_id,
gw,
Expand Down Expand Up @@ -590,22 +589,29 @@ async fn handle_vxlan_setup(
// packet traverses `nat PREROUTING`. The DNAT rule MUST already be
// installed by then, so we:
// 1. peek the initiator's bridge IP (stashed at `mark_pending`)
// 2. install DNAT with `-s <container_ip>`
// 2. install DNAT with `-s <container_ip> -d <placeholder_ip>`
// 3. mark_active → wakes the waiter, packet released into the new
// rule
if let Some(dnat_port) = message.dnat_port
&& let Ok(dnat_port) = u16::try_from(dnat_port)
&& let Ok(overlay_ip) = host_mapping.ip.parse::<Ipv4Addr>()
{
let container_key = message.docker_container.as_deref().unwrap_or("");
let container_ip = triggers_state.peek_container_ip(container_key, dnat_port);
// `host_mapping.name` is chain[0] for this trigger — the same
// literal name the NFQUEUE listener read from its port→target
// map when it observed the original packet — so recomputing the
// placeholder here independently lands on the exact same
// address it used, without needing it on the wire separately.
let dest_ip = crate::placeholder::ip_for(&host_mapping.name);
let container_ip = triggers_state.peek_container_ip(container_key, dest_ip, dnat_port);
// Only promote to Active if the DNAT rule is actually live. Waking
// the held packet without it would release the SYN into a missing
// rule (→ misroute to the original dest); instead leave it Pending
// so the listener drops at ACTIVE_TIMEOUT.
if dnat::install(dnat_port, overlay_ip, container_ip) {
if dnat::install(dnat_port, overlay_ip, container_ip, dest_ip) {
triggers_state.mark_active(
container_key,
dest_ip,
dnat_port,
vxlan_id,
overlay_ip,
Expand Down Expand Up @@ -695,30 +701,52 @@ async fn handle_vxlan_teardown(
firewall_vxlan_ports.remove(message.vxlan_id);

// remove DNAT before tearing the tunnel down so existing flows reset
// cleanly. The `container_ip` matches the `-s` we used at install time.
// `remove_by_vxlan` also matches the egress steer's sentinel-port entry
// (EGRESS_TRIGGER_PORT = 0). That path installs a policy-route steer, not a
// DNAT — its teardown runs via EgressState below — so skip DNAT removal for
// it. Only real backend DNAT ports (>0) go through `dnat::remove`; otherwise
// we'd fire a bogus `iptables -D --dport 0` and a false removal-failed event.
if let Some((_container, port, overlay_ip, container_ip)) =
// cleanly. The `container_ip`/`dst_ip` match the `-s`/`-d` we used at
// install time. `remove_by_vxlan` also matches the egress steer's
// sentinel-port entry (EGRESS_TRIGGER_PORT = 0). That path installs a
// policy-route steer, not a DNAT — its teardown runs via EgressState
// below — so skip DNAT removal for it. Only real backend DNAT ports (>0)
// go through `dnat::remove`; otherwise we'd fire a bogus
// `iptables -D --dport 0` and a false removal-failed event. Whether this
// was a backend-trigger entry (vs. an egress steer) also decides how the
// host mapping below gets torn down.
let mut is_backend_trigger_entry = false;
if let Some((_container, dst_ip, port, overlay_ip, container_ip)) =
triggers_state.remove_by_vxlan(message.vxlan_id)
&& port != crate::triggers::EGRESS_TRIGGER_PORT
&& !dnat::remove(port, overlay_ip, container_ip)
{
fire_event(
&grpc,
AgentEventKind::DnatRemovalFailed(AgentDnatRemovalFailed {
port: u32::from(port),
overlay_ip: overlay_ip.to_string(),
}),
);
is_backend_trigger_entry = true;
if !dnat::remove(port, overlay_ip, container_ip, dst_ip) {
fire_event(
&grpc,
AgentEventKind::DnatRemovalFailed(AgentDnatRemovalFailed {
port: u32::from(port),
overlay_ip: overlay_ip.to_string(),
}),
);
}
}

// remove host mapping if one was installed at setup
// Remove the host mapping installed at setup — except for a
// backend-trigger entry, where we re-seed the placeholder instead of
// deleting the line outright. Deleting would leave the initiator unable
// to resolve the name at all: the next `connect(name, ...)` would
// dead-end exactly like an un-seeded bare name does (no packet, no
// re-trigger, chain never rebuilds). Proxy-dependency mappings are
// unaffected — a fresh proxy request rebuilds the chain (and writes the
// real mapping) before forwarding, so there's no gap to cover there.
if let Some((host_mapping, docker_container)) = host_mappings_state.take_vxlan(message.vxlan_id)
{
let _ = remove_host_mapping(&host_mapping, docker_container.as_deref());
if is_backend_trigger_entry && let Some(container) = docker_container.as_deref() {
if let Err(e) = crate::placeholder::seed_placeholder(container, &host_mapping.name) {
eprintln!(
"[control_channel] failed to re-seed placeholder for '{}' in {container}: {e:?}",
host_mapping.name
);
}
} else {
let _ = remove_host_mapping(&host_mapping, docker_container.as_deref());
}
}

// teardown VXLAN on this machine
Expand Down Expand Up @@ -901,41 +929,29 @@ fn add_host_mapping(hm: &HostMapping, docker_container: Option<&str>) -> Result<
if let Some(container) = docker_container {
// container-targeted: the resolver that needs this name lives inside
// the container, so write only there and leave the host's file alone.
// Edited from the host side (see `read_container_hosts`) so a paused
// container is served just as well as a running one.
// Edited from the host side (see `host_mappings::edit_container_hosts`)
// so a paused container is served just as well as a running one.
//
// Bail rather than write on a failed read: treating a missing file as
// empty content would truncate a live `/etc/hosts` down to this one
// entry.
edit_container_hosts(container, |current| {
upsert_hosts_entry(current, &hm.name, &entry)
host_mappings::edit_container_hosts(container, |current| {
host_mappings::upsert_hosts_entry(current, &hm.name, &entry)
})
.handle_err(location!())?;
} else {
// host-targeted: upsert into the host's /etc/hosts
let content = std::fs::read_to_string(path).handle_err(location!())?;
std::fs::write(path, upsert_hosts_entry(&content, &hm.name, &entry))
.handle_err(location!())?;
std::fs::write(
path,
host_mappings::upsert_hosts_entry(&content, &hm.name, &entry),
)
.handle_err(location!())?;
}

Ok(())
}

fn upsert_hosts_entry(content: &str, name: &str, entry: &str) -> String {
let mut lines: Vec<String> = content.lines().map(ToString::to_string).collect();
let mut found = false;
for line in &mut lines {
if line.split_whitespace().skip(1).any(|tok| tok == name) {
*line = entry.to_string();
found = true;
}
}
if !found {
lines.push(entry.to_string());
}
lines.join("\n") + "\n"
}

fn remove_host_mapping(hm: &HostMapping, docker_container: Option<&str>) -> Result<(), Error> {
let path = "/etc/hosts";

Expand All @@ -949,40 +965,23 @@ fn remove_host_mapping(hm: &HostMapping, docker_container: Option<&str>) -> Resu
// the `docker pause` that follows it race, and losing that race used to
// strand the entry, leaving the name pointing at a dead overlay IP for
// as long as the container kept being resumed rather than restarted.
edit_container_hosts(container, |current| {
remove_hosts_entry(current, &hm.name, &hm.ip)
host_mappings::edit_container_hosts(container, |current| {
host_mappings::remove_hosts_entry(current, &hm.name, &hm.ip)
})
.handle_err(location!())?;
} else {
// host-targeted: drop this net's line from the host file
let content = std::fs::read_to_string(path).handle_err(location!())?;
std::fs::write(path, remove_hosts_entry(&content, &hm.name, &hm.ip))
.handle_err(location!())?;
std::fs::write(
path,
host_mappings::remove_hosts_entry(&content, &hm.name, &hm.ip),
)
.handle_err(location!())?;
}

Ok(())
}

/// Drop the line mapping `name`, but only while it still points at `ip`.
///
/// NET IDs are recycled, so a teardown can land after a *newer* net has already
/// re-installed the same name at a different overlay IP (`upsert_hosts_entry`
/// keys on the name alone, which is what makes the replacement correct).
/// Matching the IP too makes the late teardown a no-op instead of deleting a
/// mapping that belongs to a live tunnel.
fn remove_hosts_entry(content: &str, name: &str, ip: &str) -> String {
let lines: Vec<String> = content
.lines()
.filter(|line| {
let mut tokens = line.split_whitespace();
let line_ip = tokens.next();
!(line_ip == Some(ip) && tokens.any(|tok| tok == name))
})
.map(ToString::to_string)
.collect();
lines.join("\n") + "\n"
}

/// Lowercase hex encoding, used to pass the tunnel's AES key to
/// `vxlan-setup.sh`/`vxlan-teardown.sh` as a shell argument.
fn hex_encode(bytes: &[u8]) -> String {
Expand Down
Loading