fix(iac/aws): route private egress through a persistent fck-nat ENI

.github/workflows/ci.yml

@@ -613,6 +613,16 @@ jobs:
             terraform -chdir="$d" test
           done
 
+      # The fck-nat egress route must target the persistent ENI, not the
+      # current instance's own, so it survives ASG replacement (#716).
+      - name: Test AWS networking module
+        if: matrix.cloud == 'aws'
+        run: |
+          d=terraform/modules/networking/aws
+          terraform -chdir="$d" init -backend=false -lockfile=readonly -input=false
+          terraform -chdir="$d" fmt -check -recursive
+          terraform -chdir="$d" test
+
       # Failure alerts must fire on the markers the app logs (#714). GCP's
       # module tests run in the /health step below.
       - name: Test monitoring module

terraform/environments/aws/ci-cd-permissions/policy_networking.tf

@@ -33,6 +33,15 @@ resource "aws_iam_policy" "networking" {
           "ec2:CreateLaunchTemplateVersion",
           "ec2:DeleteLaunchTemplateVersions",
           "ec2:ModifyLaunchTemplate",
+          # The persistent fck-nat ENI the private routes target (#716):
+          # CreateNetworkInterface to create it, ModifyNetworkInterfaceAttribute
+          # because the provider turns source_dest_check off in a second call
+          # after the create, and DetachNetworkInterface because destroying or
+          # replacing it detaches it from the NAT instance first.
+          # DeleteNetworkInterface and DescribeNetworkInterfaces are below.
+          "ec2:CreateNetworkInterface",
+          "ec2:ModifyNetworkInterfaceAttribute",
+          "ec2:DetachNetworkInterface",
           "ec2:CreateRoute",
           "ec2:CreateRouteTable",
           "ec2:CreateSecurityGroup",

terraform/modules/networking/aws/README.md

@@ -114,6 +114,9 @@ The `enable_nat_gateway` option deploys a cost-effective NAT alternative:
 - Provides IPv4 egress for services without IPv6 support (like AWS SES)
 - Automatically deployed in the first public subnet
 - 90% cheaper than AWS NAT Gateway (~$32/month)
+- Private IPv4 routes target a persistent ENI that each instance attaches at
+  boot, so egress survives the ASG replacing the instance. With an existing
+  VPC, route private subnets to the `nat_network_interface_id` output.
 
 ## Security Groups
 
@@ -136,6 +139,8 @@ The module outputs:
 - `database_security_group_id`: Security group for database
 - `lambda_vpc_config`: Convenience object for Lambda module
 - `database_vpc_config`: Convenience object for database module
+- `nat_network_interface_id`: Persistent fck-nat ENI to route IPv4 egress to (if enabled)
+- `nat_autoscaling_group_name`: fck-nat Auto Scaling group name (if enabled)
 
 ## Cost Comparison
 

terraform/modules/networking/aws/main.tf

@@ -209,6 +209,27 @@ resource "aws_iam_instance_profile" "fck_nat" {
   })
 }
 
+# Persistent ENI the private routes target (#716). The route tables must not
+# name an instance's own primary ENI: the ASG replaces the instance (failed
+# health check, AZ event) and the replacement comes up with a new primary ENI,
+# so a route pinned to the old one blackholes IPv4 egress until the next apply.
+# This ENI outlives every instance. Each one the ASG launches attaches it at
+# boot as its second interface (see user_data), so the route target never
+# changes. It lives in the ASG's only subnet because an ENI can only attach to
+# an instance in its own AZ.
+resource "aws_network_interface" "fck_nat" {
+  count = var.enable_nat_gateway ? 1 : 0
+
+  description       = "${var.stack_name} fck-nat persistent ENI (private route target)"
+  subnet_id         = local.public_subnet_ids[0]
+  security_groups   = [aws_security_group.fck_nat[0].id]
+  source_dest_check = false # NAT forwards traffic it is neither source nor destination of
+
+  tags = merge(var.tags, {
+    Name = "${var.stack_name}-fck-nat-eni"
+  })
+}
+
 # Launch template for fck-nat instance
 resource "aws_launch_template" "fck_nat" {
   count = var.enable_nat_gateway ? 1 : 0
@@ -227,15 +248,16 @@ resource "aws_launch_template" "fck_nat" {
     security_groups             = [aws_security_group.fck_nat[0].id]
   }
 
-  # User data to disable source/destination check for NAT functionality
+  # Hand the persistent ENI to the AMI's own fck-nat service. With eni_id set
+  # it disables source/destination checking on the primary ENI, attaches
+  # eni_id as the second interface (retrying until a predecessor that is
+  # still terminating has released it), and NATs traffic arriving on it out
+  # of the primary interface.
   user_data = base64encode(<<-EOF
-    #!/bin/bash
-    # Get instance ID and ENI ID
-    INSTANCE_ID=$(ec2-metadata --instance-id | cut -d " " -f 2)
-    ENI_ID=$(aws ec2 describe-instances --instance-ids $INSTANCE_ID --region ${var.region} --query 'Reservations[0].Instances[0].NetworkInterfaces[0].NetworkInterfaceId' --output text)
-
-    # Disable source/destination check
-    aws ec2 modify-network-interface-attribute --network-interface-id $ENI_ID --region ${var.region} --no-source-dest-check
+    #!/bin/sh
+    : > /etc/fck-nat.conf
+    echo "eni_id=${aws_network_interface.fck_nat[0].id}" >> /etc/fck-nat.conf
+    service fck-nat restart
   EOF
   )
 
@@ -272,7 +294,7 @@ resource "aws_autoscaling_group" "fck_nat" {
   desired_capacity    = 1
   min_size            = 1
   max_size            = 1
-  vpc_zone_identifier = [local.public_subnet_ids[0]] # Deploy in first public subnet
+  vpc_zone_identifier = [aws_network_interface.fck_nat[0].subnet_id] # The persistent ENI's subnet (first public subnet)
 
   launch_template {
     id      = aws_launch_template.fck_nat[0].id
@@ -282,6 +304,11 @@ resource "aws_autoscaling_group" "fck_nat" {
   health_check_type         = "EC2"
   health_check_grace_period = 60
 
+  # Grant the attach permission before an instance can boot and ask for it.
+  # fck-nat retries the attach, but its one-shot source/dest-check call on
+  # the primary ENI does not, so make the order explicit.
+  depends_on = [aws_iam_role_policy.fck_nat_ec2]
+
   tag {
     key                 = "Name"
     value               = "${var.stack_name}-fck-nat-asg"
@@ -299,6 +326,16 @@ resource "aws_autoscaling_group" "fck_nat" {
 
   lifecycle {
     create_before_destroy = true
+
+    # A running instance never re-reads user data, and the launch template
+    # version is "$Latest", so a user_data change alone would leave the
+    # current instance on the old config. That matters once, on the upgrade
+    # that introduces the persistent ENI: the routes would move to an ENI no
+    # instance has attached. Replacing the ASG (create-before-destroy, so the
+    # new instance is InService before the old one goes) launches one that
+    # attaches it. user_data only changes when the ENI itself is replaced, so
+    # AMI rotations (image_id) still do not cycle the instance.
+    replace_triggered_by = [aws_launch_template.fck_nat[count.index].user_data]
   }
 }
 
@@ -311,51 +348,28 @@ resource "aws_iam_role_policy" "fck_nat_ec2" {
 
   policy = jsonencode({
     Version = "2012-10-17"
-    Statement = [{
-      Effect = "Allow"
-      Action = [
-        "ec2:DescribeInstances",
-        "ec2:ModifyNetworkInterfaceAttribute"
-      ]
-      Resource = "*"
-    }]
+    Statement = [
+      {
+        # Attach the persistent ENI to whichever instance the ASG launched.
+        # AttachNetworkInterface is authorized against both the ENI and the
+        # instance, so both ARNs are needed; the ENI one is what scopes it.
+        Sid      = "AttachPersistentENI"
+        Effect   = "Allow"
+        Action   = ["ec2:AttachNetworkInterface"]
+        Resource = [aws_network_interface.fck_nat[0].arn, "arn:aws:ec2:${var.region}:*:instance/*"]
+      },
+      {
+        # Disable source/destination checking on the instance's own primary
+        # ENI, whose ID is only known at boot.
+        Sid      = "DisableSourceDestCheck"
+        Effect   = "Allow"
+        Action   = ["ec2:ModifyNetworkInterfaceAttribute"]
+        Resource = "*"
+      },
+    ]
   })
 }
 
-# Get the instance from the ASG
-data "aws_instances" "fck_nat" {
-  count = var.enable_nat_gateway ? 1 : 0
-
-  filter {
-    name   = "tag:aws:autoscaling:groupName"
-    values = [aws_autoscaling_group.fck_nat[0].name]
-  }
-
-  filter {
-    name   = "instance-state-name"
-    values = ["running"]
-  }
-
-  depends_on = [aws_autoscaling_group.fck_nat]
-}
-
-# Get the ENI of the fck-nat instance for routing
-data "aws_network_interfaces" "fck_nat" {
-  count = var.enable_nat_gateway ? 1 : 0
-
-  filter {
-    name   = "attachment.instance-id"
-    values = data.aws_instances.fck_nat[0].ids
-  }
-
-  filter {
-    name   = "attachment.device-index"
-    values = ["0"] # Primary network interface
-  }
-
-  depends_on = [data.aws_instances.fck_nat]
-}
-
 # ==============================================
 # Availability Zones
 # ==============================================
@@ -485,7 +499,12 @@ resource "aws_route" "private_internet_ipv4" {
 
   route_table_id         = aws_route_table.private[count.index].id
   destination_cidr_block = "0.0.0.0/0"
-  network_interface_id   = data.aws_network_interfaces.fck_nat[0].ids[0]
+  network_interface_id   = aws_network_interface.fck_nat[0].id # Persistent, survives instance replacement (#716)
+
+  # Move the route only once an instance that attaches the ENI is InService,
+  # so the upgrade that introduces the ENI does not point egress at an
+  # unattached interface while the replacement ASG is still launching.
+  depends_on = [aws_autoscaling_group.fck_nat]
 }
 
 # Associate private subnets with private route tables

terraform/modules/networking/aws/outputs.tf

@@ -98,3 +98,20 @@ output "database_vpc_config" {
     security_group_id     = aws_security_group.database.id
   }
 }
+
+# The persistent fck-nat ENI (#716). The module routes its own private route
+# tables to it; with use_existing_vpc, point the VPC's private 0.0.0.0/0
+# routes here rather than at the fck-nat instance, whose primary ENI changes
+# every time the ASG replaces it.
+output "nat_network_interface_id" {
+  description = "ID of the persistent fck-nat ENI to route private IPv4 egress to (null when enable_nat_gateway is false)"
+  value       = var.enable_nat_gateway ? aws_network_interface.fck_nat[0].id : null
+}
+
+# The fck-nat ASG, for alarms on its instance replacements. It is replaced
+# (new name) when the launch template's user data changes; see the ASG's
+# replace_triggered_by.
+output "nat_autoscaling_group_name" {
+  description = "Name of the fck-nat Auto Scaling group (null when enable_nat_gateway is false)"
+  value       = var.enable_nat_gateway ? aws_autoscaling_group.fck_nat[0].name : null
+}