diff --git a/.gitignore b/.gitignore index 3380d0f8f..fc83868fa 100644 --- a/.gitignore +++ b/.gitignore @@ -41,3 +41,4 @@ bbl/debug # packrd **/*-packr.go bbl/packrd +bbl-env diff --git a/cloudconfig/aws/ops_generator.go b/cloudconfig/aws/ops_generator.go index 0c32a1c11..0a0cc0ded 100644 --- a/cloudconfig/aws/ops_generator.go +++ b/cloudconfig/aws/ops_generator.go @@ -3,6 +3,7 @@ package aws import ( "errors" "fmt" + "maps" "sort" "strings" @@ -92,6 +93,26 @@ func (o OpsGenerator) GenerateVars(state storage.State) (string, error) { "internal_az_subnet_id_mapping", "internal_az_subnet_cidr_mapping", } + cfRequiredOutputs := []string{ + "cf_router_lb_name", + "cf_router_lb_internal_security_group", + "cf_ssh_lb_name", + "cf_ssh_lb_internal_security_group", + "cf_tcp_lb_name", + "cf_tcp_lb_internal_security_group", + } + dualstackOutput, ok := terraformOutputs.Map["dualstack"] + if !ok { + return "", fmt.Errorf("missing dualstack terraform output") + } + dualstack := dualstackOutput.(bool) + if dualstack { + requiredOutputs = append(requiredOutputs, + "internal_cidr_ipv6", + "internal_az_subnet_ipv6_cidr_mapping", + ) + } + switch state.LB.Type { case "concourse": requiredOutputs = append( @@ -99,16 +120,10 @@ func (o OpsGenerator) GenerateVars(state storage.State) (string, error) { "concourse_lb_target_groups", "concourse_lb_internal_security_group", ) + case "nlb": + fallthrough case "cf": - requiredOutputs = append( - requiredOutputs, - "cf_router_lb_name", - "cf_router_lb_internal_security_group", - "cf_ssh_lb_name", - "cf_ssh_lb_internal_security_group", - "cf_tcp_lb_name", - "cf_tcp_lb_internal_security_group", - ) + requiredOutputs = append(requiredOutputs, cfRequiredOutputs...) } for _, output := range requiredOutputs { @@ -124,21 +139,41 @@ func (o OpsGenerator) GenerateVars(state storage.State) (string, error) { if err != nil { return "", err } + if dualstack { + internalAZSubnetIPv6CIDRMap := terraformOutputs.GetStringMap("internal_az_subnet_ipv6_cidr_mapping") + var azNames []string + for azName := range internalAZSubnetIDMap { + azNames = append(azNames, azName) + } + sort.Strings(azNames) + for azIndex, azName := range azNames { + cidr, ok := internalAZSubnetIPv6CIDRMap[azName] + if !ok { + return "", errors.New("missing AZ in terraform output: internal_az_subnet_ipv6_cidr_mapping") + } + v6Vars, err := azifyV6(azIndex, cidr) + if err != nil { + return "", err + } + azs = append(azs, v6Vars) + } + } varsYAML := map[string]interface{}{} - for k, v := range terraformOutputs.Map { - varsYAML[k] = v - } + maps.Copy(varsYAML, terraformOutputs.Map) + for _, az := range azs { for key, value := range az { varsYAML[key] = value } } - + // TODO: Make the ISO Segments handle IPv6 isoSegAZSubnetIDMap := terraformOutputs.GetStringMap("iso_az_subnet_id_mapping") isoSegAZSubnetCIDRMap := terraformOutputs.GetStringMap("iso_az_subnet_cidr_mapping") if len(isoSegAZSubnetIDMap) > 0 && len(isoSegAZSubnetCIDRMap) > 0 { - isoSegAzs, err := generateAZs(len(azs), isoSegAZSubnetIDMap, isoSegAZSubnetCIDRMap) + // Iso segment AZ indices start after the internal AZs + offset := len(internalAZSubnetIDMap) + isoSegAzs, err := generateAZs(offset, isoSegAZSubnetIDMap, isoSegAZSubnetCIDRMap) if err == nil { for _, az := range isoSegAzs { for key, value := range az { @@ -214,12 +249,13 @@ func createOp(opType, opPath string, value interface{}) op { func (o OpsGenerator) generateOps(state storage.State) ([]op, error) { ops := []op{} subnets := []networkSubnet{} + subnetsV6 := []networkSubnet{} azs, err := o.availabilityZones.RetrieveAZs(state.AWS.Region) if err != nil { return []op{}, fmt.Errorf("Retrieve availability zones: %s", err) //nolint:staticcheck } - + // This block doesn't seem to handle generating the OPs for isolation segments? for i := range azs { azOp := createOp("replace", "/azs/-", az{ Name: fmt.Sprintf("z%d", i+1), @@ -229,8 +265,11 @@ func (o OpsGenerator) generateOps(state storage.State) ([]op, error) { }) ops = append(ops, azOp) - subnet := generateNetworkSubnet(i) - subnets = append(subnets, subnet) + subnets = append(subnets, generateNetworkSubnet(i)) + + if state.LB.DualStack { + subnetsV6 = append(subnetsV6, generateNetworkSubnetV6(i)) + } } ops = append(ops, createOp("replace", "/networks/-", network{ @@ -245,7 +284,23 @@ func (o OpsGenerator) generateOps(state storage.State) ([]op, error) { Type: "manual", })) + if state.LB.DualStack { + ops = append(ops, createOp("replace", "/networks/-", network{ + Name: "private_v6", + Subnets: subnetsV6, + Type: "manual", + })) + + ops = append(ops, createOp("replace", "/networks/-", network{ + Name: "default_v6", + Subnets: subnetsV6, + Type: "manual", + })) + } + switch state.LB.Type { + case "nlb": + fallthrough case "cf": lbSecurityGroups := []map[string]string{ {"name": "cf-router-network-properties", "lb": "((cf_router_lb_name))", "group": "((cf_router_lb_internal_security_group))"}, @@ -307,6 +362,28 @@ func azify(az int, azName, cidr, subnet string) (map[string]string, error) { }, nil } +func azifyV6(az int, cidr string) (map[string]string, error) { + parsedCidr, err := bosh.ParseCIDRBlock(cidr) + if err != nil { + return map[string]string{}, err + } + + gateway := parsedCidr.GetNthIP(1).String() + firstReserved := parsedCidr.GetNthIP(2).String() + secondReserved := parsedCidr.GetNthIP(3).String() + lastReserved := parsedCidr.GetLastIP().String() + lastStatic := parsedCidr.GetLastIP().Subtract(1).String() + firstStatic := parsedCidr.GetLastIP().Subtract(65).String() + + return map[string]string{ + fmt.Sprintf("az%d_gateway_v6", az+1): gateway, + fmt.Sprintf("az%d_range_v6", az+1): cidr, + fmt.Sprintf("az%d_reserved_1_v6", az+1): fmt.Sprintf("%s-%s", firstReserved, secondReserved), + fmt.Sprintf("az%d_reserved_2_v6", az+1): lastReserved, + fmt.Sprintf("az%d_static_v6", az+1): fmt.Sprintf("%s-%s", firstStatic, lastStatic), + }, nil +} + func generateNetworkSubnet(az int) networkSubnet { az++ return networkSubnet{ @@ -326,3 +403,23 @@ func generateNetworkSubnet(az int) networkSubnet { }, } } + +func generateNetworkSubnetV6(az int) networkSubnet { + az++ + return networkSubnet{ + AZ: fmt.Sprintf("z%d", az), + Gateway: fmt.Sprintf("((az%d_gateway_v6))", az), + Range: fmt.Sprintf("((az%d_range_v6))", az), + Reserved: []string{ + fmt.Sprintf("((az%d_reserved_1_v6))", az), + fmt.Sprintf("((az%d_reserved_2_v6))", az), + }, + Static: []string{ + fmt.Sprintf("((az%d_static_v6))", az), + }, + CloudProperties: networkSubnetCloudProperties{ + Subnet: fmt.Sprintf("((az%d_subnet))", az), + SecurityGroups: []string{"((internal_security_group))"}, + }, + } +} diff --git a/cloudconfig/aws/ops_generator_test.go b/cloudconfig/aws/ops_generator_test.go index d866273f2..c410793c1 100644 --- a/cloudconfig/aws/ops_generator_test.go +++ b/cloudconfig/aws/ops_generator_test.go @@ -47,6 +47,7 @@ var _ = Describe("OpsGenerator", func() { "cf_tcp_lb_internal_security_group": "some-cf-tcp-lb-internal-security-group", "concourse_lb_target_groups": []string{"some-concourse-lb-target-group", "some-other-concourse-lb-target-group"}, "concourse_lb_internal_security_group": "some-concourse-lb-internal-security-group", + "dualstack": false, "internal_az_subnet_id_mapping": map[string]interface{}{ "us-east-1c": "some-internal-subnet-ids-3", "us-east-1a": "some-internal-subnet-ids-1", @@ -136,6 +137,7 @@ cf_tcp_lb_internal_security_group: some-cf-tcp-lb-internal-security-group cf_iso_router_lb_name: some-cf-iso-seg-router-lb-name concourse_lb_target_groups: [some-concourse-lb-target-group, some-other-concourse-lb-target-group] concourse_lb_internal_security_group: some-concourse-lb-internal-security-group +dualstack: false internal_az_subnet_cidr_mapping: us-east-1a: 10.0.16.0/20 us-east-1b: 10.0.32.0/20 @@ -156,6 +158,36 @@ iso_az_subnet_id_mapping: `)) }) + Context("when dualstack is enabled", func() { + BeforeEach(func() { + terraformManager.GetOutputsCall.Returns.Outputs.Map["dualstack"] = true + terraformManager.GetOutputsCall.Returns.Outputs.Map["internal_cidr_ipv6"] = "2600:1f18::/56" + terraformManager.GetOutputsCall.Returns.Outputs.Map["internal_az_subnet_ipv6_cidr_mapping"] = map[string]interface{}{ + "us-east-1a": "2600:1f18:0:1::/64", + "us-east-1b": "2600:1f18:0:2::/64", + "us-east-1c": "2600:1f18:0:3::/64", + } + }) + + It("includes _v6 suffixed variables for each AZ", func() { + varsYAML, err := opsGenerator.GenerateVars(incomingState) + Expect(err).NotTo(HaveOccurred()) + + // IPv4 vars still present + Expect(varsYAML).To(ContainSubstring("az1_gateway: 10.0.16.1")) + Expect(varsYAML).To(ContainSubstring("az1_subnet: some-internal-subnet-ids-1")) + + // IPv6 _v6 vars present + Expect(varsYAML).To(ContainSubstring("az1_gateway_v6: 2600:1f18:0:1::1")) + Expect(varsYAML).To(ContainSubstring("az1_range_v6: 2600:1f18:0:1::/64")) + Expect(varsYAML).To(ContainSubstring("az2_gateway_v6: 2600:1f18:0:2::1")) + Expect(varsYAML).To(ContainSubstring("az3_gateway_v6: 2600:1f18:0:3::1")) + + // No offset-based az4+ IPv6 entries + Expect(varsYAML).NotTo(ContainSubstring("az4_gateway_v6")) + }) + }) + Context("failure cases", func() { Context("when the az subnet id map has a key not in the cidr map", func() { BeforeEach(func() { @@ -191,7 +223,7 @@ iso_az_subnet_id_mapping: Expect(err).To(MatchError(fmt.Sprintf("missing %s terraform output", outputKey))) }, Entry("when internal_security_group is missing", "internal_security_group", ""), - + Entry("when dualstack is missing", "dualstack", "nlb"), Entry("when internal_az_subnet_id_mapping is missing", "internal_az_subnet_id_mapping", "cf"), Entry("when internal_az_subnet_cidr_mapping is missing", "internal_az_subnet_cidr_mapping", "cf"), Entry("when cf_router_lb_name is missing", "cf_router_lb_name", "cf"), @@ -247,6 +279,32 @@ iso_az_subnet_id_mapping: }) }) + Context("when there are cf lbs with dualstack", func() { + It("generates ops with separate IPv6 networks", func() { + incomingState.LB.Type = "cf" + incomingState.LB.DualStack = true + opsYAML, err := opsGenerator.Generate(incomingState) + Expect(err).NotTo(HaveOccurred()) + + // IPv6 subnets are in separate networks, not mixed into private/default + Expect(opsYAML).To(ContainSubstring("name: private_v6")) + Expect(opsYAML).To(ContainSubstring("name: default_v6")) + + // IPv6 networks use _v6 variable references + numAZs := len(availabilityZones.RetrieveAZsCall.Returns.AZs) + for i := range numAZs { + az := fmt.Sprintf("az%d", i+1) + Expect(opsYAML).To(ContainSubstring(fmt.Sprintf("((%s_gateway_v6))", az))) + Expect(opsYAML).To(ContainSubstring(fmt.Sprintf("((%s_range_v6))", az))) + Expect(opsYAML).To(ContainSubstring(fmt.Sprintf("((%s_reserved_1_v6))", az))) + Expect(opsYAML).To(ContainSubstring(fmt.Sprintf("((%s_static_v6))", az))) + } + + // IPv6 subnets share the same AWS subnet as IPv4 + Expect(opsYAML).To(ContainSubstring("((az1_subnet))")) + }) + }) + Context("when there is a concourse lb", func() { BeforeEach(func() { baseOpsYAMLContents, err := os.ReadFile(filepath.Join("fixtures", "aws-ops.yml")) diff --git a/commands/commands_usage.go b/commands/commands_usage.go index 268621eac..156803e3a 100644 --- a/commands/commands_usage.go +++ b/commands/commands_usage.go @@ -59,11 +59,12 @@ const ( LBUsage = ` Load Balancer options: - --lb-type Load balancer(s) type: "concourse" or "cf" - --lb-cert Path to SSL certificate (supported when type="cf") - --lb-key Path to SSL certificate key (supported when type="cf") + --lb-type Load balancer(s) type: "concourse", "cf", or "nlb" + --lb-cert Path to SSL certificate (supported when type="cf" or "nlb") + --lb-key Path to SSL certificate key (supported when type="cf" or "nlb") --lb-chain Path to SSL certificate chain (supported when iaas="aws") - --lb-domain Creates a DNS zone and records for the given domain (supported when type="cf")` + --lb-domain Creates a DNS zone and records for the given domain (supported when type="cf") + --dual-stack Enable dual-stack (IPv4+IPv6) networking (currently AWS only, requires --lb-type=nlb)` PlanCommandUsage = `Populates a state directory with the latest config without applying it diff --git a/commands/commands_usage_test.go b/commands/commands_usage_test.go index 38970e6e8..b01788937 100644 --- a/commands/commands_usage_test.go +++ b/commands/commands_usage_test.go @@ -69,11 +69,12 @@ var _ = Describe("Commands Usage", func() { --cloudstack-iso-segment CloudStack Activate iso segment env: $BBL_CLOUDSTACK_ISO_SEGMENT Load Balancer options: - --lb-type Load balancer(s) type: "concourse" or "cf" - --lb-cert Path to SSL certificate (supported when type="cf") - --lb-key Path to SSL certificate key (supported when type="cf") + --lb-type Load balancer(s) type: "concourse", "cf", or "nlb" + --lb-cert Path to SSL certificate (supported when type="cf" or "nlb") + --lb-key Path to SSL certificate key (supported when type="cf" or "nlb") --lb-chain Path to SSL certificate chain (supported when iaas="aws") - --lb-domain Creates a DNS zone and records for the given domain (supported when type="cf")`)) + --lb-domain Creates a DNS zone and records for the given domain (supported when type="cf") + --dual-stack Enable dual-stack (IPv4+IPv6) networking (currently AWS only, requires --lb-type=nlb)`)) }) }) }) diff --git a/commands/lb_args_handler.go b/commands/lb_args_handler.go index fc6bdb88e..db949d867 100644 --- a/commands/lb_args_handler.go +++ b/commands/lb_args_handler.go @@ -19,6 +19,7 @@ type LBArgs struct { KeyPath string ChainPath string Domain string + DualStack bool } func NewLBArgsHandler(certificateValidator certificateValidator) LBArgsHandler { @@ -28,6 +29,10 @@ func NewLBArgsHandler(certificateValidator certificateValidator) LBArgsHandler { } func (l LBArgsHandler) GetLBState(iaas string, args LBArgs) (storage.LB, error) { + if args.DualStack && !(iaas == "aws" && args.LBType == "nlb") { + return storage.LB{}, errors.New("dual stack networking requires AWS with the 'nlb' load balancer type. Set --lb-type=nlb on AWS, or remove the --dual-stack flag.") //nolint:staticcheck + } + if args.LBType == "" { return storage.LB{}, nil } @@ -61,11 +66,12 @@ func (l LBArgsHandler) GetLBState(iaas string, args LBArgs) (storage.LB, error) } return storage.LB{ - Type: args.LBType, - Cert: string(certData.Cert), - Key: string(certData.Key), - Chain: string(certData.Chain), - Domain: args.Domain, + Type: args.LBType, + Cert: string(certData.Cert), + Key: string(certData.Key), + Chain: string(certData.Chain), + Domain: args.Domain, + DualStack: args.DualStack, }, nil } @@ -78,6 +84,10 @@ func (l LBArgsHandler) Merge(new storage.LB, old storage.LB) storage.LB { if new.Type == "" { new.Type = old.Type } + + if !new.DualStack { + new.DualStack = old.DualStack + } } return new diff --git a/commands/lb_args_handler_test.go b/commands/lb_args_handler_test.go index 78623b3ce..82faaf295 100644 --- a/commands/lb_args_handler_test.go +++ b/commands/lb_args_handler_test.go @@ -66,6 +66,7 @@ var _ = Describe("LBArgsHandler", func() { Expect(lbState.Key).To(Equal("")) Expect(lbState.Chain).To(Equal("")) Expect(lbState.Domain).To(Equal("")) + Expect(lbState.DualStack).To(BeFalse()) Expect(certificateValidator.ReadAndValidateCall.CallCount).To(Equal(0)) }) }) @@ -80,6 +81,24 @@ var _ = Describe("LBArgsHandler", func() { }) }) + Context("when lb type is nlb with dualstack", func() { + It("returns a storage.LB object with DualStack set", func() { + lbState, err := handler.GetLBState("aws", commands.LBArgs{ + LBType: "nlb", + CertPath: "/path/to/cert", + KeyPath: "/path/to/key", + ChainPath: "/path/to/chain", + DualStack: true, + }) + Expect(err).NotTo(HaveOccurred()) + Expect(lbState.Type).To(Equal("nlb")) + Expect(lbState.Cert).To(Equal("some-cert")) + Expect(lbState.Key).To(Equal("some-key")) + Expect(lbState.Chain).To(Equal("some-chain")) + Expect(lbState.DualStack).To(BeTrue()) + }) + }) + Context("when iaas is azure and lb type is cf", func() { BeforeEach(func() { certDataPKCS12 := certs.CertData{ @@ -137,6 +156,37 @@ var _ = Describe("LBArgsHandler", func() { Expect(err).To(MatchError("domain is not implemented for concourse load balancers. Remove the --lb-domain flag and try again.")) }) }) + + Context("when dualstack is set on non-AWS IaaS", func() { + It("returns an error", func() { + _, err := handler.GetLBState("gcp", commands.LBArgs{ + LBType: "nlb", + DualStack: true, + }) + Expect(err).To(MatchError("dual stack networking requires AWS with the 'nlb' load balancer type. Set --lb-type=nlb on AWS, or remove the --dual-stack flag.")) + }) + }) + + Context("when dualstack is set without nlb type", func() { + It("returns an error", func() { + _, err := handler.GetLBState("aws", commands.LBArgs{ + LBType: "cf", + CertPath: "/path/to/cert", + KeyPath: "/path/to/key", + DualStack: true, + }) + Expect(err).To(MatchError("dual stack networking requires AWS with the 'nlb' load balancer type. Set --lb-type=nlb on AWS, or remove the --dual-stack flag.")) + }) + }) + + Context("when dualstack is set with empty lb type", func() { + It("returns an error", func() { + _, err := handler.GetLBState("aws", commands.LBArgs{ + DualStack: true, + }) + Expect(err).To(MatchError("dual stack networking requires AWS with the 'nlb' load balancer type. Set --lb-type=nlb on AWS, or remove the --dual-stack flag.")) + }) + }) }) }) @@ -146,18 +196,20 @@ var _ = Describe("LBArgsHandler", func() { BeforeEach(func() { new = storage.LB{ - Type: "new-type", - Cert: "new-cert", - Key: "new-key", - Chain: "new-chain", - Domain: "new-domain", + Type: "new-type", + Cert: "new-cert", + Key: "new-key", + Chain: "new-chain", + Domain: "new-domain", + DualStack: true, } old = storage.LB{ - Type: "old-type", - Cert: "old-cert", - Key: "old-key", - Chain: "old-chain", - Domain: "old-domain", + Type: "old-type", + Cert: "old-cert", + Key: "old-key", + Chain: "old-chain", + Domain: "old-domain", + DualStack: true, } }) @@ -176,11 +228,12 @@ var _ = Describe("LBArgsHandler", func() { }) Context("when the new state is empty", func() { - It("keeps the old domain and type", func() { + It("keeps the old domain, type, and dualstack", func() { merged := handler.Merge(storage.LB{}, old) Expect(merged).To(Equal(storage.LB{ - Type: "old-type", - Domain: "old-domain", + Type: "old-type", + Domain: "old-domain", + DualStack: true, })) }) }) diff --git a/commands/plan.go b/commands/plan.go index 7a06fba21..9981c89b0 100644 --- a/commands/plan.go +++ b/commands/plan.go @@ -88,6 +88,7 @@ func (p Plan) ParseArgs(args []string, state storage.State) (PlanConfig, error) planFlags.String(&lbArgs.CertPath, "lb-cert", "") planFlags.String(&lbArgs.KeyPath, "lb-key", "") planFlags.String(&lbArgs.Domain, "lb-domain", "") + planFlags.Bool(&lbArgs.DualStack, "dual-stack") if state.IAAS == "aws" { planFlags.String(&lbArgs.ChainPath, "lb-chain", "") } diff --git a/commands/plan_test.go b/commands/plan_test.go index 2202e9679..db0ebeaf3 100644 --- a/commands/plan_test.go +++ b/commands/plan_test.go @@ -136,6 +136,28 @@ var _ = Describe("Plan", func() { Expect(envIDManager.SyncCall.Receives.State.LB).To(Equal(lb)) }) }) + + Context("aws with dual-stack", func() { + It("sets LB args with DualStack on the state", func() { + err := command.Execute( + []string{ + "--lb-type", "nlb", + "--lb-cert", "cert", + "--lb-key", "key", + "--lb-chain", "chain", + "--dual-stack", + }, storage.State{IAAS: "aws"}) + Expect(err).NotTo(HaveOccurred()) + Expect(lbArgsHandler.GetLBStateCall.CallCount).To(Equal(1)) + Expect(lbArgsHandler.GetLBStateCall.Receives.Args).To(Equal(commands.LBArgs{ + LBType: "nlb", + CertPath: "cert", + KeyPath: "key", + ChainPath: "chain", + DualStack: true, + })) + }) + }) }) Describe("failure cases", func() { @@ -322,6 +344,30 @@ var _ = Describe("Plan", func() { }) }) + Context("aws with dual-stack", func() { + It("sets LB args with DualStack on the state", func() { + config, err := command.ParseArgs( + []string{ + "--lb-type", "nlb", + "--lb-cert", "cert", + "--lb-key", "key", + "--lb-chain", "chain", + "--dual-stack", + }, storage.State{IAAS: "aws"}) + Expect(err).NotTo(HaveOccurred()) + Expect(lbArgsHandler.GetLBStateCall.CallCount).To(Equal(1)) + Expect(lbArgsHandler.GetLBStateCall.Receives.Args).To(Equal(commands.LBArgs{ + LBType: "nlb", + CertPath: "cert", + KeyPath: "key", + ChainPath: "chain", + DualStack: true, + })) + + Expect(config.LB).To(Equal(lb)) + }) + }) + Context("gcp", func() { It("doesn't use --lb-chain", func() { _, err := command.ParseArgs( diff --git a/docs/advanced-configuration.md b/docs/advanced-configuration.md index 2b9e3f96a..53bf26ac3 100644 --- a/docs/advanced-configuration.md +++ b/docs/advanced-configuration.md @@ -4,6 +4,7 @@ * Using a BOSH ops-file with bbl * Customizing IaaS Paving with Terraform * Using VM Extensions for Cost Optimization +* IPv6 Dual-Stack Networking (AWS) * Applying and authoring plan patches, bundled modifications to default bbl configurations. ## Using a BOSH ops-file with bbl @@ -103,6 +104,33 @@ instance_groups: - Not recommended for singleton instances or databases - For legacy compatibility, the `preemptible` vm_extension is also available (uses the older GCP API) +## IPv6 Dual-Stack Networking (AWS) + +On AWS, you can enable IPv6 dual-stack networking for your BOSH environment. This assigns IPv6 CIDRs to all VPCs and subnets, and configures Network Load Balancers (NLBs) for dual-stack operation. + +Dual-stack requires the `nlb` load balancer type, as classic AWS Elastic Load Balancers do not support IPv6: + +```sh +bbl plan \ + --lb-type nlb \ + --lb-cert path/to/cert.pem \ + --lb-key path/to/key.pem \ + --lb-chain path/to/chain.pem \ + --dual-stack + +bbl up +``` + +This configures: +- IPv6 CIDR blocks on the VPC and all subnets (internal, LB, isolation segments) +- NLB `ip_address_type` set to `dualstack` (IPv4 when `--dual-stack` is not used) +- IPv6 ingress/egress rules on all security groups +- Separate `default_v6` and `private_v6` BOSH cloud-config networks for IPv6 subnets, alongside the existing `default` and `private` IPv4 networks + +Deployments that want to use IPv6 should reference the `default_v6` network as a secondary network in their manifest. The IPv4 network should remain the primary (with `default: [dns, gateway]`) for control plane communication. + +Dual-stack is currently only supported on AWS. Using `--dual-stack` on other IaaS providers will result in an error. + ## [Plan Patches](https://github.com/cloudfoundry/bosh-bootloader/tree/master/plan-patches) Through operations files and terraform overrides, all sorts of wild modifications can be done to the vanilla bosh environments that bbl creates. The basic principle of a plan patch is to make several modifications to a `bbl plan` in override files that bbl finds under `terraform/`, `cloud-config/`, and `{create,delete}-{jumpbox,director}.sh` . BBL will read and merge those into it's plan when you run `bbl up`. diff --git a/storage/lb.go b/storage/lb.go index 70fae2162..d636fb996 100644 --- a/storage/lb.go +++ b/storage/lb.go @@ -1,9 +1,10 @@ package storage type LB struct { - Type string `json:"type"` - Cert string `json:"cert"` - Key string `json:"key"` - Chain string `json:"chain"` - Domain string `json:"domain,omitempty"` + Type string `json:"type"` + Cert string `json:"cert"` + Key string `json:"key"` + Chain string `json:"chain"` + Domain string `json:"domain,omitempty"` + DualStack bool `json:"dual_stack,omitempty"` } diff --git a/terraform/aws/input_generator.go b/terraform/aws/input_generator.go index db9a30b1e..694dce8d6 100644 --- a/terraform/aws/input_generator.go +++ b/terraform/aws/input_generator.go @@ -44,7 +44,7 @@ func (i InputGenerator) Generate(state storage.State) (map[string]interface{}, e "availability_zones": azs, } - if state.LB.Type == "cf" { + if state.LB.Type == "cf" || state.LB.Type == "nlb" { inputs["ssl_certificate"] = state.LB.Cert inputs["ssl_certificate_private_key"] = state.LB.Key inputs["ssl_certificate_chain"] = state.LB.Chain @@ -59,6 +59,8 @@ func (i InputGenerator) Generate(state storage.State) (map[string]interface{}, e } } + inputs["dualstack"] = state.LB.DualStack + return inputs, nil } diff --git a/terraform/aws/input_generator_test.go b/terraform/aws/input_generator_test.go index dcdc459eb..3355a236c 100644 --- a/terraform/aws/input_generator_test.go +++ b/terraform/aws/input_generator_test.go @@ -64,6 +64,7 @@ var _ = Describe("InputGenerator", func() { "short_env_id": "some-env-id", "region": "some-region", "availability_zones": []string{"z1", "z2", "z3"}, + "dualstack": false, })) }) @@ -102,6 +103,7 @@ var _ = Describe("InputGenerator", func() { "ssl_certificate": "some-cert", "ssl_certificate_chain": "some-chain", "ssl_certificate_private_key": "some-key", + "dualstack": false, })) }) @@ -128,6 +130,67 @@ var _ = Describe("InputGenerator", func() { "ssl_certificate_private_key": "some-key", "system_domain": "some-domain", "parent_zone": "zone-id", + "dualstack": false, + })) + }) + }) + }) + + Context("when an nlb lb exists", func() { + var state storage.State + + BeforeEach(func() { + state = storage.State{ + IAAS: "aws", + EnvID: "some-env-id", + AWS: storage.AWS{ + AccessKeyID: "some-access-key-id", + SecretAccessKey: "some-secret-access-key", + Region: "some-region", + }, + LB: storage.LB{ + Type: "nlb", + Cert: "some-cert", + Chain: "some-chain", + Key: "some-key", + }, + } + }) + + It("returns a map with ssl certificate inputs and dualstack false", func() { + inputs, err := inputGenerator.Generate(state) + Expect(err).NotTo(HaveOccurred()) + + Expect(inputs).To(Equal(map[string]interface{}{ + "env_id": "some-env-id", + "short_env_id": "some-env-id", + "region": "some-region", + "availability_zones": []string{"z1", "z2", "z3"}, + "ssl_certificate": "some-cert", + "ssl_certificate_chain": "some-chain", + "ssl_certificate_private_key": "some-key", + "dualstack": false, + })) + }) + + Context("when dualstack is enabled", func() { + BeforeEach(func() { + state.LB.DualStack = true + }) + + It("returns a map with dualstack true", func() { + inputs, err := inputGenerator.Generate(state) + Expect(err).NotTo(HaveOccurred()) + + Expect(inputs).To(Equal(map[string]interface{}{ + "env_id": "some-env-id", + "short_env_id": "some-env-id", + "region": "some-region", + "availability_zones": []string{"z1", "z2", "z3"}, + "ssl_certificate": "some-cert", + "ssl_certificate_chain": "some-chain", + "ssl_certificate_private_key": "some-key", + "dualstack": true, })) }) }) diff --git a/terraform/aws/template_generator.go b/terraform/aws/template_generator.go index ca38c9634..9108ac9b9 100644 --- a/terraform/aws/template_generator.go +++ b/terraform/aws/template_generator.go @@ -13,8 +13,10 @@ type templates struct { iam string lbSubnet string cfLB string + cfNLB string cfDNS string concourseLB string + cfCommon string sslCertificate string isoSeg string vpc string @@ -43,7 +45,13 @@ func (tg TemplateGenerator) Generate(state storage.State) string { case "concourse": template = strings.Join([]string{template, tmpls.lbSubnet, tmpls.concourseLB}, "\n") case "cf": - template = strings.Join([]string{template, tmpls.lbSubnet, tmpls.cfLB, tmpls.sslCertificate, tmpls.isoSeg}, "\n") + template = strings.Join([]string{template, tmpls.lbSubnet, tmpls.cfLB, tmpls.cfCommon, tmpls.sslCertificate, tmpls.isoSeg}, "\n") + + if state.LB.Domain != "" { + template = strings.Join([]string{template, tmpls.cfDNS}, "\n") + } + case "nlb": + template = strings.Join([]string{template, tmpls.lbSubnet, tmpls.cfNLB, tmpls.cfCommon, tmpls.sslCertificate, tmpls.isoSeg}, "\n") if state.LB.Domain != "" { template = strings.Join([]string{template, tmpls.cfDNS}, "\n") @@ -60,6 +68,8 @@ func (t TemplateGenerator) readTemplates() templates { "lb_subnet.tf": "", "cf_lb.tf": "", "cf_dns.tf": "", + "cf_lb_common.tf": "", + "cf_nlb.tf": "", "concourse_lb.tf": "", "ssl_certificate.tf": "", "iso_segments.tf": "", @@ -94,8 +104,10 @@ func (t TemplateGenerator) readTemplates() templates { base: listings["base.tf"], iam: listings["iam.tf"], lbSubnet: listings["lb_subnet.tf"], + cfCommon: listings["cf_lb_common.tf"], cfLB: listings["cf_lb.tf"], cfDNS: listings["cf_dns.tf"], + cfNLB: listings["cf_nlb.tf"], concourseLB: listings["concourse_lb.tf"], sslCertificate: listings["ssl_certificate.tf"], isoSeg: listings["iso_segments.tf"], diff --git a/terraform/aws/template_generator_test.go b/terraform/aws/template_generator_test.go index e9285d1f0..e27265453 100644 --- a/terraform/aws/template_generator_test.go +++ b/terraform/aws/template_generator_test.go @@ -52,7 +52,7 @@ var _ = Describe("TemplateGenerator", func() { Context("when a CF lb type is provided with no system domain", func() { BeforeEach(func() { - expectedTemplate = expectTemplate("base", "iam", "vpc", "lb_subnet", "cf_lb", "ssl_certificate", "iso_segments") + expectedTemplate = expectTemplate("base", "iam", "vpc", "lb_subnet", "cf_lb", "cf_lb_common", "ssl_certificate", "iso_segments") lb = storage.LB{ Type: "cf", } @@ -65,7 +65,7 @@ var _ = Describe("TemplateGenerator", func() { Context("when a CF lb type is provided with a system domain", func() { BeforeEach(func() { - expectedTemplate = expectTemplate("base", "iam", "vpc", "lb_subnet", "cf_lb", "ssl_certificate", "iso_segments", "cf_dns") + expectedTemplate = expectTemplate("base", "iam", "vpc", "lb_subnet", "cf_lb", "cf_lb_common", "ssl_certificate", "iso_segments", "cf_dns") lb = storage.LB{ Type: "cf", Domain: "some-domain", @@ -76,6 +76,33 @@ var _ = Describe("TemplateGenerator", func() { checkTemplate(template, expectedTemplate) }) }) + + Context("when an NLB lb type is provided with no system domain", func() { + BeforeEach(func() { + expectedTemplate = expectTemplate("base", "iam", "vpc", "lb_subnet", "cf_nlb", "cf_lb_common", "ssl_certificate", "iso_segments") + lb = storage.LB{ + Type: "nlb", + } + }) + It("adds the lb subnet, cf nlb, ssl cert and iso seg to the base template", func() { + template := templateGenerator.Generate(storage.State{LB: lb}) + checkTemplate(template, expectedTemplate) + }) + }) + + Context("when an NLB lb type is provided with a system domain", func() { + BeforeEach(func() { + expectedTemplate = expectTemplate("base", "iam", "vpc", "lb_subnet", "cf_nlb", "cf_lb_common", "ssl_certificate", "iso_segments", "cf_dns") + lb = storage.LB{ + Type: "nlb", + Domain: "some-domain", + } + }) + It("adds the domain", func() { + template := templateGenerator.Generate(storage.State{LB: lb}) + checkTemplate(template, expectedTemplate) + }) + }) }) }) diff --git a/terraform/aws/templates/base.tf b/terraform/aws/templates/base.tf index 7fc6ca066..4b5dd4f1c 100644 --- a/terraform/aws/templates/base.tf +++ b/terraform/aws/templates/base.tf @@ -392,7 +392,7 @@ resource "aws_subnet" "bosh_subnet" { cidr_block = cidrsubnet(var.vpc_cidr, 8, 0) ipv6_cidr_block = var.dualstack ? "${cidrsubnet(aws_vpc.vpc[0].ipv6_cidr_block, 8, 0)}" : null - assign_ipv6_address_on_creation = var.dualstack + assign_ipv6_address_on_creation = false enable_dns64 = var.dualstack tags = { @@ -429,7 +429,7 @@ resource "aws_subnet" "internal_subnets" { availability_zone = element(var.availability_zones, count.index) ipv6_cidr_block = var.dualstack ? "${cidrsubnet(aws_vpc.vpc[0].ipv6_cidr_block, 8, count.index + 1)}" : null - assign_ipv6_address_on_creation = var.dualstack + assign_ipv6_address_on_creation = false enable_dns64 = var.dualstack tags = { diff --git a/terraform/aws/templates/cf_dns.tf b/terraform/aws/templates/cf_dns.tf index f2b37b8cb..15a882506 100644 --- a/terraform/aws/templates/cf_dns.tf +++ b/terraform/aws/templates/cf_dns.tf @@ -39,7 +39,7 @@ resource "aws_route53_record" "wildcard_dns" { type = "CNAME" ttl = 300 - records = ["${aws_elb.cf_router_lb.dns_name}"] + records = var.dualstack ? [aws_lb.cf_router_lb.dns_name] : ["${aws_elb.cf_router_lb.dns_name}"] } resource "aws_route53_record" "ssh" { @@ -48,7 +48,7 @@ resource "aws_route53_record" "ssh" { type = "CNAME" ttl = 300 - records = ["${aws_elb.cf_ssh_lb.dns_name}"] + records = var.dualstack ? [aws_lb.cf_ssh_lb.dns_name] : ["${aws_elb.cf_ssh_lb.dns_name}"] } resource "aws_route53_record" "bosh" { @@ -66,7 +66,7 @@ resource "aws_route53_record" "tcp" { type = "CNAME" ttl = 300 - records = ["${aws_elb.cf_tcp_lb.dns_name}"] + records = var.dualstack ? [aws_lb.cf_tcp_lb.dns_name] : ["${aws_elb.cf_tcp_lb.dns_name}"] } resource "aws_route53_record" "iso" { diff --git a/terraform/aws/templates/cf_lb.tf b/terraform/aws/templates/cf_lb.tf index 8521f8cb2..1c9a51303 100644 --- a/terraform/aws/templates/cf_lb.tf +++ b/terraform/aws/templates/cf_lb.tf @@ -1,75 +1,3 @@ -variable "elb_idle_timeout" { - type = number - default = 60 -} - -resource "aws_security_group" "cf_ssh_lb_security_group" { - name = "${var.env_id}-cf-ssh-lb-security-group" - description = "CF SSH" - vpc_id = local.vpc_id - - ingress { - cidr_blocks = ["0.0.0.0/0"] - ipv6_cidr_blocks = var.dualstack ? ["::/0"] : null - protocol = "tcp" - from_port = 2222 - to_port = 2222 - } - - egress { - from_port = 0 - to_port = 0 - protocol = "-1" - cidr_blocks = ["0.0.0.0/0"] - ipv6_cidr_blocks = var.dualstack ? ["::/0"] : null - } - - tags = { - Name = "${var.env_id}-cf-ssh-lb-security-group" - } - - lifecycle { - ignore_changes = [name] - } -} - -output "cf_ssh_lb_security_group" { - value = aws_security_group.cf_ssh_lb_security_group.id -} - -resource "aws_security_group" "cf_ssh_lb_internal_security_group" { - name = "${var.env_id}-cf-ssh-lb-internal-security-group" - description = "CF SSH Internal" - vpc_id = local.vpc_id - - ingress { - security_groups = ["${aws_security_group.cf_ssh_lb_security_group.id}"] - protocol = "tcp" - from_port = 2222 - to_port = 2222 - } - - egress { - from_port = 0 - to_port = 0 - protocol = "-1" - cidr_blocks = ["0.0.0.0/0"] - ipv6_cidr_blocks = var.dualstack ? ["::/0"] : null - } - - tags = { - Name = "${var.env_id}-cf-ssh-lb-internal-security-group" - } - - lifecycle { - ignore_changes = [name] - } -} - -output "cf_ssh_lb_internal_security_group" { - value = aws_security_group.cf_ssh_lb_internal_security_group.id -} - resource "aws_elb" "cf_ssh_lb" { name = "${var.short_env_id}-cf-ssh-lb" cross_zone_load_balancing = true @@ -107,88 +35,6 @@ output "cf_ssh_lb_url" { value = aws_elb.cf_ssh_lb.dns_name } -resource "aws_security_group" "cf_router_lb_security_group" { - name = "${var.env_id}-cf-router-lb-security-group" - description = "CF Router" - vpc_id = local.vpc_id - - ingress { - cidr_blocks = ["0.0.0.0/0"] - ipv6_cidr_blocks = var.dualstack ? ["::/0"] : null - protocol = "tcp" - from_port = 80 - to_port = 80 - } - - ingress { - cidr_blocks = ["0.0.0.0/0"] - ipv6_cidr_blocks = var.dualstack ? ["::/0"] : null - protocol = "tcp" - from_port = 443 - to_port = 443 - } - - ingress { - cidr_blocks = ["0.0.0.0/0"] - ipv6_cidr_blocks = var.dualstack ? ["::/0"] : null - protocol = "tcp" - from_port = 4443 - to_port = 4443 - } - - egress { - from_port = 0 - to_port = 0 - protocol = "-1" - cidr_blocks = ["0.0.0.0/0"] - ipv6_cidr_blocks = var.dualstack ? ["::/0"] : null - } - - tags = { - Name = "${var.env_id}-cf-router-lb-security-group" - } - - lifecycle { - ignore_changes = [name] - } -} - -output "cf_router_lb_security_group" { - value = aws_security_group.cf_router_lb_security_group.id -} - -resource "aws_security_group" "cf_router_lb_internal_security_group" { - name = "${var.env_id}-cf-router-lb-internal-security-group" - description = "CF Router Internal" - vpc_id = local.vpc_id - - ingress { - security_groups = ["${aws_security_group.cf_router_lb_security_group.id}"] - protocol = "tcp" - from_port = 80 - to_port = 80 - } - - egress { - from_port = 0 - to_port = 0 - protocol = "-1" - cidr_blocks = ["0.0.0.0/0"] - ipv6_cidr_blocks = var.dualstack ? ["::/0"] : null - } - - tags = { - Name = "${var.env_id}-cf-router-lb-internal-security-group" - } - - lifecycle { - ignore_changes = [name] - } -} - -output "cf_router_lb_internal_security_group" { - value = aws_security_group.cf_router_lb_internal_security_group.id -} resource "aws_elb" "cf_router_lb" { name = "${var.short_env_id}-cf-router-lb" @@ -235,21 +81,6 @@ resource "aws_elb" "cf_router_lb" { } } -resource "aws_lb_target_group" "cf_router_4443" { - name = "${var.short_env_id}-routertg-4443" - port = 4443 - protocol = "TCP" - vpc_id = local.vpc_id - - health_check { - protocol = "TCP" - } - - tags = { - Name = "${var.env_id}" - } -} - output "cf_router_lb_name" { value = aws_elb.cf_router_lb.name } @@ -258,80 +89,6 @@ output "cf_router_lb_url" { value = aws_elb.cf_router_lb.dns_name } -resource "aws_security_group" "cf_tcp_lb_security_group" { - name = "${var.env_id}-cf-tcp-lb-security-group" - description = "CF TCP" - vpc_id = local.vpc_id - - ingress { - cidr_blocks = ["0.0.0.0/0"] - ipv6_cidr_blocks = var.dualstack ? ["::/0"] : null - protocol = "tcp" - from_port = 1024 - to_port = 1123 - } - - egress { - from_port = 0 - to_port = 0 - protocol = "-1" - cidr_blocks = ["0.0.0.0/0"] - ipv6_cidr_blocks = var.dualstack ? ["::/0"] : null - } - - tags = { - Name = "${var.env_id}-cf-tcp-lb-security-group" - } - - lifecycle { - ignore_changes = [name] - } -} - -output "cf_tcp_lb_security_group" { - value = aws_security_group.cf_tcp_lb_security_group.id -} - -resource "aws_security_group" "cf_tcp_lb_internal_security_group" { - name = "${var.env_id}-cf-tcp-lb-internal-security-group" - description = "CF TCP Internal" - vpc_id = local.vpc_id - - ingress { - security_groups = ["${aws_security_group.cf_tcp_lb_security_group.id}"] - protocol = "tcp" - from_port = 1024 - to_port = 1123 - } - - ingress { - security_groups = ["${aws_security_group.cf_tcp_lb_security_group.id}"] - protocol = "tcp" - from_port = 80 - to_port = 80 - } - - egress { - from_port = 0 - to_port = 0 - protocol = "-1" - cidr_blocks = ["0.0.0.0/0"] - ipv6_cidr_blocks = var.dualstack ? ["::/0"] : null - } - - tags = { - Name = "${var.env_id}-cf-tcp-lb-security-group" - } - - lifecycle { - ignore_changes = [name] - } -} - -output "cf_tcp_lb_internal_security_group" { - value = aws_security_group.cf_tcp_lb_internal_security_group.id -} - resource "aws_elb" "cf_tcp_lb" { name = "${var.short_env_id}-cf-tcp-lb" cross_zone_load_balancing = true diff --git a/terraform/aws/templates/cf_lb_common.tf b/terraform/aws/templates/cf_lb_common.tf new file mode 100644 index 000000000..46e3dcdfb --- /dev/null +++ b/terraform/aws/templates/cf_lb_common.tf @@ -0,0 +1,247 @@ +variable "elb_idle_timeout" { + type = number + default = 60 +} + +resource "aws_lb_target_group" "cf_router_4443" { + name = "${var.short_env_id}-routertg-4443" + port = 4443 + protocol = "TCP" + vpc_id = local.vpc_id + + health_check { + protocol = "TCP" + } + + tags = { + Name = "${var.env_id}" + } +} + +resource "aws_security_group" "cf_ssh_lb_security_group" { + name = "${var.env_id}-cf-ssh-lb-security-group" + description = "CF SSH" + vpc_id = local.vpc_id + + ingress { + cidr_blocks = ["0.0.0.0/0"] + ipv6_cidr_blocks = var.dualstack ? ["::/0"] : null + protocol = "tcp" + from_port = 2222 + to_port = 2222 + } + + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + ipv6_cidr_blocks = var.dualstack ? ["::/0"] : null + } + + tags = { + Name = "${var.env_id}-cf-ssh-lb-security-group" + } + + lifecycle { + ignore_changes = [name] + } +} + +resource "aws_security_group" "cf_ssh_lb_internal_security_group" { + name = "${var.env_id}-cf-ssh-lb-internal-security-group" + description = "CF SSH Internal" + vpc_id = local.vpc_id + + ingress { + security_groups = ["${aws_security_group.cf_ssh_lb_security_group.id}"] + protocol = "tcp" + from_port = 2222 + to_port = 2222 + } + + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + ipv6_cidr_blocks = var.dualstack ? ["::/0"] : null + } + + tags = { + Name = "${var.env_id}-cf-ssh-lb-internal-security-group" + } + + lifecycle { + ignore_changes = [name] + } +} + +resource "aws_security_group" "cf_router_lb_security_group" { + name = "${var.env_id}-cf-router-lb-security-group" + description = "CF Router" + vpc_id = local.vpc_id + + ingress { + cidr_blocks = ["0.0.0.0/0"] + ipv6_cidr_blocks = var.dualstack ? ["::/0"] : null + protocol = "tcp" + from_port = 80 + to_port = 80 + } + + ingress { + cidr_blocks = ["0.0.0.0/0"] + ipv6_cidr_blocks = var.dualstack ? ["::/0"] : null + protocol = "tcp" + from_port = 443 + to_port = 443 + } + + ingress { + cidr_blocks = ["0.0.0.0/0"] + ipv6_cidr_blocks = var.dualstack ? ["::/0"] : null + protocol = "tcp" + from_port = 4443 + to_port = 4443 + } + + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + ipv6_cidr_blocks = var.dualstack ? ["::/0"] : null + } + + tags = { + Name = "${var.env_id}-cf-router-lb-security-group" + } + + lifecycle { + ignore_changes = [name] + } +} + +resource "aws_security_group" "cf_router_lb_internal_security_group" { + name = "${var.env_id}-cf-router-lb-internal-security-group" + description = "CF Router Internal" + vpc_id = local.vpc_id + + ingress { + security_groups = ["${aws_security_group.cf_router_lb_security_group.id}"] + protocol = "tcp" + from_port = 80 + to_port = 80 + } + + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + ipv6_cidr_blocks = var.dualstack ? ["::/0"] : null + } + + tags = { + Name = "${var.env_id}-cf-router-lb-internal-security-group" + } + + lifecycle { + ignore_changes = [name] + } +} + + +resource "aws_security_group" "cf_tcp_lb_security_group" { + name = "${var.env_id}-cf-tcp-lb-security-group" + description = "CF TCP" + vpc_id = local.vpc_id + + ingress { + cidr_blocks = ["0.0.0.0/0"] + ipv6_cidr_blocks = var.dualstack ? ["::/0"] : null + protocol = "tcp" + from_port = 1024 + to_port = 1123 + } + + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + ipv6_cidr_blocks = var.dualstack ? ["::/0"] : null + } + + tags = { + Name = "${var.env_id}-cf-tcp-lb-security-group" + } + + lifecycle { + ignore_changes = [name] + } +} + + +resource "aws_security_group" "cf_tcp_lb_internal_security_group" { + name = "${var.env_id}-cf-tcp-lb-internal-security-group" + description = "CF TCP Internal" + vpc_id = local.vpc_id + + ingress { + security_groups = ["${aws_security_group.cf_tcp_lb_security_group.id}"] + protocol = "tcp" + from_port = 1024 + to_port = 1123 + } + + ingress { + security_groups = ["${aws_security_group.cf_tcp_lb_security_group.id}"] + protocol = "tcp" + from_port = 80 + to_port = 80 + } + + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + ipv6_cidr_blocks = var.dualstack ? ["::/0"] : null + } + + tags = { + Name = "${var.env_id}-cf-tcp-lb-security-group" + } + + lifecycle { + ignore_changes = [name] + } +} + +output "cf_tcp_lb_security_group" { + value = aws_security_group.cf_tcp_lb_security_group.id +} + +output "cf_tcp_lb_internal_security_group" { + value = aws_security_group.cf_tcp_lb_internal_security_group.id +} + +output "cf_router_lb_internal_security_group" { + value = aws_security_group.cf_router_lb_internal_security_group.id +} + +output "cf_router_lb_security_group" { + value = aws_security_group.cf_router_lb_security_group.id +} + +output "cf_ssh_lb_internal_security_group" { + value = aws_security_group.cf_ssh_lb_internal_security_group.id +} + + +output "cf_ssh_lb_security_group" { + value = aws_security_group.cf_ssh_lb_security_group.id +} + diff --git a/terraform/aws/templates/cf_nlb.tf b/terraform/aws/templates/cf_nlb.tf new file mode 100644 index 000000000..6c3d69481 --- /dev/null +++ b/terraform/aws/templates/cf_nlb.tf @@ -0,0 +1,202 @@ +resource "aws_lb" "cf_ssh_lb" { + name = "${var.short_env_id}-cf-ssh-lb" + internal = false + load_balancer_type = "network" + security_groups = [aws_security_group.cf_ssh_lb_security_group.id] + subnets = [for subnet in aws_subnet.lb_subnets : subnet.id] + + enable_deletion_protection = false + enable_cross_zone_load_balancing = true + + # idle_timeout = var.elb_idle_timeout + ip_address_type = var.dualstack ? "dualstack" : "ipv4" + + tags = { + Name = var.env_id + } +} + +resource "aws_lb" "cf_router_lb" { + name = "${var.short_env_id}-cf-router-lb" + internal = false + load_balancer_type = "network" + security_groups = [aws_security_group.cf_router_lb_security_group.id] + subnets = [for subnet in aws_subnet.lb_subnets : subnet.id] + + enable_deletion_protection = false + enable_cross_zone_load_balancing = true + + # idle_timeout = var.elb_idle_timeout + ip_address_type = var.dualstack ? "dualstack" : "ipv4" + + tags = { + Name = var.env_id + } +} + +resource "aws_lb" "cf_tcp_lb" { + name = "${var.short_env_id}-cf-tcp-lb" + internal = false + load_balancer_type = "network" + security_groups = [aws_security_group.cf_tcp_lb_security_group.id] + subnets = [for subnet in aws_subnet.lb_subnets : subnet.id] + + enable_deletion_protection = false + enable_cross_zone_load_balancing = true + + # idle_timeout = var.elb_idle_timeout + ip_address_type = var.dualstack ? "dualstack" : "ipv4" + + tags = { + Name = var.env_id + } +} + +resource "aws_lb_listener" "cf_tcp_lb" { + for_each = toset([for x in range(1024, 1074, 1) : tostring(x)]) + + load_balancer_arn = aws_lb.cf_tcp_lb.arn + port = each.value + protocol = "TCP" + + default_action { + type = "forward" + target_group_arn = aws_lb_target_group.cf_tcp_nlb[each.value].arn + } + + depends_on = [ + aws_lb_target_group.cf_tcp_nlb + ] +} + +resource "aws_lb_target_group" "cf_tcp_nlb" { + for_each = toset([for x in range(1024, 1074, 1) : tostring(x)]) + + name = "${var.short_env_id}-cf-tcp-nlb-${each.value}" + port = each.value + protocol = "TCP" + vpc_id = local.vpc_id + + health_check { + healthy_threshold = 6 + unhealthy_threshold = 3 + interval = 15 + protocol = "TCP" + port = 80 + } + + tags = { + Name = "${var.env_id}-${each.value}" + } +} + +resource "aws_lb_target_group" "cf_ssh_nlb" { + name = "${var.short_env_id}-cf-ssh-nlb" + port = 2222 + protocol = "TCP" + vpc_id = local.vpc_id + + health_check { + healthy_threshold = 5 + unhealthy_threshold = 2 + interval = 12 + protocol = "TCP" + port = 2222 + } + + tags = { + Name = "${var.env_id}" + } +} + + +resource "aws_lb_target_group" "cf_router_nlb" { + name = "${var.short_env_id}-cf-router-nlb" + port = 80 + protocol = "TCP" + vpc_id = local.vpc_id + + health_check { + healthy_threshold = 5 + unhealthy_threshold = 2 + interval = 15 + protocol = "TCP" + port = 80 + } + + tags = { + Name = "${var.env_id}" + } +} + +resource "aws_lb_listener" "cf_ssh" { + load_balancer_arn = aws_lb.cf_ssh_lb.arn + port = "2222" + protocol = "TCP" + + default_action { + type = "forward" + target_group_arn = aws_lb_target_group.cf_ssh_nlb.arn + } +} + +resource "aws_lb_listener" "cf_router_http" { + load_balancer_arn = aws_lb.cf_router_lb.arn + port = "80" + protocol = "TCP" + + default_action { + type = "forward" + target_group_arn = aws_lb_target_group.cf_router_nlb.arn + } +} + +resource "aws_lb_listener" "cf_router_https" { + load_balancer_arn = aws_lb.cf_router_lb.arn + port = "443" + protocol = "TLS" + ssl_policy = "ELBSecurityPolicy-2016-08" + certificate_arn = aws_iam_server_certificate.lb_cert.arn + + default_action { + type = "forward" + target_group_arn = aws_lb_target_group.cf_router_nlb.arn + } +} + +resource "aws_lb_listener" "cf_router_4443" { + load_balancer_arn = aws_lb.cf_router_lb.arn + port = "4443" + protocol = "TLS" + ssl_policy = "ELBSecurityPolicy-2016-08" + certificate_arn = aws_iam_server_certificate.lb_cert.arn + + default_action { + type = "forward" + target_group_arn = aws_lb_target_group.cf_router_nlb.arn + } +} + +output "cf_ssh_lb_name" { + value = aws_lb.cf_ssh_lb.name +} + +output "cf_ssh_lb_url" { + value = aws_lb.cf_ssh_lb.dns_name +} + +output "cf_router_lb_name" { + value = aws_lb.cf_router_lb.name +} + +output "cf_router_lb_url" { + value = aws_lb.cf_router_lb.dns_name +} + +output "cf_tcp_lb_name" { + value = aws_lb.cf_tcp_lb.name +} + +output "cf_tcp_lb_url" { + value = aws_lb.cf_tcp_lb.dns_name +} diff --git a/terraform/aws/templates/iso_segments.tf b/terraform/aws/templates/iso_segments.tf index fc9945dfa..3c34c0e97 100644 --- a/terraform/aws/templates/iso_segments.tf +++ b/terraform/aws/templates/iso_segments.tf @@ -5,17 +5,17 @@ variable "isolation_segments" { } variable "iso_to_bosh_ports" { - type = list(any) + type = list(number) default = [22, 6868, 2555, 4222, 25250] } variable "iso_to_shared_tcp_ports" { - type = list(any) + type = list(number) default = [9090, 9091, 8082, 8300, 8301, 8889, 8443, 3000, 4443, 8080, 3457, 9023, 9022, 4222] } variable "iso_to_shared_udp_ports" { - type = list(any) + type = list(number) default = [8301, 8302, 8600] } @@ -29,7 +29,7 @@ resource "aws_subnet" "iso_subnets" { cidr_block = cidrsubnet(var.vpc_cidr, 4, count.index + length(var.availability_zones) + 1) ipv6_cidr_block = var.dualstack ? "${cidrsubnet(aws_vpc.vpc[0].ipv6_cidr_block, 8, count.index + 2 + length(var.availability_zones))}" : null availability_zone = element(var.availability_zones, count.index) - assign_ipv6_address_on_creation = var.dualstack + assign_ipv6_address_on_creation = false enable_dns64 = var.dualstack tags = { @@ -39,12 +39,13 @@ resource "aws_subnet" "iso_subnets" { resource "aws_route_table_association" "route_iso_subnets" { count = local.iso_az_count - subnet_id = element(aws_subnet.iso_subnets.*.id, count.index) + subnet_id = aws_subnet.iso_subnets[count.index].id route_table_id = aws_route_table.nated_route_table.id } + resource "aws_elb" "iso_router_lb" { - count = var.isolation_segments + count = var.isolation_segments == "1" && var.dualstack == false ? 1 : 0 name = "${var.short_env_id}-iso-router-lb" cross_zone_load_balancing = true @@ -88,6 +89,85 @@ resource "aws_elb" "iso_router_lb" { } } +resource "aws_lb" "iso_router_nlb" { + count = var.isolation_segments == "1" && var.dualstack ? 1 : 0 + name = "${var.short_env_id}-iso-router-lb" + internal = false + load_balancer_type = "network" + security_groups = [aws_security_group.cf_router_lb_security_group.id] + subnets = [for subnet in aws_subnet.lb_subnets : subnet.id] + + enable_deletion_protection = false + enable_cross_zone_load_balancing = true + + # idle_timeout = var.elb_idle_timeout + ip_address_type = "dualstack" + + tags = { + Name = var.env_id + } +} + +resource "aws_lb_target_group" "iso_router_nlb_http" { + count = var.isolation_segments == "1" && var.dualstack ? 1 : 0 + name = "${var.short_env_id}-iso-router-nlb-http" + port = 80 + protocol = "HTTP" + vpc_id = local.vpc_id + + health_check { + healthy_threshold = 5 + unhealthy_threshold = 2 + interval = 15 + protocol = "TCP" + port = 80 + } + + tags = { + Name = "${var.env_id}" + } +} + +resource "aws_lb_listener" "iso_router_nlb_http" { + count = var.isolation_segments == "1" && var.dualstack ? 1 : 0 + load_balancer_arn = aws_lb.iso_router_nlb[0].arn + port = "80" + protocol = "TCP" + + default_action { + type = "forward" + target_group_arn = aws_lb_target_group.iso_router_nlb_http[0].arn + } +} + +resource "aws_lb_listener" "iso_router_nlb_https" { + count = var.isolation_segments == "1" && var.dualstack ? 1 : 0 + load_balancer_arn = aws_lb.iso_router_nlb[0].arn + port = "443" + protocol = "TLS" + ssl_policy = "ELBSecurityPolicy-2016-08" + certificate_arn = aws_iam_server_certificate.lb_cert.arn + + default_action { + type = "forward" + target_group_arn = aws_lb_target_group.iso_router_nlb_http[0].arn + } +} + +resource "aws_lb_listener" "iso_router_nlb_4443" { + count = var.isolation_segments == "1" && var.dualstack ? 1 : 0 + load_balancer_arn = aws_lb.iso_router_nlb[0].arn + port = "4443" + protocol = "TLS" + ssl_policy = "ELBSecurityPolicy-2016-08" + certificate_arn = aws_iam_server_certificate.lb_cert.arn + + default_action { + type = "forward" + target_group_arn = aws_lb_target_group.iso_router_nlb_http[0].arn + } +} + resource "aws_lb_target_group" "iso_router_lb_4443" { count = var.isolation_segments name = "${var.short_env_id}-isotg-4443" @@ -211,7 +291,7 @@ resource "aws_security_group_rule" "nat_to_isolated_cells_rule" { } output "cf_iso_router_lb_name" { - value = one(aws_elb.iso_router_lb[*].name) + value = var.dualstack ? one(aws_lb.iso_router_nlb[*].name) : one(aws_elb.iso_router_lb[*].name) } output "iso_security_group_id" { diff --git a/terraform/aws/templates/lb_subnet.tf b/terraform/aws/templates/lb_subnet.tf index a20f9bf65..965e06f09 100644 --- a/terraform/aws/templates/lb_subnet.tf +++ b/terraform/aws/templates/lb_subnet.tf @@ -4,7 +4,7 @@ resource "aws_subnet" "lb_subnets" { cidr_block = cidrsubnet(var.vpc_cidr, 8, count.index + 2) ipv6_cidr_block = var.dualstack ? "${cidrsubnet(aws_vpc.vpc[0].ipv6_cidr_block, 8, count.index + 1 + length(var.availability_zones))}" : null availability_zone = element(var.availability_zones, count.index) - assign_ipv6_address_on_creation = var.dualstack + assign_ipv6_address_on_creation = false enable_dns64 = var.dualstack tags = {