diff --git a/cmd/autobahn-e2e/aws.go b/cmd/autobahn-e2e/aws.go index 77f1dc2304..5d1ae33f62 100644 --- a/cmd/autobahn-e2e/aws.go +++ b/cmd/autobahn-e2e/aws.go @@ -2,6 +2,7 @@ package main import ( "context" + "encoding/json" "fmt" "io" "net" @@ -14,7 +15,15 @@ import ( "time" ) -const ubuntuARM64AMIParameter = "/aws/service/canonical/ubuntu/server/24.04/stable/current/arm64/hvm/ebs-gp3/ami-id" +const ( + ubuntuARM64AMIParameter = "/aws/service/canonical/ubuntu/server/24.04/stable/current/arm64/hvm/ebs-gp3/ami-id" + ubuntuAMD64AMIParameter = "/aws/service/canonical/ubuntu/server/24.04/stable/current/amd64/hvm/ebs-gp3/ami-id" + nativeBuildReadyFile = "autobahn-native-build.ready" + nativeBuildFailedFile = "autobahn-native-build.failed" + nativeBuildLogFile = "autobahn-native-build.log" + nativeBuildStatusReady = "ready" + nativeBuildStatusFailed = "failed" +) var sshUserPattern = regexp.MustCompile(`^[a-z_][a-z0-9_-]*$`) @@ -41,6 +50,12 @@ func (c awsClient) environment() []string { } func (a *application) deployAWS(ctx context.Context, options deployOptions) error { + if options.architecture == "" { + options.architecture = "arm64" + } + if options.goGC == "" { + options.goGC = "200" + } if options.volumeSize < 20 { return fmt.Errorf("--volume-size must be at least 20 GiB") } @@ -50,7 +65,17 @@ func (a *application) deployAWS(ctx context.Context, options deployOptions) erro if !sshUserPattern.MatchString(options.sshUser) { return fmt.Errorf("invalid --ssh-user %q", options.sshUser) } - for _, name := range []string{"aws", "git", "ssh"} { + amiParameter, err := ubuntuAMIParameter(options.architecture) + if err != nil { + return err + } + if options.goMaxProcs < 0 { + return fmt.Errorf("--gomaxprocs cannot be negative") + } + if err := validateGoGC(options.goGC); err != nil { + return err + } + for _, name := range []string{"aws", "git", "scp", "ssh"} { if err := a.runner.lookPath(name); err != nil { return err } @@ -67,7 +92,7 @@ func (a *application) deployAWS(ctx context.Context, options deployOptions) erro if amiID == "" { amiID, err = client.output(ctx, "ssm", "get-parameter", - "--name", ubuntuARM64AMIParameter, + "--name", amiParameter, "--query", "Parameter.Value", "--output", "text", ) @@ -91,14 +116,17 @@ func (a *application) deployAWS(ctx context.Context, options deployOptions) erro Target: targetAWS, Status: "provisioning", CreatedAt: time.Now().UTC(), - Nodes: clusterNodes(dockerClusterSize), + Nodes: nativeClusterNodes(awsClusterSize), AWS: &awsState{ - Region: options.region, - Profile: options.profile, - SSHUser: options.sshUser, - RemoteDir: filepath.Join("/home", options.sshUser, "sei-chain-"+options.name), - RepoURL: repoURL, - Ref: ref, + Region: options.region, + Profile: options.profile, + SSHUser: options.sshUser, + RemoteDir: filepath.Join("/home", options.sshUser, "sei-chain-"+options.name), + RepoURL: repoURL, + Ref: ref, + GoMaxProcs: options.goMaxProcs, + GoGC: options.goGC, + Architecture: options.architecture, }, } fail := func(cause error) error { @@ -132,6 +160,13 @@ func (a *application) deployAWS(ctx context.Context, options deployOptions) erro ); err != nil { return fail(err) } + if _, err := client.output(ctx, + "ec2", "authorize-security-group-ingress", + "--group-id", state.AWS.SecurityGroupID, + "--ip-permissions", fmt.Sprintf("IpProtocol=-1,UserIdGroupPairs=[{GroupId=%s}]", state.AWS.SecurityGroupID), + ); err != nil { + return fail(err) + } if _, err := client.output(ctx, "ec2", "authorize-security-group-ingress", "--group-id", state.AWS.SecurityGroupID, @@ -173,7 +208,7 @@ func (a *application) deployAWS(ctx context.Context, options deployOptions) erro return err } - userDataPath, err := writeUserData(a.stateDir, options.name, options.sshUser) + userDataPath, err := writeUserData(a.stateDir, options.name) if err != nil { return fail(err) } @@ -188,39 +223,61 @@ func (a *application) deployAWS(ctx context.Context, options deployOptions) erro "--metadata-options", "HttpTokens=required,HttpEndpoint=enabled", "--block-device-mappings", fmt.Sprintf("DeviceName=/dev/sda1,Ebs={VolumeSize=%d,VolumeType=gp3,DeleteOnTermination=true}", options.volumeSize), "--user-data", "file://" + userDataPath, + "--count", fmt.Sprint(awsClusterSize), "--tag-specifications", fmt.Sprintf("ResourceType=instance,Tags=[{Key=Name,Value=sei-autobahn-e2e-%s},{Key=sei-autobahn-e2e-cluster,Value=%s}]", options.name, options.name), - "--query", "Instances[0].InstanceId", + "--query", "Instances[].InstanceId", "--output", "text", } if options.subnetID != "" { runArgs = append(runArgs, "--subnet-id", options.subnetID) } - instanceID, err := client.output(ctx, runArgs...) + instanceIDsOutput, err := client.output(ctx, runArgs...) if err != nil { return fail(err) } - state.AWS.InstanceID = strings.TrimSpace(instanceID) + instanceIDs := strings.Fields(instanceIDsOutput) + if len(instanceIDs) != awsClusterSize { + return fail(fmt.Errorf("AWS returned %d instances, expected %d", len(instanceIDs), awsClusterSize)) + } + state.AWS.Instances = make([]awsInstanceState, len(instanceIDs)) + for nodeIndex, instanceID := range instanceIDs { + state.AWS.Instances[nodeIndex] = awsInstanceState{NodeIndex: nodeIndex, InstanceID: instanceID} + if _, err := client.output(ctx, + "ec2", "create-tags", + "--resources", instanceID, + "--tags", fmt.Sprintf("Key=Name,Value=sei-autobahn-e2e-%s-node-%d", options.name, nodeIndex), + ); err != nil { + return fail(err) + } + } if err := a.store().save(state); err != nil { return err } - if err := client.stream(ctx, "ec2", "wait", "instance-running", "--instance-ids", state.AWS.InstanceID); err != nil { + waitArgs := append([]string{"ec2", "wait", "instance-running", "--instance-ids"}, instanceIDs...) + if err := client.stream(ctx, waitArgs...); err != nil { return fail(err) } - if err := client.stream(ctx, "ec2", "wait", "instance-status-ok", "--instance-ids", state.AWS.InstanceID); err != nil { + waitArgs = append([]string{"ec2", "wait", "instance-status-ok", "--instance-ids"}, instanceIDs...) + if err := client.stream(ctx, waitArgs...); err != nil { return fail(err) } - publicIP, err := client.output(ctx, - "ec2", "describe-instances", - "--instance-ids", state.AWS.InstanceID, - "--query", "Reservations[0].Instances[0].PublicIpAddress", - "--output", "text", - ) + instances, err := describeAWSInstances(ctx, client, instanceIDs) if err != nil { return fail(err) } - state.AWS.PublicIP = strings.TrimSpace(publicIP) - if state.AWS.PublicIP == "" || state.AWS.PublicIP == "None" { - return fail(fmt.Errorf("ec2 instance has no public IP; choose a subnet that assigns public addresses")) + for nodeIndex, instanceID := range instanceIDs { + instance, ok := instances[instanceID] + if !ok { + return fail(fmt.Errorf("describe-instances omitted %s", instanceID)) + } + if instance.PublicIP == "" { + return fail(fmt.Errorf("ec2 instance %s has no public IP; choose a subnet that assigns public addresses", instanceID)) + } + if instance.PrivateIP == "" { + return fail(fmt.Errorf("ec2 instance %s has no private IP", instanceID)) + } + instance.NodeIndex = nodeIndex + state.AWS.Instances[nodeIndex] = instance } if err := a.store().save(state); err != nil { return err @@ -241,10 +298,68 @@ func (a *application) deployAWS(ctx context.Context, options deployOptions) erro if err := a.store().save(state); err != nil { return err } - _, _ = fmt.Fprintf(a.stdout, "Cluster %s is ready on EC2 instance %s (%s).\n", state.Name, state.AWS.InstanceID, state.AWS.PublicIP) + _, _ = fmt.Fprintf(a.stdout, "Cluster %s is ready on %d native EC2 instances.\n", state.Name, len(state.AWS.Instances)) return nil } +func ubuntuAMIParameter(architecture string) (string, error) { + switch architecture { + case "arm64": + return ubuntuARM64AMIParameter, nil + case "amd64": + return ubuntuAMD64AMIParameter, nil + default: + return "", fmt.Errorf("unsupported --architecture %q; use arm64 or amd64", architecture) + } +} + +func validateGoGC(value string) error { + if value == "off" { + return nil + } + parsed, err := strconv.Atoi(value) + if err != nil || parsed < 0 { + return fmt.Errorf("--gogc must be a non-negative integer or off") + } + return nil +} + +func describeAWSInstances(ctx context.Context, client awsClient, instanceIDs []string) (map[string]awsInstanceState, error) { + args := append([]string{"ec2", "describe-instances", "--instance-ids"}, instanceIDs...) + args = append(args, + "--query", "Reservations[].Instances[].{InstanceID:InstanceId,PublicIP:PublicIpAddress,PrivateIP:PrivateIpAddress}", + "--output", "json", + ) + value, err := client.output(ctx, args...) + if err != nil { + return nil, err + } + var instances []struct { + InstanceID string `json:"InstanceID"` + PublicIP *string `json:"PublicIP"` + PrivateIP *string `json:"PrivateIP"` + } + if err := json.Unmarshal([]byte(value), &instances); err != nil { + return nil, fmt.Errorf("decode EC2 instance addresses: %w", err) + } + result := make(map[string]awsInstanceState, len(instances)) + for _, instance := range instances { + result[instance.InstanceID] = awsInstanceState{ + InstanceID: instance.InstanceID, + PublicIP: optionalString(instance.PublicIP), + PrivateIP: optionalString(instance.PrivateIP), + } + } + return result, nil +} + +func optionalString(value *string) string { + if value == nil { + return "" + } + return *value +} + func (a *application) ensureAWSCredentials(ctx context.Context, client awsClient) error { if _, err := client.output(ctx, "sts", "get-caller-identity", "--output", "json"); err == nil { return nil @@ -366,7 +481,7 @@ func cidrFlag(cidr string) string { return "--cidr" } -func writeUserData(dir, clusterName, sshUser string) (string, error) { +func writeUserData(dir, clusterName string) (string, error) { if err := os.MkdirAll(dir, 0o700); err != nil { return "", fmt.Errorf("create state directory: %w", err) } @@ -376,13 +491,11 @@ func writeUserData(dir, clusterName, sshUser string) (string, error) { } path := file.Name() defer func() { _ = file.Close() }() - script := fmt.Sprintf(`#!/usr/bin/env bash + script := `#!/usr/bin/env bash set -euxo pipefail export DEBIAN_FRONTEND=noninteractive apt-get update -apt-get install -y ca-certificates curl git jq make build-essential docker.io docker-compose-v2 -systemctl enable --now docker -usermod -aG docker %s +apt-get install -y ca-certificates curl git jq make build-essential python3 case "$(uname -m)" in aarch64|arm64) go_arch=arm64 ;; x86_64|amd64) go_arch=amd64 ;; @@ -393,7 +506,7 @@ rm -rf /usr/local/go tar -C /usr/local -xzf /tmp/go.tgz ln -sf /usr/local/go/bin/go /usr/local/bin/go touch /var/lib/autobahn-e2e-ready -`, sshUser) +` if _, err := file.WriteString(script); err != nil { _ = os.Remove(path) return "", fmt.Errorf("write EC2 user data: %w", err) @@ -406,64 +519,208 @@ touch /var/lib/autobahn-e2e-ready } func (a *application) waitForEC2Bootstrap(ctx context.Context, state clusterState) error { - ticker := time.NewTicker(5 * time.Second) + for _, instance := range state.AWS.Instances { + if err := a.waitForRemoteCommand(ctx, state, instance, "test -f /var/lib/autobahn-e2e-ready"); err != nil { + return fmt.Errorf("wait for EC2 bootstrap on node-%d: %w", instance.NodeIndex, err) + } + } + return nil +} + +func (a *application) startRemoteCluster(ctx context.Context, state clusterState) error { + aws := state.AWS + buildDir := filepath.Join(aws.RemoteDir, "build") + readyMarker := filepath.Join(buildDir, nativeBuildReadyFile) + failedMarker := filepath.Join(buildDir, nativeBuildFailedFile) + buildLog := filepath.Join(buildDir, nativeBuildLogFile) + for _, instance := range aws.Instances { + command := strings.Join([]string{ + "if test ! -d " + shellQuote(filepath.Join(aws.RemoteDir, ".git")) + "; then git clone --filter=blob:none " + shellQuote(aws.RepoURL) + " " + shellQuote(aws.RemoteDir) + "; fi", + "cd " + shellQuote(aws.RemoteDir), + "git fetch --depth=1 origin " + shellQuote(aws.Ref), + "git checkout --detach FETCH_HEAD", + "mkdir -p " + shellQuote(buildDir), + "rm -f " + shellQuote(readyMarker) + " " + shellQuote(failedMarker), + "nohup integration_test/autobahn/scripts/build_native_node.sh " + shellQuote(aws.RemoteDir) + " " + shellQuote(buildLog) + " 2>&1 &", + }, " && ") + if err := a.runner.stream(ctx, sshCommandForInstance(state, instance, command)); err != nil { + return fmt.Errorf("start native build on node-%d: %w", instance.NodeIndex, err) + } + } + if err := a.waitForNativeBuilds(ctx, state, readyMarker, failedMarker, buildLog); err != nil { + return fmt.Errorf("wait for native builds: %w", err) + } + + privateIPs := make([]string, len(aws.Instances)) + for _, instance := range aws.Instances { + privateIPs[instance.NodeIndex] = instance.PrivateIP + } + coordinator := aws.Instances[0] + prepareArgs := []string{ + "integration_test/autobahn/scripts/prepare_native_cluster.sh", + shellQuote(filepath.Join("/home", aws.SSHUser)), + } + for _, privateIP := range privateIPs { + prepareArgs = append(prepareArgs, shellQuote(privateIP)) + } + prepareCommand := "cd " + shellQuote(aws.RemoteDir) + " && " + strings.Join(prepareArgs, " ") + if err := a.runner.stream(ctx, sshCommandForInstance(state, coordinator, prepareCommand)); err != nil { + return fmt.Errorf("prepare native cluster configuration: %w", err) + } + + stagingDir, err := os.MkdirTemp("", "autobahn-e2e-native-*") + if err != nil { + return fmt.Errorf("create native cluster staging directory: %w", err) + } + defer func() { _ = os.RemoveAll(stagingDir) }() + for _, instance := range aws.Instances { + archiveName := fmt.Sprintf("node-%d.tgz", instance.NodeIndex) + localArchive := filepath.Join(stagingDir, archiveName) + remoteArchive := filepath.Join(aws.RemoteDir, "build", "autobahn-native", archiveName) + if err := a.runner.stream(ctx, scpDownloadCommand(state, coordinator, remoteArchive, localArchive)); err != nil { + return fmt.Errorf("download configuration for node-%d: %w", instance.NodeIndex, err) + } + targetArchive := filepath.Join("/tmp", state.Name+"-"+archiveName) + if err := a.runner.stream(ctx, scpUploadCommand(state, instance, localArchive, targetArchive)); err != nil { + return fmt.Errorf("upload configuration for node-%d: %w", instance.NodeIndex, err) + } + installCommand := strings.Join([]string{ + "cd " + shellQuote(aws.RemoteDir), + "integration_test/autobahn/scripts/install_native_node.sh " + shellQuote(targetArchive) + " " + shellQuote(aws.SSHUser) + " " + fmt.Sprint(aws.GoMaxProcs) + " " + shellQuote(aws.GoGC), + }, " && ") + if err := a.runner.stream(ctx, sshCommandForInstance(state, instance, installCommand)); err != nil { + return fmt.Errorf("install native validator node-%d: %w", instance.NodeIndex, err) + } + } + for _, instance := range aws.Instances { + if err := a.runner.stream(ctx, sshCommandForInstance(state, instance, "sudo systemctl start seid.service")); err != nil { + return fmt.Errorf("start native validator node-%d: %w", instance.NodeIndex, err) + } + } + return nil +} + +func (a *application) waitForNativeBuilds( + ctx context.Context, + state clusterState, + readyMarker string, + failedMarker string, + buildLog string, +) error { + command := "if test -f " + shellQuote(failedMarker) + "; then printf " + shellQuote(nativeBuildStatusFailed) + + "; elif test -f " + shellQuote(readyMarker) + "; then printf " + shellQuote(nativeBuildStatusReady) + "; fi" + ticker := time.NewTicker(2 * time.Second) defer ticker.Stop() + readyNodes := make(map[int]struct{}, len(state.AWS.Instances)) for { - _, err := a.runner.output(ctx, sshCommand(state, "test -f /var/lib/autobahn-e2e-ready")) - if err == nil { + for _, instance := range state.AWS.Instances { + if _, ok := readyNodes[instance.NodeIndex]; ok { + continue + } + status, err := a.runner.output(ctx, sshCommandForInstance(state, instance, command)) + if err != nil { + continue + } + switch strings.TrimSpace(status) { + case nativeBuildStatusReady: + readyNodes[instance.NodeIndex] = struct{}{} + case nativeBuildStatusFailed: + logOutput, logErr := a.runner.output(ctx, sshCommandForInstance(state, instance, "tail -n 200 "+shellQuote(buildLog))) + if logErr != nil { + return fmt.Errorf("node-%d native build failed; read %s: %w", instance.NodeIndex, buildLog, logErr) + } + return fmt.Errorf("node-%d native build failed:\n%s", instance.NodeIndex, strings.TrimSpace(logOutput)) + } + } + if len(readyNodes) == len(state.AWS.Instances) { return nil } select { case <-ctx.Done(): - return fmt.Errorf("wait for EC2 bootstrap: %w", ctx.Err()) + return ctx.Err() case <-ticker.C: } } } -func (a *application) startRemoteCluster(ctx context.Context, state clusterState) error { - aws := state.AWS - command := strings.Join([]string{ - "git clone --filter=blob:none " + shellQuote(aws.RepoURL) + " " + shellQuote(aws.RemoteDir), - "cd " + shellQuote(aws.RemoteDir), - "git checkout --detach " + shellQuote(aws.Ref), - "AUTOBAHN=true AUTOBAHN_EVMONLY_IN_MEMORY=true DOCKER_DETACH=true make docker-cluster-start", - }, " && ") - if err := a.runner.stream(ctx, sshCommand(state, command)); err != nil { - return fmt.Errorf("start remote cluster: %w", err) +func (a *application) waitForRemoteCluster(ctx context.Context, state clusterState) error { + command := "systemctl is-active --quiet seid.service && curl -fsS -X POST -H 'content-type: application/json' --data '{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"eth_sendRawTransaction\",\"params\":[\"0x01\"]}' http://127.0.0.1:8545 >/dev/null" + for _, instance := range state.AWS.Instances { + if err := a.waitForRemoteCommand(ctx, state, instance, command); err != nil { + return fmt.Errorf("wait for native validator node-%d: %w", instance.NodeIndex, err) + } } return nil } -func (a *application) waitForRemoteCluster(ctx context.Context, state clusterState) error { - command := "test \"$(wc -l < " + shellQuote(filepath.Join(state.AWS.RemoteDir, "build/generated/launch.complete")) + ")\" -ge " + strconv.Itoa(dockerClusterSize) +func (a *application) waitForRemoteCommand(ctx context.Context, state clusterState, instance awsInstanceState, command string) error { ticker := time.NewTicker(2 * time.Second) defer ticker.Stop() for { - if _, err := a.runner.output(ctx, sshCommand(state, command)); err == nil { + if _, err := a.runner.output(ctx, sshCommandForInstance(state, instance, command)); err == nil { return nil } select { case <-ctx.Done(): - return fmt.Errorf("wait for remote cluster: %w", ctx.Err()) + return ctx.Err() case <-ticker.C: } } } -func sshCommand(state clusterState, remoteCommand string) commandSpec { - return commandSpec{name: "ssh", args: append(sshBaseArgs(state), remoteCommand)} +func sshCommandForInstance(state clusterState, instance awsInstanceState, remoteCommand string) commandSpec { + return commandSpec{name: "ssh", args: append(sshBaseArgs(state, instance), remoteCommand)} } -func sshBaseArgs(state clusterState) []string { +func sshBaseArgs(state clusterState, instance awsInstanceState) []string { aws := state.AWS return []string{ "-i", expandHome(aws.SSHKeyPath), "-o", "BatchMode=yes", "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=accept-new", - aws.SSHUser + "@" + aws.PublicIP, + aws.SSHUser + "@" + instance.PublicIP, + } +} + +func scpDownloadCommand(state clusterState, instance awsInstanceState, remotePath, localPath string) commandSpec { + args := scpBaseArgs(state) + args = append(args, state.AWS.SSHUser+"@"+instance.PublicIP+":"+remotePath, localPath) + return commandSpec{name: "scp", args: args} +} + +func scpUploadCommand(state clusterState, instance awsInstanceState, localPath, remotePath string) commandSpec { + args := scpBaseArgs(state) + args = append(args, localPath, state.AWS.SSHUser+"@"+instance.PublicIP+":"+remotePath) + return commandSpec{name: "scp", args: args} +} + +func scpBaseArgs(state clusterState) []string { + return []string{ + "-i", expandHome(state.AWS.SSHKeyPath), + "-o", "BatchMode=yes", + "-o", "ConnectTimeout=10", + "-o", "StrictHostKeyChecking=accept-new", + } +} + +func awsInstanceForNode(state clusterState, node node) (awsInstanceState, error) { + if state.AWS == nil { + return awsInstanceState{}, fmt.Errorf("aws metadata is missing") + } + for _, instance := range state.AWS.Instances { + if instance.NodeIndex == node.Index { + return instance, nil + } + } + if state.AWS.InstanceID != "" { + return awsInstanceState{ + NodeIndex: node.Index, + InstanceID: state.AWS.InstanceID, + PublicIP: state.AWS.PublicIP, + }, nil } + return awsInstanceState{}, fmt.Errorf("AWS instance for %s is missing", node.Name) } func shellQuote(value string) string { diff --git a/cmd/autobahn-e2e/command_test.go b/cmd/autobahn-e2e/command_test.go index a2f4267925..961e8d48ef 100644 --- a/cmd/autobahn-e2e/command_test.go +++ b/cmd/autobahn-e2e/command_test.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "errors" + "fmt" "os" "path/filepath" "strings" @@ -78,6 +79,15 @@ func TestClusterNodesMatchDockerComposePorts(t *testing.T) { }, clusterNodes(4)) } +func TestNativeClusterNodesUseOneRPCPortPerHost(t *testing.T) { + require.Equal(t, []node{ + {Index: 0, Name: "node-0", EVMHostPort: 8545}, + {Index: 1, Name: "node-1", EVMHostPort: 8545}, + {Index: 2, Name: "node-2", EVMHostPort: 8545}, + {Index: 3, Name: "node-3", EVMHostPort: 8545}, + }, nativeClusterNodes(4)) +} + func TestFindNodeAcceptsNamesAndIndexes(t *testing.T) { nodes := clusterNodes(4) for _, selector := range []string{"2", "node-2", "sei-node-2"} { @@ -104,9 +114,16 @@ func TestAWSDeployCreatesManagedResourcesAndReadyState(t *testing.T) { case strings.Contains(joined, "create-key-pair"): return "-----BEGIN OPENSSH PRIVATE KEY-----\ntest\n-----END OPENSSH PRIVATE KEY-----\n", nil case strings.Contains(joined, "run-instances"): - return "i-123\n", nil + return "i-100\ti-101\ti-102\ti-103\n", nil case strings.Contains(joined, "describe-instances"): - return "203.0.113.10\n", nil + return `[ + {"InstanceID":"i-100","PublicIP":"203.0.113.10","PrivateIP":"10.0.0.10"}, + {"InstanceID":"i-101","PublicIP":"203.0.113.11","PrivateIP":"10.0.0.11"}, + {"InstanceID":"i-102","PublicIP":"203.0.113.12","PrivateIP":"10.0.0.12"}, + {"InstanceID":"i-103","PublicIP":"203.0.113.13","PrivateIP":"10.0.0.13"} +]`, nil + case spec.name == "ssh" && strings.Contains(joined, nativeBuildFailedFile): + return nativeBuildStatusReady, nil case spec.name == "ssh": return "", nil default: @@ -133,8 +150,13 @@ func TestAWSDeployCreatesManagedResourcesAndReadyState(t *testing.T) { state, err := app.store().load(options.name) require.NoError(t, err) require.Equal(t, "ready", state.Status) - require.Equal(t, "i-123", state.AWS.InstanceID) - require.Equal(t, "203.0.113.10", state.AWS.PublicIP) + require.Equal(t, []awsInstanceState{ + {NodeIndex: 0, InstanceID: "i-100", PublicIP: "203.0.113.10", PrivateIP: "10.0.0.10"}, + {NodeIndex: 1, InstanceID: "i-101", PublicIP: "203.0.113.11", PrivateIP: "10.0.0.11"}, + {NodeIndex: 2, InstanceID: "i-102", PublicIP: "203.0.113.12", PrivateIP: "10.0.0.12"}, + {NodeIndex: 3, InstanceID: "i-103", PublicIP: "203.0.113.13", PrivateIP: "10.0.0.13"}, + }, state.AWS.Instances) + require.Equal(t, nativeClusterNodes(4), state.Nodes) require.True(t, state.AWS.ManagedKey) require.FileExists(t, state.AWS.SSHKeyPath) keyInfo, err := os.Stat(state.AWS.SSHKeyPath) @@ -145,7 +167,12 @@ func TestAWSDeployCreatesManagedResourcesAndReadyState(t *testing.T) { commands := joinedCommands(runner.commands) require.Contains(t, commands, "authorize-security-group-ingress") require.Contains(t, commands, "--cidr 198.51.100.4/32") - require.Contains(t, commands, "AUTOBAHN_EVMONLY_IN_MEMORY=true") + require.Contains(t, commands, "--count 4") + require.Contains(t, commands, "UserIdGroupPairs=[{GroupId=sg-123}]") + require.Contains(t, commands, "prepare_native_cluster.sh") + require.Contains(t, commands, "install_native_node.sh") + require.Contains(t, commands, "systemctl start seid.service") + require.NotContains(t, commands, "docker") require.Contains(t, commands, "-o StrictHostKeyChecking=accept-new") } @@ -190,6 +217,68 @@ func TestAWSDeployRetainsFailedState(t *testing.T) { require.Equal(t, "sg-123", state.AWS.SecurityGroupID) } +func TestAWSRuntimeOptionValidation(t *testing.T) { + parameter, err := ubuntuAMIParameter("amd64") + require.NoError(t, err) + require.Equal(t, ubuntuAMD64AMIParameter, parameter) + parameter, err = ubuntuAMIParameter("arm64") + require.NoError(t, err) + require.Equal(t, ubuntuARM64AMIParameter, parameter) + _, err = ubuntuAMIParameter("riscv64") + require.Error(t, err) + + for _, value := range []string{"off", "0", "100", "200"} { + require.NoError(t, validateGoGC(value)) + } + for _, value := range []string{"", "-1", "invalid"} { + require.Error(t, validateGoGC(value)) + } +} + +func TestDescribeAWSInstancesHandlesMissingPublicIP(t *testing.T) { + runner := &fakeRunner{outputFn: func(commandSpec) (string, error) { + return `[{"InstanceID":"i-100","PublicIP":null,"PrivateIP":"10.0.0.10"}]`, nil + }} + instances, err := describeAWSInstances(t.Context(), awsClient{runner: runner}, []string{"i-100"}) + require.NoError(t, err) + require.Equal(t, awsInstanceState{InstanceID: "i-100", PrivateIP: "10.0.0.10"}, instances["i-100"]) +} + +func TestWaitForNativeBuildReturnsRemoteLogOnFailure(t *testing.T) { + pendingInstance := awsInstanceState{NodeIndex: 1, PublicIP: "203.0.113.11"} + instance := awsInstanceState{NodeIndex: 2, PublicIP: "203.0.113.12"} + state := clusterState{AWS: &awsState{ + SSHUser: "ubuntu", + SSHKeyPath: "/tmp/test.pem", + Instances: []awsInstanceState{pendingInstance, instance}, + }} + runner := &fakeRunner{outputFn: func(spec commandSpec) (string, error) { + joined := strings.Join(spec.args, " ") + if strings.Contains(joined, "tail -n 200") { + return "compile failed: missing library\n", nil + } + if strings.Contains(joined, pendingInstance.PublicIP) { + return "", nil + } + return nativeBuildStatusFailed, nil + }} + app := &application{runner: runner, stdout: &bytes.Buffer{}, stderr: &bytes.Buffer{}} + + err := app.waitForNativeBuilds( + t.Context(), + state, + "/remote/build/"+nativeBuildReadyFile, + "/remote/build/"+nativeBuildFailedFile, + "/remote/build/"+nativeBuildLogFile, + ) + + require.ErrorContains(t, err, "native build failed") + require.ErrorContains(t, err, "node-2") + require.ErrorContains(t, err, "compile failed: missing library") + require.Len(t, runner.commands, 3) + require.Contains(t, strings.Join(runner.commands[2].args, " "), "tail -n 200") +} + func TestAWSForwardUsesChosenNodePort(t *testing.T) { stateDir := t.TempDir() state := clusterState{ @@ -197,11 +286,16 @@ func TestAWSForwardUsesChosenNodePort(t *testing.T) { Name: "forward-test", Target: "aws", Status: "ready", - Nodes: clusterNodes(4), + Nodes: nativeClusterNodes(4), AWS: &awsState{ - PublicIP: "203.0.113.10", SSHUser: "ubuntu", SSHKeyPath: "/tmp/test.pem", + Instances: []awsInstanceState{ + {NodeIndex: 0, InstanceID: "i-100", PublicIP: "203.0.113.10", PrivateIP: "10.0.0.10"}, + {NodeIndex: 1, InstanceID: "i-101", PublicIP: "203.0.113.11", PrivateIP: "10.0.0.11"}, + {NodeIndex: 2, InstanceID: "i-102", PublicIP: "203.0.113.12", PrivateIP: "10.0.0.12"}, + {NodeIndex: 3, InstanceID: "i-103", PublicIP: "203.0.113.13", PrivateIP: "10.0.0.13"}, + }, }, } require.NoError(t, newStateStore(stateDir).save(state)) @@ -218,8 +312,8 @@ func TestAWSForwardUsesChosenNodePort(t *testing.T) { require.Len(t, runner.commands, 1) require.Equal(t, "ssh", runner.commands[0].name) joined := strings.Join(runner.commands[0].args, " ") - require.Contains(t, joined, "-L 127.0.0.1:18545:127.0.0.1:8551") - require.True(t, strings.HasSuffix(joined, "ubuntu@203.0.113.10")) + require.Contains(t, joined, "-L 127.0.0.1:18545:127.0.0.1:8545") + require.True(t, strings.HasSuffix(joined, "ubuntu@203.0.113.13")) } func TestListShowsPartialAWSDeploymentWithoutCredentials(t *testing.T) { @@ -247,6 +341,55 @@ func TestListShowsPartialAWSDeploymentWithoutCredentials(t *testing.T) { require.Empty(t, runner.commands) } +func TestListInspectsEachNativeAWSInstance(t *testing.T) { + stateDir := t.TempDir() + state := clusterState{ + Version: stateVersion, + Name: "native-aws", + Target: targetAWS, + Status: "ready", + Nodes: nativeClusterNodes(4), + AWS: &awsState{ + Region: "us-west-2", + SSHUser: "ubuntu", + SSHKeyPath: "/tmp/test.pem", + Instances: []awsInstanceState{ + {NodeIndex: 0, InstanceID: "i-100", PublicIP: "203.0.113.10", PrivateIP: "10.0.0.10"}, + {NodeIndex: 1, InstanceID: "i-101", PublicIP: "203.0.113.11", PrivateIP: "10.0.0.11"}, + {NodeIndex: 2, InstanceID: "i-102", PublicIP: "203.0.113.12", PrivateIP: "10.0.0.12"}, + {NodeIndex: 3, InstanceID: "i-103", PublicIP: "203.0.113.13", PrivateIP: "10.0.0.13"}, + }, + }, + } + require.NoError(t, newStateStore(stateDir).save(state)) + runner := &fakeRunner{outputFn: func(spec commandSpec) (string, error) { + joined := strings.Join(spec.args, " ") + switch { + case strings.Contains(joined, "sts get-caller-identity"): + return `{}`, nil + case strings.Contains(joined, "describe-instances"): + return "running\n", nil + case spec.name == "ssh" && strings.Contains(joined, "systemctl is-active"): + return "active\n", nil + case spec.name == "ssh" && strings.Contains(joined, "26660/metrics"): + return "tendermint_internal_autobahn_data_next_block{stage=\"execute\"} 43\n", nil + default: + return "", nil + } + }} + var stdout bytes.Buffer + app := &application{runner: runner, stdout: &stdout, stderr: &bytes.Buffer{}, stateDir: stateDir} + + require.NoError(t, app.list(context.Background(), listOptions{name: state.Name})) + for nodeIndex := range 4 { + require.Contains(t, stdout.String(), "node-"+fmt.Sprint(nodeIndex)) + require.Contains(t, stdout.String(), "i-10"+fmt.Sprint(nodeIndex)) + require.Contains(t, stdout.String(), "203.0.113.1"+fmt.Sprint(nodeIndex)) + } + require.Equal(t, 4, strings.Count(stdout.String(), "active")) + require.Equal(t, 4, strings.Count(stdout.String(), "42")) +} + func TestAWSTeardownToleratesAlreadyDeletedManagedResources(t *testing.T) { stateDir := t.TempDir() keyPath := filepath.Join(stateDir, "managed.pem") @@ -281,6 +424,51 @@ func TestAWSTeardownToleratesAlreadyDeletedManagedResources(t *testing.T) { require.NoFileExists(t, store.path(state.Name)) } +func TestAWSTeardownStopsAndTerminatesNativeInstances(t *testing.T) { + stateDir := t.TempDir() + keyPath := filepath.Join(stateDir, "managed.pem") + require.NoError(t, os.WriteFile(keyPath, []byte("key"), 0o600)) + state := clusterState{ + Version: stateVersion, + Name: "native-teardown", + Target: targetAWS, + Status: "ready", + Nodes: nativeClusterNodes(4), + AWS: &awsState{ + Region: "us-west-2", + SSHUser: "ubuntu", + SSHKeyPath: keyPath, + SecurityGroupID: "sg-native", + KeyName: "key-native", + ManagedKey: true, + Instances: []awsInstanceState{ + {NodeIndex: 0, InstanceID: "i-100", PublicIP: "203.0.113.10"}, + {NodeIndex: 1, InstanceID: "i-101", PublicIP: "203.0.113.11"}, + {NodeIndex: 2, InstanceID: "i-102", PublicIP: "203.0.113.12"}, + {NodeIndex: 3, InstanceID: "i-103", PublicIP: "203.0.113.13"}, + }, + }, + } + store := newStateStore(stateDir) + require.NoError(t, store.save(state)) + runner := &fakeRunner{outputFn: func(spec commandSpec) (string, error) { + if strings.Contains(strings.Join(spec.args, " "), "sts get-caller-identity") { + return `{}`, nil + } + return "", nil + }} + app := &application{runner: runner, stdout: &bytes.Buffer{}, stderr: &bytes.Buffer{}, stateDir: stateDir} + + require.NoError(t, app.teardown(context.Background(), teardownOptions{name: state.Name})) + commands := joinedCommands(runner.commands) + require.Equal(t, 4, strings.Count(commands, "systemctl stop seid.service")) + require.Contains(t, commands, "terminate-instances --instance-ids i-100 i-101 i-102 i-103") + require.Contains(t, commands, "instance-terminated --instance-ids i-100 i-101 i-102 i-103") + require.NotContains(t, commands, "docker-cluster-stop") + require.NoFileExists(t, keyPath) + require.NoFileExists(t, store.path(state.Name)) +} + func TestLocalTeardownRemovesState(t *testing.T) { stateDir := t.TempDir() state := clusterState{ @@ -311,14 +499,15 @@ func TestParseAutobahnExecutedHeight(t *testing.T) { require.Equal(t, "-", parseAutobahnExecutedHeight("not-prometheus")) } -func TestWriteUserDataUsesSelectedSSHUser(t *testing.T) { - path, err := writeUserData(t.TempDir(), "test", "ec2-user") +func TestWriteUserDataInstallsNativeBuildDependencies(t *testing.T) { + path, err := writeUserData(t.TempDir(), "test") require.NoError(t, err) data, err := os.ReadFile(path) require.NoError(t, err) - require.Contains(t, string(data), "usermod -aG docker ec2-user") + require.Contains(t, string(data), "build-essential python3") require.Contains(t, string(data), "go1.25.6") require.Contains(t, string(data), "/var/lib/autobahn-e2e-ready") + require.NotContains(t, string(data), "docker") } func TestShellQuote(t *testing.T) { diff --git a/cmd/autobahn-e2e/deploy.go b/cmd/autobahn-e2e/deploy.go index 95635f0bc8..05465aa91f 100644 --- a/cmd/autobahn-e2e/deploy.go +++ b/cmd/autobahn-e2e/deploy.go @@ -13,6 +13,7 @@ import ( ) const dockerClusterSize = 4 +const awsClusterSize = 4 type deployOptions struct { name string @@ -21,6 +22,7 @@ type deployOptions struct { region string profile string instanceType string + architecture string amiID string subnetID string sshCIDR string @@ -28,6 +30,8 @@ type deployOptions struct { keyName string sshKeyPath string volumeSize int + goMaxProcs int + goGC string repoURL string ref string } @@ -44,17 +48,20 @@ func (a *application) newDeployCommand() *cobra.Command { flags := cmd.Flags() flags.StringVar(&options.name, "name", defaultClusterName, "cluster name") flags.StringVar(&options.target, "target", targetLocal, "deployment target: local or aws") - flags.DurationVar(&options.timeout, "timeout", 20*time.Minute, "deployment readiness timeout") + flags.DurationVar(&options.timeout, "timeout", 45*time.Minute, "deployment readiness timeout") flags.StringVar(&options.region, "region", "us-west-2", "AWS region") flags.StringVar(&options.profile, "profile", "", "AWS CLI profile") flags.StringVar(&options.instanceType, "instance-type", "c7g.2xlarge", "EC2 instance type") - flags.StringVar(&options.amiID, "ami-id", "", "EC2 AMI ID; defaults to Ubuntu 24.04 ARM64") + flags.StringVar(&options.architecture, "architecture", "arm64", "EC2 architecture used to resolve the default AMI: arm64 or amd64") + flags.StringVar(&options.amiID, "ami-id", "", "EC2 AMI ID; defaults to Ubuntu 24.04 for --architecture") flags.StringVar(&options.subnetID, "subnet-id", "", "EC2 subnet; defaults to a default VPC subnet") flags.StringVar(&options.sshCIDR, "ssh-cidr", "", "CIDR allowed to SSH; defaults to the caller's public IP") flags.StringVar(&options.sshUser, "ssh-user", "ubuntu", "EC2 SSH user") flags.StringVar(&options.keyName, "key-name", "", "existing EC2 key pair name; omitted creates a managed key") flags.StringVar(&options.sshKeyPath, "ssh-key", "", "private key for --key-name") flags.IntVar(&options.volumeSize, "volume-size", 100, "EC2 root volume size in GiB") + flags.IntVar(&options.goMaxProcs, "gomaxprocs", 0, "GOMAXPROCS for each validator; 0 uses all instance CPUs") + flags.StringVar(&options.goGC, "gogc", "200", "GOGC for each validator, or off") flags.StringVar(&options.repoURL, "repo-url", "", "Git repository cloned on EC2; defaults to origin") flags.StringVar(&options.ref, "ref", "", "Git ref deployed on EC2; defaults to the current commit") return cmd diff --git a/cmd/autobahn-e2e/forward.go b/cmd/autobahn-e2e/forward.go index 804ac86b9f..0a1031c839 100644 --- a/cmd/autobahn-e2e/forward.go +++ b/cmd/autobahn-e2e/forward.go @@ -60,8 +60,12 @@ func (a *application) forward(ctx context.Context, options forwardOptions) error if state.AWS == nil { return fmt.Errorf("aws metadata is missing") } - _, _ = fmt.Fprintf(a.stdout, "Forwarding %s to %s:8545 through %s. Press Ctrl-C to stop.\n", localAddress, node.Name, state.AWS.PublicIP) - baseArgs := sshBaseArgs(state) + instance, err := awsInstanceForNode(state, node) + if err != nil { + return err + } + _, _ = fmt.Fprintf(a.stdout, "Forwarding %s to %s:8545 through %s. Press Ctrl-C to stop.\n", localAddress, node.Name, instance.PublicIP) + baseArgs := sshBaseArgs(state, instance) destination := baseArgs[len(baseArgs)-1] args := append(baseArgs[:len(baseArgs)-1], "-o", "ExitOnForwardFailure=yes", diff --git a/cmd/autobahn-e2e/list.go b/cmd/autobahn-e2e/list.go index 1d114a4442..f642b8a89e 100644 --- a/cmd/autobahn-e2e/list.go +++ b/cmd/autobahn-e2e/list.go @@ -28,7 +28,10 @@ type nodeReport struct { PublicIP string `json:"public_ip,omitempty"` } -const autobahnNextExecutedBlockMetric = "tendermint_internal_autobahn_data_next_block" +const ( + autobahnNextExecutedBlockMetric = "tendermint_internal_autobahn_data_next_block" + statusRunning = "running" +) func (a *application) newListCommand() *cobra.Command { options := listOptions{} @@ -117,7 +120,7 @@ func (a *application) inspectLocalCluster(ctx context.Context, state clusterStat } status = strings.TrimSpace(status) height := "-" - if status == "running" { + if status == statusRunning { metrics, metricsErr := a.runner.output(ctx, commandSpec{ name: "docker", args: []string{"exec", node.Container, "curl", "-fsS", "http://127.0.0.1:26660/metrics"}, @@ -142,7 +145,7 @@ func (a *application) inspectAWSCluster(ctx context.Context, state clusterState) if state.AWS == nil { return nil, fmt.Errorf("aws metadata is missing") } - if state.AWS.InstanceID == "" { + if len(state.AWS.Instances) == 0 && state.AWS.InstanceID == "" { reports := make([]nodeReport, len(state.Nodes)) for i, node := range state.Nodes { reports[i] = nodeReport{ @@ -156,6 +159,58 @@ func (a *application) inspectAWSCluster(ctx context.Context, state clusterState) } return reports, nil } + if len(state.AWS.Instances) == 0 { + return a.inspectLegacyAWSCluster(ctx, state) + } + client := awsClient{runner: a.runner, region: state.AWS.Region, profile: state.AWS.Profile} + if err := a.ensureAWSCredentials(ctx, client); err != nil { + return nil, err + } + reports := make([]nodeReport, len(state.Nodes)) + for i, node := range state.Nodes { + instance, err := awsInstanceForNode(state, node) + if err != nil { + return nil, err + } + instanceStatus, err := client.output(ctx, + "ec2", "describe-instances", + "--instance-ids", instance.InstanceID, + "--query", "Reservations[0].Instances[0].State.Name", + "--output", "text", + ) + if err != nil { + return nil, err + } + instanceStatus = strings.TrimSpace(instanceStatus) + status := instanceStatus + height := "-" + if instanceStatus == statusRunning && instance.PublicIP != "" { + value, inspectErr := a.runner.output(ctx, sshCommandForInstance(state, instance, + "systemctl is-active seid.service")) + if inspectErr == nil { + status = strings.TrimSpace(value) + } + value, heightErr := a.runner.output(ctx, sshCommandForInstance(state, instance, + "curl -fsS http://127.0.0.1:26660/metrics")) + if heightErr == nil { + height = parseAutobahnExecutedHeight(value) + } + } + reports[i] = nodeReport{ + Cluster: state.Name, + Target: state.Target, + Node: node.Name, + Status: status, + Height: height, + EVMTarget: fmt.Sprintf("SSH→127.0.0.1:%d", node.EVMHostPort), + InstanceID: instance.InstanceID, + PublicIP: instance.PublicIP, + } + } + return reports, nil +} + +func (a *application) inspectLegacyAWSCluster(ctx context.Context, state clusterState) ([]nodeReport, error) { client := awsClient{runner: a.runner, region: state.AWS.Region, profile: state.AWS.Profile} if err := a.ensureAWSCredentials(ctx, client); err != nil { return nil, err @@ -171,16 +226,17 @@ func (a *application) inspectAWSCluster(ctx context.Context, state clusterState) } instanceStatus = strings.TrimSpace(instanceStatus) reports := make([]nodeReport, len(state.Nodes)) + instance := awsInstanceState{InstanceID: state.AWS.InstanceID, PublicIP: state.AWS.PublicIP} for i, node := range state.Nodes { status := instanceStatus height := "-" - if instanceStatus == "running" && state.AWS.PublicIP != "" { - value, inspectErr := a.runner.output(ctx, sshCommand(state, + if instanceStatus == statusRunning && instance.PublicIP != "" { + value, inspectErr := a.runner.output(ctx, sshCommandForInstance(state, instance, "docker inspect --format '{{.State.Status}}' "+shellQuote(node.Container))) if inspectErr == nil { status = strings.TrimSpace(value) } - value, heightErr := a.runner.output(ctx, sshCommand(state, + value, heightErr := a.runner.output(ctx, sshCommandForInstance(state, instance, "docker exec "+shellQuote(node.Container)+" curl -fsS http://127.0.0.1:26660/metrics")) if heightErr == nil { height = parseAutobahnExecutedHeight(value) diff --git a/cmd/autobahn-e2e/state.go b/cmd/autobahn-e2e/state.go index 582e8d7e48..8c4a62d934 100644 --- a/cmd/autobahn-e2e/state.go +++ b/cmd/autobahn-e2e/state.go @@ -28,23 +28,34 @@ type clusterState struct { type node struct { Index int `json:"index"` Name string `json:"name"` - Container string `json:"container"` + Container string `json:"container,omitempty"` EVMHostPort int `json:"evm_host_port"` } type awsState struct { - Region string `json:"region"` - Profile string `json:"profile,omitempty"` - InstanceID string `json:"instance_id,omitempty"` - PublicIP string `json:"public_ip,omitempty"` - SecurityGroupID string `json:"security_group_id,omitempty"` - KeyName string `json:"key_name,omitempty"` - SSHKeyPath string `json:"ssh_key_path,omitempty"` - SSHUser string `json:"ssh_user"` - RemoteDir string `json:"remote_dir"` - ManagedKey bool `json:"managed_key"` - RepoURL string `json:"repo_url"` - Ref string `json:"ref"` + Region string `json:"region"` + Profile string `json:"profile,omitempty"` + Instances []awsInstanceState `json:"instances,omitempty"` + InstanceID string `json:"instance_id,omitempty"` + PublicIP string `json:"public_ip,omitempty"` + SecurityGroupID string `json:"security_group_id,omitempty"` + KeyName string `json:"key_name,omitempty"` + SSHKeyPath string `json:"ssh_key_path,omitempty"` + SSHUser string `json:"ssh_user"` + RemoteDir string `json:"remote_dir"` + ManagedKey bool `json:"managed_key"` + RepoURL string `json:"repo_url"` + Ref string `json:"ref"` + GoMaxProcs int `json:"gomaxprocs,omitempty"` + GoGC string `json:"gogc,omitempty"` + Architecture string `json:"architecture,omitempty"` +} + +type awsInstanceState struct { + NodeIndex int `json:"node_index"` + InstanceID string `json:"instance_id"` + PublicIP string `json:"public_ip"` + PrivateIP string `json:"private_ip"` } type stateStore struct { @@ -187,3 +198,15 @@ func clusterNodes(count int) []node { } return nodes } + +func nativeClusterNodes(count int) []node { + nodes := make([]node, count) + for i := range count { + nodes[i] = node{ + Index: i, + Name: fmt.Sprintf("node-%d", i), + EVMHostPort: 8545, + } + } + return nodes +} diff --git a/cmd/autobahn-e2e/teardown.go b/cmd/autobahn-e2e/teardown.go index 4f0235af94..0b6f4a6b21 100644 --- a/cmd/autobahn-e2e/teardown.go +++ b/cmd/autobahn-e2e/teardown.go @@ -59,18 +59,37 @@ func (a *application) teardownAWS(ctx context.Context, state clusterState) error if err := a.ensureAWSCredentials(ctx, client); err != nil { return err } - if state.AWS.PublicIP != "" && state.AWS.RemoteDir != "" { + if len(state.AWS.Instances) == 0 && state.AWS.PublicIP != "" && state.AWS.RemoteDir != "" { command := "cd " + shellQuote(state.AWS.RemoteDir) + " && make docker-cluster-stop" - if err := a.runner.stream(ctx, sshCommand(state, command)); err != nil { + instance := awsInstanceState{InstanceID: state.AWS.InstanceID, PublicIP: state.AWS.PublicIP} + if err := a.runner.stream(ctx, sshCommandForInstance(state, instance, command)); err != nil { _, _ = fmt.Fprintf(a.stderr, "warning: remote Docker teardown failed: %v\n", err) } } var errs []error - if state.AWS.InstanceID != "" { - if _, err := client.output(ctx, "ec2", "terminate-instances", "--instance-ids", state.AWS.InstanceID); err != nil { - errs = append(errs, err) - } else if err := client.stream(ctx, "ec2", "wait", "instance-terminated", "--instance-ids", state.AWS.InstanceID); err != nil { + instanceIDs := make([]string, 0, len(state.AWS.Instances)+1) + for _, instance := range state.AWS.Instances { + if instance.PublicIP != "" { + if err := a.runner.stream(ctx, sshCommandForInstance(state, instance, "sudo systemctl stop seid.service")); err != nil { + _, _ = fmt.Fprintf(a.stderr, "warning: stop native validator node-%d failed: %v\n", instance.NodeIndex, err) + } + } + if instance.InstanceID != "" { + instanceIDs = append(instanceIDs, instance.InstanceID) + } + } + if len(instanceIDs) == 0 && state.AWS.InstanceID != "" { + instanceIDs = append(instanceIDs, state.AWS.InstanceID) + } + if len(instanceIDs) > 0 { + terminateArgs := append([]string{"ec2", "terminate-instances", "--instance-ids"}, instanceIDs...) + if _, err := client.output(ctx, terminateArgs...); err != nil { errs = append(errs, err) + } else { + waitArgs := append([]string{"ec2", "wait", "instance-terminated", "--instance-ids"}, instanceIDs...) + if err := client.stream(ctx, waitArgs...); err != nil { + errs = append(errs, err) + } } } if state.AWS.SecurityGroupID != "" { diff --git a/docker/localnode/scripts/step2_genesis.sh b/docker/localnode/scripts/step2_genesis.sh index 9373ac6745..d85274aa97 100755 --- a/docker/localnode/scripts/step2_genesis.sh +++ b/docker/localnode/scripts/step2_genesis.sh @@ -61,7 +61,8 @@ cp -r build/generated/gentx/* ~/.sei/config/gentx cp -r build/generated/exported_keys ~/exported_keys # add validators to genesis -/usr/bin/add_validator_to_gensis.sh +ADD_VALIDATOR_SCRIPT=${ADD_VALIDATOR_SCRIPT:-/usr/bin/add_validator_to_gensis.sh} +"$ADD_VALIDATOR_SCRIPT" # collect gentxs echo "Collecting all gentx" diff --git a/docker/localnode/scripts/step4_config_override.sh b/docker/localnode/scripts/step4_config_override.sh index edcee91187..034faf9286 100755 --- a/docker/localnode/scripts/step4_config_override.sh +++ b/docker/localnode/scripts/step4_config_override.sh @@ -38,8 +38,8 @@ if [ "$VALIDATOR" != "true" ]; then fi # Override up persistent peers -NODE_IP=$(hostname -i | awk '{print $1}') -PEERS=$(cat build/generated/persistent_peers.txt |grep -v "$NODE_IP" | paste -sd "," -) +NODE_IP=${NODE_IP:-$(hostname -i | awk '{print $1}')} +PEERS=$(grep -F -v "@$NODE_IP:" build/generated/persistent_peers.txt | paste -sd "," -) sed -i'' -e 's/persistent-peers = ""/persistent-peers = "'$PEERS'"/g' ~/.sei/config/config.toml # Override snapshot directory diff --git a/integration_test/autobahn/README.md b/integration_test/autobahn/README.md index 7da72101ae..709ec737d8 100644 --- a/integration_test/autobahn/README.md +++ b/integration_test/autobahn/README.md @@ -38,9 +38,15 @@ inside its container, so cluster inspection does not require Tendermint RPC. ## AWS EC2 -The AWS target provisions one Ubuntu EC2 host and runs the identical four-node -Docker topology on it. Only SSH is opened in the managed security group; EVM -JSON-RPC remains private and is reached with `forward`. +The AWS target provisions four Ubuntu EC2 instances, one per validator. Each +instance builds the selected revision and runs `seid` directly as a systemd +service. Docker is not installed or used on the validator hosts. Builds run in +parallel, then node 0 generates the shared genesis and Autobahn configuration +that the command distributes to all four hosts. + +Only SSH is exposed from the managed security group to the configured caller +CIDR. The instances may communicate with each other inside the security group; +EVM JSON-RPC remains private and is reached with `forward`. ```sh ./autobahn-e2e deploy --target aws \ @@ -74,11 +80,27 @@ deployment time. Use `--ssh-cidr` when running behind a VPN, through NAT with a different egress address, or from an IPv6 network. The default EC2 shape is `c7g.2xlarge` with the current Ubuntu 24.04 ARM64 AMI -resolved from AWS Systems Manager. Override `--instance-type` and `--ami-id` -together when using another architecture. `--repo-url` and `--ref` select the -source deployed remotely; they default to the current checkout's origin and -commit. +resolved from AWS Systems Manager. The readiness timeout defaults to 45 minutes +to cover initial package installation and a cold native build; override it with +`--timeout`. For the throughput-test topology, select an AMD64 compute instance +explicitly and cap the Go scheduler at the measured knee: + +```sh +./autobahn-e2e deploy --target aws \ + --name throughput \ + --region us-east-2 \ + --architecture amd64 \ + --instance-type c8i.48xlarge \ + --gomaxprocs 24 \ + --gogc 200 +``` + +Use `--ami-id` to override the Ubuntu image selected for `--architecture`. +`--repo-url` and `--ref` select the source deployed remotely; they default to +the current checkout's origin and commit. `--gomaxprocs 0` uses every logical +CPU on each instance. `--gogc off` is accepted for short ceiling tests but can +consume hundreds of GiB under sustained native-transfer load. If provisioning fails after AWS resources are created, the state is retained with status `failed`. Run `list` to inspect it and `teardown` to remove the -instance, security group, and any managed key pair. +instances, security group, and any managed key pair. diff --git a/integration_test/autobahn/scripts/build_native_node.sh b/integration_test/autobahn/scripts/build_native_node.sh new file mode 100755 index 0000000000..ea7d4fc2b4 --- /dev/null +++ b/integration_test/autobahn/scripts/build_native_node.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_dir=${1:?repository directory is required} +cd "$repo_dir" + +build_dir=$repo_dir/build +ready_marker=$build_dir/autobahn-native-build.ready +failed_marker=$build_dir/autobahn-native-build.failed +mkdir -p "$build_dir" +rm -f "$ready_marker" "$failed_marker" + +record_build_failure() { + local exit_code=$? + if (( exit_code != 0 )); then + printf '%s\n' "$exit_code" > "$failed_marker" + fi +} +trap record_build_failure EXIT + +export LEDGER_ENABLED=false +export PATH="/usr/local/go/bin:$PATH" +make build-linux + +case "$(uname -m)" in + aarch64|arm64) wasm_arch=aarch64 ;; + x86_64|amd64) wasm_arch=x86_64 ;; + *) echo "unsupported architecture: $(uname -m)" >&2; exit 1 ;; +esac + +sudo install -d -m 0755 /opt/seid/bin /opt/seid/lib +sudo install -m 0755 build/seid /opt/seid/bin/seid +sudo install -m 0755 "sei-wasmvm/internal/api/libwasmvm.${wasm_arch}.so" /opt/seid/lib/ +sudo install -m 0755 "sei-wasmd/x/wasm/artifacts/v152/api/libwasmvm152.${wasm_arch}.so" /opt/seid/lib/ +sudo install -m 0755 "sei-wasmd/x/wasm/artifacts/v155/api/libwasmvm155.${wasm_arch}.so" /opt/seid/lib/ + +touch "$ready_marker" diff --git a/integration_test/autobahn/scripts/install_native_node.sh b/integration_test/autobahn/scripts/install_native_node.sh new file mode 100755 index 0000000000..9bb5dfd861 --- /dev/null +++ b/integration_test/autobahn/scripts/install_native_node.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +set -euo pipefail + +archive=${1:?node archive is required} +service_user=${2:?service user is required} +go_max_procs=${3:?GOMAXPROCS value is required} +go_gc=${4:?GOGC value is required} +service_home=$(getent passwd "$service_user" | cut -d: -f6) +if [[ -z "$service_home" ]]; then + echo "home directory not found for $service_user" >&2 + exit 1 +fi + +sudo systemctl stop seid.service 2>/dev/null || true +if [[ -e "$service_home/.sei" ]]; then + mv "$service_home/.sei" "$service_home/.sei.autobahn-backup-$(date +%s)" +fi +tar -C "$service_home" -xzf "$archive" + +sudo install -m 0644 /dev/null /etc/seid-native.env +{ + echo 'LD_LIBRARY_PATH=/opt/seid/lib' + if [[ "$go_max_procs" != "0" ]]; then + echo "GOMAXPROCS=$go_max_procs" + fi + echo "GOGC=$go_gc" +} | sudo tee /etc/seid-native.env >/dev/null + +sed \ + -e "s|__SERVICE_USER__|$service_user|g" \ + -e "s|__SERVICE_HOME__|$service_home|g" \ + integration_test/autobahn/systemd/seid.service | \ + sudo tee /etc/systemd/system/seid.service >/dev/null + +sudo install -m 0644 integration_test/autobahn/systemd/99-seid-native.conf /etc/sysctl.d/99-seid-native.conf +sudo sysctl --system >/dev/null +sudo systemctl daemon-reload +sudo systemctl enable seid.service >/dev/null diff --git a/integration_test/autobahn/scripts/prepare_native_cluster.sh b/integration_test/autobahn/scripts/prepare_native_cluster.sh new file mode 100755 index 0000000000..8ff3daaff2 --- /dev/null +++ b/integration_test/autobahn/scripts/prepare_native_cluster.sh @@ -0,0 +1,74 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ $# -lt 2 ]]; then + echo "usage: $0 FINAL_HOME NODE0_IP [NODE1_IP ...]" >&2 + exit 2 +fi + +final_home=$1 +shift +private_ips=("$@") +node_count=${#private_ips[@]} +repo_dir=$(git rev-parse --show-toplevel) +cd "$repo_dir" + +generated_dir=$repo_dir/build/generated +native_dir=$repo_dir/build/autobahn-native +homes_dir=$native_dir/homes +rm -rf "$generated_dir" "$native_dir" +mkdir -p "$generated_dir" "$homes_dir" + +for ((node_index = 0; node_index < node_count; node_index++)); do + node_home=$homes_dir/node-$node_index + mkdir -p "$node_home/go/bin" + HOME=$node_home \ + GOBIN=$node_home/go/bin \ + PATH="$node_home/go/bin:$repo_dir/build:/usr/local/go/bin:$PATH" \ + ID=$node_index \ + NUM_ACCOUNTS=0 \ + VALIDATOR=true \ + docker/localnode/scripts/step1_configure_init.sh +done + +: > "$generated_dir/persistent_peers.txt" +for ((node_index = 0; node_index < node_count; node_index++)); do + node_home=$homes_dir/node-$node_index + seid=$node_home/go/bin/seid + node_id=$(HOME=$node_home "$seid" tendermint show-node-id) + printf '%s@%s:26656\n' "$node_id" "${private_ips[$node_index]}" >> "$generated_dir/persistent_peers.txt" + printf '%s:26656\n' "${private_ips[$node_index]}" > "$generated_dir/node_$node_index/autobahn_address.txt" + printf 'http://%s:8545\n' "${private_ips[$node_index]}" > "$generated_dir/node_$node_index/evmrpc_url.txt" +done + +node_zero_home=$homes_dir/node-0 +HOME=$node_zero_home \ + PATH="$node_zero_home/go/bin:/usr/local/go/bin:$PATH" \ + ADD_VALIDATOR_SCRIPT=$repo_dir/docker/localnode/scripts/step3_add_validator_to_genesis.sh \ + docker/localnode/scripts/step2_genesis.sh + +for ((node_index = 0; node_index < node_count; node_index++)); do + node_home=$homes_dir/node-$node_index + HOME=$node_home \ + PATH="$node_home/go/bin:/usr/local/go/bin:$PATH" \ + ID=$node_index \ + CLUSTER_SIZE=$node_count \ + NODE_IP=${private_ips[$node_index]} \ + AUTOBAHN=true \ + AUTOBAHN_EVMONLY_IN_MEMORY=true \ + GIGA_EXECUTOR=true \ + GIGA_OCC=true \ + docker/localnode/scripts/step4_config_override.sh + + sed -i \ + -e "s|^autobahn-config-file = .*|autobahn-config-file = \"$final_home/.sei/config/autobahn.json\"|" \ + "$node_home/.sei/config/config.toml" + sed -i \ + -e "s|^snapshot-directory = .*|snapshot-directory = \"$final_home/.sei/data/snapshots\"|" \ + "$node_home/.sei/config/app.toml" + mkdir -p "$node_home/.sei/data" + printf '{"height":"0","round":0,"step":0}\n' > "$node_home/.sei/data/priv_validator_state.json" + tar -C "$node_home" -czf "$native_dir/node-$node_index.tgz" .sei +done + +touch "$native_dir/configuration.ready" diff --git a/integration_test/autobahn/systemd/99-seid-native.conf b/integration_test/autobahn/systemd/99-seid-native.conf new file mode 100644 index 0000000000..0d81fa42ad --- /dev/null +++ b/integration_test/autobahn/systemd/99-seid-native.conf @@ -0,0 +1,5 @@ +fs.file-max = 2097152 +net.core.somaxconn = 65535 +net.core.netdev_max_backlog = 250000 +net.core.rmem_max = 134217728 +net.core.wmem_max = 134217728 diff --git a/integration_test/autobahn/systemd/seid.service b/integration_test/autobahn/systemd/seid.service new file mode 100644 index 0000000000..b875a45031 --- /dev/null +++ b/integration_test/autobahn/systemd/seid.service @@ -0,0 +1,21 @@ +[Unit] +Description=Sei Autobahn native validator +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=__SERVICE_USER__ +WorkingDirectory=__SERVICE_HOME__ +EnvironmentFile=/etc/seid-native.env +ExecStart=/opt/seid/bin/seid start --home __SERVICE_HOME__/.sei --chain-id sei --inv-check-period 0 +Restart=on-failure +RestartSec=2 +TimeoutStopSec=90 +LimitNOFILE=1048576 +LimitNPROC=infinity +LimitMEMLOCK=infinity +TasksMax=infinity + +[Install] +WantedBy=multi-user.target