diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..fcee75a --- /dev/null +++ b/.dockerignore @@ -0,0 +1,11 @@ +# Docker build context 排除(T01 code review I-3) +# 两个 Dockerfile 都用 context: ../..(仓库根),这里裁掉无关内容 +# 必须保留进 context:internal/ cmd/ go.mod go.sum tests/e2e/fixtures/ +.git/ +bin/ +docs/ +.firecrawl/ +graphify-out/ +*.md +.DS_Store +.claude/ diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml new file mode 100644 index 0000000..474a7d8 --- /dev/null +++ b/.github/workflows/e2e.yml @@ -0,0 +1,31 @@ +name: e2e + +on: + push: + branches: [master] + pull_request: + branches: [master] + +jobs: + e2e: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + + - name: Run e2e (quick mode) + run: bash tests/e2e/run.sh --quick + + - name: Collect report + if: always() + run: | + mkdir -p e2e-report + cp -r tests/e2e/results ./e2e-report/ 2>/dev/null || true + + - uses: actions/upload-artifact@v4 + if: always() + with: + name: e2e-report + path: e2e-report/ diff --git a/.gitignore b/.gitignore index e660fd9..748c26c 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,4 @@ bin/ + +# e2e 测试产物 +tests/e2e/results/ diff --git a/README.md b/README.md index 6254f00..d498d0f 100644 --- a/README.md +++ b/README.md @@ -285,6 +285,30 @@ mesh leave # 注销并清理 | 服务器 | 10.100.0.1 | | 客户端(自动分配) | 10.100.0.2 ~ 10.100.0.254 | +## e2e 测试 + +基于 Docker 的端到端测试(3 容器:1 server + 2 client),验证连通性、性能、故障恢复。 + +```bash +# 快速模式(连通性 + 性能,约 2-3 分钟) +bash tests/e2e/run.sh --quick + +# 全量(含故障场景) +bash tests/e2e/run.sh --all + +# 单个场景 +bash tests/e2e/run.sh --scenario 01 + +# 性能硬门槛(release 用) +STRICT=1 bash tests/e2e/run.sh --quick +``` + +结果输出到 `tests/e2e/results//`,含各场景 JSON 指标和 `summary.txt`。 + +> 注:macOS 本机若 docker 走 colima,需先 `export DOCKER_HOST=unix://$HOME/.colima/default/docker.sock`;CI(ubuntu-latest)无需。 + +详见 `docs/superpowers/specs/2026-07-08-e2e-docker-design.md`。 + ## 技术栈 - Go 1.22+ diff --git a/cmd/mesh/main.go b/cmd/mesh/main.go index 4be0fc1..2d2e4de 100644 --- a/cmd/mesh/main.go +++ b/cmd/mesh/main.go @@ -26,12 +26,13 @@ func main() { func joinCmd() *cobra.Command { var tok string + var insecure bool cmd := &cobra.Command{ Use: "join ", Args: cobra.ExactArgs(1), Short: "向服务器注册并加入 mesh 网络(无需 root)", RunE: func(c *cobra.Command, args []string) error { - if err := client.Join(args[0], tok); err != nil { + if err := client.Join(args[0], tok, insecure); err != nil { return err } fmt.Println("Now run 'sudo mesh up' to start the tunnel.") @@ -39,6 +40,7 @@ func joinCmd() *cobra.Command { }, } cmd.Flags().StringVar(&tok, "token", "", "注册令牌(由服务器管理员提供)") + cmd.Flags().BoolVar(&insecure, "insecure", false, "跳过 TLS 证书校验(仅用于 e2e 测试)") _ = cmd.MarkFlagRequired("token") return cmd } diff --git a/docs/superpowers/plans/2026-07-08-e2e-impl.md b/docs/superpowers/plans/2026-07-08-e2e-impl.md new file mode 100644 index 0000000..1ba9b3a --- /dev/null +++ b/docs/superpowers/plans/2026-07-08-e2e-impl.md @@ -0,0 +1,1315 @@ +# e2e 测试(基于 Docker)实施计划 + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** 为 mesh VPN 建立一套基于 Docker 的 e2e 测试:3 个 Linux 容器(1 server + 2 client),通过 TUN + tc netem 验证连通性、性能、故障恢复,输出 JSON 指标和 PASS/FAIL 报告。 + +**Architecture:** server/client 跑在 privileged Linux 容器内(`/dev/net/tun` + `NET_ADMIN`);引入 `MESH_TEST_TLS=off` 开关让 server 用自签证书、client 跳过证书校验;bash 脚本驱动 3 个场景(连通性/性能/故障),结果落到 `results//` 并聚合为 `summary.txt`。 + +**Tech Stack:** Go 1.22+(mesh 本身)、Docker + docker-compose v2、ubuntu:24.04 镜像、iperf3、nuttcp、tc/netem、bash、jq。 + +**Spec:** `docs/superpowers/specs/2026-07-08-e2e-docker-design.md` + +--- + +## 文件结构 + +本计划产出/修改的文件: + +**代码改动(T00):** +- `internal/config/config.go` — 增加 `TLSTestMode bool` 字段与 `MESH_TEST_TLS` 环境变量读取。 +- `internal/api/server.go` — `ListenAndServeTLS` 支持 test mode 自签证书路径。 +- `internal/client/join.go` — `Join` 支持 `insecureTLS` 参数,自签时跳过校验。 +- `internal/client/up.go` — `Up` 读取 client config 的 insecure 标记,传给 tunnel client。 +- `internal/client/config.go` — `ClientConfig` 增加 `InsecureTLS bool`。 +- `internal/tunnel/client.go` — `NewTunnelClient` 增加 `tlsConfig *tls.Config` 参数。 + +**测试基建(T01–T08):** +- `tests/e2e/Dockerfile.server` — server 容器镜像。 +- `tests/e2e/Dockerfile.client` — client 容器镜像。 +- `tests/e2e/docker-compose.yml` — 3 容器编排。 +- `tests/e2e/fixtures/meshd.yaml` — server 配置模板。 +- `tests/e2e/fixtures/entrypoint-server.sh` — server 容器启动逻辑。 +- `tests/e2e/fixtures/entrypoint-client.sh` — client 容器启动逻辑(等待→join→up)。 +- `tests/e2e/fixtures/netem.sh` — tc netem 预设封装。 +- `tests/e2e/lib/helpers.sh` — wait_ready / register / show_logs 等通用函数。 +- `tests/e2e/lib/metrics.sh` — RTT/吞吐/丢包采集与 JSON 输出。 +- `tests/e2e/scenarios/01-connectivity.sh` — P0 连通性场景。 +- `tests/e2e/scenarios/02-performance.sh` — P0 性能场景。 +- `tests/e2e/scenarios/03-failure.sh` — P1 故障场景。 +- `tests/e2e/run.sh` — 一键入口(启动/跑场景/收尾/聚合)。 +- `.github/workflows/e2e.yml` — CI 触发。 + +--- + +## Task 0: MESH_TEST_TLS 开关(自签证书支持) + +**Files:** +- Modify: `internal/config/config.go` +- Modify: `internal/api/server.go` +- Modify: `internal/client/config.go` +- Modify: `internal/client/join.go` +- Modify: `internal/client/up.go` +- Modify: `internal/tunnel/client.go` +- Test: `internal/config/config_test.go` +- Test: `internal/client/join_test.go`(新建) + +**背景:** 容器内 server 拿不到 Let's Encrypt 证书。需要让 server 在 `MESH_TEST_TLS=off` 时用自签证书,client 端跳过证书校验。这是整个 e2e 的前置条件。 + +- [ ] **Step 1: 写 config 解析的失败测试** + +在 `internal/config/config_test.go` 末尾追加: + +```go +func TestLoadTLSTestModeFromEnv(t *testing.T) { + t.Setenv("MESH_TEST_TLS", "off") + cfg := Default() + cfg.applyTestMode() + if !cfg.TLSTestMode { + t.Fatal("expected TLSTestMode=true when MESH_TEST_TLS=off") + } + + t.Setenv("MESH_TEST_TLS", "") + cfg2 := Default() + cfg2.applyTestMode() + if cfg2.TLSTestMode { + t.Fatal("expected TLSTestMode=false when MESH_TEST_TLS unset") + } +} +``` + +- [ ] **Step 2: 运行测试,确认失败** + +Run: `go test ./internal/config/... -run TestLoadTLSTestModeFromEnv -v` +Expected: FAIL — `cfg.TLSTestMode undefined` / `cfg.applyTestMode undefined`。 + +- [ ] **Step 3: 实现 config 字段与方法** + +在 `internal/config/config.go` 的 `Config` struct 加字段,并在 `Load` 末尾调用 `applyTestMode`: + +```go +type Config struct { + Domain string `yaml:"domain"` + ListenAddr string `yaml:"listen_addr"` + Network string `yaml:"network"` + DataDir string `yaml:"data_dir"` + CertDir string `yaml:"cert_dir"` + TunName string `yaml:"tun_name"` + TunMTU int `yaml:"tun_mtu"` + TLSTestMode bool `yaml:"-"` // 由 MESH_TEST_TLS 环境变量控制,不入 yaml +} + +// applyTestMode reads MESH_TEST_TLS env and enables self-signed TLS mode. +// MESH_TEST_TLS=off (or 1/true/on) switches the server off Let's Encrypt. +func (c *Config) applyTestMode() { + v := strings.ToLower(strings.TrimSpace(os.Getenv("MESH_TEST_TLS"))) + switch v { + case "off", "1", "true", "on", "self": + c.TLSTestMode = true + } +} +``` + +并在 `Load` 的 `return cfg, nil` 前插入 `cfg.applyTestMode()`。import 增加 `"strings"`。 + +- [ ] **Step 4: 运行测试,确认通过** + +Run: `go test ./internal/config/... -run TestLoadTLSTestModeFromEnv -v` +Expected: PASS。 + +- [ ] **Step 5: 让 server 在 test mode 下用自签证书** + +修改 `internal/api/server.go` 的 `ListenAndServeTLS`。替换整个函数体为: + +```go +// ListenAndServeTLS starts the HTTPS server. In production it uses autocert +// (Let's Encrypt); when cfg.TLSTestMode is set it uses an in-memory +// self-signed certificate so the server can run in containers without a +// real domain. +func (s *Server) ListenAndServeTLS(ctx context.Context) error { + srv := &http.Server{ + Addr: s.cfg.ListenAddr, + Handler: s.Handler(), + } + + go func() { + <-ctx.Done() + srv.Close() //nolint:errcheck + }() + + if s.cfg.TLSTestMode { + tlsCfg, err := selfSignedTLSConfig() + if err != nil { + return fmt.Errorf("self-signed cert: %w", err) + } + srv.TLSConfig = tlsCfg + return srv.ListenAndServeTLS("", "") + } + + m := &autocert.Manager{ + Cache: autocert.DirCache(s.cfg.CertDir), + Prompt: autocert.AcceptTOS, + HostPolicy: autocert.HostWhitelist(s.cfg.Domain), + } + srv.TLSConfig = &tls.Config{ + GetCertificate: m.GetCertificate, + MinVersion: tls.VersionTLS12, + } + // HTTP-01 ACME challenge listener on :80 + go http.ListenAndServe(":80", m.HTTPHandler(nil)) //nolint:errcheck + return srv.ListenAndServeTLS("", "") +} + +// selfSignedTLSConfig builds an in-memory self-signed certificate valid for +// the configured domain and common loopback names, used only in test mode. +func selfSignedTLSConfig() (*tls.Config, error) { + host := s.cfg.Domain + if host == "" { + host = "localhost" + } + tpl := x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: host}, + NotBefore: time.Now(), + NotAfter: time.Now().Add(24 * time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + BasicConstraintsValid: true, + IsCA: true, + DNSNames: []string{host, "localhost", "server"}, + IPAddresses: []net.IP{net.IPv4(127, 0, 0, 1)}, + } + priv, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + return nil, err + } + der, err := x509.CreateCertificate(rand.Reader, &tpl, &tpl, &priv.PublicKey, priv) + if err != nil { + return nil, err + } + return &tls.Config{ + Certificates: []tls.Certificate{{Certificate: [][]byte{der}, PrivateKey: priv}}, + MinVersion: tls.VersionTLS12, + }, nil +} +``` + +注意 `selfSignedTLSConfig` 需要访问 `s.cfg.Domain`,所以把它改成方法 `func (s *Server) selfSignedTLSConfig()`,并把上面调用处 `selfSignedTLSConfig()` 改为 `s.selfSignedTLSConfig()`。 + +新增 import:`"crypto/ecdsa"`、`"crypto/elliptic"`、`"crypto/rand"`、`"crypto/x509"`、`"crypto/x509/pkix"`、`"math/big"`、`"net"`、`"time"`。 + +- [ ] **Step 6: 让 client config 携带 insecure 标记** + +修改 `internal/client/config.go` 的 `ClientConfig`: + +```go +type ClientConfig struct { + ServerDomain string `json:"server_domain"` + DeviceSecret string `json:"device_secret"` + DeviceIP string `json:"device_ip"` + DeviceID string `json:"device_id"` + NetworkCIDR string `json:"network_cidr"` + InsecureTLS bool `json:"insecure_tls,omitempty"` +} +``` + +- [ ] **Step 7: 让 join 接受 insecure 选项并保存到 config** + +修改 `internal/client/join.go` 的 `Join` 签名与实现: + +```go +func Join(domain, tok string, insecureTLS bool) error { + if err := os.MkdirAll(ConfigDir(), 0700); err != nil { + return err + } + if _, err := LoadClientConfig(); err == nil { + return fmt.Errorf("already registered; run 'mesh leave' first") + } + + hostname, _ := os.Hostname() + reqBody, _ := json.Marshal(map[string]string{ + "token": tok, + "hostname": hostname, + }) + + httpClient := &http.Client{Timeout: 10 * time.Second} + if insecureTLS { + httpClient.Transport = &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + } + } + url := fmt.Sprintf("https://%s/api/devices/register", domain) + resp, err := httpClient.Post(url, "application/json", bytes.NewReader(reqBody)) + if err != nil { + return fmt.Errorf("connect: %w", err) + } + defer resp.Body.Close() + + switch resp.StatusCode { + case http.StatusUnauthorized: + return fmt.Errorf("invalid token") + case http.StatusOK: + default: + return fmt.Errorf("registration failed: HTTP %d", resp.StatusCode) + } + + var regResp struct { + AssignedIP string `json:"assigned_ip"` + DeviceSecret string `json:"device_secret"` + DeviceID string `json:"device_id"` + NetworkCIDR string `json:"network_cidr"` + } + if err := json.NewDecoder(resp.Body).Decode(®Resp); err != nil { + return fmt.Errorf("parse response: %w", err) + } + + cfg := &ClientConfig{ + ServerDomain: domain, + DeviceSecret: regResp.DeviceSecret, + DeviceIP: regResp.AssignedIP, + DeviceID: regResp.DeviceID, + NetworkCIDR: regResp.NetworkCIDR, + InsecureTLS: insecureTLS, + } + if err := SaveClientConfig(cfg); err != nil { + return err + } + + fmt.Printf("Registered!\n IP: %s\n Network: %s\n", regResp.AssignedIP, regResp.NetworkCIDR) + return nil +} +``` + +新增 import:`"crypto/tls"`、`"time"`。 + +更新 `cmd/mesh/main.go` 的 `joinCmd`:增加 `--insecure` flag,传给 `client.Join`: + +```go +func joinCmd() *cobra.Command { + var tok, insecureStr string + cmd := &cobra.Command{ + Use: "join ", + Args: cobra.ExactArgs(1), + Short: "向服务器注册并加入 mesh 网络(无需 root)", + RunE: func(c *cobra.Command, args []string) error { + insecure := insecureStr != "" + if err := client.Join(args[0], tok, insecure); err != nil { + return err + } + fmt.Println("Now run 'sudo mesh up' to start the tunnel.") + return nil + }, + } + cmd.Flags().StringVar(&tok, "token", "", "注册令牌(由服务器管理员提供)") + cmd.Flags().StringVar(&insecureStr, "insecure", "", "跳过 TLS 证书校验(用于自签测试环境),任意非空值启用") + _ = cmd.MarkFlagRequired("token") + return cmd +} +``` + +- [ ] **Step 8: 让 tunnel client 接受 TLS 配置** + +修改 `internal/tunnel/client.go` 的 `NewTunnelClient`,增加 `tlsConfig *tls.Config` 参数,并在 `connect` 的 `websocket.Dial` 里传: + +```go +import "crypto/tls" + +func NewTunnelClient(serverURL, secret, localIP, network string, mtu int, statusDir string, tlsConfig *tls.Config) (*TunnelClient, error) { + // ... 原有 TUN 创建逻辑不变 ... + return &TunnelClient{ + serverURL: serverURL, + secret: secret, + mtu: mtu, + tun: dev, + statusDir: statusDir, + tlsConfig: tlsConfig, + }, nil +} +``` + +在 `TunnelClient` struct 加字段 `tlsConfig *tls.Config`。 + +在 `connect` 里改 dial: + +```go +conn, _, err := websocket.Dial(ctx, tc.serverURL, &websocket.DialOptions{ + HTTPHeader: header, + HTTPClient: &http.Client{Transport: &http.Transport{TLSClientConfig: tc.tlsConfig}}, +}) +``` + +新增 import `"net/http"`。 + +修改 `internal/client/up.go`,根据 `cfg.InsecureTLS` 构造 tlsConfig: + +```go +func Up(ctx context.Context) error { + cfg, err := LoadClientConfig() + if err != nil { + return fmt.Errorf("not registered; run 'mesh join' first") + } + + wsURL := fmt.Sprintf("wss://%s/tunnel", cfg.ServerDomain) + var tlsCfg *tls.Config + if cfg.InsecureTLS { + tlsCfg = &tls.Config{InsecureSkipVerify: true} + } + tc, err := tunnel.NewTunnelClient(wsURL, cfg.DeviceSecret, cfg.DeviceIP, cfg.NetworkCIDR, 1300, ConfigDir(), tlsCfg) + if err != nil { + return fmt.Errorf("setup tunnel: %w", err) + } + defer tc.Close() + + fmt.Printf("Mesh VPN up (IP: %s)\n", cfg.DeviceIP) + return tc.Run(ctx) +} +``` + +新增 import `"crypto/tls"`。 + +- [ ] **Step 9: 更新 NewTunnelClient 的调用点** + +`NewTunnelClient` 签名加了 `tlsConfig` 参数,全仓调用点需同步。当前唯一调用点是 `internal/client/up.go:19`(已在 Step 8 改过)。跑一次确认无遗漏: + +Run: `grep -rn "NewTunnelClient" --include="*.go" .` +Expected: 只剩定义处(`internal/tunnel/client.go`)和 `up.go` 已带 `tlsCfg` 参数的调用,无其它调用点。若有遗漏,给该调用末尾补 `nil`(测试场景无 TLS)。 + +- [ ] **Step 10: 编译 + 全量测试** + +Run: `go build ./... && go vet ./... && go test ./... -count=1` +Expected: 全部通过。 + +- [ ] **Step 11: 提交** + +```bash +git add internal/ cmd/ +git commit -m "$(cat <<'EOF' +feat(e2e): T00 增加 MESH_TEST_TLS 开关支持自签证书 + +- config 增加 TLSTestMode,由 MESH_TEST_TLS 环境变量控制 +- server 在 test mode 用内存自签证书,不依赖 Let's Encrypt +- client join/up 支持 --insecure 跳过证书校验 +- NewTunnelClient 增加 tlsConfig 参数 + +Co-Authored-By: Claude +EOF +)" +``` + +--- + +## Task 1: Docker 镜像与 compose 编排 + +**Files:** +- Create: `tests/e2e/Dockerfile.server` +- Create: `tests/e2e/Dockerfile.client` +- Create: `tests/e2e/docker-compose.yml` +- Create: `tests/e2e/fixtures/meshd.yaml` +- Create: `tests/e2e/fixtures/entrypoint-server.sh` +- Create: `tests/e2e/fixtures/entrypoint-client.sh` + +**背景:** 三个容器需要 root + NET_ADMIN + /dev/net/tun。server 镜像含 meshd,client 镜像含 mesh + iperf3 + tc。 + +- [ ] **Step 1: 写 server Dockerfile** + +`tests/e2e/Dockerfile.server`: + +```dockerfile +FROM golang:1.25-bookworm AS build +WORKDIR /src +COPY go.mod go.sum ./ +RUN go mod download +COPY . . +RUN CGO_ENABLED=1 go build -o /out/meshd ./cmd/meshd + +FROM ubuntu:24.04 +RUN apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates iproute2 iputils-ping curl jq bash && \ + rm -rf /var/lib/apt/lists/* +COPY --from=build /out/meshd /usr/local/bin/meshd +COPY tests/e2e/fixtures/meshd.yaml /etc/mesh/meshd.yaml +COPY tests/e2e/fixtures/entrypoint-server.sh /entrypoint.sh +RUN chmod +x /entrypoint.sh +ENTRYPOINT ["/entrypoint.sh"] +``` + +- [ ] **Step 2: 写 client Dockerfile** + +`tests/e2e/Dockerfile.client`: + +```dockerfile +FROM golang:1.25-bookworm AS build +WORKDIR /src +COPY go.mod go.sum ./ +RUN go mod download +COPY . . +RUN CGO_ENABLED=1 go build -o /out/mesh ./cmd/mesh + +FROM ubuntu:24.04 +RUN apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates iproute2 iputils-ping iperf3 nuttcp fping curl jq bash && \ + rm -rf /var/lib/apt/lists/* +COPY --from=build /out/mesh /usr/local/bin/mesh +COPY tests/e2e/fixtures/entrypoint-client.sh /entrypoint.sh +RUN chmod +x /entrypoint.sh +ENTRYPOINT ["/entrypoint.sh"] +``` + +- [ ] **Step 3: 写 server 配置** + +`tests/e2e/fixtures/meshd.yaml`: + +```yaml +domain: "server" +listen_addr: ":443" +network: "10.100.0.0/24" +data_dir: "/etc/mesh" +cert_dir: "/etc/mesh/certs" +tun_name: "mesh0" +tun_mtu: 1300 +``` + +- [ ] **Step 4: 写 server entrypoint** + +`tests/e2e/fixtures/entrypoint-server.sh`: + +```bash +#!/usr/bin/env bash +set -euo pipefail + +export MESH_TEST_TLS=off + +meshd init +exec meshd run +``` + +- [ ] **Step 5: 写 client entrypoint(占位,正式 join 逻辑在 T04)** + +`tests/e2e/fixtures/entrypoint-client.sh`: + +```bash +#!/usr/bin/env bash +set -euo pipefail + +# 等待 server 起来 +echo "waiting for server..." +for i in $(seq 1 60); do + if curl -kfsS https://server:443/api/devices >/dev/null 2>&1; then + echo "server is up" + break + fi + sleep 1 +done + +# 读取由 run.sh 注入的 TOKEN,注册本设备 +: "${MESH_TOKEN:?MESH_TOKEN must be set}" +HOSTNAME_ID="${HOSTNAME_ID:-$(hostname)}" +mesh join server --token "$MESH_TOKEN" --insecure + +# 启动隧道(前台) +exec mesh up +``` + +- [ ] **Step 6: 写 docker-compose.yml** + +`tests/e2e/docker-compose.yml`: + +```yaml +version: "3.9" +services: + server: + build: + context: ../.. + dockerfile: tests/e2e/Dockerfile.server + image: mesh-e2e/server:dev + container_name: mesh-server + privileged: true + cap_add: [NET_ADMIN, NET_RAW] + networks: + meshnet: + aliases: [server] + + client-a: + build: + context: ../.. + dockerfile: tests/e2e/Dockerfile.client + image: mesh-e2e/client:dev + container_name: mesh-client-a + privileged: true + cap_add: [NET_ADMIN, NET_RAW] + devices: ["/dev/net/tun:/dev/net/tun"] + environment: + - MESH_TOKEN + - HOSTNAME_ID=client-a + depends_on: [server] + networks: [meshnet] + + client-b: + build: + context: ../.. + dockerfile: tests/e2e/Dockerfile.client + image: mesh-e2e/client:dev + container_name: mesh-client-b + privileged: true + cap_add: [NET_ADMIN, NET_RAW] + devices: ["/dev/net/tun:/dev/net/tun"] + environment: + - MESH_TOKEN + - HOSTNAME_ID=client-b + depends_on: [server] + networks: [meshnet] + +networks: + meshnet: + driver: bridge +``` + +- [ ] **Step 7: 验证 server 镜像能构建并启动** + +Run: +```bash +MESH_TOKEN=dummy docker compose -f tests/e2e/docker-compose.yml build server +MESH_TOKEN=dummy docker compose -f tests/e2e/docker-compose.yml up -d server +sleep 5 +docker compose -f tests/e2e/docker-compose.yml logs server | head -20 +``` +Expected: 日志出现 `Mesh VPN server starting` 和 `Initialized.`;`curl -k https://localhost:8443` 在容器内能拿到 cover page(需端口映射,下一步验证用 exec): +```bash +docker exec mesh-server curl -kfsS https://server:443/api/devices +``` +Expected: 返回 `[]`(空设备列表)。 + +收尾:`docker compose -f tests/e2e/docker-compose.yml down`。 + +- [ ] **Step 8: 提交** + +```bash +git add tests/e2e/ +git commit -m "$(cat <<'EOF' +feat(e2e): T01 Docker 镜像与 compose 编排 + +- Dockerfile.server / Dockerfile.client(多阶段构建) +- docker-compose 3 容器拓扑(privileged + NET_ADMIN + /dev/net/tun) +- fixtures: meshd.yaml / entrypoint-server.sh / entrypoint-client.sh + +Co-Authored-By: Claude +EOF +)" +``` + +--- + +## Task 2: lib/helpers.sh 与 lib/metrics.sh + +**Files:** +- Create: `tests/e2e/lib/helpers.sh` +- Create: `tests/e2e/lib/metrics.sh` + +**背景:** 场景脚本需要通用函数:容器执行、等待、注册、日志收集、指标采集与 JSON 写入。 + +- [ ] **Step 1: 写 helpers.sh** + +`tests/e2e/lib/helpers.sh`: + +```bash +#!/usr/bin/env bash +# 通用辅助函数。所有场景脚本 source 本文件。 + +# 项目根目录(tests/e2e 的上两级) +E2E_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +REPO_ROOT="$(cd "$E2E_DIR/../.." && pwd)" +COMPOSE="$E2E_DIR/docker-compose.yml" +RESULTS_DIR="${RESULTS_DIR:-$E2E_DIR/results}" + +# 在指定容器内执行命令 +dex() { docker exec "$@"; } + +# 等 server 容器内 meshd 可达 +wait_for_server() { + echo "waiting for server meshd..." + for i in $(seq 1 60); do + if dex mesh-server curl -kfsS https://server:443/api/devices >/dev/null 2>&1; then + return 0 + fi + sleep 1 + done + echo "ERROR: server did not become ready" >&2 + return 1 +} + +# 等 client 容器内 mesh0 起来并有 IP +wait_for_client() { + local c="$1" + for i in $(seq 1 60); do + if dex "$c" ip -o addr show mesh0 2>/dev/null | grep -q inet; then + return 0 + fi + sleep 1 + done + echo "ERROR: client $c mesh0 did not come up" >&2 + return 1 +} + +# 从 server 容器取 register token +get_token() { + dex mesh-server sh -c 'sqlite3 /etc/mesh/mesh.db "SELECT token FROM tokens LIMIT 1" 2>/dev/null || true' + # sqlite3 可能不在 server 镜像里;fallback 用 meshd 启动日志里打印的 token +} + +# 收集所有容器最近日志到 results 目录 +dump_logs() { + local out="$1" + mkdir -p "$out" + for c in mesh-server mesh-client-a mesh-client-b; do + docker logs "$c" >"$out/$c.log" 2>&1 || true + done +} + +# 断言:命令退出码符合预期 +assert() { + local label="$1"; shift + if "$@"; then + echo " [PASS] $label" + echo "1" >>"$ASSERT_OK" + else + echo " [FAIL] $label" + echo "1" >>"$ASSERT_FAIL" + fi +} +``` + +注意 `get_token` 的 sqlite3 路径:server 镜像没装 sqlite3。改方案——在 entrypoint-server.sh 里把 token 写到 `/etc/mesh/token`: + +回 T01 的 `entrypoint-server.sh`,改为: + +```bash +#!/usr/bin/env bash +set -euo pipefail + +export MESH_TEST_TLS=off + +meshd init | tee /tmp/init.log +# meshd init 打印 "Token: xxx",提取写入文件供测试读取 +grep -oP 'Token: \K\S+' /tmp/init.log > /etc/mesh/token 2>/dev/null || true + +exec meshd run +``` + +`get_token` 改为: + +```bash +get_token() { + dex mesh-server cat /etc/mesh/token +} +``` + +- [ ] **Step 2: 写 metrics.sh** + +`tests/e2e/lib/metrics.sh`: + +```bash +#!/usr/bin/env bash +# 指标采集与 JSON 输出。 + +# ping RTT 统计:输出 "p50 p95 p99 loss_pct" +# 用法: rtt_stats +rtt_stats() { + local c="$1" dst="$2" count="${3:-200}" + local tmp; tmp=$(dex "$c" ping -c "$count" -i 0.05 -q "$dst" 2>&1) || true + # 最后一行: rtt min/avg/max/mdev = x/y/z/w ms + local rttline; rttline=$(echo "$tmp" | grep -oP 'rtt min/avg/max/mdev = \K[^ ]+(?= ms)' || true) + local loss; loss=$(echo "$tmp" | grep -oP '\K(\d+)(?=% packet loss)' || echo "100") + # ping -q 不直接给百分位,用 avg/mdev 近似;精确百分位留给 fping + echo "${rttline:-0} ${loss}" +} + +# iperf3 TCP 吞吐(Mbps) +iperf_tcp() { + local client="$1" dst="$2" streams="${3:-1}" time="${4:-10}" + dex "$client" iperf3 -c "$dst" -t "$time" -P "$streams" -J 2>/dev/null \ + | jq -r '.end.sum_received.bits_per_second / 1e6' 2>/dev/null || echo "0" +} + +# iperf3 UDP 丢包率 +iperf_udp_loss() { + local client="$1" dst="$2" rate="${3:-100M}" time="${4:-10}" + dex "$client" iperf3 -c "$dst" -u -b "$rate" -t "$time" -J 2>/dev/null \ + | jq -r '.end.sum.lost_percent // 100' 2>/dev/null || echo "100" +} + +# 往 JSON 结果文件写一个 key/value +json_set() { + local file="$1" key="$2" val="$3" + jq --arg k "$key" --argjson v "$val" '.[$k] = $v' "$file" >"$file.tmp" && mv "$file.tmp" "$file" +} +``` + +- [ ] **Step 3: 提交** + +```bash +git add tests/e2e/lib/ tests/e2e/fixtures/entrypoint-server.sh +git commit -m "$(cat <<'EOF' +feat(e2e): T02 helpers.sh 与 metrics.sh + +- helpers: 容器执行/等待/日志收集/断言 +- metrics: rtt/iperf TCP/iperf UDP 丢包 + JSON 写入 +- entrypoint-server 导出 token 到 /etc/mesh/token + +Co-Authored-By: Claude +EOF +)" +``` + +--- + +## Task 3: fixtures/netem.sh(tc netem 预设) + +**Files:** +- Create: `tests/e2e/fixtures/netem.sh` + +**背景:** 场景脚本需要按预设切换链路质量。封装成函数避免重复。 + +- [ ] **Step 1: 写 netem.sh** + +`tests/e2e/fixtures/netem.sh`: + +```bash +#!/usr/bin/env bash +# 在容器内设置 tc netem 预设。用容器内 eth0。 +# 用法: netem (在目标容器内执行) +# preset: clean | baseline | wan | bad | satellite + +netem() { + local preset="${1:-clean}" + case "$preset" in + clean|baseline) + tc qdisc del dev eth0 root 2>/dev/null || true + ;; + wan) + tc qdisc del dev eth0 root 2>/dev/null || true + tc qdisc add dev eth0 root netem delay 80ms 10ms loss 1% + ;; + bad) + tc qdisc del dev eth0 root 2>/dev/null || true + tc qdisc add dev eth0 root netem delay 200ms 50ms loss 5% + ;; + satellite) + tc qdisc del dev eth0 root 2>/dev/null || true + tc qdisc add dev eth0 root netem delay 600ms 100ms loss 2% + ;; + *) + echo "unknown netem preset: $preset" >&2 + return 1 + ;; + esac +} +``` + +- [ ] **Step 2: 把 netem.sh 拷进 client 镜像** + +回 T01 的 `Dockerfile.client`,在 COPY mesh 后加一行: + +```dockerfile +COPY tests/e2e/fixtures/netem.sh /usr/local/bin/netem-preset.sh +RUN chmod +x /usr/local/bin/netem-preset.sh +``` + +(不直接放到 PATH,避免误触发;场景脚本里 `source` 它。) + +- [ ] **Step 3: 手测 netem 能用** + +Run: +```bash +MESH_TOKEN=x docker compose -f tests/e2e/docker-compose.yml run --rm client-a bash -c \ + 'source /usr/local/bin/netem-preset.sh; netem wan; tc qdisc show dev eth0' +``` +Expected: 输出含 `qdisc netem` 与 `delay 80ms`。 + +- [ ] **Step 4: 提交** + +```bash +git add tests/e2e/fixtures/netem.sh tests/e2e/Dockerfile.client +git commit -m "$(cat <<'EOF' +feat(e2e): T03 netem 预设封装(clean/baseline/wan/bad/satellite) + +Co-Authored-By: Claude +EOF +)" +``` + +--- + +## Task 4: 场景 01 — 连通性与路由(P0) + +**Files:** +- Create: `tests/e2e/scenarios/01-connectivity.sh` + +**背景:** 验证 TUN 路由注入、跨 client/server ping、不存在 IP 丢包、client 断开/重连。这是 e2e 最核心的正确性证明。 + +- [ ] **Step 1: 写场景脚本** + +`tests/e2e/scenarios/01-connectivity.sh`: + +```bash +#!/usr/bin/env bash +set -euo pipefail +source "$(dirname "$0")/../lib/helpers.sh" + +OUT="$RESULTS_DIR/01-connectivity" +mkdir -p "$OUT" +ASSERT_OK="$OUT/ok"; : >"$ASSERT_OK" +ASSERT_FAIL="$OUT/fail"; : >"$ASSERT_FAIL" +LOG="$OUT/01-connectivity.log"; : >"$LOG" +exec > >(tee -a "$LOG") 2>&1 + +echo "=== Scenario 01: connectivity & routing ===" + +wait_for_server +wait_for_client mesh-client-a +wait_for_client mesh-client-b + +SERVER_IP="10.100.0.1" +A_IP=$(dex mesh-client-a ip -o -4 addr show mesh0 | awk '{print $4}' | cut -d/ -f1) +B_IP=$(dex mesh-client-b ip -o -4 addr show mesh0 | awk '{print $4}' | cut -d/ -f1) +echo "client-a=$A_IP client-b=$B_IP" + +# 01.3 路由表:10.100.0.0/24 走 mesh0 +assert "client-a has mesh route" \ + bash -c "dex mesh-client-a ip route | grep -q '10.100.0.0/24.*mesh0'" + +# 01.4 ping server +assert "client-a ping server" \ + dex mesh-client-a ping -c 3 -W 2 "$SERVER_IP" + +# 01.5 ping peer +assert "client-a ping client-b" \ + dex mesh-client-a ping -c 3 -W 2 "$B_IP" + +# 01.6 不存在的 IP 必须 100% 丢包 +NONEXIST=$(dex mesh-client-a ping -c 3 -W 1 10.100.0.99 2>&1 || true) +LOSS=$(echo "$NONEXIST" | grep -oP '\K\d+(?=% packet loss)' || echo 0) +assert "unreachable IP 100% loss" \ + bash -c "[ '$LOSS' -eq 100 ]" + +# 01.7 断开 client-b +docker compose -f "$COMPOSE" stop client-b >/dev/null +sleep 3 +assert "client-b offline: a->b fails" \ + bash -c "! dex mesh-client-a ping -c 3 -W 2 $B_IP 2>/dev/null" + +# 01.8 重启 client-b,恢复 +docker compose -f "$COMPOSE" start client-b >/dev/null +wait_for_client mesh-client-b +assert "client-b reconnect: a->b works" \ + dex mesh-client-a ping -c 3 -W 3 "$B_IP" + +# 01.9 fping 并发不丢 +assert "fping burst no loss" \ + bash -c "dex mesh-client-a fping -q -c 20 -p 100 $B_IP 2>&1 | grep -q '0% loss' || \ + dex mesh-client-a fping -q -c 20 -p 100 $B_IP 2>&1 | grep -oP '\K\d+(?=% loss)' | head -1 | awk '{exit !(\$1<5)}'" + +OK=$(wc -l <"$ASSERT_OK" | tr -d ' ') +FAIL=$(wc -l <"$ASSERT_FAIL" | tr -d ' ') +echo "=== 01 result: PASS=$OK FAIL=$FAIL ===" +jq -n --arg ok "$OK" --arg fail "$FAIL" '{pass:($ok|tonumber), fail:($fail|tonumber)}' >"$OUT/01-connectivity.json" +test "$FAIL" -eq 0 +``` + +- [ ] **Step 2: 跑场景验证** + +Run: +```bash +MESH_TOKEN=$(docker exec mesh-server cat /etc/mesh/token 2>/dev/null || echo x) \ + bash tests/e2e/scenarios/01-connectivity.sh +``` +Expected: 末尾 `01 result: PASS=6 FAIL=0`,退出码 0。 + +- [ ] **Step 3: 提交** + +```bash +git add tests/e2e/scenarios/01-connectivity.sh +git commit -m "$(cat <<'EOF' +feat(e2e): T04 场景 01 连通性与路由 + +- mesh0 路由注入、ping server/peer、不存在 IP 100% 丢包 +- client-b 断开/重连、fping 并发 + +Co-Authored-By: Claude +EOF +)" +``` + +--- + +## Task 5: 场景 02 — 性能与抖动(P0) + +**Files:** +- Create: `tests/e2e/scenarios/02-performance.sh` + +**背景:** 量化 TCP 吞吐、UDP 丢包、RTT,在 baseline / wan / bad 三种链路下对比,输出 JSON。 + +- [ ] **Step 1: 写场景脚本** + +`tests/e2e/scenarios/02-performance.sh`: + +```bash +#!/usr/bin/env bash +set -euo pipefail +source "$(dirname "$0")/../lib/helpers.sh" +source "$(dirname "$0")/../lib/metrics.sh" + +OUT="$RESULTS_DIR/02-performance" +mkdir -p "$OUT" +JSON="$OUT/02-performance.json" +echo '{}' >"$JSON" +LOG="$OUT/02-performance.log"; : >"$LOG" +exec > >(tee -a "$LOG") 2>&1 + +echo "=== Scenario 02: performance ===" +wait_for_client mesh-client-a +wait_for_client mesh-client-b +B_IP=$(dex mesh-client-b ip -o -4 addr show mesh0 | awk '{print $4}' | cut -d/ -f1) + +# 在 client-b 起一个 iperf3 server(daemon) +dex mesh-client-b sh -c 'pkill iperf3 2>/dev/null; iperf3 -s -D' + +# ---- baseline ---- +dex mesh-client-b sh -c 'source /usr/local/bin/netem-preset.sh; netem baseline' +TCP1=$(iperf_tcp mesh-client-a "$B_IP" 1 10) +json_set "$JSON" tcp_1stream_mbps "$TCP1" +TCP4=$(iperf_tcp mesh-client-a "$B_IP" 4 10) +json_set "$JSON" tcp_4stream_mbps "$TCP4" +UDPL=$(iperf_udp_loss mesh-client-a "$B_IP" 100M 10) +json_set "$JSON" udp_100m_loss_pct "$UDPL" + +# ---- wan ---- +dex mesh-client-b sh -c 'source /usr/local/bin/netem-preset.sh; netem wan' +WAN_TCP=$(iperf_tcp mesh-client-a "$B_IP" 1 10) +json_set "$JSON" wan_tcp_mbps "$WAN_TCP" +WAN_UDP=$(iperf_udp_loss mesh-client-a "$B_IP" 100M 10) +json_set "$JSON" wan_udp_loss_pct "$WAN_UDP" + +# ---- RTT (baseline) ---- +dex mesh-client-b sh -c 'source /usr/local/bin/netem-preset.sh; netem baseline' +RTT=$(rtt_stats mesh-client-a "$B_IP" 200) +AVG=$(echo "$RTT" | awk '{print $1}') +LOSS=$(echo "$RTT" | awk '{print $2}') +json_set "$JSON" rtt_avg_ms "$AVG" +json_set "$JSON" rtt_loss_pct "$LOSS" + +# ---- strict 门槛(可选)---- +if [ "${STRICT:-0}" = "1" ]; then + awk_exit=0 + awk -v tcp="$TCP1" -v udp="$WAN_UDP" ' + BEGIN{ + if (tcp+0 < 30) {print "STRICT FAIL: tcp_1stream="tcp" < 30"; awk_exit=1} + if (udp+0 > 5) {print "STRICT FAIL: wan_udp_loss="udp" > 5"; awk_exit=1} + exit awk_exit + }' +fi + +dex mesh-client-b pkill iperf3 2>/dev/null || true +echo "=== 02 done ===" +cat "$JSON" +``` + +- [ ] **Step 2: 跑场景验证** + +Run: +```bash +bash tests/e2e/scenarios/02-performance.sh +``` +Expected: 输出 JSON 含 `tcp_1stream_mbps`、`wan_tcp_mbps` 等字段,数值非 0。 + +- [ ] **Step 3: 提交** + +```bash +git add tests/e2e/scenarios/02-performance.sh +git commit -m "$(cat <<'EOF' +feat(e2e): T05 场景 02 性能与抖动 + +- iperf3 TCP 1流/4流、UDP 丢包 +- baseline/wan 链路对比 +- RTT 采集,可选 STRICT 硬门槛 + +Co-Authored-By: Claude +EOF +)" +``` + +--- + +## Task 6: 场景 03 — 故障 / 重连 / 优雅退出(P1) + +**Files:** +- Create: `tests/e2e/scenarios/03-failure.sh` + +- [ ] **Step 1: 写场景脚本** + +`tests/e2e/scenarios/03-failure.sh`: + +```bash +#!/usr/bin/env bash +set -euo pipefail +source "$(dirname "$0")/../lib/helpers.sh" + +OUT="$RESULTS_DIR/03-failure" +mkdir -p "$OUT" +ASSERT_OK="$OUT/ok"; : >"$ASSERT_OK" +ASSERT_FAIL="$OUT/fail"; : >"$ASSERT_FAIL" +LOG="$OUT/03-failure.log"; : >"$LOG" +exec > >(tee -a "$LOG") 2>&1 + +echo "=== Scenario 03: failure / reconnect / graceful shutdown ===" +wait_for_client mesh-client-a +wait_for_client mesh-client-b +B_IP=$(dex mesh-client-b ip -o -4 addr show mesh0 | awk '{print $4}' | cut -d/ -f1) + +# 03.1 kill server,client 应报错并准备重连 +docker compose -f "$COMPOSE" stop server >/dev/null +sleep 5 +assert "client-a logs reconnect attempt" \ + bash -c "docker logs mesh-client-a 2>&1 | grep -qiE 'connection lost|reconnect'" + +# 03.2 server 重启,client 恢复 +docker compose -f "$COMPOSE" start server >/dev/null +wait_for_server +# client 自动重连(TunnelClient.Run 内 3s 重连) +for i in $(seq 1 60); do + if dex mesh-client-a ping -c 1 -W 2 "$B_IP" >/dev/null 2>&1; then break; fi + sleep 1 +done +assert "server restart: a->b recovers" \ + dex mesh-client-a ping -c 3 -W 3 "$B_IP" + +# 03.3 长流中 server 短暂中断后恢复 +dex mesh-client-b sh -c 'pkill iperf3; iperf3 -s -D' +( dex mesh-client-a iperf3 -c "$B_IP" -t 20 >/dev/null 2>&1 ) & +IPERF_PID=$! +sleep 5 +docker compose -f "$COMPOSE" restart server >/dev/null +wait_for_server +wait $IPERF_PID || true +assert "server restart during iperf: connectivity restored" \ + dex mesh-client-a ping -c 3 -W 3 "$B_IP" + +# 03.6 SIGTERM client 优雅退出(<=2s) +START=$(date +%s) +docker compose -f "$COMPOSE" kill -s SIGTERM client-a >/dev/null +for i in $(seq 1 40); do + state=$(docker inspect -f '{{.State.Status}}' mesh-client-a 2>/dev/null || echo running) + [ "$state" = "exited" ] && break + sleep 0.1 +done +END=$(date +%s) +ELAPSED=$((END - START)) +assert "client SIGTERM exits <=2s" bash -c "[ $ELAPSED -le 2 ]" + +# 重新拉起 client-a 供后续 +docker compose -f "$COMPOSE" start client-a >/dev/null +wait_for_client mesh-client-a + +OK=$(wc -l <"$ASSERT_OK" | tr -d ' ') +FAIL=$(wc -l <"$ASSERT_FAIL" | tr -d ' ') +echo "=== 03 result: PASS=$OK FAIL=$FAIL ===" +jq -n --arg ok "$OK" --arg fail "$FAIL" '{pass:($ok|tonumber), fail:($fail|tonumber)}' >"$OUT/03-failure.json" +test "$FAIL" -eq 0 +``` + +- [ ] **Step 2: 跑场景验证** + +Run: `bash tests/e2e/scenarios/03-failure.sh` +Expected: `03 result: PASS=4 FAIL=0`,退出码 0。 + +- [ ] **Step 3: 提交** + +```bash +git add tests/e2e/scenarios/03-failure.sh +git commit -m "$(cat <<'EOF' +feat(e2e): T06 场景 03 故障/重连/优雅退出 + +- server kill→restart client 自动恢复 +- 长流中 server 重启 +- SIGTERM <=2s 退出 + +Co-Authored-By: Claude +EOF +)" +``` + +--- + +## Task 7: run.sh 入口与结果聚合 + +**Files:** +- Create: `tests/e2e/run.sh` + +**背景:** 一键:起容器 → 跑场景 → 收日志 → 生成 summary.txt → 收尾。 + +- [ ] **Step 1: 写 run.sh** + +`tests/e2e/run.sh`: + +```bash +#!/usr/bin/env bash +set -euo pipefail +E2E_DIR="$(cd "$(dirname "$0")" && pwd)" +source "$E2E_DIR/lib/helpers.sh" + +STRICT="${STRICT:-0}" +SCENARIOS=() + +usage() { echo "Usage: $0 [--all|--quick|--strict|--scenario N]"; exit 1; } + +while [ $# -gt 0 ]; do + case "$1" in + --all) SCENARIOS=(01 02 03);; + --quick) SCENARIOS=(01 02);; # 跳过耗时最长的 03 + --strict) STRICT=1; shift; continue;; + --scenario) SCENARIOS+=("$2"); shift 2;; + *) usage;; + esac + shift || true +done +[ ${#SCENARIOS[@]} -eq 0 ] && SCENARIOS=(01 02 03) + +TS="$(date -u +%Y-%m-%dT%H-%M-%S)" +export RESULTS_DIR="$E2E_DIR/results/$TS" +mkdir -p "$RESULTS_DIR" + +echo "=== e2e run $TS (scenarios: ${SCENARIOS[*]}, strict: $STRICT) ===" + +# 启动容器(需要 MESH_TOKEN) +export MESH_TOKEN="${MESH_TOKEN:-$(uuidgen)}" +docker compose -f "$COMPOSE" up -d --build +trap 'docker compose -f "$COMPOSE" down >/dev/null 2>&1 || true' EXIT + +wait_for_server +# 把真实 token 注入两个 client 容器(entrypoint 已 join) +# 重新读 token 确保 client 用对 +REAL_TOKEN=$(get_token) +docker compose -f "$COMPOSE" stop client-a client-b >/dev/null +MESH_TOKEN="$REAL_TOKEN" docker compose -f "$COMPOSE" start client-a client-b >/dev/null +wait_for_client mesh-client-a +wait_for_client mesh-client-b + +OVERALL=0 +for s in "${SCENARIOS[@]}"; do + echo "--- scenario $s ---" + if ! STRICT="$STRICT" bash "$E2E_DIR/scenarios/${s}-"*.sh; then + OVERALL=1 + fi +done + +dump_logs "$RESULTS_DIR" + +# 聚合 summary +{ + for s in "${SCENARIOS[@]}"; do + json=$(ls "$RESULTS_DIR"/0${s#0}-*/0${s#0}-*.json 2>/dev/null | head -1) + [ -z "$json" ] && json=$(ls "$RESULTS_DIR/${s}"*/*.json 2>/dev/null | head -1) + if [ -n "$json" ]; then + pf=$(jq -r '.pass' "$json"); fl=$(jq -r '.fail' "$json") + st="PASS"; [ "$fl" != "0" ] && st="FAIL" + echo "[${st}] scenario $s: pass=$pf fail=$fl" + fi + done + perf="$RESULTS_DIR/02-performance/02-performance.json" + [ -f "$perf" ] && echo "[PERF] $(jq -r '"tcp=\(.tcp_1stream_mbps)Mbps wan_tcp=\(.wan_tcp_mbps)Mbps rtt_avg=\(.rtt_avg_ms)ms"' "$perf")" + [ "$OVERALL" -eq 0 ] && echo "Overall: PASS" || echo "Overall: FAIL" +} | tee "$RESULTS_DIR/summary.txt" + +exit $OVERALL +``` + +- [ ] **Step 2: 跑完整 e2e** + +Run: `bash tests/e2e/run.sh --all` +Expected: 末尾 `Overall: PASS`,`results//summary.txt` 存在。 + +- [ ] **Step 3: 提交** + +```bash +git add tests/e2e/run.sh +git commit -m "$(cat <<'EOF' +feat(e2e): T07 run.sh 一键入口与结果聚合 + +- 起/收容器、注入 token、跑场景、聚合 summary.txt +- 支持 --all/--quick/--strict/--scenario N +- trap 保证容器 always down + +Co-Authored-By: Claude +EOF +)" +``` + +--- + +## Task 8: CI workflow + +**Files:** +- Create: `.github/workflows/e2e.yml` + +**背景:** push 触发 e2e;失败也要上传 artifact(CodeRabbit 提的 `if: always()`)。 + +- [ ] **Step 1: 写 workflow** + +`.github/workflows/e2e.yml`: + +```yaml +name: e2e +on: + push: + branches: [master] + pull_request: + branches: [master] + +jobs: + e2e: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + - name: Build e2e images & run + run: bash tests/e2e/run.sh --quick + - name: Collect report + if: always() + run: | + mkdir -p e2e-report + cp -r tests/e2e/results ./e2e-report/ 2>/dev/null || true + - uses: actions/upload-artifact@v4 + if: always() + with: + name: e2e-report + path: e2e-report/ +``` + +- [ ] **Step 2: 提交** + +```bash +git add .github/workflows/e2e.yml +git commit -m "$(cat <<'EOF' +ci(e2e): T08 GitHub Actions e2e workflow + +- push/PR 触发,--quick 模式 +- if: always() 保证 artifact 上传 + +Co-Authored-By: Claude +EOF +)" +``` + +- [ ] **Step 3: 推送触发 CI 验证** + +Run: `git push -u origin e2e/impl` +打开 PR,确认 CI e2e job 跑通(`Overall: PASS`)。 + +--- + +## 完成标准 + +- [ ] `bash tests/e2e/run.sh --all` 本地 PASS +- [ ] CI `e2e` job 在 PR 上绿 +- [ ] `results//summary.txt` 含 PASS 行 +- [ ] 性能 JSON 含非零吞吐与 RTT +- [ ] T00 的 `MESH_TEST_TLS=off` 在普通单测里也覆盖(不影响生产路径) diff --git a/docs/superpowers/specs/2026-07-08-e2e-docker-design.md b/docs/superpowers/specs/2026-07-08-e2e-docker-design.md index 477e5bf..e3e21fd 100644 --- a/docs/superpowers/specs/2026-07-08-e2e-docker-design.md +++ b/docs/superpowers/specs/2026-07-08-e2e-docker-design.md @@ -112,7 +112,7 @@ tests/e2e/ ### 6.1 TLS / 证书 - 容器内 server 拿不到 Let's Encrypt 证书。 -- 引入 `MESH_TEST_TLS=off` 开关(推荐在 `internal/config` 落地): +- 引入 `MESH_TEST_TLS=on` 开关(推荐在 `internal/config` 落地): - server 跳过 `acme/autocert`,改用自签证书。 - client `mesh join` 端允许自签(`InsecureSkipVerify`,已在 `internal/client/peers.go` 使用过)。 - 退出测试时无需清理证书目录,容器销毁即可。 @@ -302,7 +302,7 @@ tests/e2e/ 完整 e2e(容器 + TUN + tc + 性能) | `tc` 在 macOS host 不可用 | e2e 全在 Linux 容器内 | | Docker privileged 模式安全风险 | 仅 CI/本地开发,文档明示 | | 性能数据受 host 负载影响 | 软门槛 + 历史趋势对比 | -| server 拿不到 ACME 证书 | 引入 `MESH_TEST_TLS=off` 开关 | +| server 拿不到 ACME 证书 | 引入 `MESH_TEST_TLS=on` 开关 | | `mesh join` 在 TUN 起来前需要 DNS | 容器内 `/etc/hosts` 注入 server 别名 | | 长时间 iperf 占用 CI 资源 | 默认 30s 短流,CI 用 `--quick` 模式 | diff --git a/docs/todo/README.md b/docs/todo/README.md index 1e9d126..23a5d99 100644 --- a/docs/todo/README.md +++ b/docs/todo/README.md @@ -8,6 +8,7 @@ |------|------|------| | P | performance | 当前中转架构下的程序级性能、延迟、吞吐和可观测性优化 | | A | architecture | 参考 Tailscale / ZeroTier / Cloudflare One 的 Mesh 架构演进方向 | +| B | bug | e2e/codereview/线上发现的 mesh 缺陷修复(功能坏了,必须修) | ## 状态标记 @@ -29,6 +30,7 @@ |------|--------|------|--------| | performance | 1 | 10 | 10% | | architecture | 0 | 12 | 0% | +| bug | 2 | 2 | 100% | ## 目录结构 diff --git a/docs/todo/bug/B00.md b/docs/todo/bug/B00.md new file mode 100644 index 0000000..9d5b851 --- /dev/null +++ b/docs/todo/bug/B00.md @@ -0,0 +1,89 @@ +# B00. Linux TUN IFF_VNET_HDR offload 导致 TCP checksum 损坏 — 详细分析 + +> 状态: `[x]` | 优先级: P0 | 预估工作量: 1-2 天 | 关联: e2e T05, T04 offset 修复 + +## 实施记录 + +- 分支:`e2e/impl`,commit `4b4a943` +- 方案:D(自开 `/dev/net/tun` 用 `IFF_TUN|IFF_NO_PI` 无 `IFF_VNET_HDR` + `CreateTUNFromFile`) +- 改动: + - `internal/tun/tun_linux.go`(新建 `createTUNNative` 自开 fd) + - `internal/tun/tun_other.go`(新建,非 linux 回退 `wgtun.CreateTUN`) + - `internal/tun/tun.go`(`CreateTUN` 调用分发;`Offset()` Linux 回 0,撤 T04 offset=10 workaround;darwin 仍 4) +- 关键设计:库 `initFromFlags` 检测无 `IFF_VNET_HDR` 时 `vnetHdr=false` → kernel 不 offload → TCP checksum 软件算好 → mesh 转发天然正确 +- 验证:e2e T05 `tcp_1stream_mbps` **0.0016 → 877.64 Mbps**,`udp_100m_loss_pct` 100%→0%,`wan_tcp_mbps` 0→31.40。连带修复 UDP(`IFF_VNET_HDR` offload 也影响 USO) +- review:✅ Approved(库源码逐一核对 `vnetHdr=false` 下 Read/Write offset=0 安全,macOS 不回归) + +## 1. 背景与动机 + +e2e T05 性能场景(`tests/e2e/scenarios/02-performance.sh`)在 Linux 容器实测发现:mesh 在 Linux 上 **所有 TCP over mesh 流量完全不可用**(~0 Mbps),而 ping(ICMP)和 UDP 完全正常。 + +抓包定位(implementer 报告): +- iperf3 TCP 能完成握手但传 0 字节数据,`cwnd` 卡在 1.22KB,持续重传。 +- client-b 的 mesh0 TUN 发出的所有 TCP 包 `checksum incorrect`(如 `0x14fb (incorrect -> 0xc3f9)`)。 +- client-a kernel 因校验和不通过丢弃所有入站 TCP 数据段,cwnd 永不增长。 +- UDP/ICMP 因校验更宽松(UDP checksum=0 可接受)而幸免。 + +**影响**:Linux 上 mesh 的 TCP 完全不可用。这是生产 blocker——README 声称支持 Linux。 + +## 2. 当前项目状态与根因 + +- `internal/tun/tun.go` 用 `golang.zx2c4.com/wireguard/tun` 的 `CreateTUN`。 +- 库在 Linux 以 `IFF_TUN | IFF_NO_PI | IFF_VNET_HDR` 打开 TUN(`tun_linux.go:566`),启用 virtio offload。 +- kernel 发 TCP 包到 TUN 时**不软件计算 checksum**(预期硬件/用户态 offload 填),TCP checksum 是 partial/incorrect。 +- mesh 读 TUN 后(跳过 offset 字节拿 IP 包)直接加密转发,**从不修正 checksum**。 +- 对端 mesh 写 TUN 时填 virtio header,但 IP 包 checksum 仍 incorrect,对端 kernel 校验失败丢弃 TCP 数据段。 + +与 T04(offset bug)同源:都是 `IFF_VNET_HDR` 带来的 virtio offload 复杂性。T04 修了 offset(让写 TUN 不报 `invalid offset`),但 checksum 问题仍在。 + +## 3. 实施方案(待 brainstorm 选定) + +### 方案 A:通过库禁用 VNET_HDR —— 不可行 + +调研确认:`CreateTUN` 硬编码 `IFF_VNET_HDR`,`vnetHdr` 是内部字段(`tun_linux.go:40`),库**不暴露禁用 API**。排除。 + +### 方案 B:mesh 读 TUN 后重算 checksum + GSO 分段 —— 复杂 + +mesh 读到 IP 包后解析 IP/TCP/UDP header,重算 checksum 填入;若包是 GSO 大包(virtio header 标记 gsoSize),还需分段。 +- 等于重实现 wireguard/tun 库内部的 offload 处理(`offload_linux.go`)。 +- 工作量大,易错。 + +### 方案 D(推荐倾向):mesh 自开 /dev/net/tun(无 VNET_HDR)+ 库 CreateTUNFromFile + +mesh 自己用 `golang.org/x/sys/unix` 打开 `/dev/net/tun`,`IFF_TUN | IFF_NO_PI`(**不带 IFF_VNET_HDR**),拿到 fd 后传给库的 `CreateTUNFromFile(fd, mtu)`。 +- kernel 不启用 offload,TCP 包软件算好 checksum,mesh 转发的包天然正确。 +- `Offset()` Linux **恢复 0**(无 virtio header)——这同时正确解决 T04 的 offset(撤销 offset=10 的 workaround,从源头不需要 offset)。 +- 复用库的 Read/Write/Events 实现,只换 fd 创建,比方案 B 轻量。 + +**需验证**: +1. `CreateTUNFromFile` 是否不复核/重设 flags(不复核则 mesh 自开 fd 的 flags 生效)。 +2. `CreateTUNFromFile` 后库内部 `vnetHdr` 字段是否默认 false(false 则 Read/Write 不走 virtio/handleGRO 路径,offset 需求回到 0)。 +3. offset 恢复 0 后,T04 验证的"写 TUN 不 invalid offset"是否仍成立(应成立:vnetHdr=false 不走 handleGRO 校验)。 + +### 方案 C:mesh 读 TUN 后只重算 checksum(不处理 GSO)—— 风险 + +mesh MTU=1300,若 kernel TSO 基于 MTU,GSO 大包可能不多,只重算 checksum或够用。但 GSO 大包会超 MTU 写失败,不稳定。不推荐。 + +## 4. 测试计划 + +- 复用 e2e T05 性能场景:修复后 `tcp_1stream_mbps` 应 >0(baseline bridge 网络通常 >100Mbps)。 +- e2e T04 连通性回归(offset 改动后 ICMP 仍通)。 +- 单测:checksum 修正逻辑(若方案 B/C)或 fd 创建(若方案 D)。 +- macOS 路径回归(darwin offset 4 不变)。 + +## 5. 关联 TODO + +- e2e T05:发现本 bug 的场景,修复后重测拿真实 TCP 数值。 +- e2e T04 offset 修复(commit 56a0a19):方案 D 会调整 offset(Linux 10→0)。 +- e2e T06:TCP 长流场景受本 bug 影响,暂用 UDP 规避。 + +## 6. 上线策略 + +- 单独分支 `fix/b00-tun-checksum`,先 codereview 再合并。 +- 合并后重跑 e2e 全量(T04-T06)确认 TCP 恢复 + 无回归。 + +## 7. 参考资料 + +- 库源码:`golang.zx2c4.com/wireguard/tun/tun_linux.go`(CreateTUN:551, IFF_VNET_HDR:566, vnetHdr:40/536)、`offload_linux.go`(handleGRO:865, virtioNetHdrLen:57) +- e2e 发现:`tests/e2e/scenarios/02-performance.sh`(T05) +- T04 offset 修复:`internal/tun/tun.go`(commit 56a0a19) diff --git a/docs/todo/bug/B01.md b/docs/todo/bug/B01.md new file mode 100644 index 0000000..859109f --- /dev/null +++ b/docs/todo/bug/B01.md @@ -0,0 +1,84 @@ +# B01. TunnelClient main loop 卡 tun.Read,ctx cancel/SIGTERM 无法打断 — 详细分析 + +> 状态: `[x]` | 优先级: P1 | 预估工作量: 0.5-1 天 | 关联: e2e T06, P02 + +## 实施记录 + +- 分支:`e2e/impl`,commit `a3868b8` +- 方案:偏离确认的方案 A(close tun),改用 **channel 解耦方案** + - 方案 A 缺陷:`NativeTun.Close()` 后 fd 永久失效,server 故障重连时新 connect 的 `tun.Read` 在已关 fd 上立即返回 `os.ErrClosed` → 无限失败死循环(核对库 `closeOnce` + `tunFile.Close` 不可逆) + - channel 方案:Run 启动持久 `tunReadLoop` goroutine 执行 `tun.Read`(根 ctx,跨重连复用同一 TUN fd),通过 `pktCh chan []byte`(buffer 128)投递给 connect 的 main loop;main loop 改 `select{pktCh; ctx.Done}` +- 改动:`internal/tunnel/client.go` +- 关键设计:TUN fd 生命周期独立于易断的 WS 连接;无流量时 main loop 立即响应 ctx cancel;读写独立 mutex 无 data race +- 验证:无背景流量 SIGTERM **>10s/80s → 116ms**;e2e T06 03-failure 4/4 PASS +- review:✅ Approved(方案 A 死循环论证成立,channel 方案 5 项全过,偏离是正确必要决策) +- 注:T06 场景的主动 ping workaround 现 B01 已修可去掉,留作 e2e 纯净化 follow-up + +## 1. 背景与动机 + +e2e T06 故障场景(`tests/e2e/scenarios/03-failure.sh`)发现:mesh client 在**无 TUN 入流量**时: + +1. **SIGTERM 无法优雅退出**:PID 1(`mesh up`)收到 SIGTERM → ctx cancel,但 main loop 卡在 `tun.Read`,进程不退出(实测 80s+ 仍 running,远超 2s 预期)。 +2. **server 故障后不自动重连**:形成死锁——client-b 不重连 → server 无 b 路由 → a→b 包被丢 → b 无回包 → b 不触发检测。 + +implementer 用"场景脚本主动后台 ping 产生 TUN 流量"workaround 让 main loop 从 `tun.Read` 返回,从而能响应 ctx cancel。这模拟了真实 VPN 场景(总有 DNS/keepalive 背景流量),但"优雅退出/重连依赖外部流量"是脆弱设计。 + +## 2. 当前项目状态与根因 + +`internal/tunnel/client.go` 的 `connect` 主循环(TUN→WS 路径,约 line 152-179): + +```go +for { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + n, err := tc.tun.Read(bufs, sizes, meshtun.Offset()) + ... +} +``` + +问题:`select` 的 `default` 分支在 ctx 未触发时立即走 `tun.Read`。`tun.Read` 在无包时**阻塞**,阻塞期间不会再执行 `select` 检查 `ctx.Done`。所以 ctx cancel 后,必须等 `tun.Read` 返回(有包读或 fd 关闭)才能下一轮 select 检测到 cancel。 + +WebSocket ping loop(5s ticker + conn.Ping)能检测死连接并 cancel ctx,但 cancel 后 main loop 仍卡 `tun.Read`(无 TUN 流量)→ 不 return → `Run` 不进入 3s 重连。 + +## 3. 实施方案(待定) + +### 方案 A(推荐):ctx cancel 时关闭 TUN fd 触发 Read 返回 + +`TunnelClient` 监听 ctx.Done,触发时 `tc.tun.Close()`,让阻塞的 `tun.Read` 返回 error,main loop 退出。 +- 改动小:在 `connect` 起一个 goroutine `go func(){ <-ctx.Done(); tc.tun.Close() }()`。 +- 风险:Close 后 Read 返回 error,main loop return,`Run` 的 defer `tc.Close()` 再次 Close(需幂等,wireguard/tun 的 Close 应可重入)。 + +### 方案 B:用 tun 事件 channel + select + +wireguard/tun 的 Device 有 `Events() chan Event`(接口)。main loop 改 `select { case <-ctx.Done(): return; case <-tunEvents: ...; case pkt := <-readCh: ... }`,把 tun.Read 放独立 goroutine 喂 channel。 +- 更优雅,但要重构 read loop 为 goroutine + channel。 + +### 方案 C:tun.Read 设超时 + +wireguard/tun 的 Read 无原生超时。可用 SetReadDeadline(若支持)或外层超时。库支持情况待查。 + +## 4. 测试计划 + +- e2e T06 03.5:去掉 workaround(不主动 ping),SIGTERM 后应 ≤2s 退出。 +- e2e T06 03.2:去掉 workaround,server 故障后 client 应自动重连(不依赖外部流量)。 +- 单测:ctx cancel 触发 tun.Close,Read 返回。 +- 回归:正常转发不受影响。 + +## 5. 关联 TODO + +- e2e T06:发现本 bug,修复后可去掉 workaround(让场景更纯净)。 +- P02(异步队列):B01 修复后,writeLoop/model 更清晰,可一并审视。 + +## 6. 上线策略 + +- 与 B00 同批或独立分支修复。 +- 修复后 e2e T06 去 workaround 重测。 + +## 7. 参考资料 + +- 当前实现:`internal/tunnel/client.go`(connect 主循环、ping loop、Run 重连) +- e2e 发现:`tests/e2e/scenarios/03-failure.sh`(T06,workaround 注释) +- wireguard/tun Device 接口(Events/Close) diff --git a/docs/todo/bug/bug.md b/docs/todo/bug/bug.md new file mode 100644 index 0000000..fe50dbc --- /dev/null +++ b/docs/todo/bug/bug.md @@ -0,0 +1,29 @@ +# bug TODO 索引 + +> e2e 测试、codereview 或线上反馈发现的 mesh 缺陷修复。与 performance(优化)和 architecture(演进)区分:bug 是"功能坏了",必须修。 + +每个条目都有独立详细分析文档,路径见下表。 + +| ID | 标题 | 优先级 | 状态 | 详细分析 | +|----|------|--------|------|----------| +| B00 | Linux TUN IFF_VNET_HDR offload 导致 TCP checksum 损坏(TCP over mesh 全断) | P0 | [x] | [B00.md](B00.md) | +| B01 | TunnelClient main loop 卡 tun.Read,无背景流量时 SIGTERM 不退出/不自动重连 | P1 | [x] | [B01.md](B01.md) | + +--- + +## 各条目摘要 + +### B00. Linux TUN IFF_VNET_HDR offload 导致 TCP checksum 损坏 + +mesh 在 Linux 上所有 TCP over mesh 流量完全不可用(~0 Mbps),UDP/ICMP 正常。根因:`golang.zx2c4.com/wireguard/tun` 的 `CreateTUN` 强制 `IFF_VNET_HDR`,kernel 对 TCP 包不软件算 checksum(留给 offload),mesh 读 TUN 后直接转发不修正,对端 kernel 因 checksum incorrect 丢弃所有 TCP 数据段。e2e T05 性能场景抓包定位。修复涉及架构决策(与 T04 offset 同源),待 brainstorm 方案。 + +### B01. TunnelClient main loop 卡 tun.Read,SIGTERM/重连依赖外部流量 + +`connect` 主循环 `select{ctx.Done; default} + tun.Read`,一旦阻塞在 `tun.Read`(无 TUN 入流量)就不响应 ctx cancel。后果:无背景流量时 SIGTERM 不退出(实测 80s+)、server 故障后不自动重连(死锁)。e2e T06 发现,workaround 是场景脚本主动 ping 产生流量。修复方向:ctx cancel 时 close TUN fd 触发 Read 返回。 + +--- + +## 推荐执行顺序 + +1. ~~B00 — TCP checksum(P0)~~ ✅ 已修复(commit 4b4a943,TCP 877Mbps) +2. ~~B01 — main loop ctx cancel 死锁(P1)~~ ✅ 已修复(commit a3868b8,SIGTERM 116ms) diff --git a/internal/api/server.go b/internal/api/server.go index 9fb4248..6479a63 100644 --- a/internal/api/server.go +++ b/internal/api/server.go @@ -2,10 +2,18 @@ package api import ( "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" "database/sql" "fmt" + "math/big" + "net" "net/http" + "time" "golang.org/x/crypto/acme/autocert" @@ -51,31 +59,85 @@ func (s *Server) handleTunnel(w http.ResponseWriter, r *http.Request) { s.tunnel.HandleWebSocket(w, r) } -// ListenAndServeTLS starts the HTTPS server using autocert (Let's Encrypt). -// It also starts a :80 listener to serve the HTTP-01 ACME challenge. +// ListenAndServeTLS starts the HTTPS server. +// +// 生产环境使用 autocert(Let's Encrypt)+ :80 ACME challenge listener。 +// 当 cfg.TLSTestMode 为真(即 e2e 测试设置了 MESH_TEST_TLS=on)时, +// 改用内存自签证书,完全不依赖 Let's Encrypt,也不监听 :80。 func (s *Server) ListenAndServeTLS(ctx context.Context) error { - m := &autocert.Manager{ - Cache: autocert.DirCache(s.cfg.CertDir), - Prompt: autocert.AcceptTOS, - HostPolicy: autocert.HostWhitelist(s.cfg.Domain), - } - srv := &http.Server{ Addr: s.cfg.ListenAddr, Handler: s.Handler(), - TLSConfig: &tls.Config{ - GetCertificate: m.GetCertificate, - MinVersion: tls.VersionTLS12, - }, } - // HTTP-01 ACME challenge listener on :80 - go http.ListenAndServe(":80", m.HTTPHandler(nil)) //nolint:errcheck - go func() { <-ctx.Done() srv.Close() //nolint:errcheck }() + if s.cfg.TLSTestMode { + tlsCfg, err := s.selfSignedTLSConfig() + if err != nil { + return fmt.Errorf("self-signed cert: %w", err) + } + srv.TLSConfig = tlsCfg + return srv.ListenAndServeTLS("", "") + } + + // 生产路径:autocert + Let's Encrypt + m := &autocert.Manager{ + Cache: autocert.DirCache(s.cfg.CertDir), + Prompt: autocert.AcceptTOS, + HostPolicy: autocert.HostWhitelist(s.cfg.Domain), + } + srv.TLSConfig = &tls.Config{ + GetCertificate: m.GetCertificate, + MinVersion: tls.VersionTLS12, + } + + // HTTP-01 ACME challenge listener on :80 + go http.ListenAndServe(":80", m.HTTPHandler(nil)) //nolint:errcheck + return srv.ListenAndServeTLS("", "") } + +// selfSignedTLSConfig 生成一份内存中的 ECDSA P-256 自签证书, +// 仅供 e2e 测试使用。NotAfter 24h,IsCA=true, +// DNSNames 含 domain/localhost/server,IPAddresses 含 127.0.0.1。 +func (s *Server) selfSignedTLSConfig() (*tls.Config, error) { + priv, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + return nil, fmt.Errorf("generate key: %w", err) + } + + serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128)) + if err != nil { + return nil, fmt.Errorf("serial: %w", err) + } + + now := time.Now() + tmpl := x509.Certificate{ + SerialNumber: serial, + Subject: pkix.Name{ + CommonName: s.cfg.Domain, + }, + NotBefore: now.Add(-1 * time.Hour), + NotAfter: now.Add(24 * time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + BasicConstraintsValid: true, + IsCA: true, + DNSNames: []string{s.cfg.Domain, "localhost", "server"}, + IPAddresses: []net.IP{net.ParseIP("127.0.0.1")}, + } + + der, err := x509.CreateCertificate(rand.Reader, &tmpl, &tmpl, &priv.PublicKey, priv) + if err != nil { + return nil, fmt.Errorf("create cert: %w", err) + } + + return &tls.Config{ + Certificates: []tls.Certificate{{Certificate: [][]byte{der}, PrivateKey: priv}}, + MinVersion: tls.VersionTLS12, + }, nil +} diff --git a/internal/client/config.go b/internal/client/config.go index 0aee062..c7f7e56 100644 --- a/internal/client/config.go +++ b/internal/client/config.go @@ -14,6 +14,7 @@ type ClientConfig struct { DeviceIP string `json:"device_ip"` DeviceID string `json:"device_id"` NetworkCIDR string `json:"network_cidr"` + InsecureTLS bool `json:"insecure_tls,omitempty"` } // ConfigDir 返回客户端配置目录(~/.mesh)。 diff --git a/internal/client/join.go b/internal/client/join.go index 77ae87f..a9869fa 100644 --- a/internal/client/join.go +++ b/internal/client/join.go @@ -2,15 +2,19 @@ package client import ( "bytes" + "crypto/tls" "encoding/json" "fmt" "net/http" "os" + "time" ) // Join 向服务器注册当前设备,并将返回的配置保存到本地。 // 该操作不需要 root 权限。 -func Join(domain, tok string) error { +// +// insecureTLS 为真时跳过 TLS 证书校验,仅在 e2e 测试(server 使用自签证书)时使用。 +func Join(domain, tok string, insecureTLS bool) error { if err := os.MkdirAll(ConfigDir(), 0700); err != nil { return err } @@ -25,7 +29,14 @@ func Join(domain, tok string) error { }) url := fmt.Sprintf("https://%s/api/devices/register", domain) - resp, err := http.Post(url, "application/json", bytes.NewReader(reqBody)) + httpClient := &http.Client{Timeout: 10 * time.Second} + if insecureTLS { + httpClient.Transport = &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + } + } + + resp, err := httpClient.Post(url, "application/json", bytes.NewReader(reqBody)) if err != nil { return fmt.Errorf("connect: %w", err) } @@ -56,6 +67,7 @@ func Join(domain, tok string) error { DeviceIP: regResp.AssignedIP, DeviceID: regResp.DeviceID, NetworkCIDR: regResp.NetworkCIDR, + InsecureTLS: insecureTLS, } if err := SaveClientConfig(cfg); err != nil { return err diff --git a/internal/client/join_test.go b/internal/client/join_test.go new file mode 100644 index 0000000..c2b16d6 --- /dev/null +++ b/internal/client/join_test.go @@ -0,0 +1,57 @@ +package client + +import ( + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +// registerHandler 返回一个固定的注册成功响应。 +func registerHandler(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, `{"assigned_ip":"10.100.0.5","device_secret":"s","device_id":"d1","network_cidr":"10.100.0.0/24"}`) +} + +// TestJoinInsecureTLS 覆盖 Join 的 insecure 分支: +// - insecure=true 能连上自签 TLS server(间接证明设置了 InsecureSkipVerify), +// 并把 InsecureTLS=true 持久化到 config。 +// - insecure=false 面对自签证书应当连接失败且不保存 config(证明未跳过校验, +// 生产路径安全)。 +// +// 用 t.Setenv("HOME", ...) 隔离 ConfigDir,避免污染本机 ~/.mesh/config.json。 +func TestJoinInsecureTLS(t *testing.T) { + // httptest.NewTLSServer 使用自签证书,默认客户端校验会失败。 + mux := http.NewServeMux() + mux.HandleFunc("POST /api/devices/register", registerHandler) + srv := httptest.NewTLSServer(mux) + defer srv.Close() + + domain := strings.TrimPrefix(srv.URL, "https://") + + t.Run("insecure_true_connects_and_persists", func(t *testing.T) { + t.Setenv("HOME", t.TempDir()) + if err := Join(domain, "tok", true); err != nil { + t.Fatalf("Join insecure=true failed: %v", err) + } + cfg, err := LoadClientConfig() + if err != nil { + t.Fatalf("LoadClientConfig: %v", err) + } + if !cfg.InsecureTLS { + t.Fatal("expected InsecureTLS=true persisted in config") + } + }) + + t.Run("insecure_false_rejects_self_signed", func(t *testing.T) { + t.Setenv("HOME", t.TempDir()) + // 自签证书 + insecure=false:连接应因证书校验失败而失败,且不保存 config。 + if err := Join(domain, "tok", false); err == nil { + t.Fatal("expected Join to fail against self-signed server when insecure=false") + } + if _, err := LoadClientConfig(); err == nil { + t.Fatal("expected no config saved when Join fails on cert verification") + } + }) +} diff --git a/internal/client/peers.go b/internal/client/peers.go index 4d3c74d..b74ddd4 100644 --- a/internal/client/peers.go +++ b/internal/client/peers.go @@ -23,9 +23,15 @@ func Peers() error { return fmt.Errorf("not registered; run 'mesh join' first") } + // 仅在 e2e 测试模式(join 时带 --insecure)下跳过证书校验, + // 生产环境必须正常校验 Let's Encrypt 证书。 + tlsCfg := &tls.Config{} + if cfg.InsecureTLS { + tlsCfg.InsecureSkipVerify = true + } client := &http.Client{ Timeout: 5 * time.Second, - Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}, + Transport: &http.Transport{TLSClientConfig: tlsCfg}, } resp, err := client.Get(fmt.Sprintf("https://%s/api/devices", cfg.ServerDomain)) if err != nil { diff --git a/internal/client/up.go b/internal/client/up.go index d5c3d58..55a64a8 100644 --- a/internal/client/up.go +++ b/internal/client/up.go @@ -2,6 +2,7 @@ package client import ( "context" + "crypto/tls" "fmt" "github.com/maxyu/mesh/internal/tunnel" @@ -16,7 +17,13 @@ func Up(ctx context.Context) error { } wsURL := fmt.Sprintf("wss://%s/tunnel", cfg.ServerDomain) - tc, err := tunnel.NewTunnelClient(wsURL, cfg.DeviceSecret, cfg.DeviceIP, cfg.NetworkCIDR, 1300, ConfigDir()) + + var tlsCfg *tls.Config + if cfg.InsecureTLS { + tlsCfg = &tls.Config{InsecureSkipVerify: true} + } + + tc, err := tunnel.NewTunnelClient(wsURL, cfg.DeviceSecret, cfg.DeviceIP, cfg.NetworkCIDR, 1300, ConfigDir(), tlsCfg) if err != nil { return fmt.Errorf("setup tunnel: %w", err) } diff --git a/internal/config/config.go b/internal/config/config.go index 4f98fae..c42deee 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -3,6 +3,7 @@ package config import ( "fmt" "os" + "strings" "gopkg.in/yaml.v3" ) @@ -15,10 +16,24 @@ type Config struct { CertDir string `yaml:"cert_dir"` TunName string `yaml:"tun_name"` TunMTU int `yaml:"tun_mtu"` + + // TLSTestMode 在 e2e 测试场景启用自签证书,由 MESH_TEST_TLS 环境变量控制。 + // yaml tag "-" 表示该字段不入配置文件,仅供测试开关使用。 + TLSTestMode bool `yaml:"-"` +} + +// applyTestMode 根据环境变量 MESH_TEST_TLS 决定是否启用测试模式(自签证书)。 +// 生产环境不设此变量,行为不受影响。 +func (c *Config) applyTestMode() { + v := strings.ToLower(strings.TrimSpace(os.Getenv("MESH_TEST_TLS"))) + switch v { + case "1", "true", "on", "self": + c.TLSTestMode = true + } } func Default() *Config { - return &Config{ + cfg := &Config{ Domain: "localhost", ListenAddr: ":443", Network: "10.100.0.0/24", @@ -27,6 +42,10 @@ func Default() *Config { TunName: "mesh0", TunMTU: 1300, } + // 在这里调用,确保所有获取 Config 的路径(Default 直用、Load 成功、Load 失败回退) + // 都能读取 MESH_TEST_TLS 环境变量,e2e 测试模式下不会误走 Let's Encrypt。 + cfg.applyTestMode() + return cfg } func Load(path string) (*Config, error) { diff --git a/internal/config/config_test.go b/internal/config/config_test.go index e08bc58..f69211c 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -25,3 +25,31 @@ func TestLoad(t *testing.T) { t.Fatalf("unexpected: %+v", cfg) } } + +func TestLoadTLSTestModeFromEnv(t *testing.T) { + cases := []struct { + name string + env string + want bool + }{ + {"off", "off", false}, + {"empty", "", false}, + {"true", "true", true}, + {"self", "self", true}, + {"no_invalid", "no", false}, + {"uppercase_off", "OFF", false}, + {"trim_spaces", " off ", false}, + {"on", "on", true}, + {"1", "1", true}, + {"garbage", "garbage", false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Setenv("MESH_TEST_TLS", tc.env) + cfg := Default() + if cfg.TLSTestMode != tc.want { + t.Fatalf("MESH_TEST_TLS=%q: expected TLSTestMode=%v, got %v", tc.env, tc.want, cfg.TLSTestMode) + } + }) + } +} diff --git a/internal/tun/tun.go b/internal/tun/tun.go index a382825..bf42dd5 100644 --- a/internal/tun/tun.go +++ b/internal/tun/tun.go @@ -1,7 +1,6 @@ package tun import ( - "fmt" "runtime" wgtun "golang.zx2c4.com/wireguard/tun" @@ -10,25 +9,24 @@ import ( type Device = wgtun.Device // Offset 返回平台所需的 TUN 读写偏移量。 -// macOS utun 需要 4 字节 packet info header,Linux 不需要。 +// +// darwin: utun 每帧前缀 4 字节 AF family packet info,offset = 4。 +// linux: B00 方案 D 后 mesh 自开 /dev/net/tun 不带 IFF_VNET_HDR(见 +// +// createTUNNative),wireguard/tun 库检测到无 VNET_HDR 时 vnetHdr=false, +// 无 virtio net header,offset = 0(撤销 T04 的 offset=10 workaround)。 func Offset() int { - if runtime.GOOS == "darwin" { + switch runtime.GOOS { + case "darwin": return 4 + default: // linux 及其它:无 virtio header(B00 方案 D 不带 IFF_VNET_HDR) + return 0 } - return 0 } +// CreateTUN 创建 TUN 设备,平台分发由 createTUNNative 的 build tag 处理。 func CreateTUN(name string, mtu int) (Device, string, error) { - dev, err := wgtun.CreateTUN(name, mtu) - if err != nil { - return nil, "", fmt.Errorf("create TUN %s: %w", name, err) - } - actualName, err := dev.Name() - if err != nil { - dev.Close() - return nil, "", fmt.Errorf("get TUN name: %w", err) - } - return dev, actualName, nil + return createTUNNative(name, mtu) } func DefaultTUNName() string { diff --git a/internal/tun/tun_linux.go b/internal/tun/tun_linux.go index 886ea23..189d964 100644 --- a/internal/tun/tun_linux.go +++ b/internal/tun/tun_linux.go @@ -5,10 +5,57 @@ package tun import ( "fmt" "net" + "os" "os/exec" "strings" + + "golang.org/x/sys/unix" + wgtun "golang.zx2c4.com/wireguard/tun" ) +// createTUNNative opens /dev/net/tun WITHOUT IFF_VNET_HDR, then wraps the fd +// with the wireguard/tun library's CreateTUNFromFile. +// +// 不带 IFF_VNET_HDR 是关键:库的 initFromFlags 检测到无 VNET_HDR 时设 +// vnetHdr=false,kernel 不做 TSO/GSO/checksum offload,TCP 包由 kernel 软件 +// 算好 checksum,mesh 转发的包天然正确(修复 B00 TCP checksum 损坏)。 +// 代价:batchSize=1(无批量读),但 mesh 本就单包处理,无性能影响。 +func createTUNNative(name string, mtu int) (Device, string, error) { + fd, err := unix.Open("/dev/net/tun", unix.O_RDWR|unix.O_CLOEXEC, 0) + if err != nil { + return nil, "", fmt.Errorf("open /dev/net/tun: %w", err) + } + + ifr, err := unix.NewIfreq(name) + if err != nil { + unix.Close(fd) + return nil, "", fmt.Errorf("new ifreq %s: %w", name, err) + } + ifr.SetUint16(unix.IFF_TUN | unix.IFF_NO_PI) // 注意:无 IFF_VNET_HDR + if err := unix.IoctlIfreq(fd, unix.TUNSETIFF, ifr); err != nil { + unix.Close(fd) + return nil, "", fmt.Errorf("TUNSETIFF: %w", err) + } + + if err := unix.SetNonblock(fd, true); err != nil { + unix.Close(fd) + return nil, "", fmt.Errorf("set nonblock: %w", err) + } + + file := os.NewFile(uintptr(fd), "/dev/net/tun") + dev, err := wgtun.CreateTUNFromFile(file, mtu) + if err != nil { + file.Close() // CreateTUNFromFile 失败时确保 fd 不泄漏 + return nil, "", fmt.Errorf("create TUN %s: %w", name, err) + } + actualName, err := dev.Name() + if err != nil { + dev.Close() + return nil, "", fmt.Errorf("get TUN name: %w", err) + } + return dev, actualName, nil +} + func ConfigureInterface(ifaceName, localIP, network string) error { _, ipNet, err := net.ParseCIDR(network) if err != nil { diff --git a/internal/tun/tun_other.go b/internal/tun/tun_other.go new file mode 100644 index 0000000..ef43941 --- /dev/null +++ b/internal/tun/tun_other.go @@ -0,0 +1,24 @@ +//go:build !linux + +package tun + +import ( + "fmt" + + wgtun "golang.zx2c4.com/wireguard/tun" +) + +// createTUNNative 非 linux 平台(如 darwin)直接用 wireguard/tun 库的 +// CreateTUN。这些平台无 IFF_VNET_HDR/offload 问题,库的默认行为正确。 +func createTUNNative(name string, mtu int) (Device, string, error) { + dev, err := wgtun.CreateTUN(name, mtu) + if err != nil { + return nil, "", fmt.Errorf("create TUN %s: %w", name, err) + } + actualName, err := dev.Name() + if err != nil { + dev.Close() + return nil, "", fmt.Errorf("get TUN name: %w", err) + } + return dev, actualName, nil +} diff --git a/internal/tunnel/client.go b/internal/tunnel/client.go index 7e90b15..515b1b0 100644 --- a/internal/tunnel/client.go +++ b/internal/tunnel/client.go @@ -2,6 +2,7 @@ package tunnel import ( "context" + "crypto/tls" "encoding/json" "fmt" "log" @@ -18,11 +19,12 @@ import ( // TunnelClient connects to the mesh VPN server via WebSocket and shuttles // IP packets through a local TUN device. type TunnelClient struct { - serverURL string - secret string - mtu int - tun meshtun.Device - statusDir string + serverURL string + secret string + mtu int + tun meshtun.Device + statusDir string + httpClient *http.Client connected atomic.Int32 connectedAt atomic.Int64 // unix nano @@ -36,7 +38,9 @@ type TunnelClient struct { // NewTunnelClient creates a TunnelClient, initializes the TUN device, and // configures the network interface. -func NewTunnelClient(serverURL, secret, localIP, network string, mtu int, statusDir string) (*TunnelClient, error) { +// +// tlsConfig 可为 nil(生产环境走系统默认证书校验);e2e 测试传 InsecureSkipVerify 的配置。 +func NewTunnelClient(serverURL, secret, localIP, network string, mtu int, statusDir string, tlsConfig *tls.Config) (*TunnelClient, error) { tunName := meshtun.DefaultTUNName() dev, name, err := meshtun.CreateTUN(tunName, mtu) if err != nil { @@ -52,6 +56,8 @@ func NewTunnelClient(serverURL, secret, localIP, network string, mtu int, status mtu: mtu, tun: dev, statusDir: statusDir, + // 复用单个 http.Client,避免每次重连新建 Transport 造成空闲连接/fd 泄漏。 + httpClient: &http.Client{Transport: &http.Transport{TLSClientConfig: tlsConfig}}, }, nil } @@ -60,8 +66,20 @@ func NewTunnelClient(serverURL, secret, localIP, network string, mtu int, status // is cancelled. func (tc *TunnelClient) Run(ctx context.Context) error { go tc.writeStatusLoop(ctx) + + // B01: 持久的 TUN 读 goroutine,跨重连复用同一个 TUN fd。tun.Read 在 + // 这里执行并通过 pktCh 投递包,connect 的 main loop 用 select 监听 + // pktCh + ctx.Done,从而能在无背景流量时响应 ctx cancel(SIGTERM/连接 + // 丢失),不再阻塞在 tun.Read 上。 + // + // 不采用 "ctx cancel 时 close TUN" 方案:close 后 TUN fd 永久失效,Run + // 的进程内重连会因 tun.Read 立即报错而无限失败。channel 解耦让 TUN fd + // 生命周期独立于 WS 连接,重连时不需重建 TUN。 + pktCh := make(chan []byte, 128) + go tc.tunReadLoop(ctx, pktCh) + for { - err := tc.connect(ctx) + err := tc.connect(ctx, pktCh) if ctx.Err() != nil { return ctx.Err() } @@ -75,14 +93,47 @@ func (tc *TunnelClient) Run(ctx context.Context) error { } } +// tunReadLoop 持续从 TUN 读包并通过 pktCh 投递。在 Run 启动,跨重连复用 +// 同一个 TUN fd,直到根 ctx 取消(进程退出)或 TUN 关闭/读出错。 +// +// 解耦目的:把会阻塞的 tun.Read 从 connect 的 main loop 移出,使 main loop +// 能用 select 响应 ctx cancel。connect 退出(重连)时本 goroutine 继续运行、 +// 继续持有 TUN fd;重连后的新 connect 继续从 pktCh 读,TUN 无需重建。 +func (tc *TunnelClient) tunReadLoop(ctx context.Context, pktCh chan<- []byte) { + bufs := make([][]byte, 1) + bufs[0] = make([]byte, meshtun.Offset()+tc.mtu+100) + sizes := make([]int, 1) + for { + n, err := tc.tun.Read(bufs, sizes, meshtun.Offset()) + if err != nil { + // ctx 取消(正常关闭)静默退出;其余为 unexpected error,记录日志后退出。 + if ctx.Err() == nil { + log.Printf("tun read error: %v", err) + } + return + } + if n == 0 { + continue + } + pkt := make([]byte, sizes[0]) + copy(pkt, bufs[0][meshtun.Offset():meshtun.Offset()+sizes[0]]) + select { + case <-ctx.Done(): + return + case pktCh <- pkt: + } + } +} + // connect dials the WebSocket server and runs the bidirectional packet loop // until either the context is cancelled or an error occurs. -func (tc *TunnelClient) connect(ctx context.Context) error { +func (tc *TunnelClient) connect(ctx context.Context, pktCh <-chan []byte) error { header := http.Header{} header.Set("Authorization", "Bearer "+tc.secret) conn, _, err := websocket.Dial(ctx, tc.serverURL, &websocket.DialOptions{ HTTPHeader: header, + HTTPClient: tc.httpClient, }) if err != nil { return fmt.Errorf("dial: %w", err) @@ -137,38 +188,23 @@ func (tc *TunnelClient) connect(ctx context.Context) error { } }() - // TUN → WS: main loop reads packets from TUN and sends over WebSocket. - bufs := make([][]byte, 1) - bufs[0] = make([]byte, meshtun.Offset()+tc.mtu+100) - sizes := make([]int, 1) - + // TUN → WS: main loop reads packets (produced by the persistent + // tunReadLoop via pktCh) and sends over WebSocket. + // + // B01: tun.Read 已移到 tunReadLoop,main loop 只需 select pktCh + ctx.Done, + // 无背景流量时也能立即响应 ctx cancel(SIGTERM/连接丢失),不再阻塞。 for { select { case <-ctx.Done(): return ctx.Err() - default: - } - - n, err := tc.tun.Read(bufs, sizes, meshtun.Offset()) - if err != nil { - if ctx.Err() != nil { - return ctx.Err() + case pkt := <-pktCh: + if err := conn.Write(ctx, websocket.MessageBinary, pkt); err != nil { + return fmt.Errorf("write WS: %w", err) } - return fmt.Errorf("read TUN: %w", err) - } - if n == 0 { - continue - } - - pkt := make([]byte, sizes[0]) - copy(pkt, bufs[0][meshtun.Offset():meshtun.Offset()+sizes[0]]) - - if err := conn.Write(ctx, websocket.MessageBinary, pkt); err != nil { - return fmt.Errorf("write WS: %w", err) + tc.txPackets.Add(1) + tc.txBytes.Add(uint64(len(pkt))) + tc.lastActive.Store(time.Now().UnixNano()) } - tc.txPackets.Add(1) - tc.txBytes.Add(uint64(len(pkt))) - tc.lastActive.Store(time.Now().UnixNano()) } } diff --git a/tests/e2e/Dockerfile.client b/tests/e2e/Dockerfile.client new file mode 100644 index 0000000..e9deef1 --- /dev/null +++ b/tests/e2e/Dockerfile.client @@ -0,0 +1,49 @@ +# syntax=docker/dockerfile:1 +# T01: mesh 客户端镜像(多阶段构建) +# build context 为仓库根(compose 里 context: ../..) + +######################################## +# Stage 1: 构建 mesh 二进制 +######################################## +FROM golang:1.25-bookworm AS builder + +WORKDIR /src + +# 先拷依赖以利用 Docker 层缓存 +COPY go.mod go.sum ./ +RUN go mod download + +# 拷源码并编译 +COPY . . +RUN CGO_ENABLED=1 go build -trimpath -ldflags="-s -w" -o /out/mesh ./cmd/mesh + +######################################## +# Stage 2: 运行时 +######################################## +FROM ubuntu:24.04 + +# 装运行时依赖 + 性能测试工具(iperf3 / nuttcp / fping) +RUN apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates \ + iproute2 \ + iputils-ping \ + iperf3 \ + nuttcp \ + fping \ + curl \ + jq \ + bash \ + && rm -rf /var/lib/apt/lists/* + +# 拷二进制 +COPY --from=builder /out/mesh /usr/local/bin/mesh + +# 拷 entrypoint(路径相对仓库根,即 build context) +COPY tests/e2e/fixtures/entrypoint-client.sh /entrypoint.sh +RUN chmod +x /entrypoint.sh + +# 拷 netem 预设库(场景脚本 source /usr/local/bin/netem-preset.sh 后调用 netem ) +COPY tests/e2e/fixtures/netem.sh /usr/local/bin/netem-preset.sh +RUN chmod +x /usr/local/bin/netem-preset.sh + +ENTRYPOINT ["/entrypoint.sh"] diff --git a/tests/e2e/Dockerfile.server b/tests/e2e/Dockerfile.server new file mode 100644 index 0000000..f30cce4 --- /dev/null +++ b/tests/e2e/Dockerfile.server @@ -0,0 +1,44 @@ +# syntax=docker/dockerfile:1 +# T01: meshd 服务端镜像(多阶段构建) +# build context 为仓库根(compose 里 context: ../..) + +######################################## +# Stage 1: 构建 meshd 二进制 +######################################## +FROM golang:1.25-bookworm AS builder + +WORKDIR /src + +# 先拷依赖以利用 Docker 层缓存 +COPY go.mod go.sum ./ +RUN go mod download + +# 拷源码并编译(CGO 保留 modernc.org/sqlite 兼容性,bookworm 自带 gcc) +COPY . . +RUN CGO_ENABLED=1 go build -trimpath -ldflags="-s -w" -o /out/meshd ./cmd/meshd + +######################################## +# Stage 2: 运行时 +######################################## +FROM ubuntu:24.04 + +# 装运行时依赖:ca-certificates(TLS)、iproute2/iputils-ping/curl(排障)、 +# jq(解析 JSON)、bash +RUN apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates \ + iproute2 \ + iputils-ping \ + curl \ + jq \ + bash \ + && rm -rf /var/lib/apt/lists/* + +# 拷二进制 +COPY --from=builder /out/meshd /usr/local/bin/meshd + +# 拷配置、entrypoint(路径相对仓库根,即 build context) +COPY tests/e2e/fixtures/meshd.yaml /etc/mesh/meshd.yaml +COPY tests/e2e/fixtures/entrypoint-server.sh /entrypoint.sh +RUN chmod +x /entrypoint.sh + +ENTRYPOINT ["/entrypoint.sh"] diff --git a/tests/e2e/docker-compose.yml b/tests/e2e/docker-compose.yml new file mode 100644 index 0000000..785bc99 --- /dev/null +++ b/tests/e2e/docker-compose.yml @@ -0,0 +1,72 @@ +# T01: mesh VPN e2e 测试编排 +# 3 容器:server / client-a / client-b +# 全部 privileged + NET_ADMIN/NET_RAW + /dev/net/tun(server 也起 TUN 做中继) +# +# 用法: +# MESH_TOKEN= docker compose -f tests/e2e/docker-compose.yml up -d +# docker compose -f tests/e2e/docker-compose.yml down + +services: + server: + build: + context: ../.. + dockerfile: tests/e2e/Dockerfile.server + image: mesh-e2e/server:latest + container_name: mesh-server + privileged: true + cap_add: + - NET_ADMIN + - NET_RAW + devices: + - /dev/net/tun:/dev/net/tun + networks: + meshnet: + aliases: + - server + # 暴露 443 供 host 排障(可选) + ports: + - "8443:443" + + client-a: + build: + context: ../.. + dockerfile: tests/e2e/Dockerfile.client + image: mesh-e2e/client:latest + container_name: mesh-client-a + hostname: client-a + privileged: true + cap_add: + - NET_ADMIN + - NET_RAW + devices: + - /dev/net/tun:/dev/net/tun + depends_on: + - server + environment: + MESH_TOKEN: ${MESH_TOKEN:?MESH_TOKEN must be set} + networks: + - meshnet + + client-b: + build: + context: ../.. + dockerfile: tests/e2e/Dockerfile.client + image: mesh-e2e/client:latest + container_name: mesh-client-b + hostname: client-b + privileged: true + cap_add: + - NET_ADMIN + - NET_RAW + devices: + - /dev/net/tun:/dev/net/tun + depends_on: + - server + environment: + MESH_TOKEN: ${MESH_TOKEN:?MESH_TOKEN must be set} + networks: + - meshnet + +networks: + meshnet: + driver: bridge diff --git a/tests/e2e/fixtures/entrypoint-client.sh b/tests/e2e/fixtures/entrypoint-client.sh new file mode 100755 index 0000000..c810090 --- /dev/null +++ b/tests/e2e/fixtures/entrypoint-client.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# T01: mesh 客户端 entrypoint +# 等待 server 就绪 -> join -> up(前台) +# +# 幂等:容器重启(docker compose stop/start、restart)时 config.json 仍在, +# mesh join 会拒绝并报 "already registered"。检测到已注册则跳过 join 直接 up, +# 这样容器重启能自愈,e2e reconnect 场景也能通过。 +set -euo pipefail + +# 等待 server 起来(自签证书,curl 必须 -k) +echo "waiting for server..." +for i in $(seq 1 60); do + if curl -kfsS https://server:443/api/devices >/dev/null 2>&1; then + echo "server is up" + break + fi + sleep 1 +done +if ! curl -kfsS https://server:443/api/devices >/dev/null 2>&1; then + echo "ERROR: server did not become ready within 60s" >&2 + exit 1 +fi + +# 已注册则跳过 join(容器重启场景)。config 路径 = $HOME/.mesh/config.json +# (见 internal/client/config.go ConfigDir)。 +if [ -f "$HOME/.mesh/config.json" ]; then + echo "already registered, skip join" +else + : "${MESH_TOKEN:?MESH_TOKEN must be set}" + mesh join server --token "$MESH_TOKEN" --insecure +fi + +# 启动隧道(前台) +exec mesh up diff --git a/tests/e2e/fixtures/entrypoint-server.sh b/tests/e2e/fixtures/entrypoint-server.sh new file mode 100755 index 0000000..0a141c8 --- /dev/null +++ b/tests/e2e/fixtures/entrypoint-server.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# T01: meshd 服务端 entrypoint +# 自签证书模式(MESH_TEST_TLS=on),init 后提取 token 供 e2e 测试读取 +set -euo pipefail + +export MESH_TEST_TLS=on + +if [ -f /etc/mesh/mesh.db ]; then + echo "already initialized, skip init" +else + meshd init 2>&1 | tee /tmp/init.log + # meshd init 输出 "Initialized.\nToken: xxx",提取 token 写入文件供 e2e 读取 + # 用 sed 比 grep -oP 更可移植(不依赖 PCRE) + sed -n 's/^Token: //p' /tmp/init.log > /etc/mesh/token 2>/dev/null || true + if [ ! -s /etc/mesh/token ]; then + echo "ERROR: failed to extract token from init output" >&2 + exit 1 + fi +fi + +exec meshd run diff --git a/tests/e2e/fixtures/meshd.yaml b/tests/e2e/fixtures/meshd.yaml new file mode 100644 index 0000000..a2277f0 --- /dev/null +++ b/tests/e2e/fixtures/meshd.yaml @@ -0,0 +1,7 @@ +domain: "server" +listen_addr: ":443" +network: "10.100.0.0/24" +data_dir: "/etc/mesh" +cert_dir: "/etc/mesh/certs" +tun_name: "mesh0" +tun_mtu: 1300 diff --git a/tests/e2e/fixtures/netem.sh b/tests/e2e/fixtures/netem.sh new file mode 100644 index 0000000..c80eefd --- /dev/null +++ b/tests/e2e/fixtures/netem.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# tc netem 预设封装。在容器内对 eth0 设置丢包/延迟模拟。 +# +# 本文件为库,场景脚本(或容器内手工排障)source 后调用: +# source /usr/local/bin/netem-preset.sh +# netem wan +# +# 预设: +# clean 清除所有 qdisc(恢复默认) +# baseline 等价 clean(无干扰基线) +# wan 典型公网:80ms±10ms 延迟,1% 丢包 +# bad 恶劣网络:200ms±50ms 延迟,5% 丢包 +# satellite 卫星链路:600ms±100ms 延迟,2% 丢包 +# +# 依赖:iproute2(tc),容器需 NET_ADMIN + privileged(compose 已配置)。 +# 注意:root qdisc 不存在时 `tc qdisc del` 会报错,统一 `|| true` 忽略。 + +netem() { + local preset="${1:-clean}" + case "$preset" in + clean|baseline) + tc qdisc del dev eth0 root 2>/dev/null || true + ;; + wan) + tc qdisc del dev eth0 root 2>/dev/null || true + tc qdisc add dev eth0 root netem delay 80ms 10ms loss 1% + ;; + bad) + tc qdisc del dev eth0 root 2>/dev/null || true + tc qdisc add dev eth0 root netem delay 200ms 50ms loss 5% + ;; + satellite) + tc qdisc del dev eth0 root 2>/dev/null || true + tc qdisc add dev eth0 root netem delay 600ms 100ms loss 2% + ;; + *) + echo "unknown netem preset: $preset" >&2 + return 1 + ;; + esac +} diff --git a/tests/e2e/lib/helpers.sh b/tests/e2e/lib/helpers.sh new file mode 100644 index 0000000..f819cf2 --- /dev/null +++ b/tests/e2e/lib/helpers.sh @@ -0,0 +1,85 @@ +#!/usr/bin/env bash +# 通用辅助函数。所有场景脚本 source 本文件。 +# +# 提供: +# - dex : 在指定容器内执行命令 +# - wait_for_server : 等 server 容器内 meshd 可达(自签,curl -k) +# - wait_for_client : 等 client 容器内 mesh0 起来并有 IPv4 +# - get_token : 从 server 容器读 /etc/mesh/token(T01 entrypoint 已写入) +# - dump_logs : 收集所有容器最近日志到 results 目录 +# - assert : 断言命令退出码符合预期,写 ok/fail 计数 + +# 项目根目录(本文件位于 tests/e2e/lib/,往上两级到 e2e 目录,再往上两级到仓库根) +E2E_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +REPO_ROOT="$(cd "$E2E_DIR/../.." && pwd)" +COMPOSE="$E2E_DIR/docker-compose.yml" +RESULTS_DIR="${RESULTS_DIR:-$E2E_DIR/results}" + +# 断言计数文件(assert 函数使用)。调用方应在场景脚本开头初始化: +# ASSERT_OK="$RESULTS_DIR/.ok"; ASSERT_FAIL="$RESULTS_DIR/.fail" +# 未设置时给个默认,避免 set -u 报错。 +ASSERT_OK="${ASSERT_OK:-/tmp/mesh-e2e-ok.$$}" +ASSERT_FAIL="${ASSERT_FAIL:-/tmp/mesh-e2e-fail.$$}" + +# 在指定容器内执行命令 +dex() { docker exec "$@"; } + +# 等 server 容器内 meshd 可达(自签,curl -k) +# 用 server 的 compose 网络 alias "server" 访问,避免依赖 host 端口映射。 +wait_for_server() { + echo "waiting for server meshd..." + local i + for i in $(seq 1 60); do + if dex mesh-server curl -kfsS https://server:443/api/devices >/dev/null 2>&1; then + return 0 + fi + sleep 1 + done + echo "ERROR: server did not become ready" >&2 + return 1 +} + +# 等 client 容器内 mesh0 起来并有 IPv4 +# 用法: wait_for_client +wait_for_client() { + local c="$1" + local i + for i in $(seq 1 60); do + if dex "$c" ip -o addr show mesh0 2>/dev/null | grep -q inet; then + return 0 + fi + sleep 1 + done + echo "ERROR: client $c mesh0 did not come up" >&2 + return 1 +} + +# 从 server 容器读 /etc/mesh/token(T01 entrypoint 已写入) +get_token() { + dex mesh-server cat /etc/mesh/token +} + +# 收集所有容器最近日志到 results 目录 +# 用法: dump_logs +dump_logs() { + local out="$1" + mkdir -p "$out" + local c + for c in mesh-server mesh-client-a mesh-client-b; do + docker logs "$c" >"$out/$c.log" 2>&1 || true + done +} + +# 断言:命令退出码符合预期。写 ok/fail 计数到 $ASSERT_OK/$ASSERT_FAIL 文件 +# 用法: assert "