diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 497249b..b250f1c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -17,3 +17,5 @@ jobs: go-version: "1.27.x" - run: go test ./... - run: go vet ./... + - name: ShellCheck + run: shellcheck scripts/*.sh diff --git a/README.md b/README.md index 8b08c09..8eeb72b 100644 --- a/README.md +++ b/README.md @@ -22,33 +22,62 @@ With pinned telemetrygen v0.157, each generated trace exports one parent and one ## Quick start -Prerequisites: Go 1.27+, [just](https://github.com/casey/just), SSH, and a clean SSH-accessible Linux host. The host may be a cloud instance, dedicated machine, or bare metal. +Prerequisites: [hcloud](https://github.com/hetznercloud/cli) with an active context (or `HCLOUD_TOKEN`), a registered Hetzner SSH key available to your SSH client, Go 1.27+, `ssh-keygen`, [just](https://github.com/casey/just), and SSH/SCP. ```sh -just build +export FANOUT_BENCH_HCLOUD_SSH_KEY='benchmark-key' +just hetzner --fanout-version v2026.10.1-alpha --preset quick +``` + +The script runs `just build` before provisioning, creates a private network and two fresh Hetzner Cloud VMs with unique names, waits for SSH on both, runs the harness for each group, and downloads the evidence. Cleanup attempts to delete both servers and then the network on every exit, including failures and Ctrl-C. After three failed attempts it names each resource requiring manual deletion and exits nonzero. Hetzner bills every started hour for each VM, so grouping runs shares the billing time of both machines. + +The defaults are a `ccx33` for Fanout (8 dedicated vCPUs, 32 GB RAM) and a separate `ccx43` for generators (16 dedicated vCPUs, 64 GB RAM), both in `fsn1` on `ubuntu-24.04`. OTLP travels over a private network (`10.77.0.0/16`, cloud subnet `10.77.0.0/24`) in that location’s network zone. Override the VM types with `FANOUT_BENCH_HCLOUD_TYPE` and `FANOUT_BENCH_HCLOUD_GENERATOR_TYPE`, and location/image with `FANOUT_BENCH_HCLOUD_LOCATION` and `FANOUT_BENCH_HCLOUD_IMAGE`. The hourly VM cost is **ccx33 hourly rate + ccx43 hourly rate**, plus applicable public IPv4 charges and tax; budget for both machines for every started hour. Check the selected location’s current rates before provisioning. `FANOUT_BENCH_HCLOUD_SSH_KEY` also accepts comma-separated key names. Each invocation shares one temporary known-hosts file across its groups, leaving your SSH known-hosts file untouched. Pass `--ssh-identity ~/.ssh/benchmark` if the registered key is not available through your SSH agent. + +Separate complete sets of `fanout-bench run` flags with a standalone `--`. Groups must be nonempty and use matching SSH port and identity settings (defaults: port 22 and the SSH client's default identity). All groups run in order even if one fails, with a one-line summary per group at the end and a nonzero exit if any failed. They share both VMs, the private network, and both host descriptions; each harness run creates fresh stages and a fresh Fanout data directory. Both VMs and the network are deleted after the last group. + +A plain `--host` run on any clean SSH-accessible Linux host runs Fanout and telemetrygen together on that host: +```sh +just build ./bin/fanout-bench run \ - --host root@203.0.113.10 \ - --fanout-version v2026.8.11 \ - --preset standard \ - --query-rate 50 + --host 'root@' \ + --host-description 'Linux benchmark host' \ + --fanout-version v2026.10.1-alpha \ + --preset quick ``` -That one command: +Replace `` with the address of your host. The remote harness: 1. detects the host's Linux architecture over SSH; 2. downloads the matching Fanout release and verifies it against the release `SHA256SUMS`; 3. cross-builds pinned telemetrygen and the benchmark agent for the host; 4. uploads all three binaries to a private, isolated `/tmp/fanout-bench-*` directory; -5. starts a fresh Fanout data directory on loopback-only random ports; +5. starts a fresh Fanout data directory on one loopback-only random port; 6. creates one-time admin, ingest, and query credentials without printing them; 7. runs every phase and then `fanout repair verify`; 8. downloads `report.json`, `report.md`, and `samples.jsonl` locally; 9. stops Fanout and removes the remote run directory. -Pass `--ssh-identity ~/.ssh/bench` or `--ssh-port 2222` when needed. `--keep-remote` preserves the isolated remote directory for investigation; teardown is the default. +For an existing host, pass `--ssh-identity ~/.ssh/benchmark`, `--ssh-port 2222`, or `--ssh-known-hosts ` when needed. `--host-description ` records the provider or machine configuration beside the detected CPU model. Reports redact the SSH address and omit the run-scoped known-hosts path from reproduction commands. `--keep-remote` preserves the Fanout directory on an existing host for investigation; split runs still remove the generator stage and ephemeral SSH key. The Hetzner script always deletes both VMs and the network. -Nothing is installed system-wide. Existing Fanout services and data directories are not touched. The remote host needs no Go toolchain, Docker daemon, or preconfigured Fanout account. +To use two existing clean Linux hosts, add all three split flags together: + +```sh +./bin/fanout-bench run \ + --host 'root@' \ + --generator-host 'root@' \ + --generator-address 10.77.0.3 \ + --fanout-listen 10.77.0.2 \ + --host-description 'Fanout host' \ + --generator-host-description 'Generator host' \ + --fanout-version v2026.10.1-alpha --preset quick +``` + +The public SSH targets are used by your local machine for provisioning. The two IP flags must be reachable between the hosts, with SSH from Fanout to the generator and OTLP from the generator to Fanout. Provisioning detects both Linux architectures, builds the harness for Fanout and telemetrygen for the generator, and uploads each binary only where it runs. A local temporary ed25519 key grants the Fanout host access to the generator for that run; its public key has a unique authorized-keys marker, and the private key is uploaded with mode `0600` into the Fanout stage. Both stages use mode `0700`. + +`managed` remains the sole orchestrator on the Fanout host: it binds Fanout to the supplied private IP on a free port, performs setup, samples metrics, issues read load, verifies storage, and writes the report. Every telemetrygen process runs through SSH on the generator host, bounded by a remote timeout. Cleanup removes the marked authorized-key line, the private key, and both stages. Reports record both host descriptions and `generator_on_target_host: false`; reproduction commands redact both public targets and omit the private IP flags. + +Nothing is installed system-wide. Existing Fanout services and data directories are not touched. Neither remote host needs a Go toolchain or Docker daemon, and Fanout needs no preconfigured account. The Fanout host needs an SSH client; the generator host needs an SSH server and GNU `timeout` (included in the default Ubuntu image). > `telemetrygen` currently accepts OTLP headers through a command-line flag and prints those headers in its startup configuration. Fanout Bench suppresses and redacts child output, but the short-lived ingest token can still be visible to local process-listing tools while a generator is running. Use a dedicated token and revoke it after the run. @@ -62,40 +91,42 @@ Nothing is installed system-wide. Existing Fanout services and data directories The default topology is 8 workers per process at 2,500 items/s per worker with 1,000-item OTLP batches. A mixed step launches the same topology independently for traces, logs, and metrics. Without read load, the sustained proof reruns the mixed topology with the highest **accepted** throughput. With `--query-rate`, it chooses the highest-throughput topology that also completed at least 95% of the requested reads with no errors or scheduler shedding. If no sweep point qualifies, it repeats the best read-completion point and fails the sustained-read quality gate rather than publishing a write-only peak as mixed capacity. -For a publication run, use at least three repetitions: +For publication runs, dedicate the default ccx33 to Fanout and use the separate ccx43 for generators, with three repetitions. Run ingest-only and read-under-write as two groups sharing the same fresh pair of VMs; see [the publication commands](docs/publishing.md#publication-runs). For example: ```sh -./bin/fanout-bench run \ - --host root@bench-host \ - --fanout-version v2026.8.11 \ - --preset publish \ - --repeat 3 \ - --query-rate 50 \ - --output results/publish-2026-08-28 +scripts/hetzner.sh \ + --fanout-version v2026.10.1-alpha \ + --preset publish --repeat 3 \ + --output results/publish-ingest-2026-10-14 \ + -- \ + --fanout-version v2026.10.1-alpha \ + --preset publish --repeat 3 --query-rate 20 \ + --output results/publish-read-write-2026-10-14 ``` -Provisioned runs deliberately place Fanout and generators on the same remote host, measuring the complete single-node product under load. The report labels this an observed same-host ceiling and warns that generator contention can make it lower than Fanout's isolated backend ceiling. +Publication runs measure Fanout’s ingest ceiling with all eight ccx33 vCPUs available to Fanout; telemetrygen runs on the separate ccx43 over the private network. A plain `--host` run without the three split flags remains a same-host run, and its report warns about generator contention. ## Read and write together The write-only suite answers “how quickly can Fanout ingest?” It does not answer “what happens to dashboard latency while ingesting?” Add automatically authenticated read load to mixed phases: ```sh -./bin/fanout-bench run \ - --host root@bench-host \ - --fanout-version v2026.8.11 \ +scripts/hetzner.sh \ + --fanout-version v2026.10.1-alpha \ --preset standard \ --query-rate 50 ``` -The harness uses a bounded open-loop scheduler and rotates across overview, topology, performance, trace, and log queries with 15-minute through 24-hour windows. It reports attempted and completed rates, phase-end cancellations, explicit scheduler shedding, failures, mean, p50, p95, p99, and maximum latency. The concurrency bound prevents a stalled server from creating unbounded requests. +The harness uses a bounded open-loop scheduler targeting user operations per second at 15-minute, 1-hour, 6-hour, 12-hour, and 24-hour windows. Operations rotate through dashboard refresh, service-map trace drill-down, and trace detail. A refresh sends the full System overview panel batch exactly as the UI's initial all-visible load does, then requests annotations over the returned time-panel bounds. Layout widths model a fixed 1664-pixel viewport and are rounded as in the UI. Drill-down uses `POST /api/panels/exemplars` with the selected service and the panel response's absolute window. Trace detail uses `GET /api/traces/:id` with a discovered ID, namespace, and that same window, retained separately for each range; an empty detail is a failure. Until a trace is discovered, detail slots perform drill-down; until a selectable panel is cached, those slots perform dashboard refresh instead. Each slot performs one user operation. The current System overview has no log drill or standalone log/trace search operation. Each scheduled read measures every request in its operation, and any request or panel error fails the operation. The report records per-operation counts and latency in its methodology alongside the dashboard definition hash; aggregate fields and quality gates retain their schema. + +It reports attempted and completed rates, phase-end cancellations, explicit scheduler shedding, failures, mean, p50, p95, p99, and maximum latency. The concurrency bound prevents a stalled server from creating unbounded requests. ## Output Every run produces one self-contained directory: ```text -results/20260828T190000Z/ +results/20261014T190000Z/ ├── report.json authoritative schema-versioned result ├── report.md readable, publication-ready report ├── samples.jsonl per-second Fanout resource samples @@ -109,10 +140,10 @@ The terminal output is optimized for decisions: ╭────────────────────────────────────────────────────────────────────╮ │ FANOUT BENCH • accepted-row performance, with evidence │ ╰────────────────────────────────────────────────────────────────────╯ - Target localhost:4317 (plaintext gRPC) - Generator telemetrygen v0.157.0 same-host=true + Target 10.77.0.2:39281 (plaintext gRPC) + Generator telemetrygen v0.157.0 same-host=false Suite standard repeat=1 18 phases ~5.2 min load time - Evidence Fanout counters @ /-/metrics samples every 1.0s + Evidence Fanout counters @ /metrics samples every 1.0s ◇ mixed-p4-r1 offered 240,000 items/s for 15s … accepted …/s ratio …% CPU … cores peak RSS … @@ -140,8 +171,8 @@ Regenerate Markdown without rerunning load: ```sh ./bin/fanout-bench render \ - --input results/publish-2026-08-28/report.json \ - --output results/publish-2026-08-28/report.md + --input results/publish-2026-10-14/report.json \ + --output results/publish-2026-10-14/report.md ``` ## Metrics captured @@ -189,20 +220,23 @@ The project uses only the Go standard library at runtime. The generator is a sep ## Existing local Fanout process -Remote provisioning is the publication path. For local development against an already-running Fanout, install telemetrygen, provide short-lived credentials through environment variables, and omit `--host`: +Fresh Hetzner Cloud VMs are the publication path. For local development against an already-running Fanout, install telemetrygen, provide short-lived credentials through environment variables, and omit `--host`: ```sh just telemetrygen export FANOUT_BENCH_INGEST_TOKEN='short-lived-benchmark-token' +export FANOUT_BENCH_METRICS_TOKEN='matching-Fanout-metrics-token' ./bin/fanout-bench doctor --telemetrygen ./bin/telemetrygen ./bin/fanout-bench run \ --preset quick \ --telemetrygen ./bin/telemetrygen \ - --endpoint localhost:4317 \ - --metrics-url http://localhost:7520/-/metrics \ + --endpoint localhost:7520 \ + --metrics-url http://localhost:7520/metrics \ --insecure \ --same-host ``` +Managed runs set `FANOUT_ADDR` to one random port shared by OTLP/gRPC, OTLP/HTTP, the API, and `/metrics`. Split runs bind the private listen IP; same-host runs bind loopback. Metrics stay private: a per-run `FANOUT_METRICS_TOKEN` is sent as `Authorization: Bearer …` and never written to artifacts. MCP, alerts, and chat stay disabled. + Private local metrics endpoints use `FANOUT_BENCH_METRICS_TOKEN`. Local authenticated read load uses `FANOUT_BENCH_QUERY_COOKIE`. Secrets have no value flags and are never persisted. telemetrygen does expose its OTLP header in its process arguments while running, so use only a dedicated short-lived token and revoke it afterward. Managed remote runs create and destroy this token automatically. diff --git a/cmd/fanout-bench/main.go b/cmd/fanout-bench/main.go index 5faa73c..e33e977 100644 --- a/cmd/fanout-bench/main.go +++ b/cmd/fanout-bench/main.go @@ -63,13 +63,21 @@ func commonFlags(name string, args []string) (bench.Config, error) { fs := flag.NewFlagSet(name, flag.ContinueOnError) cfg := bench.Config{} var tokenEnv, metricsTokenEnv, queryCookieEnv string - fs.StringVar(&cfg.Host, "host", "", "SSH target to provision (for example root@bench-host); empty runs locally") + fs.StringVar(&cfg.Host, "host", "", "SSH target to provision (for example root@example.com); empty runs locally") fs.StringVar(&cfg.SSHIdentity, "ssh-identity", "", "SSH private key path") + fs.StringVar(&cfg.SSHKnownHosts, "ssh-known-hosts", "", "run-scoped SSH known-hosts file") + fs.StringVar(&cfg.HostDescription, "host-description", "", "host description recorded alongside hardware evidence") + if name == "run" { + fs.StringVar(&cfg.GeneratorHost, "generator-host", "", "SSH target to provision as a separate generator host") + fs.StringVar(&cfg.GeneratorAddress, "generator-address", "", "generator host IP reachable from the Fanout host") + fs.StringVar(&cfg.FanoutListen, "fanout-listen", "", "Fanout host IP reachable from the generator host") + fs.StringVar(&cfg.GeneratorHostDescription, "generator-host-description", "", "generator host description recorded in the report") + } fs.IntVar(&cfg.SSHPort, "ssh-port", 22, "SSH port") fs.BoolVar(&cfg.KeepRemote, "keep-remote", false, "preserve the isolated remote run directory after downloading artifacts") fs.StringVar(&cfg.Preset, "preset", "standard", "suite: quick, standard, or publish") - fs.StringVar(&cfg.Endpoint, "endpoint", "localhost:4317", "Fanout OTLP/gRPC endpoint") - fs.StringVar(&cfg.MetricsURL, "metrics-url", "http://localhost:7520/-/metrics", "Fanout Prometheus metrics URL (required evidence source)") + fs.StringVar(&cfg.Endpoint, "endpoint", "localhost:7520", "Fanout OTLP/gRPC endpoint") + fs.StringVar(&cfg.MetricsURL, "metrics-url", "http://localhost:7520/metrics", "Fanout Prometheus metrics URL (required evidence source)") fs.BoolVar(&cfg.Insecure, "insecure", false, "use plaintext OTLP/gRPC") fs.StringVar(&cfg.Telemetrygen, "telemetrygen", "./bin/telemetrygen", "path to pinned telemetrygen binary") fs.StringVar(&cfg.TelemetrygenVersion, "telemetrygen-version", "v0.157.0", "version recorded for the generator binary") @@ -87,7 +95,7 @@ func commonFlags(name string, args []string) (bench.Config, error) { fs.BoolVar(&cfg.SameHost, "same-host", false, "declare that Fanout and generators share a host") fs.IntVar(&cfg.BatchSize, "batch-size", 1000, "telemetrygen OTLP batch size") fs.StringVar(&cfg.QueryURL, "query-url", "", "Fanout HTTP base URL for authenticated read-under-write load") - fs.Float64Var(&cfg.QueryRate, "query-rate", 0, "aggregate dashboard queries/s during mixed phases") + fs.Float64Var(&cfg.QueryRate, "query-rate", 0, "aggregate UI operations/s during mixed phases") fs.StringVar(&queryCookieEnv, "query-cookie-env", "FANOUT_BENCH_QUERY_COOKIE", "environment variable containing the authenticated session Cookie header") fs.IntVar(&cfg.ProcessOverride, "processes", 0, "override sweep with one process count per signal") fs.IntVar(&cfg.WorkerOverride, "workers", 0, "override workers per telemetrygen process") @@ -96,6 +104,9 @@ func commonFlags(name string, args []string) (bench.Config, error) { if err := fs.Parse(args); err != nil { return cfg, err } + if err := bench.ValidateGeneratorHosts(cfg); err != nil { + return cfg, err + } cfg.Token = os.Getenv(tokenEnv) cfg.MetricsToken = os.Getenv(metricsTokenEnv) cfg.QueryCookie = os.Getenv(queryCookieEnv) @@ -143,18 +154,25 @@ func managed(ctx context.Context, args []string) error { var cfg bench.ManagedConfig fs.StringVar(&cfg.FanoutPath, "fanout", "./fanout", "provisioned Fanout binary") fs.StringVar(&cfg.TelemetrygenPath, "telemetrygen", "./telemetrygen", "provisioned telemetrygen binary") + fs.StringVar(&cfg.Listen, "listen", "127.0.0.1", "Fanout listen IP for OTLP, metrics, and queries") + fs.StringVar(&cfg.GeneratorSSH, "generator-ssh", "", "SSH target for telemetrygen launches") + fs.IntVar(&cfg.GeneratorSSHPort, "generator-ssh-port", 22, "SSH port for telemetrygen launches") + fs.StringVar(&cfg.GeneratorSSHIdentity, "generator-ssh-identity", "", "ephemeral SSH private key for the generator host") + fs.StringVar(&cfg.GeneratorTelemetrygen, "generator-telemetrygen", "", "telemetrygen path on the generator host") fs.StringVar(&cfg.OutputDir, "output", "./results", "remote evidence directory") fs.StringVar(&cfg.FanoutVersion, "fanout-version", "unknown", "Fanout version") fs.StringVar(&cfg.FanoutArtifact, "fanout-artifact", "", "verified release artifact identity") fs.StringVar(&cfg.Preset, "preset", "standard", "suite preset") fs.IntVar(&cfg.Repeat, "repeat", 1, "sweep repetitions") fs.IntVar(&cfg.BatchSize, "batch-size", 1000, "OTLP batch size") - fs.Float64Var(&cfg.QueryRate, "query-rate", 0, "read-under-write queries/s") + fs.Float64Var(&cfg.QueryRate, "query-rate", 0, "read-under-write UI operations/s") fs.DurationVar(&cfg.SampleInterval, "sample-interval", time.Second, "resource sample interval") fs.DurationVar(&cfg.Cooldown, "cooldown", 15*time.Second, "post-load cooldown") fs.StringVar(&cfg.HarnessVersion, "harness-version", version, "harness version") fs.StringVar(&cfg.TelemetrygenVersion, "telemetrygen-version", "v0.157.0", "telemetrygen version") - fs.StringVar(&cfg.ProvisionedHost, "provisioned-host", "", "SSH target recorded in remote evidence") + fs.StringVar(&cfg.ProvisionedHost, "provisioned-host", "", "SSH target used to construct the redacted reproduction command") + fs.StringVar(&cfg.HostDescription, "host-description", "", "host description recorded alongside hardware evidence") + fs.StringVar(&cfg.GeneratorHostDescription, "generator-host-description", "", "generator host description recorded in the report") fs.IntVar(&cfg.ProvisionedSSHPort, "provisioned-ssh-port", 22, "SSH port recorded in remote evidence") fs.StringVar(&cfg.ProvisionedSSHIdentity, "provisioned-ssh-identity", "", "SSH identity path recorded in remote evidence") fs.IntVar(&cfg.ProcessOverride, "processes", 0, "override process count per signal") diff --git a/cmd/fanout-bench/main_test.go b/cmd/fanout-bench/main_test.go new file mode 100644 index 0000000..ede4b02 --- /dev/null +++ b/cmd/fanout-bench/main_test.go @@ -0,0 +1,111 @@ +package main + +import ( + "context" + "os" + "os/exec" + "strings" + "testing" + + "github.com/labstack/fanout-bench/internal/bench" +) + +func TestLocalDefaultsUseSharedFanoutListener(t *testing.T) { + cfg, err := commonFlags("test", nil) + if err != nil { + t.Fatal(err) + } + if cfg.Endpoint != "localhost:7520" || cfg.MetricsURL != "http://localhost:7520/metrics" { + t.Fatalf("endpoint=%q metrics=%q", cfg.Endpoint, cfg.MetricsURL) + } +} + +func TestSplitReproductionCommandPassesValidationAfterSubstitution(t *testing.T) { + executable, err := os.Executable() + if err != nil { + t.Fatal(err) + } + cfg, err := commonFlags("run", []string{ + "--host", "root@192.0.2.10", "--generator-host", "root@192.0.2.11", + "--generator-address", "10.77.0.3", "--fanout-listen", "10.77.0.2", + "--ssh-port", "2222", "--telemetrygen", executable, "--preset", "quick", + }) + if err != nil { + t.Fatal(err) + } + cfg.Token = "test-token" + ctx, cancel := context.WithCancel(context.Background()) + cancel() + runner := bench.Runner{Config: cfg} + report, err := runner.Run(ctx) + if err == nil || report == nil { + t.Fatalf("expected partial report on cancellation, got report=%v err=%v", report, err) + } + command := strings.NewReplacer( + "", "192.0.2.20", "", "192.0.2.21", + "", "10.77.0.13", "", "10.77.0.12", + ).Replace(report.ReproductionCommand) + // Parse shell quoting without executing the reproduction command. + output, err := exec.Command("bash", "-c", "set -- "+command+"; printf '%s\\0' \"$@\"").Output() + if err != nil { + t.Fatal(err) + } + args := strings.Split(strings.TrimSuffix(string(output), "\x00"), "\x00") + reproduced, err := commonFlags("run", args[2:]) + if err != nil { + t.Fatalf("substituted reproduction failed validation: %v\n%s", err, command) + } + if reproduced.Host != "root@192.0.2.20" || reproduced.GeneratorHost != "root@192.0.2.21" || + reproduced.GeneratorAddress != "10.77.0.13" || reproduced.FanoutListen != "10.77.0.12" || reproduced.SSHPort != 2222 { + t.Fatalf("reproduction lost topology: %+v", reproduced) + } +} + +func TestHostFlags(t *testing.T) { + cfg, err := commonFlags("test", []string{"--ssh-known-hosts", "/tmp/run scope/known_hosts", "--host-description", "Hetzner Cloud ccx33, fsn1, ubuntu-24.04"}) + if err != nil { + t.Fatal(err) + } + if cfg.SSHKnownHosts != "/tmp/run scope/known_hosts" || cfg.HostDescription != "Hetzner Cloud ccx33, fsn1, ubuntu-24.04" { + t.Fatalf("known-hosts=%q description=%q", cfg.SSHKnownHosts, cfg.HostDescription) + } +} + +func TestSplitFlagsRequiredTogetherAndRemoteOnly(t *testing.T) { + flags := [][]string{{"--generator-host", "root@generator.example"}, {"--generator-address", "10.77.0.3"}, {"--fanout-listen", "10.77.0.2"}} + for mask := 0; mask < 8; mask++ { + for _, remote := range []bool{false, true} { + var args []string + if remote { + args = append(args, "--host", "root@fanout.example") + } + for i, pair := range flags { + if mask&(1<