diff --git a/.github/scripts/benchstat-summary.py b/.github/scripts/benchstat-summary.py new file mode 100644 index 000000000..c1acb882d --- /dev/null +++ b/.github/scripts/benchstat-summary.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python3 +""" +benchstat-summary.py — parse benchstat output and produce a markdown summary. + +Usage: benchstat-summary.py [--threshold PCT] benchstat.txt + +Extracts significant regressions/improvements from benchstat output. +Exit code 1 if any regression exceeds the threshold (default: 5%). +""" + +import argparse +import re +import sys +from dataclasses import dataclass + + +@dataclass +class Change: + name: str + base: str + head: str + pct: float + pval: str + + +# Matches lines like: +# BenchmarkName-4 1.030m ± 3% 1.304m ± 5% +26.57% (p=0.000 n=10) +# BenchmarkName-4 3.997 ± 1% 3.910 ± 0% -2.16% (p=0.000 n=10) +LINE_RE = re.compile( + r"^(\S+)" # benchmark name + r"\s+" + r"(\S+)" # base value + r"\s+±\s+\d+%" # base variance + r"\s+" + r"(\S+)" # head value + r"\s+±\s+\d+%" # head variance + r"\s+" + r"([+-]\d+\.\d+)%" # percentage change + r"\s+" + r"\(p=(\d+\.\d+)" # p-value +) + + +def parse_benchstat(path: str) -> list[Change]: + changes = [] + with open(path) as f: + for line in f: + stripped = line.strip() + if stripped.startswith("geomean"): + continue + m = LINE_RE.match(stripped) + if not m: + continue + changes.append( + Change( + name=m.group(1), + base=m.group(2), + head=m.group(3), + pct=float(m.group(4)), + pval=m.group(5), + ) + ) + return changes + + +def render_markdown(changes: list[Change], threshold: float) -> str: + regressions = sorted([c for c in changes if c.pct > 0], key=lambda c: -c.pct) + improvements = sorted([c for c in changes if c.pct < 0], key=lambda c: c.pct) + + if not regressions and not improvements: + return "### No significant performance changes detected\n" + + lines: list[str] = [] + + if regressions: + above = [r for r in regressions if r.pct > threshold] + if above: + lines.append( + f"### {len(regressions)} regression(s) detected (threshold: >{threshold:g}%)\n" + ) + else: + lines.append( + f"### {len(regressions)} minor regression(s) (all within {threshold:g}% threshold)\n" + ) + + lines.append("| Benchmark | Base | Head | Change | p-value |") + lines.append("|-----------|------|------|--------|---------|") + for r in regressions: + change = f"+{r.pct:.2f}%" + if r.pct > threshold: + change = f"**{change}**" + lines.append(f"| `{r.name}` | {r.base} | {r.head} | {change} | {r.pval} |") + lines.append("") + + if improvements: + lines.append(f"
") + lines.append(f"{len(improvements)} improvement(s)\n") + lines.append("| Benchmark | Base | Head | Change | p-value |") + lines.append("|-----------|------|------|--------|---------|") + for imp in improvements: + lines.append( + f"| `{imp.name}` | {imp.base} | {imp.head} | {imp.pct:.2f}% | {imp.pval} |" + ) + lines.append("") + lines.append("
") + lines.append("") + + return "\n".join(lines) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("input", help="Path to benchstat output file") + parser.add_argument( + "--threshold", + type=float, + default=5, + help="Regression percentage threshold to flag as failure (default: 5)", + ) + args = parser.parse_args() + + changes = parse_benchstat(args.input) + summary = render_markdown(changes, args.threshold) + print(summary) + + # Exit 1 if any regression exceeds the threshold + regressions_above_threshold = [c for c in changes if c.pct > args.threshold] + if regressions_above_threshold: + print(f"\nFailed: {len(regressions_above_threshold)} benchmark(s) regressed by more than {args.threshold:g}%:") + for c in sorted(regressions_above_threshold, key=lambda c: -c.pct): + print(f" {c.name}: {c.base} -> {c.head} (+{c.pct:.2f}%)") + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml new file mode 100644 index 000000000..6244e231e --- /dev/null +++ b/.github/workflows/benchmark.yml @@ -0,0 +1,153 @@ +on: + pull_request: + +name: Benchmark + +permissions: + contents: read + issues: write + pull-requests: write + +jobs: + bench: + name: Benchmark + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - name: Install Go + uses: buildjet/setup-go@v5 + with: + go-version: 1.25.x + cache: false + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Post "benchmark running" comment + uses: actions/github-script@v9 + with: + script: | + const marker = ''; + const body = [ + marker, + 'Benchmark running.', + '', + `Base: \`${{ github.event.pull_request.base.sha }}\``, + `Head: \`${{ github.event.pull_request.head.sha }}\``, + '', + 'This comment will be updated when benchmarks complete.', + ].join('\n'); + + const comments = await github.paginate(github.rest.issues.listComments, { + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + per_page: 100, + }); + + const existing = comments.find(c => (c.body || '').includes(marker)); + + if (existing) { + await github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: existing.id, + body, + }); + } else { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + body, + }); + } + - name: Install benchstat + run: | + GOBIN="$PWD/.bin" go install golang.org/x/perf/cmd/benchstat@latest + echo "$PWD/.bin" >> "$GITHUB_PATH" + - name: Prepare base worktree + run: | + mkdir -p .bench + git worktree add .bench/base "${{ github.event.pull_request.base.sha }}" + # Override the base's bench file with HEAD's so both runs execute + # the same benchmark suite — only the implementation under test differs. + cp serialize_bench_test.go .bench/base/serialize_bench_test.go + - name: Benchmark base + run: | + cd .bench/base + go test -run=^$ -bench=BenchmarkSerialize -count=6 -timeout 20m \ + github.com/flanksource/gomplate/v3 | tee "$GITHUB_WORKSPACE/bench-base.txt" + - name: Benchmark head + run: | + go test -run=^$ -bench=BenchmarkSerialize -count=6 -timeout 20m \ + github.com/flanksource/gomplate/v3 | tee bench-head.txt + - name: Compare + run: | + benchstat bench-base.txt bench-head.txt > benchstat.txt + + # Generate the summary (ignore exit code here; we check it in the next step) + python3 .github/scripts/benchstat-summary.py benchstat.txt > bench-summary.md || true + + { + echo "" + echo "## Benchstat" + echo "" + echo "Base: \`${{ github.event.pull_request.base.sha }}\`" + echo "Head: \`${{ github.event.pull_request.head.sha }}\`" + echo "" + cat bench-summary.md + echo "" + echo '
' + echo 'Full benchstat output' + echo "" + echo '```text' + cat benchstat.txt + echo '```' + echo "" + echo '
' + } > bench-report.md + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: benchmark-results + path: | + bench-base.txt + bench-head.txt + benchstat.txt + bench-report.md + retention-days: 14 + - name: Post report to PR + uses: actions/github-script@v9 + with: + script: | + const fs = require('fs'); + const body = fs.readFileSync('bench-report.md', 'utf8'); + const marker = ''; + + const comments = await github.paginate(github.rest.issues.listComments, { + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + per_page: 100, + }); + + const existing = comments.find(c => (c.body || '').includes(marker)); + + if (existing) { + await github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: existing.id, + body: body + }); + } else { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + body: body + }); + } + - name: Check for regressions + run: python3 .github/scripts/benchstat-summary.py --threshold 5 benchstat.txt > /dev/null diff --git a/serialize.go b/serialize.go index 108718f6d..ab15438ec 100644 --- a/serialize.go +++ b/serialize.go @@ -31,6 +31,11 @@ type AsMapper interface { AsMap(fields ...string) map[string]any } +type nativeType struct { + path jp.Expr + val any +} + // Serialize iterates over each key-value pair in the input map // serializes any struct value to map[string]any. func Serialize(in map[string]any) (out map[string]any, err error) { @@ -40,37 +45,37 @@ func Serialize(in map[string]any) (out map[string]any, err error) { defer func() { if r := recover(); r != nil { - if _err, ok := r.(error); ok { - err = _err - } err = fmt.Errorf("panic during serialization: %v", r) } }() // cel supports time.Duration natively - save original and then replace it after decomposition // FIXME: This does not work for anything inside Structs - nativeTypes := make(map[string]any, len(in)) + nativeTypes := make([]nativeType, 0, len(in)) jp.Walk(in, func(path jp.Expr, value any) { + add := func(v any) { + // Copy path so later Walk mutations can not affect the stored expression. + nativeTypes = append(nativeTypes, nativeType{path: append(jp.Expr(nil), path...), val: v}) + } + switch v := value.(type) { case AsMapper: - nativeTypes[path.String()] = v.AsMap() + add(v.AsMap()) case uuid.UUID: - nativeTypes[path.String()] = v.String() + add(v.String()) case *uuid.UUID: - nativeTypes[path.String()] = v.String() + if v != nil { + add(v.String()) + } case time.Duration: - nativeTypes[path.String()] = v + add(v) } }) out = alt.Alter(in, &opts).(map[string]any) - for path, v := range nativeTypes { - expr, err := jp.ParseString(path) - if err != nil { - return nil, err - } - if err := expr.SetOne(out, v); err != nil { + for _, native := range nativeTypes { + if err := native.path.SetOne(out, native.val); err != nil { return nil, err } } diff --git a/serialize_bench_test.go b/serialize_bench_test.go new file mode 100644 index 000000000..9a907f051 --- /dev/null +++ b/serialize_bench_test.go @@ -0,0 +1,130 @@ +package gomplate + +import ( + "fmt" + "testing" + "time" + + "github.com/google/uuid" +) + +type benchAddress struct { + City string `json:"city_name"` + Country string `json:"country,omitempty"` +} + +type benchPerson struct { + Name string `json:"name"` + Age int `json:"age,omitempty"` + ID uuid.UUID `json:"id"` + Duration time.Duration `json:"duration"` + Address *benchAddress `json:",omitempty"` + MetaData map[string]any `json:",omitempty"` + Codes []string `json:",omitempty"` + Addresses []benchAddress `json:"addresses,omitempty"` +} + +func BenchmarkSerialize(b *testing.B) { + sizes := []int{10, 100, 1000, 10000} + + for _, size := range sizes { + b.Run(fmt.Sprintf("Size-%d", size), func(b *testing.B) { + input := newSerializeBenchmarkInput(size) + + b.ReportAllocs() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + if _, err := Serialize(input); err != nil { + b.Fatal(err) + } + } + }) + } +} + +// BenchmarkSerialize_NoNativeTypes measures the path where Walk finds no +// uuid/duration/AsMapper values — isolates the Alter cost from the SetOne +// fixup loop optimized in the last commit. +func BenchmarkSerialize_NoNativeTypes(b *testing.B) { + sizes := []int{100, 1000, 10000} + + for _, size := range sizes { + b.Run(fmt.Sprintf("Size-%d", size), func(b *testing.B) { + input := newPlainBenchmarkInput(size) + + b.ReportAllocs() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + if _, err := Serialize(input); err != nil { + b.Fatal(err) + } + } + }) + } +} + +func newSerializeBenchmarkInput(size int) map[string]any { + items := make([]any, size) + for i := range items { + items[i] = benchPerson{ + Name: fmt.Sprintf("person-%d", i), + Age: i % 100, + ID: uuid.New(), + Duration: time.Duration(i) * time.Millisecond, + Address: &benchAddress{ + City: "Kathmandu", + Country: "Nepal", + }, + MetaData: map[string]any{ + "index": i, + "enabled": i%2 == 0, + "uuid": uuid.New(), + "duration": time.Duration(i) * time.Second, + }, + Codes: []string{"GO", "JS", "CEL"}, + Addresses: []benchAddress{ + {City: "Kathmandu", Country: "Nepal"}, + {City: "Lalitpur", Country: "Nepal"}, + {City: "Bhaktapur", Country: "Nepal"}, + }, + } + } + + return map[string]any{ + "id": uuid.New(), + "started": time.Now(), + "duration": 5 * time.Minute, + "items": items, + "nested": map[string]any{ + "bytes": []byte("hello world"), + "labels": map[string]string{"app": "gomplate", "bench": "serialize"}, + "durations": []time.Duration{time.Second, time.Minute, time.Hour}, + }, + } +} + +func newPlainBenchmarkInput(size int) map[string]any { + items := make([]any, size) + for i := range items { + items[i] = map[string]any{ + "name": fmt.Sprintf("person-%d", i), + "age": i % 100, + "enabled": i%2 == 0, + "codes": []string{"GO", "JS", "CEL"}, + "address": map[string]any{ + "city": "Kathmandu", + "country": "Nepal", + }, + } + } + + return map[string]any{ + "started": "2026-01-01T00:00:00Z", + "items": items, + "nested": map[string]any{ + "labels": map[string]string{"app": "gomplate", "bench": "serialize"}, + }, + } +}