Optimize protobuf encoding and add CI performance reports

.github/performance/.gitignore

@@ -0,0 +1 @@
+__pycache__/

.github/performance/compare.py

@@ -0,0 +1,131 @@
+#!/usr/bin/env python3
+"""Build two immutable revisions, then compare JMH on this machine/JDK."""
+import argparse
+import hashlib
+import json
+import os
+from pathlib import Path
+import platform
+import re
+import shutil
+import subprocess
+import tarfile
+import tempfile
+
+from report import METHODS, SMOKE_METHODS, PREFIX, compare, render, validate
+
+
+def output(command, cwd=None):
+    return subprocess.check_output(command, cwd=cwd, text=True).strip()
+
+
+def revision(repo, ref):
+    return output(['git', 'rev-parse', '--verify', '--end-of-options', ref + '^{commit}'], repo)
+
+
+def export(repo, sha, destination):
+    destination.mkdir()
+    archive = destination.parent / (destination.name + '.tar')
+    with archive.open('wb') as stream:
+        subprocess.run(['git', 'archive', '--format=tar', sha], cwd=repo, stdout=stream, check=True)
+    with tarfile.open(archive) as source:
+        # Reject archive entries escaping the exported checkout, including symlinks.
+        source.extractall(destination, filter='data')
+    archive.unlink()
+
+
+def run_logged(command, log, cwd=None):
+    with log.open('w') as stream:
+        try:
+            subprocess.run(command, cwd=cwd, stdout=stream, stderr=subprocess.STDOUT, check=True)
+        except subprocess.CalledProcessError:
+            print(log.read_text()[-8000:], flush=True)
+            raise
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('--base', required=True, help='Immutable SHA or local ref')
+    parser.add_argument('--candidate', default='HEAD')
+    parser.add_argument('--output', required=True)
+    parser.add_argument('--java', required=True, choices=['21', '25'])
+    parser.add_argument('--smoke', action='store_true', help='Build both revisions; run only 3 short cases')
+    args = parser.parse_args()
+    repo = Path(output(['git', 'rev-parse', '--show-toplevel']))
+    dest = Path(args.output).resolve()
+    dest.mkdir(parents=True, exist_ok=False)
+    java_home = Path(os.environ['JAVA_HOME']).resolve()
+    java = str(java_home / 'bin/java')
+    javac = output([str(java_home / 'bin/javac'), '-version'])
+    if not javac.startswith('javac ' + args.java + '.'):
+        raise ValueError('JAVA_HOME does not match requested Java version')
+    base, candidate = revision(repo, args.base), revision(repo, args.candidate)
+    harness = output(['git', 'rev-parse', candidate + ':buff-json-benchmarks/src'], repo)
+    methods = SMOKE_METHODS if args.smoke else METHODS
+    metadata = {'base_sha': base, 'candidate_sha': candidate, 'harness_sha': harness, 'java': args.java,
+                'compiler': javac, 'os': platform.platform(), 'architecture': platform.machine(),
+                'cpu': platform.processor(), 'runner_image': os.environ.get('ImageVersion', 'local'),
+                'run_id': os.environ.get('GITHUB_RUN_ID', 'local')}
+    if Path('/proc/cpuinfo').exists():
+        metadata['cpu'] = next((line.split(':', 1)[1].strip() for line in Path('/proc/cpuinfo').read_text().splitlines()
+                                if line.startswith('model name')), metadata['cpu'])
+    (dest / 'metadata.json').write_text(json.dumps(metadata, indent=2) + '\n')
+    all_results = {'base': [], 'candidate': []}
+    with tempfile.TemporaryDirectory(prefix='buff-json-performance-') as work:
+        work = Path(work)
+        trees = {name: work / name for name in all_results}
+        for name, sha in (('base', base), ('candidate', candidate)):
+            export(repo, sha, trees[name])
+        # Compare identical workload definitions, including generated proto fixtures.
+        # Each revision still uses its OWN runtime, protoc plugin and dependency versions.
+        source_path = 'buff-json-benchmarks/src'
+        shutil.rmtree(trees['base'] / source_path)
+        shutil.copytree(trees['candidate'] / source_path, trees['base'] / source_path)
+        jars = {}
+        for name, tree in trees.items():
+            print('BUILD', name, flush=True)
+            cache_root = Path(os.environ.get('PERFORMANCE_MAVEN_REPO', str(work / 'maven')))
+            local_repo = cache_root / name
+            local_repo.mkdir(parents=True, exist_ok=True)
+            # Cached third-party dependencies are reusable; our SNAPSHOT artifacts
+            # must always come from the exact revision being built.
+            shutil.rmtree(local_repo / 'io/github/suboptimal-solutions', ignore_errors=True)
+            command = ['mvn', '-B', '-ntp', 'clean', 'package', '-DskipTests', '-Dspotless.skip=true',
+                       '-Dmaven.repo.local=' + str(local_repo)]
+            run_logged(command, dest / (name + '-build.log'), tree)
+            jars[name] = tree / 'buff-json-benchmarks/target/benchmarks.jar'
+            metadata[name + '_jar_sha256'] = hashlib.sha256(jars[name].read_bytes()).hexdigest()
+        # All builds finish before measuring. Alternate variant order per method,
+        # reversing it on Java 25; both forks for a method share a single JMH run.
+        for index, method in enumerate(methods):
+            order = ('base', 'candidate') if (index + int(args.java == '25')) % 2 == 0 else ('candidate', 'base')
+            for name in order:
+                stem = dest / (name + '-' + method)
+                command = [java, '-jar', str(jars[name]), '^' + re.escape(PREFIX + method) + '$',
+                           '-t', '1', '-f', '1' if args.smoke else '2',
+                           '-wi', '1' if args.smoke else '3', '-i', '3' if args.smoke else '4',
+                           '-w', '200ms' if args.smoke else '1s', '-r', '200ms' if args.smoke else '1s',
+                           '-prof', 'gc', '-foe', 'true', '-jvmArgs', '-Xms256m -Xmx256m',
+                           '-rf', 'json', '-rff', str(stem) + '.json']
+                print('MEASURE', method, name, flush=True)
+                run_logged(command, Path(str(stem) + '.log'))
+                measured = json.loads(Path(str(stem) + '.json').read_text())
+                if len(measured) != 1:
+                    raise ValueError('Expected exactly one completed benchmark')
+                all_results[name].extend(measured)
+    for name, results in all_results.items():
+        (dest / (name + '.json')).write_text(json.dumps(results, indent=2) + '\n')
+    summary = compare(all_results['base'], all_results['candidate'], metadata, methods)
+    validate(summary, full=not args.smoke)
+    (dest / 'summary.json').write_text(json.dumps(summary, indent=2, allow_nan=False) + '\n')
+    (dest / 'metadata.json').write_text(json.dumps(metadata, indent=2) + '\n')
+    report = render(summary)
+    (dest / 'report.md').write_text(report)
+    if os.environ.get('GITHUB_STEP_SUMMARY'):
+        with open(os.environ['GITHUB_STEP_SUMMARY'], 'a') as stream:
+            stream.write(report)
+    print(report, flush=True)
+
+
+if __name__ == '__main__':
+    main()

.github/performance/publish.py

@@ -0,0 +1,160 @@
+"""Trusted workflow_run publisher; benchmark artifacts are data, never code."""
+import io
+import json
+import os
+from pathlib import Path
+import urllib.request
+import urllib.parse
+import zipfile
+
+from report import render, validate, SHA
+
+MARKER = '<!-- buff-json-performance-report -->'
+MAX_ARCHIVE_BYTES = 1_000_000
+MAX_JSON_BYTES = 250_000
+
+
+class SafeRedirect(urllib.request.HTTPRedirectHandler):
+    def redirect_request(self, request, fp, code, message, headers, newurl):
+        if urllib.parse.urlparse(newurl).scheme != 'https':
+            raise ValueError('Refusing non-HTTPS artifact redirect')
+        redirected = super().redirect_request(request, fp, code, message, headers, newurl)
+        if urllib.parse.urlparse(request.full_url).netloc != urllib.parse.urlparse(newurl).netloc:
+            redirected.remove_header('Authorization')
+        return redirected
+
+
+class GitHub:
+    def __init__(self, token, repository):
+        self.repository = repository
+        self.prefix = 'https://api.github.com/repos/' + repository
+        self.token = token
+
+    def request(self, path, method='GET', data=None, binary=False):
+        url = self.prefix + path
+        request = urllib.request.Request(url, method=method,
+                    data=None if data is None else json.dumps(data).encode(),
+                    headers={'Authorization': 'Bearer ' + self.token,
+                             'Accept': 'application/vnd.github+json',
+                             'X-GitHub-Api-Version': '2022-11-28',
+                             'Content-Type': 'application/json'})
+        with urllib.request.build_opener(SafeRedirect()).open(request, timeout=60) as response:
+            content = response.read(MAX_ARCHIVE_BYTES + 1 if binary else 4_000_000)
+        if binary:
+            if len(content) > MAX_ARCHIVE_BYTES:
+                raise ValueError('Oversized artifact archive')
+            return content
+        return json.loads(content) if content else None
+
+    def pages(self, path, key=None):
+        for page in range(1, 101):
+            separator = '&' if '?' in path else '?'
+            data = self.request(path + separator + f'per_page=100&page={page}')
+            items = data[key] if key else data
+            yield from items
+            if len(items) < 100:
+                return
+        raise ValueError('API pagination limit exceeded')
+
+
+def read_summary(archive, java, candidate):
+    with zipfile.ZipFile(io.BytesIO(archive)) as package:
+        entries = package.infolist()
+        if len(entries) != 1 or entries[0].filename != 'summary.json' or entries[0].file_size > MAX_JSON_BYTES:
+            raise ValueError('Expected one bounded summary.json artifact')
+        data = json.loads(package.read(entries[0]))
+    validate(data)
+    if data['metadata']['java'] != java or data['metadata']['candidate_sha'] != candidate:
+        raise ValueError('Artifact does not match triggering run')
+    return data
+
+
+def current_pull_request(api, run, base):
+    # Derive association from GitHub, not a PR number supplied by an artifact.
+    if run['event'] != 'pull_request':
+        return None
+    candidates = list(api.pages('/commits/' + run['head_sha'] + '/pulls'))
+    matching = []
+    for candidate in candidates:
+        pr = api.request('/pulls/' + str(int(candidate['number'])))
+        if (pr['state'] == 'open' and pr['base']['repo']['full_name'] == api.repository
+                and pr['head']['sha'] == run['head_sha'] and pr['base']['sha'] == base):
+            matching.append(pr)
+    # Ambiguity or a newer PR revision must not update an unrelated/current report.
+    return matching[0] if len(matching) == 1 else None
+
+
+def publish(api, run_id):
+    run = api.request('/actions/runs/' + str(run_id))
+    workflow = api.request('/actions/workflows/' + str(run['workflow_id']))
+    if (workflow['path'] != '.github/workflows/performance.yml'
+            or run['event'] not in ('pull_request', 'push', 'workflow_dispatch')
+            or run['status'] != 'completed' or not SHA.fullmatch(run['head_sha'])):
+        raise ValueError('Unexpected triggering workflow')
+    run_url = f'https://github.com/{api.repository}/actions/runs/{run_id}'
+    body = [MARKER, '## Performance comparison', '', f'[Workflow and raw JMH artifacts]({run_url})', f"Commit: {run['head_sha']}", '']
+    summaries = []
+    error = None
+    try:
+        artifacts = list(api.pages('/actions/runs/' + str(run_id) + '/artifacts', 'artifacts'))
+        for java in ('21', '25'):
+            selected = [a for a in artifacts if a['name'] == 'performance-summary-java' + java and not a['expired']]
+            if len(selected) != 1:
+                raise ValueError('Missing or ambiguous Java ' + java + ' summary')
+            artifact = selected[0]
+            if artifact['size_in_bytes'] > MAX_ARCHIVE_BYTES:
+                raise ValueError('Oversized summary artifact')
+            archive = api.request('/actions/artifacts/' + str(int(artifact['id'])) + '/zip', binary=True)
+            summaries.append(read_summary(archive, java, run['head_sha']))
+        if len({(s['metadata']['base_sha'], s['metadata']['harness_sha']) for s in summaries}) != 1:
+            raise ValueError('Matrix jobs compared different sources')
+    except (ValueError, KeyError, TypeError, zipfile.BadZipFile) as problem:
+        # Never interpolate arbitrary artifact strings into a privileged comment.
+        print('Incomplete/invalid performance data:', type(problem).__name__)
+        error = 'Performance measurements are incomplete or invalid. Inspect the workflow logs; this is not a passing performance result.'
+    if error:
+        body += [error]
+    else:
+        body += [render(s) for s in summaries]
+    if run['conclusion'] != 'success':
+        body += ['', 'The measurement workflow did not succeed; results may be incomplete.']
+    text = '\n'.join(body)
+    conclusion = 'failure' if error or run['conclusion'] != 'success' else 'neutral'
+    # Neutral reports intentionally do not turn noisy wall-clock changes into a gate.
+    check = {'name': 'Performance report', 'head_sha': run['head_sha'], 'status': 'completed',
+             'conclusion': conclusion, 'details_url': run_url, 'external_id': 'performance-' + str(run_id),
+             'output': {'title': 'JMH comparison (Java 21 and 25)', 'summary': text}}
+    existing = [c for c in api.pages('/commits/' + run['head_sha'] + '/check-runs?check_name=Performance%20report', 'check_runs')
+                if c.get('external_id') == check['external_id']]
+    if existing:
+        del check['head_sha']
+        api.request('/check-runs/' + str(existing[0]['id']), 'PATCH', check)
+    else:
+        api.request('/check-runs', 'POST', check)
+    baseline = summaries[0]['metadata']['base_sha'] if not error else None
+    if error:
+        linked = [p for p in run.get('pull_requests', []) if p['head']['sha'] == run['head_sha']]
+        if len(linked) == 1 and SHA.fullmatch(linked[0]['base']['sha']):
+            baseline = linked[0]['base']['sha']
+    if baseline:
+        pr = current_pull_request(api, run, baseline)
+        if pr:
+            path = '/issues/' + str(pr['number']) + '/comments'
+            comments = [c for c in api.pages(path) if c['user']['login'] == 'github-actions[bot]'
+                        and c.get('body', '').startswith(MARKER)]
+            # Recheck immediately before writing to avoid replacing a newer report.
+            latest = api.request('/pulls/' + str(pr['number']))
+            if latest['head']['sha'] == run['head_sha'] and latest['base']['sha'] == baseline:
+                if comments:
+                    api.request('/issues/comments/' + str(comments[0]['id']), 'PATCH', {'body': text})
+                else:
+                    api.request(path, 'POST', {'body': text})
+    return text
+
+
+if __name__ == '__main__':
+    event = json.loads(Path(os.environ['GITHUB_EVENT_PATH']).read_text())
+    api = GitHub(os.environ['GITHUB_TOKEN'], os.environ['GITHUB_REPOSITORY'])
+    result = publish(api, int(event['workflow_run']['id']))
+    with open(os.environ['GITHUB_STEP_SUMMARY'], 'a') as stream:
+        stream.write(result)