proxygen / proxygen-coro: shared source tree + performance work

frameworks/proxygen-coro/README.md

@@ -0,0 +1,61 @@
+# proxygen-coro
+
+The same workload as the [`proxygen`](../proxygen/README.md) entry, served
+through Proxygen's native coroutine stack instead of the callback
+`RequestHandler` / `HTTPTransactionHandler` APIs.
+
+`src/CoroServer.cpp` implements `proxygen::coro::HTTPHandler`, consumes requests
+through `HTTPSourceHolder` and returns `HTTPFixedSource` responses. Uploads are
+counted while asynchronously draining BODY events. WebSockets use a long-lived
+custom `HTTPSource`: its response calls `setEgressWebsocketUpgrade()`, then it
+parses and emits RFC 6455 frames over the raw upgraded BODY event stream.
+
+Response compression is proxygen's coro `CompressionFilter`, attached by the
+handler to the JSON responses rather than registered as a
+`ServerCompressionFilterFactory` on `HTTPServer::Config`. The reason is that
+`HTTPFilterFactoryHandler` calls `makeFilters()` *before* the request headers
+have been read, so a server-level factory cannot be conditional: it allocates a
+`SharedCtx`, a `VisitorFilter` holding a capturing lambda, a `CompressionFilter`
+and a coroutine frame on **every** request, including `baseline`, where nothing
+is compressible. That measured at ~10% of baseline throughput on this workload.
+The classic `HTTPServer` path has no equivalent cost, because
+`CompressionFilterFactory::onRequest` can see the request and returns the
+handler unwrapped when there is nothing to compress.
+
+## Listener layout
+
+One process owns all five listeners:
+
+- `8080/tcp` HTTP/1.1 and WebSockets
+- `8081/tcp` HTTP/1.1 over TLS, ALPN `http/1.1`
+- `8082/tcp` prior-knowledge h2c
+- `8443/tcp` HTTP/2 over TLS, ALPN `h2`
+- `8443/udp` HTTP/3 over QUIC, ALPN `h3`
+
+The four TCP listeners are acceptors on one coro `HTTPServer`, so they share a
+single affinity-aware I/O pool. Proxygen's coro API selects either TCP or QUIC
+per server, so HTTP/3 uses a second in-process pool; whichever transport is not
+being benchmarked stays idle. Size the two pools with `PROXYGEN_THREADS` and
+`PROXYGEN_H3_THREADS` (`0` = available CPUs).
+
+HTTP/2 advertises 1024 concurrent streams with a 1 MiB stream window and a
+10 MiB connection window. HTTP/3 mirrors Proxygen's coroutine benchmark
+settings: GSO batches of 48 packets, continuous-memory writes, a large
+congestion window and a 48-packet connection write limit.
+
+## Build
+
+This entry has no Dockerfile of its own. `build.sh` builds the
+`frameworks/proxygen` context with `--build-arg TARGET=coro`, the same pattern
+`sark-h3` uses to reuse `sark`. See
+[the proxygen README](../proxygen/README.md#shared-source-tree) for the layout
+and image details.
+
+```bash
+./scripts/validate.sh proxygen-coro
+./scripts/run.sh proxygen-coro
+```
+
+The implementation follows the upstream coroutine echo server and the
+`H12DownstreamSessionTest.WebSocketUpgrade` test, which documents upgraded raw
+bytes flowing through coroutine BODY events.

frameworks/proxygen-coro/build.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+# proxygen-coro reuses the proxygen build context and Dockerfile, selecting the
+# coroutine server with --build-arg TARGET=coro (the sark-h3 -> sark pattern).
+set -euo pipefail
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
+CONTEXT_DIR="$(cd "$SCRIPT_DIR/../proxygen" && pwd -P)"
+docker build -t httparena-proxygen-coro \
+    --build-arg TARGET=coro \
+    -f "$CONTEXT_DIR/Dockerfile" \
+    "$CONTEXT_DIR"

frameworks/proxygen-coro/meta.json

@@ -0,0 +1,27 @@
+{
+  "display_name": "proxygen-coro",
+  "language": "C++",
+  "type": "engine",
+  "engine": "proxygen",
+  "description": "Meta's Proxygen native coroutine HTTPServer and HTTPSource APIs across HTTP/1.1, HTTP/1.1 TLS, h2c, HTTP/2 TLS, HTTP/3 QUIC, and RFC 6455 WebSockets.",
+  "repo": "https://github.com/facebook/proxygen",
+  "enabled": true,
+  "tests": [
+    "baseline",
+    "json-comp",
+    "json-tls",
+    "static-tls",
+    "pipelined",
+    "limited-conn",
+    "baseline-h2",
+    "baseline-h2c",
+    "json-h2c",
+    "static-h2",
+    "baseline-h3",
+    "static-h3",
+    "echo-ws",
+    "echo-ws-pipeline",
+    "echo-ws-limited"
+  ],
+  "maintainers": []
+}

frameworks/proxygen/CMakeLists.txt

@@ -0,0 +1,56 @@
+cmake_minimum_required(VERSION 3.20)
+
+project(httparena-proxygen LANGUAGES CXX)
+
+set(CMAKE_CXX_STANDARD 20)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_CXX_EXTENSIONS OFF)
+
+# Which server to build. Both HttpArena entries share this source tree:
+#   classic -> proxygen::HTTPServer + the mvfst-backed HQServer   (`proxygen`)
+#   coro    -> proxygen::coro::HTTPServer                         (`proxygen-coro`)
+set(SERVER_TARGET "classic" CACHE STRING "classic or coro")
+
+# Proxygen's exported Fizz package calls find_dependency(Sodium). The official
+# builder image keeps that upstream find module with the Proxygen source tree.
+list(APPEND CMAKE_MODULE_PATH "/proxygen/build/fbcode_builder/CMake")
+
+# Proxygen's export refers to the historical un-namespaced c-ares target.
+# Resolve the installed package and provide the alias expected by that export.
+find_package(c-ares CONFIG REQUIRED)
+add_library(cares ALIAS c-ares::cares)
+
+find_package(proxygen CONFIG REQUIRED)
+
+if(SERVER_TARGET STREQUAL "coro")
+  add_executable(arena-server src/CoroServer.cpp)
+  target_link_libraries(
+    arena-server
+    PRIVATE
+      proxygen::proxygen
+      proxygen::proxygen_coro
+      proxygen::proxygen_coro_server
+      proxygen::proxygen_http_coro_filters_compression_filter_factory
+      Folly::folly_init_init
+      Folly::folly_portability_gflags
+  )
+elseif(SERVER_TARGET STREQUAL "classic")
+  add_executable(arena-server src/ClassicServer.cpp src/ArenaHQServer.cpp)
+  target_link_libraries(
+    arena-server
+    PRIVATE
+      proxygen::proxygen
+      proxygen::proxygenhttpserver
+      proxygen::proxygen_hq_samples
+      proxygen::proxygen_hq_server
+      proxygen::proxygen_transport_persistent_quic_psk_cache
+      proxygen::proxygen_httpserver
+      Folly::folly_init_init
+      Folly::folly_portability_gflags
+  )
+else()
+  message(FATAL_ERROR "SERVER_TARGET must be 'classic' or 'coro', got '${SERVER_TARGET}'")
+endif()
+
+target_include_directories(arena-server PRIVATE src)
+target_compile_options(arena-server PRIVATE -Wall -Wextra -Wpedantic)

frameworks/proxygen/Dockerfile

@@ -0,0 +1,66 @@
+# Shared image for both HttpArena Proxygen entries.
+#
+#   TARGET=classic -> proxygen::HTTPServer + mvfst HQServer   (`proxygen`)
+#   TARGET=coro    -> proxygen::coro::HTTPServer              (`proxygen-coro`)
+#
+# `frameworks/proxygen-coro/build.sh` builds this same context with
+# `--build-arg TARGET=coro`, so every fix to src/ lands in both entries at once.
+#
+# PROXYGEN_BASE is pinned by digest so a benchmark round is reproducible; bump
+# it deliberately rather than tracking a moving `:latest`.
+ARG PROXYGEN_BASE=ghcr.io/facebook/proxygen/base@sha256:3f9c745ea7a3e065bdf6887f7255f2547c17c67fc1210ff120e9e52daa8eb3dc
+ARG MIMALLOC_TAG=v2.1.7
+
+# glibc malloc is the bottleneck on the allocation-heavy profiles: the JSON
+# routes build a live folly::dynamic per request (up to 50 item copies plus the
+# serialized string), and proxygen's coroutine server allocates a frame per
+# event. Measured on this workload, mimalloc is worth +35% on `json` and
+# +39%/+88% (classic/coro) on `json-comp`, against roughly -5% on `static`,
+# where the large per-file IOBuf hits mimalloc's large-object path.
+FROM ${PROXYGEN_BASE} AS mimalloc
+ARG MIMALLOC_TAG
+RUN git clone --depth 1 -b "${MIMALLOC_TAG}" https://github.com/microsoft/mimalloc.git /src/mimalloc \
+    && cmake -S /src/mimalloc -B /src/build -DCMAKE_BUILD_TYPE=Release -DMI_BUILD_TESTS=OFF \
+    && cmake --build /src/build --parallel "$(nproc)" \
+    && cp "$(readlink -f /src/build/libmimalloc.so)" /src/libmimalloc.so
+
+FROM ${PROXYGEN_BASE} AS build
+ARG TARGET=classic
+
+WORKDIR /arena
+COPY CMakeLists.txt ./
+COPY src ./src
+RUN cmake -S . -B build -DCMAKE_BUILD_TYPE=Release -DSERVER_TARGET="${TARGET}" \
+    && cmake --build build --parallel "$(nproc)" \
+    && strip build/arena-server
+
+# Follow Proxygen's quic-interop image pattern: preserve the resolved shared
+# library paths, then copy only those libraries and the server binary to
+# a same-distro runtime image.
+RUN set -eux; \
+    ldd build/arena-server \
+      | awk '/=> \// { print $3 } /^\// { print $1 }' | sort -u > /tmp/runtime-libs.txt; \
+    tar -chf /tmp/runtime-libs.tar --files-from=/tmp/runtime-libs.txt
+
+FROM ubuntu:24.04@sha256:019e8eb29a85e74d64925745884f2ec79aa27e3feab36353d24656f4d6b89467
+
+ENV LD_LIBRARY_PATH=/opt/proxygen/lib \
+    LD_PRELOAD=/usr/local/lib/libmimalloc.so \
+    MIMALLOC_PURGE_DELAY=1
+
+COPY --from=build /tmp/runtime-libs.tar /tmp/runtime-libs.tar
+RUN tar -xf /tmp/runtime-libs.tar -C / \
+    && rm /tmp/runtime-libs.tar
+
+COPY --from=mimalloc /src/libmimalloc.so /usr/local/lib/libmimalloc.so
+COPY --from=build /arena/build/arena-server /usr/local/bin/arena-server
+COPY entrypoint.sh /usr/local/bin/arena-entrypoint
+RUN chmod +x /usr/local/bin/arena-entrypoint \
+    && groupadd --system --gid 10001 httparena \
+    && useradd --system --uid 10001 --gid httparena --no-create-home \
+        --home-dir /nonexistent --shell /usr/sbin/nologin httparena
+
+EXPOSE 8080/tcp 8081/tcp 8082/tcp 8443/tcp 8443/udp
+
+USER httparena
+ENTRYPOINT ["/usr/local/bin/arena-entrypoint"]

frameworks/proxygen/README.md

@@ -0,0 +1,108 @@
+# Proxygen
+
+[Meta's Proxygen](https://github.com/facebook/proxygen) serving every protocol
+HttpArena exercises, using Proxygen's callback server APIs:
+
+- `proxygen::HTTPServer` on TCP 8080 for HTTP/1.1 and RFC 6455 WebSocket
+  upgrades, TCP 8081 for HTTP/1.1 over TLS (ALPN `http/1.1` only), TCP 8082 for
+  prior-knowledge h2c, and TCP 8443 with TLS/ALPN `h2`.
+- Proxygen's mvfst-backed `HQServer` with ALPN `h3` on UDP 8443.
+
+Both server APIs run in one process, each with its own affinity-aware I/O pool,
+so whichever transport is being benchmarked can use the full CPU set while the
+other pool sleeps. Size them independently with `PROXYGEN_THREADS` and
+`PROXYGEN_H3_THREADS` (`0` = available CPUs).
+
+| Listener | Endpoints | Subscribed profiles |
+| --- | --- | --- |
+| HTTP/1.1 `:8080` | `/baseline11`, `/pipeline`, `/json/{count}`, `/upload`, `/static/*`, `/ws` | `baseline`, `pipelined`, `limited-conn`, `json-comp`, `echo-ws`, `echo-ws-pipeline`, `echo-ws-limited` |
+| HTTP/1.1 TLS `:8081` | `/json/{count}`, `/static/*` | `json-tls`, `static-tls` |
+| h2c `:8082` | `/baseline2`, `/json/{count}` | `baseline-h2c`, `json-h2c` |
+| HTTP/2 TLS `:8443` | `/baseline2`, `/static/*` | `baseline-h2`, `static-h2` |
+| HTTP/3 QUIC `:8443` | `/baseline2`, `/static/*` | `baseline-h3`, `static-h3` |
+
+The JSON routes load the immutable dataset once, then build each requested slice
+and its derived `total` fields per request and serialize the live object with
+Folly. Uploads count bytes delivered through the body callbacks rather than
+trusting `Content-Length`.
+
+## Static assets
+
+Files are read from the mounted directory on every request — one `open`, one
+`fstat`, one `readFull` straight into the response buffer, no intermediate
+`std::string`. There is no cache: the arena's static rules require the served
+bytes to follow the disk, and explicitly exclude a cache assembled in the entry
+("no reading the directory into a map at startup"). `validate.sh` enforces this
+with a staleness probe that swaps the file underneath a running server.
+
+The precompressed `.br`/`.gz` siblings are served when the client accepts them,
+which the rules allow "by selecting the variant in the entry off
+`Accept-Encoding`" — those bytes already exist on disk, so choosing one is a
+file read rather than compression. Selection is a delimited token match that
+honours `q=0`, preferring brotli, then gzip, then the byte-exact original,
+which is what a client sending no `Accept-Encoding` always gets.
+
+That removes the on-the-fly gzip of every CSS/JS/HTML response, which was the
+dominant cost in the static profiles.
+
+Response compression via Proxygen's `CompressionFilter` is therefore scoped to
+`application/json`, the only content type still compressed at request time and
+the one `json-comp` is scored on. The gzip level is Proxygen's default
+(`Z_DEFAULT_COMPRESSION`); level 9 measured as a net loss, trading 2.3% of
+throughput for 0.35% smaller bodies under the `(minBpr/myBpr)²` scoring.
+
+The WebSocket handler uses Proxygen's upgrade handshake (including its
+per-connection `Sec-WebSocket-Accept` calculation) and implements incremental
+RFC 6455 frame parsing: client frames are unmasked before text or binary data is
+echoed, and fragmented messages, multiple frames per read, ping/pong and close
+frames are all handled. This entry claims WebSocket support over HTTP/1.1 only,
+which is the protocol HttpArena's WebSocket profiles exercise.
+
+## Shared source tree
+
+`frameworks/proxygen` is the build context for **both** Proxygen entries. The
+`TARGET` build arg picks the server, the same way `sark-h3` reuses `sark`:
+
+| Entry | Build | Server API |
+| --- | --- | --- |
+| `proxygen` | `docker build frameworks/proxygen` | `src/ClassicServer.cpp` + `src/ArenaHQServer.cpp` |
+| `proxygen-coro` | `frameworks/proxygen-coro/build.sh` (`--build-arg TARGET=coro`) | `src/CoroServer.cpp` |
+
+`src/ArenaCommon.h` holds the routing, parsing, static-asset and validation
+helpers both servers share, so a fix lands in both entries at once.
+
+## Allocator
+
+Both images `LD_PRELOAD` mimalloc (pinned by tag in the Dockerfile). glibc malloc
+is the binding constraint on the allocation-heavy profiles — the JSON routes build
+a live `folly::dynamic` per request (up to 50 item copies plus the serialized
+string), and the coroutine server allocates a frame per event. Measured here it is
+worth roughly +41%/+82% (classic/coro) on `json-comp`.
+`MIMALLOC_PURGE_DELAY=1` is set because the default (10 ms) retains ~17% more
+memory on `json-comp` at 16384 connections for no throughput gain; going to 0
+would cut memory 4x further but costs 44% of that profile, so it is not used.
+
+## Image
+
+The builder tracks `ghcr.io/facebook/proxygen/base`, pinned by digest in
+`ARG PROXYGEN_BASE` so a benchmark round is reproducible — bump it deliberately
+rather than tracking `:latest`. The final stage contains only the Arena binary
+and the shared libraries `ldd` resolves for it, on a digest-pinned Ubuntu 24.04,
+running as the unprivileged `httparena` user (UID/GID 10001).
+
+Note that the prebuilt Proxygen, folly, fizz, wangle and mvfst libraries in the
+base image are built `RelWithDebInfo` (`-O2 -g -DNDEBUG`): `getdeps.py` defaults
+`--build-type` to `RelWithDebInfo` and Proxygen's `docker/base.Dockerfile` does
+not override it. Only this repository's translation unit is `-O3`.
+
+HttpArena mounts `/certs/server.crt`, `/certs/server.key`, `/data/dataset.json`
+and `/data/static` at runtime; no benchmark data is baked into the image.
+
+## Local validation
+
+From the repository root, on a host where the standard ports are free:
+
+```bash
+./scripts/validate.sh proxygen
+./scripts/benchmark.sh proxygen baseline
+```

frameworks/proxygen/entrypoint.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Shared by the `proxygen` and `proxygen-coro` images; the binary is the same
+# name in both, only the server API compiled into it differs.
+#
+# PROXYGEN_THREADS    TCP (H1/H2) I/O threads, 0 = available CPUs
+# PROXYGEN_H3_THREADS QUIC I/O threads, 0 = available CPUs
+exec /usr/local/bin/arena-server \
+    --ip=:: \
+    --http_port=8080 \
+    --tls_port=8081 \
+    --h2c_port=8082 \
+    --h2_port=8443 \
+    --h3_port=8443 \
+    --cert=/certs/server.crt \
+    --key=/certs/server.key \
+    --threads="${PROXYGEN_THREADS:-0}" \
+    --h3_threads="${PROXYGEN_H3_THREADS:-0}"

frameworks/proxygen/meta.json

@@ -0,0 +1,27 @@
+{
+  "display_name": "proxygen",
+  "language": "C++",
+  "type": "engine",
+  "engine": "proxygen",
+  "description": "Meta's Proxygen HTTP engine: HTTPServer for HTTP/1.1, HTTP/1.1 TLS, h2c, HTTP/2 TLS, and RFC 6455 WebSockets, plus the mvfst-backed HQ server for HTTP/3 over QUIC.",
+  "repo": "https://github.com/facebook/proxygen",
+  "enabled": true,
+  "tests": [
+    "baseline",
+    "json-comp",
+    "json-tls",
+    "static-tls",
+    "pipelined",
+    "limited-conn",
+    "baseline-h2",
+    "baseline-h2c",
+    "json-h2c",
+    "static-h2",
+    "baseline-h3",
+    "static-h3",
+    "echo-ws",
+    "echo-ws-pipeline",
+    "echo-ws-limited"
+  ],
+  "maintainers": []
+}