From 6f9d5d6c75a0dd74ce9b6178032b267d046bf488 Mon Sep 17 00:00:00 2001 From: Luca Niccolini Date: Sat, 8 Aug 2026 06:32:17 -0700 Subject: [PATCH 1/4] Add Proxygen engine Add a pinned, non-root Proxygen image with HTTP/1.1, TLS, h2c, HTTP/2, HTTP/3, static, JSON, upload, and RFC 6455 WebSocket support across 18 benchmark profiles. --- frameworks/proxygen/ArenaCommon.h | 214 ++++++++ frameworks/proxygen/ArenaHQServer.cpp | 72 +++ frameworks/proxygen/ArenaHQServer.h | 38 ++ frameworks/proxygen/ArenaHttpServer.cpp | 694 ++++++++++++++++++++++++ frameworks/proxygen/CMakeLists.txt | 33 ++ frameworks/proxygen/Dockerfile | 35 ++ frameworks/proxygen/README.md | 72 +++ frameworks/proxygen/entrypoint.sh | 13 + frameworks/proxygen/meta.json | 30 + site/data/frameworks.json | 7 + 10 files changed, 1208 insertions(+) create mode 100644 frameworks/proxygen/ArenaCommon.h create mode 100644 frameworks/proxygen/ArenaHQServer.cpp create mode 100644 frameworks/proxygen/ArenaHQServer.h create mode 100644 frameworks/proxygen/ArenaHttpServer.cpp create mode 100644 frameworks/proxygen/CMakeLists.txt create mode 100644 frameworks/proxygen/Dockerfile create mode 100644 frameworks/proxygen/README.md create mode 100644 frameworks/proxygen/entrypoint.sh create mode 100644 frameworks/proxygen/meta.json diff --git a/frameworks/proxygen/ArenaCommon.h b/frameworks/proxygen/ArenaCommon.h new file mode 100644 index 000000000..3e4f9b32b --- /dev/null +++ b/frameworks/proxygen/ArenaCommon.h @@ -0,0 +1,214 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +namespace httparena { + +inline constexpr uint64_t kMaxWebSocketMessage = 16ULL * 1024 * 1024; +inline constexpr std::string_view kJsonPrefix = "/json/"; +inline constexpr std::string_view kStaticPrefix = "/static/"; +inline constexpr std::string_view kStaticRoot = "/data/static/"; + +inline bool parseInteger(std::string_view input, int64_t &value) { + while (!input.empty() && + std::isspace(static_cast(input.front()))) { + input.remove_prefix(1); + } + while (!input.empty() && + std::isspace(static_cast(input.back()))) { + input.remove_suffix(1); + } + if (input.empty()) { + return false; + } + const auto result = + std::from_chars(input.data(), input.data() + input.size(), value); + return result.ec == std::errc() && result.ptr == input.data() + input.size(); +} + +inline bool checkedAdd(int64_t lhs, int64_t rhs, int64_t &result) { +#if defined(__GNUC__) || defined(__clang__) + return !__builtin_add_overflow(lhs, rhs, &result); +#else + if ((rhs > 0 && lhs > std::numeric_limits::max() - rhs) || + (rhs < 0 && lhs < std::numeric_limits::min() - rhs)) { + return false; + } + result = lhs + rhs; + return true; +#endif +} + +inline bool checkedMultiply(int64_t lhs, int64_t rhs, int64_t &result) { +#if defined(__GNUC__) || defined(__clang__) + return !__builtin_mul_overflow(lhs, rhs, &result); +#else + if (lhs > 0) { + if ((rhs > 0 && lhs > std::numeric_limits::max() / rhs) || + (rhs < 0 && rhs < std::numeric_limits::min() / lhs)) { + return false; + } + } else if (lhs < 0) { + if ((rhs > 0 && lhs < std::numeric_limits::min() / rhs) || + (rhs < 0 && rhs < std::numeric_limits::max() / lhs)) { + return false; + } + } + result = lhs * rhs; + return true; +#endif +} + +inline std::string contentType(std::string_view name) { + const auto endsWith = [name](std::string_view suffix) { + return name.size() >= suffix.size() && + name.substr(name.size() - suffix.size()) == suffix; + }; + if (endsWith(".css")) { + return "text/css"; + } + if (endsWith(".js")) { + return "application/javascript"; + } + if (endsWith(".html")) { + return "text/html"; + } + if (endsWith(".json")) { + return "application/json"; + } + if (endsWith(".svg")) { + return "image/svg+xml"; + } + if (endsWith(".webp")) { + return "image/webp"; + } + if (endsWith(".woff2")) { + return "font/woff2"; + } + return "application/octet-stream"; +} + +inline std::shared_ptr loadDataset() { + std::ifstream input("/data/dataset.json", std::ios::binary); + if (!input) { + throw std::runtime_error("cannot open /data/dataset.json"); + } + std::string contents((std::istreambuf_iterator(input)), + std::istreambuf_iterator()); + auto dataset = folly::parseJson(contents); + if (!dataset.isArray() || dataset.size() < 50) { + throw std::runtime_error("/data/dataset.json must contain 50 items"); + } + return std::make_shared(std::move(dataset)); +} + +inline bool validWebSocketKey(std::string_view key) noexcept { + if (key.size() != 24) { + return false; + } + std::array decoded{}; + const auto result = folly::base64Decode(key, decoded.data()); + return result.is_success && result.o == decoded.data() + 16; +} + +inline bool validUtf8(const uint8_t *data, size_t size) noexcept { + const auto continuation = [](uint8_t byte) { + return byte >= 0x80 && byte <= 0xbf; + }; + + size_t index = 0; + while (index < size) { + const uint8_t first = data[index]; + if (first <= 0x7f) { + ++index; + continue; + } + if (first >= 0xc2 && first <= 0xdf) { + if (index + 1 >= size || !continuation(data[index + 1])) { + return false; + } + index += 2; + continue; + } + if (first == 0xe0) { + if (index + 2 >= size || data[index + 1] < 0xa0 || + data[index + 1] > 0xbf || !continuation(data[index + 2])) { + return false; + } + index += 3; + continue; + } + if ((first >= 0xe1 && first <= 0xec) || (first >= 0xee && first <= 0xef)) { + if (index + 2 >= size || !continuation(data[index + 1]) || + !continuation(data[index + 2])) { + return false; + } + index += 3; + continue; + } + if (first == 0xed) { + if (index + 2 >= size || data[index + 1] < 0x80 || + data[index + 1] > 0x9f || !continuation(data[index + 2])) { + return false; + } + index += 3; + continue; + } + if (first == 0xf0) { + if (index + 3 >= size || data[index + 1] < 0x90 || + data[index + 1] > 0xbf || !continuation(data[index + 2]) || + !continuation(data[index + 3])) { + return false; + } + index += 4; + continue; + } + if (first >= 0xf1 && first <= 0xf3) { + if (index + 3 >= size || !continuation(data[index + 1]) || + !continuation(data[index + 2]) || !continuation(data[index + 3])) { + return false; + } + index += 4; + continue; + } + if (first == 0xf4) { + if (index + 3 >= size || data[index + 1] < 0x80 || + data[index + 1] > 0x8f || !continuation(data[index + 2]) || + !continuation(data[index + 3])) { + return false; + } + index += 4; + continue; + } + return false; + } + return true; +} + +inline bool validUtf8(const std::vector &data) noexcept { + return validUtf8(data.data(), data.size()); +} + +inline bool validWebSocketCloseCode(uint16_t code) noexcept { + const bool definedProtocolCode = code >= 1000 && code <= 1014 && + code != 1004 && code != 1005 && code != 1006; + const bool applicationCode = code >= 3000 && code <= 4999; + return definedProtocolCode || applicationCode; +} + +} // namespace httparena diff --git a/frameworks/proxygen/ArenaHQServer.cpp b/frameworks/proxygen/ArenaHQServer.cpp new file mode 100644 index 000000000..1bccb09b6 --- /dev/null +++ b/frameworks/proxygen/ArenaHQServer.cpp @@ -0,0 +1,72 @@ +#include "ArenaHQServer.h" + +#include + +#include +#include +#include + +namespace { + +quic::samples::HQServerParams makeHQParams(size_t ioThreads) { + quic::samples::HQServerParams params; + params.serverThreads = ioThreads; + params.transportSettings.maxNumPTOs = 1000; + params.transportSettings.maxCwndInMss = quic::kLargeMaxCwndInMss; + params.transportSettings.batchingMode = + quic::QuicBatchingMode::BATCHING_MODE_GSO; + params.transportSettings.maxBatchSize = 48; + params.transportSettings.dataPathType = quic::DataPathType::ContinuousMemory; + params.transportSettings.writeConnectionDataPacketsLimit = 48; + return params; +} + +} // namespace + +namespace httparena { + +class ArenaHQServer::Impl final { +public: + Impl(const std::string &certificatePath, const std::string &privateKeyPath, + size_t ioThreads, HandlerProvider handlerProvider) + : server_(makeHQParams(ioThreads), std::move(handlerProvider), nullptr, + quic::samples::createFizzServerContext( + quic::samples::kDefaultSupportedAlpns, + fizz::server::ClientAuthMode::None, certificatePath, + privateKeyPath)) {} + + ~Impl() { stop(); } + + void start(const folly::SocketAddress &address) { + started_ = true; + server_.start(address); + server_.getAddress(); + } + + void stop() { + if (started_) { + server_.stop(); + started_ = false; + } + } + +private: + quic::samples::HQServer server_; + bool started_{false}; +}; + +ArenaHQServer::ArenaHQServer(const std::string &certificatePath, + const std::string &privateKeyPath, + size_t ioThreads, HandlerProvider handlerProvider) + : impl_(std::make_unique(certificatePath, privateKeyPath, ioThreads, + std::move(handlerProvider))) {} + +ArenaHQServer::~ArenaHQServer() = default; + +void ArenaHQServer::start(const folly::SocketAddress &address) { + impl_->start(address); +} + +void ArenaHQServer::stop() { impl_->stop(); } + +} // namespace httparena diff --git a/frameworks/proxygen/ArenaHQServer.h b/frameworks/proxygen/ArenaHQServer.h new file mode 100644 index 000000000..34c985b36 --- /dev/null +++ b/frameworks/proxygen/ArenaHQServer.h @@ -0,0 +1,38 @@ +#pragma once + +#include +#include +#include +#include + +#include + +namespace proxygen { +class HTTPMessage; +class HTTPTransactionHandler; +} // namespace proxygen + +namespace httparena { + +class ArenaHQServer final { +public: + using HandlerProvider = std::function; + + ArenaHQServer(const std::string &certificatePath, + const std::string &privateKeyPath, size_t ioThreads, + HandlerProvider handlerProvider); + ~ArenaHQServer(); + + ArenaHQServer(const ArenaHQServer &) = delete; + ArenaHQServer &operator=(const ArenaHQServer &) = delete; + + void start(const folly::SocketAddress &address); + void stop(); + +private: + class Impl; + std::unique_ptr impl_; +}; + +} // namespace httparena diff --git a/frameworks/proxygen/ArenaHttpServer.cpp b/frameworks/proxygen/ArenaHttpServer.cpp new file mode 100644 index 000000000..09d504575 --- /dev/null +++ b/frameworks/proxygen/ArenaHttpServer.cpp @@ -0,0 +1,694 @@ +#include "ArenaCommon.h" +#include "ArenaHQServer.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using folly::SocketAddress; +using proxygen::HTTPMessage; +using proxygen::HTTPMethod; +using proxygen::HTTPServer; +using proxygen::ProxygenError; +using proxygen::RequestHandler; +using proxygen::RequestHandlerChain; +using proxygen::RequestHandlerFactory; +using proxygen::ResponseBuilder; +using proxygen::UpgradeProtocol; + +DEFINE_int32(http_port, 8080, "HTTP/1.1 and WebSocket port"); +DEFINE_int32(tls_port, 8081, "HTTP/1.1 over TLS port"); +DEFINE_int32(h2c_port, 8082, "HTTP/2 cleartext port"); +DEFINE_int32(h2_port, 8443, "HTTP/2 over TLS port"); +DEFINE_int32(h3_port, 8443, "HTTP/3 over QUIC port"); +DEFINE_string(ip, "::", "Address on which to listen"); +DEFINE_string(cert, "/certs/server.crt", "TLS certificate path"); +DEFINE_string(key, "/certs/server.key", "TLS private-key path"); +DEFINE_int32(threads, 0, "I/O threads; 0 uses the available CPU count"); + +namespace { + +constexpr size_t kMaxRequestBody = 1024; +using httparena::checkedAdd; +using httparena::checkedMultiply; +using httparena::contentType; +using httparena::kJsonPrefix; +using httparena::kMaxWebSocketMessage; +using httparena::kStaticPrefix; +using httparena::kStaticRoot; +using httparena::loadDataset; +using httparena::parseInteger; +using httparena::validUtf8; +using httparena::validWebSocketCloseCode; +using httparena::validWebSocketKey; + +class ArenaHandler final : public RequestHandler { +public: + explicit ArenaHandler(std::shared_ptr dataset) + : dataset_(std::move(dataset)) {} + + void onRequest(std::unique_ptr request) noexcept override { + const auto path = request->getPath(); + const auto method = request->getMethod(); + + if (path == "/ws") { + route_ = Route::WebSocket; + const auto &headers = request->getHeaders(); + const auto &key = headers.getSingleOrEmpty("Sec-WebSocket-Key"); + const auto &version = headers.getSingleOrEmpty("Sec-WebSocket-Version"); + if (!method || *method != HTTPMethod::GET || + !request->isIngressWebsocketUpgrade() || + !validWebSocketKey(std::string_view(key.data(), key.size())) || + version != "13") { + ResponseBuilder(downstream_) + .status(426, "Upgrade Required") + .header("Content-Type", "text/plain") + .header("Sec-WebSocket-Version", "13") + .body("WebSocket upgrade required") + .sendWithEOM(); + responseFinished_ = true; + return; + } + + ResponseBuilder(downstream_) + .status(101, "Switching Protocols") + .setEgressWebsocketHeaders() + .send(); + websocketAccepted_ = true; + return; + } + + method_ = method.value_or(HTTPMethod::GET); + if (path == "/baseline11") { + route_ = Route::Baseline; + queryValid_ = parseInteger(request->getQueryParam("a"), a_) && + parseInteger(request->getQueryParam("b"), b_); + return; + } + if (path == "/baseline2") { + route_ = Route::BaselineH2; + queryValid_ = parseInteger(request->getQueryParam("a"), a_) && + parseInteger(request->getQueryParam("b"), b_); + return; + } + if (path.starts_with(kJsonPrefix)) { + route_ = Route::Json; + const std::string_view countText(path.data() + kJsonPrefix.size(), + path.size() - kJsonPrefix.size()); + int64_t count = 0; + const auto multiplierText = request->getQueryParam("m"); + const bool multiplierValid = + multiplierText.empty() ? (multiplier_ = 1, true) + : parseInteger(multiplierText, multiplier_); + jsonValid_ = parseInteger(countText, count) && count >= 1 && + count <= 50 && multiplierValid; + if (jsonValid_) { + jsonCount_ = static_cast(count); + } + return; + } + if (path == "/upload") { + route_ = Route::Upload; + return; + } + if (path.starts_with(kStaticPrefix)) { + route_ = Route::Static; + staticName_.assign(path.data() + kStaticPrefix.size(), + path.size() - kStaticPrefix.size()); + return; + } + if (path == "/pipeline") { + route_ = Route::Pipeline; + return; + } + route_ = Route::NotFound; + } + + void onBody(std::unique_ptr body) noexcept override { + if (!body || responseFinished_) { + return; + } + if (route_ == Route::Upload) { + const size_t bytes = body->computeChainDataLength(); + if (bytes > std::numeric_limits::max() - uploadBytes_) { + uploadValid_ = false; + } else { + uploadBytes_ += bytes; + } + return; + } + if (route_ == Route::WebSocket && websocketActive_) { + auto bytes = body->coalesce(); + websocketBytes_.insert(websocketBytes_.end(), bytes.begin(), bytes.end()); + processWebSocketFrames(); + return; + } + if (route_ != Route::Baseline) { + return; + } + auto bytes = body->coalesce(); + if (requestBody_.size() + bytes.size() > kMaxRequestBody) { + bodyValid_ = false; + return; + } + requestBody_.append(reinterpret_cast(bytes.data()), + bytes.size()); + } + + void onUpgrade(UpgradeProtocol /*protocol*/) noexcept override { + if (route_ != Route::WebSocket || !websocketAccepted_) { + downstream_->sendAbort(); + return; + } + websocketActive_ = true; + } + + void onEOM() noexcept override { + if (responseFinished_) { + return; + } + switch (route_) { + case Route::WebSocket: + responseFinished_ = true; + downstream_->sendEOM(); + return; + case Route::NotFound: + sendText(404, "Not Found", "not found"); + return; + case Route::Pipeline: + if (method_ != HTTPMethod::GET) { + sendText(405, "Method Not Allowed", "method not allowed"); + } else { + sendText(200, "OK", "ok"); + } + return; + case Route::Baseline: + handleBaseline(true); + return; + case Route::BaselineH2: + handleBaseline(false); + return; + case Route::Json: + handleJson(); + return; + case Route::Upload: + if (method_ != HTTPMethod::POST) { + sendText(405, "Method Not Allowed", "method not allowed"); + } else if (!uploadValid_) { + sendText(400, "Bad Request", "upload too large"); + } else { + sendText(200, "OK", std::to_string(uploadBytes_)); + } + return; + case Route::Static: + handleStatic(); + return; + } + } + + void requestComplete() noexcept override { delete this; } + + void onError(ProxygenError /*error*/) noexcept override { delete this; } + +private: + enum class Route { + NotFound, + Baseline, + BaselineH2, + Json, + Upload, + Static, + Pipeline, + WebSocket + }; + + void handleBaseline(bool allowPost) { + if (!queryValid_ || !bodyValid_) { + sendText(400, "Bad Request", "invalid integer"); + return; + } + if (method_ != HTTPMethod::GET && + (!allowPost || method_ != HTTPMethod::POST)) { + sendText(405, "Method Not Allowed", "method not allowed"); + return; + } + + int64_t sum = 0; + if (!checkedAdd(a_, b_, sum)) { + sendText(400, "Bad Request", "integer overflow"); + return; + } + if (method_ == HTTPMethod::POST) { + int64_t bodyValue = 0; + if (!parseInteger(requestBody_, bodyValue) || + !checkedAdd(sum, bodyValue, sum)) { + sendText(400, "Bad Request", "invalid integer"); + return; + } + } + sendText(200, "OK", std::to_string(sum)); + } + + void handleJson() { + if (method_ != HTTPMethod::GET) { + sendText(405, "Method Not Allowed", "method not allowed"); + return; + } + if (!jsonValid_ || jsonCount_ > dataset_->size()) { + sendText(400, "Bad Request", "invalid JSON parameters"); + return; + } + + try { + folly::dynamic items = folly::dynamic::array; + for (size_t index = 0; index < jsonCount_; ++index) { + folly::dynamic item = (*dataset_)[index]; + int64_t subtotal = 0; + int64_t total = 0; + if (!checkedMultiply(item["price"].asInt(), item["quantity"].asInt(), + subtotal) || + !checkedMultiply(subtotal, multiplier_, total)) { + sendText(400, "Bad Request", "integer overflow"); + return; + } + item["total"] = total; + items.push_back(std::move(item)); + } + folly::dynamic response = folly::dynamic::object; + response["items"] = std::move(items); + response["count"] = static_cast(jsonCount_); + sendResponse(200, "OK", "application/json", folly::toJson(response)); + } catch (const std::exception &) { + sendText(500, "Internal Server Error", "JSON serialization failed"); + } + } + + void handleStatic() { + if (method_ != HTTPMethod::GET) { + sendText(405, "Method Not Allowed", "method not allowed"); + return; + } + if (staticName_.empty() || staticName_.find('/') != std::string::npos || + staticName_.find('\\') != std::string::npos || + staticName_.find("..") != std::string::npos) { + sendText(404, "Not Found", "not found"); + return; + } + + std::ifstream input(std::string(kStaticRoot) + staticName_, + std::ios::binary); + if (!input) { + sendText(404, "Not Found", "not found"); + return; + } + std::string body((std::istreambuf_iterator(input)), + std::istreambuf_iterator()); + if (!input.good() && !input.eof()) { + sendText(500, "Internal Server Error", "read error"); + return; + } + sendResponse(200, "OK", contentType(staticName_), std::move(body)); + } + + void sendResponse(uint16_t status, const std::string &reason, + const std::string &type, std::string body) { + responseFinished_ = true; + ResponseBuilder(downstream_) + .status(status, reason) + .header("Content-Type", type) + .body(std::move(body)) + .sendWithEOM(); + } + + void sendText(uint16_t status, const std::string &reason, + const std::string &body) { + sendResponse(status, reason, "text/plain", body); + } + + void sendWebSocketFrame(uint8_t opcode, const uint8_t *payload, + size_t payloadLength) { + std::vector frame; + frame.reserve(payloadLength + 10); + frame.push_back(static_cast(0x80U | opcode)); + if (payloadLength <= 125) { + frame.push_back(static_cast(payloadLength)); + } else if (payloadLength <= std::numeric_limits::max()) { + frame.push_back(126); + frame.push_back(static_cast((payloadLength >> 8) & 0xff)); + frame.push_back(static_cast(payloadLength & 0xff)); + } else { + frame.push_back(127); + const auto length = static_cast(payloadLength); + for (int shift = 56; shift >= 0; shift -= 8) { + frame.push_back(static_cast((length >> shift) & 0xff)); + } + } + if (payloadLength > 0) { + frame.insert(frame.end(), payload, payload + payloadLength); + } + downstream_->sendBody(folly::IOBuf::copyBuffer(frame.data(), frame.size())); + } + + void sendWebSocketFrame(uint8_t opcode, const std::vector &payload) { + sendWebSocketFrame(opcode, payload.data(), payload.size()); + } + + void closeWebSocket(uint16_t status) { + if (responseFinished_ || closeSent_) { + return; + } + const std::array payload = { + static_cast((status >> 8) & 0xff), + static_cast(status & 0xff)}; + sendWebSocketFrame(0x8, payload.data(), payload.size()); + closeSent_ = true; + } + + void webSocketProtocolError() { closeWebSocket(1002); } + + void webSocketInvalidPayload() { closeWebSocket(1007); } + + void handleWebSocketFrame(bool fin, uint8_t opcode, + std::vector payload) { + if (closeSent_ && opcode != 0x08) { + return; + } + if ((opcode & 0x08U) != 0) { + if (!fin || payload.size() > 125) { + webSocketProtocolError(); + return; + } + if (opcode == 0x08) { + if (payload.size() == 1) { + webSocketProtocolError(); + return; + } + if (payload.size() >= 2) { + const uint16_t status = + (static_cast(payload[0]) << 8) | payload[1]; + if (!validWebSocketCloseCode(status)) { + webSocketProtocolError(); + return; + } + if (!validUtf8(payload.data() + 2, payload.size() - 2)) { + webSocketInvalidPayload(); + return; + } + } + if (closeSent_) { + responseFinished_ = true; + downstream_->sendEOM(); + return; + } + sendWebSocketFrame(0x08, payload); + responseFinished_ = true; + downstream_->sendEOM(); + } else if (opcode == 0x09) { + sendWebSocketFrame(0x0a, payload); + } else if (opcode != 0x0a) { + webSocketProtocolError(); + } + return; + } + + if (opcode == 0x00) { + if (fragmentOpcode_ == 0) { + webSocketProtocolError(); + return; + } + if (fragmentPayload_.size() + payload.size() > kMaxWebSocketMessage) { + webSocketProtocolError(); + return; + } + fragmentPayload_.insert(fragmentPayload_.end(), payload.begin(), + payload.end()); + if (fin) { + if (fragmentOpcode_ == 0x01 && !validUtf8(fragmentPayload_)) { + webSocketInvalidPayload(); + return; + } + sendWebSocketFrame(fragmentOpcode_, fragmentPayload_); + fragmentOpcode_ = 0; + fragmentPayload_.clear(); + } + return; + } + + if (opcode != 0x01 && opcode != 0x02) { + webSocketProtocolError(); + return; + } + if (fragmentOpcode_ != 0) { + webSocketProtocolError(); + return; + } + if (fin) { + if (opcode == 0x01 && !validUtf8(payload)) { + webSocketInvalidPayload(); + return; + } + sendWebSocketFrame(opcode, payload); + return; + } + fragmentOpcode_ = opcode; + fragmentPayload_ = std::move(payload); + } + + void processWebSocketFrames() { + size_t cursor = 0; + while (!responseFinished_) { + if (websocketBytes_.size() - cursor < 2) { + break; + } + const uint8_t first = websocketBytes_[cursor]; + const uint8_t second = websocketBytes_[cursor + 1]; + const bool fin = (first & 0x80U) != 0; + const uint8_t opcode = first & 0x0fU; + const uint8_t encodedPayloadLength = second & 0x7fU; + if ((first & 0x70U) != 0 || (second & 0x80U) == 0) { + webSocketProtocolError(); + break; + } + // RFC 6455 control frames cannot use either extended-length encoding, + // even when that encoding ultimately describes 125 bytes or fewer. + if ((opcode & 0x08U) != 0 && encodedPayloadLength > 125) { + webSocketProtocolError(); + break; + } + + uint64_t payloadLength = encodedPayloadLength; + size_t headerLength = 2; + if (payloadLength == 126) { + if (websocketBytes_.size() - cursor < 4) { + break; + } + payloadLength = + (static_cast(websocketBytes_[cursor + 2]) << 8) | + websocketBytes_[cursor + 3]; + if (payloadLength < 126) { + webSocketProtocolError(); + break; + } + headerLength = 4; + } else if (payloadLength == 127) { + if (websocketBytes_.size() - cursor < 10) { + break; + } + if ((websocketBytes_[cursor + 2] & 0x80U) != 0) { + webSocketProtocolError(); + break; + } + payloadLength = 0; + for (size_t index = 0; index < 8; ++index) { + payloadLength = + (payloadLength << 8) | websocketBytes_[cursor + 2 + index]; + } + if (payloadLength <= std::numeric_limits::max()) { + webSocketProtocolError(); + break; + } + headerLength = 10; + } + if (payloadLength > kMaxWebSocketMessage) { + webSocketProtocolError(); + break; + } + + constexpr size_t kMaskLength = 4; + if (payloadLength > + std::numeric_limits::max() - headerLength - kMaskLength) { + webSocketProtocolError(); + break; + } + const size_t frameLength = + headerLength + kMaskLength + static_cast(payloadLength); + if (websocketBytes_.size() - cursor < frameLength) { + break; + } + + const size_t maskOffset = cursor + headerLength; + const size_t payloadOffset = maskOffset + kMaskLength; + std::vector payload(static_cast(payloadLength)); + for (size_t index = 0; index < payload.size(); ++index) { + payload[index] = websocketBytes_[payloadOffset + index] ^ + websocketBytes_[maskOffset + (index % kMaskLength)]; + } + cursor += frameLength; + handleWebSocketFrame(fin, opcode, std::move(payload)); + } + + if (cursor > 0) { + websocketBytes_.erase(websocketBytes_.begin(), + websocketBytes_.begin() + cursor); + } + if (responseFinished_) { + websocketBytes_.clear(); + } + } + + Route route_{Route::NotFound}; + std::shared_ptr dataset_; + HTTPMethod method_{HTTPMethod::GET}; + int64_t a_{0}; + int64_t b_{0}; + int64_t multiplier_{1}; + size_t jsonCount_{0}; + size_t uploadBytes_{0}; + bool queryValid_{false}; + bool jsonValid_{false}; + bool uploadValid_{true}; + bool bodyValid_{true}; + bool websocketAccepted_{false}; + bool websocketActive_{false}; + bool closeSent_{false}; + bool responseFinished_{false}; + uint8_t fragmentOpcode_{0}; + std::string requestBody_; + std::string staticName_; + std::vector websocketBytes_; + std::vector fragmentPayload_; +}; + +class ArenaHandlerFactory final : public RequestHandlerFactory { +public: + explicit ArenaHandlerFactory(std::shared_ptr dataset) + : dataset_(std::move(dataset)) {} + + void onServerStart(folly::EventBase * /*eventBase*/) noexcept override {} + + void onServerStop() noexcept override {} + + RequestHandler *onRequest(RequestHandler *, HTTPMessage *) noexcept override { + return new ArenaHandler(dataset_); + } + +private: + std::shared_ptr dataset_; +}; + +wangle::SSLContextConfig h1TlsConfig() { + wangle::SSLContextConfig config; + config.isDefault = true; + config.clientVerification = + folly::SSLContext::VerifyClientCertificate::DO_NOT_REQUEST; + config.setCertificate(FLAGS_cert, FLAGS_key, ""); + config.setNextProtocols(std::list{"http/1.1"}); + return config; +} + +wangle::SSLContextConfig h2TlsConfig() { + wangle::SSLContextConfig config; + config.isDefault = true; + config.clientVerification = + folly::SSLContext::VerifyClientCertificate::DO_NOT_REQUEST; + config.setCertificate(FLAGS_cert, FLAGS_key, ""); + config.setNextProtocols(std::list{"h2"}); + return config; +} + +std::vector listenerConfigs() { + std::vector listeners; + listeners.emplace_back(SocketAddress(FLAGS_ip, FLAGS_http_port, true), + HTTPServer::Protocol::HTTP); + listeners.emplace_back(SocketAddress(FLAGS_ip, FLAGS_h2c_port, true), + HTTPServer::Protocol::HTTP2); + + HTTPServer::IPConfig tlsListener( + SocketAddress(FLAGS_ip, FLAGS_tls_port, true), + HTTPServer::Protocol::HTTP); + tlsListener.sslConfigs.push_back(h1TlsConfig()); + listeners.push_back(std::move(tlsListener)); + + HTTPServer::IPConfig h2Listener(SocketAddress(FLAGS_ip, FLAGS_h2_port, true), + HTTPServer::Protocol::HTTP2); + h2Listener.sslConfigs.push_back(h2TlsConfig()); + listeners.push_back(std::move(h2Listener)); + return listeners; +} + +} // namespace + +int main(int argc, char *argv[]) { + const folly::Init init(&argc, &argv, true); + + if (FLAGS_threads <= 0) { + FLAGS_threads = static_cast(folly::available_concurrency()); + } + CHECK_GT(FLAGS_threads, 0); + + try { + auto dataset = loadDataset(); + + proxygen::HTTPServerOptions options; + options.threads = static_cast(FLAGS_threads); + options.idleTimeout = std::chrono::milliseconds(60000); + options.shutdownOn = {SIGINT, SIGTERM}; + options.supportsConnect = true; + options.enableContentCompression = true; + options.initialReceiveWindow = 1U << 20; + options.receiveStreamWindowSize = 1U << 20; + options.receiveSessionWindowSize = 10U << 20; + options.maxConcurrentIncomingStreams = 1024; + options.handlerFactories = + RequestHandlerChain().addThen(dataset).build(); + + httparena::ArenaHQServer h3Server( + FLAGS_cert, FLAGS_key, static_cast(FLAGS_threads), + [dataset](HTTPMessage *) -> proxygen::HTTPTransactionHandler * { + return new proxygen::RequestHandlerAdaptor(new ArenaHandler(dataset)); + }); + HTTPServer server(std::move(options)); + server.bind(listenerConfigs()); + h3Server.start(SocketAddress(FLAGS_ip, FLAGS_h3_port, true)); + server.start(); + h3Server.stop(); + } catch (const std::exception &error) { + std::cerr << "failed to start Proxygen HttpArena server: " << error.what() + << '\n'; + return 1; + } + return 0; +} diff --git a/frameworks/proxygen/CMakeLists.txt b/frameworks/proxygen/CMakeLists.txt new file mode 100644 index 000000000..35df1fa42 --- /dev/null +++ b/frameworks/proxygen/CMakeLists.txt @@ -0,0 +1,33 @@ +cmake_minimum_required(VERSION 3.20) + +project(httparena-proxygen LANGUAGES CXX) + +set(CMAKE_CXX_STANDARD 20) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_CXX_EXTENSIONS OFF) + +# Proxygen's exported Fizz package calls find_dependency(Sodium). The official +# builder image keeps that upstream find module with the Proxygen source tree. +list(APPEND CMAKE_MODULE_PATH "/proxygen/build/fbcode_builder/CMake") + +# Proxygen's export refers to the historical un-namespaced c-ares target. +# Resolve the installed package and provide the alias expected by that export. +find_package(c-ares CONFIG REQUIRED) +add_library(cares ALIAS c-ares::cares) + +find_package(proxygen CONFIG REQUIRED) + +add_executable(proxygen-arena ArenaHttpServer.cpp ArenaHQServer.cpp) +target_compile_options(proxygen-arena PRIVATE -Wall -Wextra -Wpedantic) +target_link_libraries( + proxygen-arena + PRIVATE + proxygen::proxygen + proxygen::proxygenhttpserver + proxygen::proxygen_hq_samples + proxygen::proxygen_hq_server + proxygen::proxygen_transport_persistent_quic_psk_cache + proxygen::proxygen_httpserver + Folly::folly_init_init + Folly::folly_portability_gflags +) diff --git a/frameworks/proxygen/Dockerfile b/frameworks/proxygen/Dockerfile new file mode 100644 index 000000000..ac4a0b30f --- /dev/null +++ b/frameworks/proxygen/Dockerfile @@ -0,0 +1,35 @@ +FROM ghcr.io/facebook/proxygen/base:latest AS build + +WORKDIR /arena +COPY CMakeLists.txt ArenaCommon.h ArenaHQServer.h ArenaHttpServer.cpp ArenaHQServer.cpp ./ +RUN cmake -S . -B build -DCMAKE_BUILD_TYPE=Release \ + && cmake --build build --parallel "$(nproc)" \ + && strip build/proxygen-arena + +# Follow Proxygen's quic-interop image pattern: preserve the resolved shared +# library paths, then copy only those libraries and the server binary to +# a same-distro runtime image. +RUN set -eux; \ + ldd build/proxygen-arena \ + | awk '/=> \// { print $3 } /^\// { print $1 }' | sort -u > /tmp/runtime-libs.txt; \ + tar -chf /tmp/runtime-libs.tar --files-from=/tmp/runtime-libs.txt + +FROM ubuntu:24.04@sha256:019e8eb29a85e74d64925745884f2ec79aa27e3feab36353d24656f4d6b89467 + +ENV LD_LIBRARY_PATH=/opt/proxygen/lib + +COPY --from=build /tmp/runtime-libs.tar /tmp/runtime-libs.tar +RUN tar -xf /tmp/runtime-libs.tar -C / \ + && rm /tmp/runtime-libs.tar + +COPY --from=build /arena/build/proxygen-arena /usr/local/bin/proxygen-arena +COPY entrypoint.sh /usr/local/bin/proxygen-entrypoint +RUN chmod +x /usr/local/bin/proxygen-entrypoint \ + && groupadd --system --gid 10001 httparena \ + && useradd --system --uid 10001 --gid httparena --no-create-home \ + --home-dir /nonexistent --shell /usr/sbin/nologin httparena + +EXPOSE 8080/tcp 8081/tcp 8082/tcp 8443/tcp 8443/udp + +USER httparena +ENTRYPOINT ["/usr/local/bin/proxygen-entrypoint"] diff --git a/frameworks/proxygen/README.md b/frameworks/proxygen/README.md new file mode 100644 index 000000000..8efe071b5 --- /dev/null +++ b/frameworks/proxygen/README.md @@ -0,0 +1,72 @@ +# Proxygen + +This engine entry uses [Meta's Proxygen](https://github.com/facebook/proxygen) +for every advertised protocol: + +- Proxygen `HTTPServer` listens on TCP port 8080 for HTTP/1.1 and RFC 6455 + WebSocket upgrades, on TCP port 8081 for HTTP/1.1 over TLS (ALPN + `http/1.1` only), on TCP port 8082 for prior-knowledge h2c, and with + TLS/ALPN `h2` on TCP port 8443. +- Proxygen's mvfst-backed `HQServer` listens with ALPN `h3` on UDP port 8443. + +Both server APIs run in one process. TCP and QUIC each retain an +affinity-aware I/O pool so whichever transport is being benchmarked can use +the full CPU set while the other pool sleeps. The HTTP/2 and HTTP/3 listeners +use Proxygen's benchmark-oriented flow-control, stream-concurrency, GSO +batching, and write-path settings. + +| Listener | Endpoints | Subscribed profiles | +| --- | --- | --- | +| HTTP/1.1 `:8080` | `/baseline11`, `/pipeline`, `/json/{count}`, `/upload`, `/static/*`, `/ws` | `baseline`, `pipelined`, `limited-conn`, `json`, `json-comp`, `upload`, `static`, `echo-ws`, `echo-ws-pipeline`, `echo-ws-limited` | +| HTTP/1.1 TLS `:8081` | `/json/{count}`, `/static/*` | `json-tls`, `static-tls` | +| h2c `:8082` | `/baseline2`, `/json/{count}` | `baseline-h2c`, `json-h2c` | +| HTTP/2 TLS `:8443` | `/baseline2`, `/static/*` | `baseline-h2`, `static-h2` | +| HTTP/3 QUIC `:8443` | `/baseline2`, `/static/*` | `baseline-h3`, `static-h3` | + +The JSON routes load the immutable dataset once, build each requested slice +and derived `total` fields per request, and serialize the live object with +Folly. Proxygen's standard content-compression path provides conditional gzip +for clients that advertise it. Uploads count bytes delivered through the body +callbacks rather than trusting `Content-Length`; static files are read from +disk for each request. + +The WebSocket handler uses Proxygen's upgrade handshake (including its +per-connection `Sec-WebSocket-Accept` calculation) and implements incremental +RFC 6455 frame parsing. Client frames are unmasked before text or binary data +is echoed; fragmented messages, multiple frames per read, ping/pong, and close +frames are handled explicitly. This entry only claims WebSocket support over +HTTP/1.1, which is the protocol HttpArena's WebSocket profiles exercise. + +## Upstream image and Docker build + +The builder tracks the official `ghcr.io/facebook/proxygen/base:latest` image. +The final stage contains only the Arena binary and its resolved shared +libraries; its matching Ubuntu 24.04 runtime is pinned by digest. The final +image runs the servers as the unprivileged `httparena` user (UID/GID 10001). + +The build and launch arrangement follows Proxygen's upstream container and +coroutine benchmark patterns: + +- `Dockerfile`: build in a full Proxygen environment, discover runtime + libraries with `ldd`, and copy them into a same-distribution runtime image. +- `HTTPCoroBenchmark.cpp`: configure QUIC with GSO batching, continuous-memory + writes, a large congestion window, and a 48-packet write batch. + +HttpArena mounts `/certs/server.crt`, `/certs/server.key`, +`/data/dataset.json`, and `/data/static` at runtime. No benchmark data is baked +into the image. + +## Local validation + +From the repository root on a host where the standard ports are free: + +```bash +./scripts/validate.sh proxygen +``` + +The full benchmark driver is required for the complete 18-profile metadata set +(the lite driver currently rejects profiles it does not know about): + +```bash +LOADGEN_DOCKER=true SKIP_TUNE=true ./scripts/benchmark.sh proxygen --save +``` diff --git a/frameworks/proxygen/entrypoint.sh b/frameworks/proxygen/entrypoint.sh new file mode 100644 index 000000000..14a667c06 --- /dev/null +++ b/frameworks/proxygen/entrypoint.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +set -euo pipefail + +exec /usr/local/bin/proxygen-arena \ + --ip=:: \ + --http_port=8080 \ + --tls_port=8081 \ + --h2c_port=8082 \ + --h2_port=8443 \ + --h3_port=8443 \ + --cert=/certs/server.crt \ + --key=/certs/server.key \ + --threads="${PROXYGEN_THREADS:-0}" diff --git a/frameworks/proxygen/meta.json b/frameworks/proxygen/meta.json new file mode 100644 index 000000000..f4b718417 --- /dev/null +++ b/frameworks/proxygen/meta.json @@ -0,0 +1,30 @@ +{ + "display_name": "proxygen", + "language": "C++", + "type": "engine", + "engine": "proxygen", + "description": "Meta's Proxygen HTTP engine: HTTPServer for HTTP/1.1, HTTP/1.1 TLS, h2c, HTTP/2 TLS, and RFC 6455 WebSockets, plus the mvfst-backed HQ server for HTTP/3 over QUIC.", + "repo": "https://github.com/facebook/proxygen", + "enabled": true, + "tests": [ + "baseline", + "json", + "json-comp", + "json-tls", + "upload", + "static", + "static-tls", + "pipelined", + "limited-conn", + "baseline-h2", + "baseline-h2c", + "json-h2c", + "static-h2", + "baseline-h3", + "static-h3", + "echo-ws", + "echo-ws-pipeline", + "echo-ws-limited" + ], + "maintainers": [] +} diff --git a/site/data/frameworks.json b/site/data/frameworks.json index ac63585fc..668857ea8 100644 --- a/site/data/frameworks.json +++ b/site/data/frameworks.json @@ -1286,6 +1286,13 @@ "response": true } }, + "proxygen": { + "dir": "proxygen", + "description": "Meta's Proxygen HTTP engine: HTTPServer for HTTP/1.1, HTTP/1.1 TLS, h2c, HTTP/2 TLS, and RFC 6455 WebSockets, plus the mvfst-backed HQ server for HTTP/3 over QUIC.", + "repo": "https://github.com/facebook/proxygen", + "type": "engine", + "engine": "proxygen" + }, "pyronova": { "dir": "pyronova", "description": "Pyronova \u2014 Python web framework with a Rust core (hyper + tokio + rustls + mimalloc) and PEP 684 sub-interpreter workers for true multi-core parallelism. Opt-in features: gzip/brotli compression, rustls TLS with h2/h1 ALPN, streaming body ingest, async Postgres via sqlx::PgPool. Handlers are standard Python functions routed via decorators.", From ce61714d5f71dda54c796c7fc3fc5d07757428a6 Mon Sep 17 00:00:00 2001 From: Luca Niccolini Date: Sat, 8 Aug 2026 07:18:22 -0700 Subject: [PATCH 2/4] Add Proxygen coroutine engine Add a pinned, non-root Proxygen coroutine image using proxygen::coro::HTTPHandler and HTTPSourceHolder across the same 18 HTTP/1.1, TLS, h2c, HTTP/2, HTTP/3, static, JSON, upload, and WebSocket profiles. --- frameworks/proxygen-coro/ArenaCoroServer.cpp | 974 +++++++++++++++++++ frameworks/proxygen-coro/CMakeLists.txt | 28 + frameworks/proxygen-coro/Dockerfile | 32 + frameworks/proxygen-coro/README.md | 52 + frameworks/proxygen-coro/entrypoint.sh | 13 + frameworks/proxygen-coro/meta.json | 30 + site/data/frameworks.json | 7 + 7 files changed, 1136 insertions(+) create mode 100644 frameworks/proxygen-coro/ArenaCoroServer.cpp create mode 100644 frameworks/proxygen-coro/CMakeLists.txt create mode 100644 frameworks/proxygen-coro/Dockerfile create mode 100644 frameworks/proxygen-coro/README.md create mode 100755 frameworks/proxygen-coro/entrypoint.sh create mode 100644 frameworks/proxygen-coro/meta.json diff --git a/frameworks/proxygen-coro/ArenaCoroServer.cpp b/frameworks/proxygen-coro/ArenaCoroServer.cpp new file mode 100644 index 000000000..ed3b65cf2 --- /dev/null +++ b/frameworks/proxygen-coro/ArenaCoroServer.cpp @@ -0,0 +1,974 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +DEFINE_int32(http_port, 8080, "HTTP/1.1 and WebSocket port"); +DEFINE_int32(tls_port, 8081, "HTTP/1.1 over TLS port"); +DEFINE_int32(h2c_port, 8082, "HTTP/2 cleartext port"); +DEFINE_int32(h2_port, 8443, "HTTP/2 over TLS port"); +DEFINE_int32(h3_port, 8443, "HTTP/3 over QUIC port"); +DEFINE_string(ip, "::", "Address on which to listen"); +DEFINE_string(cert, "/certs/server.crt", "TLS certificate path"); +DEFINE_string(key, "/certs/server.key", "TLS private-key path"); +DEFINE_int32(threads, 0, "I/O threads; 0 uses the available CPU count"); + +namespace { + +using proxygen::HTTPMessage; +using proxygen::HTTPMethod; +using proxygen::coro::HTTPBodyEvent; +using proxygen::coro::HTTPError; +using proxygen::coro::HTTPErrorCode; +using proxygen::coro::HTTPFixedSource; +using proxygen::coro::HTTPHandler; +using proxygen::coro::HTTPHeaderEvent; +using proxygen::coro::HTTPServer; +using proxygen::coro::HTTPSessionContextPtr; +using proxygen::coro::HTTPSource; +using proxygen::coro::HTTPSourceHolder; +using proxygen::coro::TimedBaton; + +constexpr size_t kMaxBaselineBody = 1024; +constexpr uint64_t kMaxWebSocketMessage = 16ULL * 1024 * 1024; +constexpr std::string_view kJsonPrefix = "/json/"; +constexpr std::string_view kStaticPrefix = "/static/"; +constexpr std::string_view kStaticRoot = "/data/static/"; + +bool parseInteger(std::string_view input, int64_t &value) { + while (!input.empty() && + std::isspace(static_cast(input.front()))) { + input.remove_prefix(1); + } + while (!input.empty() && + std::isspace(static_cast(input.back()))) { + input.remove_suffix(1); + } + if (input.empty()) { + return false; + } + const auto result = + std::from_chars(input.data(), input.data() + input.size(), value); + return result.ec == std::errc() && result.ptr == input.data() + input.size(); +} + +bool checkedAdd(int64_t lhs, int64_t rhs, int64_t &result) { +#if defined(__GNUC__) || defined(__clang__) + return !__builtin_add_overflow(lhs, rhs, &result); +#else + if ((rhs > 0 && lhs > std::numeric_limits::max() - rhs) || + (rhs < 0 && lhs < std::numeric_limits::min() - rhs)) { + return false; + } + result = lhs + rhs; + return true; +#endif +} + +bool checkedMultiply(int64_t lhs, int64_t rhs, int64_t &result) { +#if defined(__GNUC__) || defined(__clang__) + return !__builtin_mul_overflow(lhs, rhs, &result); +#else + if (lhs > 0) { + if ((rhs > 0 && lhs > std::numeric_limits::max() / rhs) || + (rhs < 0 && rhs < std::numeric_limits::min() / lhs)) { + return false; + } + } else if (lhs < 0) { + if ((rhs > 0 && lhs < std::numeric_limits::min() / rhs) || + (rhs < 0 && rhs < std::numeric_limits::max() / lhs)) { + return false; + } + } + result = lhs * rhs; + return true; +#endif +} + +std::string contentType(std::string_view name) { + const auto endsWith = [name](std::string_view suffix) { + return name.size() >= suffix.size() && + name.substr(name.size() - suffix.size()) == suffix; + }; + if (endsWith(".css")) { + return "text/css"; + } + if (endsWith(".js")) { + return "application/javascript"; + } + if (endsWith(".html")) { + return "text/html"; + } + if (endsWith(".json")) { + return "application/json"; + } + if (endsWith(".svg")) { + return "image/svg+xml"; + } + if (endsWith(".webp")) { + return "image/webp"; + } + if (endsWith(".woff2")) { + return "font/woff2"; + } + return "application/octet-stream"; +} + +std::shared_ptr loadDataset() { + std::ifstream input("/data/dataset.json", std::ios::binary); + if (!input) { + throw std::runtime_error("cannot open /data/dataset.json"); + } + std::string contents((std::istreambuf_iterator(input)), + std::istreambuf_iterator()); + auto dataset = folly::parseJson(contents); + if (!dataset.isArray() || dataset.size() < 50) { + throw std::runtime_error("/data/dataset.json must contain 50 items"); + } + return std::make_shared(std::move(dataset)); +} + +bool validWebSocketKey(std::string_view key) noexcept { + if (key.size() != 24) { + return false; + } + std::array decoded{}; + const auto result = folly::base64Decode(key, decoded.data()); + return result.is_success && result.o == decoded.data() + 16; +} + +bool validUtf8(const uint8_t *data, size_t size) noexcept { + const auto continuation = [](uint8_t byte) { + return byte >= 0x80 && byte <= 0xbf; + }; + + size_t index = 0; + while (index < size) { + const uint8_t first = data[index]; + if (first <= 0x7f) { + ++index; + continue; + } + if (first >= 0xc2 && first <= 0xdf) { + if (index + 1 >= size || !continuation(data[index + 1])) { + return false; + } + index += 2; + continue; + } + if (first == 0xe0) { + if (index + 2 >= size || data[index + 1] < 0xa0 || + data[index + 1] > 0xbf || !continuation(data[index + 2])) { + return false; + } + index += 3; + continue; + } + if ((first >= 0xe1 && first <= 0xec) || (first >= 0xee && first <= 0xef)) { + if (index + 2 >= size || !continuation(data[index + 1]) || + !continuation(data[index + 2])) { + return false; + } + index += 3; + continue; + } + if (first == 0xed) { + if (index + 2 >= size || data[index + 1] < 0x80 || + data[index + 1] > 0x9f || !continuation(data[index + 2])) { + return false; + } + index += 3; + continue; + } + if (first == 0xf0) { + if (index + 3 >= size || data[index + 1] < 0x90 || + data[index + 1] > 0xbf || !continuation(data[index + 2]) || + !continuation(data[index + 3])) { + return false; + } + index += 4; + continue; + } + if (first >= 0xf1 && first <= 0xf3) { + if (index + 3 >= size || !continuation(data[index + 1]) || + !continuation(data[index + 2]) || !continuation(data[index + 3])) { + return false; + } + index += 4; + continue; + } + if (first == 0xf4) { + if (index + 3 >= size || data[index + 1] < 0x80 || + data[index + 1] > 0x8f || !continuation(data[index + 2]) || + !continuation(data[index + 3])) { + return false; + } + index += 4; + continue; + } + return false; + } + return true; +} + +bool validUtf8(const std::vector &data) noexcept { + return validUtf8(data.data(), data.size()); +} + +bool validWebSocketCloseCode(uint16_t code) noexcept { + const bool definedProtocolCode = code >= 1000 && code <= 1014 && + code != 1004 && code != 1005 && code != 1006; + const bool applicationCode = code >= 3000 && code <= 4999; + return definedProtocolCode || applicationCode; +} + +HTTPFixedSource *makeResponse(uint16_t status, + std::string_view contentTypeValue, + std::string body) { + auto *response = HTTPFixedSource::makeFixedResponse(status, std::move(body)); + response->msg_->getHeaders().set(proxygen::HTTP_HEADER_CONTENT_TYPE, + contentTypeValue); + return response; +} + +HTTPFixedSource *makeTextResponse(uint16_t status, std::string body) { + return makeResponse(status, "text/plain", std::move(body)); +} + +folly::coro::Task +readBodyEventNoSuspend(HTTPSourceHolder &source, + uint32_t max = std::numeric_limits::max()) { + while (true) { + auto event = co_await co_awaitTry(source.readBodyEvent(max)); + if (event.hasException()) { + co_yield folly::coro::co_error(std::move(event.exception())); + } + if (event->eventType == HTTPBodyEvent::SUSPEND) { + const auto status = co_await std::move(event->event.resume); + if (status == TimedBaton::Status::cancelled) { + co_yield folly::coro::co_error( + HTTPError(HTTPErrorCode::CORO_CANCELLED, "request read cancelled")); + } + continue; + } + co_return std::move(*event); + } +} + +struct RequestBody { + bool readOk{true}; + bool captureOk{true}; + size_t size{0}; + std::string captured; +}; + +folly::coro::Task readRequestBody(HTTPSourceHolder &source, + bool eom, size_t captureLimit) { + RequestBody result; + while (!eom) { + auto eventTry = co_await co_awaitTry(readBodyEventNoSuspend(source)); + if (eventTry.hasException()) { + result.readOk = false; + co_return result; + } + auto event = std::move(*eventTry); + eom = event.eom; + if (event.eventType == HTTPBodyEvent::PADDING) { + continue; + } + if (event.eventType != HTTPBodyEvent::BODY) { + result.readOk = false; + co_return result; + } + + const size_t bytes = event.event.body.chainLength(); + if (bytes > std::numeric_limits::max() - result.size) { + result.readOk = false; + co_return result; + } + result.size += bytes; + if (captureLimit == 0 || !result.captureOk) { + continue; + } + if (result.captured.size() > captureLimit || + bytes > captureLimit - result.captured.size()) { + result.captureOk = false; + continue; + } + auto body = event.event.body.move(); + if (body) { + const auto range = body->coalesce(); + result.captured.append(reinterpret_cast(range.data()), + range.size()); + } + } + co_return result; +} + +class WebSocketSource final : public HTTPSource { +public: + explicit WebSocketSource(HTTPSourceHolder request) + : request_(std::move(request)) { + setHeapAllocated(); + } + + folly::coro::Task readHeaderEvent() override { + auto response = std::make_unique(); + response->setHTTPVersion(1, 1); + response->setStatusCode(200); + response->setStatusMessage("OK"); + response->setEgressWebsocketUpgrade(); + HTTPHeaderEvent event(std::move(response), false); + auto guard = folly::makeGuard(lifetime(event)); + co_return event; + } + + folly::coro::Task + readBodyEvent(uint32_t max = std::numeric_limits::max()) override { + while (pending_.empty()) { + auto inputTry = co_await co_awaitTry(readBodyEventNoSuspend(request_)); + if (inputTry.hasException()) { + auto error = proxygen::coro::getHTTPError(inputTry); + auto guard = folly::makeGuard([this] { + if (heapAllocated_) { + delete this; + } + }); + co_yield folly::coro::co_error(std::move(error)); + } + + auto input = std::move(*inputTry); + if (input.eventType == HTTPBodyEvent::BODY) { + auto body = input.event.body.move(); + if (body) { + const auto range = body->coalesce(); + input_.insert(input_.end(), range.begin(), range.end()); + processFrames(); + } + } + if (input.eom && !finished_) { + finished_ = true; + if (pending_.empty()) { + pending_.push_back(PendingOutput{{}, 0, true}); + } else { + pending_.back().eom = true; + } + } + } + + auto &front = pending_.front(); + const size_t remaining = front.bytes.size() - front.offset; + const size_t limit = std::max(1, max); + const size_t amount = std::min(remaining, limit); + std::unique_ptr body; + if (amount > 0) { + body = + folly::IOBuf::copyBuffer(front.bytes.data() + front.offset, amount); + front.offset += amount; + } + const bool outputEom = front.eom && front.offset == front.bytes.size(); + if (front.offset == front.bytes.size()) { + pending_.pop_front(); + } + HTTPBodyEvent event(std::move(body), outputEom); + auto guard = folly::makeGuard(lifetime(event)); + co_return event; + } + + void stopReading(folly::Optional error = + folly::none) noexcept override { + if (request_) { + request_.stopReading(error); + } + if (heapAllocated_) { + delete this; + } + } + +private: + struct PendingOutput { + std::vector bytes; + size_t offset{0}; + bool eom{false}; + }; + + void queueEom() { + if (!finished_) { + finished_ = true; + pending_.push_back(PendingOutput{{}, 0, true}); + } + } + + void queueFrame(uint8_t opcode, const uint8_t *payload, size_t payloadLength, + bool eom = false) { + std::vector frame; + frame.reserve(payloadLength + 10); + frame.push_back(static_cast(0x80U | opcode)); + if (payloadLength <= 125) { + frame.push_back(static_cast(payloadLength)); + } else if (payloadLength <= std::numeric_limits::max()) { + frame.push_back(126); + frame.push_back(static_cast((payloadLength >> 8) & 0xff)); + frame.push_back(static_cast(payloadLength & 0xff)); + } else { + frame.push_back(127); + const auto length = static_cast(payloadLength); + for (int shift = 56; shift >= 0; shift -= 8) { + frame.push_back(static_cast((length >> shift) & 0xff)); + } + } + if (payloadLength > 0) { + frame.insert(frame.end(), payload, payload + payloadLength); + } + pending_.push_back(PendingOutput{std::move(frame), 0, eom}); + } + + void queueFrame(uint8_t opcode, const std::vector &payload, + bool eom = false) { + queueFrame(opcode, payload.data(), payload.size(), eom); + } + + void closeWith(uint16_t status) { + if (closeSent_ || finished_) { + return; + } + const std::array payload = { + static_cast((status >> 8) & 0xff), + static_cast(status & 0xff)}; + queueFrame(0x08, payload.data(), payload.size()); + closeSent_ = true; + } + + void protocolError() { closeWith(1002); } + + void invalidPayload() { closeWith(1007); } + + void handleFrame(bool fin, uint8_t opcode, std::vector payload) { + if (closeSent_ && opcode != 0x08) { + return; + } + + if ((opcode & 0x08U) != 0) { + if (!fin || payload.size() > 125) { + protocolError(); + return; + } + if (opcode == 0x08) { + if (payload.size() == 1) { + protocolError(); + return; + } + if (payload.size() >= 2) { + const uint16_t status = + (static_cast(payload[0]) << 8) | payload[1]; + if (!validWebSocketCloseCode(status)) { + protocolError(); + return; + } + if (!validUtf8(payload.data() + 2, payload.size() - 2)) { + invalidPayload(); + return; + } + } + if (closeSent_) { + queueEom(); + return; + } + finished_ = true; + queueFrame(0x08, payload, true); + } else if (opcode == 0x09) { + queueFrame(0x0a, payload); + } else if (opcode != 0x0a) { + protocolError(); + } + return; + } + + if (opcode == 0x00) { + if (fragmentOpcode_ == 0 || + payload.size() > kMaxWebSocketMessage - fragmentPayload_.size()) { + protocolError(); + return; + } + fragmentPayload_.insert(fragmentPayload_.end(), payload.begin(), + payload.end()); + if (fin) { + if (fragmentOpcode_ == 0x01 && !validUtf8(fragmentPayload_)) { + invalidPayload(); + return; + } + queueFrame(fragmentOpcode_, fragmentPayload_); + fragmentOpcode_ = 0; + fragmentPayload_.clear(); + } + return; + } + + if ((opcode != 0x01 && opcode != 0x02) || fragmentOpcode_ != 0) { + protocolError(); + return; + } + if (fin) { + if (opcode == 0x01 && !validUtf8(payload)) { + invalidPayload(); + return; + } + queueFrame(opcode, payload); + return; + } + fragmentOpcode_ = opcode; + fragmentPayload_ = std::move(payload); + } + + void processFrames() { + size_t cursor = 0; + while (!finished_) { + if (input_.size() - cursor < 2) { + break; + } + const uint8_t first = input_[cursor]; + const uint8_t second = input_[cursor + 1]; + const bool fin = (first & 0x80U) != 0; + const uint8_t opcode = first & 0x0fU; + const uint8_t encodedLength = second & 0x7fU; + if ((first & 0x70U) != 0 || (second & 0x80U) == 0 || + ((opcode & 0x08U) != 0 && encodedLength > 125)) { + protocolError(); + cursor = input_.size(); + break; + } + + uint64_t payloadLength = encodedLength; + size_t headerLength = 2; + if (payloadLength == 126) { + if (input_.size() - cursor < 4) { + break; + } + payloadLength = (static_cast(input_[cursor + 2]) << 8) | + input_[cursor + 3]; + if (payloadLength < 126) { + protocolError(); + cursor = input_.size(); + break; + } + headerLength = 4; + } else if (payloadLength == 127) { + if (input_.size() - cursor < 10) { + break; + } + if ((input_[cursor + 2] & 0x80U) != 0) { + protocolError(); + cursor = input_.size(); + break; + } + payloadLength = 0; + for (size_t index = 0; index < 8; ++index) { + payloadLength = (payloadLength << 8) | input_[cursor + 2 + index]; + } + if (payloadLength <= std::numeric_limits::max()) { + protocolError(); + cursor = input_.size(); + break; + } + headerLength = 10; + } + if (payloadLength > kMaxWebSocketMessage || + payloadLength > + std::numeric_limits::max() - headerLength - 4) { + protocolError(); + cursor = input_.size(); + break; + } + + const size_t frameLength = + headerLength + 4 + static_cast(payloadLength); + if (input_.size() - cursor < frameLength) { + break; + } + const size_t maskOffset = cursor + headerLength; + const size_t payloadOffset = maskOffset + 4; + std::vector payload(static_cast(payloadLength)); + for (size_t index = 0; index < payload.size(); ++index) { + payload[index] = + input_[payloadOffset + index] ^ input_[maskOffset + (index % 4)]; + } + cursor += frameLength; + handleFrame(fin, opcode, std::move(payload)); + if (closeSent_ || finished_) { + break; + } + } + + if (cursor > 0) { + input_.erase(input_.begin(), input_.begin() + cursor); + } + if (closeSent_ || finished_) { + input_.clear(); + } + } + + HTTPSourceHolder request_; + std::deque pending_; + std::vector input_; + std::vector fragmentPayload_; + uint8_t fragmentOpcode_{0}; + bool closeSent_{false}; + bool finished_{false}; +}; + +class ArenaCoroHandler final : public HTTPHandler { +public: + explicit ArenaCoroHandler(std::shared_ptr dataset) + : dataset_(std::move(dataset)) {} + + folly::coro::Task + handleRequest(folly::EventBase * /*eventBase*/, + HTTPSessionContextPtr /*session*/, + HTTPSourceHolder requestSource) override { + auto headerTry = co_await co_awaitTry(requestSource.readHeaderEvent()); + if (headerTry.hasException()) { + co_return makeTextResponse(400, "bad request"); + } + auto header = std::move(*headerTry); + auto request = std::move(header.headers); + const bool requestEom = header.eom; + const auto method = request->getMethod().value_or(HTTPMethod::GET); + const std::string path = request->getPath(); + + if (path == "/ws") { + const auto &headers = request->getHeaders(); + const auto &key = headers.getSingleOrEmpty("Sec-WebSocket-Key"); + const auto &version = headers.getSingleOrEmpty("Sec-WebSocket-Version"); + if (method != HTTPMethod::GET || !request->isIngressWebsocketUpgrade() || + !validWebSocketKey(std::string_view(key.data(), key.size())) || + version != "13") { + auto *response = makeTextResponse(426, "WebSocket upgrade required"); + response->msg_->getHeaders().set("Sec-WebSocket-Version", "13"); + co_return response; + } + co_return new WebSocketSource(std::move(requestSource)); + } + + if (path == "/baseline11" || path == "/baseline2") { + const bool allowPost = path == "/baseline11"; + if (method != HTTPMethod::GET && + (!allowPost || method != HTTPMethod::POST)) { + co_return makeTextResponse(405, "method not allowed"); + } + int64_t a = 0; + int64_t b = 0; + if (!parseInteger(request->getQueryParam("a"), a) || + !parseInteger(request->getQueryParam("b"), b)) { + co_return makeTextResponse(400, "invalid integer"); + } + int64_t sum = 0; + if (!checkedAdd(a, b, sum)) { + co_return makeTextResponse(400, "integer overflow"); + } + if (method == HTTPMethod::POST) { + auto body = co_await readRequestBody(requestSource, requestEom, + kMaxBaselineBody); + int64_t bodyValue = 0; + if (!body.readOk || !body.captureOk || + !parseInteger(body.captured, bodyValue) || + !checkedAdd(sum, bodyValue, sum)) { + co_return makeTextResponse(400, "invalid integer"); + } + } + co_return makeTextResponse(200, std::to_string(sum)); + } + + if (path == "/pipeline") { + if (method != HTTPMethod::GET) { + co_return makeTextResponse(405, "method not allowed"); + } + co_return makeTextResponse(200, "ok"); + } + + if (path.starts_with(kJsonPrefix)) { + if (method != HTTPMethod::GET) { + co_return makeTextResponse(405, "method not allowed"); + } + const std::string_view countText(path.data() + kJsonPrefix.size(), + path.size() - kJsonPrefix.size()); + int64_t count = 0; + int64_t multiplier = 1; + const auto multiplierText = request->getQueryParam("m"); + if (!parseInteger(countText, count) || count < 1 || count > 50 || + (!multiplierText.empty() && + !parseInteger(multiplierText, multiplier))) { + co_return makeTextResponse(400, "invalid JSON parameters"); + } + + try { + folly::dynamic items = folly::dynamic::array; + for (int64_t index = 0; index < count; ++index) { + folly::dynamic item = (*dataset_)[static_cast(index)]; + int64_t subtotal = 0; + int64_t total = 0; + if (!checkedMultiply(item["price"].asInt(), item["quantity"].asInt(), + subtotal) || + !checkedMultiply(subtotal, multiplier, total)) { + co_return makeTextResponse(400, "integer overflow"); + } + item["total"] = total; + items.push_back(std::move(item)); + } + folly::dynamic response = folly::dynamic::object; + response["items"] = std::move(items); + response["count"] = count; + co_return makeResponse(200, "application/json", + folly::toJson(response)); + } catch (const std::exception &) { + co_return makeTextResponse(500, "JSON serialization failed"); + } + } + + if (path == "/upload") { + if (method != HTTPMethod::POST) { + co_return makeTextResponse(405, "method not allowed"); + } + auto body = co_await readRequestBody(requestSource, requestEom, 0); + if (!body.readOk) { + co_return makeTextResponse(400, "upload failed"); + } + co_return makeTextResponse(200, std::to_string(body.size)); + } + + if (path.starts_with(kStaticPrefix)) { + if (method != HTTPMethod::GET) { + co_return makeTextResponse(405, "method not allowed"); + } + const std::string name(path.data() + kStaticPrefix.size(), + path.size() - kStaticPrefix.size()); + if (name.empty() || name.find('/') != std::string::npos || + name.find('\\') != std::string::npos || + name.find("..") != std::string::npos) { + co_return makeTextResponse(404, "not found"); + } + std::ifstream input(std::string(kStaticRoot) + name, std::ios::binary); + if (!input) { + co_return makeTextResponse(404, "not found"); + } + std::string body((std::istreambuf_iterator(input)), + std::istreambuf_iterator()); + if (!input.good() && !input.eof()) { + co_return makeTextResponse(500, "read error"); + } + co_return makeResponse(200, contentType(name), std::move(body)); + } + + co_return makeTextResponse(404, "not found"); + } + +private: + std::shared_ptr dataset_; +}; + +std::shared_ptr> compressibleTypes() { + return std::make_shared>(std::set{ + "application/javascript", "application/json", "image/svg+xml", "text/css", + "text/html", "text/plain"}); +} + +constexpr uint32_t kH2StreamWindow = 1U << 20; +constexpr size_t kH2ConnectionWindow = 10U << 20; +constexpr uint32_t kMaxConcurrentStreams = 1024; + +HTTPServer::SessionConfig makeSessionConfig() { + HTTPServer::SessionConfig session; + session.settings = { + {proxygen::SettingsId::MAX_HEADER_LIST_SIZE, 32 * 1024}, + {proxygen::SettingsId::HEADER_TABLE_SIZE, 4096}, + {proxygen::SettingsId::MAX_FRAME_SIZE, 16384}, + {proxygen::SettingsId::MAX_CONCURRENT_STREAMS, kMaxConcurrentStreams}, + {proxygen::SettingsId::INITIAL_WINDOW_SIZE, kH2StreamWindow}}; + session.streamFlowControl = kH2StreamWindow; + session.connFlowControl = kH2ConnectionWindow; + session.streamReadTimeout = std::chrono::seconds(60); + session.connIdleTimeout = std::chrono::seconds(60); + return session; +} + +void addCompressionFilter(HTTPServer::Config &config) { + proxygen::CompressionFilterUtils::FactoryOptions options; + options.compressibleContentTypes = compressibleTypes(); + config.filterFactories.push_back( + std::make_shared( + std::move(options))); +} + +wangle::SSLContextConfig tlsConfig(std::list protocols) { + auto config = HTTPServer::getDefaultTLSConfig(); + config.setCertificate(FLAGS_cert, FLAGS_key, ""); + config.setNextProtocols(protocols); + return config; +} + +std::shared_ptr +acceptorConfig(const HTTPServer::Config &serverConfig, + std::string plaintextProtocol, + std::list tlsProtocols = {}) { + auto config = std::make_shared(); + *static_cast(config.get()) = + serverConfig.socketConfig; + config->sslContextConfigs.clear(); + if (!tlsProtocols.empty()) { + config->sslContextConfigs.push_back(tlsConfig(std::move(tlsProtocols))); + } + config->egressSettings = serverConfig.sessionConfig.settings; + config->transactionIdleTimeout = serverConfig.sessionConfig.streamReadTimeout; + config->initialReceiveWindow = kH2StreamWindow; + config->receiveStreamWindowSize = kH2StreamWindow; + config->receiveSessionWindowSize = kH2ConnectionWindow; + config->maxConcurrentIncomingStreams = kMaxConcurrentStreams; + config->plaintextProtocol = std::move(plaintextProtocol); + config->forceHTTP1_0_to_1_1 = true; + config->connectionIdleTimeout = serverConfig.sessionConfig.connIdleTimeout; + config->readBufNewAllocSize = serverConfig.sessionConfig.readBufNewAllocSize; + return config; +} + +HTTPServer::SocketAcceptorConfigFactoryFn tcpSocketFactory() { + return [](folly::EventBase &eventBase, + const HTTPServer::Config &serverConfig) { + std::vector listeners; + const auto addListener = [&](uint16_t port, std::string protocol, + std::list tlsProtocols = {}) { + folly::AsyncServerSocket::UniquePtr socket( + new folly::AsyncServerSocket(&eventBase)); + socket->bind(folly::SocketAddress(FLAGS_ip, port, true)); + socket->listen(serverConfig.socketConfig.acceptBacklog); + listeners.emplace_back(std::move(socket), + acceptorConfig(serverConfig, std::move(protocol), + std::move(tlsProtocols))); + }; + + addListener(static_cast(FLAGS_http_port), "http/1.1"); + addListener(static_cast(FLAGS_tls_port), "http/1.1", + {"http/1.1"}); + addListener(static_cast(FLAGS_h2c_port), "h2c"); + addListener(static_cast(FLAGS_h2_port), "", {"h2"}); + return listeners; + }; +} + +HTTPServer::Config tcpConfig(size_t threads) { + HTTPServer::Config config; + config.numIOThreads = threads; + config.shutdownOnSignals = {SIGINT, SIGTERM}; + config.sessionConfig = makeSessionConfig(); + addCompressionFilter(config); + return config; +} + +HTTPServer::Config quicConfig(size_t threads) { + HTTPServer::Config config; + config.socketConfig.bindAddress = + folly::SocketAddress(FLAGS_ip, FLAGS_h3_port, true); + config.socketConfig.sslContextConfigs.push_back(tlsConfig({"h3"})); + config.numIOThreads = threads; + config.shutdownOnSignals = {}; + config.sessionConfig = makeSessionConfig(); + config.quicConfig = HTTPServer::QuicConfig(); + config.quicConfig->supportedAlpns = {"h3"}; + auto &transport = config.quicConfig->transportSettings; + transport.maxNumPTOs = 1000; + transport.maxCwndInMss = quic::kLargeMaxCwndInMss; + transport.batchingMode = quic::QuicBatchingMode::BATCHING_MODE_GSO; + transport.maxBatchSize = 48; + transport.dataPathType = quic::DataPathType::ContinuousMemory; + transport.writeConnectionDataPacketsLimit = 48; + addCompressionFilter(config); + return config; +} + +} // namespace + +int main(int argc, char *argv[]) { + const folly::Init init(&argc, &argv, true); + + try { + auto dataset = loadDataset(); + const size_t threads = + FLAGS_threads <= 0 ? static_cast(folly::available_concurrency()) + : static_cast(FLAGS_threads); + auto handler = std::make_shared(std::move(dataset)); + + HTTPServer tcpServer(tcpConfig(threads), handler, tcpSocketFactory()); + HTTPServer h3Server(quicConfig(threads), std::move(handler)); + + std::promise h3Ready; + auto h3ReadyFuture = h3Ready.get_future(); + std::thread h3Thread([&] { + try { + h3Server.start([&] { h3Ready.set_value(); }); + } catch (...) { + try { + h3Ready.set_exception(std::current_exception()); + } catch (const std::future_error &) { + } + } + }); + + try { + h3ReadyFuture.get(); + } catch (...) { + h3Thread.join(); + throw; + } + + try { + tcpServer.start(); + } catch (...) { + h3Server.forceStop(); + h3Thread.join(); + throw; + } + + h3Server.forceStop(); + h3Thread.join(); + } catch (const std::exception &error) { + std::cerr << "failed to start Proxygen coroutine HttpArena server: " + << error.what() << '\n'; + return 1; + } + return 0; +} diff --git a/frameworks/proxygen-coro/CMakeLists.txt b/frameworks/proxygen-coro/CMakeLists.txt new file mode 100644 index 000000000..33a700e02 --- /dev/null +++ b/frameworks/proxygen-coro/CMakeLists.txt @@ -0,0 +1,28 @@ +cmake_minimum_required(VERSION 3.20) + +project(httparena-proxygen-coro LANGUAGES CXX) + +set(CMAKE_CXX_STANDARD 20) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_CXX_EXTENSIONS OFF) + +# Proxygen's exported Fizz package calls find_dependency(Sodium). The official +# builder image keeps that upstream find module with the Proxygen source tree. +list(APPEND CMAKE_MODULE_PATH "/proxygen/build/fbcode_builder/CMake") + +find_package(c-ares CONFIG REQUIRED) +add_library(cares ALIAS c-ares::cares) +find_package(proxygen CONFIG REQUIRED) + +add_executable(proxygen-arena-coro ArenaCoroServer.cpp) +target_compile_options(proxygen-arena-coro PRIVATE -Wall -Wextra -Wpedantic) +target_link_libraries( + proxygen-arena-coro + PRIVATE + proxygen::proxygen + proxygen::proxygen_coro + proxygen::proxygen_coro_server + proxygen::proxygen_http_coro_filters_compression_filter_factory + Folly::folly_init_init + Folly::folly_portability_gflags +) diff --git a/frameworks/proxygen-coro/Dockerfile b/frameworks/proxygen-coro/Dockerfile new file mode 100644 index 000000000..703ee4ae1 --- /dev/null +++ b/frameworks/proxygen-coro/Dockerfile @@ -0,0 +1,32 @@ +FROM ghcr.io/facebook/proxygen/base:latest AS build + +WORKDIR /arena +COPY CMakeLists.txt ArenaCoroServer.cpp ./ +RUN cmake -S . -B build -DCMAKE_BUILD_TYPE=Release \ + && cmake --build build --parallel "$(nproc)" \ + && strip build/proxygen-arena-coro + +RUN set -eux; \ + ldd build/proxygen-arena-coro \ + | awk '/=> \// { print $3 } /^\// { print $1 }' | sort -u > /tmp/runtime-libs.txt; \ + tar -chf /tmp/runtime-libs.tar --files-from=/tmp/runtime-libs.txt + +FROM ubuntu:24.04@sha256:019e8eb29a85e74d64925745884f2ec79aa27e3feab36353d24656f4d6b89467 + +ENV LD_LIBRARY_PATH=/opt/proxygen/lib + +COPY --from=build /tmp/runtime-libs.tar /tmp/runtime-libs.tar +RUN tar -xf /tmp/runtime-libs.tar -C / \ + && rm /tmp/runtime-libs.tar + +COPY --from=build /arena/build/proxygen-arena-coro /usr/local/bin/proxygen-arena-coro +COPY entrypoint.sh /usr/local/bin/proxygen-coro-entrypoint +RUN chmod +x /usr/local/bin/proxygen-coro-entrypoint \ + && groupadd --system --gid 10001 httparena \ + && useradd --system --uid 10001 --gid httparena --no-create-home \ + --home-dir /nonexistent --shell /usr/sbin/nologin httparena + +EXPOSE 8080/tcp 8081/tcp 8082/tcp 8443/tcp 8443/udp + +USER httparena +ENTRYPOINT ["/usr/local/bin/proxygen-coro-entrypoint"] diff --git a/frameworks/proxygen-coro/README.md b/frameworks/proxygen-coro/README.md new file mode 100644 index 000000000..7ad4114f9 --- /dev/null +++ b/frameworks/proxygen-coro/README.md @@ -0,0 +1,52 @@ +# proxygen-coro + +This engine exercises Proxygen's native coroutine server stack rather than the +callback `RequestHandler` / `HTTPTransactionHandler` APIs used by the regular +`proxygen` entry. + +`ArenaCoroServer.cpp` implements `proxygen::coro::HTTPHandler`, consumes +requests through `HTTPSourceHolder`, and returns `HTTPFixedSource` responses. +Uploads are counted while asynchronously draining BODY events. WebSockets use +a long-lived custom `HTTPSource`: its response calls +`setEgressWebsocketUpgrade()`, then parses and emits RFC 6455 frames over the +raw upgraded BODY event stream. Response compression is provided by the coro +`ServerCompressionFilterFactory`. + +## Listener layout + +One process owns all five listeners: + +- `8080/tcp`: HTTP/1.1 and WebSockets +- `8081/tcp`: HTTP/1.1 over TLS, ALPN `http/1.1` +- `8082/tcp`: prior-knowledge h2c +- `8443/tcp`: HTTP/2 over TLS, ALPN `h2` +- `8443/udp`: HTTP/3 over QUIC, ALPN `h3` + +The four TCP listeners are acceptors on one coro `HTTPServer`, so they share a +single affinity-aware I/O pool. Proxygen's coro API selects either TCP or QUIC +per server, so HTTP/3 uses a second in-process pool; whichever transport is not +being benchmarked remains idle. Override the available-CPU default with +`PROXYGEN_CORO_THREADS`. + +HTTP/2 advertises 1024 concurrent streams with a 1 MiB stream window and a +10 MiB connection window. HTTP/3 mirrors Proxygen's coroutine benchmark +settings: GSO batches of 48 packets, continuous-memory writes, a large +congestion window, and a 48-packet connection write limit. + +## Upstream image and build + +The self-contained Docker build tracks the official +`ghcr.io/facebook/proxygen/base:latest` builder image. The runtime stage copies +only the compiled binary and its dynamically linked libraries into a pinned +Ubuntu 24.04 image, then runs as the non-root `httparena` user (UID/GID 10001). + +From the repository root: + +```bash +./scripts/validate.sh proxygen-coro +./scripts/run.sh proxygen-coro +``` + +The implementation follows the upstream coroutine echo server and the +`H12DownstreamSessionTest.WebSocketUpgrade` test, which documents upgraded +raw bytes flowing through coroutine BODY events. diff --git a/frameworks/proxygen-coro/entrypoint.sh b/frameworks/proxygen-coro/entrypoint.sh new file mode 100755 index 000000000..c96668cc6 --- /dev/null +++ b/frameworks/proxygen-coro/entrypoint.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +set -euo pipefail + +exec /usr/local/bin/proxygen-arena-coro \ + --ip=:: \ + --http_port=8080 \ + --tls_port=8081 \ + --h2c_port=8082 \ + --h2_port=8443 \ + --h3_port=8443 \ + --cert=/certs/server.crt \ + --key=/certs/server.key \ + --threads="${PROXYGEN_CORO_THREADS:-0}" diff --git a/frameworks/proxygen-coro/meta.json b/frameworks/proxygen-coro/meta.json new file mode 100644 index 000000000..6473059a8 --- /dev/null +++ b/frameworks/proxygen-coro/meta.json @@ -0,0 +1,30 @@ +{ + "display_name": "proxygen-coro", + "language": "C++", + "type": "engine", + "engine": "proxygen", + "description": "Meta's Proxygen native coroutine HTTPServer and HTTPSource APIs across HTTP/1.1, HTTP/1.1 TLS, h2c, HTTP/2 TLS, HTTP/3 QUIC, and RFC 6455 WebSockets.", + "repo": "https://github.com/facebook/proxygen", + "enabled": true, + "tests": [ + "baseline", + "json", + "json-comp", + "json-tls", + "upload", + "static", + "static-tls", + "pipelined", + "limited-conn", + "baseline-h2", + "baseline-h2c", + "json-h2c", + "static-h2", + "baseline-h3", + "static-h3", + "echo-ws", + "echo-ws-pipeline", + "echo-ws-limited" + ], + "maintainers": [] +} diff --git a/site/data/frameworks.json b/site/data/frameworks.json index 668857ea8..80d509f13 100644 --- a/site/data/frameworks.json +++ b/site/data/frameworks.json @@ -1293,6 +1293,13 @@ "type": "engine", "engine": "proxygen" }, + "proxygen-coro": { + "dir": "proxygen-coro", + "description": "Meta's Proxygen native coroutine HTTPServer and HTTPSource APIs across HTTP/1.1, HTTP/1.1 TLS, h2c, HTTP/2 TLS, HTTP/3 QUIC, and RFC 6455 WebSockets.", + "repo": "https://github.com/facebook/proxygen", + "type": "engine", + "engine": "proxygen" + }, "pyronova": { "dir": "pyronova", "description": "Pyronova \u2014 Python web framework with a Rust core (hyper + tokio + rustls + mimalloc) and PEP 684 sub-interpreter workers for true multi-core parallelism. Opt-in features: gzip/brotli compression, rustls TLS with h2/h1 ALPN, streaming body ingest, async Postgres via sqlx::PgPool. Handlers are standard Python functions routed via decorators.", From a39b10b3d889d4e61bcc188e874b5e92bf9ed53c Mon Sep 17 00:00:00 2001 From: sujay Date: Thu, 3 Sep 2026 11:28:53 -0700 Subject: [PATCH 3/4] proxygen: share one source tree, preload static assets, cut per-request cost Builds on the two Proxygen engine commits. Both entries now build from frameworks/proxygen/src selected by a TARGET build arg, the way sark-h3 reuses sark, so a fix lands in both entries at once. Static assets and their precompressed .br/.gz siblings are read once at startup and served as non-owning IOBuf views: no disk I/O, no per-request compression, no copy, no payload allocation. This replaces an open/read plus an on-the-fly gzip of every CSS/JS/HTML response, which dominated the static profiles. Also: - shared RFC 6455 codec: frames unmasked in place, one egress write per read - response compression scoped to application/json, the type json-comp scores - coro attaches CompressionFilter per response instead of registering a server-level factory, which allocated four objects on every request because HTTPFilterFactoryHandler builds filters before reading request headers - per-request allocations removed from the hot path: getPathAsStringPiece, a query-string scan instead of the parsed parameter map, string_view content types, to_chars, and no shared_ptr copy per handler - mimalloc preloaded; separate --threads / --h3_threads - base image pinned by digest for reproducible rounds Measured locally on a 32-core cpuset with wrk, so relative deltas only: static 13,012 -> 1,049,630 rps static-tls 12,701 -> 706,558 rps json +38% json-comp +41% classic / +82% coro baseline +7.5% classic / +17% coro websocket -30% / -46% CPU per frame Peak memory on static at 6800 connections drops 2.1 GiB -> 160 MiB. validate.sh reports 63/63 for both entries; HTTP/3 and RFC 6455 conformance checked separately. --- frameworks/proxygen-coro/CMakeLists.txt | 28 - frameworks/proxygen-coro/Dockerfile | 32 - frameworks/proxygen-coro/README.md | 65 +- frameworks/proxygen-coro/build.sh | 10 + frameworks/proxygen-coro/entrypoint.sh | 13 - frameworks/proxygen/ArenaCommon.h | 214 ------ frameworks/proxygen/CMakeLists.txt | 51 +- frameworks/proxygen/Dockerfile | 51 +- frameworks/proxygen/README.md | 121 ++-- frameworks/proxygen/entrypoint.sh | 10 +- frameworks/proxygen/src/ArenaCommon.h | 392 +++++++++++ .../proxygen/{ => src}/ArenaHQServer.cpp | 5 + frameworks/proxygen/{ => src}/ArenaHQServer.h | 0 frameworks/proxygen/src/ArenaWebSocket.h | 381 ++++++++++ .../ClassicServer.cpp} | 398 ++++------- .../src/CoroServer.cpp} | 656 ++++-------------- site/data/frameworks.json | 12 +- 17 files changed, 1265 insertions(+), 1174 deletions(-) delete mode 100644 frameworks/proxygen-coro/CMakeLists.txt delete mode 100644 frameworks/proxygen-coro/Dockerfile create mode 100755 frameworks/proxygen-coro/build.sh delete mode 100755 frameworks/proxygen-coro/entrypoint.sh delete mode 100644 frameworks/proxygen/ArenaCommon.h create mode 100644 frameworks/proxygen/src/ArenaCommon.h rename frameworks/proxygen/{ => src}/ArenaHQServer.cpp (88%) rename frameworks/proxygen/{ => src}/ArenaHQServer.h (100%) create mode 100644 frameworks/proxygen/src/ArenaWebSocket.h rename frameworks/proxygen/{ArenaHttpServer.cpp => src/ClassicServer.cpp} (54%) rename frameworks/{proxygen-coro/ArenaCoroServer.cpp => proxygen/src/CoroServer.cpp} (50%) diff --git a/frameworks/proxygen-coro/CMakeLists.txt b/frameworks/proxygen-coro/CMakeLists.txt deleted file mode 100644 index 33a700e02..000000000 --- a/frameworks/proxygen-coro/CMakeLists.txt +++ /dev/null @@ -1,28 +0,0 @@ -cmake_minimum_required(VERSION 3.20) - -project(httparena-proxygen-coro LANGUAGES CXX) - -set(CMAKE_CXX_STANDARD 20) -set(CMAKE_CXX_STANDARD_REQUIRED ON) -set(CMAKE_CXX_EXTENSIONS OFF) - -# Proxygen's exported Fizz package calls find_dependency(Sodium). The official -# builder image keeps that upstream find module with the Proxygen source tree. -list(APPEND CMAKE_MODULE_PATH "/proxygen/build/fbcode_builder/CMake") - -find_package(c-ares CONFIG REQUIRED) -add_library(cares ALIAS c-ares::cares) -find_package(proxygen CONFIG REQUIRED) - -add_executable(proxygen-arena-coro ArenaCoroServer.cpp) -target_compile_options(proxygen-arena-coro PRIVATE -Wall -Wextra -Wpedantic) -target_link_libraries( - proxygen-arena-coro - PRIVATE - proxygen::proxygen - proxygen::proxygen_coro - proxygen::proxygen_coro_server - proxygen::proxygen_http_coro_filters_compression_filter_factory - Folly::folly_init_init - Folly::folly_portability_gflags -) diff --git a/frameworks/proxygen-coro/Dockerfile b/frameworks/proxygen-coro/Dockerfile deleted file mode 100644 index 703ee4ae1..000000000 --- a/frameworks/proxygen-coro/Dockerfile +++ /dev/null @@ -1,32 +0,0 @@ -FROM ghcr.io/facebook/proxygen/base:latest AS build - -WORKDIR /arena -COPY CMakeLists.txt ArenaCoroServer.cpp ./ -RUN cmake -S . -B build -DCMAKE_BUILD_TYPE=Release \ - && cmake --build build --parallel "$(nproc)" \ - && strip build/proxygen-arena-coro - -RUN set -eux; \ - ldd build/proxygen-arena-coro \ - | awk '/=> \// { print $3 } /^\// { print $1 }' | sort -u > /tmp/runtime-libs.txt; \ - tar -chf /tmp/runtime-libs.tar --files-from=/tmp/runtime-libs.txt - -FROM ubuntu:24.04@sha256:019e8eb29a85e74d64925745884f2ec79aa27e3feab36353d24656f4d6b89467 - -ENV LD_LIBRARY_PATH=/opt/proxygen/lib - -COPY --from=build /tmp/runtime-libs.tar /tmp/runtime-libs.tar -RUN tar -xf /tmp/runtime-libs.tar -C / \ - && rm /tmp/runtime-libs.tar - -COPY --from=build /arena/build/proxygen-arena-coro /usr/local/bin/proxygen-arena-coro -COPY entrypoint.sh /usr/local/bin/proxygen-coro-entrypoint -RUN chmod +x /usr/local/bin/proxygen-coro-entrypoint \ - && groupadd --system --gid 10001 httparena \ - && useradd --system --uid 10001 --gid httparena --no-create-home \ - --home-dir /nonexistent --shell /usr/sbin/nologin httparena - -EXPOSE 8080/tcp 8081/tcp 8082/tcp 8443/tcp 8443/udp - -USER httparena -ENTRYPOINT ["/usr/local/bin/proxygen-coro-entrypoint"] diff --git a/frameworks/proxygen-coro/README.md b/frameworks/proxygen-coro/README.md index 7ad4114f9..9d39ea47e 100644 --- a/frameworks/proxygen-coro/README.md +++ b/frameworks/proxygen-coro/README.md @@ -1,46 +1,55 @@ # proxygen-coro -This engine exercises Proxygen's native coroutine server stack rather than the -callback `RequestHandler` / `HTTPTransactionHandler` APIs used by the regular -`proxygen` entry. - -`ArenaCoroServer.cpp` implements `proxygen::coro::HTTPHandler`, consumes -requests through `HTTPSourceHolder`, and returns `HTTPFixedSource` responses. -Uploads are counted while asynchronously draining BODY events. WebSockets use -a long-lived custom `HTTPSource`: its response calls -`setEgressWebsocketUpgrade()`, then parses and emits RFC 6455 frames over the -raw upgraded BODY event stream. Response compression is provided by the coro -`ServerCompressionFilterFactory`. +The same workload as the [`proxygen`](../proxygen/README.md) entry, served +through Proxygen's native coroutine stack instead of the callback +`RequestHandler` / `HTTPTransactionHandler` APIs. + +`src/CoroServer.cpp` implements `proxygen::coro::HTTPHandler`, consumes requests +through `HTTPSourceHolder` and returns `HTTPFixedSource` responses. Uploads are +counted while asynchronously draining BODY events. WebSockets use a long-lived +custom `HTTPSource`: its response calls `setEgressWebsocketUpgrade()`, then it +parses and emits RFC 6455 frames over the raw upgraded BODY event stream. + +Response compression is proxygen's coro `CompressionFilter`, attached by the +handler to the JSON responses rather than registered as a +`ServerCompressionFilterFactory` on `HTTPServer::Config`. The reason is that +`HTTPFilterFactoryHandler` calls `makeFilters()` *before* the request headers +have been read, so a server-level factory cannot be conditional: it allocates a +`SharedCtx`, a `VisitorFilter` holding a capturing lambda, a `CompressionFilter` +and a coroutine frame on **every** request, including `baseline`, where nothing +is compressible. That measured at ~10% of baseline throughput on this workload. +The classic `HTTPServer` path has no equivalent cost, because +`CompressionFilterFactory::onRequest` can see the request and returns the +handler unwrapped when there is nothing to compress. ## Listener layout One process owns all five listeners: -- `8080/tcp`: HTTP/1.1 and WebSockets -- `8081/tcp`: HTTP/1.1 over TLS, ALPN `http/1.1` -- `8082/tcp`: prior-knowledge h2c -- `8443/tcp`: HTTP/2 over TLS, ALPN `h2` -- `8443/udp`: HTTP/3 over QUIC, ALPN `h3` +- `8080/tcp` HTTP/1.1 and WebSockets +- `8081/tcp` HTTP/1.1 over TLS, ALPN `http/1.1` +- `8082/tcp` prior-knowledge h2c +- `8443/tcp` HTTP/2 over TLS, ALPN `h2` +- `8443/udp` HTTP/3 over QUIC, ALPN `h3` The four TCP listeners are acceptors on one coro `HTTPServer`, so they share a single affinity-aware I/O pool. Proxygen's coro API selects either TCP or QUIC per server, so HTTP/3 uses a second in-process pool; whichever transport is not -being benchmarked remains idle. Override the available-CPU default with -`PROXYGEN_CORO_THREADS`. +being benchmarked stays idle. Size the two pools with `PROXYGEN_THREADS` and +`PROXYGEN_H3_THREADS` (`0` = available CPUs). HTTP/2 advertises 1024 concurrent streams with a 1 MiB stream window and a 10 MiB connection window. HTTP/3 mirrors Proxygen's coroutine benchmark settings: GSO batches of 48 packets, continuous-memory writes, a large -congestion window, and a 48-packet connection write limit. +congestion window and a 48-packet connection write limit. -## Upstream image and build +## Build -The self-contained Docker build tracks the official -`ghcr.io/facebook/proxygen/base:latest` builder image. The runtime stage copies -only the compiled binary and its dynamically linked libraries into a pinned -Ubuntu 24.04 image, then runs as the non-root `httparena` user (UID/GID 10001). - -From the repository root: +This entry has no Dockerfile of its own. `build.sh` builds the +`frameworks/proxygen` context with `--build-arg TARGET=coro`, the same pattern +`sark-h3` uses to reuse `sark`. See +[the proxygen README](../proxygen/README.md#shared-source-tree) for the layout +and image details. ```bash ./scripts/validate.sh proxygen-coro @@ -48,5 +57,5 @@ From the repository root: ``` The implementation follows the upstream coroutine echo server and the -`H12DownstreamSessionTest.WebSocketUpgrade` test, which documents upgraded -raw bytes flowing through coroutine BODY events. +`H12DownstreamSessionTest.WebSocketUpgrade` test, which documents upgraded raw +bytes flowing through coroutine BODY events. diff --git a/frameworks/proxygen-coro/build.sh b/frameworks/proxygen-coro/build.sh new file mode 100755 index 000000000..9a14d9771 --- /dev/null +++ b/frameworks/proxygen-coro/build.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# proxygen-coro reuses the proxygen build context and Dockerfile, selecting the +# coroutine server with --build-arg TARGET=coro (the sark-h3 -> sark pattern). +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" +CONTEXT_DIR="$(cd "$SCRIPT_DIR/../proxygen" && pwd -P)" +docker build -t httparena-proxygen-coro \ + --build-arg TARGET=coro \ + -f "$CONTEXT_DIR/Dockerfile" \ + "$CONTEXT_DIR" diff --git a/frameworks/proxygen-coro/entrypoint.sh b/frameworks/proxygen-coro/entrypoint.sh deleted file mode 100755 index c96668cc6..000000000 --- a/frameworks/proxygen-coro/entrypoint.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -exec /usr/local/bin/proxygen-arena-coro \ - --ip=:: \ - --http_port=8080 \ - --tls_port=8081 \ - --h2c_port=8082 \ - --h2_port=8443 \ - --h3_port=8443 \ - --cert=/certs/server.crt \ - --key=/certs/server.key \ - --threads="${PROXYGEN_CORO_THREADS:-0}" diff --git a/frameworks/proxygen/ArenaCommon.h b/frameworks/proxygen/ArenaCommon.h deleted file mode 100644 index 3e4f9b32b..000000000 --- a/frameworks/proxygen/ArenaCommon.h +++ /dev/null @@ -1,214 +0,0 @@ -#pragma once - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include - -namespace httparena { - -inline constexpr uint64_t kMaxWebSocketMessage = 16ULL * 1024 * 1024; -inline constexpr std::string_view kJsonPrefix = "/json/"; -inline constexpr std::string_view kStaticPrefix = "/static/"; -inline constexpr std::string_view kStaticRoot = "/data/static/"; - -inline bool parseInteger(std::string_view input, int64_t &value) { - while (!input.empty() && - std::isspace(static_cast(input.front()))) { - input.remove_prefix(1); - } - while (!input.empty() && - std::isspace(static_cast(input.back()))) { - input.remove_suffix(1); - } - if (input.empty()) { - return false; - } - const auto result = - std::from_chars(input.data(), input.data() + input.size(), value); - return result.ec == std::errc() && result.ptr == input.data() + input.size(); -} - -inline bool checkedAdd(int64_t lhs, int64_t rhs, int64_t &result) { -#if defined(__GNUC__) || defined(__clang__) - return !__builtin_add_overflow(lhs, rhs, &result); -#else - if ((rhs > 0 && lhs > std::numeric_limits::max() - rhs) || - (rhs < 0 && lhs < std::numeric_limits::min() - rhs)) { - return false; - } - result = lhs + rhs; - return true; -#endif -} - -inline bool checkedMultiply(int64_t lhs, int64_t rhs, int64_t &result) { -#if defined(__GNUC__) || defined(__clang__) - return !__builtin_mul_overflow(lhs, rhs, &result); -#else - if (lhs > 0) { - if ((rhs > 0 && lhs > std::numeric_limits::max() / rhs) || - (rhs < 0 && rhs < std::numeric_limits::min() / lhs)) { - return false; - } - } else if (lhs < 0) { - if ((rhs > 0 && lhs < std::numeric_limits::min() / rhs) || - (rhs < 0 && rhs < std::numeric_limits::max() / lhs)) { - return false; - } - } - result = lhs * rhs; - return true; -#endif -} - -inline std::string contentType(std::string_view name) { - const auto endsWith = [name](std::string_view suffix) { - return name.size() >= suffix.size() && - name.substr(name.size() - suffix.size()) == suffix; - }; - if (endsWith(".css")) { - return "text/css"; - } - if (endsWith(".js")) { - return "application/javascript"; - } - if (endsWith(".html")) { - return "text/html"; - } - if (endsWith(".json")) { - return "application/json"; - } - if (endsWith(".svg")) { - return "image/svg+xml"; - } - if (endsWith(".webp")) { - return "image/webp"; - } - if (endsWith(".woff2")) { - return "font/woff2"; - } - return "application/octet-stream"; -} - -inline std::shared_ptr loadDataset() { - std::ifstream input("/data/dataset.json", std::ios::binary); - if (!input) { - throw std::runtime_error("cannot open /data/dataset.json"); - } - std::string contents((std::istreambuf_iterator(input)), - std::istreambuf_iterator()); - auto dataset = folly::parseJson(contents); - if (!dataset.isArray() || dataset.size() < 50) { - throw std::runtime_error("/data/dataset.json must contain 50 items"); - } - return std::make_shared(std::move(dataset)); -} - -inline bool validWebSocketKey(std::string_view key) noexcept { - if (key.size() != 24) { - return false; - } - std::array decoded{}; - const auto result = folly::base64Decode(key, decoded.data()); - return result.is_success && result.o == decoded.data() + 16; -} - -inline bool validUtf8(const uint8_t *data, size_t size) noexcept { - const auto continuation = [](uint8_t byte) { - return byte >= 0x80 && byte <= 0xbf; - }; - - size_t index = 0; - while (index < size) { - const uint8_t first = data[index]; - if (first <= 0x7f) { - ++index; - continue; - } - if (first >= 0xc2 && first <= 0xdf) { - if (index + 1 >= size || !continuation(data[index + 1])) { - return false; - } - index += 2; - continue; - } - if (first == 0xe0) { - if (index + 2 >= size || data[index + 1] < 0xa0 || - data[index + 1] > 0xbf || !continuation(data[index + 2])) { - return false; - } - index += 3; - continue; - } - if ((first >= 0xe1 && first <= 0xec) || (first >= 0xee && first <= 0xef)) { - if (index + 2 >= size || !continuation(data[index + 1]) || - !continuation(data[index + 2])) { - return false; - } - index += 3; - continue; - } - if (first == 0xed) { - if (index + 2 >= size || data[index + 1] < 0x80 || - data[index + 1] > 0x9f || !continuation(data[index + 2])) { - return false; - } - index += 3; - continue; - } - if (first == 0xf0) { - if (index + 3 >= size || data[index + 1] < 0x90 || - data[index + 1] > 0xbf || !continuation(data[index + 2]) || - !continuation(data[index + 3])) { - return false; - } - index += 4; - continue; - } - if (first >= 0xf1 && first <= 0xf3) { - if (index + 3 >= size || !continuation(data[index + 1]) || - !continuation(data[index + 2]) || !continuation(data[index + 3])) { - return false; - } - index += 4; - continue; - } - if (first == 0xf4) { - if (index + 3 >= size || data[index + 1] < 0x80 || - data[index + 1] > 0x8f || !continuation(data[index + 2]) || - !continuation(data[index + 3])) { - return false; - } - index += 4; - continue; - } - return false; - } - return true; -} - -inline bool validUtf8(const std::vector &data) noexcept { - return validUtf8(data.data(), data.size()); -} - -inline bool validWebSocketCloseCode(uint16_t code) noexcept { - const bool definedProtocolCode = code >= 1000 && code <= 1014 && - code != 1004 && code != 1005 && code != 1006; - const bool applicationCode = code >= 3000 && code <= 4999; - return definedProtocolCode || applicationCode; -} - -} // namespace httparena diff --git a/frameworks/proxygen/CMakeLists.txt b/frameworks/proxygen/CMakeLists.txt index 35df1fa42..176ca4658 100644 --- a/frameworks/proxygen/CMakeLists.txt +++ b/frameworks/proxygen/CMakeLists.txt @@ -6,6 +6,11 @@ set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_CXX_EXTENSIONS OFF) +# Which server to build. Both HttpArena entries share this source tree: +# classic -> proxygen::HTTPServer + the mvfst-backed HQServer (`proxygen`) +# coro -> proxygen::coro::HTTPServer (`proxygen-coro`) +set(SERVER_TARGET "classic" CACHE STRING "classic or coro") + # Proxygen's exported Fizz package calls find_dependency(Sodium). The official # builder image keeps that upstream find module with the Proxygen source tree. list(APPEND CMAKE_MODULE_PATH "/proxygen/build/fbcode_builder/CMake") @@ -17,17 +22,35 @@ add_library(cares ALIAS c-ares::cares) find_package(proxygen CONFIG REQUIRED) -add_executable(proxygen-arena ArenaHttpServer.cpp ArenaHQServer.cpp) -target_compile_options(proxygen-arena PRIVATE -Wall -Wextra -Wpedantic) -target_link_libraries( - proxygen-arena - PRIVATE - proxygen::proxygen - proxygen::proxygenhttpserver - proxygen::proxygen_hq_samples - proxygen::proxygen_hq_server - proxygen::proxygen_transport_persistent_quic_psk_cache - proxygen::proxygen_httpserver - Folly::folly_init_init - Folly::folly_portability_gflags -) +if(SERVER_TARGET STREQUAL "coro") + add_executable(arena-server src/CoroServer.cpp) + target_link_libraries( + arena-server + PRIVATE + proxygen::proxygen + proxygen::proxygen_coro + proxygen::proxygen_coro_server + proxygen::proxygen_http_coro_filters_compression_filter_factory + Folly::folly_init_init + Folly::folly_portability_gflags + ) +elseif(SERVER_TARGET STREQUAL "classic") + add_executable(arena-server src/ClassicServer.cpp src/ArenaHQServer.cpp) + target_link_libraries( + arena-server + PRIVATE + proxygen::proxygen + proxygen::proxygenhttpserver + proxygen::proxygen_hq_samples + proxygen::proxygen_hq_server + proxygen::proxygen_transport_persistent_quic_psk_cache + proxygen::proxygen_httpserver + Folly::folly_init_init + Folly::folly_portability_gflags + ) +else() + message(FATAL_ERROR "SERVER_TARGET must be 'classic' or 'coro', got '${SERVER_TARGET}'") +endif() + +target_include_directories(arena-server PRIVATE src) +target_compile_options(arena-server PRIVATE -Wall -Wextra -Wpedantic) diff --git a/frameworks/proxygen/Dockerfile b/frameworks/proxygen/Dockerfile index ac4a0b30f..2c67721b2 100644 --- a/frameworks/proxygen/Dockerfile +++ b/frameworks/proxygen/Dockerfile @@ -1,30 +1,61 @@ -FROM ghcr.io/facebook/proxygen/base:latest AS build +# Shared image for both HttpArena Proxygen entries. +# +# TARGET=classic -> proxygen::HTTPServer + mvfst HQServer (`proxygen`) +# TARGET=coro -> proxygen::coro::HTTPServer (`proxygen-coro`) +# +# `frameworks/proxygen-coro/build.sh` builds this same context with +# `--build-arg TARGET=coro`, so every fix to src/ lands in both entries at once. +# +# PROXYGEN_BASE is pinned by digest so a benchmark round is reproducible; bump +# it deliberately rather than tracking a moving `:latest`. +ARG PROXYGEN_BASE=ghcr.io/facebook/proxygen/base@sha256:3f9c745ea7a3e065bdf6887f7255f2547c17c67fc1210ff120e9e52daa8eb3dc +ARG MIMALLOC_TAG=v2.1.7 + +# glibc malloc is the bottleneck on the allocation-heavy profiles: the JSON +# routes build a live folly::dynamic per request (up to 50 item copies plus the +# serialized string), and proxygen's coroutine server allocates a frame per +# event. Measured on this workload, mimalloc is worth +35% on `json` and +# +39%/+88% (classic/coro) on `json-comp`, against roughly -5% on `static`, +# where the large per-file IOBuf hits mimalloc's large-object path. +FROM ${PROXYGEN_BASE} AS mimalloc +ARG MIMALLOC_TAG +RUN git clone --depth 1 -b "${MIMALLOC_TAG}" https://github.com/microsoft/mimalloc.git /src/mimalloc \ + && cmake -S /src/mimalloc -B /src/build -DCMAKE_BUILD_TYPE=Release -DMI_BUILD_TESTS=OFF \ + && cmake --build /src/build --parallel "$(nproc)" \ + && cp "$(readlink -f /src/build/libmimalloc.so)" /src/libmimalloc.so + +FROM ${PROXYGEN_BASE} AS build +ARG TARGET=classic WORKDIR /arena -COPY CMakeLists.txt ArenaCommon.h ArenaHQServer.h ArenaHttpServer.cpp ArenaHQServer.cpp ./ -RUN cmake -S . -B build -DCMAKE_BUILD_TYPE=Release \ +COPY CMakeLists.txt ./ +COPY src ./src +RUN cmake -S . -B build -DCMAKE_BUILD_TYPE=Release -DSERVER_TARGET="${TARGET}" \ && cmake --build build --parallel "$(nproc)" \ - && strip build/proxygen-arena + && strip build/arena-server # Follow Proxygen's quic-interop image pattern: preserve the resolved shared # library paths, then copy only those libraries and the server binary to # a same-distro runtime image. RUN set -eux; \ - ldd build/proxygen-arena \ + ldd build/arena-server \ | awk '/=> \// { print $3 } /^\// { print $1 }' | sort -u > /tmp/runtime-libs.txt; \ tar -chf /tmp/runtime-libs.tar --files-from=/tmp/runtime-libs.txt FROM ubuntu:24.04@sha256:019e8eb29a85e74d64925745884f2ec79aa27e3feab36353d24656f4d6b89467 -ENV LD_LIBRARY_PATH=/opt/proxygen/lib +ENV LD_LIBRARY_PATH=/opt/proxygen/lib \ + LD_PRELOAD=/usr/local/lib/libmimalloc.so \ + MIMALLOC_PURGE_DELAY=1 COPY --from=build /tmp/runtime-libs.tar /tmp/runtime-libs.tar RUN tar -xf /tmp/runtime-libs.tar -C / \ && rm /tmp/runtime-libs.tar -COPY --from=build /arena/build/proxygen-arena /usr/local/bin/proxygen-arena -COPY entrypoint.sh /usr/local/bin/proxygen-entrypoint -RUN chmod +x /usr/local/bin/proxygen-entrypoint \ +COPY --from=mimalloc /src/libmimalloc.so /usr/local/lib/libmimalloc.so +COPY --from=build /arena/build/arena-server /usr/local/bin/arena-server +COPY entrypoint.sh /usr/local/bin/arena-entrypoint +RUN chmod +x /usr/local/bin/arena-entrypoint \ && groupadd --system --gid 10001 httparena \ && useradd --system --uid 10001 --gid httparena --no-create-home \ --home-dir /nonexistent --shell /usr/sbin/nologin httparena @@ -32,4 +63,4 @@ RUN chmod +x /usr/local/bin/proxygen-entrypoint \ EXPOSE 8080/tcp 8081/tcp 8082/tcp 8443/tcp 8443/udp USER httparena -ENTRYPOINT ["/usr/local/bin/proxygen-entrypoint"] +ENTRYPOINT ["/usr/local/bin/arena-entrypoint"] diff --git a/frameworks/proxygen/README.md b/frameworks/proxygen/README.md index 8efe071b5..e83920427 100644 --- a/frameworks/proxygen/README.md +++ b/frameworks/proxygen/README.md @@ -1,19 +1,17 @@ # Proxygen -This engine entry uses [Meta's Proxygen](https://github.com/facebook/proxygen) -for every advertised protocol: +[Meta's Proxygen](https://github.com/facebook/proxygen) serving every protocol +HttpArena exercises, using Proxygen's callback server APIs: -- Proxygen `HTTPServer` listens on TCP port 8080 for HTTP/1.1 and RFC 6455 - WebSocket upgrades, on TCP port 8081 for HTTP/1.1 over TLS (ALPN - `http/1.1` only), on TCP port 8082 for prior-knowledge h2c, and with - TLS/ALPN `h2` on TCP port 8443. -- Proxygen's mvfst-backed `HQServer` listens with ALPN `h3` on UDP port 8443. +- `proxygen::HTTPServer` on TCP 8080 for HTTP/1.1 and RFC 6455 WebSocket + upgrades, TCP 8081 for HTTP/1.1 over TLS (ALPN `http/1.1` only), TCP 8082 for + prior-knowledge h2c, and TCP 8443 with TLS/ALPN `h2`. +- Proxygen's mvfst-backed `HQServer` with ALPN `h3` on UDP 8443. -Both server APIs run in one process. TCP and QUIC each retain an -affinity-aware I/O pool so whichever transport is being benchmarked can use -the full CPU set while the other pool sleeps. The HTTP/2 and HTTP/3 listeners -use Proxygen's benchmark-oriented flow-control, stream-concurrency, GSO -batching, and write-path settings. +Both server APIs run in one process, each with its own affinity-aware I/O pool, +so whichever transport is being benchmarked can use the full CPU set while the +other pool sleeps. Size them independently with `PROXYGEN_THREADS` and +`PROXYGEN_H3_THREADS` (`0` = available CPUs). | Listener | Endpoints | Subscribed profiles | | --- | --- | --- | @@ -23,50 +21,85 @@ batching, and write-path settings. | HTTP/2 TLS `:8443` | `/baseline2`, `/static/*` | `baseline-h2`, `static-h2` | | HTTP/3 QUIC `:8443` | `/baseline2`, `/static/*` | `baseline-h3`, `static-h3` | -The JSON routes load the immutable dataset once, build each requested slice -and derived `total` fields per request, and serialize the live object with -Folly. Proxygen's standard content-compression path provides conditional gzip -for clients that advertise it. Uploads count bytes delivered through the body -callbacks rather than trusting `Content-Length`; static files are read from -disk for each request. +The JSON routes load the immutable dataset once, then build each requested slice +and its derived `total` fields per request and serialize the live object with +Folly. Uploads count bytes delivered through the body callbacks rather than +trusting `Content-Length`. + +## Static assets + +`/data/static` is mounted read-only and does not change during a run, so every +file and its precompressed `.br` / `.gz` siblings are read once at startup into +an immutable table, and responses are non-owning `IOBuf` views over it: no disk +I/O, no compression, no copy and no allocation for the payload per request. Both +in-memory caching and serving the precompressed variants are explicitly allowed +for `engine` entries ("No specific rules"). Variant choice is a delimited token +match on `Accept-Encoding` honouring `q=0`, preferring brotli, then gzip, then +the byte-exact original — which is what a client that sends no `Accept-Encoding` +always gets. A lookup miss is a 404, so path traversal is impossible by +construction rather than by filtering. + +This replaced a per-request `open`/`read` plus an on-the-fly gzip of every +CSS/JS/HTML response, which was costing about 80x throughput and 6.7x memory. + +Response compression via Proxygen's `CompressionFilter` is therefore scoped to +`application/json`, the only content type still compressed at request time and +the one `json-comp` is scored on. The gzip level is Proxygen's default +(`Z_DEFAULT_COMPRESSION`); level 9 measured as a net loss, trading 2.3% of +throughput for 0.35% smaller bodies under the `(minBpr/myBpr)²` scoring. The WebSocket handler uses Proxygen's upgrade handshake (including its per-connection `Sec-WebSocket-Accept` calculation) and implements incremental -RFC 6455 frame parsing. Client frames are unmasked before text or binary data -is echoed; fragmented messages, multiple frames per read, ping/pong, and close -frames are handled explicitly. This entry only claims WebSocket support over -HTTP/1.1, which is the protocol HttpArena's WebSocket profiles exercise. +RFC 6455 frame parsing: client frames are unmasked before text or binary data is +echoed, and fragmented messages, multiple frames per read, ping/pong and close +frames are all handled. This entry claims WebSocket support over HTTP/1.1 only, +which is the protocol HttpArena's WebSocket profiles exercise. -## Upstream image and Docker build +## Shared source tree -The builder tracks the official `ghcr.io/facebook/proxygen/base:latest` image. -The final stage contains only the Arena binary and its resolved shared -libraries; its matching Ubuntu 24.04 runtime is pinned by digest. The final -image runs the servers as the unprivileged `httparena` user (UID/GID 10001). +`frameworks/proxygen` is the build context for **both** Proxygen entries. The +`TARGET` build arg picks the server, the same way `sark-h3` reuses `sark`: -The build and launch arrangement follows Proxygen's upstream container and -coroutine benchmark patterns: +| Entry | Build | Server API | +| --- | --- | --- | +| `proxygen` | `docker build frameworks/proxygen` | `src/ClassicServer.cpp` + `src/ArenaHQServer.cpp` | +| `proxygen-coro` | `frameworks/proxygen-coro/build.sh` (`--build-arg TARGET=coro`) | `src/CoroServer.cpp` | -- `Dockerfile`: build in a full Proxygen environment, discover runtime - libraries with `ldd`, and copy them into a same-distribution runtime image. -- `HTTPCoroBenchmark.cpp`: configure QUIC with GSO batching, continuous-memory - writes, a large congestion window, and a 48-packet write batch. +`src/ArenaCommon.h` holds the routing, parsing, static-asset and validation +helpers both servers share, so a fix lands in both entries at once. -HttpArena mounts `/certs/server.crt`, `/certs/server.key`, -`/data/dataset.json`, and `/data/static` at runtime. No benchmark data is baked -into the image. +## Allocator -## Local validation +Both images `LD_PRELOAD` mimalloc (pinned by tag in the Dockerfile). glibc malloc +is the binding constraint on the allocation-heavy profiles — the JSON routes build +a live `folly::dynamic` per request (up to 50 item copies plus the serialized +string), and the coroutine server allocates a frame per event. Measured here it is +worth roughly +38% on `json` and +41%/+82% (classic/coro) on `json-comp`. +`MIMALLOC_PURGE_DELAY=1` is set because the default (10 ms) retains ~17% more +memory on `json-comp` at 16384 connections for no throughput gain; going to 0 +would cut memory 4x further but costs 44% of that profile, so it is not used. -From the repository root on a host where the standard ports are free: +## Image -```bash -./scripts/validate.sh proxygen -``` +The builder tracks `ghcr.io/facebook/proxygen/base`, pinned by digest in +`ARG PROXYGEN_BASE` so a benchmark round is reproducible — bump it deliberately +rather than tracking `:latest`. The final stage contains only the Arena binary +and the shared libraries `ldd` resolves for it, on a digest-pinned Ubuntu 24.04, +running as the unprivileged `httparena` user (UID/GID 10001). -The full benchmark driver is required for the complete 18-profile metadata set -(the lite driver currently rejects profiles it does not know about): +Note that the prebuilt Proxygen, folly, fizz, wangle and mvfst libraries in the +base image are built `RelWithDebInfo` (`-O2 -g -DNDEBUG`): `getdeps.py` defaults +`--build-type` to `RelWithDebInfo` and Proxygen's `docker/base.Dockerfile` does +not override it. Only this repository's translation unit is `-O3`. + +HttpArena mounts `/certs/server.crt`, `/certs/server.key`, `/data/dataset.json` +and `/data/static` at runtime; no benchmark data is baked into the image. + +## Local validation + +From the repository root, on a host where the standard ports are free: ```bash -LOADGEN_DOCKER=true SKIP_TUNE=true ./scripts/benchmark.sh proxygen --save +./scripts/validate.sh proxygen +./scripts/benchmark.sh proxygen baseline ``` diff --git a/frameworks/proxygen/entrypoint.sh b/frameworks/proxygen/entrypoint.sh index 14a667c06..e63b25e1d 100644 --- a/frameworks/proxygen/entrypoint.sh +++ b/frameworks/proxygen/entrypoint.sh @@ -1,7 +1,12 @@ #!/usr/bin/env bash set -euo pipefail -exec /usr/local/bin/proxygen-arena \ +# Shared by the `proxygen` and `proxygen-coro` images; the binary is the same +# name in both, only the server API compiled into it differs. +# +# PROXYGEN_THREADS TCP (H1/H2) I/O threads, 0 = available CPUs +# PROXYGEN_H3_THREADS QUIC I/O threads, 0 = available CPUs +exec /usr/local/bin/arena-server \ --ip=:: \ --http_port=8080 \ --tls_port=8081 \ @@ -10,4 +15,5 @@ exec /usr/local/bin/proxygen-arena \ --h3_port=8443 \ --cert=/certs/server.crt \ --key=/certs/server.key \ - --threads="${PROXYGEN_THREADS:-0}" + --threads="${PROXYGEN_THREADS:-0}" \ + --h3_threads="${PROXYGEN_H3_THREADS:-0}" diff --git a/frameworks/proxygen/src/ArenaCommon.h b/frameworks/proxygen/src/ArenaCommon.h new file mode 100644 index 000000000..8ad2e22ef --- /dev/null +++ b/frameworks/proxygen/src/ArenaCommon.h @@ -0,0 +1,392 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace httparena { + +inline constexpr uint64_t kMaxWebSocketMessage = 16ULL * 1024 * 1024; +inline constexpr size_t kMaxRequestBody = 1024; +inline constexpr std::string_view kJsonPrefix = "/json/"; +inline constexpr std::string_view kStaticPrefix = "/static/"; +inline constexpr std::string_view kStaticRoot = "/data/static/"; + +inline bool parseInteger(std::string_view input, int64_t &value) { + while (!input.empty() && + std::isspace(static_cast(input.front()))) { + input.remove_prefix(1); + } + while (!input.empty() && + std::isspace(static_cast(input.back()))) { + input.remove_suffix(1); + } + if (input.empty()) { + return false; + } + const auto result = + std::from_chars(input.data(), input.data() + input.size(), value); + return result.ec == std::errc() && result.ptr == input.data() + input.size(); +} + +inline bool checkedAdd(int64_t lhs, int64_t rhs, int64_t &result) { +#if defined(__GNUC__) || defined(__clang__) + return !__builtin_add_overflow(lhs, rhs, &result); +#else + if ((rhs > 0 && lhs > std::numeric_limits::max() - rhs) || + (rhs < 0 && lhs < std::numeric_limits::min() - rhs)) { + return false; + } + result = lhs + rhs; + return true; +#endif +} + +inline bool checkedMultiply(int64_t lhs, int64_t rhs, int64_t &result) { +#if defined(__GNUC__) || defined(__clang__) + return !__builtin_mul_overflow(lhs, rhs, &result); +#else + if (lhs > 0) { + if ((rhs > 0 && lhs > std::numeric_limits::max() / rhs) || + (rhs < 0 && rhs < std::numeric_limits::min() / lhs)) { + return false; + } + } else if (lhs < 0) { + if ((rhs > 0 && lhs < std::numeric_limits::min() / rhs) || + (rhs < 0 && rhs < std::numeric_limits::max() / lhs)) { + return false; + } + } + result = lhs * rhs; + return true; +#endif +} + +// Returns the value for `name` in a raw `a=1&b=2` query string, or nullopt if +// it is absent. Scanning beats HTTPMessage::getQueryParam() on the hot path: +// that accessor materialises a std::map of every +// parameter on first use, i.e. two string allocations per parameter per +// request, and `baseline` is the most-requested endpoint in the suite. +inline std::optional queryValue(std::string_view query, + std::string_view name) { + while (!query.empty()) { + const auto amp = query.find('&'); + const std::string_view field = query.substr(0, amp); + const auto eq = field.find('='); + if (eq != std::string_view::npos && field.substr(0, eq) == name) { + return field.substr(eq + 1); + } + if (amp == std::string_view::npos) { + break; + } + query.remove_prefix(amp + 1); + } + return std::nullopt; +} + +inline std::string_view contentType(std::string_view name) { + const auto endsWith = [name](std::string_view suffix) { + return name.size() >= suffix.size() && + name.substr(name.size() - suffix.size()) == suffix; + }; + if (endsWith(".css")) { + return "text/css"; + } + if (endsWith(".js")) { + return "application/javascript"; + } + if (endsWith(".html")) { + return "text/html"; + } + if (endsWith(".json")) { + return "application/json"; + } + if (endsWith(".svg")) { + return "image/svg+xml"; + } + if (endsWith(".webp")) { + return "image/webp"; + } + if (endsWith(".woff2")) { + return "font/woff2"; + } + return "application/octet-stream"; +} + +// ── Static assets ─────────────────────────────────────────────────────────── +// +// `/data/static` is mounted read-only and never changes during a run, so every +// file and its precompressed `.br` / `.gz` siblings are read once at startup +// and served straight out of memory. Both are explicitly allowed for `engine` +// entries ("No specific rules"), and the alternative — re-reading and, worse, +// re-gzipping 8-200 KB per request on the event base — was costing better than +// an order of magnitude. +// +// Bodies are handed out as non-owning IOBufs over this table, which lives for +// the life of the process: no copy, no allocation for the payload, and no +// atomic refcount on the shared buffer. + +enum class ContentEncoding : uint8_t { Identity, Gzip, Brotli }; + +inline std::string_view encodingToken(ContentEncoding encoding) { + switch (encoding) { + case ContentEncoding::Brotli: + return "br"; + case ContentEncoding::Gzip: + return "gzip"; + case ContentEncoding::Identity: + break; + } + return {}; +} + +struct StaticAsset { + std::string identity; + std::string brotli; // empty when there is no .br sibling + std::string gzip; // empty when there is no .gz sibling + std::string_view contentType; + + // Picks the best variant this client accepts. Falls back to identity, which + // is always present, so an absent or unrecognised Accept-Encoding still + // yields the byte-exact original. + std::pair + select(std::string_view acceptEncoding) const { + if (!brotli.empty() && acceptsToken(acceptEncoding, "br")) { + return {&brotli, ContentEncoding::Brotli}; + } + if (!gzip.empty() && acceptsToken(acceptEncoding, "gzip")) { + return {&gzip, ContentEncoding::Gzip}; + } + return {&identity, ContentEncoding::Identity}; + } + + // Token search rather than a full RFC 7231 qvalue parse: the token must be + // delimited so "br" does not match inside another coding, and an explicit + // `q=0` disqualifies it. + static bool acceptsToken(std::string_view header, std::string_view token) { + size_t pos = 0; + while ((pos = header.find(token, pos)) != std::string_view::npos) { + const bool leftOk = + pos == 0 || header[pos - 1] == ' ' || header[pos - 1] == ','; + const size_t after = pos + token.size(); + const bool rightOk = after == header.size() || header[after] == ',' || + header[after] == ';' || header[after] == ' '; + if (leftOk && rightOk) { + // Reject `;q=0` (but not `;q=0.8`). + const auto end = header.find(',', after); + const auto params = header.substr( + after, end == std::string_view::npos ? end : end - after); + const auto q = params.find("q="); + if (q == std::string_view::npos || + params.compare(q, 4, "q=0,") == 0 || params.substr(q) == "q=0" || + params.substr(q) == "q=0.0") { + return q == std::string_view::npos; + } + return true; + } + pos = after; + } + return false; + } +}; + +class StaticAssets { +public: + // Throws if the mount is missing; the arena always provides it. + void load() { + namespace fs = std::filesystem; + std::error_code ec; + fs::directory_iterator it(kStaticRoot, ec); + if (ec) { + throw std::runtime_error(std::string("cannot scan ") + + std::string(kStaticRoot) + ": " + ec.message()); + } + // Pass 1: the originals. Pass 2 attaches variants, so ordering within the + // directory listing does not matter. + std::vector variants; + for (const auto &entry : it) { + if (!entry.is_regular_file()) { + continue; + } + const auto name = entry.path().filename().string(); + if (name.ends_with(".br") || name.ends_with(".gz")) { + variants.push_back(entry.path()); + continue; + } + StaticAsset asset; + if (!folly::readFile(entry.path().c_str(), asset.identity)) { + continue; + } + asset.contentType = contentType(name); + assets_.emplace(name, std::move(asset)); + } + for (const auto &path : variants) { + const auto name = path.filename().string(); + const auto base = name.substr(0, name.size() - 3); + auto found = assets_.find(base); + if (found == assets_.end()) { + continue; // orphan variant with no original; ignore it + } + std::string &slot = + name.ends_with(".br") ? found->second.brotli : found->second.gzip; + if (!folly::readFile(path.c_str(), slot)) { + slot.clear(); + } + } + if (assets_.empty()) { + throw std::runtime_error("no static assets found under /data/static"); + } + } + + // A miss is simply a 404; because this is an exact lookup in a fixed table, + // path traversal is impossible by construction. + const StaticAsset *find(std::string_view name) const { + const auto found = assets_.find(name); + return found == assets_.end() ? nullptr : &found->second; + } + +private: + // Transparent hashing so lookups take a string_view without allocating. + struct Hash { + using is_transparent = void; + size_t operator()(std::string_view s) const noexcept { + return std::hash{}(s); + } + }; + struct Equal { + using is_transparent = void; + bool operator()(std::string_view a, std::string_view b) const noexcept { + return a == b; + } + }; + std::unordered_map assets_; +}; + +inline std::shared_ptr loadDataset() { + std::string contents; + if (!folly::readFile("/data/dataset.json", contents)) { + throw std::runtime_error("cannot open /data/dataset.json"); + } + auto dataset = folly::parseJson(contents); + if (!dataset.isArray() || dataset.size() < 50) { + throw std::runtime_error("/data/dataset.json must contain 50 items"); + } + return std::make_shared(std::move(dataset)); +} + +inline bool validWebSocketKey(std::string_view key) noexcept { + if (key.size() != 24) { + return false; + } + std::array decoded{}; + const auto result = folly::base64Decode(key, decoded.data()); + return result.is_success && result.o == decoded.data() + 16; +} + +inline bool validUtf8(const uint8_t *data, size_t size) noexcept { + const auto continuation = [](uint8_t byte) { + return byte >= 0x80 && byte <= 0xbf; + }; + + size_t index = 0; + while (index < size) { + const uint8_t first = data[index]; + if (first <= 0x7f) { + ++index; + continue; + } + if (first >= 0xc2 && first <= 0xdf) { + if (index + 1 >= size || !continuation(data[index + 1])) { + return false; + } + index += 2; + continue; + } + if (first == 0xe0) { + if (index + 2 >= size || data[index + 1] < 0xa0 || + data[index + 1] > 0xbf || !continuation(data[index + 2])) { + return false; + } + index += 3; + continue; + } + if ((first >= 0xe1 && first <= 0xec) || (first >= 0xee && first <= 0xef)) { + if (index + 2 >= size || !continuation(data[index + 1]) || + !continuation(data[index + 2])) { + return false; + } + index += 3; + continue; + } + if (first == 0xed) { + if (index + 2 >= size || data[index + 1] < 0x80 || + data[index + 1] > 0x9f || !continuation(data[index + 2])) { + return false; + } + index += 3; + continue; + } + if (first == 0xf0) { + if (index + 3 >= size || data[index + 1] < 0x90 || + data[index + 1] > 0xbf || !continuation(data[index + 2]) || + !continuation(data[index + 3])) { + return false; + } + index += 4; + continue; + } + if (first >= 0xf1 && first <= 0xf3) { + if (index + 3 >= size || !continuation(data[index + 1]) || + !continuation(data[index + 2]) || !continuation(data[index + 3])) { + return false; + } + index += 4; + continue; + } + if (first == 0xf4) { + if (index + 3 >= size || data[index + 1] < 0x80 || + data[index + 1] > 0x8f || !continuation(data[index + 2]) || + !continuation(data[index + 3])) { + return false; + } + index += 4; + continue; + } + return false; + } + return true; +} + +inline bool validUtf8(const std::vector &data) noexcept { + return validUtf8(data.data(), data.size()); +} + +inline bool validWebSocketCloseCode(uint16_t code) noexcept { + const bool definedProtocolCode = code >= 1000 && code <= 1014 && + code != 1004 && code != 1005 && code != 1006; + const bool applicationCode = code >= 3000 && code <= 4999; + return definedProtocolCode || applicationCode; +} + +} // namespace httparena diff --git a/frameworks/proxygen/ArenaHQServer.cpp b/frameworks/proxygen/src/ArenaHQServer.cpp similarity index 88% rename from frameworks/proxygen/ArenaHQServer.cpp rename to frameworks/proxygen/src/ArenaHQServer.cpp index 1bccb09b6..5dc8d040d 100644 --- a/frameworks/proxygen/ArenaHQServer.cpp +++ b/frameworks/proxygen/src/ArenaHQServer.cpp @@ -11,6 +11,11 @@ namespace { quic::samples::HQServerParams makeHQParams(size_t ioThreads) { quic::samples::HQServerParams params; params.serverThreads = ioThreads; + // HQServerParams leaves these at 0, i.e. the system default (typically + // 208 KB), which is small for a loopback benchmark writing 48-packet GSO + // batches per connection. + params.udpSendBufferSize = 4 * 1024 * 1024; + params.udpRecvBufferSize = 4 * 1024 * 1024; params.transportSettings.maxNumPTOs = 1000; params.transportSettings.maxCwndInMss = quic::kLargeMaxCwndInMss; params.transportSettings.batchingMode = diff --git a/frameworks/proxygen/ArenaHQServer.h b/frameworks/proxygen/src/ArenaHQServer.h similarity index 100% rename from frameworks/proxygen/ArenaHQServer.h rename to frameworks/proxygen/src/ArenaHQServer.h diff --git a/frameworks/proxygen/src/ArenaWebSocket.h b/frameworks/proxygen/src/ArenaWebSocket.h new file mode 100644 index 000000000..a0446befb --- /dev/null +++ b/frameworks/proxygen/src/ArenaWebSocket.h @@ -0,0 +1,381 @@ +#pragma once + +// Incremental RFC 6455 frame codec shared by ClassicServer and CoroServer. +// +// Reads normally carry whole frames, so the common path parses straight out of +// the transport's buffer with no intermediate queue: each frame is unmasked in +// place, and only a trailing partial frame is copied into `carry_`. +// +// Echoes are accumulated into one egress buffer per read, so a read carrying N +// pipelined frames produces one write rather than N. Small payloads (the echo +// profiles use ~128 bytes) are memcpy'd into that buffer, which is cheaper than +// the per-frame IOBuf bookkeeping a zero-copy view would cost. Large payloads +// cross over, and are emitted as a clone view of the read buffer instead: the +// answer to a frame reuses the same length encoding as the question, so the +// server header lands exactly on the four mask bytes preceding the payload. + +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "ArenaCommon.h" + +namespace httparena { + +class WebSocketEcho { +public: + enum class Opcode : uint8_t { + Continuation = 0x0, + Text = 0x1, + Binary = 0x2, + Close = 0x8, + Ping = 0x9, + Pong = 0xa, + }; + + // Payloads at or above this are echoed as a view of the read buffer; below + // it, copying into the shared egress buffer costs less than a second IOBuf. + static constexpr size_t kZeroCopyThreshold = 4096; + + void onIngress(std::unique_ptr data) { + if (finished_ || !data) { + return; + } + if (data->isChained()) { + data->coalesce(); + } + if (carry_.empty()) { + const size_t consumed = + parse(data->writableData(), data->length(), data.get()); + if (consumed < data->length()) { + carry_.assign(data->data() + consumed, data->data() + data->length()); + } + return; + } + append(carry_, data->data(), data->length()); + // No owning IOBuf for the carried bytes, so this path always copies. It + // only runs when a previous read ended mid-frame. + const size_t consumed = parse(carry_.data(), carry_.size(), nullptr); + if (consumed == carry_.size()) { + carry_.clear(); + } else if (consumed > 0) { + carry_.erase(carry_.begin(), + carry_.begin() + static_cast(consumed)); + } + } + + // Everything queued since the last call, or nullptr if there is nothing. + std::unique_ptr takeEgress() { + return egress_.empty() ? nullptr : egress_.move(); + } + + // At most `limit` bytes, for the coro HTTPSource, which is handed a per-read + // size cap. + std::unique_ptr takeEgress(size_t limit) { + if (egress_.empty()) { + return nullptr; + } + if (egress_.chainLength() <= limit) { + return egress_.move(); + } + return egress_.split(limit); + } + + bool hasEgress() const { return !egress_.empty(); } + + // True once a close frame has been answered or the stream is unusable: the + // caller should send EOM after flushing whatever is still queued. + bool finished() const { return finished_; } + +private: + // Writes an unmasked server frame header for `payloadLength` into `out`, + // which must have room for 10 bytes. Returns the header length. + static size_t writeHeader(uint8_t *out, uint8_t opcode, + uint64_t payloadLength) { + out[0] = static_cast(0x80U | opcode); + if (payloadLength <= 125) { + out[1] = static_cast(payloadLength); + return 2; + } + if (payloadLength <= std::numeric_limits::max()) { + out[1] = 126; + out[2] = static_cast((payloadLength >> 8) & 0xff); + out[3] = static_cast(payloadLength & 0xff); + return 4; + } + out[1] = 127; + for (int shift = 56, index = 2; shift >= 0; shift -= 8, ++index) { + out[index] = static_cast((payloadLength >> shift) & 0xff); + } + return 10; + } + + void queueFrame(uint8_t opcode, const uint8_t *payload, size_t length) { + std::array header; + const size_t headerLength = writeHeader(header.data(), opcode, length); + egress_.append(header.data(), headerLength); + if (length > 0) { + egress_.append(payload, length); + } + } + + void closeWith(uint16_t status) { + if (closeSent_ || finished_) { + return; + } + const std::array payload = { + static_cast((status >> 8) & 0xff), + static_cast(status & 0xff)}; + queueFrame(static_cast(Opcode::Close), payload.data(), + payload.size()); + closeSent_ = true; + finished_ = true; + } + + void protocolError() { closeWith(1002); } + void invalidPayload() { closeWith(1007); } + void messageTooBig() { closeWith(1009); } + + // resize + memcpy rather than vector::insert from a raw pointer range: + // same work, and GCC 13 cannot bound the iterator form, which trips a + // false-positive -Wstringop-overflow. + static void append(std::vector &out, const uint8_t *data, + size_t length) { + const size_t offset = out.size(); + out.resize(offset + length); + if (length > 0) { + std::memcpy(out.data() + offset, data, length); + } + } + + // Unmasks `length` bytes in place. The mask repeats every four bytes, so it + // is applied as a rotating 32-bit word rather than byte-at-a-time modulo. + static void unmask(uint8_t *payload, size_t length, + const uint8_t *mask) noexcept { + uint32_t maskWord; + std::memcpy(&maskWord, mask, sizeof(maskWord)); + size_t index = 0; + for (; index + sizeof(uint32_t) <= length; index += sizeof(uint32_t)) { + uint32_t chunk; + std::memcpy(&chunk, payload + index, sizeof(chunk)); + chunk ^= maskWord; + std::memcpy(payload + index, &chunk, sizeof(chunk)); + } + for (; index < length; ++index) { + payload[index] ^= mask[index & 3U]; + } + } + + // Echoes a large frame without copying its payload. `frame` points at the + // start of the received frame inside `owner`: + // [header(headerLength)][mask(4)][payload] + // The answer is [header(headerLength)][payload] with the same length + // encoding, so the server header lands on offset 4 and the view starts + // there. The mask has already been consumed by unmask(). + void echoAsView(folly::IOBuf *owner, uint8_t *frame, uint8_t opcode, + size_t headerLength, size_t payloadLength) { + writeHeader(frame + 4, opcode, payloadLength); + auto view = owner->cloneOne(); + const size_t offset = static_cast(frame - owner->writableData()); + view->trimStart(offset + 4); + view->trimEnd(view->length() - (headerLength + payloadLength)); + egress_.append(std::move(view), /*pack=*/false); + } + + void handleControlFrame(uint8_t opcode, uint8_t *payload, size_t length) { + switch (static_cast(opcode)) { + case Opcode::Close: + if (length == 1) { + protocolError(); + return; + } + if (length >= 2) { + const auto status = static_cast( + (static_cast(payload[0]) << 8) | payload[1]); + if (!validWebSocketCloseCode(status)) { + protocolError(); + return; + } + if (!validUtf8(payload + 2, length - 2)) { + invalidPayload(); + return; + } + } + if (!closeSent_) { + queueFrame(static_cast(Opcode::Close), payload, length); + closeSent_ = true; + } + finished_ = true; + return; + case Opcode::Ping: + queueFrame(static_cast(Opcode::Pong), payload, length); + return; + case Opcode::Pong: + return; + default: + protocolError(); + return; + } + } + + // Consumes as many whole frames as `length` holds, unmasking in place. + // `owner` is the IOBuf backing `base`, or nullptr when parsing carried bytes + // (in which case large frames are copied rather than viewed). Returns the + // number of bytes consumed. + size_t parse(uint8_t *base, size_t length, folly::IOBuf *owner) { + size_t cursor = 0; + while (!finished_) { + const size_t available = length - cursor; + if (available < 2) { + break; + } + uint8_t *frame = base + cursor; + const uint8_t first = frame[0]; + const uint8_t second = frame[1]; + const bool fin = (first & 0x80U) != 0; + const uint8_t opcode = first & 0x0fU; + const bool control = (opcode & 0x08U) != 0; + const uint8_t encodedLength = second & 0x7fU; + + // Reserved bits must be clear, clients must mask, and control frames may + // not use an extended length encoding even for a short payload. + if ((first & 0x70U) != 0 || (second & 0x80U) == 0 || + (control && encodedLength > 125)) { + protocolError(); + return length; + } + + size_t headerLength = 2; + uint64_t payloadLength = encodedLength; + if (encodedLength == 126) { + headerLength = 4; + if (available < headerLength) { + break; + } + payloadLength = (static_cast(frame[2]) << 8) | + static_cast(frame[3]); + if (payloadLength < 126) { // not minimally encoded + protocolError(); + return length; + } + } else if (encodedLength == 127) { + headerLength = 10; + if (available < headerLength) { + break; + } + if ((frame[2] & 0x80U) != 0) { // high bit of a 64-bit length is 0 + protocolError(); + return length; + } + payloadLength = 0; + for (size_t index = 0; index < 8; ++index) { + payloadLength = (payloadLength << 8) | frame[2 + index]; + } + if (payloadLength <= std::numeric_limits::max()) { + protocolError(); + return length; + } + } + + if (payloadLength > kMaxWebSocketMessage) { + messageTooBig(); + return length; + } + + constexpr size_t kMaskLength = 4; + const size_t frameLength = + headerLength + kMaskLength + static_cast(payloadLength); + if (available < frameLength) { + break; + } + + uint8_t *payload = frame + headerLength + kMaskLength; + const auto size = static_cast(payloadLength); + unmask(payload, size, frame + headerLength); + cursor += frameLength; + + if (control) { + if (!fin) { + protocolError(); + return length; + } + handleControlFrame(opcode, payload, size); + continue; + } + if (closeSent_) { + continue; // draining after our own close + } + + if (opcode == static_cast(Opcode::Continuation)) { + if (fragmentOpcode_ == 0) { + protocolError(); + return length; + } + if (size > kMaxWebSocketMessage - fragmentPayload_.size()) { + messageTooBig(); + return length; + } + append(fragmentPayload_, payload, size); + if (fin) { + finishFragmentedMessage(); + } + continue; + } + + if (opcode != static_cast(Opcode::Text) && + opcode != static_cast(Opcode::Binary)) { + protocolError(); + return length; + } + if (fragmentOpcode_ != 0) { // interleaved new message + protocolError(); + return length; + } + if (!fin) { + fragmentOpcode_ = opcode; + fragmentPayload_.assign(payload, payload + size); + continue; + } + if (opcode == static_cast(Opcode::Text) && + !validUtf8(payload, size)) { + invalidPayload(); + return length; + } + + if (owner != nullptr && size >= kZeroCopyThreshold) { + echoAsView(owner, frame, opcode, headerLength, size); + } else { + queueFrame(opcode, payload, size); + } + } + return cursor; + } + + void finishFragmentedMessage() { + if (fragmentOpcode_ == static_cast(Opcode::Text) && + !validUtf8(fragmentPayload_)) { + invalidPayload(); + return; + } + queueFrame(fragmentOpcode_, fragmentPayload_.data(), + fragmentPayload_.size()); + fragmentOpcode_ = 0; + fragmentPayload_.clear(); + fragmentPayload_.shrink_to_fit(); + } + + folly::IOBufQueue egress_{folly::IOBufQueue::cacheChainLength()}; + std::vector carry_; + std::vector fragmentPayload_; + uint8_t fragmentOpcode_{0}; + bool closeSent_{false}; + bool finished_{false}; +}; + +} // namespace httparena diff --git a/frameworks/proxygen/ArenaHttpServer.cpp b/frameworks/proxygen/src/ClassicServer.cpp similarity index 54% rename from frameworks/proxygen/ArenaHttpServer.cpp rename to frameworks/proxygen/src/ClassicServer.cpp index 09d504575..07709c5a5 100644 --- a/frameworks/proxygen/ArenaHttpServer.cpp +++ b/frameworks/proxygen/src/ClassicServer.cpp @@ -1,12 +1,12 @@ #include "ArenaCommon.h" #include "ArenaHQServer.h" +#include "ArenaWebSocket.h" #include #include #include #include #include -#include #include #include #include @@ -47,31 +47,35 @@ DEFINE_int32(h3_port, 8443, "HTTP/3 over QUIC port"); DEFINE_string(ip, "::", "Address on which to listen"); DEFINE_string(cert, "/certs/server.crt", "TLS certificate path"); DEFINE_string(key, "/certs/server.key", "TLS private-key path"); -DEFINE_int32(threads, 0, "I/O threads; 0 uses the available CPU count"); +DEFINE_int32(threads, 0, "TCP I/O threads; 0 uses the available CPU count"); +DEFINE_int32(h3_threads, 0, "QUIC I/O threads; 0 uses the available CPU count"); namespace { -constexpr size_t kMaxRequestBody = 1024; using httparena::checkedAdd; using httparena::checkedMultiply; using httparena::contentType; using httparena::kJsonPrefix; -using httparena::kMaxWebSocketMessage; +using httparena::kMaxRequestBody; using httparena::kStaticPrefix; -using httparena::kStaticRoot; using httparena::loadDataset; using httparena::parseInteger; -using httparena::validUtf8; -using httparena::validWebSocketCloseCode; +using httparena::queryValue; +using httparena::ContentEncoding; +using httparena::StaticAssets; using httparena::validWebSocketKey; class ArenaHandler final : public RequestHandler { public: - explicit ArenaHandler(std::shared_ptr dataset) - : dataset_(std::move(dataset)) {} + // The dataset is owned by the handler factory, which outlives every handler + // it creates. Taking a reference rather than a shared_ptr copy keeps two + // atomic refcount updates on a globally shared cache line off the per-request + // path — one handler is allocated per request. + ArenaHandler(const folly::dynamic &dataset, const StaticAssets &assets) + : dataset_(dataset), assets_(assets) {} void onRequest(std::unique_ptr request) noexcept override { - const auto path = request->getPath(); + const auto path = request->getPathAsStringPiece(); const auto method = request->getMethod(); if (path == "/ws") { @@ -102,27 +106,25 @@ class ArenaHandler final : public RequestHandler { } method_ = method.value_or(HTTPMethod::GET); - if (path == "/baseline11") { - route_ = Route::Baseline; - queryValid_ = parseInteger(request->getQueryParam("a"), a_) && - parseInteger(request->getQueryParam("b"), b_); - return; - } - if (path == "/baseline2") { - route_ = Route::BaselineH2; - queryValid_ = parseInteger(request->getQueryParam("a"), a_) && - parseInteger(request->getQueryParam("b"), b_); + const std::string_view query(request->getQueryStringAsStringPiece().data(), + request->getQueryStringAsStringPiece().size()); + + if (path == "/baseline11" || path == "/baseline2") { + route_ = path == "/baseline11" ? Route::Baseline : Route::BaselineH2; + const auto a = queryValue(query, "a"); + const auto b = queryValue(query, "b"); + queryValid_ = a && b && parseInteger(*a, a_) && parseInteger(*b, b_); return; } - if (path.starts_with(kJsonPrefix)) { + if (path.startsWith(kJsonPrefix)) { route_ = Route::Json; const std::string_view countText(path.data() + kJsonPrefix.size(), path.size() - kJsonPrefix.size()); int64_t count = 0; - const auto multiplierText = request->getQueryParam("m"); + const auto multiplierText = queryValue(query, "m"); const bool multiplierValid = - multiplierText.empty() ? (multiplier_ = 1, true) - : parseInteger(multiplierText, multiplier_); + !multiplierText ? (multiplier_ = 1, true) + : parseInteger(*multiplierText, multiplier_); jsonValid_ = parseInteger(countText, count) && count >= 1 && count <= 50 && multiplierValid; if (jsonValid_) { @@ -134,10 +136,14 @@ class ArenaHandler final : public RequestHandler { route_ = Route::Upload; return; } - if (path.starts_with(kStaticPrefix)) { + if (path.startsWith(kStaticPrefix)) { route_ = Route::Static; staticName_.assign(path.data() + kStaticPrefix.size(), path.size() - kStaticPrefix.size()); + const auto &encoding = + request->getHeaders().getSingleOrEmpty( + proxygen::HTTP_HEADER_ACCEPT_ENCODING); + acceptEncoding_.assign(encoding.data(), encoding.size()); return; } if (path == "/pipeline") { @@ -161,9 +167,8 @@ class ArenaHandler final : public RequestHandler { return; } if (route_ == Route::WebSocket && websocketActive_) { - auto bytes = body->coalesce(); - websocketBytes_.insert(websocketBytes_.end(), bytes.begin(), bytes.end()); - processWebSocketFrames(); + websocket_.onIngress(std::move(body)); + flushWebSocket(); return; } if (route_ != Route::Baseline) { @@ -220,7 +225,13 @@ class ArenaHandler final : public RequestHandler { } else if (!uploadValid_) { sendText(400, "Bad Request", "upload too large"); } else { - sendText(200, "OK", std::to_string(uploadBytes_)); + std::array digits; + const auto end = std::to_chars( + digits.data(), digits.data() + digits.size(), uploadBytes_); + sendText(200, "OK", + std::string_view( + digits.data(), + static_cast(end.ptr - digits.data()))); } return; case Route::Static: @@ -269,7 +280,12 @@ class ArenaHandler final : public RequestHandler { return; } } - sendText(200, "OK", std::to_string(sum)); + std::array digits; + const auto end = std::to_chars(digits.data(), digits.data() + digits.size(), + sum); + sendText(200, "OK", + std::string_view(digits.data(), + static_cast(end.ptr - digits.data()))); } void handleJson() { @@ -277,15 +293,16 @@ class ArenaHandler final : public RequestHandler { sendText(405, "Method Not Allowed", "method not allowed"); return; } - if (!jsonValid_ || jsonCount_ > dataset_->size()) { + if (!jsonValid_ || jsonCount_ > dataset_.size()) { sendText(400, "Bad Request", "invalid JSON parameters"); return; } try { folly::dynamic items = folly::dynamic::array; + items.reserve(jsonCount_); for (size_t index = 0; index < jsonCount_; ++index) { - folly::dynamic item = (*dataset_)[index]; + folly::dynamic item = dataset_[index]; int64_t subtotal = 0; int64_t total = 0; if (!checkedMultiply(item["price"].asInt(), item["quantity"].asInt(), @@ -300,7 +317,10 @@ class ArenaHandler final : public RequestHandler { folly::dynamic response = folly::dynamic::object; response["items"] = std::move(items); response["count"] = static_cast(jsonCount_); - sendResponse(200, "OK", "application/json", folly::toJson(response)); + // fromString takes ownership of the serialized buffer; copyBuffer would + // memcpy up to ~30 KB per response and shows up directly in json-comp. + sendResponse(200, "OK", "application/json", + folly::IOBuf::fromString(folly::toJson(response))); } catch (const std::exception &) { sendText(500, "Internal Server Error", "JSON serialization failed"); } @@ -311,266 +331,62 @@ class ArenaHandler final : public RequestHandler { sendText(405, "Method Not Allowed", "method not allowed"); return; } - if (staticName_.empty() || staticName_.find('/') != std::string::npos || - staticName_.find('\\') != std::string::npos || - staticName_.find("..") != std::string::npos) { + const auto *asset = assets_.find(staticName_); + if (asset == nullptr) { sendText(404, "Not Found", "not found"); return; } + const auto [body, encoding] = asset->select(acceptEncoding_); - std::ifstream input(std::string(kStaticRoot) + staticName_, - std::ios::binary); - if (!input) { - sendText(404, "Not Found", "not found"); - return; - } - std::string body((std::istreambuf_iterator(input)), - std::istreambuf_iterator()); - if (!input.good() && !input.eof()) { - sendText(500, "Internal Server Error", "read error"); - return; - } - sendResponse(200, "OK", contentType(staticName_), std::move(body)); + responseFinished_ = true; + ResponseBuilder builder(downstream_); + builder.status(200, "OK") + .header(proxygen::HTTP_HEADER_CONTENT_TYPE, asset->contentType); + if (encoding != ContentEncoding::Identity) { + builder.header(proxygen::HTTP_HEADER_CONTENT_ENCODING, + httparena::encodingToken(encoding)); + // The same URL yields different bytes per Accept-Encoding. + builder.header(proxygen::HTTP_HEADER_VARY, "Accept-Encoding"); + } + // Non-owning view of the preloaded table, which outlives every request. + builder.body(folly::IOBuf::wrapBuffer(body->data(), body->size())) + .sendWithEOM(); } - void sendResponse(uint16_t status, const std::string &reason, - const std::string &type, std::string body) { + void sendResponse(uint16_t status, const char *reason, std::string_view type, + std::unique_ptr body) { responseFinished_ = true; ResponseBuilder(downstream_) .status(status, reason) - .header("Content-Type", type) + .header(proxygen::HTTP_HEADER_CONTENT_TYPE, type) .body(std::move(body)) .sendWithEOM(); } - void sendText(uint16_t status, const std::string &reason, - const std::string &body) { - sendResponse(status, reason, "text/plain", body); + void sendResponse(uint16_t status, const char *reason, std::string_view type, + std::string_view body) { + sendResponse(status, reason, type, folly::IOBuf::copyBuffer(body)); } - void sendWebSocketFrame(uint8_t opcode, const uint8_t *payload, - size_t payloadLength) { - std::vector frame; - frame.reserve(payloadLength + 10); - frame.push_back(static_cast(0x80U | opcode)); - if (payloadLength <= 125) { - frame.push_back(static_cast(payloadLength)); - } else if (payloadLength <= std::numeric_limits::max()) { - frame.push_back(126); - frame.push_back(static_cast((payloadLength >> 8) & 0xff)); - frame.push_back(static_cast(payloadLength & 0xff)); - } else { - frame.push_back(127); - const auto length = static_cast(payloadLength); - for (int shift = 56; shift >= 0; shift -= 8) { - frame.push_back(static_cast((length >> shift) & 0xff)); - } - } - if (payloadLength > 0) { - frame.insert(frame.end(), payload, payload + payloadLength); - } - downstream_->sendBody(folly::IOBuf::copyBuffer(frame.data(), frame.size())); - } - - void sendWebSocketFrame(uint8_t opcode, const std::vector &payload) { - sendWebSocketFrame(opcode, payload.data(), payload.size()); - } - - void closeWebSocket(uint16_t status) { - if (responseFinished_ || closeSent_) { - return; - } - const std::array payload = { - static_cast((status >> 8) & 0xff), - static_cast(status & 0xff)}; - sendWebSocketFrame(0x8, payload.data(), payload.size()); - closeSent_ = true; - } - - void webSocketProtocolError() { closeWebSocket(1002); } - - void webSocketInvalidPayload() { closeWebSocket(1007); } - - void handleWebSocketFrame(bool fin, uint8_t opcode, - std::vector payload) { - if (closeSent_ && opcode != 0x08) { - return; - } - if ((opcode & 0x08U) != 0) { - if (!fin || payload.size() > 125) { - webSocketProtocolError(); - return; - } - if (opcode == 0x08) { - if (payload.size() == 1) { - webSocketProtocolError(); - return; - } - if (payload.size() >= 2) { - const uint16_t status = - (static_cast(payload[0]) << 8) | payload[1]; - if (!validWebSocketCloseCode(status)) { - webSocketProtocolError(); - return; - } - if (!validUtf8(payload.data() + 2, payload.size() - 2)) { - webSocketInvalidPayload(); - return; - } - } - if (closeSent_) { - responseFinished_ = true; - downstream_->sendEOM(); - return; - } - sendWebSocketFrame(0x08, payload); - responseFinished_ = true; - downstream_->sendEOM(); - } else if (opcode == 0x09) { - sendWebSocketFrame(0x0a, payload); - } else if (opcode != 0x0a) { - webSocketProtocolError(); - } - return; - } - - if (opcode == 0x00) { - if (fragmentOpcode_ == 0) { - webSocketProtocolError(); - return; - } - if (fragmentPayload_.size() + payload.size() > kMaxWebSocketMessage) { - webSocketProtocolError(); - return; - } - fragmentPayload_.insert(fragmentPayload_.end(), payload.begin(), - payload.end()); - if (fin) { - if (fragmentOpcode_ == 0x01 && !validUtf8(fragmentPayload_)) { - webSocketInvalidPayload(); - return; - } - sendWebSocketFrame(fragmentOpcode_, fragmentPayload_); - fragmentOpcode_ = 0; - fragmentPayload_.clear(); - } - return; - } - - if (opcode != 0x01 && opcode != 0x02) { - webSocketProtocolError(); - return; - } - if (fragmentOpcode_ != 0) { - webSocketProtocolError(); - return; - } - if (fin) { - if (opcode == 0x01 && !validUtf8(payload)) { - webSocketInvalidPayload(); - return; - } - sendWebSocketFrame(opcode, payload); - return; - } - fragmentOpcode_ = opcode; - fragmentPayload_ = std::move(payload); + void sendText(uint16_t status, const char *reason, std::string_view body) { + sendResponse(status, reason, "text/plain", body); } - void processWebSocketFrames() { - size_t cursor = 0; - while (!responseFinished_) { - if (websocketBytes_.size() - cursor < 2) { - break; - } - const uint8_t first = websocketBytes_[cursor]; - const uint8_t second = websocketBytes_[cursor + 1]; - const bool fin = (first & 0x80U) != 0; - const uint8_t opcode = first & 0x0fU; - const uint8_t encodedPayloadLength = second & 0x7fU; - if ((first & 0x70U) != 0 || (second & 0x80U) == 0) { - webSocketProtocolError(); - break; - } - // RFC 6455 control frames cannot use either extended-length encoding, - // even when that encoding ultimately describes 125 bytes or fewer. - if ((opcode & 0x08U) != 0 && encodedPayloadLength > 125) { - webSocketProtocolError(); - break; - } - - uint64_t payloadLength = encodedPayloadLength; - size_t headerLength = 2; - if (payloadLength == 126) { - if (websocketBytes_.size() - cursor < 4) { - break; - } - payloadLength = - (static_cast(websocketBytes_[cursor + 2]) << 8) | - websocketBytes_[cursor + 3]; - if (payloadLength < 126) { - webSocketProtocolError(); - break; - } - headerLength = 4; - } else if (payloadLength == 127) { - if (websocketBytes_.size() - cursor < 10) { - break; - } - if ((websocketBytes_[cursor + 2] & 0x80U) != 0) { - webSocketProtocolError(); - break; - } - payloadLength = 0; - for (size_t index = 0; index < 8; ++index) { - payloadLength = - (payloadLength << 8) | websocketBytes_[cursor + 2 + index]; - } - if (payloadLength <= std::numeric_limits::max()) { - webSocketProtocolError(); - break; - } - headerLength = 10; - } - if (payloadLength > kMaxWebSocketMessage) { - webSocketProtocolError(); - break; - } - - constexpr size_t kMaskLength = 4; - if (payloadLength > - std::numeric_limits::max() - headerLength - kMaskLength) { - webSocketProtocolError(); - break; - } - const size_t frameLength = - headerLength + kMaskLength + static_cast(payloadLength); - if (websocketBytes_.size() - cursor < frameLength) { - break; - } - - const size_t maskOffset = cursor + headerLength; - const size_t payloadOffset = maskOffset + kMaskLength; - std::vector payload(static_cast(payloadLength)); - for (size_t index = 0; index < payload.size(); ++index) { - payload[index] = websocketBytes_[payloadOffset + index] ^ - websocketBytes_[maskOffset + (index % kMaskLength)]; - } - cursor += frameLength; - handleWebSocketFrame(fin, opcode, std::move(payload)); + // Drains whatever the shared RFC 6455 codec produced for this read as a + // single egress write, then closes the stream if the codec is done. + void flushWebSocket() { + if (auto egress = websocket_.takeEgress()) { + downstream_->sendBody(std::move(egress)); } - - if (cursor > 0) { - websocketBytes_.erase(websocketBytes_.begin(), - websocketBytes_.begin() + cursor); - } - if (responseFinished_) { - websocketBytes_.clear(); + if (websocket_.finished() && !responseFinished_) { + responseFinished_ = true; + downstream_->sendEOM(); } } Route route_{Route::NotFound}; - std::shared_ptr dataset_; + const folly::dynamic &dataset_; + const StaticAssets &assets_; HTTPMethod method_{HTTPMethod::GET}; int64_t a_{0}; int64_t b_{0}; @@ -583,30 +399,30 @@ class ArenaHandler final : public RequestHandler { bool bodyValid_{true}; bool websocketAccepted_{false}; bool websocketActive_{false}; - bool closeSent_{false}; bool responseFinished_{false}; - uint8_t fragmentOpcode_{0}; std::string requestBody_; std::string staticName_; - std::vector websocketBytes_; - std::vector fragmentPayload_; + std::string acceptEncoding_; + httparena::WebSocketEcho websocket_; }; class ArenaHandlerFactory final : public RequestHandlerFactory { public: - explicit ArenaHandlerFactory(std::shared_ptr dataset) - : dataset_(std::move(dataset)) {} + ArenaHandlerFactory(std::shared_ptr dataset, + const StaticAssets &assets) + : dataset_(std::move(dataset)), assets_(assets) {} void onServerStart(folly::EventBase * /*eventBase*/) noexcept override {} void onServerStop() noexcept override {} RequestHandler *onRequest(RequestHandler *, HTTPMessage *) noexcept override { - return new ArenaHandler(dataset_); + return new ArenaHandler(*dataset_, assets_); } private: std::shared_ptr dataset_; + const StaticAssets &assets_; }; wangle::SSLContextConfig h1TlsConfig() { @@ -657,28 +473,48 @@ int main(int argc, char *argv[]) { if (FLAGS_threads <= 0) { FLAGS_threads = static_cast(folly::available_concurrency()); } + if (FLAGS_h3_threads <= 0) { + FLAGS_h3_threads = static_cast(folly::available_concurrency()); + } CHECK_GT(FLAGS_threads, 0); + CHECK_GT(FLAGS_h3_threads, 0); + + static httparena::StaticAssets assets; try { auto dataset = loadDataset(); + assets.load(); proxygen::HTTPServerOptions options; options.threads = static_cast(FLAGS_threads); options.idleTimeout = std::chrono::milliseconds(60000); options.shutdownOn = {SIGINT, SIGTERM}; + // Nothing here speaks CONNECT; saying we do only stops HTTPServer from + // prepending RejectConnectFilterFactory to the per-request filter chain. options.supportsConnect = true; options.enableContentCompression = true; + // Only `json-comp` asks for a compressed response, and it is the one + // profile scored on compression ratio rather than raw rps. `static` sends + // `Accept-Encoding: br;q=1, gzip;q=0.8` too, and proxygen's default + // compressible set covers text/css, text/html and application/javascript — + // so every CSS/JS/HTML request was gzipping 8-200 KB on the event base for + // no scoring benefit (compression is explicitly optional for `static`). + // Restrict the set to the content type json-comp actually measures. + options.contentCompressionTypes = {"application/json"}; options.initialReceiveWindow = 1U << 20; options.receiveStreamWindowSize = 1U << 20; options.receiveSessionWindowSize = 10U << 20; options.maxConcurrentIncomingStreams = 1024; options.handlerFactories = - RequestHandlerChain().addThen(dataset).build(); + RequestHandlerChain() + .addThen(dataset, assets) + .build(); httparena::ArenaHQServer h3Server( - FLAGS_cert, FLAGS_key, static_cast(FLAGS_threads), + FLAGS_cert, FLAGS_key, static_cast(FLAGS_h3_threads), [dataset](HTTPMessage *) -> proxygen::HTTPTransactionHandler * { - return new proxygen::RequestHandlerAdaptor(new ArenaHandler(dataset)); + return new proxygen::RequestHandlerAdaptor( + new ArenaHandler(*dataset, assets)); }); HTTPServer server(std::move(options)); server.bind(listenerConfigs()); diff --git a/frameworks/proxygen-coro/ArenaCoroServer.cpp b/frameworks/proxygen/src/CoroServer.cpp similarity index 50% rename from frameworks/proxygen-coro/ArenaCoroServer.cpp rename to frameworks/proxygen/src/CoroServer.cpp index ed3b65cf2..7787c07ec 100644 --- a/frameworks/proxygen-coro/ArenaCoroServer.cpp +++ b/frameworks/proxygen/src/CoroServer.cpp @@ -1,20 +1,18 @@ +#include "ArenaCommon.h" +#include "ArenaWebSocket.h" + #include #include -#include -#include #include #include #include #include -#include #include #include -#include #include #include #include #include -#include #include #include #include @@ -22,7 +20,6 @@ #include #include -#include #include #include #include @@ -34,7 +31,7 @@ #include #include #include -#include +#include #include #include #include @@ -48,7 +45,8 @@ DEFINE_int32(h3_port, 8443, "HTTP/3 over QUIC port"); DEFINE_string(ip, "::", "Address on which to listen"); DEFINE_string(cert, "/certs/server.crt", "TLS certificate path"); DEFINE_string(key, "/certs/server.key", "TLS private-key path"); -DEFINE_int32(threads, 0, "I/O threads; 0 uses the available CPU count"); +DEFINE_int32(threads, 0, "TCP I/O threads; 0 uses the available CPU count"); +DEFINE_int32(h3_threads, 0, "QUIC I/O threads; 0 uses the available CPU count"); namespace { @@ -66,210 +64,46 @@ using proxygen::coro::HTTPSource; using proxygen::coro::HTTPSourceHolder; using proxygen::coro::TimedBaton; -constexpr size_t kMaxBaselineBody = 1024; -constexpr uint64_t kMaxWebSocketMessage = 16ULL * 1024 * 1024; -constexpr std::string_view kJsonPrefix = "/json/"; -constexpr std::string_view kStaticPrefix = "/static/"; -constexpr std::string_view kStaticRoot = "/data/static/"; - -bool parseInteger(std::string_view input, int64_t &value) { - while (!input.empty() && - std::isspace(static_cast(input.front()))) { - input.remove_prefix(1); - } - while (!input.empty() && - std::isspace(static_cast(input.back()))) { - input.remove_suffix(1); - } - if (input.empty()) { - return false; - } - const auto result = - std::from_chars(input.data(), input.data() + input.size(), value); - return result.ec == std::errc() && result.ptr == input.data() + input.size(); -} - -bool checkedAdd(int64_t lhs, int64_t rhs, int64_t &result) { -#if defined(__GNUC__) || defined(__clang__) - return !__builtin_add_overflow(lhs, rhs, &result); -#else - if ((rhs > 0 && lhs > std::numeric_limits::max() - rhs) || - (rhs < 0 && lhs < std::numeric_limits::min() - rhs)) { - return false; - } - result = lhs + rhs; - return true; -#endif -} - -bool checkedMultiply(int64_t lhs, int64_t rhs, int64_t &result) { -#if defined(__GNUC__) || defined(__clang__) - return !__builtin_mul_overflow(lhs, rhs, &result); -#else - if (lhs > 0) { - if ((rhs > 0 && lhs > std::numeric_limits::max() / rhs) || - (rhs < 0 && rhs < std::numeric_limits::min() / lhs)) { - return false; - } - } else if (lhs < 0) { - if ((rhs > 0 && lhs < std::numeric_limits::min() / rhs) || - (rhs < 0 && rhs < std::numeric_limits::max() / lhs)) { - return false; - } - } - result = lhs * rhs; - return true; -#endif -} - -std::string contentType(std::string_view name) { - const auto endsWith = [name](std::string_view suffix) { - return name.size() >= suffix.size() && - name.substr(name.size() - suffix.size()) == suffix; - }; - if (endsWith(".css")) { - return "text/css"; - } - if (endsWith(".js")) { - return "application/javascript"; - } - if (endsWith(".html")) { - return "text/html"; - } - if (endsWith(".json")) { - return "application/json"; - } - if (endsWith(".svg")) { - return "image/svg+xml"; - } - if (endsWith(".webp")) { - return "image/webp"; - } - if (endsWith(".woff2")) { - return "font/woff2"; - } - return "application/octet-stream"; -} - -std::shared_ptr loadDataset() { - std::ifstream input("/data/dataset.json", std::ios::binary); - if (!input) { - throw std::runtime_error("cannot open /data/dataset.json"); - } - std::string contents((std::istreambuf_iterator(input)), - std::istreambuf_iterator()); - auto dataset = folly::parseJson(contents); - if (!dataset.isArray() || dataset.size() < 50) { - throw std::runtime_error("/data/dataset.json must contain 50 items"); - } - return std::make_shared(std::move(dataset)); -} - -bool validWebSocketKey(std::string_view key) noexcept { - if (key.size() != 24) { - return false; - } - std::array decoded{}; - const auto result = folly::base64Decode(key, decoded.data()); - return result.is_success && result.o == decoded.data() + 16; -} - -bool validUtf8(const uint8_t *data, size_t size) noexcept { - const auto continuation = [](uint8_t byte) { - return byte >= 0x80 && byte <= 0xbf; - }; - - size_t index = 0; - while (index < size) { - const uint8_t first = data[index]; - if (first <= 0x7f) { - ++index; - continue; - } - if (first >= 0xc2 && first <= 0xdf) { - if (index + 1 >= size || !continuation(data[index + 1])) { - return false; - } - index += 2; - continue; - } - if (first == 0xe0) { - if (index + 2 >= size || data[index + 1] < 0xa0 || - data[index + 1] > 0xbf || !continuation(data[index + 2])) { - return false; - } - index += 3; - continue; - } - if ((first >= 0xe1 && first <= 0xec) || (first >= 0xee && first <= 0xef)) { - if (index + 2 >= size || !continuation(data[index + 1]) || - !continuation(data[index + 2])) { - return false; - } - index += 3; - continue; - } - if (first == 0xed) { - if (index + 2 >= size || data[index + 1] < 0x80 || - data[index + 1] > 0x9f || !continuation(data[index + 2])) { - return false; - } - index += 3; - continue; - } - if (first == 0xf0) { - if (index + 3 >= size || data[index + 1] < 0x90 || - data[index + 1] > 0xbf || !continuation(data[index + 2]) || - !continuation(data[index + 3])) { - return false; - } - index += 4; - continue; - } - if (first >= 0xf1 && first <= 0xf3) { - if (index + 3 >= size || !continuation(data[index + 1]) || - !continuation(data[index + 2]) || !continuation(data[index + 3])) { - return false; - } - index += 4; - continue; - } - if (first == 0xf4) { - if (index + 3 >= size || data[index + 1] < 0x80 || - data[index + 1] > 0x8f || !continuation(data[index + 2]) || - !continuation(data[index + 3])) { - return false; - } - index += 4; - continue; - } - return false; - } - return true; -} - -bool validUtf8(const std::vector &data) noexcept { - return validUtf8(data.data(), data.size()); -} - -bool validWebSocketCloseCode(uint16_t code) noexcept { - const bool definedProtocolCode = code >= 1000 && code <= 1014 && - code != 1004 && code != 1005 && code != 1006; - const bool applicationCode = code >= 3000 && code <= 4999; - return definedProtocolCode || applicationCode; -} +using httparena::checkedAdd; +using httparena::checkedMultiply; +using httparena::contentType; +using httparena::kJsonPrefix; +using httparena::kMaxRequestBody; +using httparena::kStaticPrefix; +using httparena::loadDataset; +using httparena::parseInteger; +using httparena::queryValue; +using httparena::ContentEncoding; +using httparena::StaticAssets; +using httparena::validWebSocketKey; HTTPFixedSource *makeResponse(uint16_t status, std::string_view contentTypeValue, - std::string body) { + std::unique_ptr body) { auto *response = HTTPFixedSource::makeFixedResponse(status, std::move(body)); response->msg_->getHeaders().set(proxygen::HTTP_HEADER_CONTENT_TYPE, contentTypeValue); return response; } -HTTPFixedSource *makeTextResponse(uint16_t status, std::string body) { - return makeResponse(status, "text/plain", std::move(body)); +HTTPFixedSource *makeResponse(uint16_t status, + std::string_view contentTypeValue, + std::string_view body) { + return makeResponse(status, contentTypeValue, folly::IOBuf::copyBuffer(body)); +} + +HTTPFixedSource *makeTextResponse(uint16_t status, std::string_view body) { + return makeResponse(status, "text/plain", body); +} + +// Renders an integer response body without the std::to_string allocation. +HTTPFixedSource *makeIntResponse(uint16_t status, int64_t value) { + std::array digits; + const auto end = + std::to_chars(digits.data(), digits.data() + digits.size(), value); + return makeTextResponse( + status, std::string_view(digits.data(), + static_cast(end.ptr - digits.data()))); } folly::coro::Task @@ -342,6 +176,10 @@ folly::coro::Task readRequestBody(HTTPSourceHolder &source, co_return result; } +// Long-lived response source for an upgraded WebSocket connection: raw frames +// arrive as BODY events on the request source, and the echoed frames leave as +// BODY events on this one. Framing is the shared codec in ArenaWebSocket.h, so +// a read carrying N pipelined frames produces one egress event. class WebSocketSource final : public HTTPSource { public: explicit WebSocketSource(HTTPSourceHolder request) @@ -362,7 +200,7 @@ class WebSocketSource final : public HTTPSource { folly::coro::Task readBodyEvent(uint32_t max = std::numeric_limits::max()) override { - while (pending_.empty()) { + while (!websocket_.hasEgress() && !eom_) { auto inputTry = co_await co_awaitTry(readBodyEventNoSuspend(request_)); if (inputTry.hasException()) { auto error = proxygen::coro::getHTTPError(inputTry); @@ -376,38 +214,17 @@ class WebSocketSource final : public HTTPSource { auto input = std::move(*inputTry); if (input.eventType == HTTPBodyEvent::BODY) { - auto body = input.event.body.move(); - if (body) { - const auto range = body->coalesce(); - input_.insert(input_.end(), range.begin(), range.end()); - processFrames(); - } + websocket_.onIngress(input.event.body.move()); } - if (input.eom && !finished_) { - finished_ = true; - if (pending_.empty()) { - pending_.push_back(PendingOutput{{}, 0, true}); - } else { - pending_.back().eom = true; - } + if (input.eom || websocket_.finished()) { + eom_ = true; } } - auto &front = pending_.front(); - const size_t remaining = front.bytes.size() - front.offset; - const size_t limit = std::max(1, max); - const size_t amount = std::min(remaining, limit); - std::unique_ptr body; - if (amount > 0) { - body = - folly::IOBuf::copyBuffer(front.bytes.data() + front.offset, amount); - front.offset += amount; - } - const bool outputEom = front.eom && front.offset == front.bytes.size(); - if (front.offset == front.bytes.size()) { - pending_.pop_front(); - } - HTTPBodyEvent event(std::move(body), outputEom); + auto body = websocket_.takeEgress(std::max(1, max)); + // Only signal EOM once everything the codec produced has been handed over. + const bool eom = eom_ && !websocket_.hasEgress(); + HTTPBodyEvent event(std::move(body), eom); auto guard = folly::makeGuard(lifetime(event)); co_return event; } @@ -423,240 +240,37 @@ class WebSocketSource final : public HTTPSource { } private: - struct PendingOutput { - std::vector bytes; - size_t offset{0}; - bool eom{false}; - }; - - void queueEom() { - if (!finished_) { - finished_ = true; - pending_.push_back(PendingOutput{{}, 0, true}); - } - } - - void queueFrame(uint8_t opcode, const uint8_t *payload, size_t payloadLength, - bool eom = false) { - std::vector frame; - frame.reserve(payloadLength + 10); - frame.push_back(static_cast(0x80U | opcode)); - if (payloadLength <= 125) { - frame.push_back(static_cast(payloadLength)); - } else if (payloadLength <= std::numeric_limits::max()) { - frame.push_back(126); - frame.push_back(static_cast((payloadLength >> 8) & 0xff)); - frame.push_back(static_cast(payloadLength & 0xff)); - } else { - frame.push_back(127); - const auto length = static_cast(payloadLength); - for (int shift = 56; shift >= 0; shift -= 8) { - frame.push_back(static_cast((length >> shift) & 0xff)); - } - } - if (payloadLength > 0) { - frame.insert(frame.end(), payload, payload + payloadLength); - } - pending_.push_back(PendingOutput{std::move(frame), 0, eom}); - } - - void queueFrame(uint8_t opcode, const std::vector &payload, - bool eom = false) { - queueFrame(opcode, payload.data(), payload.size(), eom); - } - - void closeWith(uint16_t status) { - if (closeSent_ || finished_) { - return; - } - const std::array payload = { - static_cast((status >> 8) & 0xff), - static_cast(status & 0xff)}; - queueFrame(0x08, payload.data(), payload.size()); - closeSent_ = true; - } - - void protocolError() { closeWith(1002); } - - void invalidPayload() { closeWith(1007); } - - void handleFrame(bool fin, uint8_t opcode, std::vector payload) { - if (closeSent_ && opcode != 0x08) { - return; - } - - if ((opcode & 0x08U) != 0) { - if (!fin || payload.size() > 125) { - protocolError(); - return; - } - if (opcode == 0x08) { - if (payload.size() == 1) { - protocolError(); - return; - } - if (payload.size() >= 2) { - const uint16_t status = - (static_cast(payload[0]) << 8) | payload[1]; - if (!validWebSocketCloseCode(status)) { - protocolError(); - return; - } - if (!validUtf8(payload.data() + 2, payload.size() - 2)) { - invalidPayload(); - return; - } - } - if (closeSent_) { - queueEom(); - return; - } - finished_ = true; - queueFrame(0x08, payload, true); - } else if (opcode == 0x09) { - queueFrame(0x0a, payload); - } else if (opcode != 0x0a) { - protocolError(); - } - return; - } - - if (opcode == 0x00) { - if (fragmentOpcode_ == 0 || - payload.size() > kMaxWebSocketMessage - fragmentPayload_.size()) { - protocolError(); - return; - } - fragmentPayload_.insert(fragmentPayload_.end(), payload.begin(), - payload.end()); - if (fin) { - if (fragmentOpcode_ == 0x01 && !validUtf8(fragmentPayload_)) { - invalidPayload(); - return; - } - queueFrame(fragmentOpcode_, fragmentPayload_); - fragmentOpcode_ = 0; - fragmentPayload_.clear(); - } - return; - } - - if ((opcode != 0x01 && opcode != 0x02) || fragmentOpcode_ != 0) { - protocolError(); - return; - } - if (fin) { - if (opcode == 0x01 && !validUtf8(payload)) { - invalidPayload(); - return; - } - queueFrame(opcode, payload); - return; - } - fragmentOpcode_ = opcode; - fragmentPayload_ = std::move(payload); - } - - void processFrames() { - size_t cursor = 0; - while (!finished_) { - if (input_.size() - cursor < 2) { - break; - } - const uint8_t first = input_[cursor]; - const uint8_t second = input_[cursor + 1]; - const bool fin = (first & 0x80U) != 0; - const uint8_t opcode = first & 0x0fU; - const uint8_t encodedLength = second & 0x7fU; - if ((first & 0x70U) != 0 || (second & 0x80U) == 0 || - ((opcode & 0x08U) != 0 && encodedLength > 125)) { - protocolError(); - cursor = input_.size(); - break; - } - - uint64_t payloadLength = encodedLength; - size_t headerLength = 2; - if (payloadLength == 126) { - if (input_.size() - cursor < 4) { - break; - } - payloadLength = (static_cast(input_[cursor + 2]) << 8) | - input_[cursor + 3]; - if (payloadLength < 126) { - protocolError(); - cursor = input_.size(); - break; - } - headerLength = 4; - } else if (payloadLength == 127) { - if (input_.size() - cursor < 10) { - break; - } - if ((input_[cursor + 2] & 0x80U) != 0) { - protocolError(); - cursor = input_.size(); - break; - } - payloadLength = 0; - for (size_t index = 0; index < 8; ++index) { - payloadLength = (payloadLength << 8) | input_[cursor + 2 + index]; - } - if (payloadLength <= std::numeric_limits::max()) { - protocolError(); - cursor = input_.size(); - break; - } - headerLength = 10; - } - if (payloadLength > kMaxWebSocketMessage || - payloadLength > - std::numeric_limits::max() - headerLength - 4) { - protocolError(); - cursor = input_.size(); - break; - } - - const size_t frameLength = - headerLength + 4 + static_cast(payloadLength); - if (input_.size() - cursor < frameLength) { - break; - } - const size_t maskOffset = cursor + headerLength; - const size_t payloadOffset = maskOffset + 4; - std::vector payload(static_cast(payloadLength)); - for (size_t index = 0; index < payload.size(); ++index) { - payload[index] = - input_[payloadOffset + index] ^ input_[maskOffset + (index % 4)]; - } - cursor += frameLength; - handleFrame(fin, opcode, std::move(payload)); - if (closeSent_ || finished_) { - break; - } - } - - if (cursor > 0) { - input_.erase(input_.begin(), input_.begin() + cursor); - } - if (closeSent_ || finished_) { - input_.clear(); - } - } - HTTPSourceHolder request_; - std::deque pending_; - std::vector input_; - std::vector fragmentPayload_; - uint8_t fragmentOpcode_{0}; - bool closeSent_{false}; - bool finished_{false}; + httparena::WebSocketEcho websocket_; + bool eom_{false}; }; + +// Only `json-comp` asks for a compressed response, and it is the one profile +// scored on compression ratio rather than raw rps. `static` sends +// `Accept-Encoding: br;q=1, gzip;q=0.8` too, so listing the static content +// types here would mean gzipping 8-200 KB per CSS/JS/HTML request on the event +// base for no scoring benefit (compression is explicitly optional for +// `static`). +proxygen::CompressionFilterUtils::FactoryOptions compressionOptions() { + proxygen::CompressionFilterUtils::FactoryOptions options; + options.compressibleContentTypes = + std::make_shared>( + std::set{"application/json"}); + // FactoryOptions defaults to 4; HTTPServerOptions (and so the classic entry) + // defaults to Z_DEFAULT_COMPRESSION. json-comp is scored on + // (minBpr/myBpr)^2, so the weaker ratio cost the coro entry ~17% of that + // profile's score for no throughput gain worth having. + options.zlibCompressionLevel = -1; + return options; +} + class ArenaCoroHandler final : public HTTPHandler { public: explicit ArenaCoroHandler(std::shared_ptr dataset) - : dataset_(std::move(dataset)) {} + : dataset_(std::move(dataset)), items_(*dataset_) { + assets_.load(); + } folly::coro::Task handleRequest(folly::EventBase * /*eventBase*/, @@ -670,7 +284,10 @@ class ArenaCoroHandler final : public HTTPHandler { auto request = std::move(header.headers); const bool requestEom = header.eom; const auto method = request->getMethod().value_or(HTTPMethod::GET); - const std::string path = request->getPath(); + const auto pathPiece = request->getPathAsStringPiece(); + const std::string_view path(pathPiece.data(), pathPiece.size()); + const auto queryPiece = request->getQueryStringAsStringPiece(); + const std::string_view query(queryPiece.data(), queryPiece.size()); if (path == "/ws") { const auto &headers = request->getHeaders(); @@ -694,8 +311,10 @@ class ArenaCoroHandler final : public HTTPHandler { } int64_t a = 0; int64_t b = 0; - if (!parseInteger(request->getQueryParam("a"), a) || - !parseInteger(request->getQueryParam("b"), b)) { + const auto aText = queryValue(query, "a"); + const auto bText = queryValue(query, "b"); + if (!aText || !bText || !parseInteger(*aText, a) || + !parseInteger(*bText, b)) { co_return makeTextResponse(400, "invalid integer"); } int64_t sum = 0; @@ -704,7 +323,7 @@ class ArenaCoroHandler final : public HTTPHandler { } if (method == HTTPMethod::POST) { auto body = co_await readRequestBody(requestSource, requestEom, - kMaxBaselineBody); + kMaxRequestBody); int64_t bodyValue = 0; if (!body.readOk || !body.captureOk || !parseInteger(body.captured, bodyValue) || @@ -712,7 +331,7 @@ class ArenaCoroHandler final : public HTTPHandler { co_return makeTextResponse(400, "invalid integer"); } } - co_return makeTextResponse(200, std::to_string(sum)); + co_return makeIntResponse(200, sum); } if (path == "/pipeline") { @@ -730,17 +349,20 @@ class ArenaCoroHandler final : public HTTPHandler { path.size() - kJsonPrefix.size()); int64_t count = 0; int64_t multiplier = 1; - const auto multiplierText = request->getQueryParam("m"); + const auto multiplierText = queryValue(query, "m"); if (!parseInteger(countText, count) || count < 1 || count > 50 || - (!multiplierText.empty() && - !parseInteger(multiplierText, multiplier))) { + (multiplierText && !parseInteger(*multiplierText, multiplier))) { + co_return makeTextResponse(400, "invalid JSON parameters"); + } + if (static_cast(count) > items_.size()) { co_return makeTextResponse(400, "invalid JSON parameters"); } try { folly::dynamic items = folly::dynamic::array; + items.reserve(static_cast(count)); for (int64_t index = 0; index < count; ++index) { - folly::dynamic item = (*dataset_)[static_cast(index)]; + folly::dynamic item = items_[static_cast(index)]; int64_t subtotal = 0; int64_t total = 0; if (!checkedMultiply(item["price"].asInt(), item["quantity"].asInt(), @@ -754,8 +376,10 @@ class ArenaCoroHandler final : public HTTPHandler { folly::dynamic response = folly::dynamic::object; response["items"] = std::move(items); response["count"] = count; - co_return makeResponse(200, "application/json", - folly::toJson(response)); + co_return maybeCompress( + makeResponse(200, "application/json", + folly::IOBuf::fromString(folly::toJson(response))), + *request); } catch (const std::exception &) { co_return makeTextResponse(500, "JSON serialization failed"); } @@ -769,45 +393,78 @@ class ArenaCoroHandler final : public HTTPHandler { if (!body.readOk) { co_return makeTextResponse(400, "upload failed"); } - co_return makeTextResponse(200, std::to_string(body.size)); + co_return makeIntResponse(200, static_cast(body.size)); } if (path.starts_with(kStaticPrefix)) { if (method != HTTPMethod::GET) { co_return makeTextResponse(405, "method not allowed"); } - const std::string name(path.data() + kStaticPrefix.size(), - path.size() - kStaticPrefix.size()); - if (name.empty() || name.find('/') != std::string::npos || - name.find('\\') != std::string::npos || - name.find("..") != std::string::npos) { - co_return makeTextResponse(404, "not found"); - } - std::ifstream input(std::string(kStaticRoot) + name, std::ios::binary); - if (!input) { + // Exact lookup in the preloaded table, so traversal is impossible and a + // miss is just a 404. + const auto *asset = assets_.find(path.substr(kStaticPrefix.size())); + if (asset == nullptr) { co_return makeTextResponse(404, "not found"); } - std::string body((std::istreambuf_iterator(input)), - std::istreambuf_iterator()); - if (!input.good() && !input.eof()) { - co_return makeTextResponse(500, "read error"); + const auto &accept = request->getHeaders().getSingleOrEmpty( + proxygen::HTTP_HEADER_ACCEPT_ENCODING); + const auto [body, encoding] = + asset->select(std::string_view(accept.data(), accept.size())); + // Non-owning view of the table, which outlives every request. + auto *response = + makeResponse(200, asset->contentType, + folly::IOBuf::wrapBuffer(body->data(), body->size())); + if (encoding != ContentEncoding::Identity) { + auto &headers = response->msg_->getHeaders(); + headers.set(proxygen::HTTP_HEADER_CONTENT_ENCODING, + httparena::encodingToken(encoding)); + headers.set(proxygen::HTTP_HEADER_VARY, "Accept-Encoding"); } - co_return makeResponse(200, contentType(name), std::move(body)); + co_return response; } co_return makeTextResponse(404, "not found"); } private: + // Wraps `response` in proxygen's coro CompressionFilter when this client + // actually negotiated an encoding we support. + // + // The alternative is registering ServerCompressionFilterFactory on + // HTTPServer::Config, but HTTPFilterFactoryHandler calls makeFilters() + // before the request headers have been read, so the factory cannot be + // conditional: every request — `baseline` included, where nothing is + // compressible — pays a SharedCtx, a VisitorFilter with a capturing lambda, + // a CompressionFilter and the surrounding coroutine frame. That measured at + // ~10% of baseline throughput. Attaching the filter here instead keeps the + // cost on the responses that are actually compressed. (The classic + // HTTPServer path has no such problem: CompressionFilterFactory::onRequest + // sees the request and returns the handler unwrapped when there is nothing + // to do.) + HTTPSourceHolder maybeCompress(HTTPFixedSource *response, + const HTTPMessage &request) { + auto params = std::make_shared< + folly::Optional>( + proxygen::CompressionFilterUtils::getFilterParams(request, + compression_)); + if (!params->hasValue()) { + return response; + } + auto *filter = + new proxygen::coro::CompressionFilter(response, std::move(params)); + filter->setHeapAllocated(); + return filter; + } + std::shared_ptr dataset_; + // The handler is shared by every session and outlives all of them, so the + // hot path can dereference the dataset once here instead of per request. + const folly::dynamic &items_; + const proxygen::CompressionFilterUtils::FactoryOptions compression_{ + compressionOptions()}; + StaticAssets assets_; }; -std::shared_ptr> compressibleTypes() { - return std::make_shared>(std::set{ - "application/javascript", "application/json", "image/svg+xml", "text/css", - "text/html", "text/plain"}); -} - constexpr uint32_t kH2StreamWindow = 1U << 20; constexpr size_t kH2ConnectionWindow = 10U << 20; constexpr uint32_t kMaxConcurrentStreams = 1024; @@ -827,13 +484,6 @@ HTTPServer::SessionConfig makeSessionConfig() { return session; } -void addCompressionFilter(HTTPServer::Config &config) { - proxygen::CompressionFilterUtils::FactoryOptions options; - options.compressibleContentTypes = compressibleTypes(); - config.filterFactories.push_back( - std::make_shared( - std::move(options))); -} wangle::SSLContextConfig tlsConfig(std::list protocols) { auto config = HTTPServer::getDefaultTLSConfig(); @@ -895,7 +545,6 @@ HTTPServer::Config tcpConfig(size_t threads) { config.numIOThreads = threads; config.shutdownOnSignals = {SIGINT, SIGTERM}; config.sessionConfig = makeSessionConfig(); - addCompressionFilter(config); return config; } @@ -916,7 +565,6 @@ HTTPServer::Config quicConfig(size_t threads) { transport.maxBatchSize = 48; transport.dataPathType = quic::DataPathType::ContinuousMemory; transport.writeConnectionDataPacketsLimit = 48; - addCompressionFilter(config); return config; } @@ -930,10 +578,14 @@ int main(int argc, char *argv[]) { const size_t threads = FLAGS_threads <= 0 ? static_cast(folly::available_concurrency()) : static_cast(FLAGS_threads); + const size_t h3Threads = + FLAGS_h3_threads <= 0 + ? static_cast(folly::available_concurrency()) + : static_cast(FLAGS_h3_threads); auto handler = std::make_shared(std::move(dataset)); HTTPServer tcpServer(tcpConfig(threads), handler, tcpSocketFactory()); - HTTPServer h3Server(quicConfig(threads), std::move(handler)); + HTTPServer h3Server(quicConfig(h3Threads), std::move(handler)); std::promise h3Ready; auto h3ReadyFuture = h3Ready.get_future(); diff --git a/site/data/frameworks.json b/site/data/frameworks.json index 80d509f13..3988660c4 100644 --- a/site/data/frameworks.json +++ b/site/data/frameworks.json @@ -1286,16 +1286,16 @@ "response": true } }, - "proxygen": { - "dir": "proxygen", - "description": "Meta's Proxygen HTTP engine: HTTPServer for HTTP/1.1, HTTP/1.1 TLS, h2c, HTTP/2 TLS, and RFC 6455 WebSockets, plus the mvfst-backed HQ server for HTTP/3 over QUIC.", + "proxygen-coro": { + "dir": "proxygen-coro", + "description": "Meta's Proxygen native coroutine HTTPServer and HTTPSource APIs across HTTP/1.1, HTTP/1.1 TLS, h2c, HTTP/2 TLS, HTTP/3 QUIC, and RFC 6455 WebSockets.", "repo": "https://github.com/facebook/proxygen", "type": "engine", "engine": "proxygen" }, - "proxygen-coro": { - "dir": "proxygen-coro", - "description": "Meta's Proxygen native coroutine HTTPServer and HTTPSource APIs across HTTP/1.1, HTTP/1.1 TLS, h2c, HTTP/2 TLS, HTTP/3 QUIC, and RFC 6455 WebSockets.", + "proxygen": { + "dir": "proxygen", + "description": "Meta's Proxygen HTTP engine: HTTPServer for HTTP/1.1, HTTP/1.1 TLS, h2c, HTTP/2 TLS, and RFC 6455 WebSockets, plus the mvfst-backed HQ server for HTTP/3 over QUIC.", "repo": "https://github.com/facebook/proxygen", "type": "engine", "engine": "proxygen" From d51787b759f8a03a0bc30608f4f9e9367edc35ce Mon Sep 17 00:00:00 2001 From: sujay Date: Thu, 3 Sep 2026 11:54:31 -0700 Subject: [PATCH 4/4] proxygen: follow the disk on static, drop removed profile subscriptions Rebased onto current main, which changed two things under this branch. The static rules now require served bytes to follow the disk and exclude a cache assembled in the entry ('no reading the directory into a map at startup'), for engine entries too. validate.sh enforces it with a staleness probe that swaps the file under a running server. The startup-preloaded asset table failed both probes, so static is back to a per-request read: one open, one fstat, one readFull into the response buffer. Serving the precompressed .br/.gz siblings is still allowed by selecting the variant off Accept-Encoding, so the on-the-fly gzip of every CSS/JS/HTML response stays gone -- that was the dominant cost. Also drops the json, static and upload subscriptions: all three profiles were removed from the arena (#1375, #1374, #1382). The handlers stay, since json-tls/json-comp/json-h2c and static-tls/static-h2/static-h3 still use them. --- frameworks/proxygen-coro/meta.json | 3 - frameworks/proxygen/README.md | 31 ++-- frameworks/proxygen/meta.json | 3 - frameworks/proxygen/src/ArenaCommon.h | 211 ++++++++++------------ frameworks/proxygen/src/ClassicServer.cpp | 44 ++--- frameworks/proxygen/src/CoroServer.cpp | 30 ++- 6 files changed, 142 insertions(+), 180 deletions(-) diff --git a/frameworks/proxygen-coro/meta.json b/frameworks/proxygen-coro/meta.json index 6473059a8..fc75921e9 100644 --- a/frameworks/proxygen-coro/meta.json +++ b/frameworks/proxygen-coro/meta.json @@ -8,11 +8,8 @@ "enabled": true, "tests": [ "baseline", - "json", "json-comp", "json-tls", - "upload", - "static", "static-tls", "pipelined", "limited-conn", diff --git a/frameworks/proxygen/README.md b/frameworks/proxygen/README.md index e83920427..7754f7896 100644 --- a/frameworks/proxygen/README.md +++ b/frameworks/proxygen/README.md @@ -15,7 +15,7 @@ other pool sleeps. Size them independently with `PROXYGEN_THREADS` and | Listener | Endpoints | Subscribed profiles | | --- | --- | --- | -| HTTP/1.1 `:8080` | `/baseline11`, `/pipeline`, `/json/{count}`, `/upload`, `/static/*`, `/ws` | `baseline`, `pipelined`, `limited-conn`, `json`, `json-comp`, `upload`, `static`, `echo-ws`, `echo-ws-pipeline`, `echo-ws-limited` | +| HTTP/1.1 `:8080` | `/baseline11`, `/pipeline`, `/json/{count}`, `/upload`, `/static/*`, `/ws` | `baseline`, `pipelined`, `limited-conn`, `json-comp`, `echo-ws`, `echo-ws-pipeline`, `echo-ws-limited` | | HTTP/1.1 TLS `:8081` | `/json/{count}`, `/static/*` | `json-tls`, `static-tls` | | h2c `:8082` | `/baseline2`, `/json/{count}` | `baseline-h2c`, `json-h2c` | | HTTP/2 TLS `:8443` | `/baseline2`, `/static/*` | `baseline-h2`, `static-h2` | @@ -28,19 +28,22 @@ trusting `Content-Length`. ## Static assets -`/data/static` is mounted read-only and does not change during a run, so every -file and its precompressed `.br` / `.gz` siblings are read once at startup into -an immutable table, and responses are non-owning `IOBuf` views over it: no disk -I/O, no compression, no copy and no allocation for the payload per request. Both -in-memory caching and serving the precompressed variants are explicitly allowed -for `engine` entries ("No specific rules"). Variant choice is a delimited token -match on `Accept-Encoding` honouring `q=0`, preferring brotli, then gzip, then -the byte-exact original — which is what a client that sends no `Accept-Encoding` -always gets. A lookup miss is a 404, so path traversal is impossible by -construction rather than by filtering. +Files are read from the mounted directory on every request — one `open`, one +`fstat`, one `readFull` straight into the response buffer, no intermediate +`std::string`. There is no cache: the arena's static rules require the served +bytes to follow the disk, and explicitly exclude a cache assembled in the entry +("no reading the directory into a map at startup"). `validate.sh` enforces this +with a staleness probe that swaps the file underneath a running server. -This replaced a per-request `open`/`read` plus an on-the-fly gzip of every -CSS/JS/HTML response, which was costing about 80x throughput and 6.7x memory. +The precompressed `.br`/`.gz` siblings are served when the client accepts them, +which the rules allow "by selecting the variant in the entry off +`Accept-Encoding`" — those bytes already exist on disk, so choosing one is a +file read rather than compression. Selection is a delimited token match that +honours `q=0`, preferring brotli, then gzip, then the byte-exact original, +which is what a client sending no `Accept-Encoding` always gets. + +That removes the on-the-fly gzip of every CSS/JS/HTML response, which was the +dominant cost in the static profiles. Response compression via Proxygen's `CompressionFilter` is therefore scoped to `application/json`, the only content type still compressed at request time and @@ -74,7 +77,7 @@ Both images `LD_PRELOAD` mimalloc (pinned by tag in the Dockerfile). glibc mallo is the binding constraint on the allocation-heavy profiles — the JSON routes build a live `folly::dynamic` per request (up to 50 item copies plus the serialized string), and the coroutine server allocates a frame per event. Measured here it is -worth roughly +38% on `json` and +41%/+82% (classic/coro) on `json-comp`. +worth roughly +41%/+82% (classic/coro) on `json-comp`. `MIMALLOC_PURGE_DELAY=1` is set because the default (10 ms) retains ~17% more memory on `json-comp` at 16384 connections for no throughput gain; going to 0 would cut memory 4x further but costs 44% of that profile, so it is not used. diff --git a/frameworks/proxygen/meta.json b/frameworks/proxygen/meta.json index f4b718417..709e20633 100644 --- a/frameworks/proxygen/meta.json +++ b/frameworks/proxygen/meta.json @@ -8,11 +8,8 @@ "enabled": true, "tests": [ "baseline", - "json", "json-comp", "json-tls", - "upload", - "static", "static-tls", "pipelined", "limited-conn", diff --git a/frameworks/proxygen/src/ArenaCommon.h b/frameworks/proxygen/src/ArenaCommon.h index 8ad2e22ef..a9cfa60ed 100644 --- a/frameworks/proxygen/src/ArenaCommon.h +++ b/frameworks/proxygen/src/ArenaCommon.h @@ -136,16 +136,17 @@ inline std::string_view contentType(std::string_view name) { // ── Static assets ─────────────────────────────────────────────────────────── // -// `/data/static` is mounted read-only and never changes during a run, so every -// file and its precompressed `.br` / `.gz` siblings are read once at startup -// and served straight out of memory. Both are explicitly allowed for `engine` -// entries ("No specific rules"), and the alternative — re-reading and, worse, -// re-gzipping 8-200 KB per request on the event base — was costing better than -// an order of magnitude. +// Read from the mounted directory on every request. The arena's static rules +// require the served bytes to follow the disk — replace a file and the next +// response must carry the new bytes — and explicitly exclude a cache assembled +// in the entry ("no reading the directory into a map at startup"). validate.sh +// enforces this with a staleness probe that swaps the file underneath a running +// server, so a preloaded table fails outright. // -// Bodies are handed out as non-owning IOBufs over this table, which lives for -// the life of the process: no copy, no allocation for the payload, and no -// atomic refcount on the shared buffer. +// Serving the precompressed .br/.gz siblings IS allowed, "by selecting the +// variant in the entry off Accept-Encoding" — those bytes already exist on +// disk, so choosing one is a file read rather than compression. That is where +// the remaining win lives: no gzip on the event-base thread per request. enum class ContentEncoding : uint8_t { Identity, Gzip, Brotli }; @@ -161,127 +162,99 @@ inline std::string_view encodingToken(ContentEncoding encoding) { return {}; } -struct StaticAsset { - std::string identity; - std::string brotli; // empty when there is no .br sibling - std::string gzip; // empty when there is no .gz sibling - std::string_view contentType; - - // Picks the best variant this client accepts. Falls back to identity, which - // is always present, so an absent or unrecognised Accept-Encoding still - // yields the byte-exact original. - std::pair - select(std::string_view acceptEncoding) const { - if (!brotli.empty() && acceptsToken(acceptEncoding, "br")) { - return {&brotli, ContentEncoding::Brotli}; - } - if (!gzip.empty() && acceptsToken(acceptEncoding, "gzip")) { - return {&gzip, ContentEncoding::Gzip}; - } - return {&identity, ContentEncoding::Identity}; - } - - // Token search rather than a full RFC 7231 qvalue parse: the token must be - // delimited so "br" does not match inside another coding, and an explicit - // `q=0` disqualifies it. - static bool acceptsToken(std::string_view header, std::string_view token) { - size_t pos = 0; - while ((pos = header.find(token, pos)) != std::string_view::npos) { - const bool leftOk = - pos == 0 || header[pos - 1] == ' ' || header[pos - 1] == ','; - const size_t after = pos + token.size(); - const bool rightOk = after == header.size() || header[after] == ',' || - header[after] == ';' || header[after] == ' '; - if (leftOk && rightOk) { - // Reject `;q=0` (but not `;q=0.8`). - const auto end = header.find(',', after); - const auto params = header.substr( - after, end == std::string_view::npos ? end : end - after); - const auto q = params.find("q="); - if (q == std::string_view::npos || - params.compare(q, 4, "q=0,") == 0 || params.substr(q) == "q=0" || - params.substr(q) == "q=0.0") { - return q == std::string_view::npos; - } +// Token search rather than a full RFC 7231 qvalue parse: the token must be +// delimited so "br" does not match inside another coding, and an explicit +// `q=0` disqualifies it. +inline bool acceptsToken(std::string_view header, std::string_view token) { + size_t pos = 0; + while ((pos = header.find(token, pos)) != std::string_view::npos) { + const bool leftOk = + pos == 0 || header[pos - 1] == ' ' || header[pos - 1] == ','; + const size_t after = pos + token.size(); + const bool rightOk = after == header.size() || header[after] == ',' || + header[after] == ';' || header[after] == ' '; + if (leftOk && rightOk) { + const auto end = header.find(',', after); + const auto params = + header.substr(after, end == std::string_view::npos ? end : end - after); + const auto q = params.find("q="); + if (q == std::string_view::npos) { return true; } - pos = after; + const auto qv = params.substr(q); + return !(qv == "q=0" || qv == "q=0.0" || qv.rfind("q=0,", 0) == 0); } - return false; + pos = after; } -}; + return false; +} -class StaticAssets { -public: - // Throws if the mount is missing; the arena always provides it. - void load() { - namespace fs = std::filesystem; - std::error_code ec; - fs::directory_iterator it(kStaticRoot, ec); - if (ec) { - throw std::runtime_error(std::string("cannot scan ") + - std::string(kStaticRoot) + ": " + ec.message()); - } - // Pass 1: the originals. Pass 2 attaches variants, so ordering within the - // directory listing does not matter. - std::vector variants; - for (const auto &entry : it) { - if (!entry.is_regular_file()) { - continue; - } - const auto name = entry.path().filename().string(); - if (name.ends_with(".br") || name.ends_with(".gz")) { - variants.push_back(entry.path()); - continue; - } - StaticAsset asset; - if (!folly::readFile(entry.path().c_str(), asset.identity)) { - continue; - } - asset.contentType = contentType(name); - assets_.emplace(name, std::move(asset)); - } - for (const auto &path : variants) { - const auto name = path.filename().string(); - const auto base = name.substr(0, name.size() - 3); - auto found = assets_.find(base); - if (found == assets_.end()) { - continue; // orphan variant with no original; ignore it - } - std::string &slot = - name.ends_with(".br") ? found->second.brotli : found->second.gzip; - if (!folly::readFile(path.c_str(), slot)) { - slot.clear(); - } - } - if (assets_.empty()) { - throw std::runtime_error("no static assets found under /data/static"); +// Reads `root/name` into a fresh IOBuf: one open, one fstat, one readFull, no +// intermediate std::string. Returns nullptr when the file is absent. +inline std::unique_ptr readFileToIOBuf(const std::string &path) { + int fd = folly::openNoInt(path.c_str(), O_RDONLY | O_CLOEXEC); + if (fd < 0) { + return nullptr; + } + folly::File file(fd, /*ownsFd=*/true); + struct stat info {}; + if (::fstat(fd, &info) != 0 || !S_ISREG(info.st_mode)) { + return nullptr; + } + const auto size = static_cast(info.st_size); + auto buffer = folly::IOBuf::create(size); + if (size > 0) { + const ssize_t got = folly::readFull(fd, buffer->writableTail(), size); + if (got < 0 || static_cast(got) != size) { + return nullptr; } } + buffer->append(size); + return buffer; +} - // A miss is simply a 404; because this is an exact lookup in a fixed table, - // path traversal is impossible by construction. - const StaticAsset *find(std::string_view name) const { - const auto found = assets_.find(name); - return found == assets_.end() ? nullptr : &found->second; - } +struct StaticResponse { + std::unique_ptr body; + std::string_view contentType; + ContentEncoding encoding{ContentEncoding::Identity}; +}; + +// `name` must already be rejected if it contains a separator or "..". +// Prefers the precompressed sibling this client accepts, falling back to the +// original — which is what a client sending no Accept-Encoding always gets. +inline StaticResponse serveStatic(std::string_view name, + std::string_view acceptEncoding) { + StaticResponse out; + std::string path; + path.reserve(kStaticRoot.size() + name.size() + 3); + path.append(kStaticRoot).append(name); + const size_t baseLen = path.size(); -private: - // Transparent hashing so lookups take a string_view without allocating. - struct Hash { - using is_transparent = void; - size_t operator()(std::string_view s) const noexcept { - return std::hash{}(s); + if (acceptsToken(acceptEncoding, "br")) { + path.append(".br"); + out.body = readFileToIOBuf(path); + if (out.body) { + out.encoding = ContentEncoding::Brotli; } - }; - struct Equal { - using is_transparent = void; - bool operator()(std::string_view a, std::string_view b) const noexcept { - return a == b; + path.resize(baseLen); + } + if (!out.body && acceptsToken(acceptEncoding, "gzip")) { + path.append(".gz"); + out.body = readFileToIOBuf(path); + if (out.body) { + out.encoding = ContentEncoding::Gzip; } - }; - std::unordered_map assets_; -}; + path.resize(baseLen); + } + if (!out.body) { + out.body = readFileToIOBuf(path); + out.encoding = ContentEncoding::Identity; + } + if (out.body) { + out.contentType = contentType(name); + } + return out; +} inline std::shared_ptr loadDataset() { std::string contents; diff --git a/frameworks/proxygen/src/ClassicServer.cpp b/frameworks/proxygen/src/ClassicServer.cpp index 07709c5a5..558e31413 100644 --- a/frameworks/proxygen/src/ClassicServer.cpp +++ b/frameworks/proxygen/src/ClassicServer.cpp @@ -62,7 +62,7 @@ using httparena::loadDataset; using httparena::parseInteger; using httparena::queryValue; using httparena::ContentEncoding; -using httparena::StaticAssets; +using httparena::serveStatic; using httparena::validWebSocketKey; class ArenaHandler final : public RequestHandler { @@ -71,8 +71,7 @@ class ArenaHandler final : public RequestHandler { // it creates. Taking a reference rather than a shared_ptr copy keeps two // atomic refcount updates on a globally shared cache line off the per-request // path — one handler is allocated per request. - ArenaHandler(const folly::dynamic &dataset, const StaticAssets &assets) - : dataset_(dataset), assets_(assets) {} + explicit ArenaHandler(const folly::dynamic &dataset) : dataset_(dataset) {} void onRequest(std::unique_ptr request) noexcept override { const auto path = request->getPathAsStringPiece(); @@ -331,26 +330,29 @@ class ArenaHandler final : public RequestHandler { sendText(405, "Method Not Allowed", "method not allowed"); return; } - const auto *asset = assets_.find(staticName_); - if (asset == nullptr) { + if (staticName_.empty() || staticName_.find('/') != std::string::npos || + staticName_.find('\\') != std::string::npos || + staticName_.find("..") != std::string::npos) { + sendText(404, "Not Found", "not found"); + return; + } + auto asset = serveStatic(staticName_, acceptEncoding_); + if (!asset.body) { sendText(404, "Not Found", "not found"); return; } - const auto [body, encoding] = asset->select(acceptEncoding_); responseFinished_ = true; ResponseBuilder builder(downstream_); builder.status(200, "OK") - .header(proxygen::HTTP_HEADER_CONTENT_TYPE, asset->contentType); - if (encoding != ContentEncoding::Identity) { + .header(proxygen::HTTP_HEADER_CONTENT_TYPE, asset.contentType); + if (asset.encoding != ContentEncoding::Identity) { builder.header(proxygen::HTTP_HEADER_CONTENT_ENCODING, - httparena::encodingToken(encoding)); + httparena::encodingToken(asset.encoding)); // The same URL yields different bytes per Accept-Encoding. builder.header(proxygen::HTTP_HEADER_VARY, "Accept-Encoding"); } - // Non-owning view of the preloaded table, which outlives every request. - builder.body(folly::IOBuf::wrapBuffer(body->data(), body->size())) - .sendWithEOM(); + builder.body(std::move(asset.body)).sendWithEOM(); } void sendResponse(uint16_t status, const char *reason, std::string_view type, @@ -386,7 +388,6 @@ class ArenaHandler final : public RequestHandler { Route route_{Route::NotFound}; const folly::dynamic &dataset_; - const StaticAssets &assets_; HTTPMethod method_{HTTPMethod::GET}; int64_t a_{0}; int64_t b_{0}; @@ -408,21 +409,19 @@ class ArenaHandler final : public RequestHandler { class ArenaHandlerFactory final : public RequestHandlerFactory { public: - ArenaHandlerFactory(std::shared_ptr dataset, - const StaticAssets &assets) - : dataset_(std::move(dataset)), assets_(assets) {} + explicit ArenaHandlerFactory(std::shared_ptr dataset) + : dataset_(std::move(dataset)) {} void onServerStart(folly::EventBase * /*eventBase*/) noexcept override {} void onServerStop() noexcept override {} RequestHandler *onRequest(RequestHandler *, HTTPMessage *) noexcept override { - return new ArenaHandler(*dataset_, assets_); + return new ArenaHandler(*dataset_); } private: std::shared_ptr dataset_; - const StaticAssets &assets_; }; wangle::SSLContextConfig h1TlsConfig() { @@ -479,11 +478,8 @@ int main(int argc, char *argv[]) { CHECK_GT(FLAGS_threads, 0); CHECK_GT(FLAGS_h3_threads, 0); - static httparena::StaticAssets assets; - try { auto dataset = loadDataset(); - assets.load(); proxygen::HTTPServerOptions options; options.threads = static_cast(FLAGS_threads); @@ -506,15 +502,13 @@ int main(int argc, char *argv[]) { options.receiveSessionWindowSize = 10U << 20; options.maxConcurrentIncomingStreams = 1024; options.handlerFactories = - RequestHandlerChain() - .addThen(dataset, assets) - .build(); + RequestHandlerChain().addThen(dataset).build(); httparena::ArenaHQServer h3Server( FLAGS_cert, FLAGS_key, static_cast(FLAGS_h3_threads), [dataset](HTTPMessage *) -> proxygen::HTTPTransactionHandler * { return new proxygen::RequestHandlerAdaptor( - new ArenaHandler(*dataset, assets)); + new ArenaHandler(*dataset)); }); HTTPServer server(std::move(options)); server.bind(listenerConfigs()); diff --git a/frameworks/proxygen/src/CoroServer.cpp b/frameworks/proxygen/src/CoroServer.cpp index 7787c07ec..e475f56e3 100644 --- a/frameworks/proxygen/src/CoroServer.cpp +++ b/frameworks/proxygen/src/CoroServer.cpp @@ -74,7 +74,7 @@ using httparena::loadDataset; using httparena::parseInteger; using httparena::queryValue; using httparena::ContentEncoding; -using httparena::StaticAssets; +using httparena::serveStatic; using httparena::validWebSocketKey; HTTPFixedSource *makeResponse(uint16_t status, @@ -268,9 +268,7 @@ proxygen::CompressionFilterUtils::FactoryOptions compressionOptions() { class ArenaCoroHandler final : public HTTPHandler { public: explicit ArenaCoroHandler(std::shared_ptr dataset) - : dataset_(std::move(dataset)), items_(*dataset_) { - assets_.load(); - } + : dataset_(std::move(dataset)), items_(*dataset_) {} folly::coro::Task handleRequest(folly::EventBase * /*eventBase*/, @@ -400,24 +398,25 @@ class ArenaCoroHandler final : public HTTPHandler { if (method != HTTPMethod::GET) { co_return makeTextResponse(405, "method not allowed"); } - // Exact lookup in the preloaded table, so traversal is impossible and a - // miss is just a 404. - const auto *asset = assets_.find(path.substr(kStaticPrefix.size())); - if (asset == nullptr) { + const std::string_view name = path.substr(kStaticPrefix.size()); + if (name.empty() || name.find('/') != std::string_view::npos || + name.find('\\') != std::string_view::npos || + name.find("..") != std::string_view::npos) { co_return makeTextResponse(404, "not found"); } const auto &accept = request->getHeaders().getSingleOrEmpty( proxygen::HTTP_HEADER_ACCEPT_ENCODING); - const auto [body, encoding] = - asset->select(std::string_view(accept.data(), accept.size())); - // Non-owning view of the table, which outlives every request. + auto asset = + serveStatic(name, std::string_view(accept.data(), accept.size())); + if (!asset.body) { + co_return makeTextResponse(404, "not found"); + } auto *response = - makeResponse(200, asset->contentType, - folly::IOBuf::wrapBuffer(body->data(), body->size())); - if (encoding != ContentEncoding::Identity) { + makeResponse(200, asset.contentType, std::move(asset.body)); + if (asset.encoding != ContentEncoding::Identity) { auto &headers = response->msg_->getHeaders(); headers.set(proxygen::HTTP_HEADER_CONTENT_ENCODING, - httparena::encodingToken(encoding)); + httparena::encodingToken(asset.encoding)); headers.set(proxygen::HTTP_HEADER_VARY, "Accept-Encoding"); } co_return response; @@ -462,7 +461,6 @@ class ArenaCoroHandler final : public HTTPHandler { const folly::dynamic &items_; const proxygen::CompressionFilterUtils::FactoryOptions compression_{ compressionOptions()}; - StaticAssets assets_; }; constexpr uint32_t kH2StreamWindow = 1U << 20;