diff --git a/dd-trace-core/src/main/java/datadog/trace/lambda/ContentTypeBodyParser.java b/dd-trace-core/src/main/java/datadog/trace/lambda/ContentTypeBodyParser.java new file mode 100644 index 00000000000..29996b059ce --- /dev/null +++ b/dd-trace-core/src/main/java/datadog/trace/lambda/ContentTypeBodyParser.java @@ -0,0 +1,286 @@ +package datadog.trace.lambda; + +import datadog.trace.api.appsec.MediaType; +import datadog.trace.lambda.MultipartSplitter.Part; +import java.io.UnsupportedEncodingException; +import java.net.URLDecoder; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.StringTokenizer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Turns a Lambda request body into the shape the AppSec WAF expects. The declared {@code + * Content-Type} decides how the body is structured; a best-effort JSON parse handles the JSON-ish + * types and a top-level body that declares no type at all. + * + *

A body is never dropped: any type we cannot structure — and any parse failure — degrades to + * the raw {@link String}, which the WAF can still match string rules against. + */ +final class ContentTypeBodyParser { + + private static final Logger log = LoggerFactory.getLogger(ContentTypeBodyParser.class); + + // These bound the work done in this parser only: exceeding any of them degrades the body to a raw + // string rather than dropping content. + static final int MAX_BYTES = 1024 * 1024; + static final int MAX_PARTS = 256; + static final int MAX_DEPTH = 20; + + private ContentTypeBodyParser() {} + + /** + * State shared across a whole parse: the byte and part allowances, and the filenames collected + * along the way. A multipart part may itself hold a multipart body, so a per-call allowance would + * be re-satisfied at every nesting level. + */ + static final class ParseContext { + private int bytes; + private int parts = MAX_PARTS; + + ParseContext() { + this(MAX_BYTES); + } + + /** + * @param byteAllowance the total number of characters this parse may read, nesting included + */ + ParseContext(final int byteAllowance) { + this.bytes = byteAllowance; + } + + /** Allocated only once a file part is seen, which most bodies never do. */ + private List filenames; + + int remainingParts() { + return parts; + } + + void consumeParts(final int count) { + parts -= count; + } + + /** + * @return {@code false} when the parse can no longer afford to read {@code count} characters + */ + boolean takeBytes(final int count) { + if (bytes < count) { + return false; + } + bytes -= count; + return true; + } + + /** + * @return {@code false} once the part allowance is spent. A multipart part and an urlencoded + * parameter both draw from it. + */ + boolean takePart() { + if (parts == 0) { + return false; + } + parts--; + return true; + } + + void addFilename(final String filename) { + if (filenames == null) { + filenames = new ArrayList<>(2); + } + filenames.add(filename); + } + + /** + * @return the filenames of the multipart file parts found, in body order, empty when there were + * none + */ + List filenames() { + return filenames == null ? Collections.emptyList() : filenames; + } + } + + /** + * Parses a decoded request body according to its {@code Content-Type}. + * + * @param context also collects the filenames of any multipart file parts found, which the caller + * reports separately from the body + */ + static Object parseBody(final String body, final String contentType, final ParseContext context) { + return dispatch(body, contentType, 0, context); + } + + static Object dispatch( + final String body, final String contentType, final int depth, final ParseContext context) { + if (body == null) { + return null; + } + if (depth >= MAX_DEPTH) { + log.debug("Body nesting depth {} reached, keeping raw string", depth); + return body; + } + if (!context.takeBytes(body.length())) { + log.debug( + "Byte allowance cannot cover a body of {} chars, keeping raw string", body.length()); + return body; + } + final MediaType mediaType = MediaType.parse(contentType); + if (isJsonOrUntyped(mediaType)) { + final Object parsed = LambdaEventParser.parseBodyAsJson(body); + return parsed != null ? parsed : body; + } + if ("application".equals(mediaType.getType()) + && "x-www-form-urlencoded".equals(mediaType.getSubtype())) { + final Object parsed = parseUrlEncoded(body, context); + return parsed != null ? parsed : body; + } + if ("multipart".equals(mediaType.getType())) { + final Object parsed = parseMultipart(body, contentType, depth, context); + return parsed != null ? parsed : body; + } + // text/* and everything else stay raw strings. In particular a text/plain body of "12345" must + // reach the WAF as a String, not as the Double a JSON parse would produce. + return body; + } + + static boolean isJsonOrUntyped(final MediaType mediaType) { + final String subtype = mediaType.getSubtype(); + return mediaType.getType() == null + || (subtype != null && (subtype.contains("json") || subtype.contains("javascript"))); + } + + /** + * Parses an {@code application/x-www-form-urlencoded} body into a multimap, matching the shape + * produced for query parameters. + * + * @return the parsed parameters, or {@code null} if nothing usable was found or the body exhausts + * the part allowance + */ + private static Map> parseUrlEncoded( + final String body, final ParseContext context) { + if (body.isEmpty()) { + return null; + } + final Map> parameters = new LinkedHashMap<>(); + final StringTokenizer tokenizer = new StringTokenizer(body, "&"); + while (tokenizer.hasMoreTokens()) { + if (!context.takePart()) { + log.debug("Part allowance exhausted, keeping urlencoded body as a raw string"); + return null; + } + final String pair = tokenizer.nextToken(); + final int equals = pair.indexOf('='); + final String name = decode(equals == -1 ? pair : pair.substring(0, equals)); + if (!name.isEmpty()) { + parameters + .computeIfAbsent(name, k -> new ArrayList<>(1)) + .add(equals == -1 ? "" : decode(pair.substring(equals + 1))); + } + } + if (parameters.isEmpty()) { + return null; + } + log.debug("Body parsed as {} urlencoded parameters", parameters.size()); + return parameters; + } + + /** + * Parses a {@code multipart/*} body into its form fields. + * + * @return the fields found, or {@code null} if the body has no usable boundary, it holds more + * parts than the allowance, or it yields no field + */ + private static Object parseMultipart( + final String body, final String contentType, final int depth, final ParseContext context) { + final String boundary = MultipartSplitter.extractBoundary(contentType); + if (boundary == null) { + log.debug("Multipart body without a usable boundary, keeping raw string"); + return null; + } + // One over the allowance, so that a body holding more parts than may be read is distinguishable + // from one holding exactly the allowance + final int allowance = context.remainingParts(); + final List parts = MultipartSplitter.split(body, boundary, allowance + 1); + if (parts.size() > allowance) { + log.debug("Part allowance exhausted, keeping multipart body as a raw string"); + return null; + } + context.consumeParts(parts.size()); + + final Map fields = new LinkedHashMap<>(); + final Map> promoted = new HashMap<>(); + for (final Part part : parts) { + final String disposition = part.contentDisposition; + if (disposition == null) { + continue; + } + final String filename = MultipartSplitter.parameter(disposition, "filename"); + if (filename != null) { + if (!filename.isEmpty()) { + context.addFilename(filename); + } + continue; + } + final String name = MultipartSplitter.parameter(disposition, "name"); + if (name == null || name.isEmpty()) { + continue; + } + final String partContentType = part.contentType; + final String content = body.substring(part.contentStart, part.contentEnd); + // A part that declares no type is kept as a raw string + final Object value = + partContentType == null || partContentType.isEmpty() + ? content + : dispatch(content, partContentType, depth + 1, context); + addField(fields, promoted, name, value); + } + return fields.isEmpty() ? null : fields; + } + + /** + * Accumulates a field as a scalar on first sight and promotes it to a list on repeat. + * Deliberately a different shape from urlencoded's always-a-list, matching the peer tracers. + * + * @param promoted the list each promoted field was given, by name, mutated as fields are + * promoted. Tracked rather than inferred from the stored value's type: a part whose body + * parsed as a JSON array is itself a List, and appending to it would flatten the two apart. + */ + private static void addField( + final Map fields, + final Map> promoted, + final String name, + final Object value) { + final List values = promoted.get(name); + if (values != null) { + values.add(value); + return; + } + // A part value is never null, so an absent key is exactly a null lookup + final Object existing = fields.get(name); + if (existing == null) { + fields.put(name, value); + } else { + final List promotion = new ArrayList<>(2); + promotion.add(existing); + promotion.add(value); + fields.put(name, promotion); + promoted.put(name, promotion); + } + } + + /** Percent-decodes a single token, keeping it undecoded rather than dropping it on failure. */ + private static String decode(final String value) { + if (value.isEmpty()) { + return value; + } + try { + return URLDecoder.decode(value, "UTF-8"); + } catch (final UnsupportedEncodingException | IllegalArgumentException e) { + return value; + } + } +} diff --git a/dd-trace-core/src/main/java/datadog/trace/lambda/LambdaAppSecHandler.java b/dd-trace-core/src/main/java/datadog/trace/lambda/LambdaAppSecHandler.java index 4aac101e98a..3aae3404281 100644 --- a/dd-trace-core/src/main/java/datadog/trace/lambda/LambdaAppSecHandler.java +++ b/dd-trace-core/src/main/java/datadog/trace/lambda/LambdaAppSecHandler.java @@ -39,6 +39,7 @@ import java.io.ByteArrayOutputStream; import java.nio.charset.StandardCharsets; import java.util.Collections; +import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; import java.util.function.BiFunction; @@ -58,7 +59,7 @@ public class LambdaAppSecHandler { private static final RatelimitedLogger rlLog = new RatelimitedLogger(log, 5, TimeUnit.MINUTES); /** - * Marks an invocation AppSec did not process because the trigger is not HTTP, or if the even is + * Marks an invocation AppSec did not process because the trigger is not HTTP, or if the event is * unreadable (not a {@code ByteArrayInputStream}, empty, oversized, or unparseable). */ private static final String UNSUPPORTED_EVENT_TYPE_METRIC = "_dd.appsec.unsupported_event_type"; @@ -477,6 +478,20 @@ private static AgentSpanContext processAppSecRequestData( log.debug("requestBodyProcessed callback is null"); } } + + // Call requestFilesFilenames. Only the names are reported: the file content shares the + // body's UTF-8 decode, so for anything that is not text it is already lossy. + if (!eventData.filenames.isEmpty()) { + BiFunction, Flow> filenamesCallback = + tracer + .getCallbackProvider(RequestContextSlot.APPSEC) + .getCallback(EVENTS.requestFilesFilenames()); + if (filenamesCallback != null) { + filenamesCallback.apply(requestContext, eventData.filenames); + } else { + log.debug("requestFilesFilenames callback is null"); + } + } } return tagContext; } diff --git a/dd-trace-core/src/main/java/datadog/trace/lambda/LambdaEventParser.java b/dd-trace-core/src/main/java/datadog/trace/lambda/LambdaEventParser.java index 1848e78b620..5b8ad1fc35e 100644 --- a/dd-trace-core/src/main/java/datadog/trace/lambda/LambdaEventParser.java +++ b/dd-trace-core/src/main/java/datadog/trace/lambda/LambdaEventParser.java @@ -3,6 +3,8 @@ import com.squareup.moshi.JsonAdapter; import com.squareup.moshi.Moshi; import datadog.trace.api.Config; +import datadog.trace.api.appsec.MediaType; +import datadog.trace.lambda.ContentTypeBodyParser.ParseContext; import java.io.ByteArrayInputStream; import java.io.IOException; import java.net.URLEncoder; @@ -94,8 +96,8 @@ static LambdaRequestData parseEvent(String json) { case ALB_MULTI_VALUE: return extractAlbData(event, triggerType); default: - // Unsupported trigger: AppSec skips the invocation entirely, so there is nothing to - // extract. The trigger type is carried by the caller, not by this result. + // Unsupported trigger: returning EMPTY makes the caller skip the invocation, so there is + // nothing to extract sinc ethe event is not supported. return LambdaRequestData.EMPTY; } } catch (Exception e) { @@ -160,20 +162,11 @@ static LambdaResponseData parseResponse(String json) { } if (bodyString != null) { - String contentType = headers.get("content-type"); - - // If JSON content-type or unknown, attempt JSON parsing - // Normalise casing: media type tokens are case-insensitive per RFC 7231 - String contentTypeLower = - contentType == null ? null : contentType.toLowerCase(Locale.ROOT); - if (contentTypeLower == null - || contentTypeLower.contains("json") - || contentTypeLower.contains("javascript")) { - Object parsed = parseBodyAsJson(bodyString); - body = parsed != null ? parsed : bodyString; - } else { - body = bodyString; - } + // A response body is only ever structured as JSON, never as urlencoded or multipart + MediaType mediaType = MediaType.parse(headers.get("content-type")); + Object parsed = + ContentTypeBodyParser.isJsonOrUntyped(mediaType) ? parseBodyAsJson(bodyString) : null; + body = parsed != null ? parsed : bodyString; } } @@ -247,7 +240,8 @@ private static LambdaRequestData extractApiGatewayV1Data(Map eve if (queryParameters.isEmpty()) { queryParameters = extractQueryParameters(event.get("queryStringParameters")); } - Object body = extractBody(event); + ParseContext parseContext = new ParseContext(); + Object body = extractBody(event, headers, parseContext); Map requestContext = (Map) event.get("requestContext"); String method = (String) requestContext.get("httpMethod"); @@ -273,7 +267,8 @@ private static LambdaRequestData extractApiGatewayV1Data(Map eve extractHost(requestContext, headers), // REST APIs expose the parameterized route as the top-level "resource" stringOrNull(event.get("resource")), - null); + null, + parseContext.filenames()); } /** Extracts data from API Gateway v2 (HTTP API) or Lambda URL event */ @@ -283,7 +278,8 @@ private static LambdaRequestData extractApiGatewayV2HttpData( Map pathParameters = extractPathParameters(event.get("pathParameters")); Map> queryParameters = extractQueryParameters(event.get("queryStringParameters")); - Object body = extractBody(event); + ParseContext parseContext = new ParseContext(); + Object body = extractBody(event, headers, parseContext); Map requestContext = (Map) event.get("requestContext"); Map http = (Map) requestContext.get("http"); @@ -311,7 +307,8 @@ private static LambdaRequestData extractApiGatewayV2HttpData( body, extractHost(requestContext, headers), extractRouteKey(requestContext), - extractRawUri(event)); + extractRawUri(event), + parseContext.filenames()); } /** @@ -337,7 +334,8 @@ private static LambdaRequestData extractApiGatewayV2WebSocketData(Map pathParameters = extractPathParameters(event.get("pathParameters")); Map> queryParameters = extractQueryParameters(event.get("queryStringParameters")); - Object body = extractBody(event); + ParseContext parseContext = new ParseContext(); + Object body = extractBody(event, headers, parseContext); Map requestContext = (Map) event.get("requestContext"); @@ -365,7 +363,8 @@ private static LambdaRequestData extractApiGatewayV2WebSocketData(Map extractHeadersWithCookies(Map } /** Helper method to extract and parse body from event */ - private static Object extractBody(Map event) { + private static Object extractBody( + Map event, Map headers, ParseContext parseContext) { Object bodyObj = event.get("body"); if (bodyObj == null) { return null; @@ -693,20 +695,11 @@ private static Object extractBody(Map event) { } } - // Try to parse as JSON - Object parsedBody = parseBodyAsJson(bodyString); - if (parsedBody != null) { - log.debug("Body parsed as JSON successfully"); - return parsedBody; - } - - // If not JSON, return the raw string - log.debug("Body is not JSON, returning raw string"); - return bodyString; + return ContentTypeBodyParser.parseBody(bodyString, headers.get("content-type"), parseContext); } /** Helper method to parse body as JSON */ - private static Object parseBodyAsJson(String body) { + static Object parseBodyAsJson(String body) { if (body == null || body.isEmpty() || "null".equals(body)) { return null; } @@ -772,6 +765,9 @@ static class LambdaRequestData { */ final String rawUri; + /** Filenames of the multipart file parts carried by the body, empty when there are none. */ + final List filenames; + static final LambdaRequestData EMPTY = new LambdaRequestData( Collections.emptyMap(), @@ -782,32 +778,11 @@ static class LambdaRequestData { LambdaTriggerType.UNKNOWN, Collections.emptyMap(), Collections.emptyMap(), - null); - - LambdaRequestData( - Map headers, - String method, - String path, - String sourceIp, - Integer sourcePort, - LambdaTriggerType triggerType, - Map pathParameters, - Map> queryParameters, - Object body) { - this( - headers, - method, - path, - sourceIp, - sourcePort, - triggerType, - pathParameters, - queryParameters, - body, - null, - null, - null); - } + null, + null, + null, + null, + Collections.emptyList()); LambdaRequestData( Map headers, @@ -821,7 +796,8 @@ static class LambdaRequestData { Object body, String host, String route, - String rawUri) { + String rawUri, + List filenames) { this.headers = headers; this.method = method; this.path = path; @@ -834,6 +810,7 @@ static class LambdaRequestData { this.host = host; this.route = route; this.rawUri = rawUri; + this.filenames = filenames; } } diff --git a/dd-trace-core/src/main/java/datadog/trace/lambda/MultipartSplitter.java b/dd-trace-core/src/main/java/datadog/trace/lambda/MultipartSplitter.java new file mode 100644 index 00000000000..a64159f94f5 --- /dev/null +++ b/dd-trace-core/src/main/java/datadog/trace/lambda/MultipartSplitter.java @@ -0,0 +1,329 @@ +package datadog.trace.lambda; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** + * Splits a {@code multipart/*} body into its parts and reads {@code Content-Type} / {@code + * Content-Disposition} parameters. + */ +final class MultipartSplitter { + + private static final int MAX_BOUNDARY_LENGTH = 70; + + private MultipartSplitter() {} + + /** + * One part of a multipart body. Content is an index range into the body passed to {@link #split}. + */ + static final class Part { + /** {@code null} when the part declares no such header. Duplicates keep the last seen value. */ + final String contentDisposition; + + final String contentType; + final int contentStart; + final int contentEnd; + + private Part( + final String contentDisposition, + final String contentType, + final int contentStart, + final int contentEnd) { + this.contentDisposition = contentDisposition; + this.contentType = contentType; + this.contentStart = contentStart; + this.contentEnd = contentEnd; + } + } + + /** + * Splits a multipart body into at most {@code partBudget} parts. + * + * @return the parts found, in order; empty if the body holds none or is not parseable as + * multipart + */ + static List split(final String body, final String boundary, final int partBudget) { + final List parts = new ArrayList<>(); + if (body == null || boundary == null || boundary.isEmpty() || partBudget <= 0) { + return parts; + } + final String delimiter = "--" + boundary; + final int length = body.length(); + int position = body.startsWith(delimiter) ? 0 : nextDelimiter(body, delimiter, 0); + + while (position >= 0 && parts.size() < partBudget) { + final int afterDelimiter = position + delimiter.length(); + if (body.startsWith("--", afterDelimiter)) { + // Close delimiter: anything past it is the epilogue + break; + } + final int headerStart = lineStart(body, afterDelimiter); + if (headerStart < 0) { + // Not a part boundary after all — the delimiter was part of some part's content + position = nextDelimiter(body, delimiter, afterDelimiter); + continue; + } + // Only these two headers are ever read, so the others are matched and dropped rather than + // collected: a part declaring thousands of them costs nothing but the scan. + String contentDisposition = null; + String contentType = null; + int cursor = headerStart; + boolean headersComplete = false; + while (cursor < length) { + if (body.startsWith(delimiter, cursor) && endsLine(body, cursor + delimiter.length())) { + // This part's headers are not followed by a blank line, so no conforming parser reads + // this body as multipart. Report nothing rather than the parts that happen to survive: + // the caller then keeps the raw string, and the WAF sees every byte of it. + return Collections.emptyList(); + } + final int newline = body.indexOf('\n', cursor); + final int lineEnd = newline < 0 ? length : newline; + final int trimmed = + lineEnd > cursor && body.charAt(lineEnd - 1) == '\r' ? lineEnd - 1 : lineEnd; + if (trimmed == cursor) { + headersComplete = true; + cursor = newline < 0 ? length : newline + 1; + break; + } + final String disposition = headerValue(body, cursor, trimmed, "Content-Disposition"); + if (disposition != null) { + contentDisposition = disposition; + } else { + final String type = headerValue(body, cursor, trimmed, "Content-Type"); + if (type != null) { + contentType = type; + } + } + if (newline < 0) { + cursor = length; + break; + } + cursor = newline + 1; + } + if (!headersComplete) { + // The body ended inside this part's headers. They are content in their own right — a + // filename lives there — and no parser accepts a body cut short like this, so report + // nothing and let the caller keep the raw string. + return Collections.emptyList(); + } + final int next = nextDelimiter(body, delimiter, cursor); + parts.add(new Part(contentDisposition, contentType, cursor, contentEnd(body, cursor, next))); + position = next; + } + return parts; + } + + /** + * Reads the {@code boundary} parameter of a {@code Content-Type} header, case preserved. + * + * @return the boundary, or {@code null} if absent or unusable + */ + static String extractBoundary(final String contentType) { + final String boundary = parameter(contentType, "boundary"); + if (boundary == null || boundary.isEmpty() || boundary.length() > MAX_BOUNDARY_LENGTH) { + return null; + } + return boundary; + } + + /** + * Reads a header parameter, case preserved. The name is matched case-insensitively and only at a + * parameter boundary, so {@code filename} does not satisfy a lookup for {@code name}. Quoted + * values may hold separators and {@code \"} escapes, and are skipped whole: a field named {@code + * "; filename=x"} does not read as a {@code filename} parameter. + * + * @return the parameter value, {@code ""} when it is present but empty, or {@code null} when the + * parameter is absent + */ + static String parameter(final String headerValue, final String paramName) { + if (headerValue == null || paramName == null) { + return null; + } + final int length = headerValue.length(); + final int nameLength = paramName.length(); + boolean atBoundary = true; + for (int i = 0; i < length; i++) { + final char c = headerValue.charAt(i); + if (c == '"') { + i = endOfQuoted(headerValue, i); + atBoundary = false; + continue; + } + if (atBoundary && headerValue.regionMatches(true, i, paramName, 0, nameLength)) { + final int equals = skipOptionalWhitespace(headerValue, i + nameLength); + if (equals < length && headerValue.charAt(equals) == '=') { + return value(headerValue, skipOptionalWhitespace(headerValue, equals + 1)); + } + } + atBoundary = isParameterSeparator(c); + } + return null; + } + + /** + * Walks a quoted value, honouring {@code \"} escapes. + * + * @param openQuote the index of the opening quote + * @return the index of the closing quote, or the body length when the quote is unterminated + */ + private static int endOfQuoted(final String headerValue, final int openQuote) { + final int length = headerValue.length(); + for (int i = openQuote + 1; i < length; i++) { + final char c = headerValue.charAt(i); + if (c == '\\' && i + 1 < length) { + i++; + } else if (c == '"') { + return i; + } + } + return length; + } + + private static String value(final String headerValue, final int from) { + final int length = headerValue.length(); + if (from < length && headerValue.charAt(from) == '"') { + // An unterminated quote ends at the body length, so what was read is kept rather than the + // parameter being dropped. + final int close = endOfQuoted(headerValue, from); + final StringBuilder unquoted = new StringBuilder(close - from); + for (int i = from + 1; i < close; i++) { + final char c = headerValue.charAt(i); + if (c == '\\' && i + 1 < length) { + unquoted.append(headerValue.charAt(++i)); + } else { + unquoted.append(c); + } + } + return unquoted.toString(); + } + int end = length; + for (int i = from; i < length; i++) { + if (isParameterSeparator(headerValue.charAt(i))) { + end = i; + break; + } + } + return headerValue.substring(from, end); + } + + private static int skipOptionalWhitespace(final String headerValue, final int from) { + int i = from; + final int length = headerValue.length(); + while (i < length && isOptionalWhitespace(headerValue.charAt(i))) { + i++; + } + return i; + } + + private static boolean isParameterSeparator(final char c) { + return c == ';' || c == ',' || c == ' ' || c == '\t'; + } + + /** + * Finds the next delimiter at or after {@code from}. + * + * @return the index the delimiter starts at, or {@code -1} if there is none + */ + private static int nextDelimiter(final String body, final String delimiter, final int from) { + if (body.startsWith(delimiter, from) && endsLine(body, from + delimiter.length())) { + return from; + } + for (int newline = body.indexOf('\n', from); + newline >= 0; + newline = body.indexOf('\n', newline + 1)) { + if (body.startsWith(delimiter, newline + 1) + && endsLine(body, newline + 1 + delimiter.length())) { + return newline + 1; + } + } + return -1; + } + + /** + * A delimiter only delimits if its line ends there, RFC 2046 transport padding aside — for the + * close delimiter, past its trailing {@code --}. A content line that merely starts with one, be + * it {@code --x-not-a-boundary} or {@code --x--not-a-close}, is data, and must not end the part + * early and hide the rest from the WAF. + * + * @param after the index just past the matched delimiter + */ + private static boolean endsLine(final String body, final int after) { + final int end = body.startsWith("--", after) ? after + 2 : after; + return end == body.length() || lineStart(body, end) >= 0; + } + + /** + * Skips the linear whitespace and line break that follow a delimiter. + * + * @return the index the header lines start at, or {@code -1} if no line break follows + */ + private static int lineStart(final String body, final int from) { + int i = from; + final int length = body.length(); + while (i < length && isOptionalWhitespace(body.charAt(i))) { + i++; + } + if (i < length && body.charAt(i) == '\r') { + i++; + } + return i < length && body.charAt(i) == '\n' ? i + 1 : -1; + } + + /** + * Ends a part's content before the line break that introduces the next delimiter, or at the end + * of the body when the closing delimiter was truncated away. + */ + private static int contentEnd(final String body, final int contentStart, final int next) { + if (next < 0) { + return body.length(); + } + int end = next > contentStart ? next - 1 : contentStart; + if (end > contentStart && body.charAt(end - 1) == '\r') { + end--; + } + return end; + } + + /** + * Reads one header line, without allocating unless the name is the one asked for. + * + * @param from the first index of the line, {@code to} the index past its last character, the + * trailing {@code \r} already excluded + * @param name the header name to match, case-insensitively + * @return the trimmed header value, or {@code null} when the line declares another header or is + * not a header line at all — there is no colon, and obs-fold is unsupported + */ + private static String headerValue( + final String body, final int from, final int to, final String name) { + int colon = -1; + for (int i = from; i < to; i++) { + if (body.charAt(i) == ':') { + colon = i; + break; + } + } + if (colon < 0) { + return null; + } + int start = from; + int end = colon; + while (start < end && isOptionalWhitespace(body.charAt(start))) { + start++; + } + while (end > start && isOptionalWhitespace(body.charAt(end - 1))) { + end--; + } + // The canonical spelling first: that comparison is intrinsified, and every real client sends it + if (end - start != name.length() + || !(body.regionMatches(start, name, 0, name.length()) + || body.regionMatches(true, start, name, 0, name.length()))) { + return null; + } + return body.substring(colon + 1, to).trim(); + } + + private static boolean isOptionalWhitespace(final char c) { + return c == ' ' || c == '\t'; + } +} diff --git a/dd-trace-core/src/test/java/datadog/trace/lambda/ContentTypeBodyParserTest.java b/dd-trace-core/src/test/java/datadog/trace/lambda/ContentTypeBodyParserTest.java new file mode 100644 index 00000000000..c55af1eaa11 --- /dev/null +++ b/dd-trace-core/src/test/java/datadog/trace/lambda/ContentTypeBodyParserTest.java @@ -0,0 +1,457 @@ +package datadog.trace.lambda; + +import static datadog.trace.lambda.ContentTypeBodyParser.MAX_DEPTH; +import static datadog.trace.lambda.ContentTypeBodyParser.MAX_PARTS; +import static datadog.trace.lambda.ContentTypeBodyParser.dispatch; +import static java.util.Arrays.asList; +import static java.util.Collections.emptyList; +import static java.util.Collections.singletonList; +import static java.util.Collections.singletonMap; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNull; + +import datadog.trace.lambda.ContentTypeBodyParser.ParseContext; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; + +class ContentTypeBodyParserTest { + + private static final String MULTIPART = "multipart/form-data; boundary=outer"; + private static final String URL_ENCODED = "application/x-www-form-urlencoded"; + + @ParameterizedTest(name = "[{index}] {1} -> {2}") + @CsvSource( + delimiter = '|', + value = { + // content type absent or blank: best-effort JSON, as before content-type dispatch existed + "{\"a\":1} | | MAP", + "not json | | STRING", + // a header holding nothing but parameters declares no type either + "{\"a\":1} | '; charset=utf-8' | MAP", + // JSON, including suffixed subtypes and parameters + "{\"a\":1} | application/json | MAP", + "{\"a\":1} | application/json; charset=utf-8 | MAP", + "{\"a\":1} | APPLICATION/JSON | MAP", + "{\"a\":1} | application/vnd.api+json | MAP", + "{\"a\":1 | application/json | STRING", + // JSON-ish vendor types: no +json suffix, but the payload is JSON + "{\"a\":1} | application/x-amz-json-1.1 | MAP", + "{\"a\":1} | application/javascript | MAP", + // urlencoded + "a=1 | application/x-www-form-urlencoded | MAP", + // a multipart body whose boundary happens to contain "json" must still reach the multipart + // parser rather than being handed to the JSON parser + "not multipart | multipart/x; boundary=--json | STRING", + // a json-ish subtype wins over the multipart branch, whatever the type says + "{\"a\":1} | multipart/json | MAP", + // text/*: never structured, even when it holds JSON. A JSON parse of "12345" would yield a + // Double, which no string rule can match + "{\"a\":1} | text/plain | STRING", + "12345 | text/plain | STRING", + // anything else + "{\"a\":1} | application/xml | STRING", + "{\"a\":1} | garbage | STRING", + // a slashless header declares a type but no subtype, so it is not JSON-ish + "{\"a\":1} | json | STRING", + }) + void dispatchesOnContentType(String body, String contentType, String expectedKind) { + Object parsed = parseBody(body, contentType); + + if ("MAP".equals(expectedKind)) { + assertInstanceOf(Map.class, parsed); + } else { + assertEquals(body, parsed); + } + } + + @Test + void returnsNullForNullBody() { + assertNull(parseBody(null, "application/json")); + } + + @Test + void keepsEmptyBodyAsEmptyString() { + assertEquals("", parseBody("", "application/json")); + assertEquals("", parseBody("", "application/x-www-form-urlencoded")); + } + + @Test + void keepsRawStringOnceMaxDepthIsReached() { + String body = "{\"a\":1}"; + + assertEquals(body, dispatch(body, "application/json", MAX_DEPTH, new ParseContext())); + assertInstanceOf( + Map.class, dispatch(body, "application/json", MAX_DEPTH - 1, new ParseContext())); + } + + @Test + void parsesUrlEncodedIntoAMultimap() { + Map parsed = urlEncoded("user=admin&role=root"); + + assertEquals(singletonList("admin"), parsed.get("user")); + assertEquals(singletonList("root"), parsed.get("role")); + assertEquals(2, parsed.size()); + } + + @Test + void groupsRepeatedUrlEncodedKeysIntoOneList() { + assertEquals(asList("a", "b", "c"), urlEncoded("x=a&x=b&x=c").get("x")); + } + + @Test + void decodesUrlEncodedPercentEscapesAndPluses() { + Map parsed = urlEncoded("na+me=hello+world&q=%7B%22a%22%3A1%7D"); + + assertEquals(singletonList("hello world"), parsed.get("na me")); + assertEquals(singletonList("{\"a\":1}"), parsed.get("q")); + } + + @Test + void keepsUndecodableUrlEncodedTokensAsIs() { + Map parsed = urlEncoded("a=%&%=b"); + + assertEquals(singletonList("%"), parsed.get("a")); + assertEquals(singletonList("b"), parsed.get("%")); + } + + @Test + void treatsValuelessUrlEncodedPairsAsEmptyValues() { + Map parsed = urlEncoded("flag&other=&last"); + + assertEquals(singletonList(""), parsed.get("flag")); + assertEquals(singletonList(""), parsed.get("other")); + assertEquals(singletonList(""), parsed.get("last")); + } + + @Test + void skipsEmptyUrlEncodedPairsAndNames() { + Map parsed = urlEncoded("&&=orphan&&a=1&&"); + + assertEquals(singletonList("1"), parsed.get("a")); + assertEquals(1, parsed.size()); + } + + @Test + void splitsUrlEncodedValuesAtTheFirstEqualsOnly() { + assertEquals(singletonList("b=c=d"), urlEncoded("a=b=c=d").get("a")); + } + + @Test + void doesNotSeparateUrlEncodedPairsOnSemicolons() { + assertEquals(singletonList("1;b=2"), urlEncoded("a=1;b=2").get("a")); + } + + @Test + void keepsUrlEncodedBodyOverThePartAllowanceAsRawString() { + StringBuilder body = new StringBuilder(); + for (int i = 0; i < MAX_PARTS; i++) { + body.append('k').append(i).append("=v&"); + } + body.append("attack=payload"); + String raw = body.toString(); + + // Truncating would hide the trailing parameter from every WAF rule, so the whole body is kept + // as a string instead — still matchable, just unstructured + assertEquals(raw, parseBody(raw, "application/x-www-form-urlencoded")); + } + + @Test + void keepsUnparseableUrlEncodedBodyAsRawString() { + // Nothing but separators: no parameter survives, so the raw body is kept + assertEquals("&&&", parseBody("&&&", "application/x-www-form-urlencoded")); + } + + @Test + void parsesMultipartFieldsIntoAMap() { + Map fields = multipart(field("user", "admin"), field("role", "root")); + + assertEquals("admin", fields.get("user")); + assertEquals("root", fields.get("role")); + assertEquals(2, fields.size()); + } + + @Test + void skipsMultipartPartsWithoutAName() { + String body = outer(part("form-data", "novalue"), part("form-data; name=", "empty")); + + assertEquals(body, parseBody(body, MULTIPART)); + } + + @Test + void skipsMultipartPartsWithoutAContentDisposition() { + Map fields = multipart(field("user", "admin"), part(null, "orphan")); + + assertEquals(1, fields.size()); + } + + @Test + void dispatchesOnEachMultipartPartsOwnContentType() { + Map fields = + multipart( + part("form-data; name=payload", "application/json", "{\"a\":1}"), + part("form-data; name=plain", "text/plain", "12345"), + part("form-data; name=form", "application/x-www-form-urlencoded", "k=v")); + + assertInstanceOf(Map.class, fields.get("payload")); + assertEquals("12345", fields.get("plain")); + assertEquals(singletonList("v"), ((Map) fields.get("form")).get("k")); + } + + @Test + void keepsMultipartPartsWithoutAUsableContentTypeAsRawStrings() { + // A part with no Content-Type is text/plain per RFC 7578, section 4.4, not a body of unknown + // type: a JSON parse would hand the WAF a Double no string rule can match. An empty header + // reads the same way, and is the only other spelling to reach here — MultipartSplitter trims. + Map fields = + multipart( + field("amount", "12345"), + field("json", "{\"a\":1}"), + part("form-data; name=\"empty\"", "", "12345")); + + assertEquals("12345", fields.get("amount")); + assertEquals("{\"a\":1}", fields.get("json")); + assertEquals("12345", fields.get("empty")); + } + + @Test + void promotesRepeatedMultipartFieldNamesToAList() { + Map fields = multipart(field("x", "a"), field("x", "b"), field("x", "c")); + + assertEquals(asList("a", "b", "c"), fields.get("x")); + } + + @Test + void nestsARepeatedJsonArrayPartValueRatherThanFlatteningIt() { + // The first value is itself a List, so appending to it would flatten two values into one + Map fields = + multipart(part("form-data; name=x", "application/json", "[1,2]"), field("x", "b")); + + assertEquals(asList(asList(1.0, 2.0), "b"), fields.get("x")); + } + + @Test + void doesNotTakeAFieldNameThatForgesAFilenameForAFilePart() { + Map fields = multipart(part("form-data; name=\"; filename=x\"", "payload")); + + assertEquals("payload", fields.get("; filename=x")); + } + + @Test + void parsesNestedMultipartBodiesAndReportsTheirFilePartsByNameOnly() { + String inner = body("inner", field("nested", "value"), file("upload", "f.txt", "data")); + String nesting = outer(part("form-data; name=group", "multipart/mixed; boundary=inner", inner)); + + Map fields = asMap(parseBody(nesting, MULTIPART)); + + assertEquals(singletonMap("nested", "value"), fields.get("group")); + assertEquals(singletonList("f.txt"), filenamesOf(nesting)); + } + + @Test + void sharesThePartAllowanceBetweenUrlEncodedParametersAndMultipartParts() { + // Two thirds of the allowance each: the second only fails if both draw from one allowance + String urlEncoded = urlEncodedPairs(MAX_PARTS * 2 / 3); + Map fields = + multipart( + part("form-data; name=first", URL_ENCODED, urlEncoded), + part("form-data; name=second", URL_ENCODED, urlEncoded)); + + assertInstanceOf(Map.class, fields.get("first")); + assertEquals(urlEncoded, fields.get("second")); + } + + private static String urlEncodedPairs(int count) { + StringBuilder body = new StringBuilder(); + for (int i = 0; i < count; i++) { + body.append(i == 0 ? "" : "&").append('k').append(i).append("=v"); + } + return body.toString(); + } + + @Test + void parsesMultipartPartsUpToThePartAllowance() { + assertEquals(MAX_PARTS, multipart(fieldParts(MAX_PARTS)).size()); + } + + @Test + void keepsMultipartBodyOverThePartAllowanceAsRawString() { + // One part over the allowance degrades the whole body, as an urlencoded body over it does + String body = outer(fieldParts(MAX_PARTS + 1)); + + assertEquals(body, parseBody(body, MULTIPART)); + } + + @Test + void sharesThePartAllowanceAcrossNestingLevels() { + // The inner body fits the allowance on its own, but the outer part it sits in has spent one + String inner = body("inner", fieldParts(MAX_PARTS)); + Map fields = + multipart(part("form-data; name=group", "multipart/mixed; boundary=inner", inner)); + + assertEquals(inner, fields.get("group")); + } + + private static String[] fieldParts(int count) { + String[] parts = new String[count]; + for (int i = 0; i < count; i++) { + parts[i] = field("k" + i, "v"); + } + return parts; + } + + @Test + void keepsMultipartBodyAsRawStringWithoutAUsableBoundary() { + String body = outer(field("user", "admin")); + + assertEquals(body, parseBody(body, "multipart/form-data")); + assertEquals(body, parseBody(body, "multipart/form-data; boundary=")); + } + + @Test + void keepsUnsplittableMultipartBodyAsRawString() { + assertEquals("no parts here", parseBody("no parts here", MULTIPART)); + } + + @Test + void keepsABodyTheByteAllowanceCannotCoverAsRawString() { + String body = outer(field("user", "admin")); + + assertInstanceOf(Map.class, parseBody(body, MULTIPART, new ParseContext(body.length()))); + assertEquals(body, parseBody(body, MULTIPART, new ParseContext(body.length() - 1))); + } + + @Test + void sharesTheByteAllowanceAcrossNestingLevels() { + // Unshared, a nested body is re-measured at every level: MAX_DEPTH times its size to copy + String inner = body("inner", field("deep", "v")); + String nested = outer(part("form-data; name=\"n\"", "multipart/mixed; boundary=inner", inner)); + + // enough for the outer body alone, one character short of also covering the nested one + ParseContext exhausted = new ParseContext(nested.length() + inner.length() - 1); + Map fields = asMap(parseBody(nested, MULTIPART, exhausted)); + assertEquals(inner, fields.get("n")); + + ParseContext sufficient = new ParseContext(nested.length() + inner.length()); + Map parsed = asMap(parseBody(nested, MULTIPART, sufficient)); + assertEquals("v", ((Map) parsed.get("n")).get("deep")); + } + + @Test + void reportsTheFilenamesOfFileParts() { + String body = + outer( + field("user", "admin"), + file("avatar", "cat.png", "bytes"), + file("doc", "report.pdf", "bytes")); + + assertEquals(asList("cat.png", "report.pdf"), filenamesOf(body)); + // The file parts are not fields, so only the field survives into the body map + assertEquals(singletonMap("user", "admin"), multipartBody(body)); + } + + @Test + void marksAFilePartWithoutReportingAnEmptyFilename() { + // Browsers send filename="" for an untouched file input: the part is still a file, but the + // empty name gives a rule nothing to match + String body = outer(file("avatar", "", "bytes"), field("user", "admin")); + + assertEquals(emptyList(), filenamesOf(body)); + assertEquals(singletonMap("user", "admin"), multipartBody(body)); + } + + @Test + void reportsFilenamesEvenWhenTheBodyDegradesToARawString() { + // Nothing but file parts, so there is no field to report: an empty map would tell the WAF the + // body was empty, so the raw string is kept and the filenames reported alongside it + String body = outer(file("avatar", "cat.png", "bytes")); + + assertEquals(body, parseBody(body, MULTIPART)); + assertEquals(singletonList("cat.png"), filenamesOf(body)); + } + + @Test + void reportsNoFilenameWhenTheMultipartBodyIsNotParsed() { + String body = outer(file("avatar", "cat.png", "bytes")); + + ParseContext sizeCapped = new ParseContext(body.length() - 1); + assertEquals(body, parseBody(body, MULTIPART, sizeCapped)); + assertEquals(emptyList(), sizeCapped.filenames()); + + ParseContext noBoundary = new ParseContext(); + assertEquals(body, parseBody(body, "multipart/form-data", noBoundary)); + assertEquals(emptyList(), noBoundary.filenames()); + } + + private static List filenamesOf(String body) { + ParseContext context = new ParseContext(); + parseBody(body, MULTIPART, context); + return context.filenames(); + } + + private static Object parseBody(String body, String contentType) { + return parseBody(body, contentType, new ParseContext()); + } + + private static Object parseBody(String body, String contentType, ParseContext context) { + return ContentTypeBodyParser.parseBody(body, contentType, context); + } + + /** Wraps the parts in a body with the default boundary and parses it. */ + private static Map multipart(String... parts) { + return multipartBody(outer(parts)); + } + + /** Distinct from {@link #multipart}, whose varargs would otherwise swallow a whole body. */ + private static Map multipartBody(String body) { + return asMap(parseBody(body, MULTIPART)); + } + + private static Map urlEncoded(String body) { + return asMap(parseBody(body, URL_ENCODED)); + } + + private static Map asMap(Object parsed) { + assertInstanceOf(Map.class, parsed); + return (Map) parsed; + } + + /** Joins the parts with CRLF and appends the close delimiter, using the default boundary. */ + private static String outer(String... parts) { + return body("outer", parts); + } + + private static String body(String boundary, String... parts) { + StringBuilder body = new StringBuilder(); + for (String part : parts) { + body.append("--").append(boundary).append("\r\n").append(part).append("\r\n"); + } + return body.append("--").append(boundary).append("--\r\n").toString(); + } + + private static String field(String name, String value) { + return part("form-data; name=\"" + name + "\"", value); + } + + private static String file(String name, String filename, String value) { + return part("form-data; name=\"" + name + "\"; filename=\"" + filename + "\"", value); + } + + private static String part(String disposition, String value) { + return part(disposition, null, value); + } + + private static String part(String disposition, String contentType, String value) { + StringBuilder part = new StringBuilder(); + if (disposition != null) { + part.append("Content-Disposition: ").append(disposition).append("\r\n"); + } + if (contentType != null) { + part.append("Content-Type: ").append(contentType).append("\r\n"); + } + return part.append("\r\n").append(value).toString(); + } +} diff --git a/dd-trace-core/src/test/java/datadog/trace/lambda/LambdaAppSecHandlerTest.java b/dd-trace-core/src/test/java/datadog/trace/lambda/LambdaAppSecHandlerTest.java index 33e68f528e2..a44598eae9b 100644 --- a/dd-trace-core/src/test/java/datadog/trace/lambda/LambdaAppSecHandlerTest.java +++ b/dd-trace-core/src/test/java/datadog/trace/lambda/LambdaAppSecHandlerTest.java @@ -3,6 +3,7 @@ import static datadog.trace.api.gateway.Events.EVENTS; import static datadog.trace.lambda.LambdaEventParser.detectTriggerType; import static datadog.trace.lambda.LambdaEventParser.parseResponse; +import static java.util.Collections.singletonMap; import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -47,6 +48,7 @@ import datadog.trace.bootstrap.instrumentation.api.Tags; import datadog.trace.bootstrap.instrumentation.api.URIDataAdapter; import datadog.trace.core.DDCoreJavaSpecification; +import datadog.trace.lambda.LambdaEventParser.LambdaResponseData; import datadog.trace.lambda.LambdaEventParser.LambdaTriggerType; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; @@ -69,6 +71,8 @@ import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; class LambdaAppSecHandlerTest extends DDCoreJavaSpecification { @@ -700,6 +704,86 @@ void handlesEmptyBodyCorrectly() { assertEquals("", capturedBody[0]); } + @Test + @SuppressWarnings("unchecked") + void reportsMultipartFilenamesToTheWaf() { + String eventJson = + "{" + + "\"body\": \"--xy\\r\\nContent-Disposition: form-data; name=\\\"user\\\"\\r\\n\\r\\nadmin" + + "\\r\\n--xy\\r\\nContent-Disposition: form-data; name=\\\"avatar\\\";" + + " filename=\\\"cat.png\\\"\\r\\n\\r\\nbytes" + + "\\r\\n--xy--\"," + + "\"headers\": {\"Content-Type\": \"multipart/form-data; boundary=xy\"}," + + "\"requestContext\": {\"httpMethod\": \"POST\"}" + + "}"; + ByteArrayInputStream event = createInputStream(eventJson); + + Object[] capturedBody = {null}; + Object[] capturedFilenames = {null}; + + setupMockCallbacks( + new Callbacks() + .onBody(body -> capturedBody[0] = body) + .onFilenames(filenames -> capturedFilenames[0] = filenames)); + + AgentSpanContext result = LambdaAppSecHandler.processRequestStart(event); + + assertNotNull(result); + assertEquals(Arrays.asList("cat.png"), capturedFilenames[0]); + // The file part is reported by name only: it is not a field, and its content is left out + Map fields = (Map) capturedBody[0]; + assertEquals("admin", fields.get("user")); + assertNull(fields.get("avatar")); + } + + @Test + void doesNotReportFilenamesForAMultipartBodyWithoutFileParts() { + String eventJson = + "{" + + "\"body\": \"--xy\\r\\nContent-Disposition: form-data; name=\\\"user\\\"\\r\\n\\r\\nadmin" + + "\\r\\n--xy--\"," + + "\"headers\": {\"Content-Type\": \"multipart/form-data; boundary=xy\"}," + + "\"requestContext\": {\"httpMethod\": \"POST\"}" + + "}"; + ByteArrayInputStream event = createInputStream(eventJson); + + Object[] capturedFilenames = {null}; + + setupMockCallbacks(new Callbacks().onFilenames(filenames -> capturedFilenames[0] = filenames)); + + AgentSpanContext result = LambdaAppSecHandler.processRequestStart(event); + + assertNotNull(result); + assertNull(capturedFilenames[0]); + } + + @Test + @SuppressWarnings("unchecked") + void appliesContentTypeDispatchToBase64DecodedBodies() { + String base64Body = + Base64.getEncoder().encodeToString("user=admin".getBytes(StandardCharsets.UTF_8)); + String eventJson = + "{" + + "\"body\": \"" + + base64Body + + "\"," + + "\"isBase64Encoded\": true," + + "\"headers\": {\"Content-Type\": \"application/x-www-form-urlencoded\"}," + + "\"requestContext\": {\"httpMethod\": \"POST\"}" + + "}"; + ByteArrayInputStream event = createInputStream(eventJson); + + Object[] capturedBody = {null}; + + setupMockCallbacks(new Callbacks().onBody(body -> capturedBody[0] = body)); + + AgentSpanContext result = LambdaAppSecHandler.processRequestStart(event); + + assertNotNull(result); + assertInstanceOf(Map.class, capturedBody[0]); + assertEquals(Arrays.asList("admin"), ((Map>) capturedBody[0]).get("user")); + } + @Test void handlesPathWithQueryStringCorrectly() { String eventJson = @@ -2033,6 +2117,30 @@ void extractResponseDataReturnsNullForEmptyString() { assertNull(parseResponse("")); } + @ParameterizedTest(name = "[{index}] content-type {0}") + @ValueSource(strings = {"", "application/json"}) + void parsesAResponseBodyAsJsonWhenTheContentTypeIsBlankOrJson(String contentType) { + // A blank content type says nothing about the body, so it gets the same best-effort JSON parse + // as an absent one, matching the request path. + LambdaResponseData response = + parseResponse( + "{\"statusCode\": 200, \"headers\": {\"content-type\": \"" + + contentType + + "\"}, \"body\": \"{\\\"a\\\":1}\"}"); + assertNotNull(response); + assertEquals(singletonMap("a", 1.0), response.body); + } + + @Test + void keepsAResponseBodyRawWhenTheContentTypeIsNotJson() { + LambdaResponseData response = + parseResponse( + "{\"statusCode\": 200, \"headers\": {\"content-type\": \"text/plain\"}," + + " \"body\": \"{\\\"a\\\":1}\"}"); + assertNotNull(response); + assertEquals("{\"a\":1}", response.body); + } + // ============================================================================ // HTTP span tags // ============================================================================ @@ -2382,6 +2490,7 @@ private static class Callbacks { BiConsumer onSocketAddress; Consumer> onPathParams; Consumer onBody; + Consumer> onFilenames; Callbacks onMethodUri(BiConsumer cb) { this.onMethodUri = cb; @@ -2407,6 +2516,11 @@ Callbacks onBody(Consumer cb) { this.onBody = cb; return this; } + + Callbacks onFilenames(Consumer> cb) { + this.onFilenames = cb; + return this; + } } @SuppressWarnings("unchecked") @@ -2483,6 +2597,19 @@ private void setupMockCallbacks(Callbacks callbacks) { .apply(any(), any()); } + BiFunction, Flow> filenamesCallback = null; + if (callbacks.onFilenames != null) { + filenamesCallback = mock(BiFunction.class); + Consumer> capture = callbacks.onFilenames; + doAnswer( + inv -> { + capture.accept(inv.getArgument(1)); + return Flow.ResultFlow.empty(); + }) + .when(filenamesCallback) + .apply(any(), any()); + } + CallbackProvider mockCallbackProvider = mock(CallbackProvider.class); when(mockCallbackProvider.getCallback(EVENTS.requestStarted())) .thenReturn(requestStartedCallback); @@ -2496,6 +2623,8 @@ private void setupMockCallbacks(Callbacks callbacks) { when(mockCallbackProvider.getCallback(EVENTS.requestPathParams())) .thenReturn(pathParamsCallback); when(mockCallbackProvider.getCallback(EVENTS.requestBodyProcessed())).thenReturn(bodyCallback); + when(mockCallbackProvider.getCallback(EVENTS.requestFilesFilenames())) + .thenReturn(filenamesCallback); AgentTracer.TracerAPI mockTracer = mock(AgentTracer.TracerAPI.class); when(mockTracer.getCallbackProvider(RequestContextSlot.APPSEC)) diff --git a/dd-trace-core/src/test/java/datadog/trace/lambda/MultipartSplitterTest.java b/dd-trace-core/src/test/java/datadog/trace/lambda/MultipartSplitterTest.java new file mode 100644 index 00000000000..8ab01aea92c --- /dev/null +++ b/dd-trace-core/src/test/java/datadog/trace/lambda/MultipartSplitterTest.java @@ -0,0 +1,234 @@ +package datadog.trace.lambda; + +import static datadog.trace.lambda.MultipartSplitter.extractBoundary; +import static datadog.trace.lambda.MultipartSplitter.parameter; +import static datadog.trace.lambda.MultipartSplitter.split; +import static java.util.concurrent.TimeUnit.SECONDS; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import datadog.trace.lambda.MultipartSplitter.Part; +import java.util.List; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; + +class MultipartSplitterTest { + + private static final int NO_PART_BUDGET_LIMIT = 256; + + @ParameterizedTest(name = "[{index}] {0} -> {1}") + @CsvSource( + delimiter = '|', + nullValues = "NULL", + value = { + // case is preserved: the boundary is matched byte-for-byte against the body + "multipart/form-data; boundary=AbC123 | AbC123", + // the parameter name is not + "multipart/form-data; BOUNDARY=xy | xy", + // quoted values may hold separators + "multipart/form-data; boundary=\"a;b c\" | a;b c", + // position among the other parameters does not matter + "multipart/form-data; boundary=xy; charset=x | xy", + // a parameter that merely ends in "boundary" is not one + "multipart/form-data; xboundary=xy | NULL", + "multipart/form-data | NULL", + "multipart/form-data; boundary= | NULL", + "NULL | NULL", + }) + void extractsTheBoundary(String contentType, String expected) { + assertEquals(expected, extractBoundary(contentType)); + } + + @Test + void rejectsABoundaryOverSeventyCharacters() { + String maximum = repeat('a', 70); + + assertEquals(maximum, extractBoundary("multipart/form-data; boundary=" + maximum)); + assertNull(extractBoundary("multipart/form-data; boundary=" + maximum + "a")); + } + + @ParameterizedTest(name = "[{index}] {1} of {0} -> {2}") + @CsvSource( + delimiter = '|', + nullValues = "NULL", + value = { + "form-data; name=user | name | user", + "form-data; name=\"user\" | name | user", + "form-data; name=\"a;b\" | name | a;b", + "form-data; NAME=user | name | user", + "form-data; name=user; charset=utf-8| name | user", + // "filename" must not answer a lookup for "name": the match has to start at a parameter + "form-data; filename=\"f\"; name=u | name | u", + // present but empty, which is what browsers send for an untouched file input + "form-data; filename=\"\" | filename | ''", + "form-data; filename= | filename | ''", + "form-data; name=user | filename | NULL", + "NULL | name | NULL", + // a quoted value cannot forge a parameter boundary: the quoted span is skipped whole, so + // this field is not mistaken for a file part and dropped + "form-data; name=\"; filename=x\" | filename | NULL", + "form-data; name=\"; filename=x\" | name | '; filename=x'", + // and the same aliasing the other way round does not rename the field + "form-data; filename=\"; name=y\" | name | NULL", + // RFC 7230 optional whitespace is tolerated on both sides of the "=" + "form-data; name =user | name | user", + // a bare parameter name is not a parameter + "form-data; name | name | NULL", + }) + void readsParameters(String headerValue, String paramName, String expected) { + assertEquals(expected, parameter(headerValue, paramName)); + } + + @Test + void unescapesQuotedParameterValues() { + assertEquals("a\"b", parameter("form-data; name=\"a\\\"b\"", "name")); + } + + @Test + void toleratesTabsAroundTheParameterEquals() { + assertEquals("user", parameter("form-data;\tname\t=\tuser", "name")); + } + + @Test + void keepsWhatWasReadOfAnUnterminatedQuotedValue() { + assertEquals("abc", parameter("form-data; name=\"abc", "name")); + assertEquals("a\"", parameter("form-data; name=\"a\\\"", "name")); + } + + @ParameterizedTest(name = "[{index}] {0} -> {2} part(s)") + @CsvSource( + delimiter = '|', + value = { + "happy path | --x@A: b@@v@--x-- | 1", + "two parts | --x@@a@--x@@b@--x-- | 2", + "preamble is discarded | junk@--x@@v@--x-- | 1", + "epilogue is discarded | --x@@v@--x--@junk | 1", + // a truncated body still yields its last part + "truncated last delimiter | --x@A: b@@v | 1", + "truncated in the headers | --x@A: b | 0", + // A part truncated inside its headers voids the parts before it too: its own headers are + // content the WAF would otherwise never see + "truncated after a part | --x@@v@--x@A: b | 0", + "no line break at all | --x | 0", + "close delimiter only | --x-- | 0", + "empty part content | --x@A: b@@ | 1", + "part without headers | --x@@v@--x-- | 1", + "unrelated delimiter | --y@@v@--y-- | 0", + // RFC 2046 transport padding between the delimiter and its line break + "padded delimiter | --x @@v@--x-- | 1", + }) + void splitsBodies(String name, String template, int expectedParts) { + assertEquals( + expectedParts, split(template.replace("@", "\r\n"), "x", NO_PART_BUDGET_LIMIT).size()); + } + + @Test + void toleratesBareLineFeeds() { + String body = "--x\nContent-Disposition: form-data; name=a\n\nvalue\n--x--\n"; + + List parts = split(body, "x", NO_PART_BUDGET_LIMIT); + + assertEquals(1, parts.size()); + assertEquals("value", content(body, parts.get(0))); + } + + @Test + void stopsAtThePartBudget() { + String body = ("--x\r\n\r\na\r\n--x\r\n\r\nb\r\n--x\r\n\r\nc\r\n--x--"); + + assertEquals(2, split(body, "x", 2).size()); + assertEquals(0, split(body, "x", 0).size()); + } + + @Test + void delimitsContentExactly() { + // Dashes, line breaks, a replacement character and a multi-byte character all inside the + // content, plus lines that start with the part and close delimiters without being either: the + // reported range must not be thrown off by a near miss on the line-feed anchor, on the + // delimiter itself, or on its trailing dashes + String content = "--not-a-boundary\r\n--x-not-a-boundary\r\n--x--not-a-close\r\n-x\nlast�é"; + String body = + "--x\r\nContent-Disposition: form-data; name=a\r\n\r\n" + content + "\r\n--x--\r\n"; + + List parts = split(body, "x", NO_PART_BUDGET_LIMIT); + + assertEquals(1, parts.size()); + assertEquals(content, content(body, parts.get(0))); + assertEquals("form-data; name=a", parts.get(0).contentDisposition); + } + + @Test + void reportsNothingForAPartWhoseHeadersRunIntoTheNextDelimiter() { + // The first part's headers are not followed by a blank line, which no conforming parser + // accepts. Reporting the second part alone would show the WAF less than the app receives. + String body = + "--x\r\nX-First: 1\r\n" + + "--x\r\nContent-Disposition: form-data; name=\"b\"\r\n\r\nsecond\r\n--x--"; + + assertTrue(split(body, "x", NO_PART_BUDGET_LIMIT).isEmpty()); + } + + @Test + void matchesHeaderNamesCaseInsensitivelyAndTrimsValues() { + String body = + "--x\r\nCONTENT-Disposition: form-data; name=a \r\n" + + "content-type \t:\ttext/plain \r\n\r\nv\r\n" + + "--x\r\nContent-Type:\r\n\r\nv\r\n--x--"; + + List parts = split(body, "x", NO_PART_BUDGET_LIMIT); + + assertEquals("form-data; name=a", parts.get(0).contentDisposition); + assertEquals("text/plain", parts.get(0).contentType); + // A header present but empty is reported as "", distinct from the null of an absent one + assertEquals("", parts.get(1).contentType); + assertNull(parts.get(1).contentDisposition); + } + + @Test + @Timeout(value = 10, unit = SECONDS) + void returnsPromptlyOnAnAdversarialBoundary() { + // Dashes are legal boundary characters, so a scan seeded on '-' would be quadratic here. + // Anchored on the mandatory line feed, of which this body has none, the scan is linear. + String boundary = repeat('-', 69) + "X"; + String body = repeat('-', 1_000_000); + + assertTrue(split(body, boundary, NO_PART_BUDGET_LIMIT).isEmpty()); + } + + @Test + @Timeout(value = 30, unit = SECONDS) + void neverThrowsOnAMutatedBody() { + String body = + "preamble\r\n--x\r\nContent-Disposition: form-data; name=\"a\"\r\n" + + "Content-Type: application/json\r\n\r\n{\"k\":1}\r\n" + + "--x\r\nContent-Disposition: form-data; name=b; filename=\"f\"\r\n\r\nfile\r\n" + + "--x--\r\nepilogue"; + // Fixed positions rather than a random seed, so a failure is reproducible + char[] substitutes = {'-', '\r', '\n', ':', ';', '"', '\\', '=', '\0', '�'}; + + for (int i = 0; i <= body.length(); i++) { + split(body.substring(0, i), "x", NO_PART_BUDGET_LIMIT); + for (char substitute : substitutes) { + if (i < body.length()) { + split( + body.substring(0, i) + substitute + body.substring(i + 1), "x", NO_PART_BUDGET_LIMIT); + } + } + } + } + + private static String content(String body, Part part) { + return body.substring(part.contentStart, part.contentEnd); + } + + private static String repeat(char c, int count) { + StringBuilder builder = new StringBuilder(count); + for (int i = 0; i < count; i++) { + builder.append(c); + } + return builder.toString(); + } +}