From 85fdd8ab9693e418c3e9f336ebfa179a24463812 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Mon, 24 Aug 2026 20:05:01 +0000 Subject: [PATCH 01/56] feat: add filesystem-backed SerDes --- README.md | 12 + docs/adr/005-filesystem-serdes.md | 19 +- docs/advanced/configuration.md | 33 +++ extra-filesystem-serdes/README.md | 49 ++++ extra-filesystem-serdes/pom.xml | 66 +++++ .../filesystem/FileSystemPathEncoding.java | 9 + .../extra/filesystem/FileSystemSerDes.java | 240 ++++++++++++++++++ .../filesystem/FileSystemStorageMode.java | 9 + .../filesystem/FileSystemSerDesTest.java | 133 ++++++++++ pom.xml | 1 + sdk-integration-tests/pom.xml | 6 + .../FileSystemSerDesIntegrationTest.java | 79 ++++++ .../testing/LocalDurableTestRunner.java | 37 ++- .../lambda/durable/testing/TestOperation.java | 40 ++- .../lambda/durable/testing/TestResult.java | 35 ++- .../local/LocalMemoryExecutionClient.java | 19 +- .../amazon/lambda/durable/DurableConfig.java | 29 +++ .../durable/execution/DurableExecutor.java | 41 ++- .../durable/execution/ExecutionManager.java | 8 + .../operation/BaseDurableOperation.java | 26 ++ .../durable/operation/InvokeOperation.java | 7 +- .../SerializableDurableOperation.java | 50 +++- .../durable/operation/StepOperation.java | 5 +- .../operation/WaitForConditionOperation.java | 18 +- .../lambda/durable/serde/SerDesContext.java | 69 +++++ .../durable/serde/SerDesContextHolder.java | 21 ++ .../durable/serde/SerDesPayloadKind.java | 24 ++ .../lambda/durable/serde/SerDesRunner.java | 103 ++++++++ .../lambda/durable/DurableConfigTest.java | 25 ++ .../operation/CallbackOperationTest.java | 3 +- .../SerializableDurableOperationTest.java | 3 +- .../durable/serde/SerDesRunnerTest.java | 126 +++++++++ 32 files changed, 1288 insertions(+), 57 deletions(-) create mode 100644 extra-filesystem-serdes/README.md create mode 100644 extra-filesystem-serdes/pom.xml create mode 100644 extra-filesystem-serdes/src/main/java/software/amazon/lambda/durable/extra/filesystem/FileSystemPathEncoding.java create mode 100644 extra-filesystem-serdes/src/main/java/software/amazon/lambda/durable/extra/filesystem/FileSystemSerDes.java create mode 100644 extra-filesystem-serdes/src/main/java/software/amazon/lambda/durable/extra/filesystem/FileSystemStorageMode.java create mode 100644 extra-filesystem-serdes/src/test/java/software/amazon/lambda/durable/extra/filesystem/FileSystemSerDesTest.java create mode 100644 sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FileSystemSerDesIntegrationTest.java create mode 100644 sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesContext.java create mode 100644 sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesContextHolder.java create mode 100644 sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesPayloadKind.java create mode 100644 sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesRunner.java create mode 100644 sdk/src/test/java/software/amazon/lambda/durable/serde/SerDesRunnerTest.java diff --git a/README.md b/README.md index 766a71b02..9c69778c4 100644 --- a/README.md +++ b/README.md @@ -50,6 +50,18 @@ Your durable function extends `DurableHandler` and implements `handleReque ``` +For filesystem-backed payload storage, add the optional module: + +```xml + + software.amazon.lambda.durable + aws-durable-execution-sdk-java-extra-filesystem-serdes + VERSION + +``` + +See [Filesystem SerDes](extra-filesystem-serdes/README.md) for configuration and durability requirements. + ### Your First Durable Function ```java diff --git a/docs/adr/005-filesystem-serdes.md b/docs/adr/005-filesystem-serdes.md index ef0a0a17e..6a92ebab2 100644 --- a/docs/adr/005-filesystem-serdes.md +++ b/docs/adr/005-filesystem-serdes.md @@ -1,6 +1,6 @@ # ADR-005: Payload Offloading for Filesystem Storage -**Status:** Proposed +**Status:** Accepted — Approach A **Date:** 2026-07-02 ## Context @@ -156,6 +156,7 @@ Add an invocation-scoped cache for successful deserialization results. The cache - Durable execution ARN. - `entityId`. - Payload kind. +- Attempt, when applicable. - Target `TypeToken` type. - A hash of the serialized checkpoint string. @@ -407,19 +408,11 @@ This approach gives the SDK one consistent policy for root payloads, operation r | Immediate delivery risk | Lower. Builds on existing customization point. | Higher. Requires new API and more runtime integration. | | Long-term design risk | Higher. Blurs SerDes semantics and may accumulate storage behavior in serializers. | Lower if offloading grows into a first-class feature, but higher if this remains a one-off filesystem parity feature. | -## AI Recommendation +## Decision -**AI recommendation:** Prefer **Approach B: Create a `PayloadOffloader` interface** if the team is willing to treat payload offloading as a first-class Java SDK capability rather than only a JavaScript parity item. - -Reasoning: - -- The problem being solved is payload storage, not serialization. A dedicated offloader keeps the domain boundary clean. -- The SDK already needs to touch every payload path for context, caching, threading, exceptions, and root input/output. Once that plumbing exists, composing SerDes plus offloader is a more durable shape than putting storage behavior inside SerDes. -- Java customers are more likely to have custom Jackson/ObjectMapper SerDes implementations. Approach B lets them keep those and add offloading independently. -- Both approaches use one optional extra package for filesystem-specific code; that is not a differentiator. The package would be either filesystem SerDes or filesystem offloader depending on the chosen approach. The differentiator is that Approach B gives future storage extras such as S3 or DynamoDB offload the same focused core offloader contract instead of encoding storage behavior as more SerDes implementations. -- SDK-owned envelopes and two-layer caching make replay behavior easier to test and reason about. - -The main reason to choose Approach A is schedule and parity: it is smaller and maps directly to the JavaScript feature request. If the team needs to satisfy #463 quickly with minimal public API design, Approach A is a reasonable incremental step, but it should be documented as payload offloading implemented through SerDes rather than as the long-term ideal boundary. +Adopt **Approach A: Reuse SerDes for Offload**. It delivers JavaScript parity with the smallest compatible public API +change, keeps filesystem behavior in an optional artifact, and leaves the existing `SerDes` interface unchanged. +Approach B remains a possible future direction if payload offloading grows into a general multi-backend capability. ## Other Alternatives Considered diff --git a/docs/advanced/configuration.md b/docs/advanced/configuration.md index bc8bf89d0..5f9f826d7 100644 --- a/docs/advanced/configuration.md +++ b/docs/advanced/configuration.md @@ -18,6 +18,7 @@ public class OrderProcessor extends DurableHandler { .withLambdaClientBuilder(lambdaClientBuilder) .withSerDes(new MyCustomSerDes()) // Custom serialization .withExecutorService(Executors.newFixedThreadPool(10)) // Custom thread pool + .withSerDesExecutorService(Executors.newFixedThreadPool(4)) // SerDes and payload I/O .withLoggerConfig(LoggerConfig.withReplayLogging()) // Enable replay logs .build(); } @@ -34,12 +35,44 @@ public class OrderProcessor extends DurableHandler { | `withLambdaClientBuilder()` | Custom AWS Lambda client | Auto-configured Lambda client | | `withSerDes()` | Serializer for step results | Jackson with default settings | | `withExecutorService()` | Thread pool for user-defined operations | Cached daemon thread pool | +| `withSerDesExecutorService()` | Thread pool for SerDes and payload storage I/O | Cached `durable-sdk-serdes-*` daemon thread pool | | `withLoggerConfig()` | Logger behavior configuration | Suppress logs during replay | | `withPollingStrategy()` | Backend polling strategy | Exponential backoff: 1s base, 2x rate, FULL jitter, 10s max | | `withCheckpointDelay()` | How often the SDK checkpoints updates | `Duration.ofSeconds(0)` (as soon as possible) | The `withExecutorService()` option configures the thread pool used for running user-defined operations. Internal SDK coordination (checkpoint batching, polling) runs on an SDK-managed thread pool. +The `withSerDesExecutorService()` option isolates serialization and blocking payload storage from user operations and +SDK coordination. The SDK sets `SerDesContext` inside each task, clears it after the call, and caches successful +deserialization results for the current invocation. + +### Filesystem-backed payload storage + +The optional `aws-durable-execution-sdk-java-extra-filesystem-serdes` artifact can store delegate-serialized payloads +on a shared filesystem: + +```java +var fileSystemSerDes = FileSystemSerDes.builder(Path.of("/mnt/efs/durable-payloads")) + .storageMode(FileSystemStorageMode.OVERFLOW) + .pathEncoding(FileSystemPathEncoding.HASH) + .delegate(new JacksonSerDes()) + .previewGenerator(value -> Map.of("type", value.getClass().getSimpleName())) + .build(); + +return DurableConfig.builder() + .withSerDes(fileSystemSerDes) + .build(); +``` + +`ALWAYS` stores every non-null payload in a file. `OVERFLOW` keeps payloads inline until the checkpoint envelope +approaches the 256 KB service limit. `URI` produces readable escaped paths; `HASH` produces fixed-length SHA-256 path +segments. + +Do not use Lambda's ephemeral `/tmp` directory: replay can run in a different execution environment. Use a durable, +shared mount such as EFS. S3 Files can have delayed synchronization, so a runtime crash before the mount flushes may +lose recent writes; use it only when that tradeoff is acceptable. The SDK does not delete offloaded files, so configure +storage lifecycle and retention separately. + ### Dynamic plugin loading Dynamic plugin loading is an opt-in alternative to registering plugins in application code. Put provider JARs on the application class path, then set `DURABLE_EXECUTION_PLUGINS` to an ordered, comma-separated list of provider names: diff --git a/extra-filesystem-serdes/README.md b/extra-filesystem-serdes/README.md new file mode 100644 index 000000000..fbdf9c4ec --- /dev/null +++ b/extra-filesystem-serdes/README.md @@ -0,0 +1,49 @@ +# Filesystem SerDes + +`aws-durable-execution-sdk-java-extra-filesystem-serdes` stores durable user payloads on a shared filesystem while +keeping small file-pointer envelopes in checkpoints. + +## Installation + +```xml + + software.amazon.lambda.durable + aws-durable-execution-sdk-java-extra-filesystem-serdes + VERSION + +``` + +## Configuration + +```java +var serDes = FileSystemSerDes.builder(Path.of("/mnt/efs/durable-payloads")) + .storageMode(FileSystemStorageMode.ALWAYS) + .pathEncoding(FileSystemPathEncoding.URI) + .delegate(new JacksonSerDes()) + .build(); + +return DurableConfig.builder() + .withSerDes(serDes) + .build(); +``` + +- `ALWAYS` writes every non-null payload to a file. +- `OVERFLOW` stores small payloads inline and offloads envelopes approaching the 256 KB checkpoint limit. +- `URI` uses readable escaped path segments. +- `HASH` uses fixed-length SHA-256 path segments. + +An optional preview can remain visible in the checkpoint envelope: + +```java +.previewGenerator(value -> Map.of("summary", "order payload")) +``` + +## Storage requirements + +Do not use Lambda's ephemeral `/tmp` directory. Durable replay may run in another execution environment where that file +does not exist. + +Use a durable shared mount such as EFS. S3 Files can synchronize writes asynchronously, so a runtime crash before a +flush can lose recent data; use it only when that durability tradeoff is acceptable. + +The SDK does not delete payload files. Configure an appropriate retention or lifecycle policy for the backing storage. diff --git a/extra-filesystem-serdes/pom.xml b/extra-filesystem-serdes/pom.xml new file mode 100644 index 000000000..c7f55d269 --- /dev/null +++ b/extra-filesystem-serdes/pom.xml @@ -0,0 +1,66 @@ + + + 4.0.0 + + software.amazon.lambda.durable + aws-durable-execution-sdk-java-parent + 2.1.1-SNAPSHOT + + aws-durable-execution-sdk-java-extra-filesystem-serdes + AWS Lambda Durable Execution SDK Filesystem SerDes + Optional filesystem-backed payload SerDes for the AWS Lambda Durable Execution SDK + + + software.amazon.lambda.durable + aws-durable-execution-sdk-java + ${project.version} + + + com.fasterxml.jackson.core + jackson-databind + + + org.junit.jupiter + junit-jupiter + test + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + + org.apache.maven.plugins + maven-surefire-plugin + + + org.apache.maven.plugins + maven-source-plugin + + + attach-sources + + jar-no-fork + + + + + + org.apache.maven.plugins + maven-javadoc-plugin + + + attach-javadocs + + jar + + + + + + + diff --git a/extra-filesystem-serdes/src/main/java/software/amazon/lambda/durable/extra/filesystem/FileSystemPathEncoding.java b/extra-filesystem-serdes/src/main/java/software/amazon/lambda/durable/extra/filesystem/FileSystemPathEncoding.java new file mode 100644 index 000000000..4c3d0907c --- /dev/null +++ b/extra-filesystem-serdes/src/main/java/software/amazon/lambda/durable/extra/filesystem/FileSystemPathEncoding.java @@ -0,0 +1,9 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.extra.filesystem; + +/** Controls how durable execution and entity identifiers are encoded as filesystem paths. */ +public enum FileSystemPathEncoding { + URI, + HASH +} diff --git a/extra-filesystem-serdes/src/main/java/software/amazon/lambda/durable/extra/filesystem/FileSystemSerDes.java b/extra-filesystem-serdes/src/main/java/software/amazon/lambda/durable/extra/filesystem/FileSystemSerDes.java new file mode 100644 index 000000000..8d9365e4a --- /dev/null +++ b/extra-filesystem-serdes/src/main/java/software/amazon/lambda/durable/extra/filesystem/FileSystemSerDes.java @@ -0,0 +1,240 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.extra.filesystem; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.AtomicMoveNotSupportedException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.HexFormat; +import java.util.Map; +import java.util.Objects; +import java.util.function.Function; +import java.util.regex.Pattern; +import software.amazon.lambda.durable.TypeToken; +import software.amazon.lambda.durable.exception.SerDesException; +import software.amazon.lambda.durable.serde.JacksonSerDes; +import software.amazon.lambda.durable.serde.SerDes; +import software.amazon.lambda.durable.serde.SerDesContext; + +/** + * A SerDes that stores delegate-serialized payloads on a durable shared filesystem. + * + *

Do not use Lambda's ephemeral {@code /tmp} storage. Use a durable shared mount such as EFS, or S3 Files only when + * its synchronization and crash-durability tradeoffs are acceptable for the workload. + */ +public final class FileSystemSerDes implements SerDes { + private static final int OVERFLOW_THRESHOLD_BYTES = 256 * 1024 - 1024; + private static final ObjectMapper ENVELOPE_MAPPER = new ObjectMapper(); + private static final Pattern DURABLE_EXECUTION_ARN_PATTERN = Pattern.compile( + "^arn:[^:]*:lambda:[^:]*:[^:]*:function:([^:/]+):[^:/]+/durable-execution/([^/]+)/([^/]+)$"); + + private final Path basePath; + private final FileSystemStorageMode storageMode; + private final FileSystemPathEncoding pathEncoding; + private final SerDes delegate; + private final Function> previewGenerator; + + private FileSystemSerDes(Builder builder) { + basePath = builder.basePath.toAbsolutePath().normalize(); + storageMode = builder.storageMode; + pathEncoding = builder.pathEncoding; + delegate = builder.delegate; + previewGenerator = builder.previewGenerator; + } + + public static Builder builder(Path basePath) { + return new Builder(basePath); + } + + @Override + public String serialize(Object value) { + if (value == null) { + return null; + } + var context = requireContext(); + var serialized = delegate.serialize(value); + if (serialized == null) { + throw new SerDesException("Delegate SerDes returned null for a non-null value"); + } + try { + var inlineEnvelope = ENVELOPE_MAPPER.writeValueAsString(Map.of("data", serialized)); + if (storageMode == FileSystemStorageMode.OVERFLOW + && inlineEnvelope.getBytes(StandardCharsets.UTF_8).length <= OVERFLOW_THRESHOLD_BYTES) { + return inlineEnvelope; + } + var file = writePayload(serialized, context); + var preview = previewGenerator != null ? previewGenerator.apply(value) : null; + return preview == null + ? ENVELOPE_MAPPER.writeValueAsString(Map.of("file", file.toString())) + : ENVELOPE_MAPPER.writeValueAsString(Map.of("file", file.toString(), "preview", preview)); + } catch (IOException e) { + throw new SerDesException("Failed to store filesystem payload for entity '" + context.entityId() + "'", e); + } + } + + @Override + public T deserialize(String data, TypeToken typeToken) { + if (data == null) { + return null; + } + var context = requireContext(); + try { + JsonNode envelope = ENVELOPE_MAPPER.readTree(data); + var hasData = envelope.has("data") && envelope.get("data").isTextual(); + var hasFile = envelope.has("file") && envelope.get("file").isTextual(); + if (hasData == hasFile) { + throw new SerDesException("Filesystem SerDes envelope must contain exactly one of 'data' or 'file'"); + } + String serialized; + if (hasData) { + serialized = envelope.get("data").textValue(); + } else { + var file = Path.of(envelope.get("file").textValue()) + .toAbsolutePath() + .normalize(); + if (!file.startsWith(basePath)) { + throw new SerDesException("Filesystem SerDes file is outside the configured base path"); + } + serialized = Files.readString(file, StandardCharsets.UTF_8); + } + return delegate.deserialize(serialized, typeToken); + } catch (SerDesException e) { + throw e; + } catch (Exception e) { + throw new SerDesException("Failed to load filesystem payload for entity '" + context.entityId() + "'", e); + } + } + + private SerDesContext requireContext() { + var context = SerDesContext.getCurrentContext(); + if (context == null + || context.durableExecutionArn() == null + || context.durableExecutionArn().isBlank() + || context.entityId() == null + || context.entityId().isBlank()) { + throw new SerDesException( + "FileSystemSerDes requires an SDK-managed SerDesContext with durableExecutionArn and entityId"); + } + return context; + } + + private Path writePayload(String serialized, SerDesContext context) throws IOException { + var directory = resolveExecutionDirectory(context.durableExecutionArn()); + Files.createDirectories(directory); + var file = directory.resolve(encode(context.entityId()) + ".json").normalize(); + if (!file.startsWith(directory)) { + throw new SerDesException("Resolved filesystem payload path is outside the execution directory"); + } + var temporary = Files.createTempFile(directory, file.getFileName().toString(), ".tmp"); + try { + Files.writeString(temporary, serialized, StandardCharsets.UTF_8); + try { + Files.move(temporary, file, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + } catch (AtomicMoveNotSupportedException e) { + Files.move(temporary, file, StandardCopyOption.REPLACE_EXISTING); + } + } finally { + Files.deleteIfExists(temporary); + } + return file; + } + + private Path resolveExecutionDirectory(String durableExecutionArn) { + Path directory; + if (pathEncoding == FileSystemPathEncoding.URI) { + var matcher = DURABLE_EXECUTION_ARN_PATTERN.matcher(durableExecutionArn); + if (matcher.matches()) { + directory = basePath.resolve(matcher.group(1)) + .resolve(matcher.group(2)) + .resolve(matcher.group(3)) + .normalize(); + if (!directory.startsWith(basePath)) { + throw new SerDesException("Resolved filesystem execution path is outside the configured base path"); + } + return directory; + } + } + directory = basePath.resolve(encode(durableExecutionArn)).normalize(); + if (!directory.startsWith(basePath)) { + throw new SerDesException("Resolved filesystem execution path is outside the configured base path"); + } + return directory; + } + + private String encode(String value) { + if (pathEncoding == FileSystemPathEncoding.HASH) { + return sha256(value); + } + var encoded = new StringBuilder(); + for (byte valueByte : value.getBytes(StandardCharsets.UTF_8)) { + int current = valueByte & 0xff; + if (current >= 'a' && current <= 'z' + || current >= 'A' && current <= 'Z' + || current >= '0' && current <= '9' + || current == '-' + || current == '_' + || current == '.' + || current == '~') { + encoded.append((char) current); + } else { + encoded.append('%'); + encoded.append(Character.toUpperCase(Character.forDigit(current >>> 4, 16))); + encoded.append(Character.toUpperCase(Character.forDigit(current & 0xf, 16))); + } + } + return encoded.toString(); + } + + private static String sha256(String value) { + try { + var digest = MessageDigest.getInstance("SHA-256").digest(value.getBytes(StandardCharsets.UTF_8)); + return HexFormat.of().formatHex(digest); + } catch (NoSuchAlgorithmException e) { + throw new IllegalStateException("SHA-256 is unavailable", e); + } + } + + /** Builder for {@link FileSystemSerDes}. */ + public static final class Builder { + private final Path basePath; + private FileSystemStorageMode storageMode = FileSystemStorageMode.ALWAYS; + private FileSystemPathEncoding pathEncoding = FileSystemPathEncoding.URI; + private SerDes delegate = new JacksonSerDes(); + private Function> previewGenerator; + + private Builder(Path basePath) { + this.basePath = Objects.requireNonNull(basePath, "basePath cannot be null"); + } + + public Builder storageMode(FileSystemStorageMode storageMode) { + this.storageMode = Objects.requireNonNull(storageMode); + return this; + } + + public Builder pathEncoding(FileSystemPathEncoding pathEncoding) { + this.pathEncoding = Objects.requireNonNull(pathEncoding); + return this; + } + + public Builder delegate(SerDes delegate) { + this.delegate = Objects.requireNonNull(delegate); + return this; + } + + public Builder previewGenerator(Function> previewGenerator) { + this.previewGenerator = Objects.requireNonNull(previewGenerator); + return this; + } + + public FileSystemSerDes build() { + return new FileSystemSerDes(this); + } + } +} diff --git a/extra-filesystem-serdes/src/main/java/software/amazon/lambda/durable/extra/filesystem/FileSystemStorageMode.java b/extra-filesystem-serdes/src/main/java/software/amazon/lambda/durable/extra/filesystem/FileSystemStorageMode.java new file mode 100644 index 000000000..254f5805c --- /dev/null +++ b/extra-filesystem-serdes/src/main/java/software/amazon/lambda/durable/extra/filesystem/FileSystemStorageMode.java @@ -0,0 +1,9 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.extra.filesystem; + +/** Controls when serialized payloads are written to the filesystem. */ +public enum FileSystemStorageMode { + ALWAYS, + OVERFLOW +} diff --git a/extra-filesystem-serdes/src/test/java/software/amazon/lambda/durable/extra/filesystem/FileSystemSerDesTest.java b/extra-filesystem-serdes/src/test/java/software/amazon/lambda/durable/extra/filesystem/FileSystemSerDesTest.java new file mode 100644 index 000000000..28c1b2762 --- /dev/null +++ b/extra-filesystem-serdes/src/test/java/software/amazon/lambda/durable/extra/filesystem/FileSystemSerDesTest.java @@ -0,0 +1,133 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.extra.filesystem; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.fasterxml.jackson.databind.ObjectMapper; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Map; +import java.util.concurrent.Executors; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import software.amazon.awssdk.services.lambda.model.OperationType; +import software.amazon.lambda.durable.TypeToken; +import software.amazon.lambda.durable.exception.SerDesException; +import software.amazon.lambda.durable.model.OperationSubType; +import software.amazon.lambda.durable.serde.SerDesContext; +import software.amazon.lambda.durable.serde.SerDesPayloadKind; +import software.amazon.lambda.durable.serde.SerDesRunner; + +class FileSystemSerDesTest { + private static final String ARN = + "arn:aws:lambda:us-east-1:123456789012:function:orders:1/durable-execution/execution-1/invocation-1"; + private static final ObjectMapper MAPPER = new ObjectMapper(); + + @TempDir + Path basePath; + + private final java.util.concurrent.ExecutorService executor = Executors.newSingleThreadExecutor(); + + @AfterEach + void tearDown() { + executor.shutdownNow(); + } + + @Test + void alwaysModeWritesDelegatePayloadAndReplaysIt() throws Exception { + var serDes = FileSystemSerDes.builder(basePath).build(); + var runner = new SerDesRunner(executor); + + var envelope = runner.serialize(serDes, Map.of("id", 42), context()); + var file = Path.of(MAPPER.readTree(envelope).get("file").textValue()); + + assertTrue(file.startsWith(basePath.resolve("orders/execution-1/invocation-1"))); + assertEquals("{\"id\":42}", Files.readString(file)); + assertEquals( + Map.of("id", 42), + runner.deserialize(serDes, envelope, new TypeToken>() {}, context())); + } + + @Test + void overflowModeKeepsSmallPayloadInlineAndWritesLargePayload() throws Exception { + var serDes = FileSystemSerDes.builder(basePath) + .storageMode(FileSystemStorageMode.OVERFLOW) + .build(); + var runner = new SerDesRunner(executor); + + var inline = runner.serialize(serDes, "small", context()); + assertTrue(MAPPER.readTree(inline).has("data")); + + var overflow = runner.serialize(serDes, "x".repeat(256 * 1024), context()); + assertTrue(MAPPER.readTree(overflow).has("file")); + } + + @Test + void hashEncodingUsesFixedLengthSegments() throws Exception { + var serDes = FileSystemSerDes.builder(basePath) + .pathEncoding(FileSystemPathEncoding.HASH) + .build(); + + var envelope = new SerDesRunner(executor).serialize(serDes, "value", context()); + var file = Path.of(MAPPER.readTree(envelope).get("file").textValue()); + + assertEquals(64, file.getParent().getFileName().toString().length()); + assertEquals(69, file.getFileName().toString().length()); + assertFalse(file.toString().contains("operation")); + } + + @Test + void includesPreviewWithoutChangingStoredPayload() throws Exception { + var serDes = FileSystemSerDes.builder(basePath) + .previewGenerator(value -> Map.of("summary", "order")) + .build(); + + var envelope = new SerDesRunner(executor).serialize(serDes, Map.of("secret", "value"), context()); + var json = MAPPER.readTree(envelope); + + assertEquals("order", json.get("preview").get("summary").textValue()); + assertEquals( + "{\"secret\":\"value\"}", + Files.readString(Path.of(json.get("file").textValue()))); + } + + @Test + void rejectsCallsWithoutSdkContextAndMalformedEnvelopes() { + var serDes = FileSystemSerDes.builder(basePath).build(); + assertThrows(SerDesException.class, () -> serDes.serialize("value")); + + var runner = new SerDesRunner(executor); + assertThrows( + SerDesException.class, () -> runner.deserialize(serDes, "{}", TypeToken.get(String.class), context())); + assertThrows( + SerDesException.class, + () -> runner.deserialize( + serDes, "{\"file\":\"/outside/payload.json\"}", TypeToken.get(String.class), context())); + } + + @Test + void rejectsExecutionPathsOutsideConfiguredBasePath() { + var serDes = FileSystemSerDes.builder(basePath).build(); + var unsafeContext = SerDesContext.forOperation( + "arn:aws:lambda:us-east-1:123456789012:function:..:1/durable-execution/../..", + "1", + "step", + null, + OperationType.STEP, + OperationSubType.STEP, + SerDesPayloadKind.RESULT, + 1); + + assertThrows(SerDesException.class, () -> new SerDesRunner(executor).serialize(serDes, "value", unsafeContext)); + } + + private static SerDesContext context() { + return SerDesContext.forOperation( + ARN, "1", "step", null, OperationType.STEP, OperationSubType.STEP, SerDesPayloadKind.RESULT, 1); + } +} diff --git a/pom.xml b/pom.xml index b224efbbd..7e6fd9c18 100644 --- a/pom.xml +++ b/pom.xml @@ -40,6 +40,7 @@ sdk + extra-filesystem-serdes sdk-testing sdk-integration-tests otel-plugin diff --git a/sdk-integration-tests/pom.xml b/sdk-integration-tests/pom.xml index 9d0459a07..a1bc76bc6 100644 --- a/sdk-integration-tests/pom.xml +++ b/sdk-integration-tests/pom.xml @@ -36,6 +36,12 @@ ${project.version} test + + software.amazon.lambda.durable + aws-durable-execution-sdk-java-extra-filesystem-serdes + ${project.version} + test + org.junit.jupiter junit-jupiter diff --git a/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FileSystemSerDesIntegrationTest.java b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FileSystemSerDesIntegrationTest.java new file mode 100644 index 000000000..20a756f79 --- /dev/null +++ b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FileSystemSerDesIntegrationTest.java @@ -0,0 +1,79 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.fasterxml.jackson.databind.ObjectMapper; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.util.concurrent.atomic.AtomicInteger; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import software.amazon.lambda.durable.config.WaitForConditionConfig; +import software.amazon.lambda.durable.extra.filesystem.FileSystemSerDes; +import software.amazon.lambda.durable.model.ExecutionStatus; +import software.amazon.lambda.durable.model.WaitForConditionResult; +import software.amazon.lambda.durable.retry.JitterStrategy; +import software.amazon.lambda.durable.retry.WaitStrategies; +import software.amazon.lambda.durable.testing.LocalDurableTestRunner; + +class FileSystemSerDesIntegrationTest { + private static final ObjectMapper MAPPER = new ObjectMapper(); + + @TempDir + Path basePath; + + @Test + void replaysStepAndWaitForConditionStateFromFilesystem() throws Exception { + var stepExecutions = new AtomicInteger(); + var pollExecutions = new AtomicInteger(); + var config = DurableConfig.builder() + .withSerDes(FileSystemSerDes.builder(basePath).build()) + .build(); + + var runner = LocalDurableTestRunner.create( + String.class, + (input, context) -> { + var stepResult = context.step("load-order", String.class, stepContext -> { + stepExecutions.incrementAndGet(); + return input + "-loaded"; + }); + var waitConfig = WaitForConditionConfig.builder() + .waitStrategy(WaitStrategies.exponentialBackoff( + 5, Duration.ofSeconds(1), Duration.ofSeconds(10), 1, JitterStrategy.NONE)) + .build(); + var pollResult = context.waitForCondition( + "poll-order", + Integer.class, + (state, stepContext) -> { + pollExecutions.incrementAndGet(); + var next = state == null ? 1 : state + 1; + return next == 2 + ? WaitForConditionResult.stopPolling(next) + : WaitForConditionResult.continuePolling(next); + }, + waitConfig); + var childResult = context.runInChildContext( + "format-order", String.class, child -> stepResult + "-child"); + return childResult + "-" + pollResult; + }, + config) + .withOutputType(String.class); + + var result = runner.runUntilComplete("order"); + + assertEquals(ExecutionStatus.SUCCEEDED, result.getStatus()); + assertEquals("order-loaded-child-2", result.getResult()); + assertEquals(1, stepExecutions.get()); + assertEquals(2, pollExecutions.get()); + assertEquals("order-loaded", result.getOperation("load-order").getStepResult(String.class)); + + var stepEnvelope = result.getOperation("load-order").getStepDetails().result(); + var stepFile = Path.of(MAPPER.readTree(stepEnvelope).get("file").textValue()); + assertTrue(Files.exists(stepFile)); + assertTrue(stepFile.startsWith(basePath)); + } +} diff --git a/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/LocalDurableTestRunner.java b/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/LocalDurableTestRunner.java index f04e1c27b..b7d7cc025 100644 --- a/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/LocalDurableTestRunner.java +++ b/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/LocalDurableTestRunner.java @@ -23,6 +23,9 @@ import software.amazon.lambda.durable.model.ExecutionStatus; import software.amazon.lambda.durable.plugin.DurableExecutionPlugin; import software.amazon.lambda.durable.serde.SerDes; +import software.amazon.lambda.durable.serde.SerDesContext; +import software.amazon.lambda.durable.serde.SerDesPayloadKind; +import software.amazon.lambda.durable.serde.SerDesRunner; import software.amazon.lambda.durable.testing.local.LocalMemoryExecutionClient; import software.amazon.lambda.durable.testing.local.OperationResult; @@ -43,6 +46,11 @@ public class LocalDurableTestRunner { private final SerDes serDes; private final DurableConfig customerConfig; private final Instant executionStartTime = Instant.now(); + private final String executionName = UUID.randomUUID().toString(); + private final String invocationId = UUID.randomUUID().toString(); + private final String executionArn = String.format( + "arn:aws:lambda:us-east-1:123456789012:function:test:$LATEST/durable-execution/%s/%s", + executionName, invocationId); private LocalDurableTestRunner( TypeToken inputType, @@ -61,11 +69,13 @@ private LocalDurableTestRunner( .withDurableExecutionClient(storage) .withSerDes(customerConfig.getSerDes()) .withExecutorService(customerConfig.getExecutorService()) + .withSerDesExecutorService(customerConfig.getSerDesExecutorService()) .withPollingStrategy(customerConfig.getPollingStrategy()) .withCheckpointDelay(customerConfig.getCheckpointDelay()) .withLoggerConfig(customerConfig.getLoggerConfig()) // Temporary: remove along with the checkpointEmptyMap flag in a future major version. .withCheckpointEmptyMap(customerConfig.shouldCheckpointEmptyMap()) + .withDeserializeAfterSerialization(customerConfig.shouldDeserializeAfterSerialization()) .withPlugins(customerConfig.getPluginRunner().getPlugins().toArray(new DurableExecutionPlugin[0])) .build(); } else { @@ -238,11 +248,13 @@ public static LocalDurableTestRunner create(TypeToken inputType, /** Run a single invocation (may return PENDING if waiting/retrying). */ public TestResult run(I input) { - var durableInput = createDurableInput(input); + var serDesRunner = new SerDesRunner(customerConfig.getSerDesExecutorService()); + var durableInput = createDurableInput(input, serDesRunner); var output = DurableExecutor.execute(durableInput, mockLambdaContext(), inputType, handler, customerConfig); - return storage.toTestResult(output, outputType, serDes); + return storage.toTestResult( + output, outputType, serDes, serDesRunner, executionArn, invocationId, executionName); } /** @@ -281,7 +293,14 @@ public void simulateFireAndForgetCheckpointLoss(String stepName) { /** Returns the {@link TestOperation} for the given operation name, or null if not found. */ public TestOperation getOperation(String name) { var op = storage.getOperationByName(name); - return op != null ? new TestOperation(op, serDes) : null; + return op != null + ? new TestOperation( + op, + List.of(), + serDes, + new SerDesRunner(customerConfig.getSerDesExecutorService()), + executionArn) + : null; } /** Get callback ID for a named callback operation. */ @@ -329,13 +348,11 @@ public void stopChainedInvoke(String name, ErrorObject error) { storage.completeChainedInvoke(name, OperationResult.stopped(error)); } - private DurableExecutionInput createDurableInput(I input) { - var executionName = UUID.randomUUID().toString(); - var invocationId = UUID.randomUUID().toString(); - var executionArn = String.format( - "arn:aws:lambda:us-east-1:123456789012:function:test:$LATEST/durable-execution/%s/%s", - executionName, invocationId); - var inputJson = serDes.serialize(input); + private DurableExecutionInput createDurableInput(I input, SerDesRunner serDesRunner) { + var inputJson = serDesRunner.serialize( + serDes, + input, + SerDesContext.forExecution(executionArn, invocationId, executionName, SerDesPayloadKind.INPUT)); var executionOp = Operation.builder() .id(invocationId) .name(executionName) diff --git a/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/TestOperation.java b/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/TestOperation.java index 31a28b988..0f4481e31 100644 --- a/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/TestOperation.java +++ b/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/TestOperation.java @@ -18,22 +18,39 @@ import software.amazon.awssdk.services.lambda.model.WaitDetails; import software.amazon.lambda.durable.TypeToken; import software.amazon.lambda.durable.execution.ExecutionManager; +import software.amazon.lambda.durable.model.OperationSubType; import software.amazon.lambda.durable.serde.SerDes; +import software.amazon.lambda.durable.serde.SerDesContext; +import software.amazon.lambda.durable.serde.SerDesPayloadKind; +import software.amazon.lambda.durable.serde.SerDesRunner; /** Wrapper for AWS SDK Operation providing convenient access methods. */ public class TestOperation { private final Operation operation; private final List events; private final SerDes serDes; + private final SerDesRunner serDesRunner; + private final String durableExecutionArn; public TestOperation(Operation operation, SerDes serDes) { this(operation, List.of(), serDes); } public TestOperation(Operation operation, List events, SerDes serDes) { + this(operation, events, serDes, null, null); + } + + public TestOperation( + Operation operation, + List events, + SerDes serDes, + SerDesRunner serDesRunner, + String durableExecutionArn) { this.operation = operation; this.events = events; this.serDes = serDes; + this.serDesRunner = serDesRunner; + this.durableExecutionArn = durableExecutionArn; } /** Returns the raw history events associated with this operation. */ @@ -119,7 +136,28 @@ public T getStepResult(TypeToken type) { if (details == null || details.result() == null) { return null; } - return serDes.deserialize(details.result(), type); + if (serDesRunner == null) { + return serDes.deserialize(details.result(), type); + } + var subType = java.util.Arrays.stream(OperationSubType.values()) + .filter(value -> value.getValue().equals(operation.subType())) + .findFirst() + .orElse(OperationSubType.STEP); + var payloadKind = + subType == OperationSubType.WAIT_FOR_CONDITION ? SerDesPayloadKind.STATE : SerDesPayloadKind.RESULT; + return serDesRunner.deserialize( + serDes, + details.result(), + type, + SerDesContext.forOperation( + durableExecutionArn, + operation.id(), + operation.name(), + operation.parentId(), + operation.type(), + subType, + payloadKind, + details.attempt())); } /** Returns the step error, or null if the step succeeded or this is not a step operation. */ diff --git a/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/TestResult.java b/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/TestResult.java index 7de85beef..f78fb80b7 100644 --- a/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/TestResult.java +++ b/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/TestResult.java @@ -15,6 +15,9 @@ import software.amazon.lambda.durable.TypeToken; import software.amazon.lambda.durable.model.ExecutionStatus; import software.amazon.lambda.durable.serde.SerDes; +import software.amazon.lambda.durable.serde.SerDesContext; +import software.amazon.lambda.durable.serde.SerDesPayloadKind; +import software.amazon.lambda.durable.serde.SerDesRunner; /** * Represents the result of a durable execution, providing access to the execution status, output, operations, and @@ -33,6 +36,8 @@ public class TestResult { private final List allEvents; private final SerDes serDes; private final TypeToken resultType; + private final SerDesRunner serDesRunner; + private final SerDesContext outputContext; public TestResult( ExecutionStatus status, @@ -42,6 +47,21 @@ public TestResult( List allEvents, TypeToken resultType, SerDes serDes) { + this(status, resultPayload, error, operations, allEvents, resultType, serDes, null, null, null, null); + } + + public TestResult( + ExecutionStatus status, + String resultPayload, + ErrorObject error, + List operations, + List allEvents, + TypeToken resultType, + SerDes serDes, + SerDesRunner serDesRunner, + String durableExecutionArn, + String executionOperationId, + String executionOperationName) { this.status = status; this.resultPayload = resultPayload; this.error = error; @@ -51,6 +71,11 @@ public TestResult( this.allEvents = List.copyOf(allEvents); this.serDes = serDes; this.resultType = resultType; + this.serDesRunner = serDesRunner; + this.outputContext = serDesRunner == null + ? null + : SerDesContext.forExecution( + durableExecutionArn, executionOperationId, executionOperationName, SerDesPayloadKind.OUTPUT); } /** Returns the execution status (SUCCEEDED, FAILED, or PENDING). */ @@ -75,12 +100,18 @@ public T getResult(TypeToken resultType) { if (resultPayload == null || resultPayload.isEmpty()) { var lastEvent = allEvents.get(allEvents.size() - 1); if (lastEvent.eventType() == EventType.EXECUTION_SUCCEEDED) { - return serDes.deserialize( + return deserialize( lastEvent.executionSucceededDetails().result().payload(), resultType); } return null; } - return serDes.deserialize(resultPayload, resultType); + return deserialize(resultPayload, resultType); + } + + private T deserialize(String payload, TypeToken type) { + return serDesRunner == null + ? serDes.deserialize(payload, type) + : serDesRunner.deserialize(serDes, payload, type, outputContext); } /** Deserializes and returns the execution output if the result type is known. */ diff --git a/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/local/LocalMemoryExecutionClient.java b/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/local/LocalMemoryExecutionClient.java index 25f016cd9..ca76f6414 100644 --- a/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/local/LocalMemoryExecutionClient.java +++ b/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/local/LocalMemoryExecutionClient.java @@ -24,6 +24,7 @@ import software.amazon.lambda.durable.client.DurableExecutionClient; import software.amazon.lambda.durable.model.DurableExecutionOutput; import software.amazon.lambda.durable.serde.SerDes; +import software.amazon.lambda.durable.serde.SerDesRunner; import software.amazon.lambda.durable.testing.TestOperation; import software.amazon.lambda.durable.testing.TestResult; @@ -130,10 +131,18 @@ public List getUpdatedOperationIdsSinceLastInvocation() { } /** Build TestResult from current state. */ - public TestResult toTestResult(DurableExecutionOutput output, TypeToken resultType, SerDes serDes) { + public TestResult toTestResult( + DurableExecutionOutput output, + TypeToken resultType, + SerDes serDes, + SerDesRunner serDesRunner, + String durableExecutionArn, + String executionOperationId, + String executionOperationName) { var testOperations = existingOperations.values().stream() .filter(op -> op.type() != OperationType.EXECUTION) - .map(op -> new TestOperation(op, eventProcessor.getEventsForOperation(op.id()), serDes)) + .map(op -> new TestOperation( + op, eventProcessor.getEventsForOperation(op.id()), serDes, serDesRunner, durableExecutionArn)) .toList(); return new TestResult<>( output.status(), @@ -142,7 +151,11 @@ public TestResult toTestResult(DurableExecutionOutput output, TypeToken { + Thread t = new Thread(r); + t.setName("durable-sdk-serdes-" + t.getId()); + t.setDaemon(true); + return t; + }); + private final DurableExecutionClient durableExecutionClient; private final SerDes serDes; private final ExecutorService executorService; + private final ExecutorService serDesExecutorService; private final LoggerConfig loggerConfig; private final PollingStrategy pollingStrategy; private final Duration checkpointDelay; @@ -109,6 +117,8 @@ private DurableConfig(Builder builder) { this.serDes = Objects.requireNonNullElseGet(builder.serDes, JacksonSerDes::new); this.executorService = Objects.requireNonNullElseGet(builder.executorService, DurableConfig::createDefaultExecutor); + this.serDesExecutorService = + Objects.requireNonNullElse(builder.serDesExecutorService, DEFAULT_SERDES_THREAD_POOL); this.loggerConfig = Objects.requireNonNullElseGet(builder.loggerConfig, LoggerConfig::defaults); this.pollingStrategy = Objects.requireNonNullElse(builder.pollingStrategy, PollingStrategies.Presets.DEFAULT); this.checkpointDelay = Objects.requireNonNullElseGet(builder.checkpointDelay, () -> Duration.ofSeconds(0)); @@ -164,6 +174,11 @@ public ExecutorService getExecutorService() { return executorService; } + /** Gets the executor used for customer SerDes calls and payload storage I/O. */ + public ExecutorService getSerDesExecutorService() { + return serDesExecutorService; + } + /** * Gets the configured LoggerConfig. * @@ -235,6 +250,9 @@ public void validateConfiguration() { if (getExecutorService() == null) { throw new IllegalStateException("ExecutorService configuration failed"); } + if (getSerDesExecutorService() == null) { + throw new IllegalStateException("SerDes ExecutorService configuration failed"); + } } /** @@ -316,6 +334,7 @@ public static final class Builder { private DurableExecutionClient durableExecutionClient; private SerDes serDes; private ExecutorService executorService; + private ExecutorService serDesExecutorService; private LoggerConfig loggerConfig; private PollingStrategy pollingStrategy; private Duration checkpointDelay; @@ -396,6 +415,16 @@ public Builder withExecutorService(ExecutorService executorService) { return this; } + /** + * Sets the executor used for customer SerDes calls and blocking payload storage I/O. If not set, a cached + * daemon thread pool named {@code durable-sdk-serdes-*} is used. + */ + public Builder withSerDesExecutorService(ExecutorService executorService) { + this.serDesExecutorService = + Objects.requireNonNull(executorService, "SerDes ExecutorService cannot be null"); + return this; + } + /** * Sets a custom LoggerConfig. If not set, defaults to suppressing replay logs. * diff --git a/sdk/src/main/java/software/amazon/lambda/durable/execution/DurableExecutor.java b/sdk/src/main/java/software/amazon/lambda/durable/execution/DurableExecutor.java index 649c7a600..44970d954 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/execution/DurableExecutor.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/execution/DurableExecutor.java @@ -12,7 +12,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import software.amazon.awssdk.services.lambda.model.ErrorObject; -import software.amazon.awssdk.services.lambda.model.Operation; import software.amazon.awssdk.services.lambda.model.OperationAction; import software.amazon.awssdk.services.lambda.model.OperationType; import software.amazon.awssdk.services.lambda.model.OperationUpdate; @@ -31,6 +30,8 @@ import software.amazon.lambda.durable.plugin.InvocationStatus; import software.amazon.lambda.durable.plugin.PluginRunner; import software.amazon.lambda.durable.serde.SerDes; +import software.amazon.lambda.durable.serde.SerDesContext; +import software.amazon.lambda.durable.serde.SerDesPayloadKind; import software.amazon.lambda.durable.util.ExceptionHelper; /** @@ -76,8 +77,7 @@ public static DurableExecutionOutput execute( I userInput = null; Throwable inputFailure = null; try { - userInput = extractUserInput( - executionManager.getExecutionOperation(), config.getSerDes(), inputType); + userInput = extractUserInput(executionManager, config.getSerDes(), inputType); } catch (Throwable t) { inputFailure = t; } @@ -159,11 +159,17 @@ public static DurableExecutionOutput execute( cause, pluginExecutionInput.get(), null); - return DurableExecutionOutput.failure(buildErrorObject(cause, config.getSerDes())); + return DurableExecutionOutput.failure( + buildErrorObject(cause, executionManager, config.getSerDes())); } // user handler complete successfully logger.debug("Execution completed"); - var outputPayload = config.getSerDes().serialize(result); + var outputPayload = executionManager + .getSerDesRunner() + .serialize( + config.getSerDes(), + result, + executionContext(executionManager, SerDesPayloadKind.OUTPUT)); var output = DurableExecutionOutput.success(handleLargePayload(executionManager, outputPayload)); fireOnInvocationEnd( @@ -228,7 +234,7 @@ private static String handleLargePayload(ExecutionManager executionManager, Stri return outputPayload; } - private static ErrorObject buildErrorObject(Throwable e, SerDes serDes) { + private static ErrorObject buildErrorObject(Throwable e, ExecutionManager executionManager, SerDes serDes) { // exceptions thrown from operations, e.g. Step if (e instanceof DurableOperationException durableOperationException) { return durableOperationException.getErrorObject(); @@ -237,16 +243,33 @@ private static ErrorObject buildErrorObject(Throwable e, SerDes serDes) { return unrecoverableDurableExecutionException.getErrorObject(); } // exceptions thrown from non-operation code - return ExceptionHelper.buildErrorObject(e, serDes); + return ErrorObject.builder() + .errorType(e.getClass().getName()) + .errorMessage(e.getMessage()) + .errorData(executionManager + .getSerDesRunner() + .serialize(serDes, e, executionContext(executionManager, SerDesPayloadKind.EXCEPTION))) + .stackTrace(ExceptionHelper.serializeStackTrace(e.getStackTrace())) + .build(); } - private static I extractUserInput(Operation executionOp, SerDes serDes, TypeToken inputType) { + private static I extractUserInput(ExecutionManager executionManager, SerDes serDes, TypeToken inputType) { + var executionOp = executionManager.getExecutionOperation(); if (executionOp.executionDetails() == null) { throw new IllegalDurableOperationException("EXECUTION operation missing executionDetails"); } var inputPayload = executionOp.executionDetails().inputPayload(); - return serDes.deserialize(inputPayload, inputType); + return executionManager + .getSerDesRunner() + .deserialize( + serDes, inputPayload, inputType, executionContext(executionManager, SerDesPayloadKind.INPUT)); + } + + private static SerDesContext executionContext(ExecutionManager executionManager, SerDesPayloadKind payloadKind) { + var operation = executionManager.getExecutionOperation(); + return SerDesContext.forExecution( + executionManager.getDurableExecutionArn(), operation.id(), operation.name(), payloadKind); } /** diff --git a/sdk/src/main/java/software/amazon/lambda/durable/execution/ExecutionManager.java b/sdk/src/main/java/software/amazon/lambda/durable/execution/ExecutionManager.java index 1c45cb0d6..6888ef931 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/execution/ExecutionManager.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/execution/ExecutionManager.java @@ -29,6 +29,7 @@ import software.amazon.lambda.durable.model.SafeCloseable; import software.amazon.lambda.durable.operation.BaseDurableOperation; import software.amazon.lambda.durable.plugin.PluginInfoConverter; +import software.amazon.lambda.durable.serde.SerDesRunner; /** * Central manager for durable execution coordination. @@ -63,6 +64,7 @@ public class ExecutionManager implements SafeCloseable { private final AtomicReference executionMode; private final DurableConfig durableConfig; private final Set updatedOperationIdsSinceLastInvocation; + private final SerDesRunner serDesRunner; // ===== Thread Coordination ===== private final Map registeredOperations = new ConcurrentHashMap<>(); @@ -77,6 +79,7 @@ public ExecutionManager(DurableExecutionInput input, DurableConfig config, Conte durableConfig = config; this.durableExecutionArn = input.durableExecutionArn(); this.lambdaContext = lambdaContext; + this.serDesRunner = new SerDesRunner(config.getSerDesExecutorService()); // Store the set of operation IDs updated since the last successful invocation this.updatedOperationIdsSinceLastInvocation = @@ -115,6 +118,11 @@ public String getDurableExecutionArn() { return durableExecutionArn; } + /** Returns the invocation-scoped SerDes runner. */ + public SerDesRunner getSerDesRunner() { + return serDesRunner; + } + /** Returns {@code true} if the execution is currently replaying completed operations. */ public boolean isReplaying() { return executionMode.get() == ExecutionMode.REPLAY; diff --git a/sdk/src/main/java/software/amazon/lambda/durable/operation/BaseDurableOperation.java b/sdk/src/main/java/software/amazon/lambda/durable/operation/BaseDurableOperation.java index 35a71f0da..8f7acedca 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/operation/BaseDurableOperation.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/operation/BaseDurableOperation.java @@ -6,6 +6,7 @@ import java.util.List; import java.util.Objects; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ForkJoinPool; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Supplier; @@ -30,6 +31,9 @@ import software.amazon.lambda.durable.plugin.PluginInfoConverter; import software.amazon.lambda.durable.plugin.PluginRunner; import software.amazon.lambda.durable.plugin.UserFunctionOutcome; +import software.amazon.lambda.durable.serde.SerDesContext; +import software.amazon.lambda.durable.serde.SerDesPayloadKind; +import software.amazon.lambda.durable.serde.SerDesRunner; import software.amazon.lambda.durable.util.ExceptionHelper; /** @@ -60,6 +64,7 @@ public abstract class BaseDurableOperation { protected final boolean isVirtual; protected final AtomicBoolean replayCompletedOperation = new AtomicBoolean(false); private final DurableContextImpl durableContext; + private final SerDesRunner serDesRunner; private final AtomicReference> runningUserHandler = new AtomicReference<>(null); protected BaseDurableOperation( @@ -86,6 +91,9 @@ protected BaseDurableOperation( this.parentOperation = parentOperation; this.durableContext = durableContext; this.executionManager = durableContext.getExecutionManager(); + var invocationSerDesRunner = executionManager.getSerDesRunner(); + this.serDesRunner = + invocationSerDesRunner != null ? invocationSerDesRunner : new SerDesRunner(ForkJoinPool.commonPool()); this.isVirtual = isVirtual; this.completionFuture = new CompletableFuture<>(); @@ -118,6 +126,24 @@ protected DurableContextImpl getContext() { return durableContext; } + /** Builds the SerDes context for a payload owned by this operation. */ + protected SerDesContext createSerDesContext(SerDesPayloadKind payloadKind, Integer attempt) { + return SerDesContext.forOperation( + executionManager.getDurableExecutionArn(), + getOperationId(), + getName(), + durableContext.getParentId(), + getType(), + getSubType(), + payloadKind, + attempt); + } + + /** Returns the invocation-scoped SerDes runner. */ + protected SerDesRunner getSerDesRunner() { + return serDesRunner; + } + /** Gets the operation type. */ public OperationType getType() { return operationIdentifier.operationType(); diff --git a/sdk/src/main/java/software/amazon/lambda/durable/operation/InvokeOperation.java b/sdk/src/main/java/software/amazon/lambda/durable/operation/InvokeOperation.java index 9e2c54ace..3796932af 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/operation/InvokeOperation.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/operation/InvokeOperation.java @@ -15,6 +15,7 @@ import software.amazon.lambda.durable.exception.InvokeTimedOutException; import software.amazon.lambda.durable.model.OperationIdentifier; import software.amazon.lambda.durable.serde.SerDes; +import software.amazon.lambda.durable.serde.SerDesPayloadKind; /** * Durable operation that invokes another Lambda function and waits for its result. @@ -70,7 +71,11 @@ private void startInvocation() { .functionName(functionName) .tenantId(invokeConfig.tenantId()) .build()) - .payload(payloadSerDes.serialize(this.payload)); + .payload(getSerDesRunner() + .serialize( + payloadSerDes, + this.payload, + createSerDesContext(SerDesPayloadKind.INVOKE_PAYLOAD, null))); sendOperationUpdate(update); } diff --git a/sdk/src/main/java/software/amazon/lambda/durable/operation/SerializableDurableOperation.java b/sdk/src/main/java/software/amazon/lambda/durable/operation/SerializableDurableOperation.java index 6457c996d..0d4b849a5 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/operation/SerializableDurableOperation.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/operation/SerializableDurableOperation.java @@ -11,6 +11,7 @@ import software.amazon.lambda.durable.exception.SerDesException; import software.amazon.lambda.durable.model.OperationIdentifier; import software.amazon.lambda.durable.serde.SerDes; +import software.amazon.lambda.durable.serde.SerDesPayloadKind; import software.amazon.lambda.durable.util.ExceptionHelper; /** @@ -85,8 +86,14 @@ protected SerializableDurableOperation( * @throws SerDesException if deserialization fails */ protected T deserializeResult(String result) { + return deserializeResult(result, SerDesPayloadKind.RESULT, null); + } + + /** Deserializes a result with explicit payload kind and attempt metadata. */ + protected T deserializeResult(String result, SerDesPayloadKind payloadKind, Integer attempt) { try { - return resultSerDes.deserialize(result, resultTypeToken); + return getSerDesRunner() + .deserialize(resultSerDes, result, resultTypeToken, createSerDesContext(payloadKind, attempt)); } catch (SerDesException e) { logger.warn( "Failed to deserialize {} result for operation name '{}'. Ensure the result is properly encoded.", @@ -106,8 +113,17 @@ protected T deserializeResult(String result) { * @return the serialized string and the deserialized result */ protected SerializedResult serializeAndDeserializeResult(T result) { - var serialized = resultSerDes.serialize(result); - var deserialized = shouldDeserializeAfterSerialization() ? deserializeResult(serialized) : result; + return serializeAndDeserializeResult(result, SerDesPayloadKind.RESULT, null); + } + + /** Serializes a result with explicit payload kind and attempt metadata. */ + protected SerializedResult serializeAndDeserializeResult( + T result, SerDesPayloadKind payloadKind, Integer attempt) { + var context = createSerDesContext(payloadKind, attempt); + var serialized = getSerDesRunner().serialize(resultSerDes, result, context); + var deserialized = shouldDeserializeAfterSerialization() + ? getSerDesRunner().deserialize(resultSerDes, serialized, resultTypeToken, context) + : result; return new SerializedResult<>(serialized, deserialized); } @@ -119,9 +135,20 @@ protected SerializedResult serializeAndDeserializeResult(T result) { */ @SuppressWarnings("ThrowableNotThrown") protected ErrorObject serializeException(Throwable throwable) { - var error = ExceptionHelper.buildErrorObject(throwable, resultSerDes); + return serializeException(throwable, null); + } + + /** Serializes a throwable with attempt metadata. */ + protected ErrorObject serializeException(Throwable throwable, Integer attempt) { + var context = createSerDesContext(SerDesPayloadKind.EXCEPTION, attempt); + var error = ErrorObject.builder() + .errorType(throwable.getClass().getName()) + .errorMessage(throwable.getMessage()) + .errorData(getSerDesRunner().serialize(resultSerDes, throwable, context)) + .stackTrace(ExceptionHelper.serializeStackTrace(throwable.getStackTrace())) + .build(); if (shouldDeserializeAfterSerialization()) { - deserializeException(error); + deserializeException(error, attempt); } return error; } @@ -139,6 +166,11 @@ private boolean shouldDeserializeAfterSerialization() { * @return the reconstructed throwable, or null if reconstruction is not possible */ protected Throwable deserializeException(ErrorObject errorObject) { + return deserializeException(errorObject, null); + } + + /** Deserializes a throwable with attempt metadata. */ + protected Throwable deserializeException(ErrorObject errorObject, Integer attempt) { Throwable original = null; if (errorObject == null) { return original; @@ -153,8 +185,12 @@ protected Throwable deserializeException(ErrorObject errorObject) { Class exceptionClass = Class.forName(errorType); if (Throwable.class.isAssignableFrom(exceptionClass)) { - original = - resultSerDes.deserialize(errorData, TypeToken.get(exceptionClass.asSubclass(Throwable.class))); + original = getSerDesRunner() + .deserialize( + resultSerDes, + errorData, + TypeToken.get(exceptionClass.asSubclass(Throwable.class)), + createSerDesContext(SerDesPayloadKind.EXCEPTION, attempt)); if (original != null) { original.setStackTrace(ExceptionHelper.deserializeStackTrace(errorObject.stackTrace())); diff --git a/sdk/src/main/java/software/amazon/lambda/durable/operation/StepOperation.java b/sdk/src/main/java/software/amazon/lambda/durable/operation/StepOperation.java index 467a87b94..6123eb56e 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/operation/StepOperation.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/operation/StepOperation.java @@ -170,7 +170,7 @@ private void handleStepFailure(Throwable exception, int attempt) { if (exception instanceof DurableOperationException durableOperationException) { errorObject = durableOperationException.getErrorObject(); } else { - errorObject = serializeException(exception); + errorObject = serializeException(exception, attempt); } var retryDecision = config.retryStrategy().makeRetryDecision(exception, attempt); @@ -216,7 +216,8 @@ public T get() { } // Attempt to reconstruct and throw the original exception - Throwable original = deserializeException(errorObject); + var attempt = op.stepDetails() != null ? op.stepDetails().attempt() : null; + Throwable original = deserializeException(errorObject, attempt); if (original != null) { ExceptionHelper.sneakyThrow(original); } diff --git a/sdk/src/main/java/software/amazon/lambda/durable/operation/WaitForConditionOperation.java b/sdk/src/main/java/software/amazon/lambda/durable/operation/WaitForConditionOperation.java index 6a653c37b..da9d23df3 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/operation/WaitForConditionOperation.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/operation/WaitForConditionOperation.java @@ -23,6 +23,7 @@ import software.amazon.lambda.durable.logging.DurableLogger; import software.amazon.lambda.durable.model.OperationIdentifier; import software.amazon.lambda.durable.model.WaitForConditionResult; +import software.amazon.lambda.durable.serde.SerDesPayloadKind; import software.amazon.lambda.durable.util.ExceptionHelper; /** @@ -76,12 +77,14 @@ public T get() { if (op.status() == OperationStatus.SUCCEEDED) { var stepDetails = op.stepDetails(); var result = (stepDetails != null) ? stepDetails.result() : null; - return deserializeResult(result); + var attempt = stepDetails != null ? stepDetails.attempt() : null; + return deserializeResult(result, SerDesPayloadKind.STATE, attempt); } else { var errorObject = op.stepDetails().error(); // Attempt to reconstruct and throw the original exception - Throwable original = deserializeException(errorObject); + var attempt = op.stepDetails() != null ? op.stepDetails().attempt() : null; + Throwable original = deserializeException(errorObject, attempt); if (original != null) { ExceptionHelper.sneakyThrow(original); } @@ -97,7 +100,7 @@ private void resumeCheckLoop(Operation existing) { var checkpointData = stepDetails != null ? stepDetails.result() : null; T currentState; // Get current state if (checkpointData != null) { - currentState = deserializeResult(checkpointData); + currentState = deserializeResult(checkpointData, SerDesPayloadKind.STATE, attempt - 1); } else { currentState = config.initialState(); } @@ -131,7 +134,8 @@ private void executeCheckLogic(T currentState, int attempt) { runUserFunction(attempt, () -> checkFunc.apply(currentState, stepContext)); // Normalize the value through SerDes so first execution matches replay. - var serializedState = serializeAndDeserializeResult(result.value()); + var serializedState = + serializeAndDeserializeResult(result.value(), SerDesPayloadKind.STATE, attempt); T deserializedValue = serializedState.deserialized(); if (result.isDone()) { @@ -161,7 +165,7 @@ private void executeCheckLogic(T currentState, int attempt) { .thenRun(() -> executeCheckLogic(deserializedValue, attempt + 1)); } } catch (Throwable e) { - handleCheckFailure(e); + handleCheckFailure(e, attempt); } } }; @@ -169,7 +173,7 @@ private void executeCheckLogic(T currentState, int attempt) { runUserHandler(userHandler, ThreadType.STEP); } - private void handleCheckFailure(Throwable exception) { + private void handleCheckFailure(Throwable exception, int attempt) { exception = ExceptionHelper.unwrapCompletableFuture(exception); if (exception instanceof SuspendExecutionException suspendExecutionException) { throw suspendExecutionException; @@ -180,7 +184,7 @@ private void handleCheckFailure(Throwable exception) { final var errorObject = (exception instanceof DurableOperationException durableOpEx) ? durableOpEx.getErrorObject() - : serializeException(exception); + : serializeException(exception, attempt); // Checkpoint FAIL var failUpdate = OperationUpdate.builder().action(OperationAction.FAIL).error(errorObject); diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesContext.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesContext.java new file mode 100644 index 000000000..71da1fdd5 --- /dev/null +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesContext.java @@ -0,0 +1,69 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.serde; + +import software.amazon.awssdk.services.lambda.model.OperationType; +import software.amazon.lambda.durable.model.OperationSubType; + +/** + * Describes the durable payload currently being processed by a {@link SerDes}. + * + *

The SDK sets this context only while invoking a configured SerDes. Direct customer calls to SerDes methods do not + * have a current context. + */ +public record SerDesContext( + String durableExecutionArn, + String entityId, + SerDesPayloadKind payloadKind, + String operationId, + String operationName, + String parentId, + OperationType operationType, + OperationSubType operationSubType, + Integer attempt) { + + /** Returns the context for the SerDes call on the current thread, or {@code null} outside SDK-managed calls. */ + public static SerDesContext getCurrentContext() { + return SerDesContextHolder.get(); + } + + /** Creates context for a root execution payload. */ + public static SerDesContext forExecution( + String durableExecutionArn, + String executionOperationId, + String executionOperationName, + SerDesPayloadKind payloadKind) { + return new SerDesContext( + durableExecutionArn, + "execution/" + executionOperationId + "/" + payloadKind.getEntitySuffix(), + payloadKind, + executionOperationId, + executionOperationName, + null, + OperationType.EXECUTION, + null, + null); + } + + /** Creates context for an operation payload. */ + public static SerDesContext forOperation( + String durableExecutionArn, + String operationId, + String operationName, + String parentId, + OperationType operationType, + OperationSubType operationSubType, + SerDesPayloadKind payloadKind, + Integer attempt) { + return new SerDesContext( + durableExecutionArn, + "operation/" + operationId + "/" + payloadKind.getEntitySuffix(), + payloadKind, + operationId, + operationName, + parentId, + operationType, + operationSubType, + attempt); + } +} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesContextHolder.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesContextHolder.java new file mode 100644 index 000000000..6d31cd897 --- /dev/null +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesContextHolder.java @@ -0,0 +1,21 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.serde; + +final class SerDesContextHolder { + private static final ThreadLocal CURRENT = new ThreadLocal<>(); + + private SerDesContextHolder() {} + + static SerDesContext get() { + return CURRENT.get(); + } + + static void set(SerDesContext context) { + CURRENT.set(context); + } + + static void clear() { + CURRENT.remove(); + } +} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesPayloadKind.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesPayloadKind.java new file mode 100644 index 000000000..098e2b565 --- /dev/null +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesPayloadKind.java @@ -0,0 +1,24 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.serde; + +/** Identifies the durable payload being serialized or deserialized. */ +public enum SerDesPayloadKind { + INPUT("input"), + OUTPUT("output"), + RESULT("result"), + INVOKE_PAYLOAD("invoke-payload"), + STATE("state"), + EXCEPTION("exception"); + + private final String entitySuffix; + + SerDesPayloadKind(String entitySuffix) { + this.entitySuffix = entitySuffix; + } + + /** Returns the stable suffix used in external payload entity identifiers. */ + public String getEntitySuffix() { + return entitySuffix; + } +} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesRunner.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesRunner.java new file mode 100644 index 000000000..b65c82dab --- /dev/null +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesRunner.java @@ -0,0 +1,103 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.serde; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.HexFormat; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; +import java.util.function.Supplier; +import software.amazon.lambda.durable.TypeToken; +import software.amazon.lambda.durable.exception.SerDesException; +import software.amazon.lambda.durable.util.ExceptionHelper; + +/** + * Runs customer SerDes calls on the configured SerDes executor with the correct {@link SerDesContext}. + * + *

Instances are invocation-scoped so successful deserialization results are cached only for one Lambda invocation. + */ +public final class SerDesRunner { + private static final Object NULL_VALUE = new Object(); + + private final ExecutorService executorService; + private final Map deserializationCache = new ConcurrentHashMap<>(); + + public SerDesRunner(ExecutorService executorService) { + this.executorService = Objects.requireNonNull(executorService, "executorService cannot be null"); + } + + /** Serializes a value with the supplied durable payload context. */ + public String serialize(SerDes serDes, Object value, SerDesContext context) { + return run("serialize", context, () -> serDes.serialize(value)); + } + + /** Deserializes a value with invocation-scoped caching. */ + @SuppressWarnings("unchecked") + public T deserialize(SerDes serDes, String data, TypeToken typeToken, SerDesContext context) { + Objects.requireNonNull(context, "SerDesContext cannot be null"); + var key = new CacheKey( + context.durableExecutionArn(), + context.entityId(), + context.payloadKind(), + context.attempt(), + typeToken, + hash(data)); + var cached = deserializationCache.get(key); + if (cached != null) { + return cached == NULL_VALUE ? null : (T) cached; + } + + T value = run("deserialize", context, () -> serDes.deserialize(data, typeToken)); + deserializationCache.putIfAbsent(key, value == null ? NULL_VALUE : value); + return value; + } + + private T run(String action, SerDesContext context, Supplier supplier) { + Objects.requireNonNull(context, "SerDesContext cannot be null"); + try { + return CompletableFuture.supplyAsync( + () -> { + SerDesContextHolder.set(context); + try { + return supplier.get(); + } finally { + SerDesContextHolder.clear(); + } + }, + executorService) + .join(); + } catch (Throwable throwable) { + var cause = ExceptionHelper.unwrapCompletableFuture(throwable); + throw new SerDesException( + String.format( + "Failed to %s %s payload for entity '%s'", + action, context.payloadKind(), context.entityId()), + cause); + } + } + + private static String hash(String data) { + if (data == null) { + return "null"; + } + try { + var digest = MessageDigest.getInstance("SHA-256").digest(data.getBytes(StandardCharsets.UTF_8)); + return HexFormat.of().formatHex(digest); + } catch (NoSuchAlgorithmException e) { + throw new IllegalStateException("SHA-256 is unavailable", e); + } + } + + private record CacheKey( + String durableExecutionArn, + String entityId, + SerDesPayloadKind payloadKind, + Integer attempt, + TypeToken typeToken, + String serializedHash) {} +} diff --git a/sdk/src/test/java/software/amazon/lambda/durable/DurableConfigTest.java b/sdk/src/test/java/software/amazon/lambda/durable/DurableConfigTest.java index c0bd54147..7a3b7aaef 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/DurableConfigTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/DurableConfigTest.java @@ -33,12 +33,14 @@ class DurableConfigTest { private DurableExecutionClient mockClient; private SerDes mockSerDes; private ExecutorService mockExecutor; + private ExecutorService mockSerDesExecutor; @BeforeEach void setUp() { mockClient = mock(DurableExecutionClient.class); mockSerDes = mock(SerDes.class); mockExecutor = mock(ExecutorService.class); + mockSerDesExecutor = mock(ExecutorService.class); } @Test @@ -52,6 +54,8 @@ void testDefaultConfig_CreatesWithDefaults() { assertInstanceOf(JacksonSerDes.class, config.getSerDes()); assertNotNull(config.getExecutorService()); assertInstanceOf(ExecutorService.class, config.getExecutorService()); + assertNotNull(config.getSerDesExecutorService()); + assertInstanceOf(ExecutorService.class, config.getSerDesExecutorService()); } @Test @@ -87,6 +91,15 @@ void testBuilder_WithCustomExecutorService() { assertNotNull(config.getSerDes()); } + @Test + void testBuilder_WithCustomSerDesExecutorService() { + var config = DurableConfig.builder() + .withSerDesExecutorService(mockSerDesExecutor) + .build(); + + assertSame(mockSerDesExecutor, config.getSerDesExecutorService()); + } + @Test void testBuilder_DeserializeAfterSerializationDefaultsToTrue() { var config = @@ -131,12 +144,14 @@ void testBuilder_WithAllCustomComponents() { .withDurableExecutionClient(mockClient) .withSerDes(mockSerDes) .withExecutorService(mockExecutor) + .withSerDesExecutorService(mockSerDesExecutor) .build(); assertNotNull(config); assertEquals(mockClient, config.getDurableExecutionClient()); assertEquals(mockSerDes, config.getSerDes()); assertEquals(mockExecutor, config.getExecutorService()); + assertEquals(mockSerDesExecutor, config.getSerDesExecutorService()); } @Test @@ -161,6 +176,15 @@ void testBuilder_NullSerDes_ThrowsException() { assertEquals("SerDes cannot be null", exception.getMessage()); } + @Test + void testBuilder_NullSerDesExecutorService_ThrowsException() { + var builder = DurableConfig.builder(); + + var exception = assertThrows(NullPointerException.class, () -> builder.withSerDesExecutorService(null)); + + assertEquals("SerDes ExecutorService cannot be null", exception.getMessage()); + } + @Test void testBuilder_FluentAPI() { var builder = DurableConfig.builder(); @@ -169,6 +193,7 @@ void testBuilder_FluentAPI() { assertSame(builder, builder.withDurableExecutionClient(mockClient)); assertSame(builder, builder.withSerDes(mockSerDes)); assertSame(builder, builder.withExecutorService(mockExecutor)); + assertSame(builder, builder.withSerDesExecutorService(mockSerDesExecutor)); assertSame(builder, builder.withDeserializeAfterSerialization(false)); } diff --git a/sdk/src/test/java/software/amazon/lambda/durable/operation/CallbackOperationTest.java b/sdk/src/test/java/software/amazon/lambda/durable/operation/CallbackOperationTest.java index c8cea7934..b1f23d30d 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/operation/CallbackOperationTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/operation/CallbackOperationTest.java @@ -365,6 +365,7 @@ void getThrowsSerDesExceptionWithHelpfulMessageWhenDeserializationFails() { operation.execute(); var exception = assertThrows(SerDesException.class, operation::get); - assertEquals("Invalid base64 encoding", exception.getMessage()); + assertTrue(exception.getMessage().contains("RESULT")); + assertEquals("Invalid base64 encoding", exception.getCause().getMessage()); } } diff --git a/sdk/src/test/java/software/amazon/lambda/durable/operation/SerializableDurableOperationTest.java b/sdk/src/test/java/software/amazon/lambda/durable/operation/SerializableDurableOperationTest.java index bc9e940b8..713ee05c0 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/operation/SerializableDurableOperationTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/operation/SerializableDurableOperationTest.java @@ -420,7 +420,8 @@ protected void replay(Operation existing) {} @Override public String get() { var thrown = assertThrows(SerDesException.class, () -> serializeAndDeserializeResult("abc")); - assertEquals("cannot deserialize", thrown.getMessage()); + assertTrue(thrown.getMessage().contains("RESULT")); + assertEquals("cannot deserialize", thrown.getCause().getMessage()); return RESULT; } }; diff --git a/sdk/src/test/java/software/amazon/lambda/durable/serde/SerDesRunnerTest.java b/sdk/src/test/java/software/amazon/lambda/durable/serde/SerDesRunnerTest.java new file mode 100644 index 000000000..8e49df35b --- /dev/null +++ b/sdk/src/test/java/software/amazon/lambda/durable/serde/SerDesRunnerTest.java @@ -0,0 +1,126 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.serde; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import software.amazon.awssdk.services.lambda.model.OperationType; +import software.amazon.lambda.durable.TypeToken; +import software.amazon.lambda.durable.exception.SerDesException; +import software.amazon.lambda.durable.model.OperationSubType; + +class SerDesRunnerTest { + private final java.util.concurrent.ExecutorService executor = Executors.newSingleThreadExecutor(r -> { + var thread = new Thread(r); + thread.setName("test-serdes"); + return thread; + }); + + @AfterEach + void tearDown() { + executor.shutdownNow(); + } + + @Test + void setsContextInsideExecutorAndClearsItAfterCall() throws Exception { + var observedContext = new AtomicReference(); + var observedThread = new AtomicReference(); + var serDes = new JacksonSerDes() { + @Override + public String serialize(Object value) { + observedContext.set(SerDesContext.getCurrentContext()); + observedThread.set(Thread.currentThread().getName()); + return super.serialize(value); + } + }; + var context = context("operation/1/result"); + + new SerDesRunner(executor).serialize(serDes, "value", context); + + assertSame(context, observedContext.get()); + assertEquals("test-serdes", observedThread.get()); + assertNull(executor.submit(SerDesContext::getCurrentContext).get()); + assertNull(SerDesContext.getCurrentContext()); + } + + @Test + void cachesByEntityTypeAndSerializedDataHash() { + var count = new AtomicInteger(); + var delegate = new JacksonSerDes(); + var serDes = new SerDes() { + @Override + public String serialize(Object value) { + return delegate.serialize(value); + } + + @Override + public T deserialize(String data, TypeToken typeToken) { + count.incrementAndGet(); + return delegate.deserialize(data, typeToken); + } + }; + var runner = new SerDesRunner(executor); + var context = context("operation/1/result"); + + assertEquals("one", runner.deserialize(serDes, "\"one\"", TypeToken.get(String.class), context)); + assertEquals("one", runner.deserialize(serDes, "\"one\"", TypeToken.get(String.class), context)); + assertEquals("two", runner.deserialize(serDes, "\"two\"", TypeToken.get(String.class), context)); + var nextAttempt = new SerDesContext( + context.durableExecutionArn(), + context.entityId(), + context.payloadKind(), + context.operationId(), + context.operationName(), + context.parentId(), + context.operationType(), + context.operationSubType(), + 2); + assertEquals("one", runner.deserialize(serDes, "\"one\"", TypeToken.get(String.class), nextAttempt)); + + assertEquals(3, count.get()); + } + + @Test + void wrapsFailuresWithPayloadMetadata() { + var runner = new SerDesRunner(executor); + var serDes = new SerDes() { + @Override + public String serialize(Object value) { + throw new IllegalStateException("boom"); + } + + @Override + public T deserialize(String data, TypeToken typeToken) { + return null; + } + }; + + var exception = assertThrows(SerDesException.class, () -> runner.serialize(serDes, "value", context("entity"))); + + assertTrue(exception.getMessage().contains("RESULT")); + assertTrue(exception.getMessage().contains("entity")); + assertEquals("boom", exception.getCause().getMessage()); + } + + private static SerDesContext context(String entityId) { + return new SerDesContext( + "arn:test", + entityId, + SerDesPayloadKind.RESULT, + "1", + "step", + null, + OperationType.STEP, + OperationSubType.STEP, + 1); + } +} From 7040554f99dfee7c4c1f5668fd12f6a7f0338fa5 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Mon, 24 Aug 2026 21:47:47 +0000 Subject: [PATCH 02/56] docs: design composable SerDes pipeline --- docs/adr/005-filesystem-serdes.md | 232 +++++++++++++++++++++++++----- 1 file changed, 196 insertions(+), 36 deletions(-) diff --git a/docs/adr/005-filesystem-serdes.md b/docs/adr/005-filesystem-serdes.md index 6a92ebab2..46b36f95d 100644 --- a/docs/adr/005-filesystem-serdes.md +++ b/docs/adr/005-filesystem-serdes.md @@ -1,7 +1,8 @@ # ADR-005: Payload Offloading for Filesystem Storage -**Status:** Accepted — Approach A +**Status:** Accepted — Approach A with ComposableSerDes pipeline **Date:** 2026-07-02 +**Updated:** 2026-08-24 — Added the composable SerDes pipeline design. ## Context @@ -29,19 +30,35 @@ There are a few Java-specific constraints: ### Summary -Keep the existing `SerDes` contract unchanged and implement `FileSystemSerDes` as an optional extra package. The implementation uses `SerDesContext.getCurrentContext()` to identify the durable execution and entity being serialized. +Keep the existing `SerDes` serialization methods source- and binary-compatible, add a default composition method and a +core `ComposableSerDes` implementation which together chain multiple `SerDes` instances into a processing pipeline, +and implement `FileSystemSerDes` as an optional extra package. The filesystem stage uses +`SerDesContext.getCurrentContext()` to identify the durable execution and entity being serialized. ```java public interface SerDes { String serialize(Object value); T deserialize(String data, TypeToken typeToken); + + default ComposableSerDes then(SerDes nextStage) { + return ComposableSerDes.of(this, nextStage); + } } ``` -`FileSystemSerDes` acts as both serializer and payload offloader. It serializes values through a delegate SerDes, writes payloads to the filesystem when configured to do so, and stores a small envelope in the checkpoint. +`ComposableSerDes` treats the first stage as the value codec and every later stage as a reversible string +transformation. This lets customers compose JSON encoding, compression, encryption, filesystem storage, or other +processing without each implementation needing to know about every other concern. + +`FileSystemSerDes` acts as a payload-storage stage. It writes the string produced by the previous stage to the +filesystem when configured to do so and returns a small envelope for the next stage or checkpoint. For standalone +compatibility, it may still be constructed with a value-encoding delegate; pipeline configuration is the preferred +composition model. -Because the existing `SerDes` methods do not accept context parameters, this approach needs a thread-local `SerDesContext` so `FileSystemSerDes` can discover the current payload identity without changing the `SerDes` interface. +Because the existing `SerDes` methods do not accept context parameters, this approach needs a thread-local +`SerDesContext` so `FileSystemSerDes` can discover the current payload identity without changing the +`serialize`/`deserialize` signatures. ```java public record SerDesContext( @@ -75,26 +92,132 @@ The SDK owns setting and clearing this thread-local value around SDK-managed Ser ### Configuration ```java +import software.amazon.lambda.durable.serde.ComposableSerDes; import software.amazon.lambda.durable.extra.filesystem.FileSystemSerDes; -var serDes = FileSystemSerDes.builder(Path.of("/mnt/efs/durable-payloads")) +var fileSystemStage = FileSystemSerDes.stageBuilder(Path.of("/mnt/efs/durable-payloads")) .storageMode(FileSystemStorageMode.ALWAYS) .pathEncoding(FileSystemPathEncoding.URI) - .delegate(new JacksonSerDes()) .previewGenerator(optionalPreviewGenerator) .build(); +var serDes = new JacksonSerDes().then(fileSystemStage); + return DurableConfig.builder() .withSerDes(serDes) .build(); ``` +### Composable SerDes pipeline + +`ComposableSerDes` is a core implementation of `SerDes`. It owns an immutable, ordered list of stages while preserving +the existing `serialize` and `deserialize` methods: + +```java +public final class ComposableSerDes implements SerDes { + public static ComposableSerDes of(SerDes first, SerDes... remaining); + + public static Builder builder(SerDes valueCodec); + + public ComposableSerDes then(SerDes stage); + + public static final class Builder { + public Builder then(SerDes stage); + + public ComposableSerDes build(); + } +} +``` + +The first stage is the **value codec**. It converts the user value to a string and converts the final decoded string +back to the requested `TypeToken`. Every later stage is a **string stage**: it must accept a `String` in +`serialize(Object)` and must return a `String` when `deserialize` is called with `TypeToken.get(String.class)`. + +Serialization runs from first to last: + +```text +Object + -> value codec + -> String stage 1 + -> String stage 2 + -> ... + -> checkpoint String +``` + +Deserialization runs in the opposite direction: + +```text +checkpoint String + -> last string stage, deserialized as String + -> ... + -> first string stage, deserialized as String + -> value codec, deserialized as the requested TypeToken + -> T +``` + +Equivalent pseudocode: + +```java +String serialize(Object value) { + String current = stages.get(0).serialize(value); + for (int i = 1; i < stages.size(); i++) { + current = stages.get(i).serialize(current); + } + return current; +} + + T deserialize(String data, TypeToken targetType) { + String current = data; + for (int i = stages.size() - 1; i > 0; i--) { + current = stages.get(i).deserialize(current, TypeToken.get(String.class)); + } + return stages.get(0).deserialize(current, targetType); +} +``` + +Pipeline rules: + +- A pipeline must contain exactly one value codec in the first position and zero or more string stages. +- `SerDes.then(...)`, `ComposableSerDes.of(...)`, and the builder flatten nested `ComposableSerDes` instances while + preserving stage order. +- A `null` value at the pipeline boundary short-circuits the entire pipeline: serializing or deserializing `null` + returns `null` without invoking any stage. A stage returning `null` for non-null input is an error. +- All stages execute within the same `SerDesRunner` task and observe the same read-only `SerDesContext`. +- `ComposableSerDes` is immutable. It is safe for concurrent use only when every contained stage is also safe for + concurrent use, matching the existing `SerDes` requirement. +- A failure must identify the stage index and implementation class in `SerDesException`; `SerDesRunner` adds durable + entity and payload-kind metadata around the pipeline failure. +- A string stage must be reversible. Compression, encryption, signing envelopes, and external payload storage are + suitable stages; lossy redaction is not. +- Stage order is meaningful. For example, `JSON -> compression -> encryption -> filesystem` writes encrypted, + compressed data to the filesystem, while `JSON -> filesystem -> encryption` encrypts only the file-reference + envelope. +- The ordered stage list and each stage's configuration are part of the persisted checkpoint format. They must remain + replay-compatible for in-flight executions. Reordering, removing, or incompatibly reconfiguring a stage requires a + versioned envelope or an explicit migration boundary. +- `ComposableSerDes` does not add a generic pipeline envelope or persist stage names. Stages that need format + evolution must version their own output. +- Global and operation-level SerDes selection continues to select one `SerDes` instance. A `ComposableSerDes` is + treated as that single instance; operation-level selection replaces the whole pipeline rather than merging stages. +- Invocation-scoped caching wraps the complete pipeline. Cache keys use the final checkpoint string and target type, so + cache hits skip every reverse-processing stage, including filesystem reads. + +The default `SerDes.then(...)` method and immutable `ComposableSerDes.then(...)` method provide a concise form for +independently reusable processing chains: + +```java +var securePayloads = new JacksonSerDes() + .then(compressionSerDes) + .then(encryptionSerDes) + .then(fileSystemStage); +``` + Storage modes: | Mode | Behavior | |------|----------| -| `ALWAYS` | Always write the delegate-serialized value to a file and store a file envelope in the checkpoint. | -| `OVERFLOW` | Store inline until the checkpoint envelope approaches the service payload limit, then write to a file. | +| `ALWAYS` | Always write the incoming stage string to a file and return a file envelope. | +| `OVERFLOW` | Return an inline envelope until it approaches the service payload limit, then write the incoming stage string to a file. | Path encodings: @@ -106,26 +229,35 @@ Path encodings: Envelope format: ```json -{"data":""} +{"data":""} {"file":""} {"file":"","preview":{ "...": "..." }} ``` -`FileSystemSerDes` must reject calls when `SerDesContext.getCurrentContext()` is `null` or does not include `durableExecutionArn` and `entityId`. +`FileSystemSerDes` must reject calls when `SerDesContext.getCurrentContext()` is `null` or does not include +`durableExecutionArn` and `entityId`. When used as a pipeline stage, it must also reject non-string input or a +deserialization target other than `String`. + +In stage mode, the preview generator receives the string produced by the preceding stage, not the original domain +object. A preview that needs domain fields should either parse that representation, be produced by an earlier stage, or +use standalone compatibility mode where `FileSystemSerDes` receives the original value. ### Runtime flow ```java SerDesContextHolder.set(context); try { - var checkpointPayload = fileSystemSerDes.serialize(value); + var checkpointPayload = composableSerDes.serialize(value); sendCheckpoint(checkpointPayload); } finally { SerDesContextHolder.clear(); } ``` -On deserialization, `FileSystemSerDes` parses its envelope. If the envelope contains `data`, it delegates directly to the inner SerDes. If the envelope contains `file`, it reads file contents and delegates to the inner SerDes. +On deserialization, `FileSystemSerDes` parses its envelope. If the envelope contains `data`, it returns the inline +string. If the envelope contains `file`, it reads and returns the file contents. `ComposableSerDes` then passes that +string to the preceding stage. In standalone compatibility mode, `FileSystemSerDes` instead passes the resolved string +to its configured value-encoding delegate. ### Threading @@ -183,22 +315,39 @@ Root user input and output payloads should route through `SerDesRunner` so `File ### Implementation plan -1. Add `SerDesContext`, `SerDesPayloadKind`, and package-private TLS setter/clearer support. Leave the `SerDes` interface unchanged. -2. Add `SerDesRunner` and a `SerDesExecutor` default pool. Add `DurableConfig.withSerDesExecutorService(...)` and validation. -3. Update root input/output handling in `DurableExecutor` to run user payload SerDes through `SerDesRunner` while leaving `DurableInputOutputSerDes` internal. -4. Update `SerializableDurableOperation`, `InvokeOperation`, `StepOperation`, `WaitForConditionOperation`, `CallbackOperation`, `ChildContextOperation`, `MapOperation`, and test helpers to use `SerDesRunner`. -5. Add invocation-scoped deserialization caching keyed by entity, payload kind, type, and serialized data hash. -6. Update exception serialization and deserialization paths to set `SerDesPayloadKind.EXCEPTION` in TLS. -7. Add the `extra-filesystem-serdes` Maven module with artifact ID `aws-durable-execution-sdk-java-extra-filesystem-serdes`, depending on the core SDK. -8. Implement `FileSystemSerDes` in `software.amazon.lambda.durable.extra.filesystem` with `ALWAYS` and `OVERFLOW` modes, `URI` and `HASH` path encodings, envelope parsing, atomic file writes where supported by the filesystem, and clear validation errors for missing context. -9. Add unit tests for context construction, unchanged `SerDes` compatibility, TLS scoping and clearing, thread-pool isolation, cache hits, cache invalidation when serialized data changes, exception reconstruction, malformed filesystem envelopes, and extra-module packaging. -10. Add integration tests with `LocalDurableTestRunner` for step results, wait-for-condition state, invoke payload/result, child context results, map results, repeated `get()`, replay from file pointers, and custom exception types. -11. Update README and advanced configuration docs with FileSystemSerDes dependency coordinates, FileSystemSerDes examples, and warnings about `/tmp`, S3 Files flush behavior, and EFS/S3 Files operational requirements. +1. Add `SerDesContext`, `SerDesPayloadKind`, and package-private TLS setter/clearer support. Leave the existing + `SerDes` methods unchanged. +2. Add the binary-compatible `SerDes.then(...)` default method and `ComposableSerDes` with immutable stage ordering, + forward serialization, reverse deserialization, null short-circuiting, and stage-aware errors. +3. Add `SerDesRunner` and a `SerDesExecutor` default pool. Add + `DurableConfig.withSerDesExecutorService(...)` and validation. +4. Update root input/output handling in `DurableExecutor` to run user payload SerDes through `SerDesRunner` while + leaving `DurableInputOutputSerDes` internal. +5. Update `SerializableDurableOperation`, `InvokeOperation`, `StepOperation`, `WaitForConditionOperation`, + `CallbackOperation`, `ChildContextOperation`, `MapOperation`, and test helpers to use `SerDesRunner`. +6. Add invocation-scoped deserialization caching keyed by entity, payload kind, type, and serialized data hash. +7. Update exception serialization and deserialization paths to set `SerDesPayloadKind.EXCEPTION` in TLS. +8. Add the `extra-filesystem-serdes` Maven module with artifact ID + `aws-durable-execution-sdk-java-extra-filesystem-serdes`, depending on the core SDK. +9. Implement `FileSystemSerDes` in `software.amazon.lambda.durable.extra.filesystem` with standalone compatibility and + string-stage modes, `ALWAYS` and `OVERFLOW` storage, `URI` and `HASH` path encodings, envelope parsing, atomic file + writes where supported by the filesystem, and clear validation errors for missing context or invalid stage input. +10. Add unit tests for pipeline ordering, reverse processing, nulls, invalid intermediate stage types, stage failures, + context construction, TLS scoping and clearing, thread-pool isolation, cache hits, cache invalidation, exception + reconstruction, malformed filesystem envelopes, and extra-module packaging. +11. Add integration tests with `LocalDurableTestRunner` for multi-stage pipelines, step results, wait-for-condition + state, invoke payload/result, child context results, map results, repeated `get()`, replay from file pointers, and + custom exception types. +12. Update README and advanced configuration docs with pipeline examples, FileSystemSerDes dependency coordinates, + filesystem configuration, and warnings about `/tmp`, S3 Files flush behavior, and EFS/S3 Files operational + requirements. ### Pros - Delivers the requested parity feature with the smallest new public API surface. - Uses an extension point customers already understand and can configure per operation. +- Makes serialization, compression, encryption, and storage independently composable without adding a storage-specific + core interface. - Keeps the first implementation in an optional `aws-durable-execution-sdk-java-extra-*` module. - Avoids committing the core SDK to a generalized offloading envelope before the storage use cases are proven. - Closest to the current JavaScript `createFileSystemSerdes` model and issue #463 wording. @@ -206,10 +355,11 @@ Root user input and output payloads should route through `SerDesRunner` so `File ### Cons - Uses serialization as a storage hook, so the name `SerDes` no longer means only object-to-string conversion. -- Forces customers who already have a custom SerDes to wrap or compose it with FileSystemSerDes. +- Requires non-codec stages to obey a string-to-string convention that the current `SerDes` type system cannot enforce. - May lead to one-off storage SerDes implementations if S3, DynamoDB, or other backends are added later. - Makes it harder for the SDK to reason separately about serialized text size, offloaded payload references, and storage lifecycle. - The SDK treats the checkpoint envelope as opaque serialized data, so lifecycle and preview behavior are owned by the SerDes implementation. +- Makes pipeline order and configuration part of checkpoint compatibility for in-flight executions. ## Approach B: Create a PayloadOffloader Interface @@ -396,23 +546,25 @@ This approach gives the SDK one consistent policy for root payloads, operation r | Dimension | Approach A: Reuse SerDes | Approach B: PayloadOffloader | |-----------|--------------------------|------------------------------| -| Responsibility boundary | Combines value serialization and storage-reference creation in one implementation. | Keeps object encoding in `SerDes` and storage movement in a separate offloader. | -| User configuration | Users replace or wrap their SerDes with `FileSystemSerDes`. Operation-level SerDes selection already exists. | Users configure both a SerDes and an offloader. The SDK must define global, per-operation, and per-payload precedence. | +| Responsibility boundary | Combines value serialization and storage-reference creation in ordered, composable SerDes stages. | Keeps object encoding in `SerDes` and storage movement in a separate offloader. | +| User configuration | Users configure one SerDes or a `ComposableSerDes` pipeline. Operation-level SerDes selection replaces the whole pipeline. | Users configure both a SerDes and an offloader. The SDK must define global, per-operation, and per-payload precedence. | | Parity with JS issue | Closest to the current JavaScript `createFileSystemSerdes` model and issue #463 wording. | Diverges from JavaScript naming and shape, though it may be architecturally cleaner for Java. | -| Core SDK changes | Requires `SerDesContext` TLS because the existing SerDes contract has no context parameter. | Requires a new core extension point, envelope type, config surface, payload pipeline, and migration story. | -| Applicability | Only payloads using the filesystem SerDes are offloaded. Other SerDes implementations must implement their own offload behavior or be wrapped. | Any SerDes output can be offloaded uniformly after serialization. Users can combine Jackson/custom SerDes with any offloader. | +| Core SDK changes | Adds the compatible `SerDes.then(...)` default method, `ComposableSerDes`, and `SerDesContext` TLS because the existing serialization methods have no context parameter. | Requires a new core extension point, envelope type, config surface, payload pipeline, and migration story. | +| Applicability | Any compatible SerDes stages can be chained, but storage stages still own their envelope and lifecycle behavior. | Any SerDes output can be offloaded uniformly after serialization. Users can combine Jackson/custom SerDes with any offloader. | | Envelope ownership | FileSystemSerDes owns the checkpoint envelope (`data`, `file`, preview), so the SDK treats it as opaque serialized data. | SDK owns the checkpoint/offload envelope and must guarantee it composes with replay, errors, callbacks, and test utilities. | | Caching | SDK can cache deserialized values, but FileSystemSerDes may still do file reads internally unless cache hits happen before SerDes. | SDK can cache at both layers: resolved offloaded payload text and final deserialized object. | | Exception handling | Works if every exception serialization path is routed through SerDes with `SerDesPayloadKind.EXCEPTION`. | Works uniformly because exception `errorData` is another serialized payload that can be offloaded after SerDes. | -| Third-party storage | Filesystem-specific; S3/DynamoDB would likely become more SerDes wrappers or extra packages. | Natural home for multiple storage backends: filesystem, S3, DynamoDB, EFS, S3 Files, or custom customer storage. | +| Third-party storage | S3, DynamoDB, and other backends can be implemented as additional reversible SerDes stages in extra packages. | Natural home for multiple storage backends: filesystem, S3, DynamoDB, EFS, S3 Files, or custom customer storage. | | Immediate delivery risk | Lower. Builds on existing customization point. | Higher. Requires new API and more runtime integration. | | Long-term design risk | Higher. Blurs SerDes semantics and may accumulate storage behavior in serializers. | Lower if offloading grows into a first-class feature, but higher if this remains a one-off filesystem parity feature. | ## Decision -Adopt **Approach A: Reuse SerDes for Offload**. It delivers JavaScript parity with the smallest compatible public API -change, keeps filesystem behavior in an optional artifact, and leaves the existing `SerDes` interface unchanged. -Approach B remains a possible future direction if payload offloading grows into a general multi-backend capability. +Adopt **Approach A: Reuse SerDes for Offload**, extended with a core `ComposableSerDes` pipeline. It delivers +JavaScript parity, keeps filesystem behavior in an optional artifact, leaves the existing `SerDes` methods unchanged, +and lets customers assemble value encoding, compression, encryption, and storage as independently reusable stages. +Approach B remains a possible future direction if payload offloading grows into a general multi-backend capability +that requires SDK-owned storage envelopes and lifecycle policy. ## Other Alternatives Considered @@ -426,7 +578,10 @@ Rejected. Filesystem-backed storage is optional, storage-specific functionality. ### Add context-aware SerDes overloads -Rejected for Approach A. Explicit overloads are more discoverable, but they expand the public `SerDes` interface and force context into every custom implementation's method surface. Approach A uses `SerDesContext` TLS only to keep the existing `SerDes` contract unchanged. Approach B does not need SerDes TLS because `PayloadOffloader` receives `PayloadOffloadContext` explicitly. +Rejected for Approach A. Explicit overloads are more discoverable, but they force context into every custom +implementation's serialization method surface. Approach A uses `SerDesContext` TLS to keep the existing +`serialize`/`deserialize` signatures unchanged. Approach B does not need SerDes TLS because `PayloadOffloader` receives +`PayloadOffloadContext` explicitly. ### Make SerDes async @@ -452,10 +607,12 @@ Rejected. The backend request/response envelope is protocol data. User payload c Positive: -- Both approaches enable filesystem-backed payload storage without changing the existing `SerDes` interface. +- Both approaches enable filesystem-backed payload storage without changing the existing `serialize`/`deserialize` + signatures. - Filesystem-specific functionality stays out of the core SDK artifact. - The repository gets a repeatable `aws-durable-execution-sdk-java-extra-xxx` artifact pattern for optional packages. - Custom payload implementations get enough context to use external storage safely. +- Customers can compose reusable SerDes stages without creating a bespoke wrapper for each combination. - Blocking payload work is isolated from user operation and SDK coordination threads. - Repeated file reads and repeated object reconstruction can be reduced within an invocation. - User exception type reconstruction remains supported. @@ -465,6 +622,9 @@ Negative: - Adds executor, context, and caching machinery that must stay deterministic. - Adds at least one Maven module and published artifact to release and document. - Approach A requires thread-local SerDes context because the existing `SerDes` methods do not accept context. +- Approach A relies on a documented string-stage convention that is validated at runtime rather than by Java's type + system. +- Pipeline ordering and stage configuration must remain compatible with persisted checkpoints. - Repeated `get()` calls may return the same object instance in one invocation. - Filesystem-backed storage introduces operational durability requirements outside the SDK's control. - Approach A risks overloading the meaning of SerDes. @@ -472,8 +632,8 @@ Negative: Deferred: -- Choosing whether payload offloading is a first-class SDK concept or a parity feature implemented through SerDes. -- A fully async Java SerDes or payload pipeline contract. +- A generalized SDK-owned payload-offloading abstraction beyond the SerDes pipeline. +- A fully async Java SerDes or async pipeline contract. - A separate, explicitly dangerous protocol-envelope customization API. - File cleanup, retention policies, and lifecycle management for offloaded payloads. From 46646669185567891ab3a058ae35a60440180e32 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Mon, 24 Aug 2026 21:56:00 +0000 Subject: [PATCH 03/56] feat: add retryable SerDes decorator --- docs/adr/005-filesystem-serdes.md | 66 +++++-- docs/advanced/configuration.md | 10 +- docs/advanced/error-handling.md | 1 + docs/design.md | 6 +- extra-filesystem-serdes/README.md | 17 ++ .../extra/filesystem/FileSystemSerDes.java | 14 +- .../filesystem/FileSystemSerDesTest.java | 9 + .../exception/RetryableSerDesException.java | 19 ++ .../lambda/durable/serde/RetrySerDes.java | 116 ++++++++++++ .../lambda/durable/serde/RetrySerDesTest.java | 179 ++++++++++++++++++ 10 files changed, 421 insertions(+), 16 deletions(-) create mode 100644 sdk/src/main/java/software/amazon/lambda/durable/exception/RetryableSerDesException.java create mode 100644 sdk/src/main/java/software/amazon/lambda/durable/serde/RetrySerDes.java create mode 100644 sdk/src/test/java/software/amazon/lambda/durable/serde/RetrySerDesTest.java diff --git a/docs/adr/005-filesystem-serdes.md b/docs/adr/005-filesystem-serdes.md index 46b36f95d..04aea0d98 100644 --- a/docs/adr/005-filesystem-serdes.md +++ b/docs/adr/005-filesystem-serdes.md @@ -212,6 +212,43 @@ var securePayloads = new JacksonSerDes() .then(fileSystemStage); ``` +### Retryable SerDes stages + +Transient failures are explicit. `RetryableSerDesException` extends `SerDesException` and marks a failure that may +succeed when attempted again. `RetrySerDes` decorates another SerDes instance and applies an existing `RetryStrategy`: + +```java +var resilientFileSystemStage = new RetrySerDes( + fileSystemStage, + RetryStrategies.exponentialBackoff( + 3, + Duration.ofSeconds(1), + Duration.ofSeconds(5), + 2.0, + JitterStrategy.FULL)); + +var serDes = new JacksonSerDes().then(resilientFileSystemStage); +``` + +Retry rules: + +- Only `RetryableSerDesException` is retried. Ordinary `SerDesException` and other failures propagate immediately. +- `RetryStrategy.makeRetryDecision(error, attempt)` receives the transient failure and a 1-based attempt number. +- When the strategy returns `fail`, `RetrySerDes` rethrows the last `RetryableSerDesException`. +- The same read-only `SerDesContext` remains installed for every attempt because retrying happens inside the original + `SerDesRunner` task. +- A retry delay blocks only the dedicated SerDes executor thread. It is an in-invocation infrastructure retry, not a + durable wait or checkpoint. Strategies must therefore use short, bounded delays that fit within the Lambda + invocation timeout. +- If the invocation is interrupted or times out, replay may execute the SerDes pipeline again. Any stage with side + effects must use stable addressing and idempotent writes. +- `RetrySerDes` can wrap an individual stage or the complete pipeline. Wrapping the smallest transient stage avoids + repeating deterministic encoding, compression, or encryption work. +- Pipeline error decoration must preserve retryability: a stage-level `RetryableSerDesException` must remain that type, + with stage metadata added to its message or cause, so an enclosing `RetrySerDes` can recognize it. +- Filesystem read and write `IOException`s are retryable. Malformed envelopes, invalid paths, unsupported types, and + delegate encoding errors are permanent. + Storage modes: | Mode | Behavior | @@ -319,26 +356,29 @@ Root user input and output payloads should route through `SerDesRunner` so `File `SerDes` methods unchanged. 2. Add the binary-compatible `SerDes.then(...)` default method and `ComposableSerDes` with immutable stage ordering, forward serialization, reverse deserialization, null short-circuiting, and stage-aware errors. -3. Add `SerDesRunner` and a `SerDesExecutor` default pool. Add +3. Add `RetryableSerDesException` and `RetrySerDes`, reusing `RetryStrategy` for bounded in-invocation retries. +4. Add `SerDesRunner` and a `SerDesExecutor` default pool. Add `DurableConfig.withSerDesExecutorService(...)` and validation. -4. Update root input/output handling in `DurableExecutor` to run user payload SerDes through `SerDesRunner` while +5. Update root input/output handling in `DurableExecutor` to run user payload SerDes through `SerDesRunner` while leaving `DurableInputOutputSerDes` internal. -5. Update `SerializableDurableOperation`, `InvokeOperation`, `StepOperation`, `WaitForConditionOperation`, +6. Update `SerializableDurableOperation`, `InvokeOperation`, `StepOperation`, `WaitForConditionOperation`, `CallbackOperation`, `ChildContextOperation`, `MapOperation`, and test helpers to use `SerDesRunner`. -6. Add invocation-scoped deserialization caching keyed by entity, payload kind, type, and serialized data hash. -7. Update exception serialization and deserialization paths to set `SerDesPayloadKind.EXCEPTION` in TLS. -8. Add the `extra-filesystem-serdes` Maven module with artifact ID +7. Add invocation-scoped deserialization caching keyed by entity, payload kind, type, and serialized data hash. +8. Update exception serialization and deserialization paths to set `SerDesPayloadKind.EXCEPTION` in TLS. +9. Add the `extra-filesystem-serdes` Maven module with artifact ID `aws-durable-execution-sdk-java-extra-filesystem-serdes`, depending on the core SDK. -9. Implement `FileSystemSerDes` in `software.amazon.lambda.durable.extra.filesystem` with standalone compatibility and +10. Implement `FileSystemSerDes` in `software.amazon.lambda.durable.extra.filesystem` with standalone compatibility and string-stage modes, `ALWAYS` and `OVERFLOW` storage, `URI` and `HASH` path encodings, envelope parsing, atomic file - writes where supported by the filesystem, and clear validation errors for missing context or invalid stage input. -10. Add unit tests for pipeline ordering, reverse processing, nulls, invalid intermediate stage types, stage failures, - context construction, TLS scoping and clearing, thread-pool isolation, cache hits, cache invalidation, exception - reconstruction, malformed filesystem envelopes, and extra-module packaging. -11. Add integration tests with `LocalDurableTestRunner` for multi-stage pipelines, step results, wait-for-condition + writes where supported by the filesystem, retryable I/O failures, and clear validation errors for missing context or + invalid stage input. +11. Add unit tests for pipeline ordering, reverse processing, nulls, invalid intermediate stage types, stage failures, + retry selection, exhaustion, delay handling, interruption, context construction, TLS scoping and clearing, + thread-pool isolation, cache hits, cache invalidation, exception reconstruction, malformed filesystem envelopes, + and extra-module packaging. +12. Add integration tests with `LocalDurableTestRunner` for multi-stage pipelines, step results, wait-for-condition state, invoke payload/result, child context results, map results, repeated `get()`, replay from file pointers, and custom exception types. -12. Update README and advanced configuration docs with pipeline examples, FileSystemSerDes dependency coordinates, +13. Update README and advanced configuration docs with pipeline and retry examples, FileSystemSerDes dependency coordinates, filesystem configuration, and warnings about `/tmp`, S3 Files flush behavior, and EFS/S3 Files operational requirements. diff --git a/docs/advanced/configuration.md b/docs/advanced/configuration.md index 5f9f826d7..6bf1cef58 100644 --- a/docs/advanced/configuration.md +++ b/docs/advanced/configuration.md @@ -59,8 +59,12 @@ var fileSystemSerDes = FileSystemSerDes.builder(Path.of("/mnt/efs/durable-payloa .previewGenerator(value -> Map.of("type", value.getClass().getSimpleName())) .build(); +var retryingSerDes = new RetrySerDes( + fileSystemSerDes, + RetryStrategies.fixedDelay(3, Duration.ofSeconds(1))); + return DurableConfig.builder() - .withSerDes(fileSystemSerDes) + .withSerDes(retryingSerDes) .build(); ``` @@ -68,6 +72,10 @@ return DurableConfig.builder() approaches the 256 KB service limit. `URI` produces readable escaped paths; `HASH` produces fixed-length SHA-256 path segments. +`RetrySerDes` retries only failures marked with `RetryableSerDesException`. Filesystem read and write I/O use this +marker; malformed envelopes and delegate encoding failures fail immediately. Backoff occurs on the dedicated SerDes +executor within the current Lambda invocation, so use short, bounded retry strategies. + Do not use Lambda's ephemeral `/tmp` directory: replay can run in a different execution environment. Use a durable, shared mount such as EFS. S3 Files can have delayed synchronization, so a runtime crash before the mount flushes may lose recent writes; use it only when that tradeoff is acceptable. The SDK does not delete offloaded files, so configure diff --git a/docs/advanced/error-handling.md b/docs/advanced/error-handling.md index e81deeb61..ddd7552b4 100644 --- a/docs/advanced/error-handling.md +++ b/docs/advanced/error-handling.md @@ -12,6 +12,7 @@ Error RuntimeException └── DurableExecutionException - General durable exception ├── SerDesException - Serialization and deserialization exception. + │ └── RetryableSerDesException - Transient SerDes failure eligible for RetrySerDes. ├── UnrecoverableDurableExecutionException - Execution cannot be recovered. The durable execution will be immediately terminated. │ ├── NonDeterministicExecutionException - Code changed between original execution and replay. Fix code to maintain determinism; don't change step order/names. │ └── IllegalDurableOperationException - An illegal operation was detected. The execution will be immediately terminated. diff --git a/docs/design.md b/docs/design.md index eaba54af0..ed3f94412 100644 --- a/docs/design.md +++ b/docs/design.md @@ -349,6 +349,7 @@ software.amazon.lambda.durable ├── serde/ │ ├── SerDes # Interface │ ├── JacksonSerDes # Jackson impl +│ ├── RetrySerDes # Retry decorator for transient SerDes failures │ └── AwsSdkV2Module # SDK type support │ └── exception/ @@ -372,7 +373,8 @@ software.amazon.lambda.durable ├── ChildContextFailedException ├── MapIterationFailedException ├── ParallelBranchFailedException - └── SerDesException + ├── SerDesException + └── RetryableSerDesException ``` --- @@ -463,6 +465,7 @@ sequenceDiagram ``` DurableExecutionException (base) ├── SerDesException # Serialization error +│ └── RetryableSerDesException # Transient SerDes error ├── UnrecoverableDurableExecutionException # Execution cannot be recovered │ ├── NonDeterministicExecutionException # Replay mismatch │ └── IllegalDurableOperationException # Illegal operation detected @@ -503,6 +506,7 @@ SuspendExecutionException # Internal: triggers suspension (not | `NonDeterministicExecutionException` | Replay finds different operation than expected | Bug in handler (non-deterministic code) | | `IllegalDurableOperationException` | Illegal operation detected | Bug in handler | | `SerDesException` | Jackson fails to serialize/deserialize | Fix data model or custom SerDes | +| `RetryableSerDesException` | Transient SerDes or payload storage failure | Wrap the SerDes with `RetrySerDes` and a bounded retry strategy | --- diff --git a/extra-filesystem-serdes/README.md b/extra-filesystem-serdes/README.md index fbdf9c4ec..90ef5569e 100644 --- a/extra-filesystem-serdes/README.md +++ b/extra-filesystem-serdes/README.md @@ -38,6 +38,23 @@ An optional preview can remain visible in the checkpoint envelope: .previewGenerator(value -> Map.of("summary", "order payload")) ``` +## Retries + +Filesystem read and write I/O failures are reported as `RetryableSerDesException`. Wrap the filesystem SerDes with +`RetrySerDes` to apply any SDK `RetryStrategy`: + +```java +var serDes = new RetrySerDes( + FileSystemSerDes.builder(Path.of("/mnt/efs/durable-payloads")) + .storageMode(FileSystemStorageMode.ALWAYS) + .build(), + RetryStrategies.fixedDelay(3, Duration.ofSeconds(1))); +``` + +Only `RetryableSerDesException` is retried. Malformed envelopes, invalid paths, and delegate encoding failures are +treated as permanent. Retry delays run on the dedicated SerDes executor and consume time in the current Lambda +invocation, so keep attempts and delays bounded. + ## Storage requirements Do not use Lambda's ephemeral `/tmp` directory. Durable replay may run in another execution environment where that file diff --git a/extra-filesystem-serdes/src/main/java/software/amazon/lambda/durable/extra/filesystem/FileSystemSerDes.java b/extra-filesystem-serdes/src/main/java/software/amazon/lambda/durable/extra/filesystem/FileSystemSerDes.java index 8d9365e4a..488fd7509 100644 --- a/extra-filesystem-serdes/src/main/java/software/amazon/lambda/durable/extra/filesystem/FileSystemSerDes.java +++ b/extra-filesystem-serdes/src/main/java/software/amazon/lambda/durable/extra/filesystem/FileSystemSerDes.java @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 package software.amazon.lambda.durable.extra.filesystem; +import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import java.io.IOException; @@ -18,6 +19,7 @@ import java.util.function.Function; import java.util.regex.Pattern; import software.amazon.lambda.durable.TypeToken; +import software.amazon.lambda.durable.exception.RetryableSerDesException; import software.amazon.lambda.durable.exception.SerDesException; import software.amazon.lambda.durable.serde.JacksonSerDes; import software.amazon.lambda.durable.serde.SerDes; @@ -74,8 +76,12 @@ public String serialize(Object value) { return preview == null ? ENVELOPE_MAPPER.writeValueAsString(Map.of("file", file.toString())) : ENVELOPE_MAPPER.writeValueAsString(Map.of("file", file.toString(), "preview", preview)); + } catch (JsonProcessingException e) { + throw new SerDesException( + "Failed to encode filesystem payload envelope for entity '" + context.entityId() + "'", e); } catch (IOException e) { - throw new SerDesException("Failed to store filesystem payload for entity '" + context.entityId() + "'", e); + throw new RetryableSerDesException( + "Failed to store filesystem payload for entity '" + context.entityId() + "'", e); } } @@ -105,6 +111,12 @@ public T deserialize(String data, TypeToken typeToken) { serialized = Files.readString(file, StandardCharsets.UTF_8); } return delegate.deserialize(serialized, typeToken); + } catch (JsonProcessingException e) { + throw new SerDesException( + "Failed to decode filesystem payload envelope for entity '" + context.entityId() + "'", e); + } catch (IOException e) { + throw new RetryableSerDesException( + "Failed to load filesystem payload for entity '" + context.entityId() + "'", e); } catch (SerDesException e) { throw e; } catch (Exception e) { diff --git a/extra-filesystem-serdes/src/test/java/software/amazon/lambda/durable/extra/filesystem/FileSystemSerDesTest.java b/extra-filesystem-serdes/src/test/java/software/amazon/lambda/durable/extra/filesystem/FileSystemSerDesTest.java index 28c1b2762..fa6c95c9c 100644 --- a/extra-filesystem-serdes/src/test/java/software/amazon/lambda/durable/extra/filesystem/FileSystemSerDesTest.java +++ b/extra-filesystem-serdes/src/test/java/software/amazon/lambda/durable/extra/filesystem/FileSystemSerDesTest.java @@ -4,6 +4,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -17,6 +18,7 @@ import org.junit.jupiter.api.io.TempDir; import software.amazon.awssdk.services.lambda.model.OperationType; import software.amazon.lambda.durable.TypeToken; +import software.amazon.lambda.durable.exception.RetryableSerDesException; import software.amazon.lambda.durable.exception.SerDesException; import software.amazon.lambda.durable.model.OperationSubType; import software.amazon.lambda.durable.serde.SerDesContext; @@ -108,6 +110,13 @@ void rejectsCallsWithoutSdkContextAndMalformedEnvelopes() { SerDesException.class, () -> runner.deserialize( serDes, "{\"file\":\"/outside/payload.json\"}", TypeToken.get(String.class), context())); + + var missingFile = basePath.resolve("missing.json").toAbsolutePath(); + var missingFileFailure = assertThrows( + SerDesException.class, + () -> runner.deserialize( + serDes, "{\"file\":\"" + missingFile + "\"}", TypeToken.get(String.class), context())); + assertInstanceOf(RetryableSerDesException.class, missingFileFailure.getCause()); } @Test diff --git a/sdk/src/main/java/software/amazon/lambda/durable/exception/RetryableSerDesException.java b/sdk/src/main/java/software/amazon/lambda/durable/exception/RetryableSerDesException.java new file mode 100644 index 000000000..be06a762f --- /dev/null +++ b/sdk/src/main/java/software/amazon/lambda/durable/exception/RetryableSerDesException.java @@ -0,0 +1,19 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.exception; + +/** + * Indicates a transient serialization or deserialization failure that may succeed when retried. + * + *

{@link software.amazon.lambda.durable.serde.RetrySerDes} retries only this exception type. Other + * {@link SerDesException} instances are treated as permanent failures. + */ +public class RetryableSerDesException extends SerDesException { + public RetryableSerDesException(String message, Throwable cause) { + super(message, cause); + } + + public RetryableSerDesException(String message) { + super(message); + } +} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/RetrySerDes.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/RetrySerDes.java new file mode 100644 index 000000000..a32fe922b --- /dev/null +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/RetrySerDes.java @@ -0,0 +1,116 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.serde; + +import java.time.Duration; +import java.util.Objects; +import java.util.concurrent.TimeUnit; +import java.util.function.Supplier; +import software.amazon.lambda.durable.TypeToken; +import software.amazon.lambda.durable.exception.RetryableSerDesException; +import software.amazon.lambda.durable.exception.SerDesException; +import software.amazon.lambda.durable.retry.RetryDecision; +import software.amazon.lambda.durable.retry.RetryStrategy; + +/** + * A SerDes decorator that retries transient failures from another {@link SerDes}. + * + *

Only {@link RetryableSerDesException} is retried. Other failures are propagated immediately. Retry delays block + * the calling thread, which is the dedicated SerDes executor thread for SDK-managed calls. + */ +public final class RetrySerDes implements SerDes { + private static final Sleeper DEFAULT_SLEEPER = delay -> { + if (delay.getSeconds() > 0) { + TimeUnit.SECONDS.sleep(delay.getSeconds()); + } + if (delay.getNano() > 0) { + TimeUnit.NANOSECONDS.sleep(delay.getNano()); + } + }; + + private final SerDes delegate; + private final RetryStrategy retryStrategy; + private final Sleeper sleeper; + + /** + * Creates a retrying SerDes decorator. + * + * @param delegate the SerDes to invoke + * @param retryStrategy strategy that controls attempts and delays + */ + public RetrySerDes(SerDes delegate, RetryStrategy retryStrategy) { + this(delegate, retryStrategy, DEFAULT_SLEEPER); + } + + RetrySerDes(SerDes delegate, RetryStrategy retryStrategy, Sleeper sleeper) { + this.delegate = Objects.requireNonNull(delegate, "delegate cannot be null"); + this.retryStrategy = Objects.requireNonNull(retryStrategy, "retryStrategy cannot be null"); + this.sleeper = Objects.requireNonNull(sleeper, "sleeper cannot be null"); + } + + @Override + public String serialize(Object value) { + return execute("serialization", () -> delegate.serialize(value)); + } + + @Override + public T deserialize(String data, TypeToken typeToken) { + return execute("deserialization", () -> delegate.deserialize(data, typeToken)); + } + + private T execute(String action, Supplier operation) { + int attempt = 1; + while (true) { + try { + return operation.get(); + } catch (RetryableSerDesException failure) { + var decision = makeRetryDecision(action, failure, attempt); + if (!decision.shouldRetry()) { + throw failure; + } + waitForRetry(action, failure, attempt, decision.delay()); + attempt++; + } + } + } + + private RetryDecision makeRetryDecision(String action, RetryableSerDesException failure, int attempt) { + try { + var decision = retryStrategy.makeRetryDecision(failure, attempt); + if (decision == null) { + throw new SerDesException( + String.format("Retry strategy returned null for SerDes %s attempt %d", action, attempt)); + } + return decision; + } catch (SerDesException e) { + throw e; + } catch (RuntimeException e) { + throw new SerDesException( + String.format("Retry strategy failed for SerDes %s attempt %d", action, attempt), e); + } + } + + private void waitForRetry(String action, RetryableSerDesException failure, int attempt, Duration delay) { + if (delay == null || delay.isNegative()) { + throw new SerDesException(String.format( + "Retry strategy returned an invalid delay for SerDes %s attempt %d", action, attempt)); + } + if (delay.isZero()) { + return; + } + try { + sleeper.sleep(delay); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + var interrupted = new SerDesException( + String.format("Interrupted while waiting to retry SerDes %s after attempt %d", action, attempt), e); + interrupted.addSuppressed(failure); + throw interrupted; + } + } + + @FunctionalInterface + interface Sleeper { + void sleep(Duration delay) throws InterruptedException; + } +} diff --git a/sdk/src/test/java/software/amazon/lambda/durable/serde/RetrySerDesTest.java b/sdk/src/test/java/software/amazon/lambda/durable/serde/RetrySerDesTest.java new file mode 100644 index 000000000..b2a6d8503 --- /dev/null +++ b/sdk/src/test/java/software/amazon/lambda/durable/serde/RetrySerDesTest.java @@ -0,0 +1,179 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.serde; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import org.junit.jupiter.api.Test; +import software.amazon.lambda.durable.TypeToken; +import software.amazon.lambda.durable.exception.RetryableSerDesException; +import software.amazon.lambda.durable.exception.SerDesException; +import software.amazon.lambda.durable.retry.RetryDecision; +import software.amazon.lambda.durable.retry.RetryStrategies; + +class RetrySerDesTest { + + @Test + void retriesSerializationWithStrategyDelays() { + var calls = new AtomicInteger(); + var strategyAttempts = new ArrayList(); + var delays = new ArrayList(); + var delegate = new SerDes() { + @Override + public String serialize(Object value) { + if (calls.incrementAndGet() < 3) { + throw new RetryableSerDesException("transient"); + } + return "serialized"; + } + + @Override + public T deserialize(String data, TypeToken typeToken) { + return null; + } + }; + var retrySerDes = new RetrySerDes( + delegate, + (error, attempt) -> { + strategyAttempts.add(attempt); + return RetryDecision.retry(Duration.ofMillis(attempt)); + }, + delays::add); + + assertEquals("serialized", retrySerDes.serialize("value")); + assertEquals(3, calls.get()); + assertEquals(List.of(1, 2), strategyAttempts); + assertEquals(List.of(Duration.ofMillis(1), Duration.ofMillis(2)), delays); + } + + @Test + void retriesDeserialization() { + var calls = new AtomicInteger(); + var delegate = new JacksonSerDes() { + @Override + public T deserialize(String data, TypeToken typeToken) { + if (calls.incrementAndGet() == 1) { + throw new RetryableSerDesException("transient"); + } + return super.deserialize(data, typeToken); + } + }; + var retrySerDes = + new RetrySerDes(delegate, (error, attempt) -> RetryDecision.retry(Duration.ZERO), delay -> {}); + + assertEquals("value", retrySerDes.deserialize("\"value\"", TypeToken.get(String.class))); + assertEquals(2, calls.get()); + } + + @Test + void doesNotRetryPermanentSerDesFailure() { + var calls = new AtomicInteger(); + var delegate = new SerDes() { + @Override + public String serialize(Object value) { + calls.incrementAndGet(); + throw new SerDesException("permanent"); + } + + @Override + public T deserialize(String data, TypeToken typeToken) { + return null; + } + }; + var retrySerDes = new RetrySerDes(delegate, (error, attempt) -> RetryDecision.retry(Duration.ZERO), delay -> { + throw new AssertionError("permanent failures must not sleep"); + }); + + var failure = assertThrows(SerDesException.class, () -> retrySerDes.serialize("value")); + assertEquals("permanent", failure.getMessage()); + assertEquals(1, calls.get()); + } + + @Test + void rejectsInvalidConfigurationAndRetryDelay() { + var delegate = new JacksonSerDes(); + + assertThrows(NullPointerException.class, () -> new RetrySerDes(null, RetryStrategies.Presets.NO_RETRY)); + assertThrows(NullPointerException.class, () -> new RetrySerDes(delegate, null)); + + var retrySerDes = new RetrySerDes( + new SerDes() { + @Override + public String serialize(Object value) { + throw new RetryableSerDesException("transient"); + } + + @Override + public T deserialize(String data, TypeToken typeToken) { + return null; + } + }, + (error, attempt) -> RetryDecision.retry(Duration.ofSeconds(-1)), + delay -> {}); + + var failure = assertThrows(SerDesException.class, () -> retrySerDes.serialize("value")); + assertTrue(failure.getMessage().contains("invalid delay")); + } + + @Test + void rethrowsLastRetryableFailureWhenRetriesAreExhausted() { + var calls = new AtomicInteger(); + var lastFailure = new AtomicReference(); + var delegate = new SerDes() { + @Override + public String serialize(Object value) { + var failure = new RetryableSerDesException("attempt-" + calls.incrementAndGet()); + lastFailure.set(failure); + throw failure; + } + + @Override + public T deserialize(String data, TypeToken typeToken) { + return null; + } + }; + var retrySerDes = new RetrySerDes(delegate, RetryStrategies.fixedDelay(2, Duration.ofSeconds(1)), delay -> {}); + + var thrown = assertThrows(RetryableSerDesException.class, () -> retrySerDes.serialize("value")); + assertSame(lastFailure.get(), thrown); + assertEquals("attempt-2", thrown.getMessage()); + assertEquals(2, calls.get()); + } + + @Test + void restoresInterruptStatusWhenBackoffIsInterrupted() { + var retryable = new RetryableSerDesException("transient"); + var delegate = new SerDes() { + @Override + public String serialize(Object value) { + throw retryable; + } + + @Override + public T deserialize(String data, TypeToken typeToken) { + return null; + } + }; + var retrySerDes = + new RetrySerDes(delegate, (error, attempt) -> RetryDecision.retry(Duration.ofSeconds(1)), delay -> { + throw new InterruptedException("stop"); + }); + + try { + var thrown = assertThrows(SerDesException.class, () -> retrySerDes.serialize("value")); + assertTrue(thrown.getMessage().contains("Interrupted")); + assertTrue(Thread.currentThread().isInterrupted()); + assertSame(retryable, thrown.getSuppressed()[0]); + } finally { + Thread.interrupted(); + } + } +} From 04455c4a0fedd9ded0d6339940558d0aa91f2e2d Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Mon, 24 Aug 2026 22:27:17 +0000 Subject: [PATCH 04/56] docs: make SerDes executor opt-in --- docs/adr/005-filesystem-serdes.md | 93 +++++++++++++++++++++++-------- 1 file changed, 69 insertions(+), 24 deletions(-) diff --git a/docs/adr/005-filesystem-serdes.md b/docs/adr/005-filesystem-serdes.md index 04aea0d98..389837c17 100644 --- a/docs/adr/005-filesystem-serdes.md +++ b/docs/adr/005-filesystem-serdes.md @@ -2,7 +2,7 @@ **Status:** Accepted — Approach A with ComposableSerDes pipeline **Date:** 2026-07-02 -**Updated:** 2026-08-24 — Added the composable SerDes pipeline design. +**Updated:** 2026-08-24 — Added the composable SerDes pipeline and optional executor design. ## Context @@ -105,6 +105,7 @@ var serDes = new JacksonSerDes().then(fileSystemStage); return DurableConfig.builder() .withSerDes(serDes) + .withSerDesExecutorService(customSerDesExecutor) .build(); ``` @@ -182,7 +183,8 @@ Pipeline rules: preserving stage order. - A `null` value at the pipeline boundary short-circuits the entire pipeline: serializing or deserializing `null` returns `null` without invoking any stage. A stage returning `null` for non-null input is an error. -- All stages execute within the same `SerDesRunner` task and observe the same read-only `SerDesContext`. +- All stages execute within the same `SerDesRunner` invocation and observe the same read-only `SerDesContext`, whether + the runner executes inline or dispatches to a configured executor. - `ComposableSerDes` is immutable. It is safe for concurrent use only when every contained stage is also safe for concurrent use, matching the existing `SerDes` requirement. - A failure must identify the stage index and implementation class in `SerDesException`; `SerDesRunner` adds durable @@ -237,9 +239,9 @@ Retry rules: - When the strategy returns `fail`, `RetrySerDes` rethrows the last `RetryableSerDesException`. - The same read-only `SerDesContext` remains installed for every attempt because retrying happens inside the original `SerDesRunner` task. -- A retry delay blocks only the dedicated SerDes executor thread. It is an in-invocation infrastructure retry, not a - durable wait or checkpoint. Strategies must therefore use short, bounded delays that fit within the Lambda - invocation timeout. +- A retry delay blocks the thread executing the SerDes call. This is the caller thread by default or a SerDes executor + thread when one is explicitly configured. It is an in-invocation infrastructure retry, not a durable wait or + checkpoint. Strategies must therefore use short, bounded delays that fit within the Lambda invocation timeout. - If the invocation is interrupted or times out, replay may execute the SerDes pipeline again. Any stage with side effects must use stable addressing and idempotent writes. - `RetrySerDes` can wrap an individual stage or the complete pipeline. Wrapping the smallest transient stage avoids @@ -298,7 +300,11 @@ to its configured value-encoding delegate. ### Threading -Add a separate executor to `DurableConfig`: +Preserve the current SDK behavior by executing SerDes inline on the calling thread by default. Do not create a default +SerDes thread pool. This avoids a queue operation, `CompletableFuture` allocation, and thread hop for ordinary in-memory +serialization such as `JacksonSerDes`. + +Customers can explicitly configure a separate executor when a SerDes performs blocking I/O or retry backoff: ```java DurableConfig.builder() @@ -306,17 +312,42 @@ DurableConfig.builder() .build(); ``` -The default should be a cached daemon thread pool named `durable-sdk-serdes-*`. +If `withSerDesExecutorService(...)` is not called, the configured executor is absent and `SerDesRunner` invokes the +pipeline synchronously on the current thread. The builder method accepts only a non-null executor; not calling it is +how customers select inline execution. If an executor is configured, `SerDesRunner` dispatches the complete pipeline +to that executor and waits for its result. The core SDK should route user payload SerDes calls through a helper, tentatively `SerDesRunner`, that: - Builds the correct `SerDesContext`. -- Sets `SerDesContext` in TLS inside the SerDes executor task. +- Executes inline when no SerDes executor is configured. +- Dispatches to the configured executor only when one is present. +- Sets `SerDesContext` in TLS on the thread that actually invokes the SerDes. - Invokes the existing `SerDes.serialize` and `SerDes.deserialize` methods. -- Clears TLS after each SerDes call. +- Restores the previous TLS value after each SerDes call, or clears it when there was no previous value. - Wraps failures in `SerDesException` with operation and payload kind metadata. -Because TLS is bound to a single Java thread, `SerDesRunner` must set `SerDesContext` inside the SerDes executor task before calling the user SerDes. It must not rely on inheritable thread-local propagation from the operation thread because cached pool threads can be reused across operations and invocations. +Equivalent execution flow: + +```java +T run(SerDesContext context, Supplier operation) { + if (serDesExecutorService == null) { + return runWithContext(context, operation); + } + return CompletableFuture + .supplyAsync(() -> runWithContext(context, operation), serDesExecutorService) + .join(); +} +``` + +Because TLS is bound to a single Java thread, `SerDesRunner` must install the context on whichever thread executes the +operation. It must not rely on inheritable thread-local propagation. Restoring the previous value also makes nested +SDK-managed SerDes calls safe in inline mode. + +Inline execution is the compatibility and low-overhead default, not a recommendation to perform blocking storage work +on operation threads. Documentation and filesystem examples should configure a SerDes executor whenever +`FileSystemSerDes`, delayed `RetrySerDes`, or another blocking stage is used. If it is omitted, I/O and retry delays +block the calling thread. ### Caching @@ -357,8 +388,8 @@ Root user input and output payloads should route through `SerDesRunner` so `File 2. Add the binary-compatible `SerDes.then(...)` default method and `ComposableSerDes` with immutable stage ordering, forward serialization, reverse deserialization, null short-circuiting, and stage-aware errors. 3. Add `RetryableSerDesException` and `RetrySerDes`, reusing `RetryStrategy` for bounded in-invocation retries. -4. Add `SerDesRunner` and a `SerDesExecutor` default pool. Add - `DurableConfig.withSerDesExecutorService(...)` and validation. +4. Add `SerDesRunner` with inline execution by default and optional dispatch through + `DurableConfig.withSerDesExecutorService(...)`. Do not create a default SerDes pool. 5. Update root input/output handling in `DurableExecutor` to run user payload SerDes through `SerDesRunner` while leaving `DurableInputOutputSerDes` internal. 6. Update `SerializableDurableOperation`, `InvokeOperation`, `StepOperation`, `WaitForConditionOperation`, @@ -372,9 +403,9 @@ Root user input and output payloads should route through `SerDesRunner` so `File writes where supported by the filesystem, retryable I/O failures, and clear validation errors for missing context or invalid stage input. 11. Add unit tests for pipeline ordering, reverse processing, nulls, invalid intermediate stage types, stage failures, - retry selection, exhaustion, delay handling, interruption, context construction, TLS scoping and clearing, - thread-pool isolation, cache hits, cache invalidation, exception reconstruction, malformed filesystem envelopes, - and extra-module packaging. + retry selection, exhaustion, delay handling, interruption, context construction, TLS scoping and restoration, + inline execution, configured thread-pool isolation, cache hits, cache invalidation, exception reconstruction, + malformed filesystem envelopes, and extra-module packaging. 12. Add integration tests with `LocalDurableTestRunner` for multi-stage pipelines, step results, wait-for-condition state, invoke payload/result, child context results, map results, repeated `get()`, replay from file pointers, and custom exception types. @@ -386,6 +417,7 @@ Root user input and output payloads should route through `SerDesRunner` so `File - Delivers the requested parity feature with the smallest new public API surface. - Uses an extension point customers already understand and can configure per operation. +- Preserves inline SerDes execution by default, avoiding new thread-hop overhead for existing applications. - Makes serialization, compression, encryption, and storage independently composable without adding a storage-specific core interface. - Keeps the first implementation in an optional `aws-durable-execution-sdk-java-extra-*` module. @@ -505,7 +537,8 @@ The SDK owns the checkpoint/offload envelope. Storage implementations own only t ### Threading -Use a separate executor for blocking payload I/O. This can be the same configured executor as SerDes work or a distinct executor if the team wants independent tuning: +Keep ordinary SerDes work inline by default for compatibility. Blocking payload I/O can use the explicitly configured +SerDes executor or a distinct offload executor if the team wants independent tuning: ```java DurableConfig.builder() @@ -514,9 +547,11 @@ DurableConfig.builder() .build(); ``` -If a single executor is preferred, name it according to the broader responsibility, for example `durable-sdk-payload-*`. +No executor should be created by default. If a single explicitly configured executor is preferred, name it according +to the broader responsibility, for example `durable-sdk-payload-*`. -Because filesystem/S3/DynamoDB offloading can block, offload work should not run on the user operation executor or the SDK internal executor. +Because filesystem/S3/DynamoDB offloading can block, production configurations should provide an executor rather than +run that work inline. Offload work must never use the internal SDK executor. ### Caching @@ -560,7 +595,9 @@ This approach gives the SDK one consistent policy for root payloads, operation r 8. Add offloaded payload caching and deserialized object caching. 9. Add the `extra-filesystem-offloader` Maven module with artifact ID `aws-durable-execution-sdk-java-extra-filesystem-offloader`, depending on the core SDK. 10. Implement `FileSystemPayloadOffloader` in `software.amazon.lambda.durable.extra.filesystem` with `ALWAYS` and `OVERFLOW` modes, `URI` and `HASH` path encodings, atomic file writes where supported by the filesystem, and clear validation errors for missing context. -11. Add unit tests for offload envelope compatibility, precedence rules, thread-pool isolation, cache hits, cache invalidation, exception reconstruction, malformed references, and extra-module packaging. +11. Add unit tests for offload envelope compatibility, precedence rules, inline execution, explicitly configured + thread-pool isolation, cache hits, cache invalidation, exception reconstruction, malformed references, and + extra-module packaging. 12. Add integration tests with `LocalDurableTestRunner` for step results, wait-for-condition state, invoke payload/result, child context results, map results, repeated `get()`, replay from external references, and custom exception types. 13. Update README and advanced configuration docs with offloader dependency coordinates, filesystem offloader examples, and warnings about `/tmp`, S3 Files flush behavior, and EFS/S3 Files operational requirements. @@ -625,11 +662,16 @@ implementation's serialization method surface. Approach A uses `SerDesContext` T ### Make SerDes async -Deferred. The TypeScript SDK uses async SerDes because file and service I/O are naturally async in Node.js. Java can isolate blocking work with dedicated executors while preserving synchronous user-facing interfaces. A future major version can revisit `CompletionStage` and `CompletionStage` if there is a stronger need. +Deferred. The TypeScript SDK uses async SerDes because file and service I/O are naturally async in Node.js. Java can +optionally isolate blocking work with an explicitly configured executor while preserving synchronous user-facing +interfaces and inline defaults. A future major version can revisit `CompletionStage` and `CompletionStage` +if there is a stronger need. -### Run payload storage on the user executor +### Always run payload storage inline -Rejected. Filesystem-backed storage can block on mounted storage. Running that work on the user executor can starve user operation threads and make unrelated steps appear stuck. +Rejected as the only execution mode. Inline execution remains the default for backward compatibility and low overhead, +but filesystem-backed storage and delayed retries can block the calling thread. Customers should explicitly configure a +SerDes executor for those stages. ### Run payload storage on the internal SDK executor @@ -653,18 +695,21 @@ Positive: - The repository gets a repeatable `aws-durable-execution-sdk-java-extra-xxx` artifact pattern for optional packages. - Custom payload implementations get enough context to use external storage safely. - Customers can compose reusable SerDes stages without creating a bespoke wrapper for each combination. -- Blocking payload work is isolated from user operation and SDK coordination threads. +- Blocking payload work can be isolated from user operation threads with an explicitly configured SerDes executor and + never runs on the SDK coordination executor. - Repeated file reads and repeated object reconstruction can be reduced within an invocation. - User exception type reconstruction remains supported. Negative: -- Adds executor, context, and caching machinery that must stay deterministic. +- Adds optional executor, context, and caching machinery that must stay deterministic. - Adds at least one Maven module and published artifact to release and document. - Approach A requires thread-local SerDes context because the existing `SerDes` methods do not accept context. - Approach A relies on a documented string-stage convention that is validated at runtime rather than by Java's type system. - Pipeline ordering and stage configuration must remain compatible with persisted checkpoints. +- The inline default means filesystem I/O and retry delays block the caller when customers do not configure a SerDes + executor. - Repeated `get()` calls may return the same object instance in one invocation. - Filesystem-backed storage introduces operational durability requirements outside the SDK's control. - Approach A risks overloading the meaning of SerDes. From 34717213733d9f2f8f879b19d4e03c11b995b99e Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Mon, 24 Aug 2026 23:02:52 +0000 Subject: [PATCH 05/56] feat: complete composable filesystem SerDes --- .github/scripts/maven_publish.sh | 1 + .github/workflows/build.yml | 2 + .github/workflows/e2e-tests.yml | 2 + .github/workflows/publish_maven.yml | 1 + RELEASE.md | 9 +- coverage-report/pom.xml | 5 + docs/adr/005-filesystem-serdes.md | 46 ++- docs/advanced/configuration.md | 50 ++- docs/design.md | 19 +- extra-filesystem-serdes/README.md | 62 ++-- .../extra/filesystem/FileSystemSerDes.java | 343 ++++++++++++++---- .../filesystem/FileSystemSerDesTest.java | 189 ++++++++-- .../FileSystemSerDesIntegrationTest.java | 276 +++++++++++++- .../durable/testing/AsyncExecution.java | 6 +- .../testing/CloudDurableTestRunner.java | 80 +++- .../testing/LocalDurableTestRunner.java | 10 +- .../testing/cloud/HistoryEventProcessor.java | 104 +++++- .../local/LocalMemoryExecutionClient.java | 5 + .../cloud/HistoryEventProcessorTest.java | 122 +++++++ .../amazon/lambda/durable/DurableConfig.java | 31 +- .../operation/BaseDurableOperation.java | 4 +- .../durable/operation/StepOperation.java | 10 +- .../durable/serde/ComposableSerDes.java | 194 ++++++++++ .../lambda/durable/serde/RetrySerDes.java | 2 +- .../amazon/lambda/durable/serde/SerDes.java | 13 + .../lambda/durable/serde/SerDesContext.java | 6 +- .../lambda/durable/serde/SerDesRunner.java | 89 +++-- .../lambda/durable/DurableConfigTest.java | 14 +- .../durable/operation/StepOperationTest.java | 35 ++ .../durable/serde/ComposableSerDesTest.java | 146 ++++++++ .../durable/serde/SerDesRunnerTest.java | 150 ++++++++ 31 files changed, 1797 insertions(+), 229 deletions(-) create mode 100644 sdk-testing/src/test/java/software/amazon/lambda/durable/testing/cloud/HistoryEventProcessorTest.java create mode 100644 sdk/src/main/java/software/amazon/lambda/durable/serde/ComposableSerDes.java create mode 100644 sdk/src/test/java/software/amazon/lambda/durable/serde/ComposableSerDesTest.java diff --git a/.github/scripts/maven_publish.sh b/.github/scripts/maven_publish.sh index 655ad5c69..c8928c348 100644 --- a/.github/scripts/maven_publish.sh +++ b/.github/scripts/maven_publish.sh @@ -43,6 +43,7 @@ echo "settings.xml written." echo "=== Step 3: Upload to Sonatype Central Portal ===" mvn clean deploy -s "${SETTINGS_FILE}" -pl sdk -P publishing -DskipTests --no-transfer-progress +mvn clean deploy -s "${SETTINGS_FILE}" -pl extra-filesystem-serdes -P publishing -DskipTests --no-transfer-progress mvn clean deploy -s "${SETTINGS_FILE}" -pl sdk-testing -P publishing -DskipTests --no-transfer-progress mvn clean deploy -s "${SETTINGS_FILE}" -pl otel-plugin -P publishing -DskipTests --no-transfer-progress diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 26e154242..9ccde6759 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -26,6 +26,7 @@ on: - '.github/workflows/ai-pr-review.yml' - '.github/prompts/ai-pr-review.md' - 'sdk/**' + - 'extra-filesystem-serdes/**' - 'sdk-testing/**' - 'sdk-integration-tests/**' - 'examples/**' @@ -38,6 +39,7 @@ on: - '.github/workflows/ai-pr-review.yml' - '.github/prompts/ai-pr-review.md' - 'sdk/**' + - 'extra-filesystem-serdes/**' - 'sdk-testing/**' - 'sdk-integration-tests/**' - 'examples/**' diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index 8d9f9d4cd..42367a5a2 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -8,6 +8,7 @@ on: paths: - '.github/**' # for testing Github Actions - 'sdk/**' + - 'extra-filesystem-serdes/**' - 'sdk-testing/**' - 'sdk-integration-tests/**' - 'examples/**' @@ -18,6 +19,7 @@ on: paths: - '.github/**' - 'sdk/**' + - 'extra-filesystem-serdes/**' - 'sdk-testing/**' - 'sdk-integration-tests/**' - 'examples/**' diff --git a/.github/workflows/publish_maven.yml b/.github/workflows/publish_maven.yml index a664ca282..547f6dbe9 100644 --- a/.github/workflows/publish_maven.yml +++ b/.github/workflows/publish_maven.yml @@ -90,6 +90,7 @@ jobs: run: | gh release upload "$RELEASE_TAG" \ "sdk/target/aws-durable-execution-sdk-java-${RELEASE_VERSION}.jar" \ + "extra-filesystem-serdes/target/aws-durable-execution-sdk-java-extra-filesystem-serdes-${RELEASE_VERSION}.jar" \ "sdk-testing/target/aws-durable-execution-sdk-java-testing-${RELEASE_VERSION}.jar" \ "otel-plugin/target/aws-durable-execution-sdk-java-plugin-otel-${RELEASE_VERSION}.jar" \ --clobber diff --git a/RELEASE.md b/RELEASE.md index 90c347ff1..92dbae9e2 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -43,9 +43,9 @@ The publication workflow: 1. Verifies that the tag is a semantic version, points to a commit on the default branch, and matches the Maven version in the tagged POM. -2. Builds, signs, and uploads the SDK, testing library, and OpenTelemetry plugin - to Sonatype Central Portal. -3. Uploads the three JARs to the existing GitHub release. +2. Builds, signs, and uploads the SDK, filesystem SerDes extension, testing + library, and OpenTelemetry plugin to Sonatype Central Portal. +3. Uploads the four JARs to the existing GitHub release. 4. Opens a pull request for the next development version. A final release increments the patch version, so `2.1.1` produces `2.1.2-SNAPSHOT`. A prerelease keeps the same base version, so `2.1.1-rc1` produces @@ -56,7 +56,8 @@ After **Publish Maven Release** succeeds: 1. Open [Publishing Deployments](https://central.sonatype.com/publishing/deployments) in Sonatype Central Portal. 2. Find the deployments for the release version and verify that they contain - the expected SDK, testing library, and OpenTelemetry plugin artifacts. + the expected SDK, filesystem SerDes extension, testing library, and + OpenTelemetry plugin artifacts. 3. Click **Publish** for each deployment and wait for publication to complete. The workflow uses `autoPublish=false`, so this manual action is required. 4. Confirm that the GitHub release contains the expected JARs and that the diff --git a/coverage-report/pom.xml b/coverage-report/pom.xml index e820594e1..4e860d478 100644 --- a/coverage-report/pom.xml +++ b/coverage-report/pom.xml @@ -22,6 +22,11 @@ aws-durable-execution-sdk-java ${project.version} + + software.amazon.lambda.durable + aws-durable-execution-sdk-java-extra-filesystem-serdes + ${project.version} + software.amazon.lambda.durable aws-durable-execution-sdk-java-testing diff --git a/docs/adr/005-filesystem-serdes.md b/docs/adr/005-filesystem-serdes.md index 389837c17..be7681251 100644 --- a/docs/adr/005-filesystem-serdes.md +++ b/docs/adr/005-filesystem-serdes.md @@ -72,7 +72,7 @@ public record SerDesContext( OperationSubType operationSubType, Integer attempt) { public static SerDesContext getCurrentContext() { - return SerDesContextHolder.getCurrentContext(); + return SerDesContextHolder.get(); } } ``` @@ -120,6 +120,8 @@ public final class ComposableSerDes implements SerDes { public static Builder builder(SerDes valueCodec); + public SerDes getValueCodec(); + public ComposableSerDes then(SerDes stage); public static final class Builder { @@ -268,15 +270,27 @@ Path encodings: Envelope format: ```json -{"data":""} -{"file":""} -{"file":"","preview":{ "...": "..." }} +{"__durable_execution_filesystem_serdes":1,"data":""} +{"__durable_execution_filesystem_serdes":1,"file":""} +{"__durable_execution_filesystem_serdes":1,"file":"","preview":{ "...": "..." }} ``` `FileSystemSerDes` must reject calls when `SerDesContext.getCurrentContext()` is `null` or does not include `durableExecutionArn` and `entityId`. When used as a pipeline stage, it must also reject non-string input or a deserialization target other than `String`. +The marker and version distinguish filesystem envelopes from raw service-originated JSON. Initial root input, callback +results, and standard Lambda invoke results may arrive before this SerDes has processed them. For those external +payload sources only, an input without the filesystem marker passes through to the preceding pipeline stage or +standalone delegate. Missing or malformed markers on SDK-checkpointed payloads are permanent errors. + +Offloaded filenames include a content hash and are immutable. Serializing new state for the same entity creates a new +path instead of replacing a file referenced by an earlier checkpoint. Repeating the same serialization may reuse the +same content-addressed file. + +The final file envelope, including any preview, must remain below the checkpoint threshold. Oversized previews are +rejected rather than producing a checkpoint that the service cannot accept. + In stage mode, the preview generator receives the string produced by the preceding stage, not the original domain object. A preview that needs domain fields should either parse that representation, be produced by an earlier stage, or use standalone compatibility mode where `FileSystemSerDes` receives the original value. @@ -284,19 +298,25 @@ use standalone compatibility mode where `FileSystemSerDes` receives the original ### Runtime flow ```java +var previousContext = SerDesContextHolder.get(); SerDesContextHolder.set(context); try { var checkpointPayload = composableSerDes.serialize(value); sendCheckpoint(checkpointPayload); } finally { - SerDesContextHolder.clear(); + if (previousContext == null) { + SerDesContextHolder.clear(); + } else { + SerDesContextHolder.set(previousContext); + } } ``` On deserialization, `FileSystemSerDes` parses its envelope. If the envelope contains `data`, it returns the inline string. If the envelope contains `file`, it reads and returns the file contents. `ComposableSerDes` then passes that string to the preceding stage. In standalone compatibility mode, `FileSystemSerDes` instead passes the resolved string -to its configured value-encoding delegate. +to its configured value-encoding delegate. Raw external input, callback results, and standard invoke results pass +through when no versioned filesystem marker is present. ### Threading @@ -315,7 +335,8 @@ DurableConfig.builder() If `withSerDesExecutorService(...)` is not called, the configured executor is absent and `SerDesRunner` invokes the pipeline synchronously on the current thread. The builder method accepts only a non-null executor; not calling it is how customers select inline execution. If an executor is configured, `SerDesRunner` dispatches the complete pipeline -to that executor and waits for its result. +to that executor and waits for its result. Configuration rejects using the same executor instance for user operations +and SerDes because synchronous dispatch to a saturated shared pool can deadlock. The core SDK should route user payload SerDes calls through a helper, tentatively `SerDesRunner`, that: @@ -381,6 +402,11 @@ When serializing `errorData`, set `SerDesPayloadKind.EXCEPTION` and use an entit Root user input and output payloads should route through `SerDesRunner` so `FileSystemSerDes` can see `SerDesContext`. The internal `DurableExecutionInput` and `DurableExecutionOutput` envelope stays with `DurableInputOutputSerDes`. +The cloud test runner must send initial Lambda input before it receives a durable execution ARN. When configured with a +`ComposableSerDes`, it therefore serializes the invocation payload with `getValueCodec()` and applies the complete +pipeline only when reading persisted history. Standalone context-dependent SerDes implementations can configure a +separate input codec with `CloudDurableTestRunner.withInputSerDes(...)`. + ### Implementation plan 1. Add `SerDesContext`, `SerDesPayloadKind`, and package-private TLS setter/clearer support. Leave the existing @@ -737,14 +763,14 @@ Both approaches need a stable payload identity that can be used to address exter | Root input | `execution//input` | | Root output | `execution//output` | | Root exception | `execution//exception` | -| Step result | `operation//result` | -| Step exception | `operation//exception` | +| Step result | `operation//result/attempt-` | +| Step exception | `operation//exception/attempt-` | | Invoke payload | `operation//invoke-payload` | | Invoke result | `operation//result` | | Callback result | `operation//result` | | Child context result | `operation//result` | | Map result | `operation//result` | -| WaitForCondition state | `operation//state` | +| WaitForCondition state | `operation//state/attempt-` | Do not include the checkpoint token or raw user payload in the context. diff --git a/docs/advanced/configuration.md b/docs/advanced/configuration.md index 6bf1cef58..0041c0e5f 100644 --- a/docs/advanced/configuration.md +++ b/docs/advanced/configuration.md @@ -16,10 +16,10 @@ public class OrderProcessor extends DurableHandler { return DurableConfig.builder() .withLambdaClientBuilder(lambdaClientBuilder) - .withSerDes(new MyCustomSerDes()) // Custom serialization + .withSerDes(new MyCustomSerDes()) // Custom serialization .withExecutorService(Executors.newFixedThreadPool(10)) // Custom thread pool - .withSerDesExecutorService(Executors.newFixedThreadPool(4)) // SerDes and payload I/O - .withLoggerConfig(LoggerConfig.withReplayLogging()) // Enable replay logs + .withSerDesExecutorService(Executors.newFixedThreadPool(4)) // Optional SerDes/payload I/O pool + .withLoggerConfig(LoggerConfig.withReplayLogging()) // Enable replay logs .build(); } @@ -35,52 +35,66 @@ public class OrderProcessor extends DurableHandler { | `withLambdaClientBuilder()` | Custom AWS Lambda client | Auto-configured Lambda client | | `withSerDes()` | Serializer for step results | Jackson with default settings | | `withExecutorService()` | Thread pool for user-defined operations | Cached daemon thread pool | -| `withSerDesExecutorService()` | Thread pool for SerDes and payload storage I/O | Cached `durable-sdk-serdes-*` daemon thread pool | +| `withSerDesExecutorService()` | Optional thread pool for SerDes and payload storage I/O | Inline on the calling thread | | `withLoggerConfig()` | Logger behavior configuration | Suppress logs during replay | | `withPollingStrategy()` | Backend polling strategy | Exponential backoff: 1s base, 2x rate, FULL jitter, 10s max | | `withCheckpointDelay()` | How often the SDK checkpoints updates | `Duration.ofSeconds(0)` (as soon as possible) | The `withExecutorService()` option configures the thread pool used for running user-defined operations. Internal SDK coordination (checkpoint batching, polling) runs on an SDK-managed thread pool. -The `withSerDesExecutorService()` option isolates serialization and blocking payload storage from user operations and -SDK coordination. The SDK sets `SerDesContext` inside each task, clears it after the call, and caches successful -deserialization results for the current invocation. +By default, SerDes runs synchronously on the calling thread to preserve existing behavior and avoid a queue, +`CompletableFuture`, and thread-hop cost for in-memory serialization. Configure +`withSerDesExecutorService()` when a SerDes performs blocking filesystem or network I/O, or uses retry backoff. The +SerDes executor must be different from the user-operation executor to avoid deadlock when the operation pool is +saturated. + +The SDK installs `SerDesContext` on whichever thread performs the call, restores any previous nested context afterward, +and caches successful deserialization results for the current invocation. ### Filesystem-backed payload storage -The optional `aws-durable-execution-sdk-java-extra-filesystem-serdes` artifact can store delegate-serialized payloads -on a shared filesystem: +The optional `aws-durable-execution-sdk-java-extra-filesystem-serdes` artifact provides a reversible string stage for +storing serialized payloads on a shared filesystem: ```java -var fileSystemSerDes = FileSystemSerDes.builder(Path.of("/mnt/efs/durable-payloads")) +var fileSystemStage = FileSystemSerDes.stageBuilder(Path.of("/mnt/efs/durable-payloads")) .storageMode(FileSystemStorageMode.OVERFLOW) .pathEncoding(FileSystemPathEncoding.HASH) - .delegate(new JacksonSerDes()) - .previewGenerator(value -> Map.of("type", value.getClass().getSimpleName())) + .previewGenerator(json -> Map.of("format", "json")) .build(); -var retryingSerDes = new RetrySerDes( - fileSystemSerDes, +var resilientFileSystemStage = new RetrySerDes( + fileSystemStage, RetryStrategies.fixedDelay(3, Duration.ofSeconds(1))); +var serDes = new JacksonSerDes().then(resilientFileSystemStage); +var serDesExecutor = Executors.newFixedThreadPool(4); + return DurableConfig.builder() - .withSerDes(retryingSerDes) + .withSerDes(serDes) + .withSerDesExecutorService(serDesExecutor) .build(); ``` `ALWAYS` stores every non-null payload in a file. `OVERFLOW` keeps payloads inline until the checkpoint envelope approaches the 256 KB service limit. `URI` produces readable escaped paths; `HASH` produces fixed-length SHA-256 path -segments. +segments. Files are content-addressed and never overwrite data referenced by an earlier checkpoint. References are +validated against the current durable execution and entity, and symbolic-link paths are rejected. `RetrySerDes` retries only failures marked with `RetryableSerDesException`. Filesystem read and write I/O use this -marker; malformed envelopes and delegate encoding failures fail immediately. Backoff occurs on the dedicated SerDes -executor within the current Lambda invocation, so use short, bounded retry strategies. +marker; malformed envelopes and codec failures fail immediately. Backoff occurs within the current Lambda invocation, +so use short, bounded retry strategies. Without a configured SerDes executor, filesystem I/O and retry delays block the +calling thread. Do not use Lambda's ephemeral `/tmp` directory: replay can run in a different execution environment. Use a durable, shared mount such as EFS. S3 Files can have delayed synchronization, so a runtime crash before the mount flushes may lose recent writes; use it only when that tradeoff is acceptable. The SDK does not delete offloaded files, so configure storage lifecycle and retention separately. +When cloud tests use a `ComposableSerDes`, `CloudDurableTestRunner` applies only its first value-codec stage to the +initial Lambda invocation because the durable execution ARN does not exist yet. Persisted history is decoded with the +complete pipeline. For a standalone context-dependent SerDes, call `withInputSerDes(...)` with a separate input codec. + ### Dynamic plugin loading Dynamic plugin loading is an opt-in alternative to registering plugins in application code. Put provider JARs on the application class path, then set `DURABLE_EXECUTION_PLUGINS` to an ordered, comma-separated list of provider names: diff --git a/docs/design.md b/docs/design.md index ed3f94412..6c2007873 100644 --- a/docs/design.md +++ b/docs/design.md @@ -11,6 +11,7 @@ This document explains the internal architecture, threading model, and extension ``` aws-durable-execution-sdk-java/ ├── sdk/ # Core SDK - DurableHandler, DurableContext, operations +├── extra-filesystem-serdes/ # Optional filesystem-backed SerDes pipeline stage ├── sdk-testing/ # Test utilities for local and cloud testing ├── sdk-integration-tests/ # Integration tests using LocalDurableTestRunner └── examples/ # Real-world usage patterns as customers would implement them @@ -19,6 +20,7 @@ aws-durable-execution-sdk-java/ | Module | Purpose | Key Classes | |--------|---------|-------------| | `sdk` | Core runtime - extend `DurableHandler`, use `DurableContext` for durable operations | `DurableHandler`, `DurableContext`, `DurableExecutor`, `ExecutionManager` | +| `extra-filesystem-serdes` | Optional filesystem-backed payload storage for SerDes pipelines | `FileSystemSerDes`, `FileSystemStorageMode`, `FileSystemPathEncoding` | | `sdk-testing` | Test utilities: `LocalDurableTestRunner` (in-memory, simulates re-invocations and time-skipping) and `CloudDurableTestRunner` (executes against deployed Lambda) | `LocalDurableTestRunner`, `CloudDurableTestRunner`, `LocalMemoryExecutionClient`, `TestResult` | | `sdk-integration-tests` | Dogfooding tests - validates the SDK using its own test utilities. Separate module keeps dependencies acyclic: `sdk` → `sdk-testing` → `sdk-integration-tests`. | Test classes only | | `examples` | Real-world usage patterns as customers would implement them, with local and cloud tests | Example handlers, `CloudBasedIntegrationTest` | @@ -145,13 +147,14 @@ public class MyHandler extends DurableHandler { | `lambdaClientBuilder` | Auto-created `LambdaClient` for current region, primed for performance (see [`DurableConfig.java`](../sdk/src/main/java/software/amazon/lambda/durable/DurableConfig.java)) | | `serDes` | `JacksonSerDes` | | `executorService` | `Executors.newCachedThreadPool()` (for user-defined operations only) | +| `serDesExecutorService` | `null`; SerDes executes inline unless a dedicated executor is configured | | `loggerConfig` | `LoggerConfig.defaults()` (suppress replay logs) | | `pollingStrategy` | Exponential backoff: 1s base, 2x rate, FULL jitter, 10s max | | `checkpointDelay` | `Duration.ofSeconds(0)` (checkpoint as soon as possible) | ### Thread Pool Architecture -The SDK uses two separate thread pools with distinct responsibilities: +The SDK uses two always-present executors and supports one optional executor with distinct responsibilities: **User Executor (`DurableConfig.executorService`):** - Runs user-defined operations (the code passed to `ctx.step()` and `ctx.stepAsync()`) @@ -163,11 +166,17 @@ The SDK uses two separate thread pools with distinct responsibilities: - Dedicated cached thread pool with daemon threads named `durable-sdk-internal-*` - Not configurable by users +**Optional SerDes Executor (`DurableConfig.serDesExecutorService`):** +- Runs the complete configured SerDes pipeline, including blocking payload storage and retry backoff +- Configurable via `DurableConfig.builder().withSerDesExecutorService()` +- Default: absent; SerDes executes inline on the calling thread +- Must not be the same instance as the user executor + **Benefits of this separation:** | Benefit | Description | |---------|-------------| -| **Isolation** | User operations can't starve SDK internals, and vice versa | +| **Isolation** | User operations can't starve SDK internals, and blocking SerDes work can be isolated explicitly | | **No shutdown management** | Internal pool uses daemon threads; SDK coordination continues even if the user's executor is shut down | | **Efficient resource usage** | Cached thread pool creates threads on demand and reuses idle threads (60s timeout) | | **Daemon threads** | Internal threads won't prevent JVM shutdown | @@ -347,9 +356,13 @@ software.amazon.lambda.durable │ └── WaitForConditionResult # Check function return type (value + isDone) │ ├── serde/ -│ ├── SerDes # Interface +│ ├── SerDes # Interface and pipeline composition entry point +│ ├── ComposableSerDes # Immutable ordered value-codec/string-stage pipeline │ ├── JacksonSerDes # Jackson impl │ ├── RetrySerDes # Retry decorator for transient SerDes failures +│ ├── SerDesRunner # Context, optional executor dispatch, and invocation cache +│ ├── SerDesContext # Read-only durable payload identity +│ ├── SerDesPayloadKind # Input/result/state/exception/invoke payload kind │ └── AwsSdkV2Module # SDK type support │ └── exception/ diff --git a/extra-filesystem-serdes/README.md b/extra-filesystem-serdes/README.md index 90ef5569e..117eb9802 100644 --- a/extra-filesystem-serdes/README.md +++ b/extra-filesystem-serdes/README.md @@ -1,7 +1,7 @@ # Filesystem SerDes `aws-durable-execution-sdk-java-extra-filesystem-serdes` stores durable user payloads on a shared filesystem while -keeping small file-pointer envelopes in checkpoints. +keeping small, versioned file-reference envelopes in checkpoints. ## Installation @@ -13,47 +13,67 @@ keeping small file-pointer envelopes in checkpoints. ``` -## Configuration +## Pipeline configuration + +The preferred configuration uses `FileSystemSerDes` as a reversible string stage after a value codec: ```java -var serDes = FileSystemSerDes.builder(Path.of("/mnt/efs/durable-payloads")) +var fileSystemStage = FileSystemSerDes.stageBuilder(Path.of("/mnt/efs/durable-payloads")) .storageMode(FileSystemStorageMode.ALWAYS) .pathEncoding(FileSystemPathEncoding.URI) - .delegate(new JacksonSerDes()) + .previewGenerator(json -> Map.of("format", "json")) .build(); +var resilientFileSystemStage = new RetrySerDes( + fileSystemStage, + RetryStrategies.fixedDelay(3, Duration.ofSeconds(1))); + +var serDes = new JacksonSerDes().then(resilientFileSystemStage); +var serDesExecutor = Executors.newFixedThreadPool(4); + return DurableConfig.builder() .withSerDes(serDes) + .withSerDesExecutorService(serDesExecutor) .build(); ``` +Serialization runs from `JacksonSerDes` to the filesystem stage. Deserialization runs in reverse. Additional reversible +string stages such as compression or encryption can be inserted with `then(...)`. + - `ALWAYS` writes every non-null payload to a file. - `OVERFLOW` stores small payloads inline and offloads envelopes approaching the 256 KB checkpoint limit. - `URI` uses readable escaped path segments. - `HASH` uses fixed-length SHA-256 path segments. -An optional preview can remain visible in the checkpoint envelope: +The preview generator receives the incoming stage string. Its output is included only in file envelopes and the final +envelope must remain below the checkpoint threshold. -```java -.previewGenerator(value -> Map.of("summary", "order payload")) -``` +For compatibility, `FileSystemSerDes.builder(path)` creates a standalone SerDes with `JacksonSerDes` as its default +value codec. A custom standalone codec can be supplied with `.delegate(...)`. -## Retries +## Execution and retries -Filesystem read and write I/O failures are reported as `RetryableSerDesException`. Wrap the filesystem SerDes with -`RetrySerDes` to apply any SDK `RetryStrategy`: +SerDes runs inline by default. Filesystem access and retry backoff are blocking, so production configurations should +provide a dedicated executor with `withSerDesExecutorService(...)`. It must be different from the user-operation +executor. -```java -var serDes = new RetrySerDes( - FileSystemSerDes.builder(Path.of("/mnt/efs/durable-payloads")) - .storageMode(FileSystemStorageMode.ALWAYS) - .build(), - RetryStrategies.fixedDelay(3, Duration.ofSeconds(1))); -``` +Filesystem read and write I/O failures are reported as `RetryableSerDesException`. `RetrySerDes` retries only that +exception type. Malformed envelopes, invalid paths, unsupported stage types, and codec failures are permanent. Retry +delays consume time in the current Lambda invocation, so keep attempts and delays bounded. + +## Replay and envelope behavior + +Filesystem envelopes include a reserved version marker. Raw root input, callback results, and standard Lambda invoke +results pass through when they have not yet been wrapped by this SerDes. + +Offloaded files are content-addressed and immutable. Updating wait-for-condition state or retry results creates a new +path instead of replacing a file referenced by an earlier checkpoint. Repeating the same write can safely reuse the +same file. File references are bound to the current execution and entity, content hashes are verified when reading, +and symbolic-link paths are rejected. -Only `RetryableSerDesException` is retried. Malformed envelopes, invalid paths, and delegate encoding failures are -treated as permanent. Retry delays run on the dedicated SerDes executor and consume time in the current Lambda -invocation, so keep attempts and delays bounded. +`CloudDurableTestRunner` uses the first value-codec stage for the initial Lambda invocation, before an execution ARN is +available, and uses the complete pipeline for persisted history. When using standalone `FileSystemSerDes`, configure a +separate initial-input codec with `withInputSerDes(...)`. ## Storage requirements diff --git a/extra-filesystem-serdes/src/main/java/software/amazon/lambda/durable/extra/filesystem/FileSystemSerDes.java b/extra-filesystem-serdes/src/main/java/software/amazon/lambda/durable/extra/filesystem/FileSystemSerDes.java index 488fd7509..0a6a2b70d 100644 --- a/extra-filesystem-serdes/src/main/java/software/amazon/lambda/durable/extra/filesystem/FileSystemSerDes.java +++ b/extra-filesystem-serdes/src/main/java/software/amazon/lambda/durable/extra/filesystem/FileSystemSerDes.java @@ -8,31 +8,41 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; import java.nio.file.AtomicMoveNotSupportedException; +import java.nio.file.FileAlreadyExistsException; import java.nio.file.Files; +import java.nio.file.LinkOption; import java.nio.file.Path; import java.nio.file.StandardCopyOption; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.util.HexFormat; +import java.util.LinkedHashMap; import java.util.Map; import java.util.Objects; import java.util.function.Function; import java.util.regex.Pattern; +import software.amazon.awssdk.services.lambda.model.OperationType; import software.amazon.lambda.durable.TypeToken; import software.amazon.lambda.durable.exception.RetryableSerDesException; import software.amazon.lambda.durable.exception.SerDesException; import software.amazon.lambda.durable.serde.JacksonSerDes; import software.amazon.lambda.durable.serde.SerDes; import software.amazon.lambda.durable.serde.SerDesContext; +import software.amazon.lambda.durable.serde.SerDesPayloadKind; /** - * A SerDes that stores delegate-serialized payloads on a durable shared filesystem. + * A SerDes that stores payloads on a durable shared filesystem. + * + *

Use {@link #stageBuilder(Path)} when composing this implementation after a value codec. The compatibility + * {@link #builder(Path)} form includes its own value codec and can be used as a standalone SerDes. * *

Do not use Lambda's ephemeral {@code /tmp} storage. Use a durable shared mount such as EFS, or S3 Files only when * its synchronization and crash-durability tradeoffs are acceptable for the workload. */ public final class FileSystemSerDes implements SerDes { - private static final int OVERFLOW_THRESHOLD_BYTES = 256 * 1024 - 1024; + private static final String ENVELOPE_MARKER = "__durable_execution_filesystem_serdes"; + private static final int ENVELOPE_VERSION = 1; + private static final int CHECKPOINT_ENVELOPE_LIMIT_BYTES = 256 * 1024 - 1024; private static final ObjectMapper ENVELOPE_MAPPER = new ObjectMapper(); private static final Pattern DURABLE_EXECUTION_ARN_PATTERN = Pattern.compile( "^arn:[^:]*:lambda:[^:]*:[^:]*:function:([^:/]+):[^:/]+/durable-execution/([^/]+)/([^/]+)$"); @@ -42,6 +52,7 @@ public final class FileSystemSerDes implements SerDes { private final FileSystemPathEncoding pathEncoding; private final SerDes delegate; private final Function> previewGenerator; + private final boolean stageMode; private FileSystemSerDes(Builder builder) { basePath = builder.basePath.toAbsolutePath().normalize(); @@ -49,10 +60,27 @@ private FileSystemSerDes(Builder builder) { pathEncoding = builder.pathEncoding; delegate = builder.delegate; previewGenerator = builder.previewGenerator; + stageMode = builder.stageMode; } + /** + * Creates a standalone filesystem SerDes builder with {@link JacksonSerDes} as its default value codec. + * + * @param basePath durable shared filesystem root + * @return a standalone builder + */ public static Builder builder(Path basePath) { - return new Builder(basePath); + return new Builder(basePath, false); + } + + /** + * Creates a filesystem string-stage builder for use in a composable SerDes pipeline. + * + * @param basePath durable shared filesystem root + * @return a string-stage builder + */ + public static Builder stageBuilder(Path basePath) { + return new Builder(basePath, true); } @Override @@ -61,24 +89,23 @@ public String serialize(Object value) { return null; } var context = requireContext(); - var serialized = delegate.serialize(value); - if (serialized == null) { - throw new SerDesException("Delegate SerDes returned null for a non-null value"); + var serialized = serializeValue(value); + var inlineEnvelope = encodeEnvelope(serialized, null, null, context); + if (storageMode == FileSystemStorageMode.OVERFLOW && fitsCheckpoint(inlineEnvelope)) { + return inlineEnvelope; + } + + var file = resolvePayloadPath(serialized, context); + var preview = generatePreview(value, context); + var fileEnvelope = encodeEnvelope(null, file, preview, context); + if (!fitsCheckpoint(fileEnvelope)) { + throw new SerDesException("Filesystem SerDes envelope exceeds the checkpoint payload limit for entity '" + + context.entityId() + + "'"); } try { - var inlineEnvelope = ENVELOPE_MAPPER.writeValueAsString(Map.of("data", serialized)); - if (storageMode == FileSystemStorageMode.OVERFLOW - && inlineEnvelope.getBytes(StandardCharsets.UTF_8).length <= OVERFLOW_THRESHOLD_BYTES) { - return inlineEnvelope; - } - var file = writePayload(serialized, context); - var preview = previewGenerator != null ? previewGenerator.apply(value) : null; - return preview == null - ? ENVELOPE_MAPPER.writeValueAsString(Map.of("file", file.toString())) - : ENVELOPE_MAPPER.writeValueAsString(Map.of("file", file.toString(), "preview", preview)); - } catch (JsonProcessingException e) { - throw new SerDesException( - "Failed to encode filesystem payload envelope for entity '" + context.entityId() + "'", e); + writePayload(serialized, file); + return fileEnvelope; } catch (IOException e) { throw new RetryableSerDesException( "Failed to store filesystem payload for entity '" + context.entityId() + "'", e); @@ -90,40 +117,167 @@ public T deserialize(String data, TypeToken typeToken) { if (data == null) { return null; } + Objects.requireNonNull(typeToken, "typeToken cannot be null"); var context = requireContext(); - try { - JsonNode envelope = ENVELOPE_MAPPER.readTree(data); - var hasData = envelope.has("data") && envelope.get("data").isTextual(); - var hasFile = envelope.has("file") && envelope.get("file").isTextual(); - if (hasData == hasFile) { - throw new SerDesException("Filesystem SerDes envelope must contain exactly one of 'data' or 'file'"); - } - String serialized; - if (hasData) { - serialized = envelope.get("data").textValue(); - } else { - var file = Path.of(envelope.get("file").textValue()) - .toAbsolutePath() - .normalize(); - if (!file.startsWith(basePath)) { - throw new SerDesException("Filesystem SerDes file is outside the configured base path"); - } - serialized = Files.readString(file, StandardCharsets.UTF_8); + var serialized = resolveSerializedPayload(data, context); + if (stageMode) { + if (!TypeToken.get(String.class).equals(typeToken)) { + throw new SerDesException("FileSystemSerDes stage can only deserialize to String"); + } + @SuppressWarnings("unchecked") + var value = (T) serialized; + return value; + } + return delegate.deserialize(serialized, typeToken); + } + + private String serializeValue(Object value) { + if (stageMode) { + if (!(value instanceof String stringValue)) { + throw new SerDesException("FileSystemSerDes stage can only serialize String values"); } - return delegate.deserialize(serialized, typeToken); + return stringValue; + } + var serialized = delegate.serialize(value); + if (serialized == null) { + throw new SerDesException("Delegate SerDes returned null for a non-null value"); + } + return serialized; + } + + private String resolveSerializedPayload(String data, SerDesContext context) { + final JsonNode envelope; + try { + envelope = ENVELOPE_MAPPER.readTree(data); } catch (JsonProcessingException e) { - throw new SerDesException( - "Failed to decode filesystem payload envelope for entity '" + context.entityId() + "'", e); + if (acceptsExternalPayload(context)) { + return data; + } + throw malformedEnvelope(context, e); + } + + if (envelope != null && envelope.isObject() && envelope.has(ENVELOPE_MARKER)) { + if (!isFilesystemEnvelope(envelope)) { + throw malformedEnvelope(context, null); + } + } else { + if (acceptsExternalPayload(context)) { + return data; + } + throw malformedEnvelope(context, null); + } + + var hasData = envelope.has("data") && envelope.get("data").isTextual(); + var hasFile = envelope.has("file") && envelope.get("file").isTextual(); + if (hasData == hasFile) { + throw malformedEnvelope(context, null); + } + if (hasData) { + return envelope.get("data").textValue(); + } + return readPayload(envelope.get("file").textValue(), context); + } + + private String readPayload(String fileValue, SerDesContext context) { + var file = Path.of(fileValue).toAbsolutePath().normalize(); + validatePayloadPath(file, context); + try { + rejectSymbolicLinks(file); + var realBasePath = basePath.toRealPath(); + var realDirectory = file.getParent().toRealPath(); + var realFile = file.toRealPath(); + if (!realDirectory.startsWith(realBasePath) + || !realFile.getParent().equals(realDirectory) + || !realFile.equals(file.toRealPath(LinkOption.NOFOLLOW_LINKS))) { + throw new SerDesException("Filesystem SerDes file does not resolve to the expected payload path"); + } + var serialized = Files.readString(realFile, StandardCharsets.UTF_8); + var expectedFileName = payloadFileName(serialized, context); + if (!realFile.getFileName().toString().equals(expectedFileName)) { + throw new SerDesException("Filesystem SerDes file content does not match its content-addressed path"); + } + return serialized; } catch (IOException e) { throw new RetryableSerDesException( "Failed to load filesystem payload for entity '" + context.entityId() + "'", e); - } catch (SerDesException e) { - throw e; - } catch (Exception e) { - throw new SerDesException("Failed to load filesystem payload for entity '" + context.entityId() + "'", e); } } + private void validatePayloadPath(Path file, SerDesContext context) { + var expectedDirectory = resolveExecutionDirectory(context.durableExecutionArn()); + var fileName = file.getFileName(); + if (fileName == null + || file.getParent() == null + || !file.getParent().equals(expectedDirectory) + || !fileName.toString().matches(Pattern.quote(encode(context.entityId())) + "-[0-9a-f]{64}\\.json")) { + throw new SerDesException("Filesystem SerDes file is not valid for the current durable entity"); + } + } + + private void rejectSymbolicLinks(Path file) throws IOException { + var current = basePath; + for (var component : basePath.relativize(file)) { + current = current.resolve(component); + if (Files.isSymbolicLink(current)) { + throw new SerDesException("Filesystem SerDes payload path must not contain symbolic links"); + } + } + } + + private static boolean isFilesystemEnvelope(JsonNode envelope) { + return envelope != null + && envelope.isObject() + && envelope.has(ENVELOPE_MARKER) + && envelope.get(ENVELOPE_MARKER).isIntegralNumber() + && envelope.get(ENVELOPE_MARKER).intValue() == ENVELOPE_VERSION; + } + + private static boolean acceptsExternalPayload(SerDesContext context) { + return context.payloadKind() == SerDesPayloadKind.INPUT + || context.operationType() == OperationType.CALLBACK + || context.operationType() == OperationType.CHAINED_INVOKE; + } + + private static SerDesException malformedEnvelope(SerDesContext context, Throwable cause) { + var message = "Invalid filesystem SerDes envelope for entity '" + context.entityId() + "'"; + return cause == null ? new SerDesException(message) : new SerDesException(message, cause); + } + + private String encodeEnvelope(String data, Path file, Map preview, SerDesContext context) { + var envelope = new LinkedHashMap(); + envelope.put(ENVELOPE_MARKER, ENVELOPE_VERSION); + if (data != null) { + envelope.put("data", data); + } else { + envelope.put("file", file.toString()); + if (preview != null) { + envelope.put("preview", preview); + } + } + try { + return ENVELOPE_MAPPER.writeValueAsString(envelope); + } catch (JsonProcessingException e) { + throw new SerDesException( + "Failed to encode filesystem payload envelope for entity '" + context.entityId() + "'", e); + } + } + + private Map generatePreview(Object value, SerDesContext context) { + if (previewGenerator == null) { + return null; + } + try { + return previewGenerator.apply(value); + } catch (RuntimeException e) { + throw new SerDesException( + "Failed to generate filesystem payload preview for entity '" + context.entityId() + "'", e); + } + } + + private static boolean fitsCheckpoint(String envelope) { + return envelope.getBytes(StandardCharsets.UTF_8).length <= CHECKPOINT_ENVELOPE_LIMIT_BYTES; + } + private SerDesContext requireContext() { var context = SerDesContext.getCurrentContext(); if (context == null @@ -137,25 +291,79 @@ private SerDesContext requireContext() { return context; } - private Path writePayload(String serialized, SerDesContext context) throws IOException { + private Path resolvePayloadPath(String serialized, SerDesContext context) { var directory = resolveExecutionDirectory(context.durableExecutionArn()); - Files.createDirectories(directory); - var file = directory.resolve(encode(context.entityId()) + ".json").normalize(); + var fileName = payloadFileName(serialized, context); + var file = directory.resolve(fileName).normalize(); if (!file.startsWith(directory)) { throw new SerDesException("Resolved filesystem payload path is outside the execution directory"); } + return file; + } + + private String payloadFileName(String serialized, SerDesContext context) { + return encode(context.entityId()) + "-" + sha256(serialized) + ".json"; + } + + private void writePayload(String serialized, Path file) throws IOException { + var directory = file.getParent(); + createDirectoriesWithoutSymbolicLinks(directory); + rejectSymbolicLinks(file); + var realBasePath = basePath.toRealPath(); + var realDirectory = directory.toRealPath(); + if (!realDirectory.startsWith(realBasePath)) { + throw new SerDesException("Filesystem SerDes directory resolves outside the configured base path"); + } + if (Files.exists(file, LinkOption.NOFOLLOW_LINKS)) { + var existing = Files.readString(file, StandardCharsets.UTF_8); + if (!existing.equals(serialized)) { + throw new SerDesException("Filesystem SerDes content-addressed file contains unexpected data"); + } + return; + } + var temporary = Files.createTempFile(directory, file.getFileName().toString(), ".tmp"); try { Files.writeString(temporary, serialized, StandardCharsets.UTF_8); - try { - Files.move(temporary, file, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); - } catch (AtomicMoveNotSupportedException e) { - Files.move(temporary, file, StandardCopyOption.REPLACE_EXISTING); - } + moveWithoutReplacement(temporary, file); } finally { Files.deleteIfExists(temporary); } - return file; + } + + private void createDirectoriesWithoutSymbolicLinks(Path directory) throws IOException { + Files.createDirectories(basePath); + var current = basePath; + for (var component : basePath.relativize(directory)) { + current = current.resolve(component); + if (Files.exists(current, LinkOption.NOFOLLOW_LINKS)) { + if (Files.isSymbolicLink(current) || !Files.isDirectory(current, LinkOption.NOFOLLOW_LINKS)) { + throw new SerDesException("Filesystem SerDes directory path must contain only real directories"); + } + continue; + } + try { + Files.createDirectory(current); + } catch (FileAlreadyExistsException ignored) { + if (Files.isSymbolicLink(current) || !Files.isDirectory(current, LinkOption.NOFOLLOW_LINKS)) { + throw new SerDesException("Filesystem SerDes directory path must contain only real directories"); + } + } + } + } + + private static void moveWithoutReplacement(Path temporary, Path file) throws IOException { + try { + Files.move(temporary, file, StandardCopyOption.ATOMIC_MOVE); + } catch (AtomicMoveNotSupportedException e) { + try { + Files.move(temporary, file); + } catch (FileAlreadyExistsException ignored) { + // Another thread or invocation already persisted the same content-addressed payload. + } + } catch (FileAlreadyExistsException ignored) { + // Another thread or invocation already persisted the same content-addressed payload. + } } private Path resolveExecutionDirectory(String durableExecutionArn) { @@ -163,9 +371,9 @@ private Path resolveExecutionDirectory(String durableExecutionArn) { if (pathEncoding == FileSystemPathEncoding.URI) { var matcher = DURABLE_EXECUTION_ARN_PATTERN.matcher(durableExecutionArn); if (matcher.matches()) { - directory = basePath.resolve(matcher.group(1)) - .resolve(matcher.group(2)) - .resolve(matcher.group(3)) + directory = basePath.resolve(encode(matcher.group(1))) + .resolve(encode(matcher.group(2))) + .resolve(encode(matcher.group(3))) .normalize(); if (!directory.startsWith(basePath)) { throw new SerDesException("Resolved filesystem execution path is outside the configured base path"); @@ -216,32 +424,43 @@ private static String sha256(String value) { /** Builder for {@link FileSystemSerDes}. */ public static final class Builder { private final Path basePath; + private final boolean stageMode; private FileSystemStorageMode storageMode = FileSystemStorageMode.ALWAYS; private FileSystemPathEncoding pathEncoding = FileSystemPathEncoding.URI; - private SerDes delegate = new JacksonSerDes(); + private SerDes delegate; private Function> previewGenerator; - private Builder(Path basePath) { + private Builder(Path basePath, boolean stageMode) { this.basePath = Objects.requireNonNull(basePath, "basePath cannot be null"); + this.stageMode = stageMode; + this.delegate = stageMode ? null : new JacksonSerDes(); } public Builder storageMode(FileSystemStorageMode storageMode) { - this.storageMode = Objects.requireNonNull(storageMode); + this.storageMode = Objects.requireNonNull(storageMode, "storageMode cannot be null"); return this; } public Builder pathEncoding(FileSystemPathEncoding pathEncoding) { - this.pathEncoding = Objects.requireNonNull(pathEncoding); + this.pathEncoding = Objects.requireNonNull(pathEncoding, "pathEncoding cannot be null"); return this; } + /** + * Sets the value codec used by standalone mode. + * + * @throws IllegalStateException when called on a stage builder + */ public Builder delegate(SerDes delegate) { - this.delegate = Objects.requireNonNull(delegate); + if (stageMode) { + throw new IllegalStateException("FileSystemSerDes stage mode does not use a delegate"); + } + this.delegate = Objects.requireNonNull(delegate, "delegate cannot be null"); return this; } public Builder previewGenerator(Function> previewGenerator) { - this.previewGenerator = Objects.requireNonNull(previewGenerator); + this.previewGenerator = Objects.requireNonNull(previewGenerator, "previewGenerator cannot be null"); return this; } diff --git a/extra-filesystem-serdes/src/test/java/software/amazon/lambda/durable/extra/filesystem/FileSystemSerDesTest.java b/extra-filesystem-serdes/src/test/java/software/amazon/lambda/durable/extra/filesystem/FileSystemSerDesTest.java index fa6c95c9c..48aff91fc 100644 --- a/extra-filesystem-serdes/src/test/java/software/amazon/lambda/durable/extra/filesystem/FileSystemSerDesTest.java +++ b/extra-filesystem-serdes/src/test/java/software/amazon/lambda/durable/extra/filesystem/FileSystemSerDesTest.java @@ -5,6 +5,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -12,8 +13,6 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.Map; -import java.util.concurrent.Executors; -import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; import software.amazon.awssdk.services.lambda.model.OperationType; @@ -21,6 +20,7 @@ import software.amazon.lambda.durable.exception.RetryableSerDesException; import software.amazon.lambda.durable.exception.SerDesException; import software.amazon.lambda.durable.model.OperationSubType; +import software.amazon.lambda.durable.serde.JacksonSerDes; import software.amazon.lambda.durable.serde.SerDesContext; import software.amazon.lambda.durable.serde.SerDesPayloadKind; import software.amazon.lambda.durable.serde.SerDesRunner; @@ -28,26 +28,22 @@ class FileSystemSerDesTest { private static final String ARN = "arn:aws:lambda:us-east-1:123456789012:function:orders:1/durable-execution/execution-1/invocation-1"; + private static final String ENVELOPE_MARKER = "__durable_execution_filesystem_serdes"; private static final ObjectMapper MAPPER = new ObjectMapper(); @TempDir Path basePath; - private final java.util.concurrent.ExecutorService executor = Executors.newSingleThreadExecutor(); - - @AfterEach - void tearDown() { - executor.shutdownNow(); - } - @Test - void alwaysModeWritesDelegatePayloadAndReplaysIt() throws Exception { + void standaloneModeWritesDelegatePayloadAndReplaysIt() throws Exception { var serDes = FileSystemSerDes.builder(basePath).build(); - var runner = new SerDesRunner(executor); + var runner = new SerDesRunner(null); var envelope = runner.serialize(serDes, Map.of("id", 42), context()); - var file = Path.of(MAPPER.readTree(envelope).get("file").textValue()); + var json = MAPPER.readTree(envelope); + var file = Path.of(json.get("file").textValue()); + assertEquals(1, json.get(ENVELOPE_MARKER).intValue()); assertTrue(file.startsWith(basePath.resolve("orders/execution-1/invocation-1"))); assertEquals("{\"id\":42}", Files.readString(file)); assertEquals( @@ -55,12 +51,33 @@ void alwaysModeWritesDelegatePayloadAndReplaysIt() throws Exception { runner.deserialize(serDes, envelope, new TypeToken>() {}, context())); } + @Test + void stageModeComposesWithValueCodec() throws Exception { + var stage = FileSystemSerDes.stageBuilder(basePath).build(); + var pipeline = new JacksonSerDes().then(stage); + var runner = new SerDesRunner(null); + + var envelope = runner.serialize(pipeline, Map.of("id", 42), context()); + var file = Path.of(MAPPER.readTree(envelope).get("file").textValue()); + + assertEquals("{\"id\":42}", Files.readString(file)); + assertEquals( + Map.of("id", 42), + runner.deserialize(pipeline, envelope, new TypeToken>() {}, context())); + assertThrows(SerDesException.class, () -> runner.serialize(stage, Map.of("id", 42), context())); + assertThrows( + SerDesException.class, + () -> runner.deserialize(stage, envelope, TypeToken.get(Integer.class), context())); + assertThrows(IllegalStateException.class, () -> FileSystemSerDes.stageBuilder(basePath) + .delegate(new JacksonSerDes())); + } + @Test void overflowModeKeepsSmallPayloadInlineAndWritesLargePayload() throws Exception { var serDes = FileSystemSerDes.builder(basePath) .storageMode(FileSystemStorageMode.OVERFLOW) .build(); - var runner = new SerDesRunner(executor); + var runner = new SerDesRunner(null); var inline = runner.serialize(serDes, "small", context()); assertTrue(MAPPER.readTree(inline).has("data")); @@ -69,56 +86,147 @@ void overflowModeKeepsSmallPayloadInlineAndWritesLargePayload() throws Exception assertTrue(MAPPER.readTree(overflow).has("file")); } + @Test + void immutableContentAddressedFilesPreservePriorCheckpoints() throws Exception { + var serDes = FileSystemSerDes.stageBuilder(basePath).build(); + var runner = new SerDesRunner(null); + + var firstContext = context(1); + var secondContext = context(2); + var firstEnvelope = runner.serialize(serDes, "state-one", firstContext); + var secondEnvelope = runner.serialize(serDes, "state-two", secondContext); + var firstFile = Path.of(MAPPER.readTree(firstEnvelope).get("file").textValue()); + var secondFile = Path.of(MAPPER.readTree(secondEnvelope).get("file").textValue()); + + assertNotEquals(firstFile, secondFile); + assertEquals("state-one", Files.readString(firstFile)); + assertEquals("state-two", Files.readString(secondFile)); + assertEquals("state-one", runner.deserialize(serDes, firstEnvelope, TypeToken.get(String.class), firstContext)); + } + @Test void hashEncodingUsesFixedLengthSegments() throws Exception { var serDes = FileSystemSerDes.builder(basePath) .pathEncoding(FileSystemPathEncoding.HASH) .build(); - var envelope = new SerDesRunner(executor).serialize(serDes, "value", context()); + var envelope = new SerDesRunner(null).serialize(serDes, "value", context()); var file = Path.of(MAPPER.readTree(envelope).get("file").textValue()); assertEquals(64, file.getParent().getFileName().toString().length()); - assertEquals(69, file.getFileName().toString().length()); + assertEquals(134, file.getFileName().toString().length()); assertFalse(file.toString().contains("operation")); } @Test - void includesPreviewWithoutChangingStoredPayload() throws Exception { + void includesBoundedPreviewWithoutChangingStoredPayload() throws Exception { var serDes = FileSystemSerDes.builder(basePath) .previewGenerator(value -> Map.of("summary", "order")) .build(); + var runner = new SerDesRunner(null); - var envelope = new SerDesRunner(executor).serialize(serDes, Map.of("secret", "value"), context()); + var envelope = runner.serialize(serDes, Map.of("secret", "value"), context()); var json = MAPPER.readTree(envelope); assertEquals("order", json.get("preview").get("summary").textValue()); assertEquals( "{\"secret\":\"value\"}", Files.readString(Path.of(json.get("file").textValue()))); + + var oversizedPreview = FileSystemSerDes.builder(basePath) + .previewGenerator(value -> Map.of("summary", "x".repeat(256 * 1024))) + .build(); + var failure = assertThrows(SerDesException.class, () -> runner.serialize(oversizedPreview, "value", context())); + assertTrue(failure.getCause().getMessage().contains("checkpoint payload limit")); } @Test - void rejectsCallsWithoutSdkContextAndMalformedEnvelopes() { + void acceptsExternallyOriginatedRawPayloadsOnlyForSupportedSources() { + var standalone = FileSystemSerDes.builder(basePath).build(); + var stage = FileSystemSerDes.stageBuilder(basePath).build(); + var runner = new SerDesRunner(null); + + assertEquals( + Map.of("id", 42), + runner.deserialize( + standalone, + "{\"id\":42}", + new TypeToken>() {}, + executionContext(SerDesPayloadKind.INPUT))); + assertEquals( + "{\"id\":42}", + runner.deserialize( + stage, + "{\"id\":42}", + TypeToken.get(String.class), + operationContext(OperationType.CALLBACK, OperationSubType.CALLBACK))); + assertEquals( + "\"invoke-result\"", + runner.deserialize( + stage, + "\"invoke-result\"", + TypeToken.get(String.class), + operationContext(OperationType.CHAINED_INVOKE, OperationSubType.CHAINED_INVOKE))); + + assertThrows( + SerDesException.class, + () -> runner.deserialize(stage, "\"raw-step\"", TypeToken.get(String.class), context())); + } + + @Test + void rejectsCallsWithoutContextAndMalformedOrUnsafeEnvelopes() throws Exception { var serDes = FileSystemSerDes.builder(basePath).build(); assertThrows(SerDesException.class, () -> serDes.serialize("value")); - var runner = new SerDesRunner(executor); + var runner = new SerDesRunner(null); assertThrows( SerDesException.class, () -> runner.deserialize(serDes, "{}", TypeToken.get(String.class), context())); assertThrows( SerDesException.class, () -> runner.deserialize( - serDes, "{\"file\":\"/outside/payload.json\"}", TypeToken.get(String.class), context())); + serDes, envelopeWithFile("/outside/payload.json"), TypeToken.get(String.class), context())); - var missingFile = basePath.resolve("missing.json").toAbsolutePath(); + var missingEnvelope = runner.serialize(serDes, "missing", context()); + var missingFile = payloadFile(missingEnvelope); + assertTrue(Files.deleteIfExists(missingFile)); var missingFileFailure = assertThrows( - SerDesException.class, - () -> runner.deserialize( - serDes, "{\"file\":\"" + missingFile + "\"}", TypeToken.get(String.class), context())); + RetryableSerDesException.class, + () -> runner.deserialize(serDes, missingEnvelope, TypeToken.get(String.class), context())); assertInstanceOf(RetryableSerDesException.class, missingFileFailure.getCause()); } + @Test + void rejectsCrossEntityAndSymbolicLinkReferences() throws Exception { + var serDes = FileSystemSerDes.stageBuilder(basePath).build(); + var envelope = new SerDesRunner(null).serialize(serDes, "payload", context()); + + var otherEntity = SerDesContext.forOperation( + ARN, "2", "other-step", null, OperationType.STEP, OperationSubType.STEP, SerDesPayloadKind.RESULT, 1); + assertThrows(SerDesException.class, () -> new SerDesRunner(null) + .deserialize(serDes, envelope, TypeToken.get(String.class), otherEntity)); + + var file = payloadFile(envelope); + var outside = Files.createTempFile(basePath.getParent(), "outside-payload-", ".json"); + Files.writeString(outside, "payload"); + Files.delete(file); + Files.createSymbolicLink(file, outside); + + assertThrows(SerDesException.class, () -> new SerDesRunner(null) + .deserialize(serDes, envelope, TypeToken.get(String.class), context())); + } + + @Test + void rejectsSymbolicLinkDirectoriesWhenWriting() throws Exception { + var outside = Files.createTempDirectory(basePath.getParent(), "outside-payloads-"); + Files.createSymbolicLink(basePath.resolve("orders"), outside); + var serDes = FileSystemSerDes.stageBuilder(basePath).build(); + + assertThrows(SerDesException.class, () -> new SerDesRunner(null).serialize(serDes, "payload", context())); + try (var files = Files.list(outside)) { + assertEquals(0, files.count()); + } + } + @Test void rejectsExecutionPathsOutsideConfiguredBasePath() { var serDes = FileSystemSerDes.builder(basePath).build(); @@ -132,11 +240,40 @@ void rejectsExecutionPathsOutsideConfiguredBasePath() { SerDesPayloadKind.RESULT, 1); - assertThrows(SerDesException.class, () -> new SerDesRunner(executor).serialize(serDes, "value", unsafeContext)); + assertThrows(SerDesException.class, () -> new SerDesRunner(null).serialize(serDes, "value", unsafeContext)); + } + + private static String envelopeWithFile(String file) { + try { + return MAPPER.writeValueAsString(Map.of(ENVELOPE_MARKER, 1, "file", file)); + } catch (Exception e) { + throw new AssertionError(e); + } + } + + private static Path payloadFile(String envelope) { + try { + return Path.of(MAPPER.readTree(envelope).get("file").textValue()); + } catch (Exception e) { + throw new AssertionError(e); + } } private static SerDesContext context() { + return context(1); + } + + private static SerDesContext context(int attempt) { + return SerDesContext.forOperation( + ARN, "1", "step", null, OperationType.STEP, OperationSubType.STEP, SerDesPayloadKind.RESULT, attempt); + } + + private static SerDesContext executionContext(SerDesPayloadKind payloadKind) { + return SerDesContext.forExecution(ARN, "invocation-1", "execution-1", payloadKind); + } + + private static SerDesContext operationContext(OperationType operationType, OperationSubType operationSubType) { return SerDesContext.forOperation( - ARN, "1", "step", null, OperationType.STEP, OperationSubType.STEP, SerDesPayloadKind.RESULT, 1); + ARN, "1", "operation", null, operationType, operationSubType, SerDesPayloadKind.RESULT, null); } } diff --git a/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FileSystemSerDesIntegrationTest.java b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FileSystemSerDesIntegrationTest.java index 20a756f79..4146e51e6 100644 --- a/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FileSystemSerDesIntegrationTest.java +++ b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FileSystemSerDesIntegrationTest.java @@ -3,22 +3,44 @@ package software.amazon.lambda.durable; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertTrue; import com.fasterxml.jackson.databind.ObjectMapper; import java.nio.file.Files; import java.nio.file.Path; import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import software.amazon.awssdk.services.lambda.model.CheckpointUpdatedExecutionState; +import software.amazon.awssdk.services.lambda.model.ExecutionDetails; +import software.amazon.awssdk.services.lambda.model.Operation; +import software.amazon.awssdk.services.lambda.model.OperationStatus; +import software.amazon.awssdk.services.lambda.model.OperationType; +import software.amazon.lambda.durable.config.StepConfig; import software.amazon.lambda.durable.config.WaitForConditionConfig; +import software.amazon.lambda.durable.execution.DurableExecutor; import software.amazon.lambda.durable.extra.filesystem.FileSystemSerDes; +import software.amazon.lambda.durable.model.DurableExecutionInput; import software.amazon.lambda.durable.model.ExecutionStatus; import software.amazon.lambda.durable.model.WaitForConditionResult; import software.amazon.lambda.durable.retry.JitterStrategy; +import software.amazon.lambda.durable.retry.RetryStrategies; import software.amazon.lambda.durable.retry.WaitStrategies; +import software.amazon.lambda.durable.serde.JacksonSerDes; +import software.amazon.lambda.durable.serde.SerDes; +import software.amazon.lambda.durable.serde.SerDesContext; +import software.amazon.lambda.durable.serde.SerDesPayloadKind; +import software.amazon.lambda.durable.serde.SerDesRunner; import software.amazon.lambda.durable.testing.LocalDurableTestRunner; +import software.amazon.lambda.durable.testing.local.LocalMemoryExecutionClient; class FileSystemSerDesIntegrationTest { private static final ObjectMapper MAPPER = new ObjectMapper(); @@ -27,12 +49,11 @@ class FileSystemSerDesIntegrationTest { Path basePath; @Test - void replaysStepAndWaitForConditionStateFromFilesystem() throws Exception { + void pipelineReplaysStepWaitChildAndMapPayloadsFromFilesystem() throws Exception { var stepExecutions = new AtomicInteger(); var pollExecutions = new AtomicInteger(); - var config = DurableConfig.builder() - .withSerDes(FileSystemSerDes.builder(basePath).build()) - .build(); + var serDes = filesystemPipeline(); + var config = DurableConfig.builder().withSerDes(serDes).build(); var runner = LocalDurableTestRunner.create( String.class, @@ -58,7 +79,9 @@ void replaysStepAndWaitForConditionStateFromFilesystem() throws Exception { waitConfig); var childResult = context.runInChildContext( "format-order", String.class, child -> stepResult + "-child"); - return childResult + "-" + pollResult; + var mapResult = context.map( + "map-order", List.of(1, 2), Integer.class, (item, index, child) -> item * 2); + return childResult + "-" + pollResult + "-" + mapResult.results(); }, config) .withOutputType(String.class); @@ -66,14 +89,247 @@ void replaysStepAndWaitForConditionStateFromFilesystem() throws Exception { var result = runner.runUntilComplete("order"); assertEquals(ExecutionStatus.SUCCEEDED, result.getStatus()); - assertEquals("order-loaded-child-2", result.getResult()); + assertEquals("order-loaded-child-2-[2, 4]", result.getResult()); assertEquals(1, stepExecutions.get()); assertEquals(2, pollExecutions.get()); assertEquals("order-loaded", result.getOperation("load-order").getStepResult(String.class)); + assertEnvelopePointsToFile( + result.getOperation("load-order").getStepDetails().result()); + assertEnvelopePointsToFile( + result.getOperation("map-order").getContextDetails().result()); + } + + @Test + void durableExecutorAcceptsRawServiceInputBeforeFilesystemEnvelopeExists() { + var executionArn = + "arn:aws:lambda:us-east-1:123456789012:function:test:$LATEST/durable-execution/execution/raw-input"; + var invocationId = "raw-input"; + var executionName = "execution"; + var executionOperation = Operation.builder() + .id(invocationId) + .name(executionName) + .type(OperationType.EXECUTION) + .status(OperationStatus.STARTED) + .startTimestamp(Instant.now()) + .executionDetails(ExecutionDetails.builder() + .inputPayload("\"service-input\"") + .build()) + .build(); + var input = new DurableExecutionInput( + executionArn, + "checkpoint-token", + CheckpointUpdatedExecutionState.builder() + .operations(executionOperation) + .build(), + List.of()); + var client = new LocalMemoryExecutionClient(); + var serDes = filesystemPipeline(); + var config = DurableConfig.builder() + .withDurableExecutionClient(client) + .withSerDes(serDes) + .build(); + + var output = DurableExecutor.execute( + input, null, TypeToken.get(String.class), (value, context) -> value + "-output", config); + + assertEquals(ExecutionStatus.SUCCEEDED, output.status()); + var result = new SerDesRunner(null) + .deserialize( + serDes, + output.result(), + TypeToken.get(String.class), + SerDesContext.forExecution( + executionArn, invocationId, executionName, SerDesPayloadKind.OUTPUT)); + assertEquals("service-input-output", result); + } + + @Test + void acceptsRawCallbackAndInvokeResultsAndOffloadsInvokePayload() throws Exception { + var invokePayload = new AtomicReference(); + var recordingStage = identityStage((action, value) -> { + var context = SerDesContext.getCurrentContext(); + if ("serialize".equals(action) && context.payloadKind() == SerDesPayloadKind.INVOKE_PAYLOAD) { + invokePayload.set(value); + } + }); + var serDes = new JacksonSerDes() + .then(recordingStage) + .then(FileSystemSerDes.stageBuilder(basePath).build()); + var config = DurableConfig.builder().withSerDes(serDes).build(); + var runner = LocalDurableTestRunner.create( + String.class, + (input, context) -> { + var callback = context.createCallback("approval", String.class); + var approval = callback.get(); + return context.invoke( + "notify", "target-function", Map.of("approval", approval), String.class); + }, + config) + .withOutputType(String.class); + + var waitingForCallback = runner.run("input"); + assertEquals(ExecutionStatus.PENDING, waitingForCallback.getStatus()); + var callbackId = runner.getCallbackId("approval"); + assertNotNull(callbackId); + + runner.completeCallback(callbackId, "\"approved\""); + var waitingForInvoke = runner.run("input"); + assertEquals(ExecutionStatus.PENDING, waitingForInvoke.getStatus()); + assertEquals( + "approved", MAPPER.readTree(invokePayload.get()).get("approval").textValue()); + + runner.completeChainedInvoke("notify", "\"notified\""); + var completed = runner.run("input"); + + assertEquals(ExecutionStatus.SUCCEEDED, completed.getStatus()); + assertEquals("notified", completed.getResult()); + } + + @Test + void repeatedGetUsesInvocationCacheForTheCompletePipeline() { + var resultDeserializations = new AtomicInteger(); + var countingStage = identityStage((action, value) -> { + var context = SerDesContext.getCurrentContext(); + if ("deserialize".equals(action) + && context.payloadKind() == SerDesPayloadKind.RESULT + && "cached-step".equals(context.operationName())) { + resultDeserializations.incrementAndGet(); + } + }); + var serDes = new JacksonSerDes() + .then(countingStage) + .then(FileSystemSerDes.stageBuilder(basePath).build()); + var config = DurableConfig.builder().withSerDes(serDes).build(); + var runner = LocalDurableTestRunner.create( + String.class, + (input, context) -> { + var future = + context.stepAsync("cached-step", Payload.class, stepContext -> new Payload(input)); + var first = future.get(); + var second = future.get(); + assertSame(first, second); + return first.value(); + }, + config) + .withOutputType(String.class); + + var result = runner.runUntilComplete("cached"); + + assertEquals(ExecutionStatus.SUCCEEDED, result.getStatus()); + assertEquals("cached", result.getResult()); + assertEquals(1, resultDeserializations.get()); + } + + @Test + void successfulRetryUsesTheProducingAttemptForResultSerialization() { + var executions = new AtomicInteger(); + var resultAttempts = new ArrayList(); + var attemptStage = identityStage((action, value) -> { + var context = SerDesContext.getCurrentContext(); + if ("serialize".equals(action) + && context.payloadKind() == SerDesPayloadKind.RESULT + && "retry-step".equals(context.operationName())) { + resultAttempts.add(context.attempt()); + } + }); + var serDes = new JacksonSerDes() + .then(attemptStage) + .then(FileSystemSerDes.stageBuilder(basePath).build()); + var config = DurableConfig.builder().withSerDes(serDes).build(); + var stepConfig = StepConfig.builder() + .retryStrategy(RetryStrategies.fixedDelay(2, Duration.ofSeconds(1))) + .build(); + var runner = LocalDurableTestRunner.create( + String.class, + (input, context) -> context.step( + "retry-step", + String.class, + stepContext -> { + if (executions.incrementAndGet() == 1) { + throw new IllegalStateException("retry"); + } + return input + "-attempt-" + stepContext.getAttempt(); + }, + stepConfig), + config) + .withOutputType(String.class); + + var result = runner.runUntilComplete("value"); + + assertEquals(ExecutionStatus.SUCCEEDED, result.getStatus()); + assertEquals("value-attempt-2", result.getResult()); + assertEquals(List.of(2), resultAttempts); + } + + @Test + void customExceptionPayloadsRoundTripThroughFilesystem() throws Exception { + var serDes = filesystemPipeline(); + var config = DurableConfig.builder().withSerDes(serDes).build(); + var stepConfig = StepConfig.builder() + .retryStrategy(RetryStrategies.Presets.NO_RETRY) + .build(); + var runner = LocalDurableTestRunner.create( + String.class, + (input, context) -> context.step( + "fail-step", + String.class, + stepContext -> { + throw new CustomFailure("boom"); + }, + stepConfig), + config) + .withOutputType(String.class); + + var result = runner.runUntilComplete("input"); + + assertEquals(ExecutionStatus.FAILED, result.getStatus()); + assertEquals( + CustomFailure.class.getName(), result.getError().orElseThrow().errorType()); + var operationError = result.getOperation("fail-step").getError(); + assertEquals(CustomFailure.class.getName(), operationError.errorType()); + assertEnvelopePointsToFile(operationError.errorData()); + } + + private SerDes filesystemPipeline() { + return new JacksonSerDes().then(FileSystemSerDes.stageBuilder(basePath).build()); + } + + private static SerDes identityStage(RecordingFunction recorder) { + return new SerDes() { + @Override + public String serialize(Object value) { + var stringValue = (String) value; + recorder.record("serialize", stringValue); + return stringValue; + } + + @Override + @SuppressWarnings("unchecked") + public T deserialize(String data, TypeToken typeToken) { + recorder.record("deserialize", data); + return (T) data; + } + }; + } + + private void assertEnvelopePointsToFile(String envelope) throws Exception { + var file = Path.of(MAPPER.readTree(envelope).get("file").textValue()); + assertTrue(Files.exists(file)); + assertTrue(file.startsWith(basePath)); + } + + @FunctionalInterface + private interface RecordingFunction { + void record(String action, String value); + } + + record Payload(String value) {} + + public static class CustomFailure extends RuntimeException { + public CustomFailure() {} - var stepEnvelope = result.getOperation("load-order").getStepDetails().result(); - var stepFile = Path.of(MAPPER.readTree(stepEnvelope).get("file").textValue()); - assertTrue(Files.exists(stepFile)); - assertTrue(stepFile.startsWith(basePath)); + public CustomFailure(String message) { + super(message); + } } } diff --git a/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/AsyncExecution.java b/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/AsyncExecution.java index 57b6c6921..a21187b50 100644 --- a/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/AsyncExecution.java +++ b/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/AsyncExecution.java @@ -16,6 +16,7 @@ import software.amazon.lambda.durable.TypeToken; import software.amazon.lambda.durable.model.ExecutionStatus; import software.amazon.lambda.durable.serde.SerDes; +import software.amazon.lambda.durable.serde.SerDesRunner; import software.amazon.lambda.durable.testing.cloud.HistoryEventProcessor; /** @@ -30,6 +31,7 @@ public class AsyncExecution { private final Duration pollInterval; private final Duration timeout; private final HistoryEventProcessor processor; + private final SerDesRunner serDesRunner; private List currentHistory; private TestResult currentResult; @@ -47,6 +49,7 @@ public AsyncExecution( this.timeout = timeout; this.serDes = serDes; this.processor = new HistoryEventProcessor(); + this.serDesRunner = new SerDesRunner(null); } /** @@ -195,7 +198,8 @@ private void refreshHistory() { .build(); var response = lambdaClient.getDurableExecutionHistory(request); this.currentHistory = response.events(); - this.currentResult = processor.processEvents(currentHistory, outputType, serDes); + this.currentResult = + processor.processEvents(currentHistory, outputType, serDes, serDesRunner, executionArn); } catch (ResourceNotFoundException e) { // Execution doesn't exist yet - this can happen immediately after async invoke // Leave currentHistory as null, pollUntil will retry diff --git a/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/CloudDurableTestRunner.java b/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/CloudDurableTestRunner.java index b06b0dfc4..d0cb3e6e5 100644 --- a/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/CloudDurableTestRunner.java +++ b/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/CloudDurableTestRunner.java @@ -10,8 +10,10 @@ import software.amazon.awssdk.services.lambda.model.InvocationType; import software.amazon.awssdk.services.lambda.model.InvokeRequest; import software.amazon.lambda.durable.TypeToken; +import software.amazon.lambda.durable.serde.ComposableSerDes; import software.amazon.lambda.durable.serde.JacksonSerDes; import software.amazon.lambda.durable.serde.SerDes; +import software.amazon.lambda.durable.serde.SerDesRunner; import software.amazon.lambda.durable.testing.cloud.HistoryEventProcessor; import software.amazon.lambda.durable.testing.cloud.HistoryPoller; @@ -30,6 +32,7 @@ public class CloudDurableTestRunner { private final Duration pollInterval; private final Duration timeout; private final InvocationType invocationType; + private final SerDes inputSerDes; private final SerDes serDes; // Store last execution result for operation inspection private TestResult lastResult; @@ -42,6 +45,7 @@ private CloudDurableTestRunner( Duration pollInterval, Duration timeout, InvocationType invocationType, + SerDes inputSerDes, SerDes serDes) { this.functionArn = functionArn; this.inputType = inputType; @@ -52,6 +56,7 @@ private CloudDurableTestRunner( this.timeout = timeout; this.invocationType = invocationType; this.serDes = Objects.requireNonNullElseGet(serDes, JacksonSerDes::new); + this.inputSerDes = Objects.requireNonNullElse(inputSerDes, this.serDes); } private static LambdaClient createDefaultLambdaClient() { @@ -77,6 +82,7 @@ public static CloudDurableTestRunner create( Duration.ofSeconds(2), Duration.ofSeconds(300), InvocationType.REQUEST_RESPONSE, + null, null); } @@ -97,36 +103,89 @@ public static CloudDurableTestRunner create( Duration.ofSeconds(2), Duration.ofSeconds(300), InvocationType.REQUEST_RESPONSE, + null, null); } /** Returns a new runner with the specified lambda client. */ public CloudDurableTestRunner withLambdaClient(LambdaClient lambdaClient) { return new CloudDurableTestRunner<>( - functionArn, inputType, outputType, lambdaClient, pollInterval, timeout, invocationType, serDes); + functionArn, + inputType, + outputType, + lambdaClient, + pollInterval, + timeout, + invocationType, + inputSerDes, + serDes); } /** Returns a new runner with the specified poll interval between history checks. */ public CloudDurableTestRunner withPollInterval(Duration interval) { return new CloudDurableTestRunner<>( - functionArn, inputType, outputType, lambdaClient, interval, timeout, invocationType, serDes); + functionArn, + inputType, + outputType, + lambdaClient, + interval, + timeout, + invocationType, + inputSerDes, + serDes); } /** Returns a new runner with the specified maximum wait time for execution completion. */ public CloudDurableTestRunner withTimeout(Duration timeout) { return new CloudDurableTestRunner<>( - functionArn, inputType, outputType, lambdaClient, pollInterval, timeout, invocationType, serDes); + functionArn, + inputType, + outputType, + lambdaClient, + pollInterval, + timeout, + invocationType, + inputSerDes, + serDes); } /** Returns a new runner with the specified Lambda invocation type. */ public CloudDurableTestRunner withInvocationType(InvocationType type) { return new CloudDurableTestRunner<>( - functionArn, inputType, outputType, lambdaClient, pollInterval, timeout, type, serDes); + functionArn, inputType, outputType, lambdaClient, pollInterval, timeout, type, inputSerDes, serDes); } + /** Returns a new runner with the specified SerDes for persisted execution payloads. */ public CloudDurableTestRunner withSerDes(SerDes serDes) { return new CloudDurableTestRunner<>( - functionArn, inputType, outputType, lambdaClient, pollInterval, timeout, invocationType, serDes); + functionArn, + inputType, + outputType, + lambdaClient, + pollInterval, + timeout, + invocationType, + serDes, + serDes); + } + + /** + * Returns a new runner with a separate SerDes for the initial Lambda invocation payload. + * + *

This is useful with a standalone context-dependent SerDes. Composable pipelines automatically use their first + * value-codec stage for initial input. + */ + public CloudDurableTestRunner withInputSerDes(SerDes inputSerDes) { + return new CloudDurableTestRunner<>( + functionArn, + inputType, + outputType, + lambdaClient, + pollInterval, + timeout, + invocationType, + Objects.requireNonNull(inputSerDes, "inputSerDes cannot be null"), + serDes); } /** Invokes the Lambda function, polls execution history until completion, and returns the result. */ @@ -138,7 +197,7 @@ public TestResult runUntilComplete(I input) { public TestResult run(I input) { try { // Serialize input - var inputJson = serDes.serialize(input); + var inputJson = serializeInput(input); // Invoke function var invokeRequest = InvokeRequest.builder() @@ -161,7 +220,7 @@ public TestResult run(I input) { // Process events into TestResult var processor = new HistoryEventProcessor(); - var result = processor.processEvents(events, outputType, serDes); + var result = processor.processEvents(events, outputType, serDes, new SerDesRunner(null), executionArn); this.lastResult = result; return result; } catch (Exception e) { @@ -179,7 +238,7 @@ public TestResult run(I input) { public AsyncExecution startAsync(I input) { try { // Serialize input - var inputJson = serDes.serialize(input); + var inputJson = serializeInput(input); // Invoke function with EVENT type (async) var invokeRequest = InvokeRequest.builder() @@ -216,4 +275,9 @@ public TestOperation getOperation(String name) { } return lastResult.getOperation(name); } + + private String serializeInput(I input) { + var serializer = inputSerDes instanceof ComposableSerDes composable ? composable.getValueCodec() : inputSerDes; + return serializer.serialize(input); + } } diff --git a/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/LocalDurableTestRunner.java b/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/LocalDurableTestRunner.java index b7d7cc025..3eb5d11f4 100644 --- a/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/LocalDurableTestRunner.java +++ b/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/LocalDurableTestRunner.java @@ -65,19 +65,21 @@ private LocalDurableTestRunner( // Create config that uses customer's configuration but overrides the client with in-memory storage if (customerConfig != null) { // Use customer's config but override the client with our in-memory implementation - this.customerConfig = DurableConfig.builder() + var configBuilder = DurableConfig.builder() .withDurableExecutionClient(storage) .withSerDes(customerConfig.getSerDes()) .withExecutorService(customerConfig.getExecutorService()) - .withSerDesExecutorService(customerConfig.getSerDesExecutorService()) .withPollingStrategy(customerConfig.getPollingStrategy()) .withCheckpointDelay(customerConfig.getCheckpointDelay()) .withLoggerConfig(customerConfig.getLoggerConfig()) // Temporary: remove along with the checkpointEmptyMap flag in a future major version. .withCheckpointEmptyMap(customerConfig.shouldCheckpointEmptyMap()) .withDeserializeAfterSerialization(customerConfig.shouldDeserializeAfterSerialization()) - .withPlugins(customerConfig.getPluginRunner().getPlugins().toArray(new DurableExecutionPlugin[0])) - .build(); + .withPlugins(customerConfig.getPluginRunner().getPlugins().toArray(new DurableExecutionPlugin[0])); + if (customerConfig.getSerDesExecutorService() != null) { + configBuilder.withSerDesExecutorService(customerConfig.getSerDesExecutorService()); + } + this.customerConfig = configBuilder.build(); } else { // Fallback to default config with in-memory client this.customerConfig = diff --git a/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/cloud/HistoryEventProcessor.java b/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/cloud/HistoryEventProcessor.java index 4a3b8f1b2..eeefa4e52 100644 --- a/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/cloud/HistoryEventProcessor.java +++ b/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/cloud/HistoryEventProcessor.java @@ -10,6 +10,7 @@ import software.amazon.awssdk.services.lambda.model.ContextDetails; import software.amazon.awssdk.services.lambda.model.ErrorObject; import software.amazon.awssdk.services.lambda.model.Event; +import software.amazon.awssdk.services.lambda.model.EventType; import software.amazon.awssdk.services.lambda.model.Operation; import software.amazon.awssdk.services.lambda.model.OperationStatus; import software.amazon.awssdk.services.lambda.model.OperationType; @@ -18,6 +19,7 @@ import software.amazon.lambda.durable.TypeToken; import software.amazon.lambda.durable.model.ExecutionStatus; import software.amazon.lambda.durable.serde.SerDes; +import software.amazon.lambda.durable.serde.SerDesRunner; import software.amazon.lambda.durable.testing.AsyncExecution; import software.amazon.lambda.durable.testing.CloudDurableTestRunner; import software.amazon.lambda.durable.testing.TestOperation; @@ -37,11 +39,33 @@ public class HistoryEventProcessor { * @return a TestResult containing the execution status, output, and operation details */ public TestResult processEvents(List events, TypeToken outputType, SerDes serDes) { + return processEvents(events, outputType, serDes, null, null); + } + + /** + * Processes execution history using SDK-managed SerDes context. + * + * @param events the raw history events from the GetDurableExecutionHistory API + * @param outputType the expected output type for deserialization + * @param serDes the SerDes used by the durable function + * @param serDesRunner invocation-scoped SerDes runner, or {@code null} for legacy direct calls + * @param durableExecutionArn durable execution ARN, required when {@code serDesRunner} is supplied + * @param the handler output type + * @return a TestResult containing the execution status, output, and operation details + */ + public TestResult processEvents( + List events, + TypeToken outputType, + SerDes serDes, + SerDesRunner serDesRunner, + String durableExecutionArn) { var operations = new HashMap(); var operationEvents = new HashMap>(); var status = ExecutionStatus.PENDING; String result = null; ErrorObject error = null; + String executionOperationId = null; + String executionOperationName = null; for (var event : events) { var eventType = event.eventType(); @@ -56,7 +80,8 @@ public TestResult processEvents(List events, TypeToken outputTy switch (eventType) { case EXECUTION_STARTED -> { - // Execution started - no action needed, just track the event + executionOperationId = operationId; + executionOperationName = event.name(); } case INVOCATION_COMPLETED -> { var details = event.invocationCompletedDetails(); @@ -111,7 +136,14 @@ public TestResult processEvents(List events, TypeToken outputTy if (operationId != null) { operations.putIfAbsent( operationId, - createStepOperation(operationId, event.name(), null, OperationStatus.STARTED, 1)); + createStepOperation( + operationId, + event.name(), + event.parentId(), + event.subType(), + null, + OperationStatus.STARTED, + 1)); } } case STEP_SUCCEEDED -> { @@ -126,7 +158,13 @@ public TestResult processEvents(List events, TypeToken outputTy operations.put( operationId, createStepOperation( - operationId, event.name(), stepResult, OperationStatus.SUCCEEDED, attempt)); + operationId, + event.name(), + event.parentId(), + event.subType(), + stepResult, + OperationStatus.SUCCEEDED, + attempt)); } } case STEP_FAILED -> { @@ -137,7 +175,14 @@ public TestResult processEvents(List events, TypeToken outputTy : 1; operations.put( operationId, - createStepOperation(operationId, event.name(), null, OperationStatus.FAILED, attempt)); + createStepOperation( + operationId, + event.name(), + event.parentId(), + event.subType(), + null, + OperationStatus.FAILED, + attempt)); } } @@ -224,7 +269,11 @@ public TestResult processEvents(List events, TypeToken outputTy CHAINED_INVOKE_TIMED_OUT, CHAINED_INVOKE_STOPPED -> { if (operationId != null) { - operations.putIfAbsent(operationId, createInvokeOperation(operationId, event)); + if (eventType == EventType.CHAINED_INVOKE_STARTED) { + operations.putIfAbsent(operationId, createInvokeOperation(operationId, event)); + } else { + operations.put(operationId, createInvokeOperation(operationId, event)); + } } } @@ -236,14 +285,40 @@ public TestResult processEvents(List events, TypeToken outputTy var testOperations = new ArrayList(); for (var entry : operations.entrySet()) { var opEvents = operationEvents.getOrDefault(entry.getKey(), List.of()); - testOperations.add(new TestOperation(entry.getValue(), opEvents, serDes)); + testOperations.add( + serDesRunner == null + ? new TestOperation(entry.getValue(), opEvents, serDes) + : new TestOperation(entry.getValue(), opEvents, serDes, serDesRunner, durableExecutionArn)); } - return new TestResult<>(status, result, error, testOperations, events, outputType, serDes); + if (executionOperationId == null && durableExecutionArn != null) { + var parts = durableExecutionArn.split("/", -1); + executionOperationId = parts[parts.length - 1]; + } + return serDesRunner == null + ? new TestResult<>(status, result, error, testOperations, events, outputType, serDes) + : new TestResult<>( + status, + result, + error, + testOperations, + events, + outputType, + serDes, + serDesRunner, + durableExecutionArn, + executionOperationId, + executionOperationName); } private Operation createStepOperation( - String id, String name, String stepResult, OperationStatus status, Integer attempt) { + String id, + String name, + String parentId, + String subType, + String stepResult, + OperationStatus status, + Integer attempt) { var stepDetails = StepDetails.builder() .result(stepResult) .attempt(attempt != null ? attempt : 1) @@ -252,8 +327,10 @@ private Operation createStepOperation( return Operation.builder() .id(id) .name(name) + .parentId(parentId) .status(status) .type(OperationType.STEP) + .subType(subType) .stepDetails(stepDetails) .build(); } @@ -267,8 +344,10 @@ private Operation createWaitOperation(String id, String name, OperationStatus st return Operation.builder() .id(id) .name(name) + .parentId(event.parentId()) .status(status) .type(OperationType.WAIT) + .subType(event.subType()) .waitDetails(builder.build()) .build(); } @@ -302,8 +381,10 @@ private Operation createCallbackOperation(String id, String name, OperationStatu return Operation.builder() .id(id) .name(name) + .parentId(event.parentId()) .status(status) .type(OperationType.CALLBACK) + .subType(event.subType()) .callbackDetails(builder.build()) .build(); } @@ -315,7 +396,7 @@ private Operation createInvokeOperation(String id, Event event) { switch (event.eventType()) { case CHAINED_INVOKE_STARTED -> OperationStatus.STARTED; case CHAINED_INVOKE_SUCCEEDED -> { - var details = event.callbackSucceededDetails(); + var details = event.chainedInvokeSucceededDetails(); if (details != null && details.result() != null && details.result().payload() != null) { @@ -324,7 +405,7 @@ private Operation createInvokeOperation(String id, Event event) { yield OperationStatus.SUCCEEDED; } case CHAINED_INVOKE_FAILED -> { - var details = event.callbackFailedDetails(); + var details = event.chainedInvokeFailedDetails(); if (details != null && details.error() != null && details.error().payload() != null) { @@ -359,8 +440,10 @@ private Operation createInvokeOperation(String id, Event event) { return Operation.builder() .id(id) .name(event.name()) + .parentId(event.parentId()) .status(status) .type(OperationType.CHAINED_INVOKE) + .subType(event.subType()) .chainedInvokeDetails(builder.build()) .build(); } @@ -383,6 +466,7 @@ private Operation createContextOperation(String id, String name, OperationStatus return Operation.builder() .id(id) .name(name) + .parentId(event.parentId()) .status(status) .type(OperationType.CONTEXT) .subType(event.subType()) diff --git a/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/local/LocalMemoryExecutionClient.java b/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/local/LocalMemoryExecutionClient.java index ca76f6414..ca9cd0861 100644 --- a/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/local/LocalMemoryExecutionClient.java +++ b/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/local/LocalMemoryExecutionClient.java @@ -131,6 +131,11 @@ public List getUpdatedOperationIdsSinceLastInvocation() { } /** Build TestResult from current state. */ + public TestResult toTestResult(DurableExecutionOutput output, TypeToken resultType, SerDes serDes) { + return toTestResult(output, resultType, serDes, null, null, null, null); + } + + /** Build a context-aware TestResult from current state. */ public TestResult toTestResult( DurableExecutionOutput output, TypeToken resultType, diff --git a/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/cloud/HistoryEventProcessorTest.java b/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/cloud/HistoryEventProcessorTest.java new file mode 100644 index 000000000..dae253b91 --- /dev/null +++ b/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/cloud/HistoryEventProcessorTest.java @@ -0,0 +1,122 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.testing.cloud; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.util.ArrayList; +import java.util.List; +import org.junit.jupiter.api.Test; +import software.amazon.awssdk.services.lambda.model.ChainedInvokeStartedDetails; +import software.amazon.awssdk.services.lambda.model.ChainedInvokeSucceededDetails; +import software.amazon.awssdk.services.lambda.model.Event; +import software.amazon.awssdk.services.lambda.model.EventResult; +import software.amazon.awssdk.services.lambda.model.EventType; +import software.amazon.awssdk.services.lambda.model.ExecutionStartedDetails; +import software.amazon.awssdk.services.lambda.model.ExecutionSucceededDetails; +import software.amazon.awssdk.services.lambda.model.OperationStatus; +import software.amazon.awssdk.services.lambda.model.OperationType; +import software.amazon.awssdk.services.lambda.model.RetryDetails; +import software.amazon.awssdk.services.lambda.model.StepSucceededDetails; +import software.amazon.lambda.durable.TypeToken; +import software.amazon.lambda.durable.serde.SerDes; +import software.amazon.lambda.durable.serde.SerDesContext; +import software.amazon.lambda.durable.serde.SerDesPayloadKind; +import software.amazon.lambda.durable.serde.SerDesRunner; + +class HistoryEventProcessorTest { + private static final String EXECUTION_ARN = "arn:aws:lambda:us-east-1:123456789012:function:test:$LATEST" + + "/durable-execution/execution-id/invocation-id"; + + @Test + void deserializesCloudResultsWithDurablePayloadContext() { + var observedContexts = new ArrayList(); + var serDes = recordingStringSerDes(observedContexts); + var events = List.of( + Event.builder() + .id("invocation-id") + .name("execution") + .eventType(EventType.EXECUTION_STARTED) + .executionStartedDetails( + ExecutionStartedDetails.builder().build()) + .build(), + Event.builder() + .id("step-id") + .name("step") + .eventType(EventType.STEP_SUCCEEDED) + .stepSucceededDetails(StepSucceededDetails.builder() + .result(EventResult.builder() + .payload("step-result") + .build()) + .retryDetails( + RetryDetails.builder().currentAttempt(2).build()) + .build()) + .build(), + Event.builder() + .id("invoke-id") + .name("invoke") + .eventType(EventType.CHAINED_INVOKE_STARTED) + .chainedInvokeStartedDetails(ChainedInvokeStartedDetails.builder() + .functionName("target") + .build()) + .build(), + Event.builder() + .id("invoke-id") + .name("invoke") + .eventType(EventType.CHAINED_INVOKE_SUCCEEDED) + .chainedInvokeSucceededDetails(ChainedInvokeSucceededDetails.builder() + .result(EventResult.builder() + .payload("invoke-result") + .build()) + .build()) + .build(), + Event.builder() + .id("invocation-id") + .name("execution") + .eventType(EventType.EXECUTION_SUCCEEDED) + .executionSucceededDetails(ExecutionSucceededDetails.builder() + .result(EventResult.builder() + .payload("execution-result") + .build()) + .build()) + .build()); + + var result = new HistoryEventProcessor() + .processEvents(events, TypeToken.get(String.class), serDes, new SerDesRunner(null), EXECUTION_ARN); + + assertEquals("execution-result", result.getResult()); + assertEquals("step-result", result.getOperation("step").getStepResult(String.class)); + assertEquals(OperationStatus.SUCCEEDED, result.getOperation("invoke").getStatus()); + assertEquals( + "invoke-result", + result.getOperation("invoke").getChainedInvokeDetails().result()); + assertEquals(2, observedContexts.size()); + + var outputContext = observedContexts.get(0); + assertEquals(OperationType.EXECUTION, outputContext.operationType()); + assertEquals(SerDesPayloadKind.OUTPUT, outputContext.payloadKind()); + assertEquals("execution/invocation-id/output", outputContext.entityId()); + + var stepContext = observedContexts.get(1); + assertEquals(OperationType.STEP, stepContext.operationType()); + assertEquals(SerDesPayloadKind.RESULT, stepContext.payloadKind()); + assertEquals("operation/step-id/result/attempt-2", stepContext.entityId()); + assertEquals(2, stepContext.attempt()); + } + + private static SerDes recordingStringSerDes(List observedContexts) { + return new SerDes() { + @Override + public String serialize(Object value) { + return value.toString(); + } + + @Override + @SuppressWarnings("unchecked") + public T deserialize(String data, TypeToken typeToken) { + observedContexts.add(SerDesContext.getCurrentContext()); + return (T) data; + } + }; + } +} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/DurableConfig.java b/sdk/src/main/java/software/amazon/lambda/durable/DurableConfig.java index d0164722c..3bd0ad7c6 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/DurableConfig.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/DurableConfig.java @@ -92,13 +92,6 @@ public final class DurableConfig { return t; }); - private static final ExecutorService DEFAULT_SERDES_THREAD_POOL = Executors.newCachedThreadPool(r -> { - Thread t = new Thread(r); - t.setName("durable-sdk-serdes-" + t.getId()); - t.setDaemon(true); - return t; - }); - private final DurableExecutionClient durableExecutionClient; private final SerDes serDes; private final ExecutorService executorService; @@ -117,8 +110,7 @@ private DurableConfig(Builder builder) { this.serDes = Objects.requireNonNullElseGet(builder.serDes, JacksonSerDes::new); this.executorService = Objects.requireNonNullElseGet(builder.executorService, DurableConfig::createDefaultExecutor); - this.serDesExecutorService = - Objects.requireNonNullElse(builder.serDesExecutorService, DEFAULT_SERDES_THREAD_POOL); + this.serDesExecutorService = builder.serDesExecutorService; this.loggerConfig = Objects.requireNonNullElseGet(builder.loggerConfig, LoggerConfig::defaults); this.pollingStrategy = Objects.requireNonNullElse(builder.pollingStrategy, PollingStrategies.Presets.DEFAULT); this.checkpointDelay = Objects.requireNonNullElseGet(builder.checkpointDelay, () -> Duration.ofSeconds(0)); @@ -174,7 +166,11 @@ public ExecutorService getExecutorService() { return executorService; } - /** Gets the executor used for customer SerDes calls and payload storage I/O. */ + /** + * Gets the executor used for customer SerDes calls and payload storage I/O. + * + * @return the configured executor, or {@code null} when SerDes calls execute inline + */ public ExecutorService getSerDesExecutorService() { return serDesExecutorService; } @@ -250,8 +246,9 @@ public void validateConfiguration() { if (getExecutorService() == null) { throw new IllegalStateException("ExecutorService configuration failed"); } - if (getSerDesExecutorService() == null) { - throw new IllegalStateException("SerDes ExecutorService configuration failed"); + if (getSerDesExecutorService() != null && getSerDesExecutorService() == getExecutorService()) { + throw new IllegalStateException( + "SerDes ExecutorService must be different from the user operation ExecutorService"); } } @@ -416,8 +413,14 @@ public Builder withExecutorService(ExecutorService executorService) { } /** - * Sets the executor used for customer SerDes calls and blocking payload storage I/O. If not set, a cached - * daemon thread pool named {@code durable-sdk-serdes-*} is used. + * Sets the executor used for customer SerDes calls and blocking payload storage I/O. If not set, SerDes calls + * execute inline on the calling thread. + * + *

This executor must be different from the user operation executor to prevent synchronous SerDes dispatch + * from deadlocking a saturated operation pool. + * + * @param executorService the dedicated SerDes executor + * @return this builder */ public Builder withSerDesExecutorService(ExecutorService executorService) { this.serDesExecutorService = diff --git a/sdk/src/main/java/software/amazon/lambda/durable/operation/BaseDurableOperation.java b/sdk/src/main/java/software/amazon/lambda/durable/operation/BaseDurableOperation.java index 8f7acedca..19e271fb9 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/operation/BaseDurableOperation.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/operation/BaseDurableOperation.java @@ -6,7 +6,6 @@ import java.util.List; import java.util.Objects; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ForkJoinPool; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Supplier; @@ -92,8 +91,7 @@ protected BaseDurableOperation( this.durableContext = durableContext; this.executionManager = durableContext.getExecutionManager(); var invocationSerDesRunner = executionManager.getSerDesRunner(); - this.serDesRunner = - invocationSerDesRunner != null ? invocationSerDesRunner : new SerDesRunner(ForkJoinPool.commonPool()); + this.serDesRunner = invocationSerDesRunner != null ? invocationSerDesRunner : new SerDesRunner(null); this.isVirtual = isVirtual; this.completionFuture = new CompletableFuture<>(); diff --git a/sdk/src/main/java/software/amazon/lambda/durable/operation/StepOperation.java b/sdk/src/main/java/software/amazon/lambda/durable/operation/StepOperation.java index 6123eb56e..09a86ff2e 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/operation/StepOperation.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/operation/StepOperation.java @@ -25,6 +25,7 @@ import software.amazon.lambda.durable.execution.ThreadType; import software.amazon.lambda.durable.logging.DurableLogger; import software.amazon.lambda.durable.model.OperationIdentifier; +import software.amazon.lambda.durable.serde.SerDesPayloadKind; import software.amazon.lambda.durable.util.ExceptionHelper; /** @@ -117,7 +118,7 @@ private void executeStepLogic(int attempt) { // through onUserFunctionEnd; retry/checkpoint handling stays outside the boundary. T result = runUserFunction(attempt, () -> function.apply(stepContext)); - handleStepSucceeded(result); + handleStepSucceeded(result, attempt); } catch (Throwable e) { handleStepFailure(e, attempt); } @@ -144,8 +145,8 @@ private void checkpointStarted() { } } - private void handleStepSucceeded(T result) { - var serializedResult = serializeAndDeserializeResult(result); + private void handleStepSucceeded(T result, int attempt) { + var serializedResult = serializeAndDeserializeResult(result, SerDesPayloadKind.RESULT, attempt); // Send SUCCEED var successUpdate = @@ -205,8 +206,9 @@ public T get() { if (op.status() == OperationStatus.SUCCEEDED) { var stepDetails = op.stepDetails(); var result = (stepDetails != null) ? stepDetails.result() : null; + var attempt = stepDetails != null ? stepDetails.attempt() : null; - return deserializeResult(result); + return deserializeResult(result, SerDesPayloadKind.RESULT, attempt); } else { var errorObject = op.stepDetails().error(); diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/ComposableSerDes.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/ComposableSerDes.java new file mode 100644 index 000000000..75cd38faf --- /dev/null +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/ComposableSerDes.java @@ -0,0 +1,194 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.serde; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; +import software.amazon.lambda.durable.TypeToken; +import software.amazon.lambda.durable.exception.RetryableSerDesException; +import software.amazon.lambda.durable.exception.SerDesException; + +/** + * An immutable SerDes processing pipeline. + * + *

The first stage is the value codec. Every later stage must be a reversible string transformation. Serialization + * runs from first to last; deserialization runs from last to first. + */ +public final class ComposableSerDes implements SerDes { + private static final TypeToken STRING_TYPE = TypeToken.get(String.class); + + private final List stages; + + private ComposableSerDes(List stages) { + if (stages.isEmpty()) { + throw new IllegalArgumentException("ComposableSerDes requires at least one stage"); + } + this.stages = List.copyOf(stages); + } + + /** + * Creates a pipeline with a value codec followed by zero or more string stages. + * + * @param first the value codec + * @param remaining reversible string stages + * @return an immutable pipeline + */ + public static ComposableSerDes of(SerDes first, SerDes... remaining) { + Objects.requireNonNull(remaining, "remaining stages cannot be null"); + var stages = new ArrayList(); + addFlattened(stages, Objects.requireNonNull(first, "first stage cannot be null")); + Arrays.stream(remaining) + .map(stage -> Objects.requireNonNull(stage, "pipeline stage cannot be null")) + .forEach(stage -> addFlattened(stages, stage)); + return new ComposableSerDes(stages); + } + + /** + * Creates a pipeline builder. + * + * @param valueCodec the first stage which converts values to and from strings + * @return a new builder + */ + public static Builder builder(SerDes valueCodec) { + return new Builder(valueCodec); + } + + /** + * Returns the value codec at the start of this pipeline. + * + *

This is useful at external input boundaries where a durable execution context does not exist yet and only the + * domain value encoding can be applied. + * + * @return the first pipeline stage + */ + public SerDes getValueCodec() { + return stages.get(0); + } + + /** + * Returns a new pipeline with the supplied stage appended. + * + * @param stage the reversible string stage to append + * @return a new immutable pipeline + */ + @Override + public ComposableSerDes then(SerDes stage) { + var combined = new ArrayList<>(stages); + addFlattened(combined, Objects.requireNonNull(stage, "stage cannot be null")); + return new ComposableSerDes(combined); + } + + @Override + public String serialize(Object value) { + if (value == null) { + return null; + } + String current = invokeSerialize(stages.get(0), value, 0); + for (int index = 1; index < stages.size(); index++) { + current = invokeSerialize(stages.get(index), current, index); + } + return current; + } + + @Override + public T deserialize(String data, TypeToken typeToken) { + if (data == null) { + return null; + } + Objects.requireNonNull(typeToken, "typeToken cannot be null"); + String current = data; + for (int index = stages.size() - 1; index > 0; index--) { + var decoded = invokeStringStageDeserialize(stages.get(index), current, index); + if (!(decoded instanceof String stringValue)) { + throw stageFailure( + index, + stages.get(index), + "deserialize", + new SerDesException("String stage returned a non-string value")); + } + current = stringValue; + } + return invokeDeserialize(stages.get(0), current, typeToken, 0); + } + + private static Object invokeStringStageDeserialize(SerDes stage, String data, int index) { + try { + Object result = stage.deserialize(data, STRING_TYPE); + if (result == null) { + throw new SerDesException("Stage returned null for non-null data"); + } + return result; + } catch (Throwable failure) { + throw stageFailure(index, stage, "deserialize", failure); + } + } + + private static String invokeSerialize(SerDes stage, Object value, int index) { + try { + var result = stage.serialize(value); + if (result == null) { + throw new SerDesException("Stage returned null for a non-null value"); + } + return result; + } catch (Throwable failure) { + throw stageFailure(index, stage, "serialize", failure); + } + } + + private static T invokeDeserialize(SerDes stage, String data, TypeToken typeToken, int index) { + try { + var result = stage.deserialize(data, typeToken); + if (result == null) { + throw new SerDesException("Stage returned null for non-null data"); + } + return result; + } catch (Throwable failure) { + throw stageFailure(index, stage, "deserialize", failure); + } + } + + private static RuntimeException stageFailure(int index, SerDes stage, String action, Throwable failure) { + var message = String.format( + "SerDes pipeline stage %d (%s) failed to %s", + index, stage.getClass().getName(), action); + if (failure instanceof RetryableSerDesException) { + return new RetryableSerDesException(message, failure); + } + return new SerDesException(message, failure); + } + + private static void addFlattened(List target, SerDes stage) { + if (stage instanceof ComposableSerDes composable) { + target.addAll(composable.stages); + } else { + target.add(stage); + } + } + + /** Builder for an immutable {@link ComposableSerDes}. */ + public static final class Builder { + private final List stages = new ArrayList<>(); + + private Builder(SerDes valueCodec) { + addFlattened(stages, Objects.requireNonNull(valueCodec, "valueCodec cannot be null")); + } + + /** + * Appends a reversible string stage. + * + * @param stage the stage to append + * @return this builder + */ + public Builder then(SerDes stage) { + addFlattened(stages, Objects.requireNonNull(stage, "stage cannot be null")); + return this; + } + + /** Returns the immutable pipeline. */ + public ComposableSerDes build() { + return new ComposableSerDes(stages); + } + } +} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/RetrySerDes.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/RetrySerDes.java index a32fe922b..74428d9c4 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/RetrySerDes.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/RetrySerDes.java @@ -16,7 +16,7 @@ * A SerDes decorator that retries transient failures from another {@link SerDes}. * *

Only {@link RetryableSerDesException} is retried. Other failures are propagated immediately. Retry delays block - * the calling thread, which is the dedicated SerDes executor thread for SDK-managed calls. + * the thread executing the SerDes call: the caller by default or the configured SerDes executor thread. */ public final class RetrySerDes implements SerDes { private static final Sleeper DEFAULT_SLEEPER = delay -> { diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDes.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDes.java index b8f39e1c1..b353cec80 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDes.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDes.java @@ -36,4 +36,17 @@ public interface SerDes { * @return the deserialized object, or null if data is null */ T deserialize(String data, TypeToken typeToken); + + /** + * Returns an immutable processing pipeline that invokes this SerDes followed by {@code nextStage} when serializing + * and in reverse order when deserializing. + * + *

This SerDes is the value codec. The next stage must accept and return strings. + * + * @param nextStage the reversible string-processing stage to append + * @return a composable SerDes pipeline + */ + default ComposableSerDes then(SerDes nextStage) { + return ComposableSerDes.of(this, nextStage); + } } diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesContext.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesContext.java index 71da1fdd5..26d09006c 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesContext.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesContext.java @@ -55,9 +55,13 @@ public static SerDesContext forOperation( OperationSubType operationSubType, SerDesPayloadKind payloadKind, Integer attempt) { + var entityId = "operation/" + operationId + "/" + payloadKind.getEntitySuffix(); + if (attempt != null) { + entityId += "/attempt-" + attempt; + } return new SerDesContext( durableExecutionArn, - "operation/" + operationId + "/" + payloadKind.getEntitySuffix(), + entityId, payloadKind, operationId, operationName, diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesRunner.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesRunner.java index b65c82dab..ecc24d457 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesRunner.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesRunner.java @@ -13,32 +13,40 @@ import java.util.concurrent.ExecutorService; import java.util.function.Supplier; import software.amazon.lambda.durable.TypeToken; +import software.amazon.lambda.durable.exception.RetryableSerDesException; import software.amazon.lambda.durable.exception.SerDesException; import software.amazon.lambda.durable.util.ExceptionHelper; /** - * Runs customer SerDes calls on the configured SerDes executor with the correct {@link SerDesContext}. + * Runs customer SerDes calls with the correct {@link SerDesContext}. * - *

Instances are invocation-scoped so successful deserialization results are cached only for one Lambda invocation. + *

Calls execute inline unless an executor is configured. Instances are invocation-scoped so successful + * deserialization results are cached only for one Lambda invocation. */ public final class SerDesRunner { - private static final Object NULL_VALUE = new Object(); - private final ExecutorService executorService; - private final Map deserializationCache = new ConcurrentHashMap<>(); + private final Map> deserializationCache = new ConcurrentHashMap<>(); + /** + * Creates an invocation-scoped runner. + * + * @param executorService executor for SerDes calls, or {@code null} to execute inline + */ public SerDesRunner(ExecutorService executorService) { - this.executorService = Objects.requireNonNull(executorService, "executorService cannot be null"); + this.executorService = executorService; } /** Serializes a value with the supplied durable payload context. */ public String serialize(SerDes serDes, Object value, SerDesContext context) { + Objects.requireNonNull(serDes, "serDes cannot be null"); return run("serialize", context, () -> serDes.serialize(value)); } /** Deserializes a value with invocation-scoped caching. */ @SuppressWarnings("unchecked") public T deserialize(SerDes serDes, String data, TypeToken typeToken, SerDesContext context) { + Objects.requireNonNull(serDes, "serDes cannot be null"); + Objects.requireNonNull(typeToken, "typeToken cannot be null"); Objects.requireNonNull(context, "SerDesContext cannot be null"); var key = new CacheKey( context.durableExecutionArn(), @@ -47,37 +55,64 @@ public T deserialize(SerDes serDes, String data, TypeToken typeToken, Ser context.attempt(), typeToken, hash(data)); - var cached = deserializationCache.get(key); - if (cached != null) { - return cached == NULL_VALUE ? null : (T) cached; + var pending = new CompletableFuture(); + var existing = deserializationCache.putIfAbsent(key, pending); + if (existing != null) { + return (T) join(existing); } - T value = run("deserialize", context, () -> serDes.deserialize(data, typeToken)); - deserializationCache.putIfAbsent(key, value == null ? NULL_VALUE : value); - return value; + try { + T value = run("deserialize", context, () -> serDes.deserialize(data, typeToken)); + pending.complete(value); + return value; + } catch (Throwable failure) { + pending.completeExceptionally(failure); + deserializationCache.remove(key, pending); + ExceptionHelper.sneakyThrow(failure); + return null; + } } private T run(String action, SerDesContext context, Supplier supplier) { + Objects.requireNonNull(supplier, "supplier cannot be null"); Objects.requireNonNull(context, "SerDesContext cannot be null"); try { - return CompletableFuture.supplyAsync( - () -> { - SerDesContextHolder.set(context); - try { - return supplier.get(); - } finally { - SerDesContextHolder.clear(); - } - }, - executorService) + if (executorService == null) { + return runWithContext(context, supplier); + } + return CompletableFuture.supplyAsync(() -> runWithContext(context, supplier), executorService) .join(); } catch (Throwable throwable) { var cause = ExceptionHelper.unwrapCompletableFuture(throwable); - throw new SerDesException( - String.format( - "Failed to %s %s payload for entity '%s'", - action, context.payloadKind(), context.entityId()), - cause); + var message = String.format( + "Failed to %s %s payload for entity '%s'", action, context.payloadKind(), context.entityId()); + if (cause instanceof RetryableSerDesException) { + throw new RetryableSerDesException(message, cause); + } + throw new SerDesException(message, cause); + } + } + + private static T runWithContext(SerDesContext context, Supplier supplier) { + var previous = SerDesContextHolder.get(); + SerDesContextHolder.set(context); + try { + return supplier.get(); + } finally { + if (previous == null) { + SerDesContextHolder.clear(); + } else { + SerDesContextHolder.set(previous); + } + } + } + + private static Object join(CompletableFuture future) { + try { + return future.join(); + } catch (Throwable failure) { + ExceptionHelper.sneakyThrow(ExceptionHelper.unwrapCompletableFuture(failure)); + return null; } } diff --git a/sdk/src/test/java/software/amazon/lambda/durable/DurableConfigTest.java b/sdk/src/test/java/software/amazon/lambda/durable/DurableConfigTest.java index 7a3b7aaef..0843ac179 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/DurableConfigTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/DurableConfigTest.java @@ -7,6 +7,7 @@ import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNotSame; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -54,8 +55,7 @@ void testDefaultConfig_CreatesWithDefaults() { assertInstanceOf(JacksonSerDes.class, config.getSerDes()); assertNotNull(config.getExecutorService()); assertInstanceOf(ExecutorService.class, config.getExecutorService()); - assertNotNull(config.getSerDesExecutorService()); - assertInstanceOf(ExecutorService.class, config.getSerDesExecutorService()); + assertNull(config.getSerDesExecutorService()); } @Test @@ -100,6 +100,16 @@ void testBuilder_WithCustomSerDesExecutorService() { assertSame(mockSerDesExecutor, config.getSerDesExecutorService()); } + @Test + void testBuilder_RejectsAliasedOperationAndSerDesExecutors() { + var exception = assertThrows(IllegalStateException.class, () -> DurableConfig.builder() + .withExecutorService(mockExecutor) + .withSerDesExecutorService(mockExecutor) + .build()); + + assertTrue(exception.getMessage().contains("must be different")); + } + @Test void testBuilder_DeserializeAfterSerializationDefaultsToTrue() { var config = diff --git a/sdk/src/test/java/software/amazon/lambda/durable/operation/StepOperationTest.java b/sdk/src/test/java/software/amazon/lambda/durable/operation/StepOperationTest.java index be4962d71..1d8df81b3 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/operation/StepOperationTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/operation/StepOperationTest.java @@ -7,6 +7,7 @@ import java.util.List; import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicReference; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import software.amazon.awssdk.services.lambda.model.ErrorObject; @@ -25,6 +26,7 @@ import software.amazon.lambda.durable.model.OperationIdentifier; import software.amazon.lambda.durable.model.OperationSubType; import software.amazon.lambda.durable.serde.JacksonSerDes; +import software.amazon.lambda.durable.serde.SerDesContext; class StepOperationTest { @@ -94,6 +96,39 @@ void getDoesNotThrowWhenCalledFromHandlerContext() { assertEquals("cached-result", result); } + @Test + void successfulReplayUsesCheckpointedAttemptInSerDesContext() { + var observedContext = new AtomicReference(); + var serDes = new JacksonSerDes() { + @Override + public T deserialize(String data, TypeToken typeToken) { + observedContext.set(SerDesContext.getCurrentContext()); + return super.deserialize(data, typeToken); + } + }; + var op = Operation.builder() + .id(OPERATION_ID) + .name(OPERATION_NAME) + .status(OperationStatus.SUCCEEDED) + .stepDetails(StepDetails.builder() + .result("\"cached-result\"") + .attempt(3) + .build()) + .build(); + when(executionManager.getOperationAndUpdateReplayState(OPERATION_ID)).thenReturn(op); + + var operation = new StepOperation<>( + OPERATION_IDENTIFIER, + (ctx) -> RESULT, + TypeToken.get(String.class), + StepConfig.builder().serDes(serDes).build(), + durableContext); + operation.onCheckpointComplete(op); + + assertEquals("cached-result", operation.get()); + assertEquals(3, observedContext.get().attempt()); + } + @Test void getThrowsOriginalExceptionWhenClassIsAvailable() { var serDes = new JacksonSerDes(); diff --git a/sdk/src/test/java/software/amazon/lambda/durable/serde/ComposableSerDesTest.java b/sdk/src/test/java/software/amazon/lambda/durable/serde/ComposableSerDesTest.java new file mode 100644 index 000000000..471133a46 --- /dev/null +++ b/sdk/src/test/java/software/amazon/lambda/durable/serde/ComposableSerDesTest.java @@ -0,0 +1,146 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.serde; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; +import org.junit.jupiter.api.Test; +import software.amazon.lambda.durable.TypeToken; +import software.amazon.lambda.durable.exception.RetryableSerDesException; +import software.amazon.lambda.durable.exception.SerDesException; + +class ComposableSerDesTest { + + @Test + void serializesForwardAndDeserializesInReverse() { + var calls = new ArrayList(); + var first = stringStage("first", "<", ">", calls); + var second = stringStage("second", "[", "]", calls); + var pipeline = new JacksonSerDes().then(first).then(second); + + var serialized = pipeline.serialize("value"); + var deserialized = pipeline.deserialize(serialized, TypeToken.get(String.class)); + + assertEquals("[<\"value\">]", serialized); + assertEquals("value", deserialized); + assertEquals(List.of("first-serialize", "second-serialize", "second-deserialize", "first-deserialize"), calls); + } + + @Test + void factoryBuilderAndThenFlattenNestedPipelines() { + var calls = new ArrayList(); + var nested = ComposableSerDes.builder(stringStage("codec", "", "", calls)) + .then(stringStage("one", "1", "1", calls)) + .build(); + var pipeline = ComposableSerDes.of(nested).then(stringStage("two", "2", "2", calls)); + + assertEquals("21value12", pipeline.serialize("value")); + assertEquals(List.of("codec-serialize", "one-serialize", "two-serialize"), calls); + } + + @Test + void nullBoundarySkipsEveryStage() { + var calls = new AtomicInteger(); + var stage = new SerDes() { + @Override + public String serialize(Object value) { + calls.incrementAndGet(); + return "unexpected"; + } + + @Override + public T deserialize(String data, TypeToken typeToken) { + calls.incrementAndGet(); + return null; + } + }; + var pipeline = ComposableSerDes.of(stage); + + assertNull(pipeline.serialize(null)); + assertNull(pipeline.deserialize(null, TypeToken.get(String.class))); + assertEquals(0, calls.get()); + } + + @Test + void rejectsNullAndNonStringIntermediateValuesWithStageMetadata() { + var nullStage = new SerDes() { + @Override + public String serialize(Object value) { + return null; + } + + @Override + public T deserialize(String data, TypeToken typeToken) { + return null; + } + }; + var nullFailure = assertThrows( + SerDesException.class, () -> new JacksonSerDes().then(nullStage).serialize("value")); + assertTrue(nullFailure.getMessage().contains("stage 1")); + assertTrue(nullFailure.getMessage().contains(nullStage.getClass().getName())); + + var nonStringStage = new SerDes() { + @Override + public String serialize(Object value) { + return value.toString(); + } + + @Override + @SuppressWarnings("unchecked") + public T deserialize(String data, TypeToken typeToken) { + return (T) Integer.valueOf(42); + } + }; + var typeFailure = assertThrows( + SerDesException.class, + () -> new JacksonSerDes().then(nonStringStage).deserialize("\"value\"", TypeToken.get(String.class))); + assertTrue(typeFailure.getMessage().contains("stage 1")); + assertTrue(typeFailure.getCause().getMessage().contains("non-string")); + } + + @Test + void preservesRetryabilityWhenDecoratingStageFailures() { + var transientStage = new SerDes() { + @Override + public String serialize(Object value) { + throw new RetryableSerDesException("retry"); + } + + @Override + public T deserialize(String data, TypeToken typeToken) { + return null; + } + }; + + var failure = assertThrows( + RetryableSerDesException.class, + () -> new JacksonSerDes().then(transientStage).serialize("value")); + + assertInstanceOf(RetryableSerDesException.class, failure.getCause()); + assertTrue(failure.getMessage().contains("stage 1")); + } + + private static SerDes stringStage(String name, String prefix, String suffix, List calls) { + return new SerDes() { + @Override + public String serialize(Object value) { + calls.add(name + "-serialize"); + return prefix + value + suffix; + } + + @Override + @SuppressWarnings("unchecked") + public T deserialize(String data, TypeToken typeToken) { + calls.add(name + "-deserialize"); + return (T) data.substring(prefix.length(), data.length() - suffix.length()); + } + }; + } +} diff --git a/sdk/src/test/java/software/amazon/lambda/durable/serde/SerDesRunnerTest.java b/sdk/src/test/java/software/amazon/lambda/durable/serde/SerDesRunnerTest.java index 8e49df35b..a14265f3a 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/serde/SerDesRunnerTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/serde/SerDesRunnerTest.java @@ -3,11 +3,15 @@ package software.amazon.lambda.durable.serde; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import java.util.ArrayList; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.Executors; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; @@ -15,6 +19,7 @@ import org.junit.jupiter.api.Test; import software.amazon.awssdk.services.lambda.model.OperationType; import software.amazon.lambda.durable.TypeToken; +import software.amazon.lambda.durable.exception.RetryableSerDesException; import software.amazon.lambda.durable.exception.SerDesException; import software.amazon.lambda.durable.model.OperationSubType; @@ -52,6 +57,59 @@ public String serialize(Object value) { assertNull(SerDesContext.getCurrentContext()); } + @Test + void executesInlineWhenNoExecutorIsConfigured() { + var observedThread = new AtomicReference(); + var serDes = new JacksonSerDes() { + @Override + public String serialize(Object value) { + observedThread.set(Thread.currentThread()); + return super.serialize(value); + } + }; + + new SerDesRunner(null).serialize(serDes, "value", context("operation/1/result")); + + assertSame(Thread.currentThread(), observedThread.get()); + } + + @Test + void restoresPreviousContextAcrossNestedInlineCalls() { + var runner = new SerDesRunner(null); + var previous = context("previous"); + var outer = context("outer"); + var inner = context("inner"); + var duringOuter = new AtomicReference(); + var afterInner = new AtomicReference(); + var innerSerDes = new JacksonSerDes() { + @Override + public String serialize(Object value) { + assertSame(inner, SerDesContext.getCurrentContext()); + return super.serialize(value); + } + }; + var outerSerDes = new JacksonSerDes() { + @Override + public String serialize(Object value) { + duringOuter.set(SerDesContext.getCurrentContext()); + runner.serialize(innerSerDes, value, inner); + afterInner.set(SerDesContext.getCurrentContext()); + return super.serialize(value); + } + }; + + SerDesContextHolder.set(previous); + try { + runner.serialize(outerSerDes, "value", outer); + assertSame(previous, SerDesContext.getCurrentContext()); + } finally { + SerDesContextHolder.clear(); + } + + assertSame(outer, duringOuter.get()); + assertSame(outer, afterInner.get()); + } + @Test void cachesByEntityTypeAndSerializedDataHash() { var count = new AtomicInteger(); @@ -89,6 +147,77 @@ public T deserialize(String data, TypeToken typeToken) { assertEquals(3, count.get()); } + @Test + void concurrentCacheMissesDeserializeOnlyOnce() throws Exception { + var entered = new CountDownLatch(1); + var release = new CountDownLatch(1); + var count = new AtomicInteger(); + var sharedValue = new Object(); + var serDes = new SerDes() { + @Override + public String serialize(Object value) { + return value.toString(); + } + + @Override + @SuppressWarnings("unchecked") + public T deserialize(String data, TypeToken typeToken) { + count.incrementAndGet(); + entered.countDown(); + try { + release.await(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new SerDesException("interrupted", e); + } + return (T) sharedValue; + } + }; + var runner = new SerDesRunner(null); + var callers = Executors.newFixedThreadPool(8); + try { + var futures = new ArrayList>(); + for (int index = 0; index < 8; index++) { + futures.add(CompletableFuture.supplyAsync( + () -> runner.deserialize( + serDes, "data", TypeToken.get(Object.class), context("operation/1/result")), + callers)); + } + entered.await(); + release.countDown(); + + for (var future : futures) { + assertSame(sharedValue, future.join()); + } + assertEquals(1, count.get()); + } finally { + release.countDown(); + callers.shutdownNow(); + } + } + + @Test + void failedDeserializationIsRemovedFromCache() { + var calls = new AtomicInteger(); + var serDes = new JacksonSerDes() { + @Override + public T deserialize(String data, TypeToken typeToken) { + if (calls.incrementAndGet() == 1) { + throw new SerDesException("first"); + } + return super.deserialize(data, typeToken); + } + }; + var runner = new SerDesRunner(null); + var context = context("operation/1/result"); + + assertThrows( + SerDesException.class, + () -> runner.deserialize(serDes, "\"value\"", TypeToken.get(String.class), context)); + assertEquals("value", runner.deserialize(serDes, "\"value\"", TypeToken.get(String.class), context)); + assertEquals(2, calls.get()); + } + @Test void wrapsFailuresWithPayloadMetadata() { var runner = new SerDesRunner(executor); @@ -111,6 +240,27 @@ public T deserialize(String data, TypeToken typeToken) { assertEquals("boom", exception.getCause().getMessage()); } + @Test + void preservesRetryableFailureType() { + var runner = new SerDesRunner(null); + var serDes = new SerDes() { + @Override + public String serialize(Object value) { + throw new RetryableSerDesException("transient"); + } + + @Override + public T deserialize(String data, TypeToken typeToken) { + return null; + } + }; + + var exception = assertThrows( + RetryableSerDesException.class, () -> runner.serialize(serDes, "value", context("entity"))); + + assertInstanceOf(RetryableSerDesException.class, exception.getCause()); + } + private static SerDesContext context(String entityId) { return new SerDesContext( "arn:test", From a0629ea9a9453f86dfe74caaa6a264518094a2c8 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Mon, 24 Aug 2026 23:14:48 +0000 Subject: [PATCH 06/56] fix: preserve cloud operation timestamps --- .../testing/cloud/HistoryEventProcessor.java | 26 +++++++++++++++++-- .../cloud/HistoryEventProcessorTest.java | 19 ++++++++++++++ 2 files changed, 43 insertions(+), 2 deletions(-) diff --git a/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/cloud/HistoryEventProcessor.java b/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/cloud/HistoryEventProcessor.java index eeefa4e52..445ee335c 100644 --- a/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/cloud/HistoryEventProcessor.java +++ b/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/cloud/HistoryEventProcessor.java @@ -5,6 +5,7 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; +import java.util.Objects; import software.amazon.awssdk.services.lambda.model.CallbackDetails; import software.amazon.awssdk.services.lambda.model.ChainedInvokeDetails; import software.amazon.awssdk.services.lambda.model.ContextDetails; @@ -17,6 +18,7 @@ import software.amazon.awssdk.services.lambda.model.StepDetails; import software.amazon.awssdk.services.lambda.model.WaitDetails; import software.amazon.lambda.durable.TypeToken; +import software.amazon.lambda.durable.execution.ExecutionManager; import software.amazon.lambda.durable.model.ExecutionStatus; import software.amazon.lambda.durable.serde.SerDes; import software.amazon.lambda.durable.serde.SerDesRunner; @@ -285,10 +287,11 @@ public TestResult processEvents( var testOperations = new ArrayList(); for (var entry : operations.entrySet()) { var opEvents = operationEvents.getOrDefault(entry.getKey(), List.of()); + var operation = withEventTimestamps(entry.getValue(), opEvents); testOperations.add( serDesRunner == null - ? new TestOperation(entry.getValue(), opEvents, serDes) - : new TestOperation(entry.getValue(), opEvents, serDes, serDesRunner, durableExecutionArn)); + ? new TestOperation(operation, opEvents, serDes) + : new TestOperation(operation, opEvents, serDes, serDesRunner, durableExecutionArn)); } if (executionOperationId == null && durableExecutionArn != null) { @@ -311,6 +314,25 @@ public TestResult processEvents( executionOperationName); } + private Operation withEventTimestamps(Operation operation, List events) { + var startTimestamp = events.stream() + .map(Event::eventTimestamp) + .filter(Objects::nonNull) + .min(java.time.Instant::compareTo) + .orElse(operation.startTimestamp()); + var endTimestamp = ExecutionManager.isTerminalStatus(operation.status()) + ? events.stream() + .map(Event::eventTimestamp) + .filter(Objects::nonNull) + .max(java.time.Instant::compareTo) + .orElse(operation.endTimestamp()) + : operation.endTimestamp(); + return operation.toBuilder() + .startTimestamp(startTimestamp) + .endTimestamp(endTimestamp) + .build(); + } + private Operation createStepOperation( String id, String name, diff --git a/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/cloud/HistoryEventProcessorTest.java b/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/cloud/HistoryEventProcessorTest.java index dae253b91..633d52459 100644 --- a/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/cloud/HistoryEventProcessorTest.java +++ b/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/cloud/HistoryEventProcessorTest.java @@ -4,6 +4,8 @@ import static org.junit.jupiter.api.Assertions.assertEquals; +import java.time.Duration; +import java.time.Instant; import java.util.ArrayList; import java.util.List; import org.junit.jupiter.api.Test; @@ -17,6 +19,7 @@ import software.amazon.awssdk.services.lambda.model.OperationStatus; import software.amazon.awssdk.services.lambda.model.OperationType; import software.amazon.awssdk.services.lambda.model.RetryDetails; +import software.amazon.awssdk.services.lambda.model.StepStartedDetails; import software.amazon.awssdk.services.lambda.model.StepSucceededDetails; import software.amazon.lambda.durable.TypeToken; import software.amazon.lambda.durable.serde.SerDes; @@ -32,18 +35,30 @@ class HistoryEventProcessorTest { void deserializesCloudResultsWithDurablePayloadContext() { var observedContexts = new ArrayList(); var serDes = recordingStringSerDes(observedContexts); + var startedAt = Instant.parse("2026-08-24T00:00:00Z"); var events = List.of( Event.builder() .id("invocation-id") .name("execution") .eventType(EventType.EXECUTION_STARTED) + .eventTimestamp(startedAt) .executionStartedDetails( ExecutionStartedDetails.builder().build()) .build(), Event.builder() .id("step-id") .name("step") + .subType("Step") + .eventType(EventType.STEP_STARTED) + .eventTimestamp(startedAt.plusSeconds(1)) + .stepStartedDetails(StepStartedDetails.builder().build()) + .build(), + Event.builder() + .id("step-id") + .name("step") + .subType("Step") .eventType(EventType.STEP_SUCCEEDED) + .eventTimestamp(startedAt.plusSeconds(3)) .stepSucceededDetails(StepSucceededDetails.builder() .result(EventResult.builder() .payload("step-result") @@ -56,6 +71,7 @@ void deserializesCloudResultsWithDurablePayloadContext() { .id("invoke-id") .name("invoke") .eventType(EventType.CHAINED_INVOKE_STARTED) + .eventTimestamp(startedAt.plusSeconds(4)) .chainedInvokeStartedDetails(ChainedInvokeStartedDetails.builder() .functionName("target") .build()) @@ -64,6 +80,7 @@ void deserializesCloudResultsWithDurablePayloadContext() { .id("invoke-id") .name("invoke") .eventType(EventType.CHAINED_INVOKE_SUCCEEDED) + .eventTimestamp(startedAt.plusSeconds(5)) .chainedInvokeSucceededDetails(ChainedInvokeSucceededDetails.builder() .result(EventResult.builder() .payload("invoke-result") @@ -74,6 +91,7 @@ void deserializesCloudResultsWithDurablePayloadContext() { .id("invocation-id") .name("execution") .eventType(EventType.EXECUTION_SUCCEEDED) + .eventTimestamp(startedAt.plusSeconds(6)) .executionSucceededDetails(ExecutionSucceededDetails.builder() .result(EventResult.builder() .payload("execution-result") @@ -86,6 +104,7 @@ void deserializesCloudResultsWithDurablePayloadContext() { assertEquals("execution-result", result.getResult()); assertEquals("step-result", result.getOperation("step").getStepResult(String.class)); + assertEquals(Duration.ofSeconds(2), result.getOperation("step").getDuration()); assertEquals(OperationStatus.SUCCEEDED, result.getOperation("invoke").getStatus()); assertEquals( "invoke-result", From 39de91565871ef8c0ab48bedc286ddc4d1fd19d4 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Mon, 24 Aug 2026 23:31:00 +0000 Subject: [PATCH 07/56] fix: address SerDes pipeline review findings --- docs/adr/005-filesystem-serdes.md | 14 ++- extra-filesystem-serdes/README.md | 9 +- .../extra/filesystem/FileSystemSerDes.java | 66 ++++++++--- .../filesystem/FileSystemSerDesTest.java | 45 ++++++++ .../FileSystemSerDesIntegrationTest.java | 106 ++++++++++++++++++ .../testing/CloudDurableTestRunner.java | 22 ++-- .../testing/CloudDurableTestRunnerTest.java | 58 ++++++++++ .../durable/serde/ComposableSerDes.java | 6 +- .../durable/serde/ComposableSerDesTest.java | 22 ++++ 9 files changed, 310 insertions(+), 38 deletions(-) diff --git a/docs/adr/005-filesystem-serdes.md b/docs/adr/005-filesystem-serdes.md index be7681251..eac399708 100644 --- a/docs/adr/005-filesystem-serdes.md +++ b/docs/adr/005-filesystem-serdes.md @@ -184,7 +184,8 @@ Pipeline rules: - `SerDes.then(...)`, `ComposableSerDes.of(...)`, and the builder flatten nested `ComposableSerDes` instances while preserving stage order. - A `null` value at the pipeline boundary short-circuits the entire pipeline: serializing or deserializing `null` - returns `null` without invoking any stage. A stage returning `null` for non-null input is an error. + returns `null` without invoking any stage. A string stage returning `null` for non-null input is an error. The value + codec may decode a non-null representation such as the JSON literal `null` to a null domain value. - All stages execute within the same `SerDesRunner` invocation and observe the same read-only `SerDesContext`, whether the runner executes inline or dispatches to a configured executor. - `ComposableSerDes` is immutable. It is safe for concurrent use only when every contained stage is also safe for @@ -271,8 +272,8 @@ Envelope format: ```json {"__durable_execution_filesystem_serdes":1,"data":""} -{"__durable_execution_filesystem_serdes":1,"file":""} -{"__durable_execution_filesystem_serdes":1,"file":"","preview":{ "...": "..." }} +{"__durable_execution_filesystem_serdes":1,"file":"","ownerDurableExecutionArn":"","ownerEntityId":""} +{"__durable_execution_filesystem_serdes":1,"file":"","ownerDurableExecutionArn":"","ownerEntityId":"","preview":{ "...": "..." }} ``` `FileSystemSerDes` must reject calls when `SerDesContext.getCurrentContext()` is `null` or does not include @@ -288,6 +289,13 @@ Offloaded filenames include a content hash and are immutable. Serializing new st path instead of replacing a file referenced by an earlier checkpoint. Repeating the same serialization may reuse the same content-addressed file. +File envelopes identify the execution ARN and entity that produced the content. Normal checkpoint replay requires +that owner to match the current context. Initial input and chained-invoke result boundaries may consume a reference +owned by the other Lambda execution, allowing two functions configured with the same durable filesystem root and path +encoding to exchange offloaded invoke payloads and results. The declared owner must still match the content-addressed +path, and the resolved file must remain beneath the configured root. The file envelope is therefore a capability and +must be protected with the same care as the payload it references. + The final file envelope, including any preview, must remain below the checkpoint threshold. Oversized previews are rejected rather than producing a checkpoint that the service cannot accept. diff --git a/extra-filesystem-serdes/README.md b/extra-filesystem-serdes/README.md index 117eb9802..84a59397a 100644 --- a/extra-filesystem-serdes/README.md +++ b/extra-filesystem-serdes/README.md @@ -68,12 +68,15 @@ results pass through when they have not yet been wrapped by this SerDes. Offloaded files are content-addressed and immutable. Updating wait-for-condition state or retry results creates a new path instead of replacing a file referenced by an earlier checkpoint. Repeating the same write can safely reuse the -same file. File references are bound to the current execution and entity, content hashes are verified when reading, -and symbolic-link paths are rejected. +same file. File envelopes identify the producing execution and entity. Ordinary checkpoint replay must match that +owner, while invoke input and result boundaries may consume a file owned by the other Lambda execution when both +functions use the same shared root and path encoding. Treat file envelopes as capabilities. Content hashes are +verified when reading, and symbolic-link paths are rejected. `CloudDurableTestRunner` uses the first value-codec stage for the initial Lambda invocation, before an execution ARN is available, and uses the complete pipeline for persisted history. When using standalone `FileSystemSerDes`, configure a -separate initial-input codec with `withInputSerDes(...)`. +separate initial-input codec with `withInputSerDes(...)`. An explicitly supplied input SerDes is used exactly as +configured, including every stage in a composable pipeline. ## Storage requirements diff --git a/extra-filesystem-serdes/src/main/java/software/amazon/lambda/durable/extra/filesystem/FileSystemSerDes.java b/extra-filesystem-serdes/src/main/java/software/amazon/lambda/durable/extra/filesystem/FileSystemSerDes.java index 0a6a2b70d..6913923b6 100644 --- a/extra-filesystem-serdes/src/main/java/software/amazon/lambda/durable/extra/filesystem/FileSystemSerDes.java +++ b/extra-filesystem-serdes/src/main/java/software/amazon/lambda/durable/extra/filesystem/FileSystemSerDes.java @@ -90,9 +90,11 @@ public String serialize(Object value) { } var context = requireContext(); var serialized = serializeValue(value); - var inlineEnvelope = encodeEnvelope(serialized, null, null, context); - if (storageMode == FileSystemStorageMode.OVERFLOW && fitsCheckpoint(inlineEnvelope)) { - return inlineEnvelope; + if (storageMode == FileSystemStorageMode.OVERFLOW) { + var inlineEnvelope = encodeEnvelope(serialized, null, null, context); + if (fitsCheckpoint(inlineEnvelope)) { + return inlineEnvelope; + } } var file = resolvePayloadPath(serialized, context); @@ -175,12 +177,13 @@ private String resolveSerializedPayload(String data, SerDesContext context) { if (hasData) { return envelope.get("data").textValue(); } - return readPayload(envelope.get("file").textValue(), context); + var owner = payloadOwner(envelope, context); + return readPayload(envelope.get("file").textValue(), owner, context); } - private String readPayload(String fileValue, SerDesContext context) { + private String readPayload(String fileValue, PayloadOwner owner, SerDesContext context) { var file = Path.of(fileValue).toAbsolutePath().normalize(); - validatePayloadPath(file, context); + validatePayloadPath(file, owner); try { rejectSymbolicLinks(file); var realBasePath = basePath.toRealPath(); @@ -192,7 +195,7 @@ private String readPayload(String fileValue, SerDesContext context) { throw new SerDesException("Filesystem SerDes file does not resolve to the expected payload path"); } var serialized = Files.readString(realFile, StandardCharsets.UTF_8); - var expectedFileName = payloadFileName(serialized, context); + var expectedFileName = payloadFileName(serialized, owner.entityId()); if (!realFile.getFileName().toString().equals(expectedFileName)) { throw new SerDesException("Filesystem SerDes file content does not match its content-addressed path"); } @@ -203,17 +206,48 @@ private String readPayload(String fileValue, SerDesContext context) { } } - private void validatePayloadPath(Path file, SerDesContext context) { - var expectedDirectory = resolveExecutionDirectory(context.durableExecutionArn()); + private void validatePayloadPath(Path file, PayloadOwner owner) { + var expectedDirectory = resolveExecutionDirectory(owner.durableExecutionArn()); var fileName = file.getFileName(); if (fileName == null || file.getParent() == null || !file.getParent().equals(expectedDirectory) - || !fileName.toString().matches(Pattern.quote(encode(context.entityId())) + "-[0-9a-f]{64}\\.json")) { - throw new SerDesException("Filesystem SerDes file is not valid for the current durable entity"); + || !fileName.toString().matches(Pattern.quote(encode(owner.entityId())) + "-[0-9a-f]{64}\\.json")) { + throw new SerDesException("Filesystem SerDes file is not valid for its declared durable entity"); } } + private static PayloadOwner payloadOwner(JsonNode envelope, SerDesContext context) { + var hasOwnerArn = envelope.has("ownerDurableExecutionArn") + && envelope.get("ownerDurableExecutionArn").isTextual(); + var hasOwnerEntity = + envelope.has("ownerEntityId") && envelope.get("ownerEntityId").isTextual(); + if (hasOwnerArn != hasOwnerEntity) { + throw malformedEnvelope(context, null); + } + + var owner = hasOwnerArn + ? new PayloadOwner( + envelope.get("ownerDurableExecutionArn").textValue(), + envelope.get("ownerEntityId").textValue()) + : new PayloadOwner(context.durableExecutionArn(), context.entityId()); + if (owner.durableExecutionArn().isBlank() || owner.entityId().isBlank()) { + throw malformedEnvelope(context, null); + } + + var sameOwner = owner.durableExecutionArn().equals(context.durableExecutionArn()) + && owner.entityId().equals(context.entityId()); + if (!sameOwner && !acceptsCrossExecutionReference(context)) { + throw new SerDesException("Filesystem SerDes file belongs to a different durable entity"); + } + return owner; + } + + private static boolean acceptsCrossExecutionReference(SerDesContext context) { + return context.payloadKind() == SerDesPayloadKind.INPUT + || context.operationType() == OperationType.CHAINED_INVOKE; + } + private void rejectSymbolicLinks(Path file) throws IOException { var current = basePath; for (var component : basePath.relativize(file)) { @@ -250,6 +284,8 @@ private String encodeEnvelope(String data, Path file, Map previe envelope.put("data", data); } else { envelope.put("file", file.toString()); + envelope.put("ownerDurableExecutionArn", context.durableExecutionArn()); + envelope.put("ownerEntityId", context.entityId()); if (preview != null) { envelope.put("preview", preview); } @@ -293,7 +329,7 @@ private SerDesContext requireContext() { private Path resolvePayloadPath(String serialized, SerDesContext context) { var directory = resolveExecutionDirectory(context.durableExecutionArn()); - var fileName = payloadFileName(serialized, context); + var fileName = payloadFileName(serialized, context.entityId()); var file = directory.resolve(fileName).normalize(); if (!file.startsWith(directory)) { throw new SerDesException("Resolved filesystem payload path is outside the execution directory"); @@ -301,8 +337,8 @@ private Path resolvePayloadPath(String serialized, SerDesContext context) { return file; } - private String payloadFileName(String serialized, SerDesContext context) { - return encode(context.entityId()) + "-" + sha256(serialized) + ".json"; + private String payloadFileName(String serialized, String entityId) { + return encode(entityId) + "-" + sha256(serialized) + ".json"; } private void writePayload(String serialized, Path file) throws IOException { @@ -421,6 +457,8 @@ private static String sha256(String value) { } } + private record PayloadOwner(String durableExecutionArn, String entityId) {} + /** Builder for {@link FileSystemSerDes}. */ public static final class Builder { private final Path basePath; diff --git a/extra-filesystem-serdes/src/test/java/software/amazon/lambda/durable/extra/filesystem/FileSystemSerDesTest.java b/extra-filesystem-serdes/src/test/java/software/amazon/lambda/durable/extra/filesystem/FileSystemSerDesTest.java index 48aff91fc..9e5f7c0f6 100644 --- a/extra-filesystem-serdes/src/test/java/software/amazon/lambda/durable/extra/filesystem/FileSystemSerDesTest.java +++ b/extra-filesystem-serdes/src/test/java/software/amazon/lambda/durable/extra/filesystem/FileSystemSerDesTest.java @@ -173,6 +173,51 @@ void acceptsExternallyOriginatedRawPayloadsOnlyForSupportedSources() { () -> runner.deserialize(stage, "\"raw-step\"", TypeToken.get(String.class), context())); } + @Test + void fileReferencesCrossInvokeInputAndResultBoundaries() { + var serDes = + new JacksonSerDes().then(FileSystemSerDes.stageBuilder(basePath).build()); + var runner = new SerDesRunner(null); + var callerArn = + "arn:aws:lambda:us-east-1:123456789012:function:caller:1/durable-execution/caller-execution/caller-invocation"; + var calleeArn = + "arn:aws:lambda:us-east-1:123456789012:function:callee:1/durable-execution/callee-execution/callee-invocation"; + var callerInvokePayload = SerDesContext.forOperation( + callerArn, + "invoke-1", + "call-callee", + null, + OperationType.CHAINED_INVOKE, + OperationSubType.CHAINED_INVOKE, + SerDesPayloadKind.INVOKE_PAYLOAD, + null); + var calleeInput = + SerDesContext.forExecution(calleeArn, "callee-invocation", "callee-execution", SerDesPayloadKind.INPUT); + + var invokeEnvelope = runner.serialize(serDes, Map.of("request", "value"), callerInvokePayload); + assertEquals( + Map.of("request", "value"), + runner.deserialize(serDes, invokeEnvelope, new TypeToken>() {}, calleeInput)); + + var calleeOutput = SerDesContext.forExecution( + calleeArn, "callee-invocation", "callee-execution", SerDesPayloadKind.OUTPUT); + var callerInvokeResult = SerDesContext.forOperation( + callerArn, + "invoke-1", + "call-callee", + null, + OperationType.CHAINED_INVOKE, + OperationSubType.CHAINED_INVOKE, + SerDesPayloadKind.RESULT, + null); + var resultEnvelope = runner.serialize(serDes, Map.of("response", "value"), calleeOutput); + + assertEquals( + Map.of("response", "value"), + runner.deserialize( + serDes, resultEnvelope, new TypeToken>() {}, callerInvokeResult)); + } + @Test void rejectsCallsWithoutContextAndMalformedOrUnsafeEnvelopes() throws Exception { var serDes = FileSystemSerDes.builder(basePath).build(); diff --git a/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FileSystemSerDesIntegrationTest.java b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FileSystemSerDesIntegrationTest.java index 4146e51e6..b3360c076 100644 --- a/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FileSystemSerDesIntegrationTest.java +++ b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FileSystemSerDesIntegrationTest.java @@ -17,11 +17,13 @@ import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; +import java.util.function.BiFunction; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; import software.amazon.awssdk.services.lambda.model.CheckpointUpdatedExecutionState; import software.amazon.awssdk.services.lambda.model.ExecutionDetails; import software.amazon.awssdk.services.lambda.model.Operation; +import software.amazon.awssdk.services.lambda.model.OperationAction; import software.amazon.awssdk.services.lambda.model.OperationStatus; import software.amazon.awssdk.services.lambda.model.OperationType; import software.amazon.lambda.durable.config.StepConfig; @@ -41,6 +43,7 @@ import software.amazon.lambda.durable.serde.SerDesRunner; import software.amazon.lambda.durable.testing.LocalDurableTestRunner; import software.amazon.lambda.durable.testing.local.LocalMemoryExecutionClient; +import software.amazon.lambda.durable.testing.local.OperationResult; class FileSystemSerDesIntegrationTest { private static final ObjectMapper MAPPER = new ObjectMapper(); @@ -185,6 +188,79 @@ void acceptsRawCallbackAndInvokeResultsAndOffloadsInvokePayload() throws Excepti assertEquals("notified", completed.getResult()); } + @Test + void callerAndCalleeExchangeOffloadedInvokePayloadAndResult() throws Exception { + var callerArn = + "arn:aws:lambda:us-east-1:123456789012:function:caller:1/durable-execution/caller-execution/caller-invocation"; + var calleeArn = + "arn:aws:lambda:us-east-1:123456789012:function:callee:1/durable-execution/callee-execution/callee-invocation"; + var serDes = filesystemPipeline(); + var callerClient = new LocalMemoryExecutionClient(); + var callerConfig = DurableConfig.builder() + .withDurableExecutionClient(callerClient) + .withSerDes(serDes) + .build(); + BiFunction callerHandler = (input, context) -> + context.invoke("call-callee", "callee", new CrossInvokeRequest(input), CrossInvokeResponse.class); + var callerExecution = + executionOperation("caller-invocation", "caller-execution", "\"request\"", OperationStatus.STARTED); + + var pending = DurableExecutor.execute( + durableInput(callerArn, callerExecution, List.of(), List.of()), + null, + TypeToken.get(String.class), + callerHandler, + callerConfig); + + assertEquals(ExecutionStatus.PENDING, pending.status()); + var invokePayload = callerClient.getOperationUpdates().stream() + .filter(update -> + update.type() == OperationType.CHAINED_INVOKE && update.action() == OperationAction.START) + .findFirst() + .orElseThrow() + .payload(); + assertEnvelopePointsToFile(invokePayload); + + var calleeClient = new LocalMemoryExecutionClient(); + var calleeConfig = DurableConfig.builder() + .withDurableExecutionClient(calleeClient) + .withSerDes(serDes) + .build(); + var calleeExecution = + executionOperation("callee-invocation", "callee-execution", invokePayload, OperationStatus.STARTED); + var calleeOutput = DurableExecutor.execute( + durableInput(calleeArn, calleeExecution, List.of(), List.of()), + null, + TypeToken.get(CrossInvokeRequest.class), + (request, context) -> new CrossInvokeResponse("reply:" + request.value()), + calleeConfig); + + assertEquals(ExecutionStatus.SUCCEEDED, calleeOutput.status()); + assertEnvelopePointsToFile(calleeOutput.result()); + + callerClient.completeChainedInvoke("call-callee", OperationResult.succeeded(calleeOutput.result())); + var resumed = DurableExecutor.execute( + durableInput( + callerArn, + callerExecution, + callerClient.getAllOperations(), + callerClient.getUpdatedOperationIdsSinceLastInvocation()), + null, + TypeToken.get(String.class), + callerHandler, + callerConfig); + + assertEquals(ExecutionStatus.SUCCEEDED, resumed.status()); + var result = new SerDesRunner(null) + .deserialize( + serDes, + resumed.result(), + TypeToken.get(CrossInvokeResponse.class), + SerDesContext.forExecution( + callerArn, "caller-invocation", "caller-execution", SerDesPayloadKind.OUTPUT)); + assertEquals(new CrossInvokeResponse("reply:request"), result); + } + @Test void repeatedGetUsesInvocationCacheForTheCompletePipeline() { var resultDeserializations = new AtomicInteger(); @@ -294,6 +370,32 @@ private SerDes filesystemPipeline() { return new JacksonSerDes().then(FileSystemSerDes.stageBuilder(basePath).build()); } + private static DurableExecutionInput durableInput( + String executionArn, Operation executionOperation, List operations, List updatedIds) { + var allOperations = new ArrayList(); + allOperations.add(executionOperation); + allOperations.addAll(operations); + return new DurableExecutionInput( + executionArn, + "checkpoint-token", + CheckpointUpdatedExecutionState.builder() + .operations(allOperations) + .build(), + updatedIds); + } + + private static Operation executionOperation(String id, String name, String inputPayload, OperationStatus status) { + return Operation.builder() + .id(id) + .name(name) + .type(OperationType.EXECUTION) + .status(status) + .startTimestamp(Instant.now()) + .executionDetails( + ExecutionDetails.builder().inputPayload(inputPayload).build()) + .build(); + } + private static SerDes identityStage(RecordingFunction recorder) { return new SerDes() { @Override @@ -325,6 +427,10 @@ private interface RecordingFunction { record Payload(String value) {} + record CrossInvokeRequest(String value) {} + + record CrossInvokeResponse(String value) {} + public static class CustomFailure extends RuntimeException { public CustomFailure() {} diff --git a/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/CloudDurableTestRunner.java b/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/CloudDurableTestRunner.java index d0cb3e6e5..8d698ddd6 100644 --- a/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/CloudDurableTestRunner.java +++ b/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/CloudDurableTestRunner.java @@ -56,7 +56,7 @@ private CloudDurableTestRunner( this.timeout = timeout; this.invocationType = invocationType; this.serDes = Objects.requireNonNullElseGet(serDes, JacksonSerDes::new); - this.inputSerDes = Objects.requireNonNullElse(inputSerDes, this.serDes); + this.inputSerDes = inputSerDes; } private static LambdaClient createDefaultLambdaClient() { @@ -158,22 +158,15 @@ public CloudDurableTestRunner withInvocationType(InvocationType type) { /** Returns a new runner with the specified SerDes for persisted execution payloads. */ public CloudDurableTestRunner withSerDes(SerDes serDes) { return new CloudDurableTestRunner<>( - functionArn, - inputType, - outputType, - lambdaClient, - pollInterval, - timeout, - invocationType, - serDes, - serDes); + functionArn, inputType, outputType, lambdaClient, pollInterval, timeout, invocationType, null, serDes); } /** * Returns a new runner with a separate SerDes for the initial Lambda invocation payload. * - *

This is useful with a standalone context-dependent SerDes. Composable pipelines automatically use their first - * value-codec stage for initial input. + *

The supplied SerDes is used exactly as configured, including every stage in a composable pipeline. When this + * method is not called, composable persisted SerDes pipelines use only their first value-codec stage because a + * durable execution context does not exist before the Lambda invocation. */ public CloudDurableTestRunner withInputSerDes(SerDes inputSerDes) { return new CloudDurableTestRunner<>( @@ -277,7 +270,10 @@ public TestOperation getOperation(String name) { } private String serializeInput(I input) { - var serializer = inputSerDes instanceof ComposableSerDes composable ? composable.getValueCodec() : inputSerDes; + var serializer = inputSerDes; + if (serializer == null) { + serializer = serDes instanceof ComposableSerDes composable ? composable.getValueCodec() : serDes; + } return serializer.serialize(input); } } diff --git a/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/CloudDurableTestRunnerTest.java b/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/CloudDurableTestRunnerTest.java index b1fde42e9..58dc65f9a 100644 --- a/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/CloudDurableTestRunnerTest.java +++ b/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/CloudDurableTestRunnerTest.java @@ -7,8 +7,14 @@ import java.time.Duration; import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; import software.amazon.awssdk.services.lambda.LambdaClient; import software.amazon.awssdk.services.lambda.model.InvocationType; +import software.amazon.awssdk.services.lambda.model.InvokeRequest; +import software.amazon.awssdk.services.lambda.model.InvokeResponse; +import software.amazon.lambda.durable.TypeToken; +import software.amazon.lambda.durable.serde.JacksonSerDes; +import software.amazon.lambda.durable.serde.SerDes; class CloudDurableTestRunnerTest { @@ -31,4 +37,56 @@ void testPlaceholderMethods() { assertThrows(IllegalStateException.class, () -> runner.getOperation("test")); } + + @Test + void explicitComposableInputSerDesUsesTheCompletePipeline() { + var mockClient = mock(LambdaClient.class); + when(mockClient.invoke(any(InvokeRequest.class))) + .thenReturn(InvokeResponse.builder() + .durableExecutionArn("arn:aws:lambda:us-east-2:123:function:test:1/durable-execution/e/i") + .build()); + var wrappingStage = wrappingStage(); + var runner = CloudDurableTestRunner.create( + "arn:aws:lambda:us-east-2:123:function:test", String.class, String.class, mockClient) + .withInputSerDes(new JacksonSerDes().then(wrappingStage)); + + runner.startAsync("value"); + + var request = ArgumentCaptor.forClass(InvokeRequest.class); + verify(mockClient).invoke(request.capture()); + assertEquals("<\"value\">", request.getValue().payload().asUtf8String()); + } + + @Test + void persistedComposableSerDesUsesOnlyValueCodecForDefaultInput() { + var mockClient = mock(LambdaClient.class); + when(mockClient.invoke(any(InvokeRequest.class))) + .thenReturn(InvokeResponse.builder() + .durableExecutionArn("arn:aws:lambda:us-east-2:123:function:test:1/durable-execution/e/i") + .build()); + var runner = CloudDurableTestRunner.create( + "arn:aws:lambda:us-east-2:123:function:test", String.class, String.class, mockClient) + .withSerDes(new JacksonSerDes().then(wrappingStage())); + + runner.startAsync("value"); + + var request = ArgumentCaptor.forClass(InvokeRequest.class); + verify(mockClient).invoke(request.capture()); + assertEquals("\"value\"", request.getValue().payload().asUtf8String()); + } + + private static SerDes wrappingStage() { + return new SerDes() { + @Override + public String serialize(Object value) { + return "<" + value + ">"; + } + + @Override + @SuppressWarnings("unchecked") + public T deserialize(String data, TypeToken typeToken) { + return (T) data.substring(1, data.length() - 1); + } + }; + } } diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/ComposableSerDes.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/ComposableSerDes.java index 75cd38faf..2617c118d 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/ComposableSerDes.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/ComposableSerDes.java @@ -139,11 +139,7 @@ private static String invokeSerialize(SerDes stage, Object value, int index) { private static T invokeDeserialize(SerDes stage, String data, TypeToken typeToken, int index) { try { - var result = stage.deserialize(data, typeToken); - if (result == null) { - throw new SerDesException("Stage returned null for non-null data"); - } - return result; + return stage.deserialize(data, typeToken); } catch (Throwable failure) { throw stageFailure(index, stage, "deserialize", failure); } diff --git a/sdk/src/test/java/software/amazon/lambda/durable/serde/ComposableSerDesTest.java b/sdk/src/test/java/software/amazon/lambda/durable/serde/ComposableSerDesTest.java index 471133a46..dba62a3cc 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/serde/ComposableSerDesTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/serde/ComposableSerDesTest.java @@ -68,6 +68,28 @@ public T deserialize(String data, TypeToken typeToken) { assertEquals(0, calls.get()); } + @Test + void valueCodecMayDecodeNonNullRepresentationToNull() { + var intermediateCalls = new AtomicInteger(); + var identityStage = new SerDes() { + @Override + public String serialize(Object value) { + return value.toString(); + } + + @Override + @SuppressWarnings("unchecked") + public T deserialize(String data, TypeToken typeToken) { + intermediateCalls.incrementAndGet(); + return (T) data; + } + }; + var pipeline = new JacksonSerDes().then(identityStage); + + assertNull(pipeline.deserialize("null", TypeToken.get(Object.class))); + assertEquals(1, intermediateCalls.get()); + } + @Test void rejectsNullAndNonStringIntermediateValuesWithStageMetadata() { var nullStage = new SerDes() { From e9b25ab6d4f3487822cc738a01d558e903f3f686 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Mon, 24 Aug 2026 23:59:32 +0000 Subject: [PATCH 08/56] fix: enforce SerDes pipeline boundaries --- docs/adr/005-filesystem-serdes.md | 46 ++++++++++--- docs/advanced/configuration.md | 8 ++- extra-filesystem-serdes/README.md | 14 ++-- .../extra/filesystem/FileSystemSerDes.java | 37 ++++++++-- .../filesystem/FileSystemSerDesTest.java | 41 ++++++++++-- .../testing/CloudDurableTestRunner.java | 25 ++++--- .../testing/CloudDurableTestRunnerTest.java | 54 ++++++++++++++- .../durable/serde/ComposableSerDes.java | 38 ++++++----- .../lambda/durable/serde/RetrySerDes.java | 15 +++++ .../amazon/lambda/durable/serde/SerDes.java | 42 ++++++++++++ .../durable/serde/SerDesStageResult.java | 28 ++++++++ .../durable/serde/ComposableSerDesTest.java | 67 +++++++++++++++++++ .../lambda/durable/serde/RetrySerDesTest.java | 44 ++++++++++++ 13 files changed, 404 insertions(+), 55 deletions(-) create mode 100644 sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesStageResult.java diff --git a/docs/adr/005-filesystem-serdes.md b/docs/adr/005-filesystem-serdes.md index eac399708..5f2c75de1 100644 --- a/docs/adr/005-filesystem-serdes.md +++ b/docs/adr/005-filesystem-serdes.md @@ -130,6 +130,12 @@ public final class ComposableSerDes implements SerDes { public ComposableSerDes build(); } } + +public record SerDesStageResult(String value, boolean skipRemainingStages) { + public static SerDesStageResult continueWith(String value); + + public static SerDesStageResult decodeWithValueCodec(String value); +} ``` The first stage is the **value codec**. It converts the user value to a string and converts the final decoded string @@ -172,7 +178,11 @@ String serialize(Object value) { T deserialize(String data, TypeToken targetType) { String current = data; for (int i = stages.size() - 1; i > 0; i--) { - current = stages.get(i).deserialize(current, TypeToken.get(String.class)); + var decoded = stages.get(i).deserializePipelineStage(current); + current = decoded.value(); + if (decoded.skipRemainingStages()) { + break; + } } return stages.get(0).deserialize(current, targetType); } @@ -181,6 +191,8 @@ String serialize(Object value) { Pipeline rules: - A pipeline must contain exactly one value codec in the first position and zero or more string stages. +- A stage may declare that it requires durable context or that it must be terminal. A terminal stage must be the last + stage so later transformations cannot invalidate its checkpoint-size or storage decision. - `SerDes.then(...)`, `ComposableSerDes.of(...)`, and the builder flatten nested `ComposableSerDes` instances while preserving stage order. - A `null` value at the pipeline boundary short-circuits the entire pipeline: serializing or deserializing `null` @@ -194,9 +206,12 @@ Pipeline rules: entity and payload-kind metadata around the pipeline failure. - A string stage must be reversible. Compression, encryption, signing envelopes, and external payload storage are suitable stages; lossy redaction is not. +- A boundary stage may return `SerDesStageResult.decodeWithValueCodec(...)` when it receives raw data that did not pass + through the configured pipeline. `ComposableSerDes` then skips every earlier string stage and decodes the raw value + directly with the value codec. - Stage order is meaningful. For example, `JSON -> compression -> encryption -> filesystem` writes encrypted, - compressed data to the filesystem, while `JSON -> filesystem -> encryption` encrypts only the file-reference - envelope. + compressed data to the filesystem. `FileSystemSerDes` is terminal; placing encryption or any expanding + transformation after it is rejected. - The ordered stage list and each stage's configuration are part of the persisted checkpoint format. They must remain replay-compatible for in-flight executions. Reordering, removing, or incompatibly reconfiguring a stage requires a versioned envelope or an explicit migration boundary. @@ -282,8 +297,10 @@ deserialization target other than `String`. The marker and version distinguish filesystem envelopes from raw service-originated JSON. Initial root input, callback results, and standard Lambda invoke results may arrive before this SerDes has processed them. For those external -payload sources only, an input without the filesystem marker passes through to the preceding pipeline stage or -standalone delegate. Missing or malformed markers on SDK-checkpointed payloads are permanent errors. +payload sources only, an input without the filesystem marker is decoded directly with the pipeline value codec or +standalone delegate. Skipping every string stage is required because raw external data has not been compressed, +encrypted, or otherwise transformed by those stages. Missing or malformed markers on SDK-checkpointed payloads are +permanent errors. Offloaded filenames include a content hash and are immutable. Serializing new state for the same entity creates a new path instead of replacing a file referenced by an earlier checkpoint. Repeating the same serialization may reuse the @@ -299,6 +316,10 @@ must be protected with the same care as the payload it references. The final file envelope, including any preview, must remain below the checkpoint threshold. Oversized previews are rejected rather than producing a checkpoint that the service cannot accept. +`FileSystemSerDes` declares itself terminal in every mode. This makes its overflow decision apply to the final +checkpoint representation and prevents a later Base64, encryption, or other expanding stage from pushing an inline +envelope over the service limit. + In stage mode, the preview generator receives the string produced by the preceding stage, not the original domain object. A preview that needs domain fields should either parse that representation, be produced by an earlier stage, or use standalone compatibility mode where `FileSystemSerDes` receives the original value. @@ -323,8 +344,8 @@ try { On deserialization, `FileSystemSerDes` parses its envelope. If the envelope contains `data`, it returns the inline string. If the envelope contains `file`, it reads and returns the file contents. `ComposableSerDes` then passes that string to the preceding stage. In standalone compatibility mode, `FileSystemSerDes` instead passes the resolved string -to its configured value-encoding delegate. Raw external input, callback results, and standard invoke results pass -through when no versioned filesystem marker is present. +to its configured value-encoding delegate. Raw external input, callback results, and standard invoke results skip all +string stages and go directly to the value codec when no versioned filesystem marker is present. ### Threading @@ -411,16 +432,19 @@ When serializing `errorData`, set `SerDesPayloadKind.EXCEPTION` and use an entit Root user input and output payloads should route through `SerDesRunner` so `FileSystemSerDes` can see `SerDesContext`. The internal `DurableExecutionInput` and `DurableExecutionOutput` envelope stays with `DurableInputOutputSerDes`. The cloud test runner must send initial Lambda input before it receives a durable execution ARN. When configured with a -`ComposableSerDes`, it therefore serializes the invocation payload with `getValueCodec()` and applies the complete -pipeline only when reading persisted history. Standalone context-dependent SerDes implementations can configure a -separate input codec with `CloudDurableTestRunner.withInputSerDes(...)`. +context-free `ComposableSerDes`, it serializes the invocation payload with the complete configured pipeline so +compression, encryption, and other ordinary transformations remain compatible with the deployed function. When the +persisted SerDes reports that it requires durable context, the runner requires a separate context-free input codec via +`CloudDurableTestRunner.withInputSerDes(...)`. Fluent configuration preserves that explicit input codec regardless of +whether `withInputSerDes(...)` or `withSerDes(...)` is called first. ### Implementation plan 1. Add `SerDesContext`, `SerDesPayloadKind`, and package-private TLS setter/clearer support. Leave the existing `SerDes` methods unchanged. 2. Add the binary-compatible `SerDes.then(...)` default method and `ComposableSerDes` with immutable stage ordering, - forward serialization, reverse deserialization, null short-circuiting, and stage-aware errors. + forward serialization, reverse deserialization, external-boundary bypass, terminal-stage validation, null + short-circuiting, and stage-aware errors. 3. Add `RetryableSerDesException` and `RetrySerDes`, reusing `RetryStrategy` for bounded in-invocation retries. 4. Add `SerDesRunner` with inline execution by default and optional dispatch through `DurableConfig.withSerDesExecutorService(...)`. Do not create a default SerDes pool. diff --git a/docs/advanced/configuration.md b/docs/advanced/configuration.md index 0041c0e5f..cd6692e18 100644 --- a/docs/advanced/configuration.md +++ b/docs/advanced/configuration.md @@ -91,9 +91,11 @@ shared mount such as EFS. S3 Files can have delayed synchronization, so a runtim lose recent writes; use it only when that tradeoff is acceptable. The SDK does not delete offloaded files, so configure storage lifecycle and retention separately. -When cloud tests use a `ComposableSerDes`, `CloudDurableTestRunner` applies only its first value-codec stage to the -initial Lambda invocation because the durable execution ARN does not exist yet. Persisted history is decoded with the -complete pipeline. For a standalone context-dependent SerDes, call `withInputSerDes(...)` with a separate input codec. +For a context-free `ComposableSerDes`, `CloudDurableTestRunner` applies the complete pipeline to the initial Lambda +invocation. If the persisted SerDes requires durable context, such as `FileSystemSerDes`, call +`withInputSerDes(...)` with a separate context-free input codec because the durable execution ARN does not exist yet. +`FileSystemSerDes` must also be the final pipeline stage so its checkpoint-size decision cannot be invalidated by a +later expanding transformation. ### Dynamic plugin loading diff --git a/extra-filesystem-serdes/README.md b/extra-filesystem-serdes/README.md index 84a59397a..3a17b369e 100644 --- a/extra-filesystem-serdes/README.md +++ b/extra-filesystem-serdes/README.md @@ -64,7 +64,8 @@ delays consume time in the current Lambda invocation, so keep attempts and delay ## Replay and envelope behavior Filesystem envelopes include a reserved version marker. Raw root input, callback results, and standard Lambda invoke -results pass through when they have not yet been wrapped by this SerDes. +results bypass every string-processing stage and decode directly with the pipeline value codec when they have not yet +been wrapped by this SerDes. Offloaded files are content-addressed and immutable. Updating wait-for-condition state or retry results creates a new path instead of replacing a file referenced by an earlier checkpoint. Repeating the same write can safely reuse the @@ -73,10 +74,13 @@ owner, while invoke input and result boundaries may consume a file owned by the functions use the same shared root and path encoding. Treat file envelopes as capabilities. Content hashes are verified when reading, and symbolic-link paths are rejected. -`CloudDurableTestRunner` uses the first value-codec stage for the initial Lambda invocation, before an execution ARN is -available, and uses the complete pipeline for persisted history. When using standalone `FileSystemSerDes`, configure a -separate initial-input codec with `withInputSerDes(...)`. An explicitly supplied input SerDes is used exactly as -configured, including every stage in a composable pipeline. +`FileSystemSerDes` must be the final stage in a pipeline. Its overflow decision is therefore made against the final +checkpoint representation; a later expanding transform cannot push an inline envelope over the service limit. + +`CloudDurableTestRunner` cannot use `FileSystemSerDes` for the initial Lambda invocation because an execution ARN is +not available yet. Configure a separate context-free initial-input codec with `withInputSerDes(...)`. An explicitly +supplied input SerDes is used exactly as configured, including every stage in a composable pipeline. Context-free +persisted pipelines use their complete pipeline for the initial invocation. ## Storage requirements diff --git a/extra-filesystem-serdes/src/main/java/software/amazon/lambda/durable/extra/filesystem/FileSystemSerDes.java b/extra-filesystem-serdes/src/main/java/software/amazon/lambda/durable/extra/filesystem/FileSystemSerDes.java index 6913923b6..ce333dcca 100644 --- a/extra-filesystem-serdes/src/main/java/software/amazon/lambda/durable/extra/filesystem/FileSystemSerDes.java +++ b/extra-filesystem-serdes/src/main/java/software/amazon/lambda/durable/extra/filesystem/FileSystemSerDes.java @@ -29,6 +29,7 @@ import software.amazon.lambda.durable.serde.SerDes; import software.amazon.lambda.durable.serde.SerDesContext; import software.amazon.lambda.durable.serde.SerDesPayloadKind; +import software.amazon.lambda.durable.serde.SerDesStageResult; /** * A SerDes that stores payloads on a durable shared filesystem. @@ -121,7 +122,7 @@ public T deserialize(String data, TypeToken typeToken) { } Objects.requireNonNull(typeToken, "typeToken cannot be null"); var context = requireContext(); - var serialized = resolveSerializedPayload(data, context); + var serialized = resolveSerializedPayload(data, context).serialized(); if (stageMode) { if (!TypeToken.get(String.class).equals(typeToken)) { throw new SerDesException("FileSystemSerDes stage can only deserialize to String"); @@ -133,6 +134,28 @@ public T deserialize(String data, TypeToken typeToken) { return delegate.deserialize(serialized, typeToken); } + @Override + public SerDesStageResult deserializePipelineStage(String data) { + if (!stageMode) { + return SerDes.super.deserializePipelineStage(data); + } + var context = requireContext(); + var resolved = resolveSerializedPayload(data, context); + return resolved.external() + ? SerDesStageResult.decodeWithValueCodec(resolved.serialized()) + : SerDesStageResult.continueWith(resolved.serialized()); + } + + @Override + public boolean requiresDurableContext() { + return true; + } + + @Override + public boolean isTerminalPipelineStage() { + return true; + } + private String serializeValue(Object value) { if (stageMode) { if (!(value instanceof String stringValue)) { @@ -147,13 +170,13 @@ private String serializeValue(Object value) { return serialized; } - private String resolveSerializedPayload(String data, SerDesContext context) { + private ResolvedPayload resolveSerializedPayload(String data, SerDesContext context) { final JsonNode envelope; try { envelope = ENVELOPE_MAPPER.readTree(data); } catch (JsonProcessingException e) { if (acceptsExternalPayload(context)) { - return data; + return new ResolvedPayload(data, true); } throw malformedEnvelope(context, e); } @@ -164,7 +187,7 @@ private String resolveSerializedPayload(String data, SerDesContext context) { } } else { if (acceptsExternalPayload(context)) { - return data; + return new ResolvedPayload(data, true); } throw malformedEnvelope(context, null); } @@ -175,10 +198,10 @@ private String resolveSerializedPayload(String data, SerDesContext context) { throw malformedEnvelope(context, null); } if (hasData) { - return envelope.get("data").textValue(); + return new ResolvedPayload(envelope.get("data").textValue(), false); } var owner = payloadOwner(envelope, context); - return readPayload(envelope.get("file").textValue(), owner, context); + return new ResolvedPayload(readPayload(envelope.get("file").textValue(), owner, context), false); } private String readPayload(String fileValue, PayloadOwner owner, SerDesContext context) { @@ -459,6 +482,8 @@ private static String sha256(String value) { private record PayloadOwner(String durableExecutionArn, String entityId) {} + private record ResolvedPayload(String serialized, boolean external) {} + /** Builder for {@link FileSystemSerDes}. */ public static final class Builder { private final Path basePath; diff --git a/extra-filesystem-serdes/src/test/java/software/amazon/lambda/durable/extra/filesystem/FileSystemSerDesTest.java b/extra-filesystem-serdes/src/test/java/software/amazon/lambda/durable/extra/filesystem/FileSystemSerDesTest.java index 9e5f7c0f6..62abffca2 100644 --- a/extra-filesystem-serdes/src/test/java/software/amazon/lambda/durable/extra/filesystem/FileSystemSerDesTest.java +++ b/extra-filesystem-serdes/src/test/java/software/amazon/lambda/durable/extra/filesystem/FileSystemSerDesTest.java @@ -21,6 +21,7 @@ import software.amazon.lambda.durable.exception.SerDesException; import software.amazon.lambda.durable.model.OperationSubType; import software.amazon.lambda.durable.serde.JacksonSerDes; +import software.amazon.lambda.durable.serde.SerDes; import software.amazon.lambda.durable.serde.SerDesContext; import software.amazon.lambda.durable.serde.SerDesPayloadKind; import software.amazon.lambda.durable.serde.SerDesRunner; @@ -70,6 +71,7 @@ void stageModeComposesWithValueCodec() throws Exception { () -> runner.deserialize(stage, envelope, TypeToken.get(Integer.class), context())); assertThrows(IllegalStateException.class, () -> FileSystemSerDes.stageBuilder(basePath) .delegate(new JacksonSerDes())); + assertThrows(IllegalArgumentException.class, () -> pipeline.then(wrappingStage())); } @Test @@ -144,6 +146,7 @@ void includesBoundedPreviewWithoutChangingStoredPayload() throws Exception { void acceptsExternallyOriginatedRawPayloadsOnlyForSupportedSources() { var standalone = FileSystemSerDes.builder(basePath).build(); var stage = FileSystemSerDes.stageBuilder(basePath).build(); + var pipeline = new JacksonSerDes().then(wrappingStage()).then(stage); var runner = new SerDesRunner(null); assertEquals( @@ -154,16 +157,16 @@ void acceptsExternallyOriginatedRawPayloadsOnlyForSupportedSources() { new TypeToken>() {}, executionContext(SerDesPayloadKind.INPUT))); assertEquals( - "{\"id\":42}", + Map.of("id", 42), runner.deserialize( - stage, + pipeline, "{\"id\":42}", - TypeToken.get(String.class), + new TypeToken>() {}, operationContext(OperationType.CALLBACK, OperationSubType.CALLBACK))); assertEquals( - "\"invoke-result\"", + "invoke-result", runner.deserialize( - stage, + pipeline, "\"invoke-result\"", TypeToken.get(String.class), operationContext(OperationType.CHAINED_INVOKE, OperationSubType.CHAINED_INVOKE))); @@ -173,6 +176,19 @@ void acceptsExternallyOriginatedRawPayloadsOnlyForSupportedSources() { () -> runner.deserialize(stage, "\"raw-step\"", TypeToken.get(String.class), context())); } + @Test + void overflowFilesystemStageMustRemainTerminal() { + var filesystem = FileSystemSerDes.stageBuilder(basePath) + .storageMode(FileSystemStorageMode.OVERFLOW) + .build(); + + var failure = assertThrows( + IllegalArgumentException.class, + () -> new JacksonSerDes().then(filesystem).then(wrappingStage())); + + assertTrue(failure.getMessage().contains("final stage")); + } + @Test void fileReferencesCrossInvokeInputAndResultBoundaries() { var serDes = @@ -321,4 +337,19 @@ private static SerDesContext operationContext(OperationType operationType, Opera return SerDesContext.forOperation( ARN, "1", "operation", null, operationType, operationSubType, SerDesPayloadKind.RESULT, null); } + + private static SerDes wrappingStage() { + return new SerDes() { + @Override + public String serialize(Object value) { + return "<" + value + ">"; + } + + @Override + @SuppressWarnings("unchecked") + public T deserialize(String data, TypeToken typeToken) { + return (T) data.substring(1, data.length() - 1); + } + }; + } } diff --git a/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/CloudDurableTestRunner.java b/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/CloudDurableTestRunner.java index 8d698ddd6..667b2f5b6 100644 --- a/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/CloudDurableTestRunner.java +++ b/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/CloudDurableTestRunner.java @@ -10,7 +10,6 @@ import software.amazon.awssdk.services.lambda.model.InvocationType; import software.amazon.awssdk.services.lambda.model.InvokeRequest; import software.amazon.lambda.durable.TypeToken; -import software.amazon.lambda.durable.serde.ComposableSerDes; import software.amazon.lambda.durable.serde.JacksonSerDes; import software.amazon.lambda.durable.serde.SerDes; import software.amazon.lambda.durable.serde.SerDesRunner; @@ -158,15 +157,23 @@ public CloudDurableTestRunner withInvocationType(InvocationType type) { /** Returns a new runner with the specified SerDes for persisted execution payloads. */ public CloudDurableTestRunner withSerDes(SerDes serDes) { return new CloudDurableTestRunner<>( - functionArn, inputType, outputType, lambdaClient, pollInterval, timeout, invocationType, null, serDes); + functionArn, + inputType, + outputType, + lambdaClient, + pollInterval, + timeout, + invocationType, + inputSerDes, + serDes); } /** * Returns a new runner with a separate SerDes for the initial Lambda invocation payload. * - *

The supplied SerDes is used exactly as configured, including every stage in a composable pipeline. When this - * method is not called, composable persisted SerDes pipelines use only their first value-codec stage because a - * durable execution context does not exist before the Lambda invocation. + *

The supplied SerDes is used exactly as configured, including every stage in a composable pipeline. Configure a + * separate context-free input SerDes when the persisted SerDes requires a durable execution context, because that + * context does not exist before the initial Lambda invocation. */ public CloudDurableTestRunner withInputSerDes(SerDes inputSerDes) { return new CloudDurableTestRunner<>( @@ -270,9 +277,11 @@ public TestOperation getOperation(String name) { } private String serializeInput(I input) { - var serializer = inputSerDes; - if (serializer == null) { - serializer = serDes instanceof ComposableSerDes composable ? composable.getValueCodec() : serDes; + var serializer = inputSerDes != null ? inputSerDes : serDes; + if (inputSerDes == null && serializer.requiresDurableContext()) { + throw new IllegalStateException( + "Configured persisted SerDes requires a durable execution context; configure a separate " + + "context-free input SerDes with withInputSerDes(...)"); } return serializer.serialize(input); } diff --git a/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/CloudDurableTestRunnerTest.java b/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/CloudDurableTestRunnerTest.java index 58dc65f9a..5e141645f 100644 --- a/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/CloudDurableTestRunnerTest.java +++ b/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/CloudDurableTestRunnerTest.java @@ -58,7 +58,7 @@ void explicitComposableInputSerDesUsesTheCompletePipeline() { } @Test - void persistedComposableSerDesUsesOnlyValueCodecForDefaultInput() { + void persistedComposableSerDesUsesCompletePipelineForDefaultInput() { var mockClient = mock(LambdaClient.class); when(mockClient.invoke(any(InvokeRequest.class))) .thenReturn(InvokeResponse.builder() @@ -72,7 +72,57 @@ void persistedComposableSerDesUsesOnlyValueCodecForDefaultInput() { var request = ArgumentCaptor.forClass(InvokeRequest.class); verify(mockClient).invoke(request.capture()); - assertEquals("\"value\"", request.getValue().payload().asUtf8String()); + assertEquals("<\"value\">", request.getValue().payload().asUtf8String()); + } + + @Test + void contextDependentPersistedSerDesRequiresExplicitInputSerDes() { + var mockClient = mock(LambdaClient.class); + var contextStage = new SerDes() { + @Override + public String serialize(Object value) { + return value.toString(); + } + + @Override + @SuppressWarnings("unchecked") + public T deserialize(String data, TypeToken typeToken) { + return (T) data; + } + + @Override + public boolean requiresDurableContext() { + return true; + } + }; + var runner = CloudDurableTestRunner.create( + "arn:aws:lambda:us-east-2:123:function:test", String.class, String.class, mockClient) + .withSerDes(new JacksonSerDes().then(contextStage)); + + var failure = assertThrows(RuntimeException.class, () -> runner.startAsync("value")); + + assertInstanceOf(IllegalStateException.class, failure.getCause()); + assertTrue(failure.getCause().getMessage().contains("withInputSerDes")); + verifyNoInteractions(mockClient); + } + + @Test + void replacingPersistedSerDesPreservesExplicitInputSerDes() { + var mockClient = mock(LambdaClient.class); + when(mockClient.invoke(any(InvokeRequest.class))) + .thenReturn(InvokeResponse.builder() + .durableExecutionArn("arn:aws:lambda:us-east-2:123:function:test:1/durable-execution/e/i") + .build()); + var runner = CloudDurableTestRunner.create( + "arn:aws:lambda:us-east-2:123:function:test", String.class, String.class, mockClient) + .withInputSerDes(new JacksonSerDes().then(wrappingStage())) + .withSerDes(new JacksonSerDes()); + + runner.startAsync("value"); + + var request = ArgumentCaptor.forClass(InvokeRequest.class); + verify(mockClient).invoke(request.capture()); + assertEquals("<\"value\">", request.getValue().payload().asUtf8String()); } private static SerDes wrappingStage() { diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/ComposableSerDes.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/ComposableSerDes.java index 2617c118d..2badc4bad 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/ComposableSerDes.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/ComposableSerDes.java @@ -17,14 +17,19 @@ * runs from first to last; deserialization runs from last to first. */ public final class ComposableSerDes implements SerDes { - private static final TypeToken STRING_TYPE = TypeToken.get(String.class); - private final List stages; private ComposableSerDes(List stages) { if (stages.isEmpty()) { throw new IllegalArgumentException("ComposableSerDes requires at least one stage"); } + for (int index = 0; index < stages.size() - 1; index++) { + if (stages.get(index).isTerminalPipelineStage()) { + throw new IllegalArgumentException(String.format( + "SerDes pipeline stage %d (%s) must be the final stage", + index, stages.get(index).getClass().getName())); + } + } this.stages = List.copyOf(stages); } @@ -58,15 +63,22 @@ public static Builder builder(SerDes valueCodec) { /** * Returns the value codec at the start of this pipeline. * - *

This is useful at external input boundaries where a durable execution context does not exist yet and only the - * domain value encoding can be applied. - * * @return the first pipeline stage */ public SerDes getValueCodec() { return stages.get(0); } + @Override + public boolean requiresDurableContext() { + return stages.stream().anyMatch(SerDes::requiresDurableContext); + } + + @Override + public boolean isTerminalPipelineStage() { + return stages.get(stages.size() - 1).isTerminalPipelineStage(); + } + /** * Returns a new pipeline with the supplied stage appended. * @@ -101,23 +113,19 @@ public T deserialize(String data, TypeToken typeToken) { String current = data; for (int index = stages.size() - 1; index > 0; index--) { var decoded = invokeStringStageDeserialize(stages.get(index), current, index); - if (!(decoded instanceof String stringValue)) { - throw stageFailure( - index, - stages.get(index), - "deserialize", - new SerDesException("String stage returned a non-string value")); + current = decoded.value(); + if (decoded.skipRemainingStages()) { + break; } - current = stringValue; } return invokeDeserialize(stages.get(0), current, typeToken, 0); } - private static Object invokeStringStageDeserialize(SerDes stage, String data, int index) { + private static SerDesStageResult invokeStringStageDeserialize(SerDes stage, String data, int index) { try { - Object result = stage.deserialize(data, STRING_TYPE); + var result = stage.deserializePipelineStage(data); if (result == null) { - throw new SerDesException("Stage returned null for non-null data"); + throw new SerDesException("Stage returned a null pipeline result"); } return result; } catch (Throwable failure) { diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/RetrySerDes.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/RetrySerDes.java index 74428d9c4..2b0017baf 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/RetrySerDes.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/RetrySerDes.java @@ -58,6 +58,21 @@ public T deserialize(String data, TypeToken typeToken) { return execute("deserialization", () -> delegate.deserialize(data, typeToken)); } + @Override + public SerDesStageResult deserializePipelineStage(String data) { + return execute("pipeline stage deserialization", () -> delegate.deserializePipelineStage(data)); + } + + @Override + public boolean requiresDurableContext() { + return delegate.requiresDurableContext(); + } + + @Override + public boolean isTerminalPipelineStage() { + return delegate.isTerminalPipelineStage(); + } + private T execute(String action, Supplier operation) { int attempt = 1; while (true) { diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDes.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDes.java index b353cec80..648391ef7 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDes.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDes.java @@ -3,6 +3,7 @@ package software.amazon.lambda.durable.serde; import software.amazon.lambda.durable.TypeToken; +import software.amazon.lambda.durable.exception.SerDesException; /** * Interface for serialization and deserialization of objects. @@ -37,6 +38,47 @@ public interface SerDes { */ T deserialize(String data, TypeToken typeToken); + /** + * Deserializes this SerDes when it is used as a string-processing pipeline stage. + * + *

Most stages should use the default result, which continues reverse processing through earlier string stages. + * Boundary stages may return {@link SerDesStageResult#decodeWithValueCodec(String)} when the input originated + * outside the configured pipeline and must be decoded directly by the value codec. + * + * @param data the non-null string supplied to this stage + * @return the stage result + */ + default SerDesStageResult deserializePipelineStage(String data) { + Object result = deserialize(data, TypeToken.get(String.class)); + if (result == null) { + throw new SerDesException("Stage returned null for non-null data"); + } + if (!(result instanceof String stringResult)) { + throw new SerDesException("String stage returned a non-string value"); + } + return SerDesStageResult.continueWith(stringResult); + } + + /** + * Returns whether this SerDes requires an SDK-managed durable execution context. + * + *

Context-dependent SerDes implementations cannot process an initial external invocation payload unless a + * separate context-free input SerDes is configured. + */ + default boolean requiresDurableContext() { + return false; + } + + /** + * Returns whether this SerDes must be the final stage in a composable pipeline. + * + *

Stages that make size-based storage decisions should normally be terminal so later transformations cannot + * expand their output beyond checkpoint limits. + */ + default boolean isTerminalPipelineStage() { + return false; + } + /** * Returns an immutable processing pipeline that invokes this SerDes followed by {@code nextStage} when serializing * and in reverse order when deserializing. diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesStageResult.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesStageResult.java new file mode 100644 index 000000000..b28190db8 --- /dev/null +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesStageResult.java @@ -0,0 +1,28 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.serde; + +import java.util.Objects; + +/** + * Result returned when a {@link SerDes} is used as a string-processing stage in a {@link ComposableSerDes}. + * + * @param value the string produced by the stage + * @param skipRemainingStages whether deserialization should skip the remaining string stages and decode {@code value} + * directly with the pipeline's value codec + */ +public record SerDesStageResult(String value, boolean skipRemainingStages) { + public SerDesStageResult { + Objects.requireNonNull(value, "value cannot be null"); + } + + /** Continues reverse processing through the remaining string stages. */ + public static SerDesStageResult continueWith(String value) { + return new SerDesStageResult(value, false); + } + + /** Skips the remaining string stages and decodes the value directly with the pipeline's value codec. */ + public static SerDesStageResult decodeWithValueCodec(String value) { + return new SerDesStageResult(value, true); + } +} diff --git a/sdk/src/test/java/software/amazon/lambda/durable/serde/ComposableSerDesTest.java b/sdk/src/test/java/software/amazon/lambda/durable/serde/ComposableSerDesTest.java index dba62a3cc..80317ae23 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/serde/ComposableSerDesTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/serde/ComposableSerDesTest.java @@ -90,6 +90,73 @@ public T deserialize(String data, TypeToken typeToken) { assertEquals(1, intermediateCalls.get()); } + @Test + void stageMayDecodeExternalDataDirectlyWithValueCodec() { + var transformDeserializations = new AtomicInteger(); + var transform = new SerDes() { + @Override + public String serialize(Object value) { + return "<" + value + ">"; + } + + @Override + @SuppressWarnings("unchecked") + public T deserialize(String data, TypeToken typeToken) { + transformDeserializations.incrementAndGet(); + return (T) data.substring(1, data.length() - 1); + } + }; + var externalBoundary = new SerDes() { + @Override + public String serialize(Object value) { + return value.toString(); + } + + @Override + @SuppressWarnings("unchecked") + public T deserialize(String data, TypeToken typeToken) { + return (T) data; + } + + @Override + public SerDesStageResult deserializePipelineStage(String data) { + return SerDesStageResult.decodeWithValueCodec(data); + } + }; + var pipeline = new JacksonSerDes().then(transform).then(externalBoundary); + + assertEquals("value", pipeline.deserialize("\"value\"", TypeToken.get(String.class))); + assertEquals(0, transformDeserializations.get()); + } + + @Test + void rejectsStagesAfterTerminalStage() { + var terminal = new SerDes() { + @Override + public String serialize(Object value) { + return value.toString(); + } + + @Override + @SuppressWarnings("unchecked") + public T deserialize(String data, TypeToken typeToken) { + return (T) data; + } + + @Override + public boolean isTerminalPipelineStage() { + return true; + } + }; + + var failure = assertThrows( + IllegalArgumentException.class, + () -> new JacksonSerDes().then(terminal).then(stringStage("late", "", "", new ArrayList<>()))); + + assertTrue(failure.getMessage().contains("stage 1")); + assertTrue(failure.getMessage().contains("final stage")); + } + @Test void rejectsNullAndNonStringIntermediateValuesWithStageMetadata() { var nullStage = new SerDes() { diff --git a/sdk/src/test/java/software/amazon/lambda/durable/serde/RetrySerDesTest.java b/sdk/src/test/java/software/amazon/lambda/durable/serde/RetrySerDesTest.java index b2a6d8503..4c1cf0bb6 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/serde/RetrySerDesTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/serde/RetrySerDesTest.java @@ -73,6 +73,50 @@ public T deserialize(String data, TypeToken typeToken) { assertEquals(2, calls.get()); } + @Test + void retriesPipelineStageDeserializationAndDelegatesCapabilities() { + var calls = new AtomicInteger(); + var delegate = new SerDes() { + @Override + public String serialize(Object value) { + return value.toString(); + } + + @Override + public T deserialize(String data, TypeToken typeToken) { + return null; + } + + @Override + public SerDesStageResult deserializePipelineStage(String data) { + if (calls.incrementAndGet() == 1) { + throw new RetryableSerDesException("transient"); + } + return SerDesStageResult.decodeWithValueCodec(data); + } + + @Override + public boolean requiresDurableContext() { + return true; + } + + @Override + public boolean isTerminalPipelineStage() { + return true; + } + }; + var retrySerDes = + new RetrySerDes(delegate, (error, attempt) -> RetryDecision.retry(Duration.ZERO), delay -> {}); + + var result = retrySerDes.deserializePipelineStage("value"); + + assertEquals("value", result.value()); + assertTrue(result.skipRemainingStages()); + assertEquals(2, calls.get()); + assertTrue(retrySerDes.requiresDurableContext()); + assertTrue(retrySerDes.isTerminalPipelineStage()); + } + @Test void doesNotRetryPermanentSerDesFailure() { var calls = new AtomicInteger(); From 150a5f6ef53b2ef4e1468be81aabe4470a10ffc7 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Tue, 25 Aug 2026 00:39:52 +0000 Subject: [PATCH 09/56] refactor: move filesystem SerDes into core SDK --- .github/scripts/maven_publish.sh | 1 - .github/workflows/build.yml | 2 - .github/workflows/e2e-tests.yml | 2 - .github/workflows/publish_maven.yml | 1 - README.md | 14 +--- coverage-report/pom.xml | 5 -- docs/adr/005-filesystem-serdes.md | 65 +++++++++--------- docs/advanced/configuration.md | 3 +- .../advanced/filesystem-serdes.md | 6 +- docs/design.md | 6 +- extra-filesystem-serdes/pom.xml | 66 ------------------- pom.xml | 1 - sdk-integration-tests/pom.xml | 6 -- .../FileSystemSerDesIntegrationTest.java | 2 +- .../serde}/FileSystemPathEncoding.java | 2 +- .../durable/serde}/FileSystemSerDes.java | 7 +- .../durable/serde}/FileSystemStorageMode.java | 2 +- .../durable/serde}/FileSystemSerDesTest.java | 7 +- 18 files changed, 47 insertions(+), 151 deletions(-) rename extra-filesystem-serdes/README.md => docs/advanced/filesystem-serdes.md (94%) delete mode 100644 extra-filesystem-serdes/pom.xml rename {extra-filesystem-serdes/src/main/java/software/amazon/lambda/durable/extra/filesystem => sdk/src/main/java/software/amazon/lambda/durable/serde}/FileSystemPathEncoding.java (82%) rename {extra-filesystem-serdes/src/main/java/software/amazon/lambda/durable/extra/filesystem => sdk/src/main/java/software/amazon/lambda/durable/serde}/FileSystemSerDes.java (98%) rename {extra-filesystem-serdes/src/main/java/software/amazon/lambda/durable/extra/filesystem => sdk/src/main/java/software/amazon/lambda/durable/serde}/FileSystemStorageMode.java (81%) rename {extra-filesystem-serdes/src/test/java/software/amazon/lambda/durable/extra/filesystem => sdk/src/test/java/software/amazon/lambda/durable/serde}/FileSystemSerDesTest.java (97%) diff --git a/.github/scripts/maven_publish.sh b/.github/scripts/maven_publish.sh index c8928c348..655ad5c69 100644 --- a/.github/scripts/maven_publish.sh +++ b/.github/scripts/maven_publish.sh @@ -43,7 +43,6 @@ echo "settings.xml written." echo "=== Step 3: Upload to Sonatype Central Portal ===" mvn clean deploy -s "${SETTINGS_FILE}" -pl sdk -P publishing -DskipTests --no-transfer-progress -mvn clean deploy -s "${SETTINGS_FILE}" -pl extra-filesystem-serdes -P publishing -DskipTests --no-transfer-progress mvn clean deploy -s "${SETTINGS_FILE}" -pl sdk-testing -P publishing -DskipTests --no-transfer-progress mvn clean deploy -s "${SETTINGS_FILE}" -pl otel-plugin -P publishing -DskipTests --no-transfer-progress diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9ccde6759..26e154242 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -26,7 +26,6 @@ on: - '.github/workflows/ai-pr-review.yml' - '.github/prompts/ai-pr-review.md' - 'sdk/**' - - 'extra-filesystem-serdes/**' - 'sdk-testing/**' - 'sdk-integration-tests/**' - 'examples/**' @@ -39,7 +38,6 @@ on: - '.github/workflows/ai-pr-review.yml' - '.github/prompts/ai-pr-review.md' - 'sdk/**' - - 'extra-filesystem-serdes/**' - 'sdk-testing/**' - 'sdk-integration-tests/**' - 'examples/**' diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index 42367a5a2..8d9f9d4cd 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -8,7 +8,6 @@ on: paths: - '.github/**' # for testing Github Actions - 'sdk/**' - - 'extra-filesystem-serdes/**' - 'sdk-testing/**' - 'sdk-integration-tests/**' - 'examples/**' @@ -19,7 +18,6 @@ on: paths: - '.github/**' - 'sdk/**' - - 'extra-filesystem-serdes/**' - 'sdk-testing/**' - 'sdk-integration-tests/**' - 'examples/**' diff --git a/.github/workflows/publish_maven.yml b/.github/workflows/publish_maven.yml index 547f6dbe9..a664ca282 100644 --- a/.github/workflows/publish_maven.yml +++ b/.github/workflows/publish_maven.yml @@ -90,7 +90,6 @@ jobs: run: | gh release upload "$RELEASE_TAG" \ "sdk/target/aws-durable-execution-sdk-java-${RELEASE_VERSION}.jar" \ - "extra-filesystem-serdes/target/aws-durable-execution-sdk-java-extra-filesystem-serdes-${RELEASE_VERSION}.jar" \ "sdk-testing/target/aws-durable-execution-sdk-java-testing-${RELEASE_VERSION}.jar" \ "otel-plugin/target/aws-durable-execution-sdk-java-plugin-otel-${RELEASE_VERSION}.jar" \ --clobber diff --git a/README.md b/README.md index 9c69778c4..306c0d8e3 100644 --- a/README.md +++ b/README.md @@ -50,17 +50,8 @@ Your durable function extends `DurableHandler` and implements `handleReque ``` -For filesystem-backed payload storage, add the optional module: - -```xml - - software.amazon.lambda.durable - aws-durable-execution-sdk-java-extra-filesystem-serdes - VERSION - -``` - -See [Filesystem SerDes](extra-filesystem-serdes/README.md) for configuration and durability requirements. +Filesystem-backed payload storage is included in the core SDK. See +[Filesystem SerDes](docs/advanced/filesystem-serdes.md) for configuration and durability requirements. ### Your First Durable Function @@ -123,6 +114,7 @@ See [Deploy Lambda durable functions with Infrastructure as Code](https://docs.a **Advanced Topics** - [Configuration](docs/advanced/configuration.md) - Customize SDK behaviour +- [Filesystem SerDes](docs/advanced/filesystem-serdes.md) - Store durable payloads on a shared filesystem - [Error Handling](docs/advanced/error-handling.md) - SDK exceptions for handling failures - [Logging](docs/advanced/logging.md) - How to use DurableLogger - [Migrating from 1.x to 2.x](docs/migration-1.x-to-2.x.md) - Upgrade guide for breaking changes since `v1.2.1` diff --git a/coverage-report/pom.xml b/coverage-report/pom.xml index 4e860d478..e820594e1 100644 --- a/coverage-report/pom.xml +++ b/coverage-report/pom.xml @@ -22,11 +22,6 @@ aws-durable-execution-sdk-java ${project.version} - - software.amazon.lambda.durable - aws-durable-execution-sdk-java-extra-filesystem-serdes - ${project.version} - software.amazon.lambda.durable aws-durable-execution-sdk-java-testing diff --git a/docs/adr/005-filesystem-serdes.md b/docs/adr/005-filesystem-serdes.md index 5f2c75de1..c6352e352 100644 --- a/docs/adr/005-filesystem-serdes.md +++ b/docs/adr/005-filesystem-serdes.md @@ -2,7 +2,7 @@ **Status:** Accepted — Approach A with ComposableSerDes pipeline **Date:** 2026-07-02 -**Updated:** 2026-08-24 — Added the composable SerDes pipeline and optional executor design. +**Updated:** 2026-08-25 — Included FileSystemSerDes in the core SDK artifact. ## Context @@ -23,7 +23,8 @@ There are a few Java-specific constraints: - The same operation payload can be deserialized multiple times in one invocation because most operation results are not cached after deserialization. - Java uses the configured `SerDes` for both operation results and user-defined exception objects stored in `ErrorObject.errorData`. - `DurableInputOutputSerDes` is a hard-coded internal serializer for the Lambda Durable Functions request and response envelope. It is separate from the customer-facing `DurableConfig.getSerDes()`. -- Filesystem-backed storage is optional and storage-specific. It should not add filesystem-oriented public surface area to the core SDK artifact. +- The filesystem implementation uses JDK filesystem APIs and existing core dependencies. Including the initial + implementation in the core SDK avoids a second artifact and release path for the accepted parity feature. - Filesystem persistence is not automatically durable. Lambda `/tmp` is not valid for replay across environments. Mounted S3 Files may have delayed synchronization and can lose recent writes if the runtime crashes before the mount flushes. EFS or an explicitly accepted S3 Files durability tradeoff should be required for production use. ## Approach A: Reuse SerDes for Offload @@ -32,7 +33,7 @@ There are a few Java-specific constraints: Keep the existing `SerDes` serialization methods source- and binary-compatible, add a default composition method and a core `ComposableSerDes` implementation which together chain multiple `SerDes` instances into a processing pipeline, -and implement `FileSystemSerDes` as an optional extra package. The filesystem stage uses +and implement `FileSystemSerDes` in the core SDK. The filesystem stage uses `SerDesContext.getCurrentContext()` to identify the durable execution and entity being serialized. ```java @@ -83,17 +84,19 @@ The SDK owns setting and clearing this thread-local value around SDK-managed Ser | Concern | Decision | |---------|----------| -| Maven module directory | `extra-filesystem-serdes` | -| Maven artifact ID | `aws-durable-execution-sdk-java-extra-filesystem-serdes` | +| Maven module directory | `sdk` | +| Maven artifact ID | `aws-durable-execution-sdk-java` | | Maven group ID | `software.amazon.lambda.durable` | -| Java package | `software.amazon.lambda.durable.extra.filesystem` | -| Core dependency direction | Extra module depends on `aws-durable-execution-sdk-java`; core does not depend on extras. | +| Java package | `software.amazon.lambda.durable.serde` | +| Dependency impact | No additional artifact or production dependency is required. | ### Configuration ```java -import software.amazon.lambda.durable.serde.ComposableSerDes; -import software.amazon.lambda.durable.extra.filesystem.FileSystemSerDes; +import software.amazon.lambda.durable.serde.FileSystemPathEncoding; +import software.amazon.lambda.durable.serde.FileSystemSerDes; +import software.amazon.lambda.durable.serde.FileSystemStorageMode; +import software.amazon.lambda.durable.serde.JacksonSerDes; var fileSystemStage = FileSystemSerDes.stageBuilder(Path.of("/mnt/efs/durable-payloads")) .storageMode(FileSystemStorageMode.ALWAYS) @@ -454,22 +457,19 @@ whether `withInputSerDes(...)` or `withSerDes(...)` is called first. `CallbackOperation`, `ChildContextOperation`, `MapOperation`, and test helpers to use `SerDesRunner`. 7. Add invocation-scoped deserialization caching keyed by entity, payload kind, type, and serialized data hash. 8. Update exception serialization and deserialization paths to set `SerDesPayloadKind.EXCEPTION` in TLS. -9. Add the `extra-filesystem-serdes` Maven module with artifact ID - `aws-durable-execution-sdk-java-extra-filesystem-serdes`, depending on the core SDK. -10. Implement `FileSystemSerDes` in `software.amazon.lambda.durable.extra.filesystem` with standalone compatibility and +9. Implement `FileSystemSerDes` in the core `software.amazon.lambda.durable.serde` package with standalone compatibility and string-stage modes, `ALWAYS` and `OVERFLOW` storage, `URI` and `HASH` path encodings, envelope parsing, atomic file writes where supported by the filesystem, retryable I/O failures, and clear validation errors for missing context or invalid stage input. -11. Add unit tests for pipeline ordering, reverse processing, nulls, invalid intermediate stage types, stage failures, +10. Add unit tests for pipeline ordering, reverse processing, nulls, invalid intermediate stage types, stage failures, retry selection, exhaustion, delay handling, interruption, context construction, TLS scoping and restoration, inline execution, configured thread-pool isolation, cache hits, cache invalidation, exception reconstruction, - malformed filesystem envelopes, and extra-module packaging. -12. Add integration tests with `LocalDurableTestRunner` for multi-stage pipelines, step results, wait-for-condition + malformed filesystem envelopes, and core-artifact packaging. +11. Add integration tests with `LocalDurableTestRunner` for multi-stage pipelines, step results, wait-for-condition state, invoke payload/result, child context results, map results, repeated `get()`, replay from file pointers, and custom exception types. -13. Update README and advanced configuration docs with pipeline and retry examples, FileSystemSerDes dependency coordinates, - filesystem configuration, and warnings about `/tmp`, S3 Files flush behavior, and EFS/S3 Files operational - requirements. +12. Update README and advanced configuration docs with pipeline and retry examples, filesystem configuration, and + warnings about `/tmp`, S3 Files flush behavior, and EFS/S3 Files operational requirements. ### Pros @@ -478,7 +478,7 @@ whether `withInputSerDes(...)` or `withSerDes(...)` is called first. - Preserves inline SerDes execution by default, avoiding new thread-hop overhead for existing applications. - Makes serialization, compression, encryption, and storage independently composable without adding a storage-specific core interface. -- Keeps the first implementation in an optional `aws-durable-execution-sdk-java-extra-*` module. +- Makes filesystem storage available without an additional Maven dependency or release artifact. - Avoids committing the core SDK to a generalized offloading envelope before the storage use cases are proven. - Closest to the current JavaScript `createFileSystemSerdes` model and issue #463 wording. @@ -689,14 +689,14 @@ This approach gives the SDK one consistent policy for root payloads, operation r | Envelope ownership | FileSystemSerDes owns the checkpoint envelope (`data`, `file`, preview), so the SDK treats it as opaque serialized data. | SDK owns the checkpoint/offload envelope and must guarantee it composes with replay, errors, callbacks, and test utilities. | | Caching | SDK can cache deserialized values, but FileSystemSerDes may still do file reads internally unless cache hits happen before SerDes. | SDK can cache at both layers: resolved offloaded payload text and final deserialized object. | | Exception handling | Works if every exception serialization path is routed through SerDes with `SerDesPayloadKind.EXCEPTION`. | Works uniformly because exception `errorData` is another serialized payload that can be offloaded after SerDes. | -| Third-party storage | S3, DynamoDB, and other backends can be implemented as additional reversible SerDes stages in extra packages. | Natural home for multiple storage backends: filesystem, S3, DynamoDB, EFS, S3 Files, or custom customer storage. | +| Third-party storage | S3, DynamoDB, and other backends can be implemented as additional reversible SerDes stages, either in core or separate artifacts based on their dependencies and support model. | Natural home for multiple storage backends: filesystem, S3, DynamoDB, EFS, S3 Files, or custom customer storage. | | Immediate delivery risk | Lower. Builds on existing customization point. | Higher. Requires new API and more runtime integration. | | Long-term design risk | Higher. Blurs SerDes semantics and may accumulate storage behavior in serializers. | Lower if offloading grows into a first-class feature, but higher if this remains a one-off filesystem parity feature. | ## Decision Adopt **Approach A: Reuse SerDes for Offload**, extended with a core `ComposableSerDes` pipeline. It delivers -JavaScript parity, keeps filesystem behavior in an optional artifact, leaves the existing `SerDes` methods unchanged, +JavaScript parity, includes filesystem behavior in the existing core artifact, leaves the existing `SerDes` methods unchanged, and lets customers assemble value encoding, compression, encryption, and storage as independently reusable stages. Approach B remains a possible future direction if payload offloading grows into a general multi-backend capability that requires SDK-owned storage envelopes and lifecycle policy. @@ -707,9 +707,11 @@ that requires SDK-owned storage envelopes and lifecycle policy. Rejected. A filesystem-backed implementation needs stable operation identity. Without context, it cannot choose a safe file name, distinguish result and exception payloads for the same operation, or avoid collisions across durable executions. -### Put filesystem-backed offloading in the core SDK artifact +### Publish filesystem-backed offloading as a separate artifact -Rejected. Filesystem-backed storage is optional, storage-specific functionality. Keeping it in an `aws-durable-execution-sdk-java-extra-*` artifact preserves a small core SDK and creates a repeatable package shape for future optional features. +Rejected for Approach A. The implementation adds no new production dependency, is part of the accepted JavaScript +parity feature, and already relies on core SerDes context and pipeline behavior. A separate artifact would add module, +publishing, documentation, and dependency-management overhead without isolating a distinct dependency graph. ### Add context-aware SerDes overloads @@ -749,8 +751,7 @@ Positive: - Both approaches enable filesystem-backed payload storage without changing the existing `serialize`/`deserialize` signatures. -- Filesystem-specific functionality stays out of the core SDK artifact. -- The repository gets a repeatable `aws-durable-execution-sdk-java-extra-xxx` artifact pattern for optional packages. +- Approach A makes filesystem-backed storage available from the core SDK without an additional artifact. - Custom payload implementations get enough context to use external storage safely. - Customers can compose reusable SerDes stages without creating a bespoke wrapper for each combination. - Blocking payload work can be isolated from user operation threads with an explicitly configured SerDes executor and @@ -761,7 +762,7 @@ Positive: Negative: - Adds optional executor, context, and caching machinery that must stay deterministic. -- Adds at least one Maven module and published artifact to release and document. +- Adds storage-specific public API and implementation code to the core SDK artifact. - Approach A requires thread-local SerDes context because the existing `SerDes` methods do not accept context. - Approach A relies on a documented string-stage convention that is validated at runtime rather than by Java's type system. @@ -806,20 +807,20 @@ Both approaches need a stable payload identity that can be used to address exter Do not include the checkpoint token or raw user payload in the context. -### Extra package pattern - -Payload offloading implementations should live outside the core SDK artifact when they target a specific storage mechanism. +### Packaging boundary -Use the `aws-durable-execution-sdk-java-extra-xxx` artifact pattern. The filesystem payload package name depends on which approach is chosen; the repository should not publish both a filesystem SerDes package and a filesystem offloader package for the same feature. +Approach A's filesystem implementation is part of the core SDK because it adds no external production dependency and +is the concrete parity feature accepted by this ADR. A future storage stage may use a separate artifact when it brings +substantial provider-specific dependencies or has an independent support and release model. | Feature | Artifact ID | Java package | |---------|-------------|--------------| -| Filesystem payload storage, Approach A | `aws-durable-execution-sdk-java-extra-filesystem-serdes` | `software.amazon.lambda.durable.extra.filesystem` | +| Filesystem payload storage, Approach A | `aws-durable-execution-sdk-java` | `software.amazon.lambda.durable.serde` | | Filesystem payload storage, Approach B | `aws-durable-execution-sdk-java-extra-filesystem-offloader` | `software.amazon.lambda.durable.extra.filesystem` | | Event deserialization helpers | `aws-durable-execution-sdk-java-extra-event-deserialization` | `software.amazon.lambda.durable.extra.eventdeserialization` | | Virtual thread executor helpers | `aws-durable-execution-sdk-java-extra-virtual-thread-pool` | `software.amazon.lambda.durable.extra.virtualthreads` | -Extra modules should be independently documented, tested, and versioned with the repository release. They may depend on the core SDK and normal support libraries, but the core SDK should expose stable extension points without knowing about any specific extra package. For filesystem payload storage, create exactly one extra module after choosing Approach A or Approach B. +The repository should not publish both a filesystem SerDes and a filesystem offloader for the same feature. ### Protocol SerDes boundary diff --git a/docs/advanced/configuration.md b/docs/advanced/configuration.md index cd6692e18..3ed13081d 100644 --- a/docs/advanced/configuration.md +++ b/docs/advanced/configuration.md @@ -53,8 +53,7 @@ and caches successful deserialization results for the current invocation. ### Filesystem-backed payload storage -The optional `aws-durable-execution-sdk-java-extra-filesystem-serdes` artifact provides a reversible string stage for -storing serialized payloads on a shared filesystem: +The core SDK provides a reversible string stage for storing serialized payloads on a shared filesystem: ```java var fileSystemStage = FileSystemSerDes.stageBuilder(Path.of("/mnt/efs/durable-payloads")) diff --git a/extra-filesystem-serdes/README.md b/docs/advanced/filesystem-serdes.md similarity index 94% rename from extra-filesystem-serdes/README.md rename to docs/advanced/filesystem-serdes.md index 3a17b369e..4e497bfcb 100644 --- a/extra-filesystem-serdes/README.md +++ b/docs/advanced/filesystem-serdes.md @@ -1,14 +1,14 @@ # Filesystem SerDes -`aws-durable-execution-sdk-java-extra-filesystem-serdes` stores durable user payloads on a shared filesystem while -keeping small, versioned file-reference envelopes in checkpoints. +`FileSystemSerDes` stores durable user payloads on a shared filesystem while keeping small, versioned file-reference +envelopes in checkpoints. It is included in the core `aws-durable-execution-sdk-java` artifact. ## Installation ```xml software.amazon.lambda.durable - aws-durable-execution-sdk-java-extra-filesystem-serdes + aws-durable-execution-sdk-java VERSION ``` diff --git a/docs/design.md b/docs/design.md index 6c2007873..072e7590f 100644 --- a/docs/design.md +++ b/docs/design.md @@ -10,8 +10,7 @@ This document explains the internal architecture, threading model, and extension ``` aws-durable-execution-sdk-java/ -├── sdk/ # Core SDK - DurableHandler, DurableContext, operations -├── extra-filesystem-serdes/ # Optional filesystem-backed SerDes pipeline stage +├── sdk/ # Core SDK - DurableHandler, DurableContext, operations, SerDes ├── sdk-testing/ # Test utilities for local and cloud testing ├── sdk-integration-tests/ # Integration tests using LocalDurableTestRunner └── examples/ # Real-world usage patterns as customers would implement them @@ -19,8 +18,7 @@ aws-durable-execution-sdk-java/ | Module | Purpose | Key Classes | |--------|---------|-------------| -| `sdk` | Core runtime - extend `DurableHandler`, use `DurableContext` for durable operations | `DurableHandler`, `DurableContext`, `DurableExecutor`, `ExecutionManager` | -| `extra-filesystem-serdes` | Optional filesystem-backed payload storage for SerDes pipelines | `FileSystemSerDes`, `FileSystemStorageMode`, `FileSystemPathEncoding` | +| `sdk` | Core runtime - extend `DurableHandler`, use `DurableContext` for durable operations, and configure composable or filesystem-backed SerDes | `DurableHandler`, `DurableContext`, `DurableExecutor`, `ExecutionManager`, `FileSystemSerDes` | | `sdk-testing` | Test utilities: `LocalDurableTestRunner` (in-memory, simulates re-invocations and time-skipping) and `CloudDurableTestRunner` (executes against deployed Lambda) | `LocalDurableTestRunner`, `CloudDurableTestRunner`, `LocalMemoryExecutionClient`, `TestResult` | | `sdk-integration-tests` | Dogfooding tests - validates the SDK using its own test utilities. Separate module keeps dependencies acyclic: `sdk` → `sdk-testing` → `sdk-integration-tests`. | Test classes only | | `examples` | Real-world usage patterns as customers would implement them, with local and cloud tests | Example handlers, `CloudBasedIntegrationTest` | diff --git a/extra-filesystem-serdes/pom.xml b/extra-filesystem-serdes/pom.xml deleted file mode 100644 index c7f55d269..000000000 --- a/extra-filesystem-serdes/pom.xml +++ /dev/null @@ -1,66 +0,0 @@ - - - 4.0.0 - - software.amazon.lambda.durable - aws-durable-execution-sdk-java-parent - 2.1.1-SNAPSHOT - - aws-durable-execution-sdk-java-extra-filesystem-serdes - AWS Lambda Durable Execution SDK Filesystem SerDes - Optional filesystem-backed payload SerDes for the AWS Lambda Durable Execution SDK - - - software.amazon.lambda.durable - aws-durable-execution-sdk-java - ${project.version} - - - com.fasterxml.jackson.core - jackson-databind - - - org.junit.jupiter - junit-jupiter - test - - - - - - org.apache.maven.plugins - maven-compiler-plugin - - - org.apache.maven.plugins - maven-surefire-plugin - - - org.apache.maven.plugins - maven-source-plugin - - - attach-sources - - jar-no-fork - - - - - - org.apache.maven.plugins - maven-javadoc-plugin - - - attach-javadocs - - jar - - - - - - - diff --git a/pom.xml b/pom.xml index 674bfaf34..2a5c8356a 100644 --- a/pom.xml +++ b/pom.xml @@ -40,7 +40,6 @@ sdk - extra-filesystem-serdes sdk-testing sdk-integration-tests otel-plugin diff --git a/sdk-integration-tests/pom.xml b/sdk-integration-tests/pom.xml index a1bc76bc6..9d0459a07 100644 --- a/sdk-integration-tests/pom.xml +++ b/sdk-integration-tests/pom.xml @@ -36,12 +36,6 @@ ${project.version} test - - software.amazon.lambda.durable - aws-durable-execution-sdk-java-extra-filesystem-serdes - ${project.version} - test - org.junit.jupiter junit-jupiter diff --git a/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FileSystemSerDesIntegrationTest.java b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FileSystemSerDesIntegrationTest.java index b3360c076..79233c1a0 100644 --- a/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FileSystemSerDesIntegrationTest.java +++ b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FileSystemSerDesIntegrationTest.java @@ -29,13 +29,13 @@ import software.amazon.lambda.durable.config.StepConfig; import software.amazon.lambda.durable.config.WaitForConditionConfig; import software.amazon.lambda.durable.execution.DurableExecutor; -import software.amazon.lambda.durable.extra.filesystem.FileSystemSerDes; import software.amazon.lambda.durable.model.DurableExecutionInput; import software.amazon.lambda.durable.model.ExecutionStatus; import software.amazon.lambda.durable.model.WaitForConditionResult; import software.amazon.lambda.durable.retry.JitterStrategy; import software.amazon.lambda.durable.retry.RetryStrategies; import software.amazon.lambda.durable.retry.WaitStrategies; +import software.amazon.lambda.durable.serde.FileSystemSerDes; import software.amazon.lambda.durable.serde.JacksonSerDes; import software.amazon.lambda.durable.serde.SerDes; import software.amazon.lambda.durable.serde.SerDesContext; diff --git a/extra-filesystem-serdes/src/main/java/software/amazon/lambda/durable/extra/filesystem/FileSystemPathEncoding.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemPathEncoding.java similarity index 82% rename from extra-filesystem-serdes/src/main/java/software/amazon/lambda/durable/extra/filesystem/FileSystemPathEncoding.java rename to sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemPathEncoding.java index 4c3d0907c..76d53032c 100644 --- a/extra-filesystem-serdes/src/main/java/software/amazon/lambda/durable/extra/filesystem/FileSystemPathEncoding.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemPathEncoding.java @@ -1,6 +1,6 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -package software.amazon.lambda.durable.extra.filesystem; +package software.amazon.lambda.durable.serde; /** Controls how durable execution and entity identifiers are encoded as filesystem paths. */ public enum FileSystemPathEncoding { diff --git a/extra-filesystem-serdes/src/main/java/software/amazon/lambda/durable/extra/filesystem/FileSystemSerDes.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java similarity index 98% rename from extra-filesystem-serdes/src/main/java/software/amazon/lambda/durable/extra/filesystem/FileSystemSerDes.java rename to sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java index ce333dcca..6c9411f08 100644 --- a/extra-filesystem-serdes/src/main/java/software/amazon/lambda/durable/extra/filesystem/FileSystemSerDes.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java @@ -1,6 +1,6 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -package software.amazon.lambda.durable.extra.filesystem; +package software.amazon.lambda.durable.serde; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; @@ -25,11 +25,6 @@ import software.amazon.lambda.durable.TypeToken; import software.amazon.lambda.durable.exception.RetryableSerDesException; import software.amazon.lambda.durable.exception.SerDesException; -import software.amazon.lambda.durable.serde.JacksonSerDes; -import software.amazon.lambda.durable.serde.SerDes; -import software.amazon.lambda.durable.serde.SerDesContext; -import software.amazon.lambda.durable.serde.SerDesPayloadKind; -import software.amazon.lambda.durable.serde.SerDesStageResult; /** * A SerDes that stores payloads on a durable shared filesystem. diff --git a/extra-filesystem-serdes/src/main/java/software/amazon/lambda/durable/extra/filesystem/FileSystemStorageMode.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemStorageMode.java similarity index 81% rename from extra-filesystem-serdes/src/main/java/software/amazon/lambda/durable/extra/filesystem/FileSystemStorageMode.java rename to sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemStorageMode.java index 254f5805c..02ca68b98 100644 --- a/extra-filesystem-serdes/src/main/java/software/amazon/lambda/durable/extra/filesystem/FileSystemStorageMode.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemStorageMode.java @@ -1,6 +1,6 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -package software.amazon.lambda.durable.extra.filesystem; +package software.amazon.lambda.durable.serde; /** Controls when serialized payloads are written to the filesystem. */ public enum FileSystemStorageMode { diff --git a/extra-filesystem-serdes/src/test/java/software/amazon/lambda/durable/extra/filesystem/FileSystemSerDesTest.java b/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java similarity index 97% rename from extra-filesystem-serdes/src/test/java/software/amazon/lambda/durable/extra/filesystem/FileSystemSerDesTest.java rename to sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java index 62abffca2..e824198cf 100644 --- a/extra-filesystem-serdes/src/test/java/software/amazon/lambda/durable/extra/filesystem/FileSystemSerDesTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java @@ -1,6 +1,6 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -package software.amazon.lambda.durable.extra.filesystem; +package software.amazon.lambda.durable.serde; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -20,11 +20,6 @@ import software.amazon.lambda.durable.exception.RetryableSerDesException; import software.amazon.lambda.durable.exception.SerDesException; import software.amazon.lambda.durable.model.OperationSubType; -import software.amazon.lambda.durable.serde.JacksonSerDes; -import software.amazon.lambda.durable.serde.SerDes; -import software.amazon.lambda.durable.serde.SerDesContext; -import software.amazon.lambda.durable.serde.SerDesPayloadKind; -import software.amazon.lambda.durable.serde.SerDesRunner; class FileSystemSerDesTest { private static final String ARN = From 6e544b5da4e6a0c71754791e7c34da0c99ae32e9 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Tue, 25 Aug 2026 01:02:45 +0000 Subject: [PATCH 10/56] fix: bound SerDes caching and input handling --- docs/adr/005-filesystem-serdes.md | 19 +++- docs/advanced/configuration.md | 6 +- docs/advanced/filesystem-serdes.md | 7 +- .../durable/testing/AsyncExecution.java | 5 +- .../testing/CloudDurableTestRunner.java | 16 ++- .../durable/testing/AsyncExecutionTest.java | 91 ++++++++++++++++ .../testing/CloudDurableTestRunnerTest.java | 101 +++++++++++++++--- .../lambda/durable/serde/SerDesRunner.java | 97 +++++++++++++++-- .../durable/serde/SerDesRunnerTest.java | 57 ++++++++++ 9 files changed, 358 insertions(+), 41 deletions(-) create mode 100644 sdk-testing/src/test/java/software/amazon/lambda/durable/testing/AsyncExecutionTest.java diff --git a/docs/adr/005-filesystem-serdes.md b/docs/adr/005-filesystem-serdes.md index c6352e352..99646fcfd 100644 --- a/docs/adr/005-filesystem-serdes.md +++ b/docs/adr/005-filesystem-serdes.md @@ -406,6 +406,7 @@ block the calling thread. Add an invocation-scoped cache for successful deserialization results. The cache key should include: +- The identity of the SerDes instance. - Durable execution ARN. - `entityId`. - Payload kind. @@ -413,7 +414,12 @@ Add an invocation-scoped cache for successful deserialization results. The cache - Target `TypeToken` type. - A hash of the serialized checkpoint string. -The serialized string hash prevents stale results when a `WAIT_FOR_CONDITION` or retried step updates the same operation payload across attempts. Cache entries live only for the current Lambda invocation and are discarded when `ExecutionManager` closes. +The SerDes identity prevents two configured pipelines from sharing a call-order-dependent result. The serialized string +hash prevents stale results when a `WAIT_FOR_CONDITION` or retried step updates the same operation payload across +attempts. Concurrent misses share one in-flight deserialization. Completed values use a bounded weak-reference cache +so a large replay does not retain every materialized object. Cache entries live only for the current Lambda invocation +and are discarded when `ExecutionManager` closes. Cloud test polling creates a fresh cache for each history snapshot +and retains that cache only with the corresponding `TestResult`. With this approach, SDK caching can avoid repeated calls to `FileSystemSerDes.deserialize`. If a cache miss occurs, `FileSystemSerDes` may perform a file read internally. @@ -437,9 +443,11 @@ Root user input and output payloads should route through `SerDesRunner` so `File The cloud test runner must send initial Lambda input before it receives a durable execution ARN. When configured with a context-free `ComposableSerDes`, it serializes the invocation payload with the complete configured pipeline so compression, encryption, and other ordinary transformations remain compatible with the deployed function. When the -persisted SerDes reports that it requires durable context, the runner requires a separate context-free input codec via -`CloudDurableTestRunner.withInputSerDes(...)`. Fluent configuration preserves that explicit input codec regardless of -whether `withInputSerDes(...)` or `withSerDes(...)` is called first. +persisted SerDes reports that it requires durable context, the runner requires a separate context-free input value +codec via `CloudDurableTestRunner.withInputSerDes(...)`. That codec must not be a composable string-processing pipeline: +an unframed external payload does not identify which input stages ran, while a context-dependent terminal stage must +also accept raw service payloads such as callbacks and invoke results. Fluent configuration preserves that explicit +input codec regardless of whether `withInputSerDes(...)` or `withSerDes(...)` is called first. ### Implementation plan @@ -455,7 +463,8 @@ whether `withInputSerDes(...)` or `withSerDes(...)` is called first. leaving `DurableInputOutputSerDes` internal. 6. Update `SerializableDurableOperation`, `InvokeOperation`, `StepOperation`, `WaitForConditionOperation`, `CallbackOperation`, `ChildContextOperation`, `MapOperation`, and test helpers to use `SerDesRunner`. -7. Add invocation-scoped deserialization caching keyed by entity, payload kind, type, and serialized data hash. +7. Add bounded, invocation-scoped deserialization caching keyed by SerDes identity, entity, payload kind, type, and + serialized data hash. 8. Update exception serialization and deserialization paths to set `SerDesPayloadKind.EXCEPTION` in TLS. 9. Implement `FileSystemSerDes` in the core `software.amazon.lambda.durable.serde` package with standalone compatibility and string-stage modes, `ALWAYS` and `OVERFLOW` storage, `URI` and `HASH` path encodings, envelope parsing, atomic file diff --git a/docs/advanced/configuration.md b/docs/advanced/configuration.md index 3ed13081d..3e9e8b6e8 100644 --- a/docs/advanced/configuration.md +++ b/docs/advanced/configuration.md @@ -49,7 +49,7 @@ SerDes executor must be different from the user-operation executor to avoid dead saturated. The SDK installs `SerDesContext` on whichever thread performs the call, restores any previous nested context afterward, -and caches successful deserialization results for the current invocation. +and uses a bounded weak-reference cache for successful deserialization results during the current invocation. ### Filesystem-backed payload storage @@ -92,7 +92,9 @@ storage lifecycle and retention separately. For a context-free `ComposableSerDes`, `CloudDurableTestRunner` applies the complete pipeline to the initial Lambda invocation. If the persisted SerDes requires durable context, such as `FileSystemSerDes`, call -`withInputSerDes(...)` with a separate context-free input codec because the durable execution ARN does not exist yet. +`withInputSerDes(...)` with a separate context-free input value codec because the durable execution ARN does not exist +yet. In that case, the input SerDes must not include composable string-processing stages because the external payload +does not carry framing that identifies which stages ran. `FileSystemSerDes` must also be the final pipeline stage so its checkpoint-size decision cannot be invalidated by a later expanding transformation. diff --git a/docs/advanced/filesystem-serdes.md b/docs/advanced/filesystem-serdes.md index 4e497bfcb..1a93afbd9 100644 --- a/docs/advanced/filesystem-serdes.md +++ b/docs/advanced/filesystem-serdes.md @@ -78,9 +78,10 @@ verified when reading, and symbolic-link paths are rejected. checkpoint representation; a later expanding transform cannot push an inline envelope over the service limit. `CloudDurableTestRunner` cannot use `FileSystemSerDes` for the initial Lambda invocation because an execution ARN is -not available yet. Configure a separate context-free initial-input codec with `withInputSerDes(...)`. An explicitly -supplied input SerDes is used exactly as configured, including every stage in a composable pipeline. Context-free -persisted pipelines use their complete pipeline for the initial invocation. +not available yet. Configure a separate context-free initial-input value codec with `withInputSerDes(...)`. Do not use +a composable string-processing pipeline for that input boundary: the unframed external payload does not identify which +stages ran before the context-dependent filesystem stage. Context-free persisted pipelines still use their complete +pipeline for the initial invocation. ## Storage requirements diff --git a/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/AsyncExecution.java b/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/AsyncExecution.java index a21187b50..50ac079e4 100644 --- a/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/AsyncExecution.java +++ b/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/AsyncExecution.java @@ -31,7 +31,6 @@ public class AsyncExecution { private final Duration pollInterval; private final Duration timeout; private final HistoryEventProcessor processor; - private final SerDesRunner serDesRunner; private List currentHistory; private TestResult currentResult; @@ -49,7 +48,6 @@ public AsyncExecution( this.timeout = timeout; this.serDes = serDes; this.processor = new HistoryEventProcessor(); - this.serDesRunner = new SerDesRunner(null); } /** @@ -198,8 +196,9 @@ private void refreshHistory() { .build(); var response = lambdaClient.getDurableExecutionHistory(request); this.currentHistory = response.events(); + var snapshotSerDesRunner = new SerDesRunner(null); this.currentResult = - processor.processEvents(currentHistory, outputType, serDes, serDesRunner, executionArn); + processor.processEvents(currentHistory, outputType, serDes, snapshotSerDesRunner, executionArn); } catch (ResourceNotFoundException e) { // Execution doesn't exist yet - this can happen immediately after async invoke // Leave currentHistory as null, pollUntil will retry diff --git a/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/CloudDurableTestRunner.java b/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/CloudDurableTestRunner.java index 667b2f5b6..821bbea32 100644 --- a/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/CloudDurableTestRunner.java +++ b/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/CloudDurableTestRunner.java @@ -10,6 +10,7 @@ import software.amazon.awssdk.services.lambda.model.InvocationType; import software.amazon.awssdk.services.lambda.model.InvokeRequest; import software.amazon.lambda.durable.TypeToken; +import software.amazon.lambda.durable.serde.ComposableSerDes; import software.amazon.lambda.durable.serde.JacksonSerDes; import software.amazon.lambda.durable.serde.SerDes; import software.amazon.lambda.durable.serde.SerDesRunner; @@ -173,7 +174,9 @@ public CloudDurableTestRunner withSerDes(SerDes serDes) { * *

The supplied SerDes is used exactly as configured, including every stage in a composable pipeline. Configure a * separate context-free input SerDes when the persisted SerDes requires a durable execution context, because that - * context does not exist before the initial Lambda invocation. + * context does not exist before the initial Lambda invocation. In that case the input SerDes must be a value codec, + * not a composable string-processing pipeline, because context-dependent persisted stages cannot distinguish which + * input stages produced an unframed external payload. */ public CloudDurableTestRunner withInputSerDes(SerDes inputSerDes) { return new CloudDurableTestRunner<>( @@ -278,10 +281,15 @@ public TestOperation getOperation(String name) { private String serializeInput(I input) { var serializer = inputSerDes != null ? inputSerDes : serDes; - if (inputSerDes == null && serializer.requiresDurableContext()) { + if (serializer.requiresDurableContext()) { throw new IllegalStateException( - "Configured persisted SerDes requires a durable execution context; configure a separate " - + "context-free input SerDes with withInputSerDes(...)"); + "Initial input SerDes requires a durable execution context; configure a context-free " + + "input SerDes with withInputSerDes(...)"); + } + if (inputSerDes instanceof ComposableSerDes && serDes.requiresDurableContext()) { + throw new IllegalStateException( + "Initial input for a context-dependent persisted SerDes must use a value codec, not a composable " + + "string-processing pipeline"); } return serializer.serialize(input); } diff --git a/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/AsyncExecutionTest.java b/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/AsyncExecutionTest.java new file mode 100644 index 000000000..fed669502 --- /dev/null +++ b/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/AsyncExecutionTest.java @@ -0,0 +1,91 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.testing; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.time.Duration; +import java.time.Instant; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; +import org.junit.jupiter.api.Test; +import software.amazon.awssdk.services.lambda.LambdaClient; +import software.amazon.awssdk.services.lambda.model.Event; +import software.amazon.awssdk.services.lambda.model.EventResult; +import software.amazon.awssdk.services.lambda.model.EventType; +import software.amazon.awssdk.services.lambda.model.GetDurableExecutionHistoryRequest; +import software.amazon.awssdk.services.lambda.model.GetDurableExecutionHistoryResponse; +import software.amazon.awssdk.services.lambda.model.RetryDetails; +import software.amazon.awssdk.services.lambda.model.StepStartedDetails; +import software.amazon.awssdk.services.lambda.model.StepSucceededDetails; +import software.amazon.lambda.durable.TypeToken; +import software.amazon.lambda.durable.serde.SerDes; + +class AsyncExecutionTest { + private static final String EXECUTION_ARN = "arn:aws:lambda:us-east-1:123456789012:function:test:$LATEST" + + "/durable-execution/execution-id/invocation-id"; + + @Test + void scopesDeserializationCacheToOneHistorySnapshot() { + var lambdaClient = mock(LambdaClient.class); + when(lambdaClient.getDurableExecutionHistory(any(GetDurableExecutionHistoryRequest.class))) + .thenReturn(GetDurableExecutionHistoryResponse.builder() + .events(stepEvents()) + .build()); + var deserializations = new AtomicInteger(); + var serDes = new SerDes() { + @Override + public String serialize(Object value) { + return value.toString(); + } + + @Override + @SuppressWarnings("unchecked") + public T deserialize(String data, TypeToken typeToken) { + deserializations.incrementAndGet(); + return (T) data; + } + }; + var execution = new AsyncExecution<>( + EXECUTION_ARN, lambdaClient, TypeToken.get(String.class), serDes, Duration.ZERO, Duration.ofSeconds(1)); + var snapshots = new AtomicInteger(); + + execution.pollUntil(current -> { + assertEquals("step-result", current.getOperation("step").getStepResult(String.class)); + assertEquals("step-result", current.getOperation("step").getStepResult(String.class)); + return snapshots.incrementAndGet() == 2; + }); + + assertEquals(2, deserializations.get()); + } + + private static List stepEvents() { + var startedAt = Instant.parse("2026-08-25T00:00:00Z"); + return List.of( + Event.builder() + .id("step-id") + .name("step") + .subType("Step") + .eventType(EventType.STEP_STARTED) + .eventTimestamp(startedAt) + .stepStartedDetails(StepStartedDetails.builder().build()) + .build(), + Event.builder() + .id("step-id") + .name("step") + .subType("Step") + .eventType(EventType.STEP_SUCCEEDED) + .eventTimestamp(startedAt.plusSeconds(1)) + .stepSucceededDetails(StepSucceededDetails.builder() + .result(EventResult.builder() + .payload("step-result") + .build()) + .retryDetails( + RetryDetails.builder().currentAttempt(1).build()) + .build()) + .build()); + } +} diff --git a/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/CloudDurableTestRunnerTest.java b/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/CloudDurableTestRunnerTest.java index 5e141645f..9aa79ce4c 100644 --- a/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/CloudDurableTestRunnerTest.java +++ b/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/CloudDurableTestRunnerTest.java @@ -5,16 +5,22 @@ import static org.junit.jupiter.api.Assertions.*; import static org.mockito.Mockito.*; +import java.nio.file.Path; import java.time.Duration; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; import org.mockito.ArgumentCaptor; import software.amazon.awssdk.services.lambda.LambdaClient; import software.amazon.awssdk.services.lambda.model.InvocationType; import software.amazon.awssdk.services.lambda.model.InvokeRequest; import software.amazon.awssdk.services.lambda.model.InvokeResponse; import software.amazon.lambda.durable.TypeToken; +import software.amazon.lambda.durable.serde.FileSystemSerDes; import software.amazon.lambda.durable.serde.JacksonSerDes; import software.amazon.lambda.durable.serde.SerDes; +import software.amazon.lambda.durable.serde.SerDesContext; +import software.amazon.lambda.durable.serde.SerDesPayloadKind; +import software.amazon.lambda.durable.serde.SerDesRunner; class CloudDurableTestRunnerTest { @@ -78,34 +84,75 @@ void persistedComposableSerDesUsesCompletePipelineForDefaultInput() { @Test void contextDependentPersistedSerDesRequiresExplicitInputSerDes() { var mockClient = mock(LambdaClient.class); - var contextStage = new SerDes() { - @Override - public String serialize(Object value) { - return value.toString(); - } + var runner = CloudDurableTestRunner.create( + "arn:aws:lambda:us-east-2:123:function:test", String.class, String.class, mockClient) + .withSerDes(new JacksonSerDes().then(contextDependentStage())); - @Override - @SuppressWarnings("unchecked") - public T deserialize(String data, TypeToken typeToken) { - return (T) data; - } + var failure = assertThrows(RuntimeException.class, () -> runner.startAsync("value")); - @Override - public boolean requiresDurableContext() { - return true; - } - }; + assertInstanceOf(IllegalStateException.class, failure.getCause()); + assertTrue(failure.getCause().getMessage().contains("withInputSerDes")); + verifyNoInteractions(mockClient); + } + + @Test + void explicitInputSerDesMustBeContextFree() { + var mockClient = mock(LambdaClient.class); var runner = CloudDurableTestRunner.create( "arn:aws:lambda:us-east-2:123:function:test", String.class, String.class, mockClient) - .withSerDes(new JacksonSerDes().then(contextStage)); + .withInputSerDes(contextDependentStage()); var failure = assertThrows(RuntimeException.class, () -> runner.startAsync("value")); assertInstanceOf(IllegalStateException.class, failure.getCause()); - assertTrue(failure.getCause().getMessage().contains("withInputSerDes")); + assertTrue(failure.getCause().getMessage().contains("Initial input SerDes")); verifyNoInteractions(mockClient); } + @Test + void contextDependentPersistedSerDesRequiresValueCodecInput() { + var mockClient = mock(LambdaClient.class); + var runner = CloudDurableTestRunner.create( + "arn:aws:lambda:us-east-2:123:function:test", String.class, String.class, mockClient) + .withSerDes(new JacksonSerDes().then(contextDependentStage())) + .withInputSerDes(new JacksonSerDes().then(wrappingStage())); + + var failure = assertThrows(RuntimeException.class, () -> runner.startAsync("value")); + + assertInstanceOf(IllegalStateException.class, failure.getCause()); + assertTrue(failure.getCause().getMessage().contains("must use a value codec")); + verifyNoInteractions(mockClient); + } + + @Test + void valueCodecInputRoundTripsThroughFileSystemPipeline(@TempDir Path basePath) { + var mockClient = mock(LambdaClient.class); + var executionArn = "arn:aws:lambda:us-east-2:123:function:test:1/durable-execution/e/i"; + when(mockClient.invoke(any(InvokeRequest.class))) + .thenReturn(InvokeResponse.builder() + .durableExecutionArn(executionArn) + .build()); + var persistedSerDes = + new JacksonSerDes().then(FileSystemSerDes.stageBuilder(basePath).build()); + var runner = CloudDurableTestRunner.create( + "arn:aws:lambda:us-east-2:123:function:test", String.class, String.class, mockClient) + .withSerDes(persistedSerDes) + .withInputSerDes(new JacksonSerDes()); + + runner.startAsync("value"); + + var request = ArgumentCaptor.forClass(InvokeRequest.class); + verify(mockClient).invoke(request.capture()); + assertEquals( + "value", + new SerDesRunner(null) + .deserialize( + persistedSerDes, + request.getValue().payload().asUtf8String(), + TypeToken.get(String.class), + SerDesContext.forExecution(executionArn, "i", "execution", SerDesPayloadKind.INPUT))); + } + @Test void replacingPersistedSerDesPreservesExplicitInputSerDes() { var mockClient = mock(LambdaClient.class); @@ -139,4 +186,24 @@ public T deserialize(String data, TypeToken typeToken) { } }; } + + private static SerDes contextDependentStage() { + return new SerDes() { + @Override + public String serialize(Object value) { + return value.toString(); + } + + @Override + @SuppressWarnings("unchecked") + public T deserialize(String data, TypeToken typeToken) { + return (T) data; + } + + @Override + public boolean requiresDurableContext() { + return true; + } + }; + } } diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesRunner.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesRunner.java index ecc24d457..1aa501b0e 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesRunner.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesRunner.java @@ -2,10 +2,13 @@ // SPDX-License-Identifier: Apache-2.0 package software.amazon.lambda.durable.serde; +import java.lang.ref.WeakReference; import java.nio.charset.StandardCharsets; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; +import java.util.Collections; import java.util.HexFormat; +import java.util.LinkedHashMap; import java.util.Map; import java.util.Objects; import java.util.concurrent.CompletableFuture; @@ -21,11 +24,23 @@ * Runs customer SerDes calls with the correct {@link SerDesContext}. * *

Calls execute inline unless an executor is configured. Instances are invocation-scoped so successful - * deserialization results are cached only for one Lambda invocation. + * deserialization results are cached only for one Lambda invocation. Completed values use a bounded weak-reference + * cache, while concurrent calls for the same value share one in-flight deserialization. */ public final class SerDesRunner { + static final int MAX_COMPLETED_DESERIALIZATIONS = 256; + private static final Object CACHE_MISS = new Object(); + private static final Object NULL_VALUE = new Object(); + private final ExecutorService executorService; - private final Map> deserializationCache = new ConcurrentHashMap<>(); + private final Map> inFlightDeserializations = new ConcurrentHashMap<>(); + private final Map> completedDeserializations = + Collections.synchronizedMap(new LinkedHashMap<>(16, 0.75f, true) { + @Override + protected boolean removeEldestEntry(Map.Entry> eldest) { + return size() > MAX_COMPLETED_DESERIALIZATIONS; + } + }); /** * Creates an invocation-scoped runner. @@ -49,30 +64,74 @@ public T deserialize(SerDes serDes, String data, TypeToken typeToken, Ser Objects.requireNonNull(typeToken, "typeToken cannot be null"); Objects.requireNonNull(context, "SerDesContext cannot be null"); var key = new CacheKey( + serDes, context.durableExecutionArn(), context.entityId(), context.payloadKind(), context.attempt(), typeToken, hash(data)); + var cached = getCompleted(key); + if (cached != CACHE_MISS) { + return (T) unmaskNull(cached); + } + var pending = new CompletableFuture(); - var existing = deserializationCache.putIfAbsent(key, pending); + var existing = inFlightDeserializations.putIfAbsent(key, pending); if (existing != null) { - return (T) join(existing); + return (T) unmaskNull(join(existing)); } try { + // A deserialization may have completed between the first cache lookup and this caller claiming + // the in-flight slot. + cached = getCompleted(key); + if (cached != CACHE_MISS) { + pending.complete(cached); + return (T) unmaskNull(cached); + } + T value = run("deserialize", context, () -> serDes.deserialize(data, typeToken)); - pending.complete(value); + var cacheValue = maskNull(value); + putCompleted(key, cacheValue); + pending.complete(cacheValue); return value; } catch (Throwable failure) { pending.completeExceptionally(failure); - deserializationCache.remove(key, pending); ExceptionHelper.sneakyThrow(failure); return null; + } finally { + inFlightDeserializations.remove(key, pending); + } + } + + private Object getCompleted(CacheKey key) { + synchronized (completedDeserializations) { + var reference = completedDeserializations.get(key); + if (reference == null) { + return CACHE_MISS; + } + var value = reference.get(); + if (value == null) { + completedDeserializations.remove(key); + return CACHE_MISS; + } + return value; } } + private void putCompleted(CacheKey key, Object value) { + completedDeserializations.put(key, new WeakReference<>(value)); + } + + private static Object maskNull(Object value) { + return value == null ? NULL_VALUE : value; + } + + private static Object unmaskNull(Object value) { + return value == NULL_VALUE ? null : value; + } + private T run(String action, SerDesContext context, Supplier supplier) { Objects.requireNonNull(supplier, "supplier cannot be null"); Objects.requireNonNull(context, "SerDesContext cannot be null"); @@ -129,10 +188,34 @@ private static String hash(String data) { } private record CacheKey( + SerDes serDes, String durableExecutionArn, String entityId, SerDesPayloadKind payloadKind, Integer attempt, TypeToken typeToken, - String serializedHash) {} + String serializedHash) { + @Override + public boolean equals(Object other) { + return other instanceof CacheKey that + && serDes == that.serDes + && Objects.equals(durableExecutionArn, that.durableExecutionArn) + && Objects.equals(entityId, that.entityId) + && payloadKind == that.payloadKind + && Objects.equals(attempt, that.attempt) + && Objects.equals(typeToken, that.typeToken) + && Objects.equals(serializedHash, that.serializedHash); + } + + @Override + public int hashCode() { + int result = System.identityHashCode(serDes); + result = 31 * result + Objects.hashCode(durableExecutionArn); + result = 31 * result + Objects.hashCode(entityId); + result = 31 * result + Objects.hashCode(payloadKind); + result = 31 * result + Objects.hashCode(attempt); + result = 31 * result + Objects.hashCode(typeToken); + return 31 * result + Objects.hashCode(serializedHash); + } + } } diff --git a/sdk/src/test/java/software/amazon/lambda/durable/serde/SerDesRunnerTest.java b/sdk/src/test/java/software/amazon/lambda/durable/serde/SerDesRunnerTest.java index a14265f3a..0bf210003 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/serde/SerDesRunnerTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/serde/SerDesRunnerTest.java @@ -147,6 +147,48 @@ public T deserialize(String data, TypeToken typeToken) { assertEquals(3, count.get()); } + @Test + void cacheKeyIncludesSerDesIdentity() { + var runner = new SerDesRunner(null); + var context = context("operation/1/result"); + var first = fixedValueSerDes("first"); + var second = fixedValueSerDes("second"); + + assertEquals("first", runner.deserialize(first, "\"value\"", TypeToken.get(String.class), context)); + assertEquals("second", runner.deserialize(second, "\"value\"", TypeToken.get(String.class), context)); + } + + @Test + void completedCacheEvictsOldestEntries() { + var calls = new AtomicInteger(); + var serDes = new SerDes() { + @Override + public String serialize(Object value) { + return value.toString(); + } + + @Override + @SuppressWarnings("unchecked") + public T deserialize(String data, TypeToken typeToken) { + calls.incrementAndGet(); + return (T) data; + } + }; + var runner = new SerDesRunner(null); + var retainedValues = new ArrayList(); + + for (int index = 0; index <= SerDesRunner.MAX_COMPLETED_DESERIALIZATIONS; index++) { + retainedValues.add(runner.deserialize( + serDes, "value-" + index, TypeToken.get(String.class), context("operation/" + index + "/result"))); + } + assertEquals(SerDesRunner.MAX_COMPLETED_DESERIALIZATIONS + 1, calls.get()); + + assertEquals( + retainedValues.get(0), + runner.deserialize(serDes, "value-0", TypeToken.get(String.class), context("operation/0/result"))); + assertEquals(SerDesRunner.MAX_COMPLETED_DESERIALIZATIONS + 2, calls.get()); + } + @Test void concurrentCacheMissesDeserializeOnlyOnce() throws Exception { var entered = new CountDownLatch(1); @@ -261,6 +303,21 @@ public T deserialize(String data, TypeToken typeToken) { assertInstanceOf(RetryableSerDesException.class, exception.getCause()); } + private static SerDes fixedValueSerDes(String value) { + return new SerDes() { + @Override + public String serialize(Object input) { + return input.toString(); + } + + @Override + @SuppressWarnings("unchecked") + public T deserialize(String data, TypeToken typeToken) { + return (T) value; + } + }; + } + private static SerDesContext context(String entityId) { return new SerDesContext( "arn:test", From c61834db42032a16998b1b4976171bb64ac62763 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Tue, 25 Aug 2026 01:22:26 +0000 Subject: [PATCH 11/56] fix: preserve filesystem SerDes error ownership --- RELEASE.md | 9 +- .../FileSystemSerDesIntegrationTest.java | 102 ++++++++++++++++++ .../operation/ChildContextOperation.java | 2 +- .../SerializableDurableOperation.java | 48 ++++++++- .../durable/serde/FileSystemSerDes.java | 57 +++++++--- .../durable/serde/FileSystemSerDesTest.java | 19 ++++ 6 files changed, 216 insertions(+), 21 deletions(-) diff --git a/RELEASE.md b/RELEASE.md index 92dbae9e2..a91a8f130 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -43,9 +43,9 @@ The publication workflow: 1. Verifies that the tag is a semantic version, points to a commit on the default branch, and matches the Maven version in the tagged POM. -2. Builds, signs, and uploads the SDK, filesystem SerDes extension, testing - library, and OpenTelemetry plugin to Sonatype Central Portal. -3. Uploads the four JARs to the existing GitHub release. +2. Builds, signs, and uploads the SDK, testing library, and OpenTelemetry + plugin to Sonatype Central Portal. +3. Uploads the three JARs to the existing GitHub release. 4. Opens a pull request for the next development version. A final release increments the patch version, so `2.1.1` produces `2.1.2-SNAPSHOT`. A prerelease keeps the same base version, so `2.1.1-rc1` produces @@ -56,8 +56,7 @@ After **Publish Maven Release** succeeds: 1. Open [Publishing Deployments](https://central.sonatype.com/publishing/deployments) in Sonatype Central Portal. 2. Find the deployments for the release version and verify that they contain - the expected SDK, filesystem SerDes extension, testing library, and - OpenTelemetry plugin artifacts. + the expected SDK, testing library, and OpenTelemetry plugin artifacts. 3. Click **Publish** for each deployment and wait for publication to complete. The workflow uses `autoPublish=false`, so this manual action is required. 4. Confirm that the GitHub release contains the expected JARs and that the diff --git a/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FileSystemSerDesIntegrationTest.java b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FileSystemSerDesIntegrationTest.java index 79233c1a0..3520840ff 100644 --- a/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FileSystemSerDesIntegrationTest.java +++ b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FileSystemSerDesIntegrationTest.java @@ -21,6 +21,7 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; import software.amazon.awssdk.services.lambda.model.CheckpointUpdatedExecutionState; +import software.amazon.awssdk.services.lambda.model.ErrorObject; import software.amazon.awssdk.services.lambda.model.ExecutionDetails; import software.amazon.awssdk.services.lambda.model.Operation; import software.amazon.awssdk.services.lambda.model.OperationAction; @@ -42,6 +43,7 @@ import software.amazon.lambda.durable.serde.SerDesPayloadKind; import software.amazon.lambda.durable.serde.SerDesRunner; import software.amazon.lambda.durable.testing.LocalDurableTestRunner; +import software.amazon.lambda.durable.testing.TestResult; import software.amazon.lambda.durable.testing.local.LocalMemoryExecutionClient; import software.amazon.lambda.durable.testing.local.OperationResult; @@ -366,6 +368,97 @@ void customExceptionPayloadsRoundTripThroughFilesystem() throws Exception { assertEnvelopePointsToFile(operationError.errorData()); } + @Test + void nestedInvokeFailurePreservesProducerContextAcrossReplay() throws Exception { + var childExecutions = new AtomicInteger(); + var serDes = filesystemPipeline(); + var config = DurableConfig.builder().withSerDes(serDes).build(); + var runner = LocalDurableTestRunner.create( + String.class, + (input, context) -> { + try { + return context.runInChildContext("invoke-child", String.class, child -> { + childExecutions.incrementAndGet(); + return child.invoke("nested-invoke", "callee", input, String.class); + }); + } catch (CustomFailure failure) { + return "caught:" + failure.getMessage(); + } + }, + config) + .withOutputType(String.class); + + assertEquals(ExecutionStatus.PENDING, runner.run("input").getStatus()); + var calleeArn = + "arn:aws:lambda:us-east-1:123456789012:function:callee:1/durable-execution/callee-execution/callee-invocation"; + var errorData = new SerDesRunner(null) + .serialize( + serDes, + new CustomFailure("invoke-boom"), + SerDesContext.forExecution( + calleeArn, "callee-invocation", "callee-execution", SerDesPayloadKind.EXCEPTION)); + runner.failChainedInvoke( + "nested-invoke", + ErrorObject.builder() + .errorType(CustomFailure.class.getName()) + .errorMessage("invoke-boom") + .errorData(errorData) + .build()); + + var completed = runner.runUntilComplete("input"); + assertEquals(ExecutionStatus.SUCCEEDED, completed.getStatus()); + assertEquals("caught:invoke-boom", completed.getResult()); + assertForwardedErrorOwnedByChild(completed, "invoke-child"); + var executionsAfterCompletion = childExecutions.get(); + + var replay = runner.run("input"); + assertEquals(ExecutionStatus.SUCCEEDED, replay.getStatus()); + assertEquals("caught:invoke-boom", replay.getResult()); + assertEquals(executionsAfterCompletion, childExecutions.get()); + } + + @Test + void nestedCallbackFailurePreservesProducerContextAcrossReplay() throws Exception { + var childExecutions = new AtomicInteger(); + var serDes = filesystemPipeline(); + var config = DurableConfig.builder().withSerDes(serDes).build(); + var runner = LocalDurableTestRunner.create( + String.class, + (input, context) -> { + try { + return context.runInChildContext("callback-child", String.class, child -> { + childExecutions.incrementAndGet(); + return child.createCallback("nested-callback", String.class) + .get(); + }); + } catch (CustomFailure failure) { + return "caught:" + failure.getMessage(); + } + }, + config) + .withOutputType(String.class); + + assertEquals(ExecutionStatus.PENDING, runner.run("input").getStatus()); + runner.failCallback( + runner.getCallbackId("nested-callback"), + ErrorObject.builder() + .errorType(CustomFailure.class.getName()) + .errorMessage("callback-boom") + .errorData(new JacksonSerDes().serialize(new CustomFailure("callback-boom"))) + .build()); + + var completed = runner.runUntilComplete("input"); + assertEquals(ExecutionStatus.SUCCEEDED, completed.getStatus()); + assertEquals("caught:callback-boom", completed.getResult()); + assertForwardedErrorOwnedByChild(completed, "callback-child"); + var executionsAfterCompletion = childExecutions.get(); + + var replay = runner.run("input"); + assertEquals(ExecutionStatus.SUCCEEDED, replay.getStatus()); + assertEquals("caught:callback-boom", replay.getResult()); + assertEquals(executionsAfterCompletion, childExecutions.get()); + } + private SerDes filesystemPipeline() { return new JacksonSerDes().then(FileSystemSerDes.stageBuilder(basePath).build()); } @@ -420,6 +513,15 @@ private void assertEnvelopePointsToFile(String envelope) throws Exception { assertTrue(file.startsWith(basePath)); } + private void assertForwardedErrorOwnedByChild(TestResult result, String childName) throws Exception { + var child = result.getOperation(childName); + var errorData = child.getContextDetails().error().errorData(); + assertEnvelopePointsToFile(errorData); + assertEquals( + "operation/" + child.getId() + "/exception", + MAPPER.readTree(errorData).get("ownerEntityId").textValue()); + } + @FunctionalInterface private interface RecordingFunction { void record(String action, String value); diff --git a/sdk/src/main/java/software/amazon/lambda/durable/operation/ChildContextOperation.java b/sdk/src/main/java/software/amazon/lambda/durable/operation/ChildContextOperation.java index 8c299cfa4..d8d6b91db 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/operation/ChildContextOperation.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/operation/ChildContextOperation.java @@ -199,7 +199,7 @@ private void handleChildContextFailure(Throwable exception) { final ErrorObject errorObject; if (exception instanceof DurableOperationException opEx) { - errorObject = opEx.getErrorObject(); + errorObject = rebindForwardedException(opEx); } else { errorObject = serializeException(exception); } diff --git a/sdk/src/main/java/software/amazon/lambda/durable/operation/SerializableDurableOperation.java b/sdk/src/main/java/software/amazon/lambda/durable/operation/SerializableDurableOperation.java index 0d4b849a5..9c4f2b9cc 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/operation/SerializableDurableOperation.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/operation/SerializableDurableOperation.java @@ -5,12 +5,16 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import software.amazon.awssdk.services.lambda.model.ErrorObject; +import software.amazon.awssdk.services.lambda.model.Operation; import software.amazon.lambda.durable.DurableFuture; import software.amazon.lambda.durable.TypeToken; import software.amazon.lambda.durable.context.DurableContextImpl; +import software.amazon.lambda.durable.exception.DurableOperationException; import software.amazon.lambda.durable.exception.SerDesException; import software.amazon.lambda.durable.model.OperationIdentifier; +import software.amazon.lambda.durable.model.OperationSubType; import software.amazon.lambda.durable.serde.SerDes; +import software.amazon.lambda.durable.serde.SerDesContext; import software.amazon.lambda.durable.serde.SerDesPayloadKind; import software.amazon.lambda.durable.util.ExceptionHelper; @@ -153,6 +157,22 @@ protected ErrorObject serializeException(Throwable throwable, Integer attempt) { return error; } + /** + * Re-serializes an exception forwarded from another durable operation under this operation's context. + * + *

Context-dependent SerDes implementations may store the source error data under the producing operation or + * invoked execution. Rebinding reconstructable exceptions prevents a parent checkpoint from later trying to read + * that data using the parent's unrelated entity identity. + */ + protected ErrorObject rebindForwardedException(DurableOperationException exception) { + var error = exception.getErrorObject(); + if (error == null || exception.getOperation() == null) { + return error; + } + var original = deserializeExceptionWithContext(error, producerExceptionContext(exception.getOperation())); + return original != null ? serializeException(original) : error; + } + private boolean shouldDeserializeAfterSerialization() { var config = getContext().getDurableConfig(); return config == null || config.shouldDeserializeAfterSerialization(); @@ -171,6 +191,10 @@ protected Throwable deserializeException(ErrorObject errorObject) { /** Deserializes a throwable with attempt metadata. */ protected Throwable deserializeException(ErrorObject errorObject, Integer attempt) { + return deserializeExceptionWithContext(errorObject, createSerDesContext(SerDesPayloadKind.EXCEPTION, attempt)); + } + + private Throwable deserializeExceptionWithContext(ErrorObject errorObject, SerDesContext context) { Throwable original = null; if (errorObject == null) { return original; @@ -190,7 +214,7 @@ protected Throwable deserializeException(ErrorObject errorObject, Integer attemp resultSerDes, errorData, TypeToken.get(exceptionClass.asSubclass(Throwable.class)), - createSerDesContext(SerDesPayloadKind.EXCEPTION, attempt)); + context); if (original != null) { original.setStackTrace(ExceptionHelper.deserializeStackTrace(errorObject.stackTrace())); @@ -204,5 +228,27 @@ protected Throwable deserializeException(ErrorObject errorObject, Integer attemp return original; } + private SerDesContext producerExceptionContext(Operation operation) { + var attempt = operation.stepDetails() != null ? operation.stepDetails().attempt() : null; + return SerDesContext.forOperation( + getContext().getExecutionManager().getDurableExecutionArn(), + operation.id(), + operation.name(), + operation.parentId(), + operation.type(), + operationSubType(operation), + SerDesPayloadKind.EXCEPTION, + attempt); + } + + private static OperationSubType operationSubType(Operation operation) { + for (var subType : OperationSubType.values()) { + if (subType.getValue().equals(operation.subType())) { + return subType; + } + } + return null; + } + public abstract T get(); } diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java index 6c9411f08..a87c4faf4 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java @@ -11,6 +11,7 @@ import java.nio.file.FileAlreadyExistsException; import java.nio.file.Files; import java.nio.file.LinkOption; +import java.nio.file.NoSuchFileException; import java.nio.file.Path; import java.nio.file.StandardCopyOption; import java.security.MessageDigest; @@ -49,6 +50,7 @@ public final class FileSystemSerDes implements SerDes { private final SerDes delegate; private final Function> previewGenerator; private final boolean stageMode; + private volatile Path canonicalBasePath; private FileSystemSerDes(Builder builder) { basePath = builder.basePath.toAbsolutePath().normalize(); @@ -203,8 +205,8 @@ private String readPayload(String fileValue, PayloadOwner owner, SerDesContext c var file = Path.of(fileValue).toAbsolutePath().normalize(); validatePayloadPath(file, owner); try { + var realBasePath = validateBasePath(false); rejectSymbolicLinks(file); - var realBasePath = basePath.toRealPath(); var realDirectory = file.getParent().toRealPath(); var realFile = file.toRealPath(); if (!realDirectory.startsWith(realBasePath) @@ -267,6 +269,7 @@ private static boolean acceptsCrossExecutionReference(SerDesContext context) { } private void rejectSymbolicLinks(Path file) throws IOException { + validateBasePath(false); var current = basePath; for (var component : basePath.relativize(file)) { current = current.resolve(component); @@ -361,9 +364,8 @@ private String payloadFileName(String serialized, String entityId) { private void writePayload(String serialized, Path file) throws IOException { var directory = file.getParent(); - createDirectoriesWithoutSymbolicLinks(directory); + var realBasePath = createDirectoriesWithoutSymbolicLinks(directory); rejectSymbolicLinks(file); - var realBasePath = basePath.toRealPath(); var realDirectory = directory.toRealPath(); if (!realDirectory.startsWith(realBasePath)) { throw new SerDesException("Filesystem SerDes directory resolves outside the configured base path"); @@ -385,25 +387,52 @@ private void writePayload(String serialized, Path file) throws IOException { } } - private void createDirectoriesWithoutSymbolicLinks(Path directory) throws IOException { - Files.createDirectories(basePath); + private Path createDirectoriesWithoutSymbolicLinks(Path directory) throws IOException { + var realBasePath = validateBasePath(true); var current = basePath; for (var component : basePath.relativize(directory)) { current = current.resolve(component); - if (Files.exists(current, LinkOption.NOFOLLOW_LINKS)) { - if (Files.isSymbolicLink(current) || !Files.isDirectory(current, LinkOption.NOFOLLOW_LINKS)) { - throw new SerDesException("Filesystem SerDes directory path must contain only real directories"); - } - continue; + ensureRealDirectory(current, true); + } + return realBasePath; + } + + private Path validateBasePath(boolean createMissing) throws IOException { + var current = basePath.getRoot(); + if (current == null) { + throw new SerDesException("Filesystem SerDes base path must be absolute"); + } + ensureRealDirectory(current, false); + for (var component : basePath) { + current = current.resolve(component); + ensureRealDirectory(current, createMissing); + } + return retainCanonicalBasePath(basePath.toRealPath()); + } + + private static void ensureRealDirectory(Path directory, boolean createMissing) throws IOException { + if (!Files.exists(directory, LinkOption.NOFOLLOW_LINKS)) { + if (!createMissing) { + throw new NoSuchFileException(directory.toString()); } try { - Files.createDirectory(current); + Files.createDirectory(directory); } catch (FileAlreadyExistsException ignored) { - if (Files.isSymbolicLink(current) || !Files.isDirectory(current, LinkOption.NOFOLLOW_LINKS)) { - throw new SerDesException("Filesystem SerDes directory path must contain only real directories"); - } + // Validate the entry created by another writer below. } } + if (Files.isSymbolicLink(directory) || !Files.isDirectory(directory, LinkOption.NOFOLLOW_LINKS)) { + throw new SerDesException("Filesystem SerDes directory path must contain only real directories"); + } + } + + private synchronized Path retainCanonicalBasePath(Path currentBasePath) { + if (canonicalBasePath == null) { + canonicalBasePath = currentBasePath; + } else if (!canonicalBasePath.equals(currentBasePath)) { + throw new SerDesException("Filesystem SerDes base path changed after validation"); + } + return canonicalBasePath; } private static void moveWithoutReplacement(Path temporary, Path file) throws IOException { diff --git a/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java b/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java index e824198cf..5e0bb6ebf 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java @@ -283,6 +283,25 @@ void rejectsSymbolicLinkDirectoriesWhenWriting() throws Exception { } } + @Test + void rejectsSymbolicLinkConfiguredBasePathAndAncestors() throws Exception { + var outsideRoot = Files.createTempDirectory(basePath.getParent(), "outside-root-"); + var linkedRoot = basePath.resolve("linked-root"); + Files.createSymbolicLink(linkedRoot, outsideRoot); + + var rootSerDes = FileSystemSerDes.stageBuilder(linkedRoot).build(); + assertThrows(SerDesException.class, () -> new SerDesRunner(null).serialize(rootSerDes, "payload", context())); + + var outsideAncestor = Files.createTempDirectory(basePath.getParent(), "outside-ancestor-"); + var linkedAncestor = basePath.resolve("linked-ancestor"); + Files.createSymbolicLink(linkedAncestor, outsideAncestor); + var nestedSerDes = FileSystemSerDes.stageBuilder(linkedAncestor.resolve("payloads")) + .build(); + + assertThrows(SerDesException.class, () -> new SerDesRunner(null).serialize(nestedSerDes, "payload", context())); + assertFalse(Files.exists(outsideAncestor.resolve("payloads"))); + } + @Test void rejectsExecutionPathsOutsideConfiguredBasePath() { var serDes = FileSystemSerDes.builder(basePath).build(); From eea1bf147570e29566a5bc25e49ec13b851fa090 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Tue, 25 Aug 2026 01:33:44 +0000 Subject: [PATCH 12/56] fix: replay checkpointed execution outputs --- .../testing/LocalDurableTestRunner.java | 4 +++- .../testing/LocalDurableTestRunnerTest.java | 24 +++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/LocalDurableTestRunner.java b/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/LocalDurableTestRunner.java index 3eb5d11f4..d4a50ae67 100644 --- a/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/LocalDurableTestRunner.java +++ b/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/LocalDurableTestRunner.java @@ -366,7 +366,9 @@ private DurableExecutionInput createDurableInput(I input, SerDesRunner serDesRun .build(); // Load previous operations and include them in InitialExecutionState - var existingOps = storage.getAllOperations(); + var existingOps = storage.getAllOperations().stream() + .filter(op -> op.type() != OperationType.EXECUTION) + .toList(); var allOps = new ArrayList<>(List.of(executionOp)); allOps.addAll(existingOps); diff --git a/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/LocalDurableTestRunnerTest.java b/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/LocalDurableTestRunnerTest.java index 36f1bbced..8dc146f86 100644 --- a/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/LocalDurableTestRunnerTest.java +++ b/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/LocalDurableTestRunnerTest.java @@ -10,6 +10,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; import org.junit.jupiter.api.Test; import software.amazon.lambda.durable.DurableConfig; import software.amazon.lambda.durable.TypeToken; @@ -114,4 +115,27 @@ public void onInvocationStart(InvocationInfo info) { assertNotNull(executionStartTimes.get(0)); assertEquals(executionStartTimes.get(0), executionStartTimes.get(1)); } + + @Test + void checkpointedLargeOutputReplaysWithoutDuplicateExecutionOperation() { + var stepExecutions = new AtomicInteger(); + var largeResult = "x".repeat(7 * 1024 * 1024); + var runner = LocalDurableTestRunner.create(String.class, (input, context) -> { + context.step("once", Void.class, step -> { + stepExecutions.incrementAndGet(); + return null; + }); + return largeResult; + }) + .withOutputType(String.class); + + var firstResult = runner.run("test"); + var replayResult = runner.run("test"); + + assertEquals(ExecutionStatus.SUCCEEDED, firstResult.getStatus()); + assertEquals(largeResult, firstResult.getResult()); + assertEquals(ExecutionStatus.SUCCEEDED, replayResult.getStatus()); + assertEquals(largeResult, replayResult.getResult()); + assertEquals(1, stepExecutions.get()); + } } From c56b606b3a7b69893e77a58c3fe96287ebdd2329 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Tue, 25 Aug 2026 01:44:15 +0000 Subject: [PATCH 13/56] Preserve retryable SerDes replay failures --- .../SerializableDurableOperation.java | 3 + .../durable/serde/FileSystemSerDes.java | 55 ++++++++++------- .../SerializableDurableOperationTest.java | 59 +++++++++++++++++++ .../durable/serde/FileSystemSerDesTest.java | 10 +++- 4 files changed, 104 insertions(+), 23 deletions(-) diff --git a/sdk/src/main/java/software/amazon/lambda/durable/operation/SerializableDurableOperation.java b/sdk/src/main/java/software/amazon/lambda/durable/operation/SerializableDurableOperation.java index 9c4f2b9cc..d4eafb3c1 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/operation/SerializableDurableOperation.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/operation/SerializableDurableOperation.java @@ -10,6 +10,7 @@ import software.amazon.lambda.durable.TypeToken; import software.amazon.lambda.durable.context.DurableContextImpl; import software.amazon.lambda.durable.exception.DurableOperationException; +import software.amazon.lambda.durable.exception.RetryableSerDesException; import software.amazon.lambda.durable.exception.SerDesException; import software.amazon.lambda.durable.model.OperationIdentifier; import software.amazon.lambda.durable.model.OperationSubType; @@ -222,6 +223,8 @@ private Throwable deserializeExceptionWithContext(ErrorObject errorObject, SerDe } } catch (ClassNotFoundException e) { logger.warn("Cannot re-construct original exception type. Falling back to generic StepFailedException."); + } catch (RetryableSerDesException e) { + throw e; } catch (SerDesException e) { logger.warn("Cannot deserialize original exception data. Falling back to generic StepFailedException.", e); } diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java index a87c4faf4..78dd99d42 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java @@ -178,11 +178,7 @@ private ResolvedPayload resolveSerializedPayload(String data, SerDesContext cont throw malformedEnvelope(context, e); } - if (envelope != null && envelope.isObject() && envelope.has(ENVELOPE_MARKER)) { - if (!isFilesystemEnvelope(envelope)) { - throw malformedEnvelope(context, null); - } - } else { + if (!isFilesystemEnvelope(envelope)) { if (acceptsExternalPayload(context)) { return new ResolvedPayload(data, true); } @@ -191,13 +187,10 @@ private ResolvedPayload resolveSerializedPayload(String data, SerDesContext cont var hasData = envelope.has("data") && envelope.get("data").isTextual(); var hasFile = envelope.has("file") && envelope.get("file").isTextual(); - if (hasData == hasFile) { - throw malformedEnvelope(context, null); - } + var owner = payloadOwner(envelope, context); if (hasData) { return new ResolvedPayload(envelope.get("data").textValue(), false); } - var owner = payloadOwner(envelope, context); return new ResolvedPayload(readPayload(envelope.get("file").textValue(), owner, context), false); } @@ -242,15 +235,13 @@ private static PayloadOwner payloadOwner(JsonNode envelope, SerDesContext contex && envelope.get("ownerDurableExecutionArn").isTextual(); var hasOwnerEntity = envelope.has("ownerEntityId") && envelope.get("ownerEntityId").isTextual(); - if (hasOwnerArn != hasOwnerEntity) { + if (!hasOwnerArn || !hasOwnerEntity) { throw malformedEnvelope(context, null); } - var owner = hasOwnerArn - ? new PayloadOwner( - envelope.get("ownerDurableExecutionArn").textValue(), - envelope.get("ownerEntityId").textValue()) - : new PayloadOwner(context.durableExecutionArn(), context.entityId()); + var owner = new PayloadOwner( + envelope.get("ownerDurableExecutionArn").textValue(), + envelope.get("ownerEntityId").textValue()); if (owner.durableExecutionArn().isBlank() || owner.entityId().isBlank()) { throw malformedEnvelope(context, null); } @@ -280,11 +271,31 @@ private void rejectSymbolicLinks(Path file) throws IOException { } private static boolean isFilesystemEnvelope(JsonNode envelope) { - return envelope != null - && envelope.isObject() - && envelope.has(ENVELOPE_MARKER) - && envelope.get(ENVELOPE_MARKER).isIntegralNumber() - && envelope.get(ENVELOPE_MARKER).intValue() == ENVELOPE_VERSION; + if (envelope == null + || !envelope.isObject() + || !envelope.has(ENVELOPE_MARKER) + || !envelope.get(ENVELOPE_MARKER).isIntegralNumber() + || envelope.get(ENVELOPE_MARKER).intValue() != ENVELOPE_VERSION + || !envelope.has("ownerDurableExecutionArn") + || !envelope.get("ownerDurableExecutionArn").isTextual() + || envelope.get("ownerDurableExecutionArn").textValue().isBlank() + || !envelope.has("ownerEntityId") + || !envelope.get("ownerEntityId").isTextual() + || envelope.get("ownerEntityId").textValue().isBlank()) { + return false; + } + + var hasData = envelope.has("data") && envelope.get("data").isTextual(); + var hasFile = envelope.has("file") && envelope.get("file").isTextual(); + if (hasData == hasFile) { + return false; + } + + var hasPreview = envelope.has("preview"); + if (hasPreview && (hasData || !envelope.get("preview").isObject())) { + return false; + } + return envelope.size() == (hasPreview ? 5 : 4); } private static boolean acceptsExternalPayload(SerDesContext context) { @@ -301,12 +312,12 @@ private static SerDesException malformedEnvelope(SerDesContext context, Throwabl private String encodeEnvelope(String data, Path file, Map preview, SerDesContext context) { var envelope = new LinkedHashMap(); envelope.put(ENVELOPE_MARKER, ENVELOPE_VERSION); + envelope.put("ownerDurableExecutionArn", context.durableExecutionArn()); + envelope.put("ownerEntityId", context.entityId()); if (data != null) { envelope.put("data", data); } else { envelope.put("file", file.toString()); - envelope.put("ownerDurableExecutionArn", context.durableExecutionArn()); - envelope.put("ownerEntityId", context.entityId()); if (preview != null) { envelope.put("preview", preview); } diff --git a/sdk/src/test/java/software/amazon/lambda/durable/operation/SerializableDurableOperationTest.java b/sdk/src/test/java/software/amazon/lambda/durable/operation/SerializableDurableOperationTest.java index 713ee05c0..95edc7282 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/operation/SerializableDurableOperationTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/operation/SerializableDurableOperationTest.java @@ -15,14 +15,19 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; import software.amazon.awssdk.services.lambda.model.ErrorObject; import software.amazon.awssdk.services.lambda.model.Operation; import software.amazon.awssdk.services.lambda.model.OperationStatus; @@ -34,12 +39,14 @@ import software.amazon.lambda.durable.context.DurableContextImpl; import software.amazon.lambda.durable.exception.IllegalDurableOperationException; import software.amazon.lambda.durable.exception.NonDeterministicExecutionException; +import software.amazon.lambda.durable.exception.RetryableSerDesException; import software.amazon.lambda.durable.exception.SerDesException; import software.amazon.lambda.durable.execution.ExecutionManager; import software.amazon.lambda.durable.execution.ThreadContext; import software.amazon.lambda.durable.execution.ThreadType; import software.amazon.lambda.durable.model.OperationIdentifier; import software.amazon.lambda.durable.model.OperationSubType; +import software.amazon.lambda.durable.serde.FileSystemSerDes; import software.amazon.lambda.durable.serde.JacksonSerDes; import software.amazon.lambda.durable.serde.SerDes; @@ -94,8 +101,12 @@ public T deserialize(String data, TypeToken typeToken) { private static final TypeToken RESULT_TYPE = TypeToken.get(String.class); private static final SerDes SER_DES = new JacksonSerDes(); private static final String RESULT = "name"; + private static final ObjectMapper MAPPER = new ObjectMapper(); private final ExecutorService internalExecutor = Executors.newFixedThreadPool(2); + @TempDir + Path basePath; + private ExecutionManager executionManager; private DurableContextImpl durableContext; @@ -503,6 +514,54 @@ public String get() { op.get(); } + @Test + void deserializeExceptionPreservesRetryableStorageFailure() { + when(executionManager.getDurableExecutionArn()) + .thenReturn( + "arn:aws:lambda:us-east-1:123456789012:function:test:1/durable-execution/execution/invocation"); + var serDes = FileSystemSerDes.builder(basePath).build(); + var checkpointedError = new AtomicReference(); + SerializableDurableOperation producer = + new SerializableDurableOperation<>(OPERATION_IDENTIFIER, RESULT_TYPE, serDes, durableContext) { + @Override + protected void start() {} + + @Override + protected void replay(Operation existing) {} + + @Override + public String get() { + checkpointedError.set(serializeException(new RuntimeException("test exception"), 1)); + return RESULT; + } + }; + producer.get(); + try { + Files.delete(Path.of(MAPPER.readTree(checkpointedError.get().errorData()) + .get("file") + .textValue())); + } catch (Exception e) { + throw new AssertionError(e); + } + + SerializableDurableOperation replay = + new SerializableDurableOperation<>(OPERATION_IDENTIFIER, RESULT_TYPE, serDes, durableContext) { + @Override + protected void start() {} + + @Override + protected void replay(Operation existing) {} + + @Override + public String get() { + assertThrows( + RetryableSerDesException.class, () -> deserializeException(checkpointedError.get(), 1)); + return RESULT; + } + }; + replay.get(); + } + @Test void serializeExceptionValidatesRoundTrip() { var serDes = new TrackingSerDes(); diff --git a/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java b/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java index 5e0bb6ebf..7d047593f 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java @@ -165,6 +165,13 @@ void acceptsExternallyOriginatedRawPayloadsOnlyForSupportedSources() { "\"invoke-result\"", TypeToken.get(String.class), operationContext(OperationType.CHAINED_INVOKE, OperationSubType.CHAINED_INVOKE))); + assertEquals( + Map.of(ENVELOPE_MARKER, 1, "data", "domain-value"), + runner.deserialize( + standalone, + "{\"__durable_execution_filesystem_serdes\":1,\"data\":\"domain-value\"}", + new TypeToken>() {}, + executionContext(SerDesPayloadKind.INPUT))); assertThrows( SerDesException.class, @@ -320,7 +327,8 @@ void rejectsExecutionPathsOutsideConfiguredBasePath() { private static String envelopeWithFile(String file) { try { - return MAPPER.writeValueAsString(Map.of(ENVELOPE_MARKER, 1, "file", file)); + return MAPPER.writeValueAsString( + Map.of(ENVELOPE_MARKER, 1, "file", file, "ownerDurableExecutionArn", ARN, "ownerEntityId", "1")); } catch (Exception e) { throw new AssertionError(e); } From be2712fd68f09ccf7d401f8e9b69ae2e5cacd364 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Tue, 25 Aug 2026 02:04:39 +0000 Subject: [PATCH 14/56] Fix SerDes forwarding and envelope versioning --- docs/adr/005-filesystem-serdes.md | 5 +- docs/advanced/filesystem-serdes.md | 3 +- .../durable/exception/CallbackException.java | 16 +++- .../exception/CallbackFailedException.java | 6 +- .../exception/DurableOperationException.java | 21 +++++ .../durable/exception/InvokeException.java | 18 ++++- .../exception/InvokeFailedException.java | 6 +- .../durable/operation/CallbackOperation.java | 4 +- .../durable/operation/InvokeOperation.java | 4 +- .../SerializableDurableOperation.java | 26 +------ .../durable/serde/FileSystemSerDes.java | 24 +++++- .../operation/InvokeOperationTest.java | 16 ++-- .../SerializableDurableOperationTest.java | 78 +++++++++++++++++++ .../durable/serde/FileSystemSerDesTest.java | 27 ++++++- 14 files changed, 213 insertions(+), 41 deletions(-) diff --git a/docs/adr/005-filesystem-serdes.md b/docs/adr/005-filesystem-serdes.md index 99646fcfd..68a568e95 100644 --- a/docs/adr/005-filesystem-serdes.md +++ b/docs/adr/005-filesystem-serdes.md @@ -289,7 +289,7 @@ Path encodings: Envelope format: ```json -{"__durable_execution_filesystem_serdes":1,"data":""} +{"__durable_execution_filesystem_serdes":1,"data":"","ownerDurableExecutionArn":"","ownerEntityId":""} {"__durable_execution_filesystem_serdes":1,"file":"","ownerDurableExecutionArn":"","ownerEntityId":""} {"__durable_execution_filesystem_serdes":1,"file":"","ownerDurableExecutionArn":"","ownerEntityId":"","preview":{ "...": "..." }} ``` @@ -303,7 +303,8 @@ results, and standard Lambda invoke results may arrive before this SerDes has pr payload sources only, an input without the filesystem marker is decoded directly with the pipeline value codec or standalone delegate. Skipping every string stage is required because raw external data has not been compressed, encrypted, or otherwise transformed by those stages. Missing or malformed markers on SDK-checkpointed payloads are -permanent errors. +permanent errors. The marker name is reserved: malformed marked envelopes and unsupported envelope versions are +rejected at external boundaries rather than being treated as raw user data. Offloaded filenames include a content hash and are immutable. Serializing new state for the same entity creates a new path instead of replacing a file referenced by an earlier checkpoint. Repeating the same serialization may reuse the diff --git a/docs/advanced/filesystem-serdes.md b/docs/advanced/filesystem-serdes.md index 1a93afbd9..4e0da19c8 100644 --- a/docs/advanced/filesystem-serdes.md +++ b/docs/advanced/filesystem-serdes.md @@ -65,7 +65,8 @@ delays consume time in the current Lambda invocation, so keep attempts and delay Filesystem envelopes include a reserved version marker. Raw root input, callback results, and standard Lambda invoke results bypass every string-processing stage and decode directly with the pipeline value codec when they have not yet -been wrapped by this SerDes. +been wrapped by this SerDes. Payloads containing the reserved marker must be valid supported filesystem envelopes; +malformed marked envelopes and unsupported versions fail instead of falling back to raw-data decoding. Offloaded files are content-addressed and immutable. Updating wait-for-condition state or retry results creates a new path instead of replacing a file referenced by an earlier checkpoint. Repeating the same write can safely reuse the diff --git a/sdk/src/main/java/software/amazon/lambda/durable/exception/CallbackException.java b/sdk/src/main/java/software/amazon/lambda/durable/exception/CallbackException.java index a8fb6a011..92d745e0f 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/exception/CallbackException.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/exception/CallbackException.java @@ -3,6 +3,7 @@ package software.amazon.lambda.durable.exception; import software.amazon.awssdk.services.lambda.model.Operation; +import software.amazon.lambda.durable.util.ExceptionHelper; /** Thrown when a callback operation encounters an error. */ public class CallbackException extends DurableOperationException { @@ -13,7 +14,20 @@ public CallbackException(Operation operation, String message) { } public CallbackException(Operation operation, String message, Throwable cause) { - super(operation, operation.callbackDetails().error(), message, cause); + this(operation, message, cause, null); + } + + protected CallbackException(Operation operation, String message, Throwable cause, Throwable deserializedError) { + super( + operation, + operation.callbackDetails().error(), + message, + operation.callbackDetails().error() != null + ? ExceptionHelper.deserializeStackTrace( + operation.callbackDetails().error().stackTrace()) + : null, + cause, + deserializedError); this.callbackId = operation.callbackDetails().callbackId(); } diff --git a/sdk/src/main/java/software/amazon/lambda/durable/exception/CallbackFailedException.java b/sdk/src/main/java/software/amazon/lambda/durable/exception/CallbackFailedException.java index e3fb9e177..d966bbc3f 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/exception/CallbackFailedException.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/exception/CallbackFailedException.java @@ -8,7 +8,11 @@ /** Exception thrown when a callback fails due to an error from the external system. */ public class CallbackFailedException extends CallbackException { public CallbackFailedException(Operation operation) { - super(operation, buildMessage(operation.callbackDetails().error())); + this(operation, null); + } + + public CallbackFailedException(Operation operation, Throwable deserializedError) { + super(operation, buildMessage(operation.callbackDetails().error()), deserializedError, deserializedError); } private static String buildMessage(ErrorObject error) { diff --git a/sdk/src/main/java/software/amazon/lambda/durable/exception/DurableOperationException.java b/sdk/src/main/java/software/amazon/lambda/durable/exception/DurableOperationException.java index 73078ea1d..5eec194db 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/exception/DurableOperationException.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/exception/DurableOperationException.java @@ -11,6 +11,7 @@ public class DurableOperationException extends DurableExecutionException { private final Operation operation; private final ErrorObject errorObject; + private final transient Throwable deserializedError; public DurableOperationException(Operation operation, ErrorObject errorObject) { this(operation, errorObject, errorObject != null ? errorObject.errorMessage() : null); @@ -36,9 +37,20 @@ public DurableOperationException( String errorMessage, StackTraceElement[] stackTrace, Throwable cause) { + this(operation, errorObject, errorMessage, stackTrace, cause, null); + } + + protected DurableOperationException( + Operation operation, + ErrorObject errorObject, + String errorMessage, + StackTraceElement[] stackTrace, + Throwable cause, + Throwable deserializedError) { super(errorMessage, cause, stackTrace); this.operation = operation; this.errorObject = errorObject; + this.deserializedError = deserializedError; } /** Returns the error details from the failed operation. */ @@ -60,4 +72,13 @@ public OperationStatus getOperationStatus() { public String getOperationId() { return operation.id(); } + + /** + * Returns the original error reconstructed by the operation that produced this exception, when available. + * + *

This is used internally when a child context forwards an operation failure through a different SerDes. + */ + public Throwable deserializedError() { + return deserializedError; + } } diff --git a/sdk/src/main/java/software/amazon/lambda/durable/exception/InvokeException.java b/sdk/src/main/java/software/amazon/lambda/durable/exception/InvokeException.java index 37bbf2ff9..e88a3a34e 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/exception/InvokeException.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/exception/InvokeException.java @@ -3,14 +3,30 @@ package software.amazon.lambda.durable.exception; import software.amazon.awssdk.services.lambda.model.Operation; +import software.amazon.lambda.durable.util.ExceptionHelper; /** Base exception for chained invoke operation failures. */ public class InvokeException extends DurableOperationException { public InvokeException(Operation operation) { + this(operation, null); + } + + protected InvokeException(Operation operation, Throwable deserializedError) { super( operation, operation.chainedInvokeDetails() != null ? operation.chainedInvokeDetails().error() - : null); + : null, + operation.chainedInvokeDetails() != null + && operation.chainedInvokeDetails().error() != null + ? operation.chainedInvokeDetails().error().errorMessage() + : null, + operation.chainedInvokeDetails() != null + && operation.chainedInvokeDetails().error() != null + ? ExceptionHelper.deserializeStackTrace( + operation.chainedInvokeDetails().error().stackTrace()) + : null, + deserializedError, + deserializedError); } } diff --git a/sdk/src/main/java/software/amazon/lambda/durable/exception/InvokeFailedException.java b/sdk/src/main/java/software/amazon/lambda/durable/exception/InvokeFailedException.java index 45f84c341..9bd5b5e01 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/exception/InvokeFailedException.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/exception/InvokeFailedException.java @@ -8,6 +8,10 @@ public class InvokeFailedException extends InvokeException { public InvokeFailedException(Operation operation) { - super(operation); + this(operation, null); + } + + public InvokeFailedException(Operation operation, Throwable deserializedError) { + super(operation, deserializedError); } } diff --git a/sdk/src/main/java/software/amazon/lambda/durable/operation/CallbackOperation.java b/sdk/src/main/java/software/amazon/lambda/durable/operation/CallbackOperation.java index 9d9481fb9..305b04fd9 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/operation/CallbackOperation.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/operation/CallbackOperation.java @@ -77,7 +77,9 @@ public T get() { return switch (op.status()) { case SUCCEEDED -> deserializeResult(op.callbackDetails().result()); - case FAILED -> throw new CallbackFailedException(op); + case FAILED -> + throw new CallbackFailedException( + op, deserializeException(op.callbackDetails().error())); case TIMED_OUT -> throw new CallbackTimeoutException(op); default -> throw terminateExecutionWithIllegalDurableOperationException( diff --git a/sdk/src/main/java/software/amazon/lambda/durable/operation/InvokeOperation.java b/sdk/src/main/java/software/amazon/lambda/durable/operation/InvokeOperation.java index 3796932af..fc0521819 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/operation/InvokeOperation.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/operation/InvokeOperation.java @@ -92,7 +92,9 @@ public T get() { var result = invokeDetails != null ? invokeDetails.result() : null; return switch (op.status()) { case SUCCEEDED -> deserializeResult(result); - case FAILED -> throw new InvokeFailedException(op); + case FAILED -> + throw new InvokeFailedException( + op, deserializeException(op.chainedInvokeDetails().error())); case TIMED_OUT -> throw new InvokeTimedOutException(op); case STOPPED -> throw new InvokeStoppedException(op); // Unexpected status which should not happen. This is added for forward-compatibility. diff --git a/sdk/src/main/java/software/amazon/lambda/durable/operation/SerializableDurableOperation.java b/sdk/src/main/java/software/amazon/lambda/durable/operation/SerializableDurableOperation.java index d4eafb3c1..cebe55662 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/operation/SerializableDurableOperation.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/operation/SerializableDurableOperation.java @@ -5,7 +5,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import software.amazon.awssdk.services.lambda.model.ErrorObject; -import software.amazon.awssdk.services.lambda.model.Operation; import software.amazon.lambda.durable.DurableFuture; import software.amazon.lambda.durable.TypeToken; import software.amazon.lambda.durable.context.DurableContextImpl; @@ -13,7 +12,6 @@ import software.amazon.lambda.durable.exception.RetryableSerDesException; import software.amazon.lambda.durable.exception.SerDesException; import software.amazon.lambda.durable.model.OperationIdentifier; -import software.amazon.lambda.durable.model.OperationSubType; import software.amazon.lambda.durable.serde.SerDes; import software.amazon.lambda.durable.serde.SerDesContext; import software.amazon.lambda.durable.serde.SerDesPayloadKind; @@ -170,7 +168,7 @@ protected ErrorObject rebindForwardedException(DurableOperationException excepti if (error == null || exception.getOperation() == null) { return error; } - var original = deserializeExceptionWithContext(error, producerExceptionContext(exception.getOperation())); + var original = exception.deserializedError(); return original != null ? serializeException(original) : error; } @@ -231,27 +229,5 @@ private Throwable deserializeExceptionWithContext(ErrorObject errorObject, SerDe return original; } - private SerDesContext producerExceptionContext(Operation operation) { - var attempt = operation.stepDetails() != null ? operation.stepDetails().attempt() : null; - return SerDesContext.forOperation( - getContext().getExecutionManager().getDurableExecutionArn(), - operation.id(), - operation.name(), - operation.parentId(), - operation.type(), - operationSubType(operation), - SerDesPayloadKind.EXCEPTION, - attempt); - } - - private static OperationSubType operationSubType(Operation operation) { - for (var subType : OperationSubType.values()) { - if (subType.getValue().equals(operation.subType())) { - return subType; - } - } - return null; - } - public abstract T get(); } diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java index 78dd99d42..56830d805 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java @@ -178,12 +178,22 @@ private ResolvedPayload resolveSerializedPayload(String data, SerDesContext cont throw malformedEnvelope(context, e); } - if (!isFilesystemEnvelope(envelope)) { + if (!hasFilesystemMarker(envelope)) { if (acceptsExternalPayload(context)) { return new ResolvedPayload(data, true); } throw malformedEnvelope(context, null); } + var marker = envelope.get(ENVELOPE_MARKER); + if (!marker.isIntegralNumber()) { + throw malformedEnvelope(context, null); + } + if (marker.intValue() != ENVELOPE_VERSION) { + throw unsupportedEnvelopeVersion(context, marker.asText()); + } + if (!isFilesystemEnvelope(envelope)) { + throw malformedEnvelope(context, null); + } var hasData = envelope.has("data") && envelope.get("data").isTextual(); var hasFile = envelope.has("file") && envelope.get("file").isTextual(); @@ -298,6 +308,10 @@ private static boolean isFilesystemEnvelope(JsonNode envelope) { return envelope.size() == (hasPreview ? 5 : 4); } + private static boolean hasFilesystemMarker(JsonNode envelope) { + return envelope != null && envelope.isObject() && envelope.has(ENVELOPE_MARKER); + } + private static boolean acceptsExternalPayload(SerDesContext context) { return context.payloadKind() == SerDesPayloadKind.INPUT || context.operationType() == OperationType.CALLBACK @@ -309,6 +323,14 @@ private static SerDesException malformedEnvelope(SerDesContext context, Throwabl return cause == null ? new SerDesException(message) : new SerDesException(message, cause); } + private static SerDesException unsupportedEnvelopeVersion(SerDesContext context, String version) { + return new SerDesException("Unsupported filesystem SerDes envelope version " + + version + + " for entity '" + + context.entityId() + + "'"); + } + private String encodeEnvelope(String data, Path file, Map preview, SerDesContext context) { var envelope = new LinkedHashMap(); envelope.put(ENVELOPE_MARKER, ENVELOPE_VERSION); diff --git a/sdk/src/test/java/software/amazon/lambda/durable/operation/InvokeOperationTest.java b/sdk/src/test/java/software/amazon/lambda/durable/operation/InvokeOperationTest.java index 2c1d76c74..b647bb0a1 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/operation/InvokeOperationTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/operation/InvokeOperationTest.java @@ -3,6 +3,7 @@ package software.amazon.lambda.durable.operation; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -71,15 +72,18 @@ void getDoesNotThrowWhenCalledFromHandlerContext() { @Test void getInvokeFailedExceptionWhenInvocationFailed() { + var serDes = new JacksonSerDes(); + var original = new IllegalStateException("errorMessage"); + var errorData = serDes.serialize(original); var op = Operation.builder() .id(OPERATION_ID) .name(OPERATION_NAME) .status(OperationStatus.FAILED) .chainedInvokeDetails(ChainedInvokeDetails.builder() .error(ErrorObject.builder() - .errorType("errorType") + .errorType(original.getClass().getName()) .errorMessage("errorMessage") - .errorData("errorData") + .errorData(errorData) .build()) .build()) .build(); @@ -90,14 +94,16 @@ void getInvokeFailedExceptionWhenInvocationFailed() { "test-function", "{}", TypeToken.get(String.class), - InvokeConfig.builder().serDes(new JacksonSerDes()).build(), + InvokeConfig.builder().serDes(serDes).build(), durableContext); operation.onCheckpointComplete(op); InvokeFailedException ex = assertThrows(InvokeFailedException.class, () -> operation.get()); - assertEquals("errorData", ex.getErrorObject().errorData()); - assertEquals("errorType", ex.getErrorObject().errorType()); + assertEquals(errorData, ex.getErrorObject().errorData()); + assertEquals(original.getClass().getName(), ex.getErrorObject().errorType()); assertEquals("errorMessage", ex.getMessage()); + assertInstanceOf(IllegalStateException.class, ex.deserializedError()); + assertEquals("errorMessage", ex.deserializedError().getMessage()); } @Test diff --git a/sdk/src/test/java/software/amazon/lambda/durable/operation/SerializableDurableOperationTest.java b/sdk/src/test/java/software/amazon/lambda/durable/operation/SerializableDurableOperationTest.java index 95edc7282..82e4f0224 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/operation/SerializableDurableOperationTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/operation/SerializableDurableOperationTest.java @@ -28,6 +28,7 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import software.amazon.awssdk.services.lambda.model.CallbackDetails; import software.amazon.awssdk.services.lambda.model.ErrorObject; import software.amazon.awssdk.services.lambda.model.Operation; import software.amazon.awssdk.services.lambda.model.OperationStatus; @@ -36,7 +37,9 @@ import software.amazon.lambda.durable.DurableConfig; import software.amazon.lambda.durable.TypeToken; import software.amazon.lambda.durable.client.DurableExecutionClient; +import software.amazon.lambda.durable.config.CallbackConfig; import software.amazon.lambda.durable.context.DurableContextImpl; +import software.amazon.lambda.durable.exception.CallbackFailedException; import software.amazon.lambda.durable.exception.IllegalDurableOperationException; import software.amazon.lambda.durable.exception.NonDeterministicExecutionException; import software.amazon.lambda.durable.exception.RetryableSerDesException; @@ -91,6 +94,27 @@ public T deserialize(String data, TypeToken typeToken) { } } + private static final class PrefixedSerDes extends JacksonSerDes { + private final String prefix; + + private PrefixedSerDes(String prefix) { + this.prefix = prefix; + } + + @Override + public String serialize(Object value) { + return prefix + super.serialize(value); + } + + @Override + public T deserialize(String data, TypeToken typeToken) { + if (data == null || !data.startsWith(prefix)) { + throw new SerDesException("Expected SerDes prefix " + prefix); + } + return super.deserialize(data.substring(prefix.length()), typeToken); + } + } + private static final String OPERATION_ID = "1"; private static final String CONTEXT_ID = "1-step"; private static final String OPERATION_NAME = "name"; @@ -562,6 +586,60 @@ public String get() { replay.get(); } + @Test + void rebindForwardedExceptionUsesProducingOperationSerDesBeforeParentSerDes() { + var producerSerDes = new PrefixedSerDes("producer:"); + var parentSerDes = new PrefixedSerDes("parent:"); + var original = new IllegalStateException("callback failed"); + var producerOperation = Operation.builder() + .id("callback-1") + .name("callback") + .type(OperationType.CALLBACK) + .subType(OperationSubType.CALLBACK.getValue()) + .status(OperationStatus.FAILED) + .callbackDetails(CallbackDetails.builder() + .callbackId("callback-id") + .error(ErrorObject.builder() + .errorType(original.getClass().getName()) + .errorMessage(original.getMessage()) + .errorData(producerSerDes.serialize(original)) + .build()) + .build()) + .build(); + when(executionManager.getOperationAndUpdateReplayState("callback-1")).thenReturn(producerOperation); + + var producer = new CallbackOperation<>( + OperationIdentifier.of("callback-1", "callback", OperationSubType.CALLBACK), + TypeToken.get(String.class), + CallbackConfig.builder().serDes(producerSerDes).build(), + durableContext); + producer.onCheckpointComplete(producerOperation); + var forwarded = assertThrows(CallbackFailedException.class, producer::get); + assertInstanceOf(IllegalStateException.class, forwarded.deserializedError()); + + var rebound = new AtomicReference(); + SerializableDurableOperation parent = + new SerializableDurableOperation<>(OPERATION_IDENTIFIER, RESULT_TYPE, parentSerDes, durableContext) { + @Override + protected void start() {} + + @Override + protected void replay(Operation existing) {} + + @Override + public String get() { + rebound.set(rebindForwardedException(forwarded)); + return RESULT; + } + }; + + parent.get(); + + assertTrue(rebound.get().errorData().startsWith("parent:")); + var decoded = parentSerDes.deserialize(rebound.get().errorData(), TypeToken.get(IllegalStateException.class)); + assertEquals("callback failed", decoded.getMessage()); + } + @Test void serializeExceptionValidatesRoundTrip() { var serDes = new TrackingSerDes(); diff --git a/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java b/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java index 7d047593f..7db8534da 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java @@ -166,8 +166,15 @@ void acceptsExternallyOriginatedRawPayloadsOnlyForSupportedSources() { TypeToken.get(String.class), operationContext(OperationType.CHAINED_INVOKE, OperationSubType.CHAINED_INVOKE))); assertEquals( - Map.of(ENVELOPE_MARKER, 1, "data", "domain-value"), + Map.of("domainMarker", 1, "data", "domain-value"), runner.deserialize( + standalone, + "{\"domainMarker\":1,\"data\":\"domain-value\"}", + new TypeToken>() {}, + executionContext(SerDesPayloadKind.INPUT))); + assertThrows( + SerDesException.class, + () -> runner.deserialize( standalone, "{\"__durable_execution_filesystem_serdes\":1,\"data\":\"domain-value\"}", new TypeToken>() {}, @@ -178,6 +185,24 @@ void acceptsExternallyOriginatedRawPayloadsOnlyForSupportedSources() { () -> runner.deserialize(stage, "\"raw-step\"", TypeToken.get(String.class), context())); } + @Test + void rejectsUnsupportedEnvelopeVersionsAtExternalBoundaries() { + var serDes = FileSystemSerDes.builder(basePath).build(); + var futureEnvelope = "{\"__durable_execution_filesystem_serdes\":2," + + "\"ownerDurableExecutionArn\":\"" + + ARN + + "\",\"ownerEntityId\":\"1\",\"data\":\"value\"}"; + + var failure = assertThrows(SerDesException.class, () -> new SerDesRunner(null) + .deserialize( + serDes, + futureEnvelope, + TypeToken.get(String.class), + executionContext(SerDesPayloadKind.INPUT))); + + assertTrue(failure.getCause().getMessage().contains("Unsupported filesystem SerDes envelope version 2")); + } + @Test void overflowFilesystemStageMustRemainTerminal() { var filesystem = FileSystemSerDes.stageBuilder(basePath) From db52f38bad0e0c4b7850a8eedac16d9583cd1a3f Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Tue, 25 Aug 2026 02:17:56 +0000 Subject: [PATCH 15/56] Preserve SerDes errors and validate envelope versions --- .../durable/serde/ComposableSerDes.java | 3 + .../durable/serde/FileSystemSerDes.java | 3 +- .../lambda/durable/serde/SerDesRunner.java | 3 + .../durable/serde/ComposableSerDesTest.java | 56 +++++++++++++++++++ .../durable/serde/FileSystemSerDesTest.java | 19 +++++++ .../durable/serde/SerDesRunnerTest.java | 21 +++++++ 6 files changed, 104 insertions(+), 1 deletion(-) diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/ComposableSerDes.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/ComposableSerDes.java index 2badc4bad..adbd4be38 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/ComposableSerDes.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/ComposableSerDes.java @@ -154,6 +154,9 @@ private static T invokeDeserialize(SerDes stage, String data, TypeToken t } private static RuntimeException stageFailure(int index, SerDes stage, String action, Throwable failure) { + if (failure instanceof Error error) { + throw error; + } var message = String.format( "SerDes pipeline stage %d (%s) failed to %s", index, stage.getClass().getName(), action); diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java index 56830d805..618049fa0 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java @@ -188,7 +188,7 @@ private ResolvedPayload resolveSerializedPayload(String data, SerDesContext cont if (!marker.isIntegralNumber()) { throw malformedEnvelope(context, null); } - if (marker.intValue() != ENVELOPE_VERSION) { + if (!marker.canConvertToInt() || marker.intValue() != ENVELOPE_VERSION) { throw unsupportedEnvelopeVersion(context, marker.asText()); } if (!isFilesystemEnvelope(envelope)) { @@ -285,6 +285,7 @@ private static boolean isFilesystemEnvelope(JsonNode envelope) { || !envelope.isObject() || !envelope.has(ENVELOPE_MARKER) || !envelope.get(ENVELOPE_MARKER).isIntegralNumber() + || !envelope.get(ENVELOPE_MARKER).canConvertToInt() || envelope.get(ENVELOPE_MARKER).intValue() != ENVELOPE_VERSION || !envelope.has("ownerDurableExecutionArn") || !envelope.get("ownerDurableExecutionArn").isTextual() diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesRunner.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesRunner.java index 1aa501b0e..fc1a83a79 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesRunner.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesRunner.java @@ -143,6 +143,9 @@ private T run(String action, SerDesContext context, Supplier supplier) { .join(); } catch (Throwable throwable) { var cause = ExceptionHelper.unwrapCompletableFuture(throwable); + if (cause instanceof Error error) { + throw error; + } var message = String.format( "Failed to %s %s payload for entity '%s'", action, context.payloadKind(), context.entityId()); if (cause instanceof RetryableSerDesException) { diff --git a/sdk/src/test/java/software/amazon/lambda/durable/serde/ComposableSerDesTest.java b/sdk/src/test/java/software/amazon/lambda/durable/serde/ComposableSerDesTest.java index 80317ae23..9cd7a2b28 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/serde/ComposableSerDesTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/serde/ComposableSerDesTest.java @@ -5,6 +5,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -216,6 +217,61 @@ public T deserialize(String data, TypeToken typeToken) { assertTrue(failure.getMessage().contains("stage 1")); } + @Test + void preservesFatalErrorsFromEveryPipelineCall() { + var serializeError = new OutOfMemoryError("serialize"); + var serializeStage = new SerDes() { + @Override + public String serialize(Object value) { + throw serializeError; + } + + @Override + public T deserialize(String data, TypeToken typeToken) { + return null; + } + }; + assertSame(serializeError, assertThrows(OutOfMemoryError.class, () -> new JacksonSerDes() + .then(serializeStage) + .serialize("value"))); + + var stringStageError = new StackOverflowError("string-stage-deserialize"); + var stringStage = new SerDes() { + @Override + public String serialize(Object value) { + return value.toString(); + } + + @Override + public T deserialize(String data, TypeToken typeToken) { + return null; + } + + @Override + public SerDesStageResult deserializePipelineStage(String data) { + throw stringStageError; + } + }; + assertSame(stringStageError, assertThrows(StackOverflowError.class, () -> new JacksonSerDes() + .then(stringStage) + .deserialize("value", TypeToken.get(String.class)))); + + var valueCodecError = new AssertionError("value-codec-deserialize"); + var valueCodec = new SerDes() { + @Override + public String serialize(Object value) { + return value.toString(); + } + + @Override + public T deserialize(String data, TypeToken typeToken) { + throw valueCodecError; + } + }; + assertSame(valueCodecError, assertThrows(AssertionError.class, () -> ComposableSerDes.of(valueCodec) + .deserialize("value", TypeToken.get(String.class)))); + } + private static SerDes stringStage(String name, String prefix, String suffix, List calls) { return new SerDes() { @Override diff --git a/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java b/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java index 7db8534da..725728863 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java @@ -203,6 +203,25 @@ void rejectsUnsupportedEnvelopeVersionsAtExternalBoundaries() { assertTrue(failure.getCause().getMessage().contains("Unsupported filesystem SerDes envelope version 2")); } + @Test + void rejectsOutOfRangeEnvelopeVersionsWithoutTruncation() { + var serDes = FileSystemSerDes.builder(basePath).build(); + var oversizedVersion = "{\"__durable_execution_filesystem_serdes\":4294967297," + + "\"ownerDurableExecutionArn\":\"" + + ARN + + "\",\"ownerEntityId\":\"1\",\"data\":\"value\"}"; + + var failure = assertThrows(SerDesException.class, () -> new SerDesRunner(null) + .deserialize( + serDes, + oversizedVersion, + TypeToken.get(String.class), + executionContext(SerDesPayloadKind.INPUT))); + + assertTrue( + failure.getCause().getMessage().contains("Unsupported filesystem SerDes envelope version 4294967297")); + } + @Test void overflowFilesystemStageMustRemainTerminal() { var filesystem = FileSystemSerDes.stageBuilder(basePath) diff --git a/sdk/src/test/java/software/amazon/lambda/durable/serde/SerDesRunnerTest.java b/sdk/src/test/java/software/amazon/lambda/durable/serde/SerDesRunnerTest.java index 0bf210003..35187aaa4 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/serde/SerDesRunnerTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/serde/SerDesRunnerTest.java @@ -303,6 +303,27 @@ public T deserialize(String data, TypeToken typeToken) { assertInstanceOf(RetryableSerDesException.class, exception.getCause()); } + @Test + void preservesFatalErrorsWithAndWithoutExecutor() { + var fatal = new OutOfMemoryError("fatal"); + var serDes = new SerDes() { + @Override + public String serialize(Object value) { + throw fatal; + } + + @Override + public T deserialize(String data, TypeToken typeToken) { + return null; + } + }; + + assertSame(fatal, assertThrows(OutOfMemoryError.class, () -> new SerDesRunner(null) + .serialize(serDes, "value", context("entity")))); + assertSame(fatal, assertThrows(OutOfMemoryError.class, () -> new SerDesRunner(executor) + .serialize(serDes, "value", context("entity")))); + } + private static SerDes fixedValueSerDes(String value) { return new SerDes() { @Override From 26c1626b9f5f44078e6ddad9f3def6afb51dae4c Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Tue, 25 Aug 2026 05:41:35 +0000 Subject: [PATCH 16/56] feat: support typed SerDes pipeline stages --- docs/adr/005-filesystem-serdes.md | 126 ++++++----- docs/advanced/configuration.md | 15 +- docs/advanced/filesystem-serdes.md | 22 +- docs/design.md | 3 +- .../FileSystemSerDesIntegrationTest.java | 24 ++- .../testing/CloudDurableTestRunner.java | 6 +- .../testing/LocalDurableTestRunner.java | 67 ++++-- .../testing/LocalDurableTestRunnerTest.java | 83 ++++++++ .../durable/serde/ComposableSerDes.java | 183 +++++++++++----- .../durable/serde/FileSystemSerDes.java | 198 ++++++++++++++---- .../amazon/lambda/durable/serde/SerDes.java | 36 ++-- .../lambda/durable/serde/SerDesStage.java | 55 +++++ .../durable/serde/SerDesStageResult.java | 16 +- .../durable/serde/ComposableSerDesTest.java | 82 +++++++- .../durable/serde/FileSystemSerDesTest.java | 77 ++++++- 15 files changed, 769 insertions(+), 224 deletions(-) create mode 100644 sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesStage.java diff --git a/docs/adr/005-filesystem-serdes.md b/docs/adr/005-filesystem-serdes.md index 68a568e95..fc999a198 100644 --- a/docs/adr/005-filesystem-serdes.md +++ b/docs/adr/005-filesystem-serdes.md @@ -2,7 +2,7 @@ **Status:** Accepted — Approach A with ComposableSerDes pipeline **Date:** 2026-07-02 -**Updated:** 2026-08-25 — Included FileSystemSerDes in the core SDK artifact. +**Updated:** 2026-08-25 — Included FileSystemSerDes in core and generalized pipelines to typed intermediate stages. ## Context @@ -31,12 +31,20 @@ There are a few Java-specific constraints: ### Summary -Keep the existing `SerDes` serialization methods source- and binary-compatible, add a default composition method and a -core `ComposableSerDes` implementation which together chain multiple `SerDes` instances into a processing pipeline, -and implement `FileSystemSerDes` in the core SDK. The filesystem stage uses +Keep the existing `SerDes` serialization methods source- and binary-compatible, add a typed `SerDesStage` +contract and a core `ComposableSerDes` implementation which together form a processing pipeline, and implement +`FileSystemSerDes` in the core SDK. Existing `SerDes` implementations continue to participate as string-producing +stages without changing their binary contract, while dedicated intermediate stages may exchange arbitrary Java types. +The filesystem stage uses `SerDesContext.getCurrentContext()` to identify the durable execution and entity being serialized. ```java +public interface SerDesStage { + O serialize(I value); + + I deserialize(O data); +} + public interface SerDes { String serialize(Object value); @@ -45,15 +53,20 @@ public interface SerDes { default ComposableSerDes then(SerDes nextStage) { return ComposableSerDes.of(this, nextStage); } + + default ComposableSerDes then(SerDesStage nextStage) { + return ComposableSerDes.builder(this).then(nextStage).build(); + } } ``` -`ComposableSerDes` treats the first stage as the value codec and every later stage as a reversible string -transformation. This lets customers compose JSON encoding, compression, encryption, filesystem storage, or other -processing without each implementation needing to know about every other concern. +`ComposableSerDes` treats the first stage as the value codec and every later stage as a reversible typed +transformation. This lets customers compose JSON encoding, binary compression, encryption, filesystem storage, or +other processing without artificial Base64 conversion between every stage. The complete pipeline still returns a +string because checkpoints use the existing `SerDes` boundary. -`FileSystemSerDes` acts as a payload-storage stage. It writes the string produced by the previous stage to the -filesystem when configured to do so and returns a small envelope for the next stage or checkpoint. For standalone +`FileSystemSerDes` acts as a terminal payload-storage stage. It writes the string or byte array produced by the +previous stage to the filesystem when configured to do so and returns a small checkpoint envelope. For standalone compatibility, it may still be constructed with a value-encoding delegate; pipeline configuration is the preferred composition model. @@ -126,32 +139,38 @@ public final class ComposableSerDes implements SerDes { public SerDes getValueCodec(); public ComposableSerDes then(SerDes stage); + public ComposableSerDes then(SerDesStage stage); public static final class Builder { public Builder then(SerDes stage); + public Builder then(SerDesStage stage); public ComposableSerDes build(); } } -public record SerDesStageResult(String value, boolean skipRemainingStages) { - public static SerDesStageResult continueWith(String value); +public record SerDesStageResult(Object value, boolean skipRemainingStages) { + public static SerDesStageResult continueWith(Object value); public static SerDesStageResult decodeWithValueCodec(String value); } ``` The first stage is the **value codec**. It converts the user value to a string and converts the final decoded string -back to the requested `TypeToken`. Every later stage is a **string stage**: it must accept a `String` in -`serialize(Object)` and must return a `String` when `deserialize` is called with `TypeToken.get(String.class)`. +back to the requested `TypeToken`. Every later stage is a typed `SerDesStage`. Adjacent stages must be +compatible: one stage's serialized output becomes the next stage's input, and deserialization applies the inverse +mapping. Runtime stage metadata is preserved in failures because Java type erasure prevents complete validation when +heterogeneous stages are held in one immutable pipeline. Serialization runs from first to last: ```text Object -> value codec - -> String stage 1 - -> String stage 2 + -> String + -> typed stage 1 + -> intermediate type A + -> typed stage 2 -> ... -> checkpoint String ``` @@ -160,9 +179,10 @@ Deserialization runs in the opposite direction: ```text checkpoint String - -> last string stage, deserialized as String + -> last typed stage -> ... - -> first string stage, deserialized as String + -> first typed stage + -> String -> value codec, deserialized as the requested TypeToken -> T ``` @@ -171,35 +191,37 @@ Equivalent pseudocode: ```java String serialize(Object value) { - String current = stages.get(0).serialize(value); - for (int i = 1; i < stages.size(); i++) { - current = stages.get(i).serialize(current); + Object current = valueCodec.serialize(value); + for (var stage : stages) { + current = stage.serialize(current); } - return current; + return requireStringCheckpoint(current); } T deserialize(String data, TypeToken targetType) { - String current = data; - for (int i = stages.size() - 1; i > 0; i--) { + Object current = data; + for (int i = stages.size() - 1; i >= 0; i--) { var decoded = stages.get(i).deserializePipelineStage(current); current = decoded.value(); if (decoded.skipRemainingStages()) { break; } } - return stages.get(0).deserialize(current, targetType); + return valueCodec.deserialize(requireStringValueCodecInput(current), targetType); } ``` Pipeline rules: -- A pipeline must contain exactly one value codec in the first position and zero or more string stages. +- A pipeline must contain exactly one value codec in the first position and zero or more typed intermediate stages. +- Intermediate values may use any non-null Java type. The complete serialization pipeline must finish with a + `String`, and reverse processing must return a `String` to the value codec. - A stage may declare that it requires durable context or that it must be terminal. A terminal stage must be the last stage so later transformations cannot invalidate its checkpoint-size or storage decision. - `SerDes.then(...)`, `ComposableSerDes.of(...)`, and the builder flatten nested `ComposableSerDes` instances while preserving stage order. - A `null` value at the pipeline boundary short-circuits the entire pipeline: serializing or deserializing `null` - returns `null` without invoking any stage. A string stage returning `null` for non-null input is an error. The value + returns `null` without invoking any stage. A stage returning `null` for non-null input is an error. The value codec may decode a non-null representation such as the JSON literal `null` to a null domain value. - All stages execute within the same `SerDesRunner` invocation and observe the same read-only `SerDesContext`, whether the runner executes inline or dispatches to a configured executor. @@ -207,10 +229,10 @@ Pipeline rules: concurrent use, matching the existing `SerDes` requirement. - A failure must identify the stage index and implementation class in `SerDesException`; `SerDesRunner` adds durable entity and payload-kind metadata around the pipeline failure. -- A string stage must be reversible. Compression, encryption, signing envelopes, and external payload storage are +- A stage must be reversible. Compression, encryption, signing envelopes, and external payload storage are suitable stages; lossy redaction is not. - A boundary stage may return `SerDesStageResult.decodeWithValueCodec(...)` when it receives raw data that did not pass - through the configured pipeline. `ComposableSerDes` then skips every earlier string stage and decodes the raw value + through the configured pipeline. `ComposableSerDes` then skips every earlier intermediate stage and decodes the raw value directly with the value codec. - Stage order is meaningful. For example, `JSON -> compression -> encryption -> filesystem` writes encrypted, compressed data to the filesystem. `FileSystemSerDes` is terminal; placing encryption or any expanding @@ -276,8 +298,8 @@ Storage modes: | Mode | Behavior | |------|----------| -| `ALWAYS` | Always write the incoming stage string to a file and return a file envelope. | -| `OVERFLOW` | Return an inline envelope until it approaches the service payload limit, then write the incoming stage string to a file. | +| `ALWAYS` | Always write the incoming stage representation to a file and return a file envelope. | +| `OVERFLOW` | Return an inline envelope until it approaches the service payload limit, then write the incoming stage representation to a file. | Path encodings: @@ -289,19 +311,20 @@ Path encodings: Envelope format: ```json -{"__durable_execution_filesystem_serdes":1,"data":"","ownerDurableExecutionArn":"","ownerEntityId":""} -{"__durable_execution_filesystem_serdes":1,"file":"","ownerDurableExecutionArn":"","ownerEntityId":""} -{"__durable_execution_filesystem_serdes":1,"file":"","ownerDurableExecutionArn":"","ownerEntityId":"","preview":{ "...": "..." }} +{"__durable_execution_filesystem_serdes":1,"data":"","payloadType":"STRING|BYTES","ownerDurableExecutionArn":"","ownerEntityId":""} +{"__durable_execution_filesystem_serdes":1,"file":"","payloadType":"STRING|BYTES","ownerDurableExecutionArn":"","ownerEntityId":""} +{"__durable_execution_filesystem_serdes":1,"file":"","payloadType":"STRING|BYTES","ownerDurableExecutionArn":"","ownerEntityId":"","preview":{ "...": "..." }} ``` `FileSystemSerDes` must reject calls when `SerDesContext.getCurrentContext()` is `null` or does not include -`durableExecutionArn` and `entityId`. When used as a pipeline stage, it must also reject non-string input or a -deserialization target other than `String`. +`durableExecutionArn` and `entityId`. In pipeline mode it accepts `String` and `byte[]` values, records the payload type +in the envelope, stores byte arrays without text conversion, and restores the same representation during reverse +processing. Standalone mode continues to use its configured string value codec. The marker and version distinguish filesystem envelopes from raw service-originated JSON. Initial root input, callback results, and standard Lambda invoke results may arrive before this SerDes has processed them. For those external payload sources only, an input without the filesystem marker is decoded directly with the pipeline value codec or -standalone delegate. Skipping every string stage is required because raw external data has not been compressed, +standalone delegate. Skipping every intermediate stage is required because raw external data has not been compressed, encrypted, or otherwise transformed by those stages. Missing or malformed markers on SDK-checkpointed payloads are permanent errors. The marker name is reserved: malformed marked envelopes and unsupported envelope versions are rejected at external boundaries rather than being treated as raw user data. @@ -324,9 +347,9 @@ rejected rather than producing a checkpoint that the service cannot accept. checkpoint representation and prevents a later Base64, encryption, or other expanding stage from pushing an inline envelope over the service limit. -In stage mode, the preview generator receives the string produced by the preceding stage, not the original domain -object. A preview that needs domain fields should either parse that representation, be produced by an earlier stage, or -use standalone compatibility mode where `FileSystemSerDes` receives the original value. +In stage mode, the preview generator receives the `String` or `byte[]` produced by the preceding stage, not the +original domain object. A preview that needs domain fields should either parse that representation, be produced by an +earlier stage, or use standalone compatibility mode where `FileSystemSerDes` receives the original value. ### Runtime flow @@ -345,11 +368,12 @@ try { } ``` -On deserialization, `FileSystemSerDes` parses its envelope. If the envelope contains `data`, it returns the inline -string. If the envelope contains `file`, it reads and returns the file contents. `ComposableSerDes` then passes that -string to the preceding stage. In standalone compatibility mode, `FileSystemSerDes` instead passes the resolved string -to its configured value-encoding delegate. Raw external input, callback results, and standard invoke results skip all -string stages and go directly to the value codec when no versioned filesystem marker is present. +On deserialization, `FileSystemSerDes` parses its envelope. If the envelope contains `data`, it restores inline text or +Base64-decoded bytes according to `payloadType`. If the envelope contains `file`, it reads the raw file contents and +restores the same representation. `ComposableSerDes` then passes that value to the preceding typed stage. In standalone +compatibility mode, `FileSystemSerDes` passes the resolved string to its configured value-encoding delegate. Raw +external input, callback results, and standard invoke results skip all intermediate stages and go directly to the +value codec when no versioned filesystem marker is present. ### Threading @@ -444,11 +468,13 @@ Root user input and output payloads should route through `SerDesRunner` so `File The cloud test runner must send initial Lambda input before it receives a durable execution ARN. When configured with a context-free `ComposableSerDes`, it serializes the invocation payload with the complete configured pipeline so compression, encryption, and other ordinary transformations remain compatible with the deployed function. When the -persisted SerDes reports that it requires durable context, the runner requires a separate context-free input value -codec via `CloudDurableTestRunner.withInputSerDes(...)`. That codec must not be a composable string-processing pipeline: +persisted SerDes reports that it requires durable context, the cloud and local runners require a separate context-free +input value codec via `withInputSerDes(...)`. That codec must not be a composable pipeline: an unframed external payload does not identify which input stages ran, while a context-dependent terminal stage must also accept raw service payloads such as callbacks and invoke results. Fluent configuration preserves that explicit -input codec regardless of whether `withInputSerDes(...)` or `withSerDes(...)` is called first. +input codec when other runner configuration is replaced. `LocalDurableTestRunner` must create the execution operation +with this raw external payload and let `DurableExecutor` apply the persisted pipeline's external-boundary behavior; it +must not synthesize a durable context and serialize initial input through the full persisted pipeline. ### Implementation plan @@ -468,7 +494,7 @@ input codec regardless of whether `withInputSerDes(...)` or `withSerDes(...)` is serialized data hash. 8. Update exception serialization and deserialization paths to set `SerDesPayloadKind.EXCEPTION` in TLS. 9. Implement `FileSystemSerDes` in the core `software.amazon.lambda.durable.serde` package with standalone compatibility and - string-stage modes, `ALWAYS` and `OVERFLOW` storage, `URI` and `HASH` path encodings, envelope parsing, atomic file + typed terminal-stage modes, `ALWAYS` and `OVERFLOW` storage, `URI` and `HASH` path encodings, envelope parsing, atomic file writes where supported by the filesystem, retryable I/O failures, and clear validation errors for missing context or invalid stage input. 10. Add unit tests for pipeline ordering, reverse processing, nulls, invalid intermediate stage types, stage failures, @@ -774,8 +800,8 @@ Negative: - Adds optional executor, context, and caching machinery that must stay deterministic. - Adds storage-specific public API and implementation code to the core SDK artifact. - Approach A requires thread-local SerDes context because the existing `SerDes` methods do not accept context. -- Approach A relies on a documented string-stage convention that is validated at runtime rather than by Java's type - system. +- Approach A uses typed stage contracts, but heterogeneous pipeline compatibility is still validated at runtime after + type erasure. - Pipeline ordering and stage configuration must remain compatible with persisted checkpoints. - The inline default means filesystem I/O and retry delays block the caller when customers do not configure a SerDes executor. diff --git a/docs/advanced/configuration.md b/docs/advanced/configuration.md index 3e9e8b6e8..552dd1790 100644 --- a/docs/advanced/configuration.md +++ b/docs/advanced/configuration.md @@ -53,7 +53,7 @@ and uses a bounded weak-reference cache for successful deserialization results d ### Filesystem-backed payload storage -The core SDK provides a reversible string stage for storing serialized payloads on a shared filesystem: +The core SDK provides a reversible terminal stage for storing serialized text or bytes on a shared filesystem: ```java var fileSystemStage = FileSystemSerDes.stageBuilder(Path.of("/mnt/efs/durable-payloads")) @@ -75,6 +75,11 @@ return DurableConfig.builder() .build(); ``` +Pipelines may exchange non-string intermediate values. For example, a custom +`SerDesStage` can compress the JSON string and feed the resulting bytes directly into +`FileSystemSerDes`; reverse processing restores the bytes to the compression stage. The complete pipeline still +returns a string checkpoint envelope. + `ALWAYS` stores every non-null payload in a file. `OVERFLOW` keeps payloads inline until the checkpoint envelope approaches the 256 KB service limit. `URI` produces readable escaped paths; `HASH` produces fixed-length SHA-256 path segments. Files are content-addressed and never overwrite data referenced by an earlier checkpoint. References are @@ -90,11 +95,11 @@ shared mount such as EFS. S3 Files can have delayed synchronization, so a runtim lose recent writes; use it only when that tradeoff is acceptable. The SDK does not delete offloaded files, so configure storage lifecycle and retention separately. -For a context-free `ComposableSerDes`, `CloudDurableTestRunner` applies the complete pipeline to the initial Lambda -invocation. If the persisted SerDes requires durable context, such as `FileSystemSerDes`, call +For a context-free `ComposableSerDes`, the test runners apply the complete pipeline to the initial Lambda invocation. +If the persisted SerDes requires durable context, such as `FileSystemSerDes`, call `withInputSerDes(...)` with a separate context-free input value codec because the durable execution ARN does not exist -yet. In that case, the input SerDes must not include composable string-processing stages because the external payload -does not carry framing that identifies which stages ran. +yet. In that case, the input SerDes must be a value codec rather than a composable pipeline because the external +payload does not carry framing that identifies which stages ran. `FileSystemSerDes` must also be the final pipeline stage so its checkpoint-size decision cannot be invalidated by a later expanding transformation. diff --git a/docs/advanced/filesystem-serdes.md b/docs/advanced/filesystem-serdes.md index 4e0da19c8..201782310 100644 --- a/docs/advanced/filesystem-serdes.md +++ b/docs/advanced/filesystem-serdes.md @@ -15,7 +15,7 @@ envelopes in checkpoints. It is included in the core `aws-durable-execution-sdk- ## Pipeline configuration -The preferred configuration uses `FileSystemSerDes` as a reversible string stage after a value codec: +The preferred configuration uses `FileSystemSerDes` as a reversible terminal stage after a value codec: ```java var fileSystemStage = FileSystemSerDes.stageBuilder(Path.of("/mnt/efs/durable-payloads")) @@ -38,15 +38,17 @@ return DurableConfig.builder() ``` Serialization runs from `JacksonSerDes` to the filesystem stage. Deserialization runs in reverse. Additional reversible -string stages such as compression or encryption can be inserted with `then(...)`. +typed stages such as compression or encryption can be inserted with `then(...)`. A +`SerDesStage` may pass compressed or encrypted bytes directly to `FileSystemSerDes`; no Base64 adapter +is required between those stages. The complete pipeline still produces a string checkpoint envelope. - `ALWAYS` writes every non-null payload to a file. - `OVERFLOW` stores small payloads inline and offloads envelopes approaching the 256 KB checkpoint limit. - `URI` uses readable escaped path segments. - `HASH` uses fixed-length SHA-256 path segments. -The preview generator receives the incoming stage string. Its output is included only in file envelopes and the final -envelope must remain below the checkpoint threshold. +The preview generator receives the incoming stage value, which may be a `String` or `byte[]`. Its output is included +only in file envelopes and the final envelope must remain below the checkpoint threshold. For compatibility, `FileSystemSerDes.builder(path)` creates a standalone SerDes with `JacksonSerDes` as its default value codec. A custom standalone codec can be supplied with `.delegate(...)`. @@ -64,7 +66,7 @@ delays consume time in the current Lambda invocation, so keep attempts and delay ## Replay and envelope behavior Filesystem envelopes include a reserved version marker. Raw root input, callback results, and standard Lambda invoke -results bypass every string-processing stage and decode directly with the pipeline value codec when they have not yet +results bypass every intermediate stage and decode directly with the pipeline value codec when they have not yet been wrapped by this SerDes. Payloads containing the reserved marker must be valid supported filesystem envelopes; malformed marked envelopes and unsupported versions fail instead of falling back to raw-data decoding. @@ -78,11 +80,11 @@ verified when reading, and symbolic-link paths are rejected. `FileSystemSerDes` must be the final stage in a pipeline. Its overflow decision is therefore made against the final checkpoint representation; a later expanding transform cannot push an inline envelope over the service limit. -`CloudDurableTestRunner` cannot use `FileSystemSerDes` for the initial Lambda invocation because an execution ARN is -not available yet. Configure a separate context-free initial-input value codec with `withInputSerDes(...)`. Do not use -a composable string-processing pipeline for that input boundary: the unframed external payload does not identify which -stages ran before the context-dependent filesystem stage. Context-free persisted pipelines still use their complete -pipeline for the initial invocation. +The cloud and local test runners cannot use `FileSystemSerDes` for the initial Lambda invocation because an execution +ARN is not available yet. Configure a separate context-free initial-input value codec with +`withInputSerDes(...)`. Do not use a composable pipeline for that input boundary: the unframed external payload does +not identify which stages ran before the context-dependent filesystem stage. Context-free +persisted pipelines still use their complete pipeline for the initial invocation. ## Storage requirements diff --git a/docs/design.md b/docs/design.md index 072e7590f..269e98500 100644 --- a/docs/design.md +++ b/docs/design.md @@ -355,7 +355,8 @@ software.amazon.lambda.durable │ ├── serde/ │ ├── SerDes # Interface and pipeline composition entry point -│ ├── ComposableSerDes # Immutable ordered value-codec/string-stage pipeline +│ ├── SerDesStage # Reversible typed intermediate pipeline stage +│ ├── ComposableSerDes # Immutable ordered value-codec/typed-stage pipeline │ ├── JacksonSerDes # Jackson impl │ ├── RetrySerDes # Retry decorator for transient SerDes failures │ ├── SerDesRunner # Context, optional executor dispatch, and invocation cache diff --git a/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FileSystemSerDesIntegrationTest.java b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FileSystemSerDesIntegrationTest.java index 3520840ff..ee0505fe3 100644 --- a/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FileSystemSerDesIntegrationTest.java +++ b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FileSystemSerDesIntegrationTest.java @@ -8,6 +8,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import com.fasterxml.jackson.databind.ObjectMapper; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.time.Duration; @@ -42,6 +43,7 @@ import software.amazon.lambda.durable.serde.SerDesContext; import software.amazon.lambda.durable.serde.SerDesPayloadKind; import software.amazon.lambda.durable.serde.SerDesRunner; +import software.amazon.lambda.durable.serde.SerDesStage; import software.amazon.lambda.durable.testing.LocalDurableTestRunner; import software.amazon.lambda.durable.testing.TestResult; import software.amazon.lambda.durable.testing.local.LocalMemoryExecutionClient; @@ -89,6 +91,7 @@ void pipelineReplaysStepWaitChildAndMapPayloadsFromFilesystem() throws Exception return childResult + "-" + pollResult + "-" + mapResult.results(); }, config) + .withInputSerDes(new JacksonSerDes()) .withOutputType(String.class); var result = runner.runUntilComplete("order"); @@ -170,6 +173,7 @@ void acceptsRawCallbackAndInvokeResultsAndOffloadsInvokePayload() throws Excepti "notify", "target-function", Map.of("approval", approval), String.class); }, config) + .withInputSerDes(new JacksonSerDes()) .withOutputType(String.class); var waitingForCallback = runner.run("input"); @@ -289,6 +293,7 @@ void repeatedGetUsesInvocationCacheForTheCompletePipeline() { return first.value(); }, config) + .withInputSerDes(new JacksonSerDes()) .withOutputType(String.class); var result = runner.runUntilComplete("cached"); @@ -330,6 +335,7 @@ void successfulRetryUsesTheProducingAttemptForResultSerialization() { }, stepConfig), config) + .withInputSerDes(new JacksonSerDes()) .withOutputType(String.class); var result = runner.runUntilComplete("value"); @@ -356,6 +362,7 @@ void customExceptionPayloadsRoundTripThroughFilesystem() throws Exception { }, stepConfig), config) + .withInputSerDes(new JacksonSerDes()) .withOutputType(String.class); var result = runner.runUntilComplete("input"); @@ -386,6 +393,7 @@ void nestedInvokeFailurePreservesProducerContextAcrossReplay() throws Exception } }, config) + .withInputSerDes(new JacksonSerDes()) .withOutputType(String.class); assertEquals(ExecutionStatus.PENDING, runner.run("input").getStatus()); @@ -436,6 +444,7 @@ void nestedCallbackFailurePreservesProducerContextAcrossReplay() throws Exceptio } }, config) + .withInputSerDes(new JacksonSerDes()) .withOutputType(String.class); assertEquals(ExecutionStatus.PENDING, runner.run("input").getStatus()); @@ -460,7 +469,20 @@ void nestedCallbackFailurePreservesProducerContextAcrossReplay() throws Exceptio } private SerDes filesystemPipeline() { - return new JacksonSerDes().then(FileSystemSerDes.stageBuilder(basePath).build()); + SerDesStage utf8 = new SerDesStage<>() { + @Override + public byte[] serialize(String value) { + return value.getBytes(StandardCharsets.UTF_8); + } + + @Override + public String deserialize(byte[] data) { + return new String(data, StandardCharsets.UTF_8); + } + }; + return new JacksonSerDes() + .then(utf8) + .then(FileSystemSerDes.stageBuilder(basePath).build()); } private static DurableExecutionInput durableInput( diff --git a/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/CloudDurableTestRunner.java b/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/CloudDurableTestRunner.java index 821bbea32..9d81a2b22 100644 --- a/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/CloudDurableTestRunner.java +++ b/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/CloudDurableTestRunner.java @@ -175,8 +175,8 @@ public CloudDurableTestRunner withSerDes(SerDes serDes) { *

The supplied SerDes is used exactly as configured, including every stage in a composable pipeline. Configure a * separate context-free input SerDes when the persisted SerDes requires a durable execution context, because that * context does not exist before the initial Lambda invocation. In that case the input SerDes must be a value codec, - * not a composable string-processing pipeline, because context-dependent persisted stages cannot distinguish which - * input stages produced an unframed external payload. + * not a composable pipeline, because context-dependent persisted stages cannot distinguish which input stages + * produced an unframed external payload. */ public CloudDurableTestRunner withInputSerDes(SerDes inputSerDes) { return new CloudDurableTestRunner<>( @@ -289,7 +289,7 @@ private String serializeInput(I input) { if (inputSerDes instanceof ComposableSerDes && serDes.requiresDurableContext()) { throw new IllegalStateException( "Initial input for a context-dependent persisted SerDes must use a value codec, not a composable " - + "string-processing pipeline"); + + "pipeline"); } return serializer.serialize(input); } diff --git a/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/LocalDurableTestRunner.java b/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/LocalDurableTestRunner.java index d4a50ae67..18eb4f9c1 100644 --- a/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/LocalDurableTestRunner.java +++ b/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/LocalDurableTestRunner.java @@ -6,6 +6,7 @@ import java.time.Instant; import java.util.ArrayList; import java.util.List; +import java.util.Objects; import java.util.UUID; import java.util.function.BiFunction; import software.amazon.awssdk.services.lambda.model.CheckpointUpdatedExecutionState; @@ -22,9 +23,8 @@ import software.amazon.lambda.durable.model.DurableExecutionInput; import software.amazon.lambda.durable.model.ExecutionStatus; import software.amazon.lambda.durable.plugin.DurableExecutionPlugin; +import software.amazon.lambda.durable.serde.ComposableSerDes; import software.amazon.lambda.durable.serde.SerDes; -import software.amazon.lambda.durable.serde.SerDesContext; -import software.amazon.lambda.durable.serde.SerDesPayloadKind; import software.amazon.lambda.durable.serde.SerDesRunner; import software.amazon.lambda.durable.testing.local.LocalMemoryExecutionClient; import software.amazon.lambda.durable.testing.local.OperationResult; @@ -43,6 +43,7 @@ public class LocalDurableTestRunner { private final TypeToken outputType; private final BiFunction handler; private final LocalMemoryExecutionClient storage; + private final SerDes inputSerDes; private final SerDes serDes; private final DurableConfig customerConfig; private final Instant executionStartTime = Instant.now(); @@ -56,10 +57,12 @@ private LocalDurableTestRunner( TypeToken inputType, TypeToken outputType, BiFunction handlerFn, - DurableConfig customerConfig) { + DurableConfig customerConfig, + SerDes inputSerDes) { this.inputType = inputType; this.outputType = outputType; this.handler = handlerFn; + this.inputSerDes = inputSerDes; this.storage = new LocalMemoryExecutionClient(); // Create config that uses customer's configuration but overrides the client with in-memory storage @@ -100,7 +103,7 @@ private LocalDurableTestRunner( */ public static LocalDurableTestRunner create( Class inputType, BiFunction handlerFn) { - return new LocalDurableTestRunner<>(TypeToken.get(inputType), null, handlerFn, null); + return new LocalDurableTestRunner<>(TypeToken.get(inputType), null, handlerFn, null, null); } /** @@ -121,7 +124,7 @@ public static LocalDurableTestRunner create( */ public static LocalDurableTestRunner create( TypeToken inputType, BiFunction handlerFn) { - return new LocalDurableTestRunner<>(inputType, null, handlerFn, null); + return new LocalDurableTestRunner<>(inputType, null, handlerFn, null, null); } /** @@ -137,7 +140,7 @@ public static LocalDurableTestRunner create( */ public static LocalDurableTestRunner create( Class inputType, BiFunction handlerFn, DurableConfig config) { - return new LocalDurableTestRunner<>(TypeToken.get(inputType), null, handlerFn, config); + return new LocalDurableTestRunner<>(TypeToken.get(inputType), null, handlerFn, config, null); } /** * Creates a LocalDurableTestRunner that uses a custom configuration. This allows the test runner to use custom @@ -175,7 +178,7 @@ public static LocalDurableTestRunner create( */ public static LocalDurableTestRunner create( TypeToken inputType, BiFunction handlerFn, DurableConfig config) { - return new LocalDurableTestRunner<>(inputType, null, handlerFn, config); + return new LocalDurableTestRunner<>(inputType, null, handlerFn, config, null); } /** @@ -191,7 +194,7 @@ public static LocalDurableTestRunner create( */ public static LocalDurableTestRunner create(Class inputType, DurableHandler handler) { return new LocalDurableTestRunner<>( - TypeToken.get(inputType), null, handler::handleRequest, handler.getConfiguration()); + TypeToken.get(inputType), null, handler::handleRequest, handler.getConfiguration(), null); } /** @@ -199,17 +202,33 @@ public static LocalDurableTestRunner create(Class inputType, Dur * a new runner instance. */ public LocalDurableTestRunner withDurableConfig(DurableConfig config) { - return new LocalDurableTestRunner<>(inputType, outputType, handler, config); + return new LocalDurableTestRunner<>(inputType, outputType, handler, config, inputSerDes); } /** Overrides the output type for this test runner. */ public LocalDurableTestRunner withOutputType(TypeToken outputType) { - return new LocalDurableTestRunner<>(inputType, outputType, handler, customerConfig); + return new LocalDurableTestRunner<>(inputType, outputType, handler, customerConfig, inputSerDes); } /** Overrides the output type for this test runner. */ public LocalDurableTestRunner withOutputType(Class outputType) { - return new LocalDurableTestRunner<>(inputType, TypeToken.get(outputType), handler, customerConfig); + return new LocalDurableTestRunner<>(inputType, TypeToken.get(outputType), handler, customerConfig, inputSerDes); + } + + /** + * Returns a new runner with a separate SerDes for the initial Lambda invocation payload. + * + *

Configure a context-free input value codec when the persisted SerDes requires durable context. This preserves + * production behavior: the initial external payload is decoded directly by the value codec, while the persisted + * pipeline processes payloads after the durable execution context exists. + */ + public LocalDurableTestRunner withInputSerDes(SerDes inputSerDes) { + return new LocalDurableTestRunner<>( + inputType, + outputType, + handler, + customerConfig, + Objects.requireNonNull(inputSerDes, "inputSerDes cannot be null")); } /** @@ -245,13 +264,13 @@ public LocalDurableTestRunner withOutputType(Class outputType) { * @return LocalDurableTestRunner configured with the handler's settings */ public static LocalDurableTestRunner create(TypeToken inputType, DurableHandler handler) { - return new LocalDurableTestRunner<>(inputType, null, handler::handleRequest, handler.getConfiguration()); + return new LocalDurableTestRunner<>(inputType, null, handler::handleRequest, handler.getConfiguration(), null); } /** Run a single invocation (may return PENDING if waiting/retrying). */ public TestResult run(I input) { var serDesRunner = new SerDesRunner(customerConfig.getSerDesExecutorService()); - var durableInput = createDurableInput(input, serDesRunner); + var durableInput = createDurableInput(input); var output = DurableExecutor.execute(durableInput, mockLambdaContext(), inputType, handler, customerConfig); @@ -350,11 +369,8 @@ public void stopChainedInvoke(String name, ErrorObject error) { storage.completeChainedInvoke(name, OperationResult.stopped(error)); } - private DurableExecutionInput createDurableInput(I input, SerDesRunner serDesRunner) { - var inputJson = serDesRunner.serialize( - serDes, - input, - SerDesContext.forExecution(executionArn, invocationId, executionName, SerDesPayloadKind.INPUT)); + private DurableExecutionInput createDurableInput(I input) { + var inputJson = serializeInput(input); var executionOp = Operation.builder() .id(invocationId) .name(executionName) @@ -383,6 +399,21 @@ private DurableExecutionInput createDurableInput(I input, SerDesRunner serDesRun updatedOperationIds); } + private String serializeInput(I input) { + var serializer = inputSerDes != null ? inputSerDes : serDes; + if (serializer.requiresDurableContext()) { + throw new IllegalStateException( + "Initial input SerDes requires a durable execution context; configure a context-free " + + "input SerDes with withInputSerDes(...)"); + } + if (inputSerDes instanceof ComposableSerDes && serDes.requiresDurableContext()) { + throw new IllegalStateException( + "Initial input for a context-dependent persisted SerDes must use a value codec, not a composable " + + "pipeline"); + } + return serializer.serialize(input); + } + private Context mockLambdaContext() { return null; // Minimal - tests don't need real Lambda context } diff --git a/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/LocalDurableTestRunnerTest.java b/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/LocalDurableTestRunnerTest.java index 8dc146f86..7cf89f041 100644 --- a/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/LocalDurableTestRunnerTest.java +++ b/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/LocalDurableTestRunnerTest.java @@ -5,6 +5,8 @@ import static org.junit.jupiter.api.Assertions.*; import static software.amazon.lambda.durable.TypeToken.get; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; import java.time.Duration; import java.time.Instant; import java.util.ArrayList; @@ -12,11 +14,16 @@ import java.util.List; import java.util.concurrent.atomic.AtomicInteger; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; import software.amazon.lambda.durable.DurableConfig; import software.amazon.lambda.durable.TypeToken; import software.amazon.lambda.durable.model.ExecutionStatus; import software.amazon.lambda.durable.plugin.DurableExecutionPlugin; import software.amazon.lambda.durable.plugin.InvocationInfo; +import software.amazon.lambda.durable.serde.FileSystemSerDes; +import software.amazon.lambda.durable.serde.JacksonSerDes; +import software.amazon.lambda.durable.serde.SerDes; +import software.amazon.lambda.durable.serde.SerDesStage; class LocalDurableTestRunnerTest { @@ -138,4 +145,80 @@ void checkpointedLargeOutputReplaysWithoutDuplicateExecutionOperation() { assertEquals(largeResult, replayResult.getResult()); assertEquals(1, stepExecutions.get()); } + + @Test + void contextDependentPersistedSerDesRequiresExplicitInputSerDes(@TempDir Path basePath) { + var config = DurableConfig.builder() + .withSerDes(new JacksonSerDes() + .then(FileSystemSerDes.stageBuilder(basePath).build())) + .build(); + var runner = LocalDurableTestRunner.create(String.class, (input, context) -> input, config); + + var failure = assertThrows(IllegalStateException.class, () -> runner.run("value")); + + assertTrue(failure.getMessage().contains("withInputSerDes")); + } + + @Test + void contextDependentPersistedSerDesRequiresValueCodecInput(@TempDir Path basePath) { + var config = DurableConfig.builder() + .withSerDes(new JacksonSerDes() + .then(FileSystemSerDes.stageBuilder(basePath).build())) + .build(); + var runner = LocalDurableTestRunner.create(String.class, (input, context) -> input, config) + .withInputSerDes(new JacksonSerDes().then(wrappingStage(new AtomicInteger()))); + + var failure = assertThrows(IllegalStateException.class, () -> runner.run("value")); + + assertTrue(failure.getMessage().contains("must use a value codec")); + } + + @Test + void initialInputBypassesStagesBeforeFileSystemSerDes(@TempDir Path basePath) { + var deserializeCalls = new AtomicInteger(); + var persistedSerDes = new JacksonSerDes() + .then(bytesStage(deserializeCalls)) + .then(FileSystemSerDes.stageBuilder(basePath).build()); + var config = DurableConfig.builder().withSerDes(persistedSerDes).build(); + var runner = LocalDurableTestRunner.create( + String.class, (input, context) -> input + ":" + deserializeCalls.get(), config) + .withInputSerDes(new JacksonSerDes()) + .withOutputType(String.class); + + var result = runner.run("value"); + + assertEquals(ExecutionStatus.SUCCEEDED, result.getStatus()); + assertEquals("value:0", result.getResult()); + } + + private static SerDes wrappingStage(AtomicInteger deserializeCalls) { + return new SerDes() { + @Override + public String serialize(Object value) { + return "<" + value + ">"; + } + + @Override + @SuppressWarnings("unchecked") + public T deserialize(String data, TypeToken typeToken) { + deserializeCalls.incrementAndGet(); + return (T) data.substring(1, data.length() - 1); + } + }; + } + + private static SerDesStage bytesStage(AtomicInteger deserializeCalls) { + return new SerDesStage<>() { + @Override + public byte[] serialize(String value) { + return value.getBytes(StandardCharsets.UTF_8); + } + + @Override + public String deserialize(byte[] data) { + deserializeCalls.incrementAndGet(); + return new String(data, StandardCharsets.UTF_8); + } + }; + } } diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/ComposableSerDes.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/ComposableSerDes.java index adbd4be38..5484f8cd8 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/ComposableSerDes.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/ComposableSerDes.java @@ -13,41 +13,46 @@ /** * An immutable SerDes processing pipeline. * - *

The first stage is the value codec. Every later stage must be a reversible string transformation. Serialization - * runs from first to last; deserialization runs from last to first. + *

The first stage is the value codec. Later {@link SerDesStage} instances may exchange arbitrary intermediate Java + * types. Serialization runs from first to last; deserialization runs from last to first. The final serialized value and + * the value returned to the value codec during deserialization must be strings. */ public final class ComposableSerDes implements SerDes { - private final List stages; + private final SerDes valueCodec; + private final List stages; - private ComposableSerDes(List stages) { - if (stages.isEmpty()) { - throw new IllegalArgumentException("ComposableSerDes requires at least one stage"); + private ComposableSerDes(SerDes valueCodec, List stages) { + this.valueCodec = Objects.requireNonNull(valueCodec, "valueCodec cannot be null"); + if (!stages.isEmpty() && valueCodec.isTerminalPipelineStage()) { + throw terminalStageFailure(0, valueCodec); } for (int index = 0; index < stages.size() - 1; index++) { - if (stages.get(index).isTerminalPipelineStage()) { - throw new IllegalArgumentException(String.format( - "SerDes pipeline stage %d (%s) must be the final stage", - index, stages.get(index).getClass().getName())); + if (isTerminal(stages.get(index))) { + throw terminalStageFailure(index + 1, stages.get(index)); } } this.stages = List.copyOf(stages); } /** - * Creates a pipeline with a value codec followed by zero or more string stages. + * Creates a pipeline with a value codec followed by zero or more typed stages. * * @param first the value codec - * @param remaining reversible string stages + * @param remaining reversible SerDes stages * @return an immutable pipeline */ public static ComposableSerDes of(SerDes first, SerDes... remaining) { Objects.requireNonNull(remaining, "remaining stages cannot be null"); - var stages = new ArrayList(); - addFlattened(stages, Objects.requireNonNull(first, "first stage cannot be null")); + var valueCodec = Objects.requireNonNull(first, "first stage cannot be null"); + var stages = new ArrayList(); + if (valueCodec instanceof ComposableSerDes composable) { + valueCodec = composable.valueCodec; + stages.addAll(composable.stages); + } Arrays.stream(remaining) .map(stage -> Objects.requireNonNull(stage, "pipeline stage cannot be null")) .forEach(stage -> addFlattened(stages, stage)); - return new ComposableSerDes(stages); + return new ComposableSerDes(valueCodec, stages); } /** @@ -60,36 +65,35 @@ public static Builder builder(SerDes valueCodec) { return new Builder(valueCodec); } - /** - * Returns the value codec at the start of this pipeline. - * - * @return the first pipeline stage - */ + /** Returns the value codec at the start of this pipeline. */ public SerDes getValueCodec() { - return stages.get(0); + return valueCodec; } @Override public boolean requiresDurableContext() { - return stages.stream().anyMatch(SerDes::requiresDurableContext); + return valueCodec.requiresDurableContext() || stages.stream().anyMatch(ComposableSerDes::requiresContext); } @Override public boolean isTerminalPipelineStage() { - return stages.get(stages.size() - 1).isTerminalPipelineStage(); + return stages.isEmpty() ? valueCodec.isTerminalPipelineStage() : isTerminal(stages.get(stages.size() - 1)); } - /** - * Returns a new pipeline with the supplied stage appended. - * - * @param stage the reversible string stage to append - * @return a new immutable pipeline - */ + /** Returns a new pipeline with the supplied stage appended. */ @Override public ComposableSerDes then(SerDes stage) { var combined = new ArrayList<>(stages); addFlattened(combined, Objects.requireNonNull(stage, "stage cannot be null")); - return new ComposableSerDes(combined); + return new ComposableSerDes(valueCodec, combined); + } + + /** Returns a new pipeline with the supplied typed stage appended. */ + @Override + public ComposableSerDes then(SerDesStage stage) { + var combined = new ArrayList<>(stages); + addFlattened(combined, Objects.requireNonNull(stage, "stage cannot be null")); + return new ComposableSerDes(valueCodec, combined); } @Override @@ -97,11 +101,15 @@ public String serialize(Object value) { if (value == null) { return null; } - String current = invokeSerialize(stages.get(0), value, 0); - for (int index = 1; index < stages.size(); index++) { - current = invokeSerialize(stages.get(index), current, index); + Object current = invokeValueCodecSerialize(valueCodec, value); + for (int index = 0; index < stages.size(); index++) { + current = invokeStageSerialize(stages.get(index), current, index + 1); + } + if (!(current instanceof String serialized)) { + throw new SerDesException( + "SerDes pipeline final stage returned " + current.getClass().getName() + " instead of String"); } - return current; + return serialized; } @Override @@ -110,20 +118,50 @@ public T deserialize(String data, TypeToken typeToken) { return null; } Objects.requireNonNull(typeToken, "typeToken cannot be null"); - String current = data; - for (int index = stages.size() - 1; index > 0; index--) { - var decoded = invokeStringStageDeserialize(stages.get(index), current, index); + Object current = data; + for (int index = stages.size() - 1; index >= 0; index--) { + var decoded = invokeStageDeserialize(stages.get(index), current, index + 1); current = decoded.value(); if (decoded.skipRemainingStages()) { break; } } - return invokeDeserialize(stages.get(0), current, typeToken, 0); + if (!(current instanceof String valueCodecInput)) { + throw new SerDesException("SerDes pipeline produced " + + current.getClass().getName() + + " instead of String for the value codec"); + } + return invokeValueCodecDeserialize(valueCodec, valueCodecInput, typeToken); } - private static SerDesStageResult invokeStringStageDeserialize(SerDes stage, String data, int index) { + @SuppressWarnings("unchecked") + private static Object invokeStageSerialize(Object stage, Object value, int index) { try { - var result = stage.deserializePipelineStage(data); + var result = stage instanceof SerDes serDes + ? serDes.serialize(value) + : ((SerDesStage) stage).serialize(value); + if (result == null) { + throw new SerDesException("Stage returned null for a non-null value"); + } + return result; + } catch (Throwable failure) { + throw stageFailure(index, stage, "serialize", failure); + } + } + + @SuppressWarnings("unchecked") + private static SerDesStageResult invokeStageDeserialize(Object stage, Object data, int index) { + try { + SerDesStageResult result; + if (stage instanceof SerDes serDes) { + if (!(data instanceof String stringData)) { + throw new SerDesException("SerDes stage requires String input but received " + + data.getClass().getName()); + } + result = serDes.deserializePipelineStage(stringData); + } else { + result = ((SerDesStage) stage).deserializePipelineStage(data); + } if (result == null) { throw new SerDesException("Stage returned a null pipeline result"); } @@ -133,27 +171,27 @@ private static SerDesStageResult invokeStringStageDeserialize(SerDes stage, Stri } } - private static String invokeSerialize(SerDes stage, Object value, int index) { + private static String invokeValueCodecSerialize(SerDes valueCodec, Object value) { try { - var result = stage.serialize(value); + var result = valueCodec.serialize(value); if (result == null) { - throw new SerDesException("Stage returned null for a non-null value"); + throw new SerDesException("Value codec returned null for a non-null value"); } return result; } catch (Throwable failure) { - throw stageFailure(index, stage, "serialize", failure); + throw stageFailure(0, valueCodec, "serialize", failure); } } - private static T invokeDeserialize(SerDes stage, String data, TypeToken typeToken, int index) { + private static T invokeValueCodecDeserialize(SerDes valueCodec, String data, TypeToken typeToken) { try { - return stage.deserialize(data, typeToken); + return valueCodec.deserialize(data, typeToken); } catch (Throwable failure) { - throw stageFailure(index, stage, "deserialize", failure); + throw stageFailure(0, valueCodec, "deserialize", failure); } } - private static RuntimeException stageFailure(int index, SerDes stage, String action, Throwable failure) { + private static RuntimeException stageFailure(int index, Object stage, String action, Throwable failure) { if (failure instanceof Error error) { throw error; } @@ -166,36 +204,65 @@ private static RuntimeException stageFailure(int index, SerDes stage, String act return new SerDesException(message, failure); } - private static void addFlattened(List target, SerDes stage) { + private static IllegalArgumentException terminalStageFailure(int index, Object stage) { + return new IllegalArgumentException(String.format( + "SerDes pipeline stage %d (%s) must be the final stage", + index, stage.getClass().getName())); + } + + private static boolean requiresContext(Object stage) { + return stage instanceof SerDes serDes + ? serDes.requiresDurableContext() + : ((SerDesStage) stage).requiresDurableContext(); + } + + private static boolean isTerminal(Object stage) { + return stage instanceof SerDes serDes + ? serDes.isTerminalPipelineStage() + : ((SerDesStage) stage).isTerminalPipelineStage(); + } + + private static void addFlattened(List target, SerDes stage) { if (stage instanceof ComposableSerDes composable) { + target.add(composable.valueCodec); target.addAll(composable.stages); } else { target.add(stage); } } + private static void addFlattened(List target, SerDesStage stage) { + target.add(stage); + } + /** Builder for an immutable {@link ComposableSerDes}. */ public static final class Builder { - private final List stages = new ArrayList<>(); + private SerDes valueCodec; + private final List stages = new ArrayList<>(); private Builder(SerDes valueCodec) { - addFlattened(stages, Objects.requireNonNull(valueCodec, "valueCodec cannot be null")); + this.valueCodec = Objects.requireNonNull(valueCodec, "valueCodec cannot be null"); + if (valueCodec instanceof ComposableSerDes composable) { + this.valueCodec = composable.valueCodec; + stages.addAll(composable.stages); + } } - /** - * Appends a reversible string stage. - * - * @param stage the stage to append - * @return this builder - */ + /** Appends a reversible typed stage. */ public Builder then(SerDes stage) { addFlattened(stages, Objects.requireNonNull(stage, "stage cannot be null")); return this; } + /** Appends a reversible typed stage. */ + public Builder then(SerDesStage stage) { + addFlattened(stages, Objects.requireNonNull(stage, "stage cannot be null")); + return this; + } + /** Returns the immutable pipeline. */ public ComposableSerDes build() { - return new ComposableSerDes(stages); + return new ComposableSerDes(valueCodec, stages); } } } diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java index 618049fa0..903d50e1b 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java @@ -16,6 +16,8 @@ import java.nio.file.StandardCopyOption; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; +import java.util.Arrays; +import java.util.Base64; import java.util.HexFormat; import java.util.LinkedHashMap; import java.util.Map; @@ -38,6 +40,7 @@ */ public final class FileSystemSerDes implements SerDes { private static final String ENVELOPE_MARKER = "__durable_execution_filesystem_serdes"; + private static final String PAYLOAD_TYPE_FIELD = "payloadType"; private static final int ENVELOPE_VERSION = 1; private static final int CHECKPOINT_ENVELOPE_LIMIT_BYTES = 256 * 1024 - 1024; private static final ObjectMapper ENVELOPE_MAPPER = new ObjectMapper(); @@ -72,10 +75,12 @@ public static Builder builder(Path basePath) { } /** - * Creates a filesystem string-stage builder for use in a composable SerDes pipeline. + * Creates a filesystem terminal-stage builder for use in a composable SerDes pipeline. + * + *

Stage mode accepts {@link String} and {@code byte[]} values. * * @param basePath durable shared filesystem root - * @return a string-stage builder + * @return a terminal-stage builder */ public static Builder stageBuilder(Path basePath) { return new Builder(basePath, true); @@ -87,24 +92,24 @@ public String serialize(Object value) { return null; } var context = requireContext(); - var serialized = serializeValue(value); + var payload = serializeValue(value); if (storageMode == FileSystemStorageMode.OVERFLOW) { - var inlineEnvelope = encodeEnvelope(serialized, null, null, context); + var inlineEnvelope = encodeEnvelope(payload, null, null, context); if (fitsCheckpoint(inlineEnvelope)) { return inlineEnvelope; } } - var file = resolvePayloadPath(serialized, context); + var file = resolvePayloadPath(payload, context); var preview = generatePreview(value, context); - var fileEnvelope = encodeEnvelope(null, file, preview, context); + var fileEnvelope = encodeEnvelope(payload.withoutData(), file, preview, context); if (!fitsCheckpoint(fileEnvelope)) { throw new SerDesException("Filesystem SerDes envelope exceeds the checkpoint payload limit for entity '" + context.entityId() + "'"); } try { - writePayload(serialized, file); + writePayload(payload, file); return fileEnvelope; } catch (IOException e) { throw new RetryableSerDesException( @@ -119,16 +124,20 @@ public T deserialize(String data, TypeToken typeToken) { } Objects.requireNonNull(typeToken, "typeToken cannot be null"); var context = requireContext(); - var serialized = resolveSerializedPayload(data, context).serialized(); + var serialized = resolveSerializedPayload(data, context).value(); if (stageMode) { - if (!TypeToken.get(String.class).equals(typeToken)) { - throw new SerDesException("FileSystemSerDes stage can only deserialize to String"); + if ((TypeToken.get(String.class).equals(typeToken) && serialized instanceof String) + || (TypeToken.get(byte[].class).equals(typeToken) && serialized instanceof byte[])) { + @SuppressWarnings("unchecked") + var value = (T) serialized; + return value; } - @SuppressWarnings("unchecked") - var value = (T) serialized; - return value; + throw new SerDesException("FileSystemSerDes stage payload type does not match requested type " + typeToken); + } + if (!(serialized instanceof String serializedString)) { + throw new SerDesException("Standalone FileSystemSerDes cannot decode a binary stage payload"); } - return delegate.deserialize(serialized, typeToken); + return delegate.deserialize(serializedString, typeToken); } @Override @@ -139,8 +148,8 @@ public SerDesStageResult deserializePipelineStage(String data) { var context = requireContext(); var resolved = resolveSerializedPayload(data, context); return resolved.external() - ? SerDesStageResult.decodeWithValueCodec(resolved.serialized()) - : SerDesStageResult.continueWith(resolved.serialized()); + ? SerDesStageResult.decodeWithValueCodec((String) resolved.value()) + : SerDesStageResult.continueWith(resolved.value()); } @Override @@ -153,18 +162,22 @@ public boolean isTerminalPipelineStage() { return true; } - private String serializeValue(Object value) { + private SerializedPayload serializeValue(Object value) { if (stageMode) { - if (!(value instanceof String stringValue)) { - throw new SerDesException("FileSystemSerDes stage can only serialize String values"); + if (value instanceof String stringValue) { + return SerializedPayload.fromString(stringValue); + } + if (value instanceof byte[] bytes) { + return SerializedPayload.fromBytes(bytes); } - return stringValue; + throw new SerDesException("FileSystemSerDes stage supports String and byte[] values, but received " + + value.getClass().getName()); } var serialized = delegate.serialize(value); if (serialized == null) { throw new SerDesException("Delegate SerDes returned null for a non-null value"); } - return serialized; + return SerializedPayload.fromString(serialized); } private ResolvedPayload resolveSerializedPayload(String data, SerDesContext context) { @@ -197,14 +210,27 @@ private ResolvedPayload resolveSerializedPayload(String data, SerDesContext cont var hasData = envelope.has("data") && envelope.get("data").isTextual(); var hasFile = envelope.has("file") && envelope.get("file").isTextual(); + var payloadType = payloadType(envelope, context); var owner = payloadOwner(envelope, context); if (hasData) { - return new ResolvedPayload(envelope.get("data").textValue(), false); + try { + return new ResolvedPayload( + SerializedPayload.fromInlineValue( + payloadType, envelope.get("data").textValue()) + .value(), + false); + } catch (IllegalArgumentException e) { + throw malformedEnvelope(context, e); + } } - return new ResolvedPayload(readPayload(envelope.get("file").textValue(), owner, context), false); + return new ResolvedPayload( + readPayload(envelope.get("file").textValue(), payloadType, owner, context) + .value(), + false); } - private String readPayload(String fileValue, PayloadOwner owner, SerDesContext context) { + private SerializedPayload readPayload( + String fileValue, PayloadType payloadType, PayloadOwner owner, SerDesContext context) { var file = Path.of(fileValue).toAbsolutePath().normalize(); validatePayloadPath(file, owner); try { @@ -217,7 +243,7 @@ private String readPayload(String fileValue, PayloadOwner owner, SerDesContext c || !realFile.equals(file.toRealPath(LinkOption.NOFOLLOW_LINKS))) { throw new SerDesException("Filesystem SerDes file does not resolve to the expected payload path"); } - var serialized = Files.readString(realFile, StandardCharsets.UTF_8); + var serialized = new SerializedPayload(payloadType, Files.readAllBytes(realFile)); var expectedFileName = payloadFileName(serialized, owner.entityId()); if (!realFile.getFileName().toString().equals(expectedFileName)) { throw new SerDesException("Filesystem SerDes file content does not match its content-addressed path"); @@ -235,11 +261,23 @@ private void validatePayloadPath(Path file, PayloadOwner owner) { if (fileName == null || file.getParent() == null || !file.getParent().equals(expectedDirectory) - || !fileName.toString().matches(Pattern.quote(encode(owner.entityId())) + "-[0-9a-f]{64}\\.json")) { + || !fileName.toString().matches(Pattern.quote(encode(owner.entityId())) + "-[0-9a-f]{64}\\.payload")) { throw new SerDesException("Filesystem SerDes file is not valid for its declared durable entity"); } } + private static PayloadType payloadType(JsonNode envelope, SerDesContext context) { + var node = envelope.get(PAYLOAD_TYPE_FIELD); + if (node == null || !node.isTextual()) { + throw malformedEnvelope(context, null); + } + try { + return PayloadType.valueOf(node.textValue()); + } catch (IllegalArgumentException e) { + throw malformedEnvelope(context, e); + } + } + private static PayloadOwner payloadOwner(JsonNode envelope, SerDesContext context) { var hasOwnerArn = envelope.has("ownerDurableExecutionArn") && envelope.get("ownerDurableExecutionArn").isTextual(); @@ -292,7 +330,10 @@ private static boolean isFilesystemEnvelope(JsonNode envelope) { || envelope.get("ownerDurableExecutionArn").textValue().isBlank() || !envelope.has("ownerEntityId") || !envelope.get("ownerEntityId").isTextual() - || envelope.get("ownerEntityId").textValue().isBlank()) { + || envelope.get("ownerEntityId").textValue().isBlank() + || !envelope.has(PAYLOAD_TYPE_FIELD) + || !envelope.get(PAYLOAD_TYPE_FIELD).isTextual() + || !isPayloadType(envelope.get(PAYLOAD_TYPE_FIELD).textValue())) { return false; } @@ -306,7 +347,16 @@ private static boolean isFilesystemEnvelope(JsonNode envelope) { if (hasPreview && (hasData || !envelope.get("preview").isObject())) { return false; } - return envelope.size() == (hasPreview ? 5 : 4); + return envelope.size() == (hasPreview ? 6 : 5); + } + + private static boolean isPayloadType(String value) { + try { + PayloadType.valueOf(value); + return true; + } catch (IllegalArgumentException e) { + return false; + } } private static boolean hasFilesystemMarker(JsonNode envelope) { @@ -332,13 +382,15 @@ private static SerDesException unsupportedEnvelopeVersion(SerDesContext context, + "'"); } - private String encodeEnvelope(String data, Path file, Map preview, SerDesContext context) { + private String encodeEnvelope( + SerializedPayload payload, Path file, Map preview, SerDesContext context) { var envelope = new LinkedHashMap(); envelope.put(ENVELOPE_MARKER, ENVELOPE_VERSION); envelope.put("ownerDurableExecutionArn", context.durableExecutionArn()); envelope.put("ownerEntityId", context.entityId()); - if (data != null) { - envelope.put("data", data); + envelope.put(PAYLOAD_TYPE_FIELD, payload.type().name()); + if (payload.hasData()) { + envelope.put("data", payload.inlineValue()); } else { envelope.put("file", file.toString()); if (preview != null) { @@ -382,9 +434,9 @@ private SerDesContext requireContext() { return context; } - private Path resolvePayloadPath(String serialized, SerDesContext context) { + private Path resolvePayloadPath(SerializedPayload payload, SerDesContext context) { var directory = resolveExecutionDirectory(context.durableExecutionArn()); - var fileName = payloadFileName(serialized, context.entityId()); + var fileName = payloadFileName(payload, context.entityId()); var file = directory.resolve(fileName).normalize(); if (!file.startsWith(directory)) { throw new SerDesException("Resolved filesystem payload path is outside the execution directory"); @@ -392,11 +444,11 @@ private Path resolvePayloadPath(String serialized, SerDesContext context) { return file; } - private String payloadFileName(String serialized, String entityId) { - return encode(entityId) + "-" + sha256(serialized) + ".json"; + private String payloadFileName(SerializedPayload payload, String entityId) { + return encode(entityId) + "-" + sha256(payload.data()) + ".payload"; } - private void writePayload(String serialized, Path file) throws IOException { + private void writePayload(SerializedPayload payload, Path file) throws IOException { var directory = file.getParent(); var realBasePath = createDirectoriesWithoutSymbolicLinks(directory); rejectSymbolicLinks(file); @@ -405,8 +457,8 @@ private void writePayload(String serialized, Path file) throws IOException { throw new SerDesException("Filesystem SerDes directory resolves outside the configured base path"); } if (Files.exists(file, LinkOption.NOFOLLOW_LINKS)) { - var existing = Files.readString(file, StandardCharsets.UTF_8); - if (!existing.equals(serialized)) { + var existing = Files.readAllBytes(file); + if (!Arrays.equals(existing, payload.data())) { throw new SerDesException("Filesystem SerDes content-addressed file contains unexpected data"); } return; @@ -414,7 +466,7 @@ private void writePayload(String serialized, Path file) throws IOException { var temporary = Files.createTempFile(directory, file.getFileName().toString(), ".tmp"); try { - Files.writeString(temporary, serialized, StandardCharsets.UTF_8); + Files.write(temporary, payload.data()); moveWithoutReplacement(temporary, file); } finally { Files.deleteIfExists(temporary); @@ -530,8 +582,12 @@ private String encode(String value) { } private static String sha256(String value) { + return sha256(value.getBytes(StandardCharsets.UTF_8)); + } + + private static String sha256(byte[] value) { try { - var digest = MessageDigest.getInstance("SHA-256").digest(value.getBytes(StandardCharsets.UTF_8)); + var digest = MessageDigest.getInstance("SHA-256").digest(value); return HexFormat.of().formatHex(digest); } catch (NoSuchAlgorithmException e) { throw new IllegalStateException("SHA-256 is unavailable", e); @@ -540,7 +596,67 @@ private static String sha256(String value) { private record PayloadOwner(String durableExecutionArn, String entityId) {} - private record ResolvedPayload(String serialized, boolean external) {} + private record ResolvedPayload(Object value, boolean external) {} + + private enum PayloadType { + STRING, + BYTES + } + + private record SerializedPayload(PayloadType type, byte[] data) { + private SerializedPayload { + Objects.requireNonNull(type, "type cannot be null"); + data = data == null ? null : data.clone(); + } + + private static SerializedPayload fromString(String value) { + return new SerializedPayload(PayloadType.STRING, value.getBytes(StandardCharsets.UTF_8)); + } + + private static SerializedPayload fromBytes(byte[] value) { + return new SerializedPayload(PayloadType.BYTES, value); + } + + private static SerializedPayload fromInlineValue(PayloadType type, String value) { + return switch (type) { + case STRING -> fromString(value); + case BYTES -> fromBytes(Base64.getDecoder().decode(value)); + }; + } + + @Override + public byte[] data() { + return data == null ? null : data.clone(); + } + + private boolean hasData() { + return data != null; + } + + private SerializedPayload withoutData() { + return new SerializedPayload(type, null); + } + + private String inlineValue() { + if (data == null) { + throw new IllegalStateException("Serialized payload does not contain inline data"); + } + return switch (type) { + case STRING -> new String(data, StandardCharsets.UTF_8); + case BYTES -> Base64.getEncoder().encodeToString(data); + }; + } + + private Object value() { + if (data == null) { + throw new IllegalStateException("Serialized payload does not contain data"); + } + return switch (type) { + case STRING -> new String(data, StandardCharsets.UTF_8); + case BYTES -> data.clone(); + }; + } + } /** Builder for {@link FileSystemSerDes}. */ public static final class Builder { diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDes.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDes.java index 648391ef7..fa61b83bf 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDes.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDes.java @@ -6,9 +6,10 @@ import software.amazon.lambda.durable.exception.SerDesException; /** - * Interface for serialization and deserialization of objects. + * Interface for serialization and deserialization of objects at the persisted string boundary. * - *

Implementations must support both simple types via {@link Class} and complex generic types via {@link TypeToken}. + *

Implementations can also be used as string-producing stages in a {@link ComposableSerDes}. Use {@link SerDesStage} + * for typed intermediate transformations that produce or consume non-string values. */ public interface SerDes { /** @@ -39,24 +40,16 @@ public interface SerDes { T deserialize(String data, TypeToken typeToken); /** - * Deserializes this SerDes when it is used as a string-processing pipeline stage. + * Deserializes this SerDes when it is used as an intermediate pipeline stage. * - *

Most stages should use the default result, which continues reverse processing through earlier string stages. - * Boundary stages may return {@link SerDesStageResult#decodeWithValueCodec(String)} when the input originated - * outside the configured pipeline and must be decoded directly by the value codec. - * - * @param data the non-null string supplied to this stage - * @return the stage result + *

The default uses {@link String} as the intermediate value type, preserving existing SerDes behavior. */ default SerDesStageResult deserializePipelineStage(String data) { - Object result = deserialize(data, TypeToken.get(String.class)); + var result = deserialize(data, TypeToken.get(String.class)); if (result == null) { throw new SerDesException("Stage returned null for non-null data"); } - if (!(result instanceof String stringResult)) { - throw new SerDesException("String stage returned a non-string value"); - } - return SerDesStageResult.continueWith(stringResult); + return SerDesStageResult.continueWith(result); } /** @@ -83,12 +76,23 @@ default boolean isTerminalPipelineStage() { * Returns an immutable processing pipeline that invokes this SerDes followed by {@code nextStage} when serializing * and in reverse order when deserializing. * - *

This SerDes is the value codec. The next stage must accept and return strings. + *

This SerDes is the value codec. Intermediate stages may transform values into arbitrary Java types, but the + * final stage must return a string for persistence. * - * @param nextStage the reversible string-processing stage to append + * @param nextStage the reversible stage to append * @return a composable SerDes pipeline */ default ComposableSerDes then(SerDes nextStage) { return ComposableSerDes.of(this, nextStage); } + + /** + * Returns an immutable processing pipeline with a typed intermediate stage appended. + * + * @param nextStage the reversible typed stage to append + * @return a composable SerDes pipeline + */ + default ComposableSerDes then(SerDesStage nextStage) { + return ComposableSerDes.builder(this).then(nextStage).build(); + } } diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesStage.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesStage.java new file mode 100644 index 000000000..9662e87f3 --- /dev/null +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesStage.java @@ -0,0 +1,55 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.serde; + +/** + * A reversible typed stage in a {@link ComposableSerDes} pipeline. + * + *

Serialization maps {@code I} to {@code O}; deserialization applies the inverse mapping. Intermediate stages may + * use any Java types. The complete pipeline must still produce a {@link String} at its checkpoint boundary because that + * is the persisted representation required by {@link SerDes}. + * + * @param the stage input type during serialization + * @param the stage output type during serialization + */ +public interface SerDesStage { + /** + * Applies this stage during forward serialization. + * + * @param value the non-null input value + * @return the non-null transformed value + */ + O serialize(I value); + + /** + * Reverses this stage during deserialization. + * + * @param data the non-null serialized form produced by this stage + * @return the non-null value expected by the preceding stage + */ + I deserialize(O data); + + /** + * Reverses this stage with control over external-boundary processing. + * + *

Most stages should use the default result. Boundary stages may return + * {@link SerDesStageResult#decodeWithValueCodec(String)} when the input originated outside the configured pipeline + * and should bypass the remaining intermediate stages. + * + * @param data the non-null serialized form produced by this stage + * @return the stage result + */ + default SerDesStageResult deserializePipelineStage(O data) { + return SerDesStageResult.continueWith(deserialize(data)); + } + + /** Returns whether this stage requires an SDK-managed durable execution context. */ + default boolean requiresDurableContext() { + return false; + } + + /** Returns whether this stage must be the final stage in a composable pipeline. */ + default boolean isTerminalPipelineStage() { + return false; + } +} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesStageResult.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesStageResult.java index b28190db8..ed7c3a939 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesStageResult.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesStageResult.java @@ -5,23 +5,23 @@ import java.util.Objects; /** - * Result returned when a {@link SerDes} is used as a string-processing stage in a {@link ComposableSerDes}. + * Result returned when a {@link SerDesStage} is reversed in a {@link ComposableSerDes}. * - * @param value the string produced by the stage - * @param skipRemainingStages whether deserialization should skip the remaining string stages and decode {@code value} - * directly with the pipeline's value codec + * @param value the non-null value produced by the stage + * @param skipRemainingStages whether deserialization should skip the remaining intermediate stages and decode + * {@code value} directly with the pipeline's value codec */ -public record SerDesStageResult(String value, boolean skipRemainingStages) { +public record SerDesStageResult(Object value, boolean skipRemainingStages) { public SerDesStageResult { Objects.requireNonNull(value, "value cannot be null"); } - /** Continues reverse processing through the remaining string stages. */ - public static SerDesStageResult continueWith(String value) { + /** Continues reverse processing through the remaining intermediate stages. */ + public static SerDesStageResult continueWith(Object value) { return new SerDesStageResult(value, false); } - /** Skips the remaining string stages and decodes the value directly with the pipeline's value codec. */ + /** Skips the remaining intermediate stages and decodes the value directly with the pipeline's value codec. */ public static SerDesStageResult decodeWithValueCodec(String value) { return new SerDesStageResult(value, true); } diff --git a/sdk/src/test/java/software/amazon/lambda/durable/serde/ComposableSerDesTest.java b/sdk/src/test/java/software/amazon/lambda/durable/serde/ComposableSerDesTest.java index 9cd7a2b28..1fa9927f4 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/serde/ComposableSerDesTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/serde/ComposableSerDesTest.java @@ -9,7 +9,9 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.Base64; import java.util.List; import java.util.concurrent.atomic.AtomicInteger; import org.junit.jupiter.api.Test; @@ -34,6 +36,45 @@ void serializesForwardAndDeserializesInReverse() { assertEquals(List.of("first-serialize", "second-serialize", "second-deserialize", "first-deserialize"), calls); } + @Test + void supportsTypedIntermediateValues() { + var calls = new ArrayList(); + SerDesStage utf8 = new SerDesStage<>() { + @Override + public byte[] serialize(String value) { + calls.add("bytes-serialize"); + return value.getBytes(StandardCharsets.UTF_8); + } + + @Override + public String deserialize(byte[] data) { + calls.add("bytes-deserialize"); + return new String(data, StandardCharsets.UTF_8); + } + }; + SerDesStage base64 = new SerDesStage<>() { + @Override + public String serialize(byte[] value) { + calls.add("base64-serialize"); + return Base64.getEncoder().encodeToString(value); + } + + @Override + public byte[] deserialize(String data) { + calls.add("base64-deserialize"); + return Base64.getDecoder().decode(data); + } + }; + var pipeline = new JacksonSerDes().then(utf8).then(base64); + + var serialized = pipeline.serialize("value"); + var deserialized = pipeline.deserialize(serialized, TypeToken.get(String.class)); + + assertEquals(Base64.getEncoder().encodeToString("\"value\"".getBytes(StandardCharsets.UTF_8)), serialized); + assertEquals("value", deserialized); + assertEquals(List.of("bytes-serialize", "base64-serialize", "base64-deserialize", "bytes-deserialize"), calls); + } + @Test void factoryBuilderAndThenFlattenNestedPipelines() { var calls = new ArrayList(); @@ -159,7 +200,7 @@ public boolean isTerminalPipelineStage() { } @Test - void rejectsNullAndNonStringIntermediateValuesWithStageMetadata() { + void rejectsNullIntermediateAndNonStringBoundaryValues() { var nullStage = new SerDes() { @Override public String serialize(Object value) { @@ -176,23 +217,44 @@ public T deserialize(String data, TypeToken typeToken) { assertTrue(nullFailure.getMessage().contains("stage 1")); assertTrue(nullFailure.getMessage().contains(nullStage.getClass().getName())); - var nonStringStage = new SerDes() { + SerDesStage nonStringFinalStage = new SerDesStage<>() { @Override - public String serialize(Object value) { - return value.toString(); + public Integer serialize(String value) { + return value.length(); } @Override - @SuppressWarnings("unchecked") - public T deserialize(String data, TypeToken typeToken) { - return (T) Integer.valueOf(42); + public String deserialize(Integer data) { + return "x".repeat(data); } }; var typeFailure = assertThrows( SerDesException.class, - () -> new JacksonSerDes().then(nonStringStage).deserialize("\"value\"", TypeToken.get(String.class))); - assertTrue(typeFailure.getMessage().contains("stage 1")); - assertTrue(typeFailure.getCause().getMessage().contains("non-string")); + () -> new JacksonSerDes().then(nonStringFinalStage).serialize("value")); + assertTrue(typeFailure.getMessage().contains("final stage")); + assertTrue(typeFailure.getMessage().contains(Integer.class.getName())); + } + + @Test + void incompatibleTypedStagesFailWithStageMetadata() { + SerDesStage integerStage = new SerDesStage<>() { + @Override + public String serialize(Integer value) { + return value.toString(); + } + + @Override + public Integer deserialize(String data) { + return Integer.valueOf(data); + } + }; + + var failure = assertThrows( + SerDesException.class, + () -> new JacksonSerDes().then(integerStage).serialize("value")); + + assertTrue(failure.getMessage().contains("stage 1")); + assertInstanceOf(ClassCastException.class, failure.getCause()); } @Test diff --git a/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java b/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java index 725728863..0dfb14900 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 package software.amazon.lambda.durable.serde; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertInstanceOf; @@ -10,6 +11,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import com.fasterxml.jackson.databind.ObjectMapper; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.util.Map; @@ -40,6 +42,7 @@ void standaloneModeWritesDelegatePayloadAndReplaysIt() throws Exception { var file = Path.of(json.get("file").textValue()); assertEquals(1, json.get(ENVELOPE_MARKER).intValue()); + assertEquals("STRING", json.get("payloadType").textValue()); assertTrue(file.startsWith(basePath.resolve("orders/execution-1/invocation-1"))); assertEquals("{\"id\":42}", Files.readString(file)); assertEquals( @@ -69,6 +72,50 @@ void stageModeComposesWithValueCodec() throws Exception { assertThrows(IllegalArgumentException.class, () -> pipeline.then(wrappingStage())); } + @Test + void stageModeStoresAndRestoresBinaryIntermediateValues() throws Exception { + SerDesStage utf8 = new SerDesStage<>() { + @Override + public byte[] serialize(String value) { + return value.getBytes(StandardCharsets.UTF_8); + } + + @Override + public String deserialize(byte[] data) { + return new String(data, StandardCharsets.UTF_8); + } + }; + var stage = FileSystemSerDes.stageBuilder(basePath).build(); + var pipeline = new JacksonSerDes().then(utf8).then(stage); + var runner = new SerDesRunner(null); + + var envelope = runner.serialize(pipeline, Map.of("id", 42), context()); + var json = MAPPER.readTree(envelope); + var file = Path.of(json.get("file").textValue()); + + assertEquals("BYTES", json.get("payloadType").textValue()); + assertEquals("{\"id\":42}", new String(Files.readAllBytes(file), StandardCharsets.UTF_8)); + assertEquals( + Map.of("id", 42), + runner.deserialize(pipeline, envelope, new TypeToken>() {}, context())); + } + + @Test + void overflowModeKeepsSmallBinaryPayloadsInline() throws Exception { + var stage = FileSystemSerDes.stageBuilder(basePath) + .storageMode(FileSystemStorageMode.OVERFLOW) + .build(); + var runner = new SerDesRunner(null); + var value = new byte[] {0, 1, 2, -1}; + + var envelope = runner.serialize(stage, value, context()); + var json = MAPPER.readTree(envelope); + + assertEquals("BYTES", json.get("payloadType").textValue()); + assertTrue(json.has("data")); + assertArrayEquals(value, runner.deserialize(stage, envelope, TypeToken.get(byte[].class), context())); + } + @Test void overflowModeKeepsSmallPayloadInlineAndWritesLargePayload() throws Exception { var serDes = FileSystemSerDes.builder(basePath) @@ -111,7 +158,7 @@ void hashEncodingUsesFixedLengthSegments() throws Exception { var file = Path.of(MAPPER.readTree(envelope).get("file").textValue()); assertEquals(64, file.getParent().getFileName().toString().length()); - assertEquals(134, file.getFileName().toString().length()); + assertEquals(137, file.getFileName().toString().length()); assertFalse(file.toString().contains("operation")); } @@ -203,6 +250,21 @@ void rejectsUnsupportedEnvelopeVersionsAtExternalBoundaries() { assertTrue(failure.getCause().getMessage().contains("Unsupported filesystem SerDes envelope version 2")); } + @Test + void rejectsMalformedBinaryInlinePayload() { + var envelope = "{\"__durable_execution_filesystem_serdes\":1," + + "\"ownerDurableExecutionArn\":\"" + + ARN + + "\",\"ownerEntityId\":\"1\",\"payloadType\":\"BYTES\",\"data\":\"not-base64!\"}"; + + assertThrows(SerDesException.class, () -> new SerDesRunner(null) + .deserialize( + FileSystemSerDes.stageBuilder(basePath).build(), + envelope, + TypeToken.get(byte[].class), + context())); + } + @Test void rejectsOutOfRangeEnvelopeVersionsWithoutTruncation() { var serDes = FileSystemSerDes.builder(basePath).build(); @@ -371,8 +433,17 @@ void rejectsExecutionPathsOutsideConfiguredBasePath() { private static String envelopeWithFile(String file) { try { - return MAPPER.writeValueAsString( - Map.of(ENVELOPE_MARKER, 1, "file", file, "ownerDurableExecutionArn", ARN, "ownerEntityId", "1")); + return MAPPER.writeValueAsString(Map.of( + ENVELOPE_MARKER, + 1, + "file", + file, + "ownerDurableExecutionArn", + ARN, + "ownerEntityId", + "1", + "payloadType", + "STRING")); } catch (Exception e) { throw new AssertionError(e); } From 24ec5634f1e21d38b4534ac20e3366fde50f7487 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Tue, 25 Aug 2026 05:50:19 +0000 Subject: [PATCH 17/56] refactor: move SerDes chaining to stages --- docs/adr/005-filesystem-serdes.md | 50 +++---- docs/advanced/configuration.md | 2 +- docs/advanced/filesystem-serdes.md | 5 +- .../FileSystemSerDesIntegrationTest.java | 41 +++--- .../testing/CloudDurableTestRunnerTest.java | 52 ++++--- .../testing/LocalDurableTestRunnerTest.java | 32 +++-- .../durable/serde/ChainedSerDesStage.java | 112 +++++++++++++++ .../durable/serde/ComposableSerDes.java | 109 ++++----------- .../durable/serde/FileSystemSerDes.java | 13 +- .../lambda/durable/serde/RetrySerDes.java | 12 +- .../amazon/lambda/durable/serde/SerDes.java | 31 +---- .../lambda/durable/serde/SerDesStage.java | 14 ++ .../durable/serde/ComposableSerDesTest.java | 129 +++++++++--------- .../durable/serde/FileSystemSerDesTest.java | 42 ++++-- 14 files changed, 363 insertions(+), 281 deletions(-) create mode 100644 sdk/src/main/java/software/amazon/lambda/durable/serde/ChainedSerDesStage.java diff --git a/docs/adr/005-filesystem-serdes.md b/docs/adr/005-filesystem-serdes.md index fc999a198..56dbce75b 100644 --- a/docs/adr/005-filesystem-serdes.md +++ b/docs/adr/005-filesystem-serdes.md @@ -33,9 +33,8 @@ There are a few Java-specific constraints: Keep the existing `SerDes` serialization methods source- and binary-compatible, add a typed `SerDesStage` contract and a core `ComposableSerDes` implementation which together form a processing pipeline, and implement -`FileSystemSerDes` in the core SDK. Existing `SerDes` implementations continue to participate as string-producing -stages without changing their binary contract, while dedicated intermediate stages may exchange arbitrary Java types. -The filesystem stage uses +`FileSystemSerDes` in the core SDK. `SerDes` remains the persisted value-codec boundary, while intermediate +`SerDesStage` instances may exchange arbitrary Java types and compose independently. The filesystem stage uses `SerDesContext.getCurrentContext()` to identify the durable execution and entity being serialized. ```java @@ -43,19 +42,9 @@ public interface SerDesStage { O serialize(I value); I deserialize(O data); -} - -public interface SerDes { - String serialize(Object value); - T deserialize(String data, TypeToken typeToken); - - default ComposableSerDes then(SerDes nextStage) { - return ComposableSerDes.of(this, nextStage); - } - - default ComposableSerDes then(SerDesStage nextStage) { - return ComposableSerDes.builder(this).then(nextStage).build(); + default SerDesStage then(SerDesStage nextStage) { + return ChainedSerDesStage.of(this, nextStage); } } ``` @@ -117,7 +106,7 @@ var fileSystemStage = FileSystemSerDes.stageBuilder(Path.of("/mnt/efs/durable-pa .previewGenerator(optionalPreviewGenerator) .build(); -var serDes = new JacksonSerDes().then(fileSystemStage); +var serDes = ComposableSerDes.of(new JacksonSerDes(), fileSystemStage); return DurableConfig.builder() .withSerDes(serDes) @@ -132,17 +121,14 @@ the existing `serialize` and `deserialize` methods: ```java public final class ComposableSerDes implements SerDes { - public static ComposableSerDes of(SerDes first, SerDes... remaining); + public static ComposableSerDes of(SerDes valueCodec); + public static ComposableSerDes of(SerDes valueCodec, SerDesStage stage); public static Builder builder(SerDes valueCodec); public SerDes getValueCodec(); - public ComposableSerDes then(SerDes stage); - public ComposableSerDes then(SerDesStage stage); - public static final class Builder { - public Builder then(SerDes stage); public Builder then(SerDesStage stage); public ComposableSerDes build(); @@ -218,8 +204,8 @@ Pipeline rules: `String`, and reverse processing must return a `String` to the value codec. - A stage may declare that it requires durable context or that it must be terminal. A terminal stage must be the last stage so later transformations cannot invalidate its checkpoint-size or storage decision. -- `SerDes.then(...)`, `ComposableSerDes.of(...)`, and the builder flatten nested `ComposableSerDes` instances while - preserving stage order. +- `SerDesStage.then(...)`, `ComposableSerDes.of(...)`, and the builder flatten nested stage chains while preserving + stage order. - A `null` value at the pipeline boundary short-circuits the entire pipeline: serializing or deserializing `null` returns `null` without invoking any stage. A stage returning `null` for non-null input is an error. The value codec may decode a non-null representation such as the JSON literal `null` to a null domain value. @@ -247,14 +233,15 @@ Pipeline rules: - Invocation-scoped caching wraps the complete pipeline. Cache keys use the final checkpoint string and target type, so cache hits skip every reverse-processing stage, including filesystem reads. -The default `SerDes.then(...)` method and immutable `ComposableSerDes.then(...)` method provide a concise form for -independently reusable processing chains: +The default `SerDesStage.then(...)` method provides independently reusable processing chains. `ComposableSerDes` +connects one chain to the persisted value codec: ```java -var securePayloads = new JacksonSerDes() - .then(compressionSerDes) +var secureStages = compressionStage .then(encryptionSerDes) .then(fileSystemStage); + +var securePayloads = ComposableSerDes.of(new JacksonSerDes(), secureStages); ``` ### Retryable SerDes stages @@ -272,7 +259,7 @@ var resilientFileSystemStage = new RetrySerDes( 2.0, JitterStrategy.FULL)); -var serDes = new JacksonSerDes().then(resilientFileSystemStage); +var serDes = ComposableSerDes.of(new JacksonSerDes(), resilientFileSystemStage); ``` Retry rules: @@ -480,9 +467,8 @@ must not synthesize a durable context and serialize initial input through the fu 1. Add `SerDesContext`, `SerDesPayloadKind`, and package-private TLS setter/clearer support. Leave the existing `SerDes` methods unchanged. -2. Add the binary-compatible `SerDes.then(...)` default method and `ComposableSerDes` with immutable stage ordering, - forward serialization, reverse deserialization, external-boundary bypass, terminal-stage validation, null - short-circuiting, and stage-aware errors. +2. Add `SerDesStage.then(...)` and `ComposableSerDes` with immutable stage ordering, forward serialization, reverse + deserialization, external-boundary bypass, terminal-stage validation, null short-circuiting, and stage-aware errors. 3. Add `RetryableSerDesException` and `RetrySerDes`, reusing `RetryStrategy` for bounded in-invocation retries. 4. Add `SerDesRunner` with inline execution by default and optional dispatch through `DurableConfig.withSerDesExecutorService(...)`. Do not create a default SerDes pool. @@ -720,7 +706,7 @@ This approach gives the SDK one consistent policy for root payloads, operation r | Responsibility boundary | Combines value serialization and storage-reference creation in ordered, composable SerDes stages. | Keeps object encoding in `SerDes` and storage movement in a separate offloader. | | User configuration | Users configure one SerDes or a `ComposableSerDes` pipeline. Operation-level SerDes selection replaces the whole pipeline. | Users configure both a SerDes and an offloader. The SDK must define global, per-operation, and per-payload precedence. | | Parity with JS issue | Closest to the current JavaScript `createFileSystemSerdes` model and issue #463 wording. | Diverges from JavaScript naming and shape, though it may be architecturally cleaner for Java. | -| Core SDK changes | Adds the compatible `SerDes.then(...)` default method, `ComposableSerDes`, and `SerDesContext` TLS because the existing serialization methods have no context parameter. | Requires a new core extension point, envelope type, config surface, payload pipeline, and migration story. | +| Core SDK changes | Adds `SerDesStage`, `ComposableSerDes`, and `SerDesContext` TLS while leaving `SerDes.then(...)` absent and the existing serialization methods unchanged. | Requires a new core extension point, envelope type, config surface, payload pipeline, and migration story. | | Applicability | Any compatible SerDes stages can be chained, but storage stages still own their envelope and lifecycle behavior. | Any SerDes output can be offloaded uniformly after serialization. Users can combine Jackson/custom SerDes with any offloader. | | Envelope ownership | FileSystemSerDes owns the checkpoint envelope (`data`, `file`, preview), so the SDK treats it as opaque serialized data. | SDK owns the checkpoint/offload envelope and must guarantee it composes with replay, errors, callbacks, and test utilities. | | Caching | SDK can cache deserialized values, but FileSystemSerDes may still do file reads internally unless cache hits happen before SerDes. | SDK can cache at both layers: resolved offloaded payload text and final deserialized object. | diff --git a/docs/advanced/configuration.md b/docs/advanced/configuration.md index 552dd1790..9477d6511 100644 --- a/docs/advanced/configuration.md +++ b/docs/advanced/configuration.md @@ -66,7 +66,7 @@ var resilientFileSystemStage = new RetrySerDes( fileSystemStage, RetryStrategies.fixedDelay(3, Duration.ofSeconds(1))); -var serDes = new JacksonSerDes().then(resilientFileSystemStage); +var serDes = ComposableSerDes.of(new JacksonSerDes(), resilientFileSystemStage); var serDesExecutor = Executors.newFixedThreadPool(4); return DurableConfig.builder() diff --git a/docs/advanced/filesystem-serdes.md b/docs/advanced/filesystem-serdes.md index 201782310..ce83d7ad4 100644 --- a/docs/advanced/filesystem-serdes.md +++ b/docs/advanced/filesystem-serdes.md @@ -28,7 +28,7 @@ var resilientFileSystemStage = new RetrySerDes( fileSystemStage, RetryStrategies.fixedDelay(3, Duration.ofSeconds(1))); -var serDes = new JacksonSerDes().then(resilientFileSystemStage); +var serDes = ComposableSerDes.of(new JacksonSerDes(), resilientFileSystemStage); var serDesExecutor = Executors.newFixedThreadPool(4); return DurableConfig.builder() @@ -38,7 +38,8 @@ return DurableConfig.builder() ``` Serialization runs from `JacksonSerDes` to the filesystem stage. Deserialization runs in reverse. Additional reversible -typed stages such as compression or encryption can be inserted with `then(...)`. A +typed stages such as compression or encryption compose through `SerDesStage.then(...)` before the stage chain is passed +to `ComposableSerDes.of(...)`. A `SerDesStage` may pass compressed or encrypted bytes directly to `FileSystemSerDes`; no Base64 adapter is required between those stages. The complete pipeline still produces a string checkpoint envelope. diff --git a/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FileSystemSerDesIntegrationTest.java b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FileSystemSerDesIntegrationTest.java index ee0505fe3..1b600e263 100644 --- a/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FileSystemSerDesIntegrationTest.java +++ b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FileSystemSerDesIntegrationTest.java @@ -37,6 +37,7 @@ import software.amazon.lambda.durable.retry.JitterStrategy; import software.amazon.lambda.durable.retry.RetryStrategies; import software.amazon.lambda.durable.retry.WaitStrategies; +import software.amazon.lambda.durable.serde.ComposableSerDes; import software.amazon.lambda.durable.serde.FileSystemSerDes; import software.amazon.lambda.durable.serde.JacksonSerDes; import software.amazon.lambda.durable.serde.SerDes; @@ -160,9 +161,9 @@ void acceptsRawCallbackAndInvokeResultsAndOffloadsInvokePayload() throws Excepti invokePayload.set(value); } }); - var serDes = new JacksonSerDes() - .then(recordingStage) - .then(FileSystemSerDes.stageBuilder(basePath).build()); + var serDes = ComposableSerDes.of( + new JacksonSerDes(), + recordingStage.then(FileSystemSerDes.stageBuilder(basePath).build())); var config = DurableConfig.builder().withSerDes(serDes).build(); var runner = LocalDurableTestRunner.create( String.class, @@ -278,9 +279,9 @@ void repeatedGetUsesInvocationCacheForTheCompletePipeline() { resultDeserializations.incrementAndGet(); } }); - var serDes = new JacksonSerDes() - .then(countingStage) - .then(FileSystemSerDes.stageBuilder(basePath).build()); + var serDes = ComposableSerDes.of( + new JacksonSerDes(), + countingStage.then(FileSystemSerDes.stageBuilder(basePath).build())); var config = DurableConfig.builder().withSerDes(serDes).build(); var runner = LocalDurableTestRunner.create( String.class, @@ -315,9 +316,9 @@ void successfulRetryUsesTheProducingAttemptForResultSerialization() { resultAttempts.add(context.attempt()); } }); - var serDes = new JacksonSerDes() - .then(attemptStage) - .then(FileSystemSerDes.stageBuilder(basePath).build()); + var serDes = ComposableSerDes.of( + new JacksonSerDes(), + attemptStage.then(FileSystemSerDes.stageBuilder(basePath).build())); var config = DurableConfig.builder().withSerDes(serDes).build(); var stepConfig = StepConfig.builder() .retryStrategy(RetryStrategies.fixedDelay(2, Duration.ofSeconds(1))) @@ -480,9 +481,9 @@ public String deserialize(byte[] data) { return new String(data, StandardCharsets.UTF_8); } }; - return new JacksonSerDes() - .then(utf8) - .then(FileSystemSerDes.stageBuilder(basePath).build()); + return ComposableSerDes.of( + new JacksonSerDes(), + utf8.then(FileSystemSerDes.stageBuilder(basePath).build())); } private static DurableExecutionInput durableInput( @@ -511,20 +512,18 @@ private static Operation executionOperation(String id, String name, String input .build(); } - private static SerDes identityStage(RecordingFunction recorder) { - return new SerDes() { + private static SerDesStage identityStage(RecordingFunction recorder) { + return new SerDesStage<>() { @Override - public String serialize(Object value) { - var stringValue = (String) value; - recorder.record("serialize", stringValue); - return stringValue; + public String serialize(String value) { + recorder.record("serialize", value); + return value; } @Override - @SuppressWarnings("unchecked") - public T deserialize(String data, TypeToken typeToken) { + public String deserialize(String data) { recorder.record("deserialize", data); - return (T) data; + return data; } }; } diff --git a/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/CloudDurableTestRunnerTest.java b/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/CloudDurableTestRunnerTest.java index 9aa79ce4c..c9cf9ee75 100644 --- a/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/CloudDurableTestRunnerTest.java +++ b/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/CloudDurableTestRunnerTest.java @@ -15,12 +15,14 @@ import software.amazon.awssdk.services.lambda.model.InvokeRequest; import software.amazon.awssdk.services.lambda.model.InvokeResponse; import software.amazon.lambda.durable.TypeToken; +import software.amazon.lambda.durable.serde.ComposableSerDes; import software.amazon.lambda.durable.serde.FileSystemSerDes; import software.amazon.lambda.durable.serde.JacksonSerDes; import software.amazon.lambda.durable.serde.SerDes; import software.amazon.lambda.durable.serde.SerDesContext; import software.amazon.lambda.durable.serde.SerDesPayloadKind; import software.amazon.lambda.durable.serde.SerDesRunner; +import software.amazon.lambda.durable.serde.SerDesStage; class CloudDurableTestRunnerTest { @@ -54,7 +56,7 @@ void explicitComposableInputSerDesUsesTheCompletePipeline() { var wrappingStage = wrappingStage(); var runner = CloudDurableTestRunner.create( "arn:aws:lambda:us-east-2:123:function:test", String.class, String.class, mockClient) - .withInputSerDes(new JacksonSerDes().then(wrappingStage)); + .withInputSerDes(ComposableSerDes.of(new JacksonSerDes(), wrappingStage)); runner.startAsync("value"); @@ -72,7 +74,7 @@ void persistedComposableSerDesUsesCompletePipelineForDefaultInput() { .build()); var runner = CloudDurableTestRunner.create( "arn:aws:lambda:us-east-2:123:function:test", String.class, String.class, mockClient) - .withSerDes(new JacksonSerDes().then(wrappingStage())); + .withSerDes(ComposableSerDes.of(new JacksonSerDes(), wrappingStage())); runner.startAsync("value"); @@ -86,7 +88,7 @@ void contextDependentPersistedSerDesRequiresExplicitInputSerDes() { var mockClient = mock(LambdaClient.class); var runner = CloudDurableTestRunner.create( "arn:aws:lambda:us-east-2:123:function:test", String.class, String.class, mockClient) - .withSerDes(new JacksonSerDes().then(contextDependentStage())); + .withSerDes(ComposableSerDes.of(new JacksonSerDes(), contextDependentStage())); var failure = assertThrows(RuntimeException.class, () -> runner.startAsync("value")); @@ -100,7 +102,7 @@ void explicitInputSerDesMustBeContextFree() { var mockClient = mock(LambdaClient.class); var runner = CloudDurableTestRunner.create( "arn:aws:lambda:us-east-2:123:function:test", String.class, String.class, mockClient) - .withInputSerDes(contextDependentStage()); + .withInputSerDes(contextDependentSerDes()); var failure = assertThrows(RuntimeException.class, () -> runner.startAsync("value")); @@ -114,8 +116,8 @@ void contextDependentPersistedSerDesRequiresValueCodecInput() { var mockClient = mock(LambdaClient.class); var runner = CloudDurableTestRunner.create( "arn:aws:lambda:us-east-2:123:function:test", String.class, String.class, mockClient) - .withSerDes(new JacksonSerDes().then(contextDependentStage())) - .withInputSerDes(new JacksonSerDes().then(wrappingStage())); + .withSerDes(ComposableSerDes.of(new JacksonSerDes(), contextDependentStage())) + .withInputSerDes(ComposableSerDes.of(new JacksonSerDes(), wrappingStage())); var failure = assertThrows(RuntimeException.class, () -> runner.startAsync("value")); @@ -132,8 +134,8 @@ void valueCodecInputRoundTripsThroughFileSystemPipeline(@TempDir Path basePath) .thenReturn(InvokeResponse.builder() .durableExecutionArn(executionArn) .build()); - var persistedSerDes = - new JacksonSerDes().then(FileSystemSerDes.stageBuilder(basePath).build()); + var persistedSerDes = ComposableSerDes.of( + new JacksonSerDes(), FileSystemSerDes.stageBuilder(basePath).build()); var runner = CloudDurableTestRunner.create( "arn:aws:lambda:us-east-2:123:function:test", String.class, String.class, mockClient) .withSerDes(persistedSerDes) @@ -162,7 +164,7 @@ void replacingPersistedSerDesPreservesExplicitInputSerDes() { .build()); var runner = CloudDurableTestRunner.create( "arn:aws:lambda:us-east-2:123:function:test", String.class, String.class, mockClient) - .withInputSerDes(new JacksonSerDes().then(wrappingStage())) + .withInputSerDes(ComposableSerDes.of(new JacksonSerDes(), wrappingStage())) .withSerDes(new JacksonSerDes()); runner.startAsync("value"); @@ -172,22 +174,40 @@ void replacingPersistedSerDesPreservesExplicitInputSerDes() { assertEquals("<\"value\">", request.getValue().payload().asUtf8String()); } - private static SerDes wrappingStage() { - return new SerDes() { + private static SerDesStage wrappingStage() { + return new SerDesStage<>() { @Override - public String serialize(Object value) { + public String serialize(String value) { return "<" + value + ">"; } @Override - @SuppressWarnings("unchecked") - public T deserialize(String data, TypeToken typeToken) { - return (T) data.substring(1, data.length() - 1); + public String deserialize(String data) { + return data.substring(1, data.length() - 1); + } + }; + } + + private static SerDesStage contextDependentStage() { + return new SerDesStage<>() { + @Override + public String serialize(String value) { + return value; + } + + @Override + public String deserialize(String data) { + return data; + } + + @Override + public boolean requiresDurableContext() { + return true; } }; } - private static SerDes contextDependentStage() { + private static SerDes contextDependentSerDes() { return new SerDes() { @Override public String serialize(Object value) { diff --git a/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/LocalDurableTestRunnerTest.java b/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/LocalDurableTestRunnerTest.java index 7cf89f041..17abb475f 100644 --- a/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/LocalDurableTestRunnerTest.java +++ b/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/LocalDurableTestRunnerTest.java @@ -20,9 +20,9 @@ import software.amazon.lambda.durable.model.ExecutionStatus; import software.amazon.lambda.durable.plugin.DurableExecutionPlugin; import software.amazon.lambda.durable.plugin.InvocationInfo; +import software.amazon.lambda.durable.serde.ComposableSerDes; import software.amazon.lambda.durable.serde.FileSystemSerDes; import software.amazon.lambda.durable.serde.JacksonSerDes; -import software.amazon.lambda.durable.serde.SerDes; import software.amazon.lambda.durable.serde.SerDesStage; class LocalDurableTestRunnerTest { @@ -149,8 +149,9 @@ void checkpointedLargeOutputReplaysWithoutDuplicateExecutionOperation() { @Test void contextDependentPersistedSerDesRequiresExplicitInputSerDes(@TempDir Path basePath) { var config = DurableConfig.builder() - .withSerDes(new JacksonSerDes() - .then(FileSystemSerDes.stageBuilder(basePath).build())) + .withSerDes(ComposableSerDes.of( + new JacksonSerDes(), + FileSystemSerDes.stageBuilder(basePath).build())) .build(); var runner = LocalDurableTestRunner.create(String.class, (input, context) -> input, config); @@ -162,11 +163,12 @@ void contextDependentPersistedSerDesRequiresExplicitInputSerDes(@TempDir Path ba @Test void contextDependentPersistedSerDesRequiresValueCodecInput(@TempDir Path basePath) { var config = DurableConfig.builder() - .withSerDes(new JacksonSerDes() - .then(FileSystemSerDes.stageBuilder(basePath).build())) + .withSerDes(ComposableSerDes.of( + new JacksonSerDes(), + FileSystemSerDes.stageBuilder(basePath).build())) .build(); var runner = LocalDurableTestRunner.create(String.class, (input, context) -> input, config) - .withInputSerDes(new JacksonSerDes().then(wrappingStage(new AtomicInteger()))); + .withInputSerDes(ComposableSerDes.of(new JacksonSerDes(), wrappingStage(new AtomicInteger()))); var failure = assertThrows(IllegalStateException.class, () -> runner.run("value")); @@ -176,9 +178,10 @@ void contextDependentPersistedSerDesRequiresValueCodecInput(@TempDir Path basePa @Test void initialInputBypassesStagesBeforeFileSystemSerDes(@TempDir Path basePath) { var deserializeCalls = new AtomicInteger(); - var persistedSerDes = new JacksonSerDes() - .then(bytesStage(deserializeCalls)) - .then(FileSystemSerDes.stageBuilder(basePath).build()); + var persistedSerDes = ComposableSerDes.of( + new JacksonSerDes(), + bytesStage(deserializeCalls) + .then(FileSystemSerDes.stageBuilder(basePath).build())); var config = DurableConfig.builder().withSerDes(persistedSerDes).build(); var runner = LocalDurableTestRunner.create( String.class, (input, context) -> input + ":" + deserializeCalls.get(), config) @@ -191,18 +194,17 @@ void initialInputBypassesStagesBeforeFileSystemSerDes(@TempDir Path basePath) { assertEquals("value:0", result.getResult()); } - private static SerDes wrappingStage(AtomicInteger deserializeCalls) { - return new SerDes() { + private static SerDesStage wrappingStage(AtomicInteger deserializeCalls) { + return new SerDesStage<>() { @Override - public String serialize(Object value) { + public String serialize(String value) { return "<" + value + ">"; } @Override - @SuppressWarnings("unchecked") - public T deserialize(String data, TypeToken typeToken) { + public String deserialize(String data) { deserializeCalls.incrementAndGet(); - return (T) data.substring(1, data.length() - 1); + return data.substring(1, data.length() - 1); } }; } diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/ChainedSerDesStage.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/ChainedSerDesStage.java new file mode 100644 index 000000000..3ba6179ef --- /dev/null +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/ChainedSerDesStage.java @@ -0,0 +1,112 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.serde; + +import java.util.ArrayList; +import java.util.List; +import software.amazon.lambda.durable.exception.RetryableSerDesException; +import software.amazon.lambda.durable.exception.SerDesException; + +final class ChainedSerDesStage implements SerDesStage { + private final List> stages; + + private ChainedSerDesStage(List> stages) { + for (int index = 0; index < stages.size() - 1; index++) { + var stage = stages.get(index); + if (stage.isTerminalPipelineStage()) { + throw new IllegalArgumentException(String.format( + "SerDes pipeline stage %d (%s) must be the final stage", + index + 1, stage.getClass().getName())); + } + } + this.stages = List.copyOf(stages); + } + + static SerDesStage of(SerDesStage first, SerDesStage second) { + var stages = new ArrayList>(); + addFlattened(stages, first); + addFlattened(stages, second); + return new ChainedSerDesStage<>(stages); + } + + List> stages() { + return stages; + } + + @Override + @SuppressWarnings("unchecked") + public O serialize(I value) { + Object current = value; + for (int index = 0; index < stages.size(); index++) { + var stage = stages.get(index); + try { + current = ((SerDesStage) stage).serialize(current); + if (current == null) { + throw new SerDesException("Stage returned null for a non-null value"); + } + } catch (Throwable failure) { + throw stageFailure(index + 1, stage, "serialize", failure); + } + } + return (O) current; + } + + @Override + @SuppressWarnings("unchecked") + public I deserialize(O data) { + return (I) deserializePipelineStage(data).value(); + } + + @Override + @SuppressWarnings("unchecked") + public SerDesStageResult deserializePipelineStage(O data) { + Object current = data; + for (int index = stages.size() - 1; index >= 0; index--) { + var stage = stages.get(index); + try { + var result = ((SerDesStage) stage).deserializePipelineStage(current); + if (result == null) { + throw new SerDesException("Stage returned a null pipeline result"); + } + current = result.value(); + if (result.skipRemainingStages()) { + return result; + } + } catch (Throwable failure) { + throw stageFailure(index + 1, stage, "deserialize", failure); + } + } + return SerDesStageResult.continueWith(current); + } + + @Override + public boolean requiresDurableContext() { + return stages.stream().anyMatch(SerDesStage::requiresDurableContext); + } + + @Override + public boolean isTerminalPipelineStage() { + return stages.get(stages.size() - 1).isTerminalPipelineStage(); + } + + private static void addFlattened(List> target, SerDesStage stage) { + if (stage instanceof ChainedSerDesStage chained) { + target.addAll(chained.stages); + } else { + target.add(stage); + } + } + + private static RuntimeException stageFailure(int index, SerDesStage stage, String action, Throwable failure) { + if (failure instanceof Error error) { + throw error; + } + var message = String.format( + "SerDes pipeline stage %d (%s) failed to %s", + index, stage.getClass().getName(), action); + if (failure instanceof RetryableSerDesException) { + return new RetryableSerDesException(message, failure); + } + return new SerDesException(message, failure); + } +} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/ComposableSerDes.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/ComposableSerDes.java index 5484f8cd8..2277c7ed5 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/ComposableSerDes.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/ComposableSerDes.java @@ -3,7 +3,6 @@ package software.amazon.lambda.durable.serde; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import java.util.Objects; import software.amazon.lambda.durable.TypeToken; @@ -19,15 +18,15 @@ */ public final class ComposableSerDes implements SerDes { private final SerDes valueCodec; - private final List stages; + private final List> stages; - private ComposableSerDes(SerDes valueCodec, List stages) { + private ComposableSerDes(SerDes valueCodec, List> stages) { this.valueCodec = Objects.requireNonNull(valueCodec, "valueCodec cannot be null"); if (!stages.isEmpty() && valueCodec.isTerminalPipelineStage()) { throw terminalStageFailure(0, valueCodec); } for (int index = 0; index < stages.size() - 1; index++) { - if (isTerminal(stages.get(index))) { + if (stages.get(index).isTerminalPipelineStage()) { throw terminalStageFailure(index + 1, stages.get(index)); } } @@ -35,24 +34,24 @@ private ComposableSerDes(SerDes valueCodec, List stages) { } /** - * Creates a pipeline with a value codec followed by zero or more typed stages. + * Creates a pipeline containing only a value codec. * - * @param first the value codec - * @param remaining reversible SerDes stages + * @param valueCodec the value codec * @return an immutable pipeline */ - public static ComposableSerDes of(SerDes first, SerDes... remaining) { - Objects.requireNonNull(remaining, "remaining stages cannot be null"); - var valueCodec = Objects.requireNonNull(first, "first stage cannot be null"); - var stages = new ArrayList(); - if (valueCodec instanceof ComposableSerDes composable) { - valueCodec = composable.valueCodec; - stages.addAll(composable.stages); - } - Arrays.stream(remaining) - .map(stage -> Objects.requireNonNull(stage, "pipeline stage cannot be null")) - .forEach(stage -> addFlattened(stages, stage)); - return new ComposableSerDes(valueCodec, stages); + public static ComposableSerDes of(SerDes valueCodec) { + return builder(valueCodec).build(); + } + + /** + * Creates a pipeline with a value codec followed by a typed stage or stage chain. + * + * @param valueCodec the value codec + * @param stage the reversible typed stage or stage chain + * @return an immutable pipeline + */ + public static ComposableSerDes of(SerDes valueCodec, SerDesStage stage) { + return builder(valueCodec).then(stage).build(); } /** @@ -72,28 +71,14 @@ public SerDes getValueCodec() { @Override public boolean requiresDurableContext() { - return valueCodec.requiresDurableContext() || stages.stream().anyMatch(ComposableSerDes::requiresContext); + return valueCodec.requiresDurableContext() || stages.stream().anyMatch(SerDesStage::requiresDurableContext); } @Override public boolean isTerminalPipelineStage() { - return stages.isEmpty() ? valueCodec.isTerminalPipelineStage() : isTerminal(stages.get(stages.size() - 1)); - } - - /** Returns a new pipeline with the supplied stage appended. */ - @Override - public ComposableSerDes then(SerDes stage) { - var combined = new ArrayList<>(stages); - addFlattened(combined, Objects.requireNonNull(stage, "stage cannot be null")); - return new ComposableSerDes(valueCodec, combined); - } - - /** Returns a new pipeline with the supplied typed stage appended. */ - @Override - public ComposableSerDes then(SerDesStage stage) { - var combined = new ArrayList<>(stages); - addFlattened(combined, Objects.requireNonNull(stage, "stage cannot be null")); - return new ComposableSerDes(valueCodec, combined); + return stages.isEmpty() + ? valueCodec.isTerminalPipelineStage() + : stages.get(stages.size() - 1).isTerminalPipelineStage(); } @Override @@ -135,11 +120,9 @@ public T deserialize(String data, TypeToken typeToken) { } @SuppressWarnings("unchecked") - private static Object invokeStageSerialize(Object stage, Object value, int index) { + private static Object invokeStageSerialize(SerDesStage stage, Object value, int index) { try { - var result = stage instanceof SerDes serDes - ? serDes.serialize(value) - : ((SerDesStage) stage).serialize(value); + var result = ((SerDesStage) stage).serialize(value); if (result == null) { throw new SerDesException("Stage returned null for a non-null value"); } @@ -150,18 +133,9 @@ private static Object invokeStageSerialize(Object stage, Object value, int index } @SuppressWarnings("unchecked") - private static SerDesStageResult invokeStageDeserialize(Object stage, Object data, int index) { + private static SerDesStageResult invokeStageDeserialize(SerDesStage stage, Object data, int index) { try { - SerDesStageResult result; - if (stage instanceof SerDes serDes) { - if (!(data instanceof String stringData)) { - throw new SerDesException("SerDes stage requires String input but received " - + data.getClass().getName()); - } - result = serDes.deserializePipelineStage(stringData); - } else { - result = ((SerDesStage) stage).deserializePipelineStage(data); - } + var result = ((SerDesStage) stage).deserializePipelineStage(data); if (result == null) { throw new SerDesException("Stage returned a null pipeline result"); } @@ -210,35 +184,18 @@ private static IllegalArgumentException terminalStageFailure(int index, Object s index, stage.getClass().getName())); } - private static boolean requiresContext(Object stage) { - return stage instanceof SerDes serDes - ? serDes.requiresDurableContext() - : ((SerDesStage) stage).requiresDurableContext(); - } - - private static boolean isTerminal(Object stage) { - return stage instanceof SerDes serDes - ? serDes.isTerminalPipelineStage() - : ((SerDesStage) stage).isTerminalPipelineStage(); - } - - private static void addFlattened(List target, SerDes stage) { - if (stage instanceof ComposableSerDes composable) { - target.add(composable.valueCodec); - target.addAll(composable.stages); + private static void addFlattened(List> target, SerDesStage stage) { + if (stage instanceof ChainedSerDesStage chained) { + target.addAll(chained.stages()); } else { target.add(stage); } } - private static void addFlattened(List target, SerDesStage stage) { - target.add(stage); - } - /** Builder for an immutable {@link ComposableSerDes}. */ public static final class Builder { private SerDes valueCodec; - private final List stages = new ArrayList<>(); + private final List> stages = new ArrayList<>(); private Builder(SerDes valueCodec) { this.valueCodec = Objects.requireNonNull(valueCodec, "valueCodec cannot be null"); @@ -248,12 +205,6 @@ private Builder(SerDes valueCodec) { } } - /** Appends a reversible typed stage. */ - public Builder then(SerDes stage) { - addFlattened(stages, Objects.requireNonNull(stage, "stage cannot be null")); - return this; - } - /** Appends a reversible typed stage. */ public Builder then(SerDesStage stage) { addFlattened(stages, Objects.requireNonNull(stage, "stage cannot be null")); diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java index 903d50e1b..043d94eb3 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java @@ -38,7 +38,7 @@ *

Do not use Lambda's ephemeral {@code /tmp} storage. Use a durable shared mount such as EFS, or S3 Files only when * its synchronization and crash-durability tradeoffs are acceptable for the workload. */ -public final class FileSystemSerDes implements SerDes { +public final class FileSystemSerDes implements SerDes, SerDesStage { private static final String ENVELOPE_MARKER = "__durable_execution_filesystem_serdes"; private static final String PAYLOAD_TYPE_FIELD = "payloadType"; private static final int ENVELOPE_VERSION = 1; @@ -140,6 +140,17 @@ public T deserialize(String data, TypeToken typeToken) { return delegate.deserialize(serializedString, typeToken); } + @Override + public Object deserialize(String data) { + if (data == null) { + return null; + } + if (!stageMode) { + throw new SerDesException("Standalone FileSystemSerDes cannot be used as a pipeline stage"); + } + return resolveSerializedPayload(data, requireContext()).value(); + } + @Override public SerDesStageResult deserializePipelineStage(String data) { if (!stageMode) { diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/RetrySerDes.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/RetrySerDes.java index 2b0017baf..b4d96bf08 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/RetrySerDes.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/RetrySerDes.java @@ -18,7 +18,7 @@ *

Only {@link RetryableSerDesException} is retried. Other failures are propagated immediately. Retry delays block * the thread executing the SerDes call: the caller by default or the configured SerDes executor thread. */ -public final class RetrySerDes implements SerDes { +public final class RetrySerDes implements SerDes, SerDesStage { private static final Sleeper DEFAULT_SLEEPER = delay -> { if (delay.getSeconds() > 0) { TimeUnit.SECONDS.sleep(delay.getSeconds()); @@ -58,6 +58,16 @@ public T deserialize(String data, TypeToken typeToken) { return execute("deserialization", () -> delegate.deserialize(data, typeToken)); } + @Override + @SuppressWarnings("unchecked") + public Object deserialize(String data) { + if (delegate instanceof SerDesStage stage) { + return execute( + "pipeline stage deserialization", () -> ((SerDesStage) stage).deserialize(data)); + } + return execute("pipeline stage deserialization", () -> delegate.deserialize(data, TypeToken.get(String.class))); + } + @Override public SerDesStageResult deserializePipelineStage(String data) { return execute("pipeline stage deserialization", () -> delegate.deserializePipelineStage(data)); diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDes.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDes.java index fa61b83bf..7c656378e 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDes.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDes.java @@ -5,12 +5,7 @@ import software.amazon.lambda.durable.TypeToken; import software.amazon.lambda.durable.exception.SerDesException; -/** - * Interface for serialization and deserialization of objects at the persisted string boundary. - * - *

Implementations can also be used as string-producing stages in a {@link ComposableSerDes}. Use {@link SerDesStage} - * for typed intermediate transformations that produce or consume non-string values. - */ +/** Interface for serialization and deserialization of objects at the persisted string boundary. */ public interface SerDes { /** * Serializes an object to a JSON string. @@ -71,28 +66,4 @@ default boolean requiresDurableContext() { default boolean isTerminalPipelineStage() { return false; } - - /** - * Returns an immutable processing pipeline that invokes this SerDes followed by {@code nextStage} when serializing - * and in reverse order when deserializing. - * - *

This SerDes is the value codec. Intermediate stages may transform values into arbitrary Java types, but the - * final stage must return a string for persistence. - * - * @param nextStage the reversible stage to append - * @return a composable SerDes pipeline - */ - default ComposableSerDes then(SerDes nextStage) { - return ComposableSerDes.of(this, nextStage); - } - - /** - * Returns an immutable processing pipeline with a typed intermediate stage appended. - * - * @param nextStage the reversible typed stage to append - * @return a composable SerDes pipeline - */ - default ComposableSerDes then(SerDesStage nextStage) { - return ComposableSerDes.builder(this).then(nextStage).build(); - } } diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesStage.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesStage.java index 9662e87f3..cedb9250e 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesStage.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesStage.java @@ -2,6 +2,8 @@ // SPDX-License-Identifier: Apache-2.0 package software.amazon.lambda.durable.serde; +import java.util.Objects; + /** * A reversible typed stage in a {@link ComposableSerDes} pipeline. * @@ -52,4 +54,16 @@ default boolean requiresDurableContext() { default boolean isTerminalPipelineStage() { return false; } + + /** + * Returns an immutable stage chain that applies this stage followed by {@code nextStage} during serialization and + * reverses them in the opposite order during deserialization. + * + * @param nextStage the reversible stage that consumes this stage's output + * @param the next stage's output type + * @return the composed stage chain + */ + default SerDesStage then(SerDesStage nextStage) { + return ChainedSerDesStage.of(this, Objects.requireNonNull(nextStage, "nextStage cannot be null")); + } } diff --git a/sdk/src/test/java/software/amazon/lambda/durable/serde/ComposableSerDesTest.java b/sdk/src/test/java/software/amazon/lambda/durable/serde/ComposableSerDesTest.java index 1fa9927f4..62d9a0a0e 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/serde/ComposableSerDesTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/serde/ComposableSerDesTest.java @@ -26,7 +26,7 @@ void serializesForwardAndDeserializesInReverse() { var calls = new ArrayList(); var first = stringStage("first", "<", ">", calls); var second = stringStage("second", "[", "]", calls); - var pipeline = new JacksonSerDes().then(first).then(second); + var pipeline = ComposableSerDes.of(new JacksonSerDes(), first.then(second)); var serialized = pipeline.serialize("value"); var deserialized = pipeline.deserialize(serialized, TypeToken.get(String.class)); @@ -65,7 +65,7 @@ public byte[] deserialize(String data) { return Base64.getDecoder().decode(data); } }; - var pipeline = new JacksonSerDes().then(utf8).then(base64); + var pipeline = ComposableSerDes.of(new JacksonSerDes(), utf8.then(base64)); var serialized = pipeline.serialize("value"); var deserialized = pipeline.deserialize(serialized, TypeToken.get(String.class)); @@ -76,15 +76,14 @@ public byte[] deserialize(String data) { } @Test - void factoryBuilderAndThenFlattenNestedPipelines() { + void factoryBuilderAndThenFlattenNestedStageChains() { var calls = new ArrayList(); - var nested = ComposableSerDes.builder(stringStage("codec", "", "", calls)) - .then(stringStage("one", "1", "1", calls)) - .build(); - var pipeline = ComposableSerDes.of(nested).then(stringStage("two", "2", "2", calls)); + var nested = stringStage("one", "1", "1", calls).then(stringStage("two", "2", "2", calls)); + var pipeline = + ComposableSerDes.builder(new JacksonSerDes()).then(nested).build(); - assertEquals("21value12", pipeline.serialize("value")); - assertEquals(List.of("codec-serialize", "one-serialize", "two-serialize"), calls); + assertEquals("21\"value\"12", pipeline.serialize("value")); + assertEquals(List.of("one-serialize", "two-serialize"), calls); } @Test @@ -113,20 +112,19 @@ public T deserialize(String data, TypeToken typeToken) { @Test void valueCodecMayDecodeNonNullRepresentationToNull() { var intermediateCalls = new AtomicInteger(); - var identityStage = new SerDes() { + var identityStage = new SerDesStage() { @Override - public String serialize(Object value) { - return value.toString(); + public String serialize(String value) { + return value; } @Override - @SuppressWarnings("unchecked") - public T deserialize(String data, TypeToken typeToken) { + public String deserialize(String data) { intermediateCalls.incrementAndGet(); - return (T) data; + return data; } }; - var pipeline = new JacksonSerDes().then(identityStage); + var pipeline = ComposableSerDes.of(new JacksonSerDes(), identityStage); assertNull(pipeline.deserialize("null", TypeToken.get(Object.class))); assertEquals(1, intermediateCalls.get()); @@ -135,29 +133,27 @@ public T deserialize(String data, TypeToken typeToken) { @Test void stageMayDecodeExternalDataDirectlyWithValueCodec() { var transformDeserializations = new AtomicInteger(); - var transform = new SerDes() { + var transform = new SerDesStage() { @Override - public String serialize(Object value) { + public String serialize(String value) { return "<" + value + ">"; } @Override - @SuppressWarnings("unchecked") - public T deserialize(String data, TypeToken typeToken) { + public String deserialize(String data) { transformDeserializations.incrementAndGet(); - return (T) data.substring(1, data.length() - 1); + return data.substring(1, data.length() - 1); } }; - var externalBoundary = new SerDes() { + var externalBoundary = new SerDesStage() { @Override - public String serialize(Object value) { - return value.toString(); + public String serialize(String value) { + return value; } @Override - @SuppressWarnings("unchecked") - public T deserialize(String data, TypeToken typeToken) { - return (T) data; + public String deserialize(String data) { + return data; } @Override @@ -165,7 +161,7 @@ public SerDesStageResult deserializePipelineStage(String data) { return SerDesStageResult.decodeWithValueCodec(data); } }; - var pipeline = new JacksonSerDes().then(transform).then(externalBoundary); + var pipeline = ComposableSerDes.of(new JacksonSerDes(), transform.then(externalBoundary)); assertEquals("value", pipeline.deserialize("\"value\"", TypeToken.get(String.class))); assertEquals(0, transformDeserializations.get()); @@ -173,16 +169,15 @@ public SerDesStageResult deserializePipelineStage(String data) { @Test void rejectsStagesAfterTerminalStage() { - var terminal = new SerDes() { + var terminal = new SerDesStage() { @Override - public String serialize(Object value) { - return value.toString(); + public String serialize(String value) { + return value; } @Override - @SuppressWarnings("unchecked") - public T deserialize(String data, TypeToken typeToken) { - return (T) data; + public String deserialize(String data) { + return data; } @Override @@ -192,8 +187,7 @@ public boolean isTerminalPipelineStage() { }; var failure = assertThrows( - IllegalArgumentException.class, - () -> new JacksonSerDes().then(terminal).then(stringStage("late", "", "", new ArrayList<>()))); + IllegalArgumentException.class, () -> terminal.then(stringStage("late", "", "", new ArrayList<>()))); assertTrue(failure.getMessage().contains("stage 1")); assertTrue(failure.getMessage().contains("final stage")); @@ -201,19 +195,19 @@ public boolean isTerminalPipelineStage() { @Test void rejectsNullIntermediateAndNonStringBoundaryValues() { - var nullStage = new SerDes() { + var nullStage = new SerDesStage() { @Override - public String serialize(Object value) { + public String serialize(String value) { return null; } @Override - public T deserialize(String data, TypeToken typeToken) { + public String deserialize(String data) { return null; } }; - var nullFailure = assertThrows( - SerDesException.class, () -> new JacksonSerDes().then(nullStage).serialize("value")); + var nullFailure = assertThrows(SerDesException.class, () -> ComposableSerDes.of(new JacksonSerDes(), nullStage) + .serialize("value")); assertTrue(nullFailure.getMessage().contains("stage 1")); assertTrue(nullFailure.getMessage().contains(nullStage.getClass().getName())); @@ -228,9 +222,9 @@ public String deserialize(Integer data) { return "x".repeat(data); } }; - var typeFailure = assertThrows( - SerDesException.class, - () -> new JacksonSerDes().then(nonStringFinalStage).serialize("value")); + var typeFailure = + assertThrows(SerDesException.class, () -> ComposableSerDes.of(new JacksonSerDes(), nonStringFinalStage) + .serialize("value")); assertTrue(typeFailure.getMessage().contains("final stage")); assertTrue(typeFailure.getMessage().contains(Integer.class.getName())); } @@ -249,9 +243,8 @@ public Integer deserialize(String data) { } }; - var failure = assertThrows( - SerDesException.class, - () -> new JacksonSerDes().then(integerStage).serialize("value")); + var failure = assertThrows(SerDesException.class, () -> ComposableSerDes.of(new JacksonSerDes(), integerStage) + .serialize("value")); assertTrue(failure.getMessage().contains("stage 1")); assertInstanceOf(ClassCastException.class, failure.getCause()); @@ -259,21 +252,21 @@ public Integer deserialize(String data) { @Test void preservesRetryabilityWhenDecoratingStageFailures() { - var transientStage = new SerDes() { + var transientStage = new SerDesStage() { @Override - public String serialize(Object value) { + public String serialize(String value) { throw new RetryableSerDesException("retry"); } @Override - public T deserialize(String data, TypeToken typeToken) { + public String deserialize(String data) { return null; } }; var failure = assertThrows( RetryableSerDesException.class, - () -> new JacksonSerDes().then(transientStage).serialize("value")); + () -> ComposableSerDes.of(new JacksonSerDes(), transientStage).serialize("value")); assertInstanceOf(RetryableSerDesException.class, failure.getCause()); assertTrue(failure.getMessage().contains("stage 1")); @@ -282,30 +275,30 @@ public T deserialize(String data, TypeToken typeToken) { @Test void preservesFatalErrorsFromEveryPipelineCall() { var serializeError = new OutOfMemoryError("serialize"); - var serializeStage = new SerDes() { + var serializeStage = new SerDesStage() { @Override - public String serialize(Object value) { + public String serialize(String value) { throw serializeError; } @Override - public T deserialize(String data, TypeToken typeToken) { + public String deserialize(String data) { return null; } }; - assertSame(serializeError, assertThrows(OutOfMemoryError.class, () -> new JacksonSerDes() - .then(serializeStage) + assertSame(serializeError, assertThrows(OutOfMemoryError.class, () -> ComposableSerDes.of( + new JacksonSerDes(), serializeStage) .serialize("value"))); var stringStageError = new StackOverflowError("string-stage-deserialize"); - var stringStage = new SerDes() { + var stringStage = new SerDesStage() { @Override - public String serialize(Object value) { - return value.toString(); + public String serialize(String value) { + return value; } @Override - public T deserialize(String data, TypeToken typeToken) { + public String deserialize(String data) { return null; } @@ -314,8 +307,8 @@ public SerDesStageResult deserializePipelineStage(String data) { throw stringStageError; } }; - assertSame(stringStageError, assertThrows(StackOverflowError.class, () -> new JacksonSerDes() - .then(stringStage) + assertSame(stringStageError, assertThrows(StackOverflowError.class, () -> ComposableSerDes.of( + new JacksonSerDes(), stringStage) .deserialize("value", TypeToken.get(String.class)))); var valueCodecError = new AssertionError("value-codec-deserialize"); @@ -334,19 +327,19 @@ public T deserialize(String data, TypeToken typeToken) { .deserialize("value", TypeToken.get(String.class)))); } - private static SerDes stringStage(String name, String prefix, String suffix, List calls) { - return new SerDes() { + private static SerDesStage stringStage( + String name, String prefix, String suffix, List calls) { + return new SerDesStage<>() { @Override - public String serialize(Object value) { + public String serialize(String value) { calls.add(name + "-serialize"); return prefix + value + suffix; } @Override - @SuppressWarnings("unchecked") - public T deserialize(String data, TypeToken typeToken) { + public String deserialize(String data) { calls.add(name + "-deserialize"); - return (T) data.substring(prefix.length(), data.length() - suffix.length()); + return data.substring(prefix.length(), data.length() - suffix.length()); } }; } diff --git a/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java b/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java index 0dfb14900..850947f9f 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java @@ -22,6 +22,7 @@ import software.amazon.lambda.durable.exception.RetryableSerDesException; import software.amazon.lambda.durable.exception.SerDesException; import software.amazon.lambda.durable.model.OperationSubType; +import software.amazon.lambda.durable.retry.RetryStrategies; class FileSystemSerDesTest { private static final String ARN = @@ -53,7 +54,7 @@ void standaloneModeWritesDelegatePayloadAndReplaysIt() throws Exception { @Test void stageModeComposesWithValueCodec() throws Exception { var stage = FileSystemSerDes.stageBuilder(basePath).build(); - var pipeline = new JacksonSerDes().then(stage); + var pipeline = ComposableSerDes.of(new JacksonSerDes(), stage); var runner = new SerDesRunner(null); var envelope = runner.serialize(pipeline, Map.of("id", 42), context()); @@ -69,7 +70,21 @@ void stageModeComposesWithValueCodec() throws Exception { () -> runner.deserialize(stage, envelope, TypeToken.get(Integer.class), context())); assertThrows(IllegalStateException.class, () -> FileSystemSerDes.stageBuilder(basePath) .delegate(new JacksonSerDes())); - assertThrows(IllegalArgumentException.class, () -> pipeline.then(wrappingStage())); + assertThrows(IllegalArgumentException.class, () -> stage.then(wrappingStage())); + } + + @Test + void retryDecoratorPreservesFilesystemStageComposition() { + var retryStage = + new RetrySerDes(FileSystemSerDes.stageBuilder(basePath).build(), RetryStrategies.Presets.NO_RETRY); + var pipeline = ComposableSerDes.of(new JacksonSerDes(), retryStage); + var runner = new SerDesRunner(null); + + var envelope = runner.serialize(pipeline, Map.of("id", 42), context()); + + assertEquals( + Map.of("id", 42), + runner.deserialize(pipeline, envelope, new TypeToken>() {}, context())); } @Test @@ -86,7 +101,7 @@ public String deserialize(byte[] data) { } }; var stage = FileSystemSerDes.stageBuilder(basePath).build(); - var pipeline = new JacksonSerDes().then(utf8).then(stage); + var pipeline = ComposableSerDes.of(new JacksonSerDes(), utf8.then(stage)); var runner = new SerDesRunner(null); var envelope = runner.serialize(pipeline, Map.of("id", 42), context()); @@ -188,7 +203,7 @@ void includesBoundedPreviewWithoutChangingStoredPayload() throws Exception { void acceptsExternallyOriginatedRawPayloadsOnlyForSupportedSources() { var standalone = FileSystemSerDes.builder(basePath).build(); var stage = FileSystemSerDes.stageBuilder(basePath).build(); - var pipeline = new JacksonSerDes().then(wrappingStage()).then(stage); + var pipeline = ComposableSerDes.of(new JacksonSerDes(), wrappingStage().then(stage)); var runner = new SerDesRunner(null); assertEquals( @@ -290,17 +305,15 @@ void overflowFilesystemStageMustRemainTerminal() { .storageMode(FileSystemStorageMode.OVERFLOW) .build(); - var failure = assertThrows( - IllegalArgumentException.class, - () -> new JacksonSerDes().then(filesystem).then(wrappingStage())); + var failure = assertThrows(IllegalArgumentException.class, () -> filesystem.then(wrappingStage())); assertTrue(failure.getMessage().contains("final stage")); } @Test void fileReferencesCrossInvokeInputAndResultBoundaries() { - var serDes = - new JacksonSerDes().then(FileSystemSerDes.stageBuilder(basePath).build()); + var serDes = ComposableSerDes.of( + new JacksonSerDes(), FileSystemSerDes.stageBuilder(basePath).build()); var runner = new SerDesRunner(null); var callerArn = "arn:aws:lambda:us-east-1:123456789012:function:caller:1/durable-execution/caller-execution/caller-invocation"; @@ -475,17 +488,16 @@ private static SerDesContext operationContext(OperationType operationType, Opera ARN, "1", "operation", null, operationType, operationSubType, SerDesPayloadKind.RESULT, null); } - private static SerDes wrappingStage() { - return new SerDes() { + private static SerDesStage wrappingStage() { + return new SerDesStage<>() { @Override - public String serialize(Object value) { + public String serialize(String value) { return "<" + value + ">"; } @Override - @SuppressWarnings("unchecked") - public T deserialize(String data, TypeToken typeToken) { - return (T) data.substring(1, data.length() - 1); + public String deserialize(String data) { + return data.substring(1, data.length() - 1); } }; } From 54665b28e48761ab2126102465cf76c2b4bca9ab Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Tue, 25 Aug 2026 06:51:33 +0000 Subject: [PATCH 18/56] refactor: allow typed value serde representations --- docs/adr/005-filesystem-serdes.md | 108 ++++++++++-------- docs/advanced/configuration.md | 9 +- docs/advanced/filesystem-serdes.md | 11 +- docs/design.md | 5 +- .../durable/serde/ComposableSerDes.java | 97 +++++++--------- .../amazon/lambda/durable/serde/SerDes.java | 14 ++- .../durable/serde/SerDesStageResult.java | 6 +- .../lambda/durable/serde/ValueSerDes.java | 60 ++++++++++ .../durable/serde/ComposableSerDesTest.java | 103 ++++++++++++++--- .../durable/serde/FileSystemSerDesTest.java | 29 +++++ 10 files changed, 308 insertions(+), 134 deletions(-) create mode 100644 sdk/src/main/java/software/amazon/lambda/durable/serde/ValueSerDes.java diff --git a/docs/adr/005-filesystem-serdes.md b/docs/adr/005-filesystem-serdes.md index 56dbce75b..ae0ce4e26 100644 --- a/docs/adr/005-filesystem-serdes.md +++ b/docs/adr/005-filesystem-serdes.md @@ -2,7 +2,8 @@ **Status:** Accepted — Approach A with ComposableSerDes pipeline **Date:** 2026-07-02 -**Updated:** 2026-08-25 — Included FileSystemSerDes in core and generalized pipelines to typed intermediate stages. +**Updated:** 2026-08-25 — Included FileSystemSerDes in core and generalized the value codec and pipeline stages to +typed representations. ## Context @@ -31,13 +32,27 @@ There are a few Java-specific constraints: ### Summary -Keep the existing `SerDes` serialization methods source- and binary-compatible, add a typed `SerDesStage` -contract and a core `ComposableSerDes` implementation which together form a processing pipeline, and implement -`FileSystemSerDes` in the core SDK. `SerDes` remains the persisted value-codec boundary, while intermediate -`SerDesStage` instances may exchange arbitrary Java types and compose independently. The filesystem stage uses +Keep the existing `SerDes` serialization methods source- and binary-compatible, define it as the string specialization +of a typed `ValueSerDes` value codec, add a typed `SerDesStage` contract and a core `ComposableSerDes` +implementation, and implement `FileSystemSerDes` in the core SDK. The value codec and stages may exchange arbitrary +Java types; only the complete `SerDes` pipeline must produce the checkpoint string. The filesystem stage uses `SerDesContext.getCurrentContext()` to identify the durable execution and entity being serialized. ```java +public interface ValueSerDes { + R serialize(Object value); + + T deserialize(R data, TypeToken typeToken); + + default T deserializeExternal(String data, TypeToken typeToken); +} + +public interface SerDes extends ValueSerDes { + String serialize(Object value); + + T deserialize(String data, TypeToken typeToken); +} + public interface SerDesStage { O serialize(I value); @@ -49,10 +64,11 @@ public interface SerDesStage { } ``` -`ComposableSerDes` treats the first stage as the value codec and every later stage as a reversible typed -transformation. This lets customers compose JSON encoding, binary compression, encryption, filesystem storage, or -other processing without artificial Base64 conversion between every stage. The complete pipeline still returns a -string because checkpoints use the existing `SerDes` boundary. +`ComposableSerDes` treats the `ValueSerDes` as the TypeToken-aware value codec and connects its representation +directly to a reversible typed stage chain. The first `SerDesStage` therefore consumes `R`, which does not need to be a +string. This lets customers compose JSON trees, binary encoding, compression, encryption, filesystem storage, or other +processing without artificial string conversion between stages. The complete pipeline still returns a string because +checkpoints use the existing `SerDes` boundary. `FileSystemSerDes` acts as a terminal payload-storage stage. It writes the string or byte array produced by the previous stage to the filesystem when configured to do so and returns a small checkpoint envelope. For standalone @@ -116,23 +132,18 @@ return DurableConfig.builder() ### Composable SerDes pipeline -`ComposableSerDes` is a core implementation of `SerDes`. It owns an immutable, ordered list of stages while preserving -the existing `serialize` and `deserialize` methods: +`ComposableSerDes` is a core implementation of `SerDes`. It owns a typed value codec and immutable stage chain while +preserving the existing persisted-string `serialize` and `deserialize` methods: ```java -public final class ComposableSerDes implements SerDes { - public static ComposableSerDes of(SerDes valueCodec); - public static ComposableSerDes of(SerDes valueCodec, SerDesStage stage); - - public static Builder builder(SerDes valueCodec); +public final class ComposableSerDes implements SerDes { + public static ComposableSerDes of(SerDes valueCodec); - public SerDes getValueCodec(); + public static ComposableSerDes of( + ValueSerDes valueCodec, + SerDesStage stage); - public static final class Builder { - public Builder then(SerDesStage stage); - - public ComposableSerDes build(); - } + public ValueSerDes getValueCodec(); } public record SerDesStageResult(Object value, boolean skipRemainingStages) { @@ -142,18 +153,18 @@ public record SerDesStageResult(Object value, boolean skipRemainingStages) { } ``` -The first stage is the **value codec**. It converts the user value to a string and converts the final decoded string -back to the requested `TypeToken`. Every later stage is a typed `SerDesStage`. Adjacent stages must be -compatible: one stage's serialized output becomes the next stage's input, and deserialization applies the inverse -mapping. Runtime stage metadata is preserved in failures because Java type erasure prevents complete validation when -heterogeneous stages are held in one immutable pipeline. +The first component is the **value codec**. It converts the user value to representation `R` and converts the restored +`R` back to the requested `TypeToken`. The following chain contains typed `SerDesStage` instances. Adjacent +stages must be compatible: one stage's serialized output becomes the next stage's input, and deserialization applies +the inverse mapping. The complete chain must finish with `String`. Runtime stage metadata is preserved in failures +because Java type erasure prevents complete validation when heterogeneous stages are held in one immutable pipeline. Serialization runs from first to last: ```text Object -> value codec - -> String + -> representation R -> typed stage 1 -> intermediate type A -> typed stage 2 @@ -168,7 +179,7 @@ checkpoint String -> last typed stage -> ... -> first typed stage - -> String + -> representation R -> value codec, deserialized as the requested TypeToken -> T ``` @@ -193,19 +204,18 @@ String serialize(Object value) { break; } } - return valueCodec.deserialize(requireStringValueCodecInput(current), targetType); + return valueCodec.deserialize(requireValueCodecRepresentation(current), targetType); } ``` Pipeline rules: -- A pipeline must contain exactly one value codec in the first position and zero or more typed intermediate stages. -- Intermediate values may use any non-null Java type. The complete serialization pipeline must finish with a - `String`, and reverse processing must return a `String` to the value codec. +- A pipeline must contain exactly one TypeToken-aware value codec followed by a typed stage chain. +- The value codec representation and intermediate values may use any non-null Java type. The complete serialization + pipeline must finish with a `String`, and reverse processing must restore the value codec's representation type. - A stage may declare that it requires durable context or that it must be terminal. A terminal stage must be the last stage so later transformations cannot invalidate its checkpoint-size or storage decision. -- `SerDesStage.then(...)`, `ComposableSerDes.of(...)`, and the builder flatten nested stage chains while preserving - stage order. +- `SerDesStage.then(...)` and `ComposableSerDes.of(...)` flatten nested stage chains while preserving stage order. - A `null` value at the pipeline boundary short-circuits the entire pipeline: serializing or deserializing `null` returns `null` without invoking any stage. A stage returning `null` for non-null input is an error. The value codec may decode a non-null representation such as the JSON literal `null` to a null domain value. @@ -217,9 +227,11 @@ Pipeline rules: entity and payload-kind metadata around the pipeline failure. - A stage must be reversible. Compression, encryption, signing envelopes, and external payload storage are suitable stages; lossy redaction is not. -- A boundary stage may return `SerDesStageResult.decodeWithValueCodec(...)` when it receives raw data that did not pass - through the configured pipeline. `ComposableSerDes` then skips every earlier intermediate stage and decodes the raw value - directly with the value codec. +- A boundary stage may return `SerDesStageResult.decodeWithValueCodec(...)` when it receives raw string data that did + not pass through the configured pipeline. `ComposableSerDes` then skips every earlier stage and calls + `ValueSerDes.deserializeExternal(...)`. Existing string-valued `SerDes` implementations delegate that call to their + normal `deserialize(...)` method. A non-string value codec must override it when the pipeline may receive external + invocation, callback, or invoke-result payloads. - Stage order is meaningful. For example, `JSON -> compression -> encryption -> filesystem` writes encrypted, compressed data to the filesystem. `FileSystemSerDes` is terminal; placing encryption or any expanding transformation after it is rejected. @@ -237,11 +249,12 @@ The default `SerDesStage.then(...)` method provides independently reusable proce connects one chain to the persisted value codec: ```java -var secureStages = compressionStage +SerDesStage secureStages = compressionStage .then(encryptionSerDes) .then(fileSystemStage); -var securePayloads = ComposableSerDes.of(new JacksonSerDes(), secureStages); +ValueSerDes valueCodec = new JsonRepresentationSerDes(); +var securePayloads = ComposableSerDes.of(valueCodec, secureStages); ``` ### Retryable SerDes stages @@ -467,8 +480,9 @@ must not synthesize a durable context and serialize initial input through the fu 1. Add `SerDesContext`, `SerDesPayloadKind`, and package-private TLS setter/clearer support. Leave the existing `SerDes` methods unchanged. -2. Add `SerDesStage.then(...)` and `ComposableSerDes` with immutable stage ordering, forward serialization, reverse - deserialization, external-boundary bypass, terminal-stage validation, null short-circuiting, and stage-aware errors. +2. Add `ValueSerDes`, `SerDesStage.then(...)`, and `ComposableSerDes` with typed value representations, immutable + stage ordering, forward serialization, reverse deserialization, external-boundary bypass, terminal-stage validation, + null short-circuiting, and stage-aware errors. 3. Add `RetryableSerDesException` and `RetrySerDes`, reusing `RetryStrategy` for bounded in-invocation retries. 4. Add `SerDesRunner` with inline execution by default and optional dispatch through `DurableConfig.withSerDesExecutorService(...)`. Do not create a default SerDes pool. @@ -706,7 +720,7 @@ This approach gives the SDK one consistent policy for root payloads, operation r | Responsibility boundary | Combines value serialization and storage-reference creation in ordered, composable SerDes stages. | Keeps object encoding in `SerDes` and storage movement in a separate offloader. | | User configuration | Users configure one SerDes or a `ComposableSerDes` pipeline. Operation-level SerDes selection replaces the whole pipeline. | Users configure both a SerDes and an offloader. The SDK must define global, per-operation, and per-payload precedence. | | Parity with JS issue | Closest to the current JavaScript `createFileSystemSerdes` model and issue #463 wording. | Diverges from JavaScript naming and shape, though it may be architecturally cleaner for Java. | -| Core SDK changes | Adds `SerDesStage`, `ComposableSerDes`, and `SerDesContext` TLS while leaving `SerDes.then(...)` absent and the existing serialization methods unchanged. | Requires a new core extension point, envelope type, config surface, payload pipeline, and migration story. | +| Core SDK changes | Adds `ValueSerDes`, `SerDesStage`, `ComposableSerDes`, and `SerDesContext` TLS while leaving `SerDes.then(...)` absent and the existing serialization methods unchanged. | Requires a new core extension point, envelope type, config surface, payload pipeline, and migration story. | | Applicability | Any compatible SerDes stages can be chained, but storage stages still own their envelope and lifecycle behavior. | Any SerDes output can be offloaded uniformly after serialization. Users can combine Jackson/custom SerDes with any offloader. | | Envelope ownership | FileSystemSerDes owns the checkpoint envelope (`data`, `file`, preview), so the SDK treats it as opaque serialized data. | SDK owns the checkpoint/offload envelope and must guarantee it composes with replay, errors, callbacks, and test utilities. | | Caching | SDK can cache deserialized values, but FileSystemSerDes may still do file reads internally unless cache hits happen before SerDes. | SDK can cache at both layers: resolved offloaded payload text and final deserialized object. | @@ -718,8 +732,9 @@ This approach gives the SDK one consistent policy for root payloads, operation r ## Decision Adopt **Approach A: Reuse SerDes for Offload**, extended with a core `ComposableSerDes` pipeline. It delivers -JavaScript parity, includes filesystem behavior in the existing core artifact, leaves the existing `SerDes` methods unchanged, -and lets customers assemble value encoding, compression, encryption, and storage as independently reusable stages. +JavaScript parity, includes filesystem behavior in the existing core artifact, leaves the existing `SerDes` methods +unchanged, and lets customers assemble typed value encoding, compression, encryption, and storage as independently +reusable components. Approach B remains a possible future direction if payload offloading grows into a general multi-backend capability that requires SDK-owned storage envelopes and lifecycle policy. @@ -775,7 +790,8 @@ Positive: signatures. - Approach A makes filesystem-backed storage available from the core SDK without an additional artifact. - Custom payload implementations get enough context to use external storage safely. -- Customers can compose reusable SerDes stages without creating a bespoke wrapper for each combination. +- Customers can compose a non-string value representation and reusable SerDes stages without creating a bespoke + wrapper for each combination. - Blocking payload work can be isolated from user operation threads with an explicitly configured SerDes executor and never runs on the SDK coordination executor. - Repeated file reads and repeated object reconstruction can be reduced within an invocation. diff --git a/docs/advanced/configuration.md b/docs/advanced/configuration.md index 9477d6511..73e3c9b1f 100644 --- a/docs/advanced/configuration.md +++ b/docs/advanced/configuration.md @@ -75,10 +75,11 @@ return DurableConfig.builder() .build(); ``` -Pipelines may exchange non-string intermediate values. For example, a custom -`SerDesStage` can compress the JSON string and feed the resulting bytes directly into -`FileSystemSerDes`; reverse processing restores the bytes to the compression stage. The complete pipeline still -returns a string checkpoint envelope. +The value codec and pipeline stages may exchange non-string representations. Existing `SerDes` implementations are +`ValueSerDes` codecs, but a custom `ValueSerDes` may feed an `EncodedPayload` directly into the +first stage. For example, a stage chain can transform `EncodedPayload -> byte[] -> String` without an intermediate +string conversion. Reverse processing restores the same types in the opposite order. Only the complete pipeline must +return a string checkpoint envelope. `ALWAYS` stores every non-null payload in a file. `OVERFLOW` keeps payloads inline until the checkpoint envelope approaches the 256 KB service limit. `URI` produces readable escaped paths; `HASH` produces fixed-length SHA-256 path diff --git a/docs/advanced/filesystem-serdes.md b/docs/advanced/filesystem-serdes.md index ce83d7ad4..aa5e2f39d 100644 --- a/docs/advanced/filesystem-serdes.md +++ b/docs/advanced/filesystem-serdes.md @@ -37,11 +37,12 @@ return DurableConfig.builder() .build(); ``` -Serialization runs from `JacksonSerDes` to the filesystem stage. Deserialization runs in reverse. Additional reversible -typed stages such as compression or encryption compose through `SerDesStage.then(...)` before the stage chain is passed -to `ComposableSerDes.of(...)`. A -`SerDesStage` may pass compressed or encrypted bytes directly to `FileSystemSerDes`; no Base64 adapter -is required between those stages. The complete pipeline still produces a string checkpoint envelope. +Serialization runs from the value codec to the filesystem stage. Deserialization runs in reverse. Additional +reversible typed stages such as compression or encryption compose through `SerDesStage.then(...)` before the stage +chain is passed to `ComposableSerDes.of(...)`. The value codec may produce any representation type, so a custom +`ValueSerDes` can start a chain such as `EncodedPayload -> byte[] -> FileSystemSerDes`; no String or +Base64 adapter is required between those components. The complete pipeline still produces a string checkpoint +envelope. - `ALWAYS` writes every non-null payload to a file. - `OVERFLOW` stores small payloads inline and offloads envelopes approaching the 256 KB checkpoint limit. diff --git a/docs/design.md b/docs/design.md index 269e98500..62b5ee970 100644 --- a/docs/design.md +++ b/docs/design.md @@ -354,8 +354,9 @@ software.amazon.lambda.durable │ └── WaitForConditionResult # Check function return type (value + isDone) │ ├── serde/ -│ ├── SerDes # Interface and pipeline composition entry point -│ ├── SerDesStage # Reversible typed intermediate pipeline stage +│ ├── SerDes # Backward-compatible String value SerDes +│ ├── ValueSerDes # TypeToken-aware value codec with a typed representation +│ ├── SerDesStage # Reversible typed pipeline transformation │ ├── ComposableSerDes # Immutable ordered value-codec/typed-stage pipeline │ ├── JacksonSerDes # Jackson impl │ ├── RetrySerDes # Retry decorator for transient SerDes failures diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/ComposableSerDes.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/ComposableSerDes.java index 2277c7ed5..3847f60d5 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/ComposableSerDes.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/ComposableSerDes.java @@ -12,15 +12,17 @@ /** * An immutable SerDes processing pipeline. * - *

The first stage is the value codec. Later {@link SerDesStage} instances may exchange arbitrary intermediate Java - * types. Serialization runs from first to last; deserialization runs from last to first. The final serialized value and - * the value returned to the value codec during deserialization must be strings. + *

The value codec may produce any representation type. The {@link SerDesStage} chain starts with that representation + * and may exchange arbitrary intermediate Java types. Serialization runs from first to last; deserialization runs from + * last to first. Only the final serialized value must be a string. + * + * @param the representation exchanged between the value codec and the first stage */ -public final class ComposableSerDes implements SerDes { - private final SerDes valueCodec; +public final class ComposableSerDes implements SerDes { + private final ValueSerDes valueCodec; private final List> stages; - private ComposableSerDes(SerDes valueCodec, List> stages) { + private ComposableSerDes(ValueSerDes valueCodec, List> stages) { this.valueCodec = Objects.requireNonNull(valueCodec, "valueCodec cannot be null"); if (!stages.isEmpty() && valueCodec.isTerminalPipelineStage()) { throw terminalStageFailure(0, valueCodec); @@ -39,33 +41,27 @@ private ComposableSerDes(SerDes valueCodec, List> stages) { * @param valueCodec the value codec * @return an immutable pipeline */ - public static ComposableSerDes of(SerDes valueCodec) { - return builder(valueCodec).build(); + public static ComposableSerDes of(SerDes valueCodec) { + return new ComposableSerDes<>(valueCodec, List.of()); } /** - * Creates a pipeline with a value codec followed by a typed stage or stage chain. + * Creates a pipeline with a typed value codec followed by a stage or stage chain that produces the checkpoint + * string. * * @param valueCodec the value codec * @param stage the reversible typed stage or stage chain + * @param the representation exchanged between the value codec and the first stage * @return an immutable pipeline */ - public static ComposableSerDes of(SerDes valueCodec, SerDesStage stage) { - return builder(valueCodec).then(stage).build(); - } - - /** - * Creates a pipeline builder. - * - * @param valueCodec the first stage which converts values to and from strings - * @return a new builder - */ - public static Builder builder(SerDes valueCodec) { - return new Builder(valueCodec); + public static ComposableSerDes of(ValueSerDes valueCodec, SerDesStage stage) { + var stages = new ArrayList>(); + addFlattened(stages, Objects.requireNonNull(stage, "stage cannot be null")); + return new ComposableSerDes<>(valueCodec, stages); } /** Returns the value codec at the start of this pipeline. */ - public SerDes getValueCodec() { + public ValueSerDes getValueCodec() { return valueCodec; } @@ -104,19 +100,24 @@ public T deserialize(String data, TypeToken typeToken) { } Objects.requireNonNull(typeToken, "typeToken cannot be null"); Object current = data; + boolean skipToExternalCodec = false; for (int index = stages.size() - 1; index >= 0; index--) { var decoded = invokeStageDeserialize(stages.get(index), current, index + 1); current = decoded.value(); if (decoded.skipRemainingStages()) { + skipToExternalCodec = true; break; } } - if (!(current instanceof String valueCodecInput)) { - throw new SerDesException("SerDes pipeline produced " - + current.getClass().getName() - + " instead of String for the value codec"); + if (skipToExternalCodec) { + if (!(current instanceof String externalInput)) { + throw new SerDesException("SerDes pipeline external boundary produced " + + current.getClass().getName() + + " instead of String"); + } + return invokeExternalValueCodecDeserialize(valueCodec, externalInput, typeToken); } - return invokeValueCodecDeserialize(valueCodec, valueCodecInput, typeToken); + return invokeValueCodecDeserialize(valueCodec, current, typeToken); } @SuppressWarnings("unchecked") @@ -145,7 +146,7 @@ private static SerDesStageResult invokeStageDeserialize(SerDesStage stage, } } - private static String invokeValueCodecSerialize(SerDes valueCodec, Object value) { + private static Object invokeValueCodecSerialize(ValueSerDes valueCodec, Object value) { try { var result = valueCodec.serialize(value); if (result == null) { @@ -157,14 +158,25 @@ private static String invokeValueCodecSerialize(SerDes valueCodec, Object value) } } - private static T invokeValueCodecDeserialize(SerDes valueCodec, String data, TypeToken typeToken) { + @SuppressWarnings("unchecked") + private static T invokeValueCodecDeserialize( + ValueSerDes valueCodec, Object data, TypeToken typeToken) { try { - return valueCodec.deserialize(data, typeToken); + return valueCodec.deserialize((R) data, typeToken); } catch (Throwable failure) { throw stageFailure(0, valueCodec, "deserialize", failure); } } + private static T invokeExternalValueCodecDeserialize( + ValueSerDes valueCodec, String data, TypeToken typeToken) { + try { + return valueCodec.deserializeExternal(data, typeToken); + } catch (Throwable failure) { + throw stageFailure(0, valueCodec, "deserialize external payload", failure); + } + } + private static RuntimeException stageFailure(int index, Object stage, String action, Throwable failure) { if (failure instanceof Error error) { throw error; @@ -191,29 +203,4 @@ private static void addFlattened(List> target, SerDesStage> stages = new ArrayList<>(); - - private Builder(SerDes valueCodec) { - this.valueCodec = Objects.requireNonNull(valueCodec, "valueCodec cannot be null"); - if (valueCodec instanceof ComposableSerDes composable) { - this.valueCodec = composable.valueCodec; - stages.addAll(composable.stages); - } - } - - /** Appends a reversible typed stage. */ - public Builder then(SerDesStage stage) { - addFlattened(stages, Objects.requireNonNull(stage, "stage cannot be null")); - return this; - } - - /** Returns the immutable pipeline. */ - public ComposableSerDes build() { - return new ComposableSerDes(valueCodec, stages); - } - } } diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDes.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDes.java index 7c656378e..6021b3ed3 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDes.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDes.java @@ -5,8 +5,13 @@ import software.amazon.lambda.durable.TypeToken; import software.amazon.lambda.durable.exception.SerDesException; -/** Interface for serialization and deserialization of objects at the persisted string boundary. */ -public interface SerDes { +/** + * Interface for serialization and deserialization of objects at the persisted string boundary. + * + *

This is the backward-compatible string specialization of {@link ValueSerDes}. Use {@code ValueSerDes} with + * {@link ComposableSerDes} when the first pipeline stage should consume a non-string representation. + */ +public interface SerDes extends ValueSerDes { /** * Serializes an object to a JSON string. * @@ -34,6 +39,11 @@ public interface SerDes { */ T deserialize(String data, TypeToken typeToken); + @Override + default T deserializeExternal(String data, TypeToken typeToken) { + return deserialize(data, typeToken); + } + /** * Deserializes this SerDes when it is used as an intermediate pipeline stage. * diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesStageResult.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesStageResult.java index ed7c3a939..5d484e657 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesStageResult.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesStageResult.java @@ -8,8 +8,8 @@ * Result returned when a {@link SerDesStage} is reversed in a {@link ComposableSerDes}. * * @param value the non-null value produced by the stage - * @param skipRemainingStages whether deserialization should skip the remaining intermediate stages and decode - * {@code value} directly with the pipeline's value codec + * @param skipRemainingStages whether deserialization should skip the remaining stages and decode {@code value} as an + * external string with the pipeline's value codec */ public record SerDesStageResult(Object value, boolean skipRemainingStages) { public SerDesStageResult { @@ -21,7 +21,7 @@ public static SerDesStageResult continueWith(Object value) { return new SerDesStageResult(value, false); } - /** Skips the remaining intermediate stages and decodes the value directly with the pipeline's value codec. */ + /** Skips the remaining stages and decodes the external string with the pipeline's value codec. */ public static SerDesStageResult decodeWithValueCodec(String value) { return new SerDesStageResult(value, true); } diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/ValueSerDes.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/ValueSerDes.java new file mode 100644 index 000000000..e6db2237b --- /dev/null +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/ValueSerDes.java @@ -0,0 +1,60 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.serde; + +import software.amazon.lambda.durable.TypeToken; +import software.amazon.lambda.durable.exception.SerDesException; + +/** + * Converts domain values to and from a typed pipeline representation. + * + *

The representation may be any Java type. A {@link ComposableSerDes} connects it to a {@link SerDesStage} chain + * whose final serialized representation is the checkpoint {@link String}. + * + * @param the serialized representation produced for the first pipeline stage + */ +public interface ValueSerDes { + /** + * Serializes a domain value to the representation consumed by the first pipeline stage. + * + * @param value the domain value + * @return the serialized representation, or null if value is null + */ + R serialize(Object value); + + /** + * Deserializes the representation to the requested domain type. + * + * @param data the representation restored by the stage chain + * @param typeToken the requested domain type + * @param the requested domain type + * @return the deserialized value, or null if data is null + */ + T deserialize(R data, TypeToken typeToken); + + /** + * Deserializes an unframed external string that did not pass through the configured stage chain. + * + *

Codecs that can receive external invocation, callback, or invoke-result payloads should override this method. + * String-valued {@link SerDes} implementations support it automatically. + * + * @param data the external string payload + * @param typeToken the requested domain type + * @param the requested domain type + * @return the deserialized value + */ + default T deserializeExternal(String data, TypeToken typeToken) { + throw new SerDesException( + "Value SerDes " + getClass().getName() + " cannot deserialize an external String payload"); + } + + /** Returns whether this value SerDes requires an SDK-managed durable execution context. */ + default boolean requiresDurableContext() { + return false; + } + + /** Returns whether this value SerDes must be the final component of a composable pipeline. */ + default boolean isTerminalPipelineStage() { + return false; + } +} diff --git a/sdk/src/test/java/software/amazon/lambda/durable/serde/ComposableSerDesTest.java b/sdk/src/test/java/software/amazon/lambda/durable/serde/ComposableSerDesTest.java index 62d9a0a0e..1b51c500f 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/serde/ComposableSerDesTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/serde/ComposableSerDesTest.java @@ -37,19 +37,33 @@ void serializesForwardAndDeserializesInReverse() { } @Test - void supportsTypedIntermediateValues() { + void supportsCustomValueCodecAndIntermediateTypes() { var calls = new ArrayList(); - SerDesStage utf8 = new SerDesStage<>() { + var jackson = new JacksonSerDes(); + ValueSerDes valueCodec = new ValueSerDes<>() { @Override - public byte[] serialize(String value) { + public JsonRepresentation serialize(Object value) { + calls.add("codec-serialize"); + return new JsonRepresentation(jackson.serialize(value)); + } + + @Override + public T deserialize(JsonRepresentation data, TypeToken typeToken) { + calls.add("codec-deserialize"); + return jackson.deserialize(data.value(), typeToken); + } + }; + SerDesStage utf8 = new SerDesStage<>() { + @Override + public byte[] serialize(JsonRepresentation value) { calls.add("bytes-serialize"); - return value.getBytes(StandardCharsets.UTF_8); + return value.value().getBytes(StandardCharsets.UTF_8); } @Override - public String deserialize(byte[] data) { + public JsonRepresentation deserialize(byte[] data) { calls.add("bytes-deserialize"); - return new String(data, StandardCharsets.UTF_8); + return new JsonRepresentation(new String(data, StandardCharsets.UTF_8)); } }; SerDesStage base64 = new SerDesStage<>() { @@ -65,22 +79,29 @@ public byte[] deserialize(String data) { return Base64.getDecoder().decode(data); } }; - var pipeline = ComposableSerDes.of(new JacksonSerDes(), utf8.then(base64)); + var pipeline = ComposableSerDes.of(valueCodec, utf8.then(base64)); var serialized = pipeline.serialize("value"); var deserialized = pipeline.deserialize(serialized, TypeToken.get(String.class)); assertEquals(Base64.getEncoder().encodeToString("\"value\"".getBytes(StandardCharsets.UTF_8)), serialized); assertEquals("value", deserialized); - assertEquals(List.of("bytes-serialize", "base64-serialize", "base64-deserialize", "bytes-deserialize"), calls); + assertEquals( + List.of( + "codec-serialize", + "bytes-serialize", + "base64-serialize", + "base64-deserialize", + "bytes-deserialize", + "codec-deserialize"), + calls); } @Test - void factoryBuilderAndThenFlattenNestedStageChains() { + void factoryFlattensNestedStageChains() { var calls = new ArrayList(); var nested = stringStage("one", "1", "1", calls).then(stringStage("two", "2", "2", calls)); - var pipeline = - ComposableSerDes.builder(new JacksonSerDes()).then(nested).build(); + var pipeline = ComposableSerDes.of(new JacksonSerDes(), nested); assertEquals("21\"value\"12", pipeline.serialize("value")); assertEquals(List.of("one-serialize", "two-serialize"), calls); @@ -167,6 +188,49 @@ public SerDesStageResult deserializePipelineStage(String data) { assertEquals(0, transformDeserializations.get()); } + @Test + void customValueCodecMayDecodeExternalStringPayload() { + var jackson = new JacksonSerDes(); + var externalDeserializations = new AtomicInteger(); + ValueSerDes valueCodec = new ValueSerDes<>() { + @Override + public JsonRepresentation serialize(Object value) { + return new JsonRepresentation(jackson.serialize(value)); + } + + @Override + public T deserialize(JsonRepresentation data, TypeToken typeToken) { + return jackson.deserialize(data.value(), typeToken); + } + + @Override + public T deserializeExternal(String data, TypeToken typeToken) { + externalDeserializations.incrementAndGet(); + return jackson.deserialize(data, typeToken); + } + }; + SerDesStage externalBoundary = new SerDesStage<>() { + @Override + public String serialize(JsonRepresentation value) { + return value.value(); + } + + @Override + public JsonRepresentation deserialize(String data) { + return new JsonRepresentation(data); + } + + @Override + public SerDesStageResult deserializePipelineStage(String data) { + return SerDesStageResult.decodeWithValueCodec(data); + } + }; + var pipeline = ComposableSerDes.of(valueCodec, externalBoundary); + + assertEquals("value", pipeline.deserialize("\"value\"", TypeToken.get(String.class))); + assertEquals(1, externalDeserializations.get()); + } + @Test void rejectsStagesAfterTerminalStage() { var terminal = new SerDesStage() { @@ -194,7 +258,8 @@ public boolean isTerminalPipelineStage() { } @Test - void rejectsNullIntermediateAndNonStringBoundaryValues() { + @SuppressWarnings({"rawtypes", "unchecked"}) + void rejectsNullIntermediateAndNonStringBoundaryValuesFromUnsafeRawStages() { var nullStage = new SerDesStage() { @Override public String serialize(String value) { @@ -222,15 +287,16 @@ public String deserialize(Integer data) { return "x".repeat(data); } }; - var typeFailure = - assertThrows(SerDesException.class, () -> ComposableSerDes.of(new JacksonSerDes(), nonStringFinalStage) + var typeFailure = assertThrows( + SerDesException.class, () -> ComposableSerDes.of(new JacksonSerDes(), (SerDesStage) nonStringFinalStage) .serialize("value")); assertTrue(typeFailure.getMessage().contains("final stage")); assertTrue(typeFailure.getMessage().contains(Integer.class.getName())); } @Test - void incompatibleTypedStagesFailWithStageMetadata() { + @SuppressWarnings({"rawtypes", "unchecked"}) + void incompatibleUnsafeRawStagesFailWithStageMetadata() { SerDesStage integerStage = new SerDesStage<>() { @Override public String serialize(Integer value) { @@ -243,8 +309,9 @@ public Integer deserialize(String data) { } }; - var failure = assertThrows(SerDesException.class, () -> ComposableSerDes.of(new JacksonSerDes(), integerStage) - .serialize("value")); + var failure = assertThrows( + SerDesException.class, () -> ComposableSerDes.of(new JacksonSerDes(), (SerDesStage) integerStage) + .serialize("value")); assertTrue(failure.getMessage().contains("stage 1")); assertInstanceOf(ClassCastException.class, failure.getCause()); @@ -343,4 +410,6 @@ public String deserialize(String data) { } }; } + + private record JsonRepresentation(String value) {} } diff --git a/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java b/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java index 850947f9f..d0cc47146 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java @@ -115,6 +115,35 @@ public String deserialize(byte[] data) { runner.deserialize(pipeline, envelope, new TypeToken>() {}, context())); } + @Test + void stageModeAcceptsNonStringValueCodecRepresentation() throws Exception { + var jackson = new JacksonSerDes(); + ValueSerDes valueCodec = new ValueSerDes<>() { + @Override + public byte[] serialize(Object value) { + return jackson.serialize(value).getBytes(StandardCharsets.UTF_8); + } + + @Override + public T deserialize(byte[] data, TypeToken typeToken) { + return jackson.deserialize(new String(data, StandardCharsets.UTF_8), typeToken); + } + }; + var stage = FileSystemSerDes.stageBuilder(basePath).build(); + var pipeline = ComposableSerDes.of(valueCodec, stage); + var runner = new SerDesRunner(null); + + var envelope = runner.serialize(pipeline, Map.of("id", 42), context()); + var json = MAPPER.readTree(envelope); + var file = Path.of(json.get("file").textValue()); + + assertEquals("BYTES", json.get("payloadType").textValue()); + assertEquals("{\"id\":42}", new String(Files.readAllBytes(file), StandardCharsets.UTF_8)); + assertEquals( + Map.of("id", 42), + runner.deserialize(pipeline, envelope, new TypeToken>() {}, context())); + } + @Test void overflowModeKeepsSmallBinaryPayloadsInline() throws Exception { var stage = FileSystemSerDes.stageBuilder(basePath) From e1b8469c82fd650b30336399f05925ca50c523c7 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Tue, 25 Aug 2026 14:27:14 +0000 Subject: [PATCH 19/56] Revert "refactor: allow typed value serde representations" This reverts commit 54665b28e48761ab2126102465cf76c2b4bca9ab. --- docs/adr/005-filesystem-serdes.md | 108 ++++++++---------- docs/advanced/configuration.md | 9 +- docs/advanced/filesystem-serdes.md | 11 +- docs/design.md | 5 +- .../durable/serde/ComposableSerDes.java | 97 +++++++++------- .../amazon/lambda/durable/serde/SerDes.java | 14 +-- .../durable/serde/SerDesStageResult.java | 6 +- .../lambda/durable/serde/ValueSerDes.java | 60 ---------- .../durable/serde/ComposableSerDesTest.java | 103 +++-------------- .../durable/serde/FileSystemSerDesTest.java | 29 ----- 10 files changed, 134 insertions(+), 308 deletions(-) delete mode 100644 sdk/src/main/java/software/amazon/lambda/durable/serde/ValueSerDes.java diff --git a/docs/adr/005-filesystem-serdes.md b/docs/adr/005-filesystem-serdes.md index ae0ce4e26..56dbce75b 100644 --- a/docs/adr/005-filesystem-serdes.md +++ b/docs/adr/005-filesystem-serdes.md @@ -2,8 +2,7 @@ **Status:** Accepted — Approach A with ComposableSerDes pipeline **Date:** 2026-07-02 -**Updated:** 2026-08-25 — Included FileSystemSerDes in core and generalized the value codec and pipeline stages to -typed representations. +**Updated:** 2026-08-25 — Included FileSystemSerDes in core and generalized pipelines to typed intermediate stages. ## Context @@ -32,27 +31,13 @@ There are a few Java-specific constraints: ### Summary -Keep the existing `SerDes` serialization methods source- and binary-compatible, define it as the string specialization -of a typed `ValueSerDes` value codec, add a typed `SerDesStage` contract and a core `ComposableSerDes` -implementation, and implement `FileSystemSerDes` in the core SDK. The value codec and stages may exchange arbitrary -Java types; only the complete `SerDes` pipeline must produce the checkpoint string. The filesystem stage uses +Keep the existing `SerDes` serialization methods source- and binary-compatible, add a typed `SerDesStage` +contract and a core `ComposableSerDes` implementation which together form a processing pipeline, and implement +`FileSystemSerDes` in the core SDK. `SerDes` remains the persisted value-codec boundary, while intermediate +`SerDesStage` instances may exchange arbitrary Java types and compose independently. The filesystem stage uses `SerDesContext.getCurrentContext()` to identify the durable execution and entity being serialized. ```java -public interface ValueSerDes { - R serialize(Object value); - - T deserialize(R data, TypeToken typeToken); - - default T deserializeExternal(String data, TypeToken typeToken); -} - -public interface SerDes extends ValueSerDes { - String serialize(Object value); - - T deserialize(String data, TypeToken typeToken); -} - public interface SerDesStage { O serialize(I value); @@ -64,11 +49,10 @@ public interface SerDesStage { } ``` -`ComposableSerDes` treats the `ValueSerDes` as the TypeToken-aware value codec and connects its representation -directly to a reversible typed stage chain. The first `SerDesStage` therefore consumes `R`, which does not need to be a -string. This lets customers compose JSON trees, binary encoding, compression, encryption, filesystem storage, or other -processing without artificial string conversion between stages. The complete pipeline still returns a string because -checkpoints use the existing `SerDes` boundary. +`ComposableSerDes` treats the first stage as the value codec and every later stage as a reversible typed +transformation. This lets customers compose JSON encoding, binary compression, encryption, filesystem storage, or +other processing without artificial Base64 conversion between every stage. The complete pipeline still returns a +string because checkpoints use the existing `SerDes` boundary. `FileSystemSerDes` acts as a terminal payload-storage stage. It writes the string or byte array produced by the previous stage to the filesystem when configured to do so and returns a small checkpoint envelope. For standalone @@ -132,18 +116,23 @@ return DurableConfig.builder() ### Composable SerDes pipeline -`ComposableSerDes` is a core implementation of `SerDes`. It owns a typed value codec and immutable stage chain while -preserving the existing persisted-string `serialize` and `deserialize` methods: +`ComposableSerDes` is a core implementation of `SerDes`. It owns an immutable, ordered list of stages while preserving +the existing `serialize` and `deserialize` methods: ```java -public final class ComposableSerDes implements SerDes { - public static ComposableSerDes of(SerDes valueCodec); +public final class ComposableSerDes implements SerDes { + public static ComposableSerDes of(SerDes valueCodec); + public static ComposableSerDes of(SerDes valueCodec, SerDesStage stage); + + public static Builder builder(SerDes valueCodec); - public static ComposableSerDes of( - ValueSerDes valueCodec, - SerDesStage stage); + public SerDes getValueCodec(); - public ValueSerDes getValueCodec(); + public static final class Builder { + public Builder then(SerDesStage stage); + + public ComposableSerDes build(); + } } public record SerDesStageResult(Object value, boolean skipRemainingStages) { @@ -153,18 +142,18 @@ public record SerDesStageResult(Object value, boolean skipRemainingStages) { } ``` -The first component is the **value codec**. It converts the user value to representation `R` and converts the restored -`R` back to the requested `TypeToken`. The following chain contains typed `SerDesStage` instances. Adjacent -stages must be compatible: one stage's serialized output becomes the next stage's input, and deserialization applies -the inverse mapping. The complete chain must finish with `String`. Runtime stage metadata is preserved in failures -because Java type erasure prevents complete validation when heterogeneous stages are held in one immutable pipeline. +The first stage is the **value codec**. It converts the user value to a string and converts the final decoded string +back to the requested `TypeToken`. Every later stage is a typed `SerDesStage`. Adjacent stages must be +compatible: one stage's serialized output becomes the next stage's input, and deserialization applies the inverse +mapping. Runtime stage metadata is preserved in failures because Java type erasure prevents complete validation when +heterogeneous stages are held in one immutable pipeline. Serialization runs from first to last: ```text Object -> value codec - -> representation R + -> String -> typed stage 1 -> intermediate type A -> typed stage 2 @@ -179,7 +168,7 @@ checkpoint String -> last typed stage -> ... -> first typed stage - -> representation R + -> String -> value codec, deserialized as the requested TypeToken -> T ``` @@ -204,18 +193,19 @@ String serialize(Object value) { break; } } - return valueCodec.deserialize(requireValueCodecRepresentation(current), targetType); + return valueCodec.deserialize(requireStringValueCodecInput(current), targetType); } ``` Pipeline rules: -- A pipeline must contain exactly one TypeToken-aware value codec followed by a typed stage chain. -- The value codec representation and intermediate values may use any non-null Java type. The complete serialization - pipeline must finish with a `String`, and reverse processing must restore the value codec's representation type. +- A pipeline must contain exactly one value codec in the first position and zero or more typed intermediate stages. +- Intermediate values may use any non-null Java type. The complete serialization pipeline must finish with a + `String`, and reverse processing must return a `String` to the value codec. - A stage may declare that it requires durable context or that it must be terminal. A terminal stage must be the last stage so later transformations cannot invalidate its checkpoint-size or storage decision. -- `SerDesStage.then(...)` and `ComposableSerDes.of(...)` flatten nested stage chains while preserving stage order. +- `SerDesStage.then(...)`, `ComposableSerDes.of(...)`, and the builder flatten nested stage chains while preserving + stage order. - A `null` value at the pipeline boundary short-circuits the entire pipeline: serializing or deserializing `null` returns `null` without invoking any stage. A stage returning `null` for non-null input is an error. The value codec may decode a non-null representation such as the JSON literal `null` to a null domain value. @@ -227,11 +217,9 @@ Pipeline rules: entity and payload-kind metadata around the pipeline failure. - A stage must be reversible. Compression, encryption, signing envelopes, and external payload storage are suitable stages; lossy redaction is not. -- A boundary stage may return `SerDesStageResult.decodeWithValueCodec(...)` when it receives raw string data that did - not pass through the configured pipeline. `ComposableSerDes` then skips every earlier stage and calls - `ValueSerDes.deserializeExternal(...)`. Existing string-valued `SerDes` implementations delegate that call to their - normal `deserialize(...)` method. A non-string value codec must override it when the pipeline may receive external - invocation, callback, or invoke-result payloads. +- A boundary stage may return `SerDesStageResult.decodeWithValueCodec(...)` when it receives raw data that did not pass + through the configured pipeline. `ComposableSerDes` then skips every earlier intermediate stage and decodes the raw value + directly with the value codec. - Stage order is meaningful. For example, `JSON -> compression -> encryption -> filesystem` writes encrypted, compressed data to the filesystem. `FileSystemSerDes` is terminal; placing encryption or any expanding transformation after it is rejected. @@ -249,12 +237,11 @@ The default `SerDesStage.then(...)` method provides independently reusable proce connects one chain to the persisted value codec: ```java -SerDesStage secureStages = compressionStage +var secureStages = compressionStage .then(encryptionSerDes) .then(fileSystemStage); -ValueSerDes valueCodec = new JsonRepresentationSerDes(); -var securePayloads = ComposableSerDes.of(valueCodec, secureStages); +var securePayloads = ComposableSerDes.of(new JacksonSerDes(), secureStages); ``` ### Retryable SerDes stages @@ -480,9 +467,8 @@ must not synthesize a durable context and serialize initial input through the fu 1. Add `SerDesContext`, `SerDesPayloadKind`, and package-private TLS setter/clearer support. Leave the existing `SerDes` methods unchanged. -2. Add `ValueSerDes`, `SerDesStage.then(...)`, and `ComposableSerDes` with typed value representations, immutable - stage ordering, forward serialization, reverse deserialization, external-boundary bypass, terminal-stage validation, - null short-circuiting, and stage-aware errors. +2. Add `SerDesStage.then(...)` and `ComposableSerDes` with immutable stage ordering, forward serialization, reverse + deserialization, external-boundary bypass, terminal-stage validation, null short-circuiting, and stage-aware errors. 3. Add `RetryableSerDesException` and `RetrySerDes`, reusing `RetryStrategy` for bounded in-invocation retries. 4. Add `SerDesRunner` with inline execution by default and optional dispatch through `DurableConfig.withSerDesExecutorService(...)`. Do not create a default SerDes pool. @@ -720,7 +706,7 @@ This approach gives the SDK one consistent policy for root payloads, operation r | Responsibility boundary | Combines value serialization and storage-reference creation in ordered, composable SerDes stages. | Keeps object encoding in `SerDes` and storage movement in a separate offloader. | | User configuration | Users configure one SerDes or a `ComposableSerDes` pipeline. Operation-level SerDes selection replaces the whole pipeline. | Users configure both a SerDes and an offloader. The SDK must define global, per-operation, and per-payload precedence. | | Parity with JS issue | Closest to the current JavaScript `createFileSystemSerdes` model and issue #463 wording. | Diverges from JavaScript naming and shape, though it may be architecturally cleaner for Java. | -| Core SDK changes | Adds `ValueSerDes`, `SerDesStage`, `ComposableSerDes`, and `SerDesContext` TLS while leaving `SerDes.then(...)` absent and the existing serialization methods unchanged. | Requires a new core extension point, envelope type, config surface, payload pipeline, and migration story. | +| Core SDK changes | Adds `SerDesStage`, `ComposableSerDes`, and `SerDesContext` TLS while leaving `SerDes.then(...)` absent and the existing serialization methods unchanged. | Requires a new core extension point, envelope type, config surface, payload pipeline, and migration story. | | Applicability | Any compatible SerDes stages can be chained, but storage stages still own their envelope and lifecycle behavior. | Any SerDes output can be offloaded uniformly after serialization. Users can combine Jackson/custom SerDes with any offloader. | | Envelope ownership | FileSystemSerDes owns the checkpoint envelope (`data`, `file`, preview), so the SDK treats it as opaque serialized data. | SDK owns the checkpoint/offload envelope and must guarantee it composes with replay, errors, callbacks, and test utilities. | | Caching | SDK can cache deserialized values, but FileSystemSerDes may still do file reads internally unless cache hits happen before SerDes. | SDK can cache at both layers: resolved offloaded payload text and final deserialized object. | @@ -732,9 +718,8 @@ This approach gives the SDK one consistent policy for root payloads, operation r ## Decision Adopt **Approach A: Reuse SerDes for Offload**, extended with a core `ComposableSerDes` pipeline. It delivers -JavaScript parity, includes filesystem behavior in the existing core artifact, leaves the existing `SerDes` methods -unchanged, and lets customers assemble typed value encoding, compression, encryption, and storage as independently -reusable components. +JavaScript parity, includes filesystem behavior in the existing core artifact, leaves the existing `SerDes` methods unchanged, +and lets customers assemble value encoding, compression, encryption, and storage as independently reusable stages. Approach B remains a possible future direction if payload offloading grows into a general multi-backend capability that requires SDK-owned storage envelopes and lifecycle policy. @@ -790,8 +775,7 @@ Positive: signatures. - Approach A makes filesystem-backed storage available from the core SDK without an additional artifact. - Custom payload implementations get enough context to use external storage safely. -- Customers can compose a non-string value representation and reusable SerDes stages without creating a bespoke - wrapper for each combination. +- Customers can compose reusable SerDes stages without creating a bespoke wrapper for each combination. - Blocking payload work can be isolated from user operation threads with an explicitly configured SerDes executor and never runs on the SDK coordination executor. - Repeated file reads and repeated object reconstruction can be reduced within an invocation. diff --git a/docs/advanced/configuration.md b/docs/advanced/configuration.md index 73e3c9b1f..9477d6511 100644 --- a/docs/advanced/configuration.md +++ b/docs/advanced/configuration.md @@ -75,11 +75,10 @@ return DurableConfig.builder() .build(); ``` -The value codec and pipeline stages may exchange non-string representations. Existing `SerDes` implementations are -`ValueSerDes` codecs, but a custom `ValueSerDes` may feed an `EncodedPayload` directly into the -first stage. For example, a stage chain can transform `EncodedPayload -> byte[] -> String` without an intermediate -string conversion. Reverse processing restores the same types in the opposite order. Only the complete pipeline must -return a string checkpoint envelope. +Pipelines may exchange non-string intermediate values. For example, a custom +`SerDesStage` can compress the JSON string and feed the resulting bytes directly into +`FileSystemSerDes`; reverse processing restores the bytes to the compression stage. The complete pipeline still +returns a string checkpoint envelope. `ALWAYS` stores every non-null payload in a file. `OVERFLOW` keeps payloads inline until the checkpoint envelope approaches the 256 KB service limit. `URI` produces readable escaped paths; `HASH` produces fixed-length SHA-256 path diff --git a/docs/advanced/filesystem-serdes.md b/docs/advanced/filesystem-serdes.md index aa5e2f39d..ce83d7ad4 100644 --- a/docs/advanced/filesystem-serdes.md +++ b/docs/advanced/filesystem-serdes.md @@ -37,12 +37,11 @@ return DurableConfig.builder() .build(); ``` -Serialization runs from the value codec to the filesystem stage. Deserialization runs in reverse. Additional -reversible typed stages such as compression or encryption compose through `SerDesStage.then(...)` before the stage -chain is passed to `ComposableSerDes.of(...)`. The value codec may produce any representation type, so a custom -`ValueSerDes` can start a chain such as `EncodedPayload -> byte[] -> FileSystemSerDes`; no String or -Base64 adapter is required between those components. The complete pipeline still produces a string checkpoint -envelope. +Serialization runs from `JacksonSerDes` to the filesystem stage. Deserialization runs in reverse. Additional reversible +typed stages such as compression or encryption compose through `SerDesStage.then(...)` before the stage chain is passed +to `ComposableSerDes.of(...)`. A +`SerDesStage` may pass compressed or encrypted bytes directly to `FileSystemSerDes`; no Base64 adapter +is required between those stages. The complete pipeline still produces a string checkpoint envelope. - `ALWAYS` writes every non-null payload to a file. - `OVERFLOW` stores small payloads inline and offloads envelopes approaching the 256 KB checkpoint limit. diff --git a/docs/design.md b/docs/design.md index 62b5ee970..269e98500 100644 --- a/docs/design.md +++ b/docs/design.md @@ -354,9 +354,8 @@ software.amazon.lambda.durable │ └── WaitForConditionResult # Check function return type (value + isDone) │ ├── serde/ -│ ├── SerDes # Backward-compatible String value SerDes -│ ├── ValueSerDes # TypeToken-aware value codec with a typed representation -│ ├── SerDesStage # Reversible typed pipeline transformation +│ ├── SerDes # Interface and pipeline composition entry point +│ ├── SerDesStage # Reversible typed intermediate pipeline stage │ ├── ComposableSerDes # Immutable ordered value-codec/typed-stage pipeline │ ├── JacksonSerDes # Jackson impl │ ├── RetrySerDes # Retry decorator for transient SerDes failures diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/ComposableSerDes.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/ComposableSerDes.java index 3847f60d5..2277c7ed5 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/ComposableSerDes.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/ComposableSerDes.java @@ -12,17 +12,15 @@ /** * An immutable SerDes processing pipeline. * - *

The value codec may produce any representation type. The {@link SerDesStage} chain starts with that representation - * and may exchange arbitrary intermediate Java types. Serialization runs from first to last; deserialization runs from - * last to first. Only the final serialized value must be a string. - * - * @param the representation exchanged between the value codec and the first stage + *

The first stage is the value codec. Later {@link SerDesStage} instances may exchange arbitrary intermediate Java + * types. Serialization runs from first to last; deserialization runs from last to first. The final serialized value and + * the value returned to the value codec during deserialization must be strings. */ -public final class ComposableSerDes implements SerDes { - private final ValueSerDes valueCodec; +public final class ComposableSerDes implements SerDes { + private final SerDes valueCodec; private final List> stages; - private ComposableSerDes(ValueSerDes valueCodec, List> stages) { + private ComposableSerDes(SerDes valueCodec, List> stages) { this.valueCodec = Objects.requireNonNull(valueCodec, "valueCodec cannot be null"); if (!stages.isEmpty() && valueCodec.isTerminalPipelineStage()) { throw terminalStageFailure(0, valueCodec); @@ -41,27 +39,33 @@ private ComposableSerDes(ValueSerDes valueCodec, List> stag * @param valueCodec the value codec * @return an immutable pipeline */ - public static ComposableSerDes of(SerDes valueCodec) { - return new ComposableSerDes<>(valueCodec, List.of()); + public static ComposableSerDes of(SerDes valueCodec) { + return builder(valueCodec).build(); } /** - * Creates a pipeline with a typed value codec followed by a stage or stage chain that produces the checkpoint - * string. + * Creates a pipeline with a value codec followed by a typed stage or stage chain. * * @param valueCodec the value codec * @param stage the reversible typed stage or stage chain - * @param the representation exchanged between the value codec and the first stage * @return an immutable pipeline */ - public static ComposableSerDes of(ValueSerDes valueCodec, SerDesStage stage) { - var stages = new ArrayList>(); - addFlattened(stages, Objects.requireNonNull(stage, "stage cannot be null")); - return new ComposableSerDes<>(valueCodec, stages); + public static ComposableSerDes of(SerDes valueCodec, SerDesStage stage) { + return builder(valueCodec).then(stage).build(); + } + + /** + * Creates a pipeline builder. + * + * @param valueCodec the first stage which converts values to and from strings + * @return a new builder + */ + public static Builder builder(SerDes valueCodec) { + return new Builder(valueCodec); } /** Returns the value codec at the start of this pipeline. */ - public ValueSerDes getValueCodec() { + public SerDes getValueCodec() { return valueCodec; } @@ -100,24 +104,19 @@ public T deserialize(String data, TypeToken typeToken) { } Objects.requireNonNull(typeToken, "typeToken cannot be null"); Object current = data; - boolean skipToExternalCodec = false; for (int index = stages.size() - 1; index >= 0; index--) { var decoded = invokeStageDeserialize(stages.get(index), current, index + 1); current = decoded.value(); if (decoded.skipRemainingStages()) { - skipToExternalCodec = true; break; } } - if (skipToExternalCodec) { - if (!(current instanceof String externalInput)) { - throw new SerDesException("SerDes pipeline external boundary produced " - + current.getClass().getName() - + " instead of String"); - } - return invokeExternalValueCodecDeserialize(valueCodec, externalInput, typeToken); + if (!(current instanceof String valueCodecInput)) { + throw new SerDesException("SerDes pipeline produced " + + current.getClass().getName() + + " instead of String for the value codec"); } - return invokeValueCodecDeserialize(valueCodec, current, typeToken); + return invokeValueCodecDeserialize(valueCodec, valueCodecInput, typeToken); } @SuppressWarnings("unchecked") @@ -146,7 +145,7 @@ private static SerDesStageResult invokeStageDeserialize(SerDesStage stage, } } - private static Object invokeValueCodecSerialize(ValueSerDes valueCodec, Object value) { + private static String invokeValueCodecSerialize(SerDes valueCodec, Object value) { try { var result = valueCodec.serialize(value); if (result == null) { @@ -158,25 +157,14 @@ private static Object invokeValueCodecSerialize(ValueSerDes valueCodec, Objec } } - @SuppressWarnings("unchecked") - private static T invokeValueCodecDeserialize( - ValueSerDes valueCodec, Object data, TypeToken typeToken) { + private static T invokeValueCodecDeserialize(SerDes valueCodec, String data, TypeToken typeToken) { try { - return valueCodec.deserialize((R) data, typeToken); + return valueCodec.deserialize(data, typeToken); } catch (Throwable failure) { throw stageFailure(0, valueCodec, "deserialize", failure); } } - private static T invokeExternalValueCodecDeserialize( - ValueSerDes valueCodec, String data, TypeToken typeToken) { - try { - return valueCodec.deserializeExternal(data, typeToken); - } catch (Throwable failure) { - throw stageFailure(0, valueCodec, "deserialize external payload", failure); - } - } - private static RuntimeException stageFailure(int index, Object stage, String action, Throwable failure) { if (failure instanceof Error error) { throw error; @@ -203,4 +191,29 @@ private static void addFlattened(List> target, SerDesStage> stages = new ArrayList<>(); + + private Builder(SerDes valueCodec) { + this.valueCodec = Objects.requireNonNull(valueCodec, "valueCodec cannot be null"); + if (valueCodec instanceof ComposableSerDes composable) { + this.valueCodec = composable.valueCodec; + stages.addAll(composable.stages); + } + } + + /** Appends a reversible typed stage. */ + public Builder then(SerDesStage stage) { + addFlattened(stages, Objects.requireNonNull(stage, "stage cannot be null")); + return this; + } + + /** Returns the immutable pipeline. */ + public ComposableSerDes build() { + return new ComposableSerDes(valueCodec, stages); + } + } } diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDes.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDes.java index 6021b3ed3..7c656378e 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDes.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDes.java @@ -5,13 +5,8 @@ import software.amazon.lambda.durable.TypeToken; import software.amazon.lambda.durable.exception.SerDesException; -/** - * Interface for serialization and deserialization of objects at the persisted string boundary. - * - *

This is the backward-compatible string specialization of {@link ValueSerDes}. Use {@code ValueSerDes} with - * {@link ComposableSerDes} when the first pipeline stage should consume a non-string representation. - */ -public interface SerDes extends ValueSerDes { +/** Interface for serialization and deserialization of objects at the persisted string boundary. */ +public interface SerDes { /** * Serializes an object to a JSON string. * @@ -39,11 +34,6 @@ public interface SerDes extends ValueSerDes { */ T deserialize(String data, TypeToken typeToken); - @Override - default T deserializeExternal(String data, TypeToken typeToken) { - return deserialize(data, typeToken); - } - /** * Deserializes this SerDes when it is used as an intermediate pipeline stage. * diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesStageResult.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesStageResult.java index 5d484e657..ed7c3a939 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesStageResult.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesStageResult.java @@ -8,8 +8,8 @@ * Result returned when a {@link SerDesStage} is reversed in a {@link ComposableSerDes}. * * @param value the non-null value produced by the stage - * @param skipRemainingStages whether deserialization should skip the remaining stages and decode {@code value} as an - * external string with the pipeline's value codec + * @param skipRemainingStages whether deserialization should skip the remaining intermediate stages and decode + * {@code value} directly with the pipeline's value codec */ public record SerDesStageResult(Object value, boolean skipRemainingStages) { public SerDesStageResult { @@ -21,7 +21,7 @@ public static SerDesStageResult continueWith(Object value) { return new SerDesStageResult(value, false); } - /** Skips the remaining stages and decodes the external string with the pipeline's value codec. */ + /** Skips the remaining intermediate stages and decodes the value directly with the pipeline's value codec. */ public static SerDesStageResult decodeWithValueCodec(String value) { return new SerDesStageResult(value, true); } diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/ValueSerDes.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/ValueSerDes.java deleted file mode 100644 index e6db2237b..000000000 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/ValueSerDes.java +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 -package software.amazon.lambda.durable.serde; - -import software.amazon.lambda.durable.TypeToken; -import software.amazon.lambda.durable.exception.SerDesException; - -/** - * Converts domain values to and from a typed pipeline representation. - * - *

The representation may be any Java type. A {@link ComposableSerDes} connects it to a {@link SerDesStage} chain - * whose final serialized representation is the checkpoint {@link String}. - * - * @param the serialized representation produced for the first pipeline stage - */ -public interface ValueSerDes { - /** - * Serializes a domain value to the representation consumed by the first pipeline stage. - * - * @param value the domain value - * @return the serialized representation, or null if value is null - */ - R serialize(Object value); - - /** - * Deserializes the representation to the requested domain type. - * - * @param data the representation restored by the stage chain - * @param typeToken the requested domain type - * @param the requested domain type - * @return the deserialized value, or null if data is null - */ - T deserialize(R data, TypeToken typeToken); - - /** - * Deserializes an unframed external string that did not pass through the configured stage chain. - * - *

Codecs that can receive external invocation, callback, or invoke-result payloads should override this method. - * String-valued {@link SerDes} implementations support it automatically. - * - * @param data the external string payload - * @param typeToken the requested domain type - * @param the requested domain type - * @return the deserialized value - */ - default T deserializeExternal(String data, TypeToken typeToken) { - throw new SerDesException( - "Value SerDes " + getClass().getName() + " cannot deserialize an external String payload"); - } - - /** Returns whether this value SerDes requires an SDK-managed durable execution context. */ - default boolean requiresDurableContext() { - return false; - } - - /** Returns whether this value SerDes must be the final component of a composable pipeline. */ - default boolean isTerminalPipelineStage() { - return false; - } -} diff --git a/sdk/src/test/java/software/amazon/lambda/durable/serde/ComposableSerDesTest.java b/sdk/src/test/java/software/amazon/lambda/durable/serde/ComposableSerDesTest.java index 1b51c500f..62d9a0a0e 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/serde/ComposableSerDesTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/serde/ComposableSerDesTest.java @@ -37,33 +37,19 @@ void serializesForwardAndDeserializesInReverse() { } @Test - void supportsCustomValueCodecAndIntermediateTypes() { + void supportsTypedIntermediateValues() { var calls = new ArrayList(); - var jackson = new JacksonSerDes(); - ValueSerDes valueCodec = new ValueSerDes<>() { + SerDesStage utf8 = new SerDesStage<>() { @Override - public JsonRepresentation serialize(Object value) { - calls.add("codec-serialize"); - return new JsonRepresentation(jackson.serialize(value)); - } - - @Override - public T deserialize(JsonRepresentation data, TypeToken typeToken) { - calls.add("codec-deserialize"); - return jackson.deserialize(data.value(), typeToken); - } - }; - SerDesStage utf8 = new SerDesStage<>() { - @Override - public byte[] serialize(JsonRepresentation value) { + public byte[] serialize(String value) { calls.add("bytes-serialize"); - return value.value().getBytes(StandardCharsets.UTF_8); + return value.getBytes(StandardCharsets.UTF_8); } @Override - public JsonRepresentation deserialize(byte[] data) { + public String deserialize(byte[] data) { calls.add("bytes-deserialize"); - return new JsonRepresentation(new String(data, StandardCharsets.UTF_8)); + return new String(data, StandardCharsets.UTF_8); } }; SerDesStage base64 = new SerDesStage<>() { @@ -79,29 +65,22 @@ public byte[] deserialize(String data) { return Base64.getDecoder().decode(data); } }; - var pipeline = ComposableSerDes.of(valueCodec, utf8.then(base64)); + var pipeline = ComposableSerDes.of(new JacksonSerDes(), utf8.then(base64)); var serialized = pipeline.serialize("value"); var deserialized = pipeline.deserialize(serialized, TypeToken.get(String.class)); assertEquals(Base64.getEncoder().encodeToString("\"value\"".getBytes(StandardCharsets.UTF_8)), serialized); assertEquals("value", deserialized); - assertEquals( - List.of( - "codec-serialize", - "bytes-serialize", - "base64-serialize", - "base64-deserialize", - "bytes-deserialize", - "codec-deserialize"), - calls); + assertEquals(List.of("bytes-serialize", "base64-serialize", "base64-deserialize", "bytes-deserialize"), calls); } @Test - void factoryFlattensNestedStageChains() { + void factoryBuilderAndThenFlattenNestedStageChains() { var calls = new ArrayList(); var nested = stringStage("one", "1", "1", calls).then(stringStage("two", "2", "2", calls)); - var pipeline = ComposableSerDes.of(new JacksonSerDes(), nested); + var pipeline = + ComposableSerDes.builder(new JacksonSerDes()).then(nested).build(); assertEquals("21\"value\"12", pipeline.serialize("value")); assertEquals(List.of("one-serialize", "two-serialize"), calls); @@ -188,49 +167,6 @@ public SerDesStageResult deserializePipelineStage(String data) { assertEquals(0, transformDeserializations.get()); } - @Test - void customValueCodecMayDecodeExternalStringPayload() { - var jackson = new JacksonSerDes(); - var externalDeserializations = new AtomicInteger(); - ValueSerDes valueCodec = new ValueSerDes<>() { - @Override - public JsonRepresentation serialize(Object value) { - return new JsonRepresentation(jackson.serialize(value)); - } - - @Override - public T deserialize(JsonRepresentation data, TypeToken typeToken) { - return jackson.deserialize(data.value(), typeToken); - } - - @Override - public T deserializeExternal(String data, TypeToken typeToken) { - externalDeserializations.incrementAndGet(); - return jackson.deserialize(data, typeToken); - } - }; - SerDesStage externalBoundary = new SerDesStage<>() { - @Override - public String serialize(JsonRepresentation value) { - return value.value(); - } - - @Override - public JsonRepresentation deserialize(String data) { - return new JsonRepresentation(data); - } - - @Override - public SerDesStageResult deserializePipelineStage(String data) { - return SerDesStageResult.decodeWithValueCodec(data); - } - }; - var pipeline = ComposableSerDes.of(valueCodec, externalBoundary); - - assertEquals("value", pipeline.deserialize("\"value\"", TypeToken.get(String.class))); - assertEquals(1, externalDeserializations.get()); - } - @Test void rejectsStagesAfterTerminalStage() { var terminal = new SerDesStage() { @@ -258,8 +194,7 @@ public boolean isTerminalPipelineStage() { } @Test - @SuppressWarnings({"rawtypes", "unchecked"}) - void rejectsNullIntermediateAndNonStringBoundaryValuesFromUnsafeRawStages() { + void rejectsNullIntermediateAndNonStringBoundaryValues() { var nullStage = new SerDesStage() { @Override public String serialize(String value) { @@ -287,16 +222,15 @@ public String deserialize(Integer data) { return "x".repeat(data); } }; - var typeFailure = assertThrows( - SerDesException.class, () -> ComposableSerDes.of(new JacksonSerDes(), (SerDesStage) nonStringFinalStage) + var typeFailure = + assertThrows(SerDesException.class, () -> ComposableSerDes.of(new JacksonSerDes(), nonStringFinalStage) .serialize("value")); assertTrue(typeFailure.getMessage().contains("final stage")); assertTrue(typeFailure.getMessage().contains(Integer.class.getName())); } @Test - @SuppressWarnings({"rawtypes", "unchecked"}) - void incompatibleUnsafeRawStagesFailWithStageMetadata() { + void incompatibleTypedStagesFailWithStageMetadata() { SerDesStage integerStage = new SerDesStage<>() { @Override public String serialize(Integer value) { @@ -309,9 +243,8 @@ public Integer deserialize(String data) { } }; - var failure = assertThrows( - SerDesException.class, () -> ComposableSerDes.of(new JacksonSerDes(), (SerDesStage) integerStage) - .serialize("value")); + var failure = assertThrows(SerDesException.class, () -> ComposableSerDes.of(new JacksonSerDes(), integerStage) + .serialize("value")); assertTrue(failure.getMessage().contains("stage 1")); assertInstanceOf(ClassCastException.class, failure.getCause()); @@ -410,6 +343,4 @@ public String deserialize(String data) { } }; } - - private record JsonRepresentation(String value) {} } diff --git a/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java b/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java index d0cc47146..850947f9f 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java @@ -115,35 +115,6 @@ public String deserialize(byte[] data) { runner.deserialize(pipeline, envelope, new TypeToken>() {}, context())); } - @Test - void stageModeAcceptsNonStringValueCodecRepresentation() throws Exception { - var jackson = new JacksonSerDes(); - ValueSerDes valueCodec = new ValueSerDes<>() { - @Override - public byte[] serialize(Object value) { - return jackson.serialize(value).getBytes(StandardCharsets.UTF_8); - } - - @Override - public T deserialize(byte[] data, TypeToken typeToken) { - return jackson.deserialize(new String(data, StandardCharsets.UTF_8), typeToken); - } - }; - var stage = FileSystemSerDes.stageBuilder(basePath).build(); - var pipeline = ComposableSerDes.of(valueCodec, stage); - var runner = new SerDesRunner(null); - - var envelope = runner.serialize(pipeline, Map.of("id", 42), context()); - var json = MAPPER.readTree(envelope); - var file = Path.of(json.get("file").textValue()); - - assertEquals("BYTES", json.get("payloadType").textValue()); - assertEquals("{\"id\":42}", new String(Files.readAllBytes(file), StandardCharsets.UTF_8)); - assertEquals( - Map.of("id", 42), - runner.deserialize(pipeline, envelope, new TypeToken>() {}, context())); - } - @Test void overflowModeKeepsSmallBinaryPayloadsInline() throws Exception { var stage = FileSystemSerDes.stageBuilder(basePath) From eaa9dd5c6f909885c750ed7845bf085204c49b45 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Tue, 25 Aug 2026 14:29:55 +0000 Subject: [PATCH 20/56] fix: preserve operation failure metadata --- .../lambda/durable/testing/TestOperation.java | 14 ++++- .../durable/testing/TestOperationTest.java | 56 +++++++++++++++++++ .../durable/operation/InvokeOperation.java | 2 +- .../operation/InvokeOperationTest.java | 24 ++++++++ 4 files changed, 94 insertions(+), 2 deletions(-) create mode 100644 sdk-testing/src/test/java/software/amazon/lambda/durable/testing/TestOperationTest.java diff --git a/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/TestOperation.java b/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/TestOperation.java index 0f4481e31..308b7e397 100644 --- a/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/TestOperation.java +++ b/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/TestOperation.java @@ -145,6 +145,7 @@ public T getStepResult(TypeToken type) { .orElse(OperationSubType.STEP); var payloadKind = subType == OperationSubType.WAIT_FOR_CONDITION ? SerDesPayloadKind.STATE : SerDesPayloadKind.RESULT; + var resultAttempt = resultAttempt(details, subType); return serDesRunner.deserialize( serDes, details.result(), @@ -157,7 +158,18 @@ public T getStepResult(TypeToken type) { operation.type(), subType, payloadKind, - details.attempt())); + resultAttempt)); + } + + private Integer resultAttempt(StepDetails details, OperationSubType subType) { + var attempt = details.attempt(); + if (subType == OperationSubType.WAIT_FOR_CONDITION + && operation.status() == OperationStatus.FAILED + && attempt != null + && attempt > 1) { + return attempt - 1; + } + return attempt; } /** Returns the step error, or null if the step succeeded or this is not a step operation. */ diff --git a/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/TestOperationTest.java b/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/TestOperationTest.java new file mode 100644 index 000000000..42a663691 --- /dev/null +++ b/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/TestOperationTest.java @@ -0,0 +1,56 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.testing; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.util.List; +import java.util.concurrent.atomic.AtomicReference; +import org.junit.jupiter.api.Test; +import software.amazon.awssdk.services.lambda.model.Operation; +import software.amazon.awssdk.services.lambda.model.OperationStatus; +import software.amazon.awssdk.services.lambda.model.StepDetails; +import software.amazon.lambda.durable.TypeToken; +import software.amazon.lambda.durable.model.OperationSubType; +import software.amazon.lambda.durable.serde.SerDes; +import software.amazon.lambda.durable.serde.SerDesContext; +import software.amazon.lambda.durable.serde.SerDesRunner; + +class TestOperationTest { + private static final String EXECUTION_ARN = "arn:aws:lambda:us-east-1:123456789012:function:test:$LATEST" + + "/durable-execution/execution-id/invocation-id"; + + @Test + void failedWaitForConditionReadsStateFromPreviousAttempt() { + var observedContext = new AtomicReference(); + var serDes = new SerDes() { + @Override + public String serialize(Object value) { + return value.toString(); + } + + @Override + @SuppressWarnings("unchecked") + public T deserialize(String data, TypeToken typeToken) { + observedContext.set(SerDesContext.getCurrentContext()); + return (T) data; + } + }; + var operation = Operation.builder() + .id("wait-id") + .name("wait-condition") + .type(OperationSubType.WAIT_FOR_CONDITION.getOperationType()) + .subType(OperationSubType.WAIT_FOR_CONDITION.getValue()) + .status(OperationStatus.FAILED) + .stepDetails(StepDetails.builder() + .attempt(3) + .result("retained-state") + .build()) + .build(); + var testOperation = new TestOperation(operation, List.of(), serDes, new SerDesRunner(null), EXECUTION_ARN); + + assertEquals("retained-state", testOperation.getStepResult(String.class)); + assertEquals(2, observedContext.get().attempt()); + assertEquals("operation/wait-id/state/attempt-2", observedContext.get().entityId()); + } +} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/operation/InvokeOperation.java b/sdk/src/main/java/software/amazon/lambda/durable/operation/InvokeOperation.java index fc0521819..7ad3a834b 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/operation/InvokeOperation.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/operation/InvokeOperation.java @@ -94,7 +94,7 @@ public T get() { case SUCCEEDED -> deserializeResult(result); case FAILED -> throw new InvokeFailedException( - op, deserializeException(op.chainedInvokeDetails().error())); + op, deserializeException(invokeDetails != null ? invokeDetails.error() : null)); case TIMED_OUT -> throw new InvokeTimedOutException(op); case STOPPED -> throw new InvokeStoppedException(op); // Unexpected status which should not happen. This is added for forward-compatibility. diff --git a/sdk/src/test/java/software/amazon/lambda/durable/operation/InvokeOperationTest.java b/sdk/src/test/java/software/amazon/lambda/durable/operation/InvokeOperationTest.java index b647bb0a1..21363148b 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/operation/InvokeOperationTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/operation/InvokeOperationTest.java @@ -4,6 +4,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -106,6 +107,29 @@ void getInvokeFailedExceptionWhenInvocationFailed() { assertEquals("errorMessage", ex.deserializedError().getMessage()); } + @Test + void getInvokeFailedExceptionWhenInvocationDetailsAreMissing() { + var op = Operation.builder() + .id(OPERATION_ID) + .name(OPERATION_NAME) + .status(OperationStatus.FAILED) + .build(); + when(executionManager.getOperationAndUpdateReplayState(OPERATION_ID)).thenReturn(op); + + var operation = new InvokeOperation<>( + OPERATION_IDENTIFIER, + "test-function", + "{}", + TypeToken.get(String.class), + InvokeConfig.builder().serDes(new JacksonSerDes()).build(), + durableContext); + operation.onCheckpointComplete(op); + + var exception = assertThrows(InvokeFailedException.class, operation::get); + assertNull(exception.getErrorObject()); + assertNull(exception.deserializedError()); + } + @Test void getInvokeTimedOutExceptionWhenInvocationTimedOut() { var op = Operation.builder() From a02d21a9cd50fa51218f07e5e2fbd0d6440d3156 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Tue, 25 Aug 2026 14:32:39 +0000 Subject: [PATCH 21/56] Revert "refactor: move SerDes chaining to stages" This reverts commit 24ec5634f1e21d38b4534ac20e3366fde50f7487. --- docs/adr/005-filesystem-serdes.md | 50 ++++--- docs/advanced/configuration.md | 2 +- docs/advanced/filesystem-serdes.md | 5 +- .../FileSystemSerDesIntegrationTest.java | 41 +++--- .../testing/CloudDurableTestRunnerTest.java | 52 +++---- .../testing/LocalDurableTestRunnerTest.java | 32 ++--- .../durable/serde/ChainedSerDesStage.java | 112 --------------- .../durable/serde/ComposableSerDes.java | 109 +++++++++++---- .../durable/serde/FileSystemSerDes.java | 13 +- .../lambda/durable/serde/RetrySerDes.java | 12 +- .../amazon/lambda/durable/serde/SerDes.java | 31 ++++- .../lambda/durable/serde/SerDesStage.java | 14 -- .../durable/serde/ComposableSerDesTest.java | 129 +++++++++--------- .../durable/serde/FileSystemSerDesTest.java | 42 ++---- 14 files changed, 281 insertions(+), 363 deletions(-) delete mode 100644 sdk/src/main/java/software/amazon/lambda/durable/serde/ChainedSerDesStage.java diff --git a/docs/adr/005-filesystem-serdes.md b/docs/adr/005-filesystem-serdes.md index 56dbce75b..fc999a198 100644 --- a/docs/adr/005-filesystem-serdes.md +++ b/docs/adr/005-filesystem-serdes.md @@ -33,8 +33,9 @@ There are a few Java-specific constraints: Keep the existing `SerDes` serialization methods source- and binary-compatible, add a typed `SerDesStage` contract and a core `ComposableSerDes` implementation which together form a processing pipeline, and implement -`FileSystemSerDes` in the core SDK. `SerDes` remains the persisted value-codec boundary, while intermediate -`SerDesStage` instances may exchange arbitrary Java types and compose independently. The filesystem stage uses +`FileSystemSerDes` in the core SDK. Existing `SerDes` implementations continue to participate as string-producing +stages without changing their binary contract, while dedicated intermediate stages may exchange arbitrary Java types. +The filesystem stage uses `SerDesContext.getCurrentContext()` to identify the durable execution and entity being serialized. ```java @@ -42,9 +43,19 @@ public interface SerDesStage { O serialize(I value); I deserialize(O data); +} + +public interface SerDes { + String serialize(Object value); - default SerDesStage then(SerDesStage nextStage) { - return ChainedSerDesStage.of(this, nextStage); + T deserialize(String data, TypeToken typeToken); + + default ComposableSerDes then(SerDes nextStage) { + return ComposableSerDes.of(this, nextStage); + } + + default ComposableSerDes then(SerDesStage nextStage) { + return ComposableSerDes.builder(this).then(nextStage).build(); } } ``` @@ -106,7 +117,7 @@ var fileSystemStage = FileSystemSerDes.stageBuilder(Path.of("/mnt/efs/durable-pa .previewGenerator(optionalPreviewGenerator) .build(); -var serDes = ComposableSerDes.of(new JacksonSerDes(), fileSystemStage); +var serDes = new JacksonSerDes().then(fileSystemStage); return DurableConfig.builder() .withSerDes(serDes) @@ -121,14 +132,17 @@ the existing `serialize` and `deserialize` methods: ```java public final class ComposableSerDes implements SerDes { - public static ComposableSerDes of(SerDes valueCodec); - public static ComposableSerDes of(SerDes valueCodec, SerDesStage stage); + public static ComposableSerDes of(SerDes first, SerDes... remaining); public static Builder builder(SerDes valueCodec); public SerDes getValueCodec(); + public ComposableSerDes then(SerDes stage); + public ComposableSerDes then(SerDesStage stage); + public static final class Builder { + public Builder then(SerDes stage); public Builder then(SerDesStage stage); public ComposableSerDes build(); @@ -204,8 +218,8 @@ Pipeline rules: `String`, and reverse processing must return a `String` to the value codec. - A stage may declare that it requires durable context or that it must be terminal. A terminal stage must be the last stage so later transformations cannot invalidate its checkpoint-size or storage decision. -- `SerDesStage.then(...)`, `ComposableSerDes.of(...)`, and the builder flatten nested stage chains while preserving - stage order. +- `SerDes.then(...)`, `ComposableSerDes.of(...)`, and the builder flatten nested `ComposableSerDes` instances while + preserving stage order. - A `null` value at the pipeline boundary short-circuits the entire pipeline: serializing or deserializing `null` returns `null` without invoking any stage. A stage returning `null` for non-null input is an error. The value codec may decode a non-null representation such as the JSON literal `null` to a null domain value. @@ -233,15 +247,14 @@ Pipeline rules: - Invocation-scoped caching wraps the complete pipeline. Cache keys use the final checkpoint string and target type, so cache hits skip every reverse-processing stage, including filesystem reads. -The default `SerDesStage.then(...)` method provides independently reusable processing chains. `ComposableSerDes` -connects one chain to the persisted value codec: +The default `SerDes.then(...)` method and immutable `ComposableSerDes.then(...)` method provide a concise form for +independently reusable processing chains: ```java -var secureStages = compressionStage +var securePayloads = new JacksonSerDes() + .then(compressionSerDes) .then(encryptionSerDes) .then(fileSystemStage); - -var securePayloads = ComposableSerDes.of(new JacksonSerDes(), secureStages); ``` ### Retryable SerDes stages @@ -259,7 +272,7 @@ var resilientFileSystemStage = new RetrySerDes( 2.0, JitterStrategy.FULL)); -var serDes = ComposableSerDes.of(new JacksonSerDes(), resilientFileSystemStage); +var serDes = new JacksonSerDes().then(resilientFileSystemStage); ``` Retry rules: @@ -467,8 +480,9 @@ must not synthesize a durable context and serialize initial input through the fu 1. Add `SerDesContext`, `SerDesPayloadKind`, and package-private TLS setter/clearer support. Leave the existing `SerDes` methods unchanged. -2. Add `SerDesStage.then(...)` and `ComposableSerDes` with immutable stage ordering, forward serialization, reverse - deserialization, external-boundary bypass, terminal-stage validation, null short-circuiting, and stage-aware errors. +2. Add the binary-compatible `SerDes.then(...)` default method and `ComposableSerDes` with immutable stage ordering, + forward serialization, reverse deserialization, external-boundary bypass, terminal-stage validation, null + short-circuiting, and stage-aware errors. 3. Add `RetryableSerDesException` and `RetrySerDes`, reusing `RetryStrategy` for bounded in-invocation retries. 4. Add `SerDesRunner` with inline execution by default and optional dispatch through `DurableConfig.withSerDesExecutorService(...)`. Do not create a default SerDes pool. @@ -706,7 +720,7 @@ This approach gives the SDK one consistent policy for root payloads, operation r | Responsibility boundary | Combines value serialization and storage-reference creation in ordered, composable SerDes stages. | Keeps object encoding in `SerDes` and storage movement in a separate offloader. | | User configuration | Users configure one SerDes or a `ComposableSerDes` pipeline. Operation-level SerDes selection replaces the whole pipeline. | Users configure both a SerDes and an offloader. The SDK must define global, per-operation, and per-payload precedence. | | Parity with JS issue | Closest to the current JavaScript `createFileSystemSerdes` model and issue #463 wording. | Diverges from JavaScript naming and shape, though it may be architecturally cleaner for Java. | -| Core SDK changes | Adds `SerDesStage`, `ComposableSerDes`, and `SerDesContext` TLS while leaving `SerDes.then(...)` absent and the existing serialization methods unchanged. | Requires a new core extension point, envelope type, config surface, payload pipeline, and migration story. | +| Core SDK changes | Adds the compatible `SerDes.then(...)` default method, `ComposableSerDes`, and `SerDesContext` TLS because the existing serialization methods have no context parameter. | Requires a new core extension point, envelope type, config surface, payload pipeline, and migration story. | | Applicability | Any compatible SerDes stages can be chained, but storage stages still own their envelope and lifecycle behavior. | Any SerDes output can be offloaded uniformly after serialization. Users can combine Jackson/custom SerDes with any offloader. | | Envelope ownership | FileSystemSerDes owns the checkpoint envelope (`data`, `file`, preview), so the SDK treats it as opaque serialized data. | SDK owns the checkpoint/offload envelope and must guarantee it composes with replay, errors, callbacks, and test utilities. | | Caching | SDK can cache deserialized values, but FileSystemSerDes may still do file reads internally unless cache hits happen before SerDes. | SDK can cache at both layers: resolved offloaded payload text and final deserialized object. | diff --git a/docs/advanced/configuration.md b/docs/advanced/configuration.md index 9477d6511..552dd1790 100644 --- a/docs/advanced/configuration.md +++ b/docs/advanced/configuration.md @@ -66,7 +66,7 @@ var resilientFileSystemStage = new RetrySerDes( fileSystemStage, RetryStrategies.fixedDelay(3, Duration.ofSeconds(1))); -var serDes = ComposableSerDes.of(new JacksonSerDes(), resilientFileSystemStage); +var serDes = new JacksonSerDes().then(resilientFileSystemStage); var serDesExecutor = Executors.newFixedThreadPool(4); return DurableConfig.builder() diff --git a/docs/advanced/filesystem-serdes.md b/docs/advanced/filesystem-serdes.md index ce83d7ad4..201782310 100644 --- a/docs/advanced/filesystem-serdes.md +++ b/docs/advanced/filesystem-serdes.md @@ -28,7 +28,7 @@ var resilientFileSystemStage = new RetrySerDes( fileSystemStage, RetryStrategies.fixedDelay(3, Duration.ofSeconds(1))); -var serDes = ComposableSerDes.of(new JacksonSerDes(), resilientFileSystemStage); +var serDes = new JacksonSerDes().then(resilientFileSystemStage); var serDesExecutor = Executors.newFixedThreadPool(4); return DurableConfig.builder() @@ -38,8 +38,7 @@ return DurableConfig.builder() ``` Serialization runs from `JacksonSerDes` to the filesystem stage. Deserialization runs in reverse. Additional reversible -typed stages such as compression or encryption compose through `SerDesStage.then(...)` before the stage chain is passed -to `ComposableSerDes.of(...)`. A +typed stages such as compression or encryption can be inserted with `then(...)`. A `SerDesStage` may pass compressed or encrypted bytes directly to `FileSystemSerDes`; no Base64 adapter is required between those stages. The complete pipeline still produces a string checkpoint envelope. diff --git a/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FileSystemSerDesIntegrationTest.java b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FileSystemSerDesIntegrationTest.java index 1b600e263..ee0505fe3 100644 --- a/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FileSystemSerDesIntegrationTest.java +++ b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FileSystemSerDesIntegrationTest.java @@ -37,7 +37,6 @@ import software.amazon.lambda.durable.retry.JitterStrategy; import software.amazon.lambda.durable.retry.RetryStrategies; import software.amazon.lambda.durable.retry.WaitStrategies; -import software.amazon.lambda.durable.serde.ComposableSerDes; import software.amazon.lambda.durable.serde.FileSystemSerDes; import software.amazon.lambda.durable.serde.JacksonSerDes; import software.amazon.lambda.durable.serde.SerDes; @@ -161,9 +160,9 @@ void acceptsRawCallbackAndInvokeResultsAndOffloadsInvokePayload() throws Excepti invokePayload.set(value); } }); - var serDes = ComposableSerDes.of( - new JacksonSerDes(), - recordingStage.then(FileSystemSerDes.stageBuilder(basePath).build())); + var serDes = new JacksonSerDes() + .then(recordingStage) + .then(FileSystemSerDes.stageBuilder(basePath).build()); var config = DurableConfig.builder().withSerDes(serDes).build(); var runner = LocalDurableTestRunner.create( String.class, @@ -279,9 +278,9 @@ void repeatedGetUsesInvocationCacheForTheCompletePipeline() { resultDeserializations.incrementAndGet(); } }); - var serDes = ComposableSerDes.of( - new JacksonSerDes(), - countingStage.then(FileSystemSerDes.stageBuilder(basePath).build())); + var serDes = new JacksonSerDes() + .then(countingStage) + .then(FileSystemSerDes.stageBuilder(basePath).build()); var config = DurableConfig.builder().withSerDes(serDes).build(); var runner = LocalDurableTestRunner.create( String.class, @@ -316,9 +315,9 @@ void successfulRetryUsesTheProducingAttemptForResultSerialization() { resultAttempts.add(context.attempt()); } }); - var serDes = ComposableSerDes.of( - new JacksonSerDes(), - attemptStage.then(FileSystemSerDes.stageBuilder(basePath).build())); + var serDes = new JacksonSerDes() + .then(attemptStage) + .then(FileSystemSerDes.stageBuilder(basePath).build()); var config = DurableConfig.builder().withSerDes(serDes).build(); var stepConfig = StepConfig.builder() .retryStrategy(RetryStrategies.fixedDelay(2, Duration.ofSeconds(1))) @@ -481,9 +480,9 @@ public String deserialize(byte[] data) { return new String(data, StandardCharsets.UTF_8); } }; - return ComposableSerDes.of( - new JacksonSerDes(), - utf8.then(FileSystemSerDes.stageBuilder(basePath).build())); + return new JacksonSerDes() + .then(utf8) + .then(FileSystemSerDes.stageBuilder(basePath).build()); } private static DurableExecutionInput durableInput( @@ -512,18 +511,20 @@ private static Operation executionOperation(String id, String name, String input .build(); } - private static SerDesStage identityStage(RecordingFunction recorder) { - return new SerDesStage<>() { + private static SerDes identityStage(RecordingFunction recorder) { + return new SerDes() { @Override - public String serialize(String value) { - recorder.record("serialize", value); - return value; + public String serialize(Object value) { + var stringValue = (String) value; + recorder.record("serialize", stringValue); + return stringValue; } @Override - public String deserialize(String data) { + @SuppressWarnings("unchecked") + public T deserialize(String data, TypeToken typeToken) { recorder.record("deserialize", data); - return data; + return (T) data; } }; } diff --git a/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/CloudDurableTestRunnerTest.java b/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/CloudDurableTestRunnerTest.java index c9cf9ee75..9aa79ce4c 100644 --- a/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/CloudDurableTestRunnerTest.java +++ b/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/CloudDurableTestRunnerTest.java @@ -15,14 +15,12 @@ import software.amazon.awssdk.services.lambda.model.InvokeRequest; import software.amazon.awssdk.services.lambda.model.InvokeResponse; import software.amazon.lambda.durable.TypeToken; -import software.amazon.lambda.durable.serde.ComposableSerDes; import software.amazon.lambda.durable.serde.FileSystemSerDes; import software.amazon.lambda.durable.serde.JacksonSerDes; import software.amazon.lambda.durable.serde.SerDes; import software.amazon.lambda.durable.serde.SerDesContext; import software.amazon.lambda.durable.serde.SerDesPayloadKind; import software.amazon.lambda.durable.serde.SerDesRunner; -import software.amazon.lambda.durable.serde.SerDesStage; class CloudDurableTestRunnerTest { @@ -56,7 +54,7 @@ void explicitComposableInputSerDesUsesTheCompletePipeline() { var wrappingStage = wrappingStage(); var runner = CloudDurableTestRunner.create( "arn:aws:lambda:us-east-2:123:function:test", String.class, String.class, mockClient) - .withInputSerDes(ComposableSerDes.of(new JacksonSerDes(), wrappingStage)); + .withInputSerDes(new JacksonSerDes().then(wrappingStage)); runner.startAsync("value"); @@ -74,7 +72,7 @@ void persistedComposableSerDesUsesCompletePipelineForDefaultInput() { .build()); var runner = CloudDurableTestRunner.create( "arn:aws:lambda:us-east-2:123:function:test", String.class, String.class, mockClient) - .withSerDes(ComposableSerDes.of(new JacksonSerDes(), wrappingStage())); + .withSerDes(new JacksonSerDes().then(wrappingStage())); runner.startAsync("value"); @@ -88,7 +86,7 @@ void contextDependentPersistedSerDesRequiresExplicitInputSerDes() { var mockClient = mock(LambdaClient.class); var runner = CloudDurableTestRunner.create( "arn:aws:lambda:us-east-2:123:function:test", String.class, String.class, mockClient) - .withSerDes(ComposableSerDes.of(new JacksonSerDes(), contextDependentStage())); + .withSerDes(new JacksonSerDes().then(contextDependentStage())); var failure = assertThrows(RuntimeException.class, () -> runner.startAsync("value")); @@ -102,7 +100,7 @@ void explicitInputSerDesMustBeContextFree() { var mockClient = mock(LambdaClient.class); var runner = CloudDurableTestRunner.create( "arn:aws:lambda:us-east-2:123:function:test", String.class, String.class, mockClient) - .withInputSerDes(contextDependentSerDes()); + .withInputSerDes(contextDependentStage()); var failure = assertThrows(RuntimeException.class, () -> runner.startAsync("value")); @@ -116,8 +114,8 @@ void contextDependentPersistedSerDesRequiresValueCodecInput() { var mockClient = mock(LambdaClient.class); var runner = CloudDurableTestRunner.create( "arn:aws:lambda:us-east-2:123:function:test", String.class, String.class, mockClient) - .withSerDes(ComposableSerDes.of(new JacksonSerDes(), contextDependentStage())) - .withInputSerDes(ComposableSerDes.of(new JacksonSerDes(), wrappingStage())); + .withSerDes(new JacksonSerDes().then(contextDependentStage())) + .withInputSerDes(new JacksonSerDes().then(wrappingStage())); var failure = assertThrows(RuntimeException.class, () -> runner.startAsync("value")); @@ -134,8 +132,8 @@ void valueCodecInputRoundTripsThroughFileSystemPipeline(@TempDir Path basePath) .thenReturn(InvokeResponse.builder() .durableExecutionArn(executionArn) .build()); - var persistedSerDes = ComposableSerDes.of( - new JacksonSerDes(), FileSystemSerDes.stageBuilder(basePath).build()); + var persistedSerDes = + new JacksonSerDes().then(FileSystemSerDes.stageBuilder(basePath).build()); var runner = CloudDurableTestRunner.create( "arn:aws:lambda:us-east-2:123:function:test", String.class, String.class, mockClient) .withSerDes(persistedSerDes) @@ -164,7 +162,7 @@ void replacingPersistedSerDesPreservesExplicitInputSerDes() { .build()); var runner = CloudDurableTestRunner.create( "arn:aws:lambda:us-east-2:123:function:test", String.class, String.class, mockClient) - .withInputSerDes(ComposableSerDes.of(new JacksonSerDes(), wrappingStage())) + .withInputSerDes(new JacksonSerDes().then(wrappingStage())) .withSerDes(new JacksonSerDes()); runner.startAsync("value"); @@ -174,40 +172,22 @@ void replacingPersistedSerDesPreservesExplicitInputSerDes() { assertEquals("<\"value\">", request.getValue().payload().asUtf8String()); } - private static SerDesStage wrappingStage() { - return new SerDesStage<>() { + private static SerDes wrappingStage() { + return new SerDes() { @Override - public String serialize(String value) { + public String serialize(Object value) { return "<" + value + ">"; } @Override - public String deserialize(String data) { - return data.substring(1, data.length() - 1); - } - }; - } - - private static SerDesStage contextDependentStage() { - return new SerDesStage<>() { - @Override - public String serialize(String value) { - return value; - } - - @Override - public String deserialize(String data) { - return data; - } - - @Override - public boolean requiresDurableContext() { - return true; + @SuppressWarnings("unchecked") + public T deserialize(String data, TypeToken typeToken) { + return (T) data.substring(1, data.length() - 1); } }; } - private static SerDes contextDependentSerDes() { + private static SerDes contextDependentStage() { return new SerDes() { @Override public String serialize(Object value) { diff --git a/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/LocalDurableTestRunnerTest.java b/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/LocalDurableTestRunnerTest.java index 17abb475f..7cf89f041 100644 --- a/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/LocalDurableTestRunnerTest.java +++ b/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/LocalDurableTestRunnerTest.java @@ -20,9 +20,9 @@ import software.amazon.lambda.durable.model.ExecutionStatus; import software.amazon.lambda.durable.plugin.DurableExecutionPlugin; import software.amazon.lambda.durable.plugin.InvocationInfo; -import software.amazon.lambda.durable.serde.ComposableSerDes; import software.amazon.lambda.durable.serde.FileSystemSerDes; import software.amazon.lambda.durable.serde.JacksonSerDes; +import software.amazon.lambda.durable.serde.SerDes; import software.amazon.lambda.durable.serde.SerDesStage; class LocalDurableTestRunnerTest { @@ -149,9 +149,8 @@ void checkpointedLargeOutputReplaysWithoutDuplicateExecutionOperation() { @Test void contextDependentPersistedSerDesRequiresExplicitInputSerDes(@TempDir Path basePath) { var config = DurableConfig.builder() - .withSerDes(ComposableSerDes.of( - new JacksonSerDes(), - FileSystemSerDes.stageBuilder(basePath).build())) + .withSerDes(new JacksonSerDes() + .then(FileSystemSerDes.stageBuilder(basePath).build())) .build(); var runner = LocalDurableTestRunner.create(String.class, (input, context) -> input, config); @@ -163,12 +162,11 @@ void contextDependentPersistedSerDesRequiresExplicitInputSerDes(@TempDir Path ba @Test void contextDependentPersistedSerDesRequiresValueCodecInput(@TempDir Path basePath) { var config = DurableConfig.builder() - .withSerDes(ComposableSerDes.of( - new JacksonSerDes(), - FileSystemSerDes.stageBuilder(basePath).build())) + .withSerDes(new JacksonSerDes() + .then(FileSystemSerDes.stageBuilder(basePath).build())) .build(); var runner = LocalDurableTestRunner.create(String.class, (input, context) -> input, config) - .withInputSerDes(ComposableSerDes.of(new JacksonSerDes(), wrappingStage(new AtomicInteger()))); + .withInputSerDes(new JacksonSerDes().then(wrappingStage(new AtomicInteger()))); var failure = assertThrows(IllegalStateException.class, () -> runner.run("value")); @@ -178,10 +176,9 @@ void contextDependentPersistedSerDesRequiresValueCodecInput(@TempDir Path basePa @Test void initialInputBypassesStagesBeforeFileSystemSerDes(@TempDir Path basePath) { var deserializeCalls = new AtomicInteger(); - var persistedSerDes = ComposableSerDes.of( - new JacksonSerDes(), - bytesStage(deserializeCalls) - .then(FileSystemSerDes.stageBuilder(basePath).build())); + var persistedSerDes = new JacksonSerDes() + .then(bytesStage(deserializeCalls)) + .then(FileSystemSerDes.stageBuilder(basePath).build()); var config = DurableConfig.builder().withSerDes(persistedSerDes).build(); var runner = LocalDurableTestRunner.create( String.class, (input, context) -> input + ":" + deserializeCalls.get(), config) @@ -194,17 +191,18 @@ void initialInputBypassesStagesBeforeFileSystemSerDes(@TempDir Path basePath) { assertEquals("value:0", result.getResult()); } - private static SerDesStage wrappingStage(AtomicInteger deserializeCalls) { - return new SerDesStage<>() { + private static SerDes wrappingStage(AtomicInteger deserializeCalls) { + return new SerDes() { @Override - public String serialize(String value) { + public String serialize(Object value) { return "<" + value + ">"; } @Override - public String deserialize(String data) { + @SuppressWarnings("unchecked") + public T deserialize(String data, TypeToken typeToken) { deserializeCalls.incrementAndGet(); - return data.substring(1, data.length() - 1); + return (T) data.substring(1, data.length() - 1); } }; } diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/ChainedSerDesStage.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/ChainedSerDesStage.java deleted file mode 100644 index 3ba6179ef..000000000 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/ChainedSerDesStage.java +++ /dev/null @@ -1,112 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 -package software.amazon.lambda.durable.serde; - -import java.util.ArrayList; -import java.util.List; -import software.amazon.lambda.durable.exception.RetryableSerDesException; -import software.amazon.lambda.durable.exception.SerDesException; - -final class ChainedSerDesStage implements SerDesStage { - private final List> stages; - - private ChainedSerDesStage(List> stages) { - for (int index = 0; index < stages.size() - 1; index++) { - var stage = stages.get(index); - if (stage.isTerminalPipelineStage()) { - throw new IllegalArgumentException(String.format( - "SerDes pipeline stage %d (%s) must be the final stage", - index + 1, stage.getClass().getName())); - } - } - this.stages = List.copyOf(stages); - } - - static SerDesStage of(SerDesStage first, SerDesStage second) { - var stages = new ArrayList>(); - addFlattened(stages, first); - addFlattened(stages, second); - return new ChainedSerDesStage<>(stages); - } - - List> stages() { - return stages; - } - - @Override - @SuppressWarnings("unchecked") - public O serialize(I value) { - Object current = value; - for (int index = 0; index < stages.size(); index++) { - var stage = stages.get(index); - try { - current = ((SerDesStage) stage).serialize(current); - if (current == null) { - throw new SerDesException("Stage returned null for a non-null value"); - } - } catch (Throwable failure) { - throw stageFailure(index + 1, stage, "serialize", failure); - } - } - return (O) current; - } - - @Override - @SuppressWarnings("unchecked") - public I deserialize(O data) { - return (I) deserializePipelineStage(data).value(); - } - - @Override - @SuppressWarnings("unchecked") - public SerDesStageResult deserializePipelineStage(O data) { - Object current = data; - for (int index = stages.size() - 1; index >= 0; index--) { - var stage = stages.get(index); - try { - var result = ((SerDesStage) stage).deserializePipelineStage(current); - if (result == null) { - throw new SerDesException("Stage returned a null pipeline result"); - } - current = result.value(); - if (result.skipRemainingStages()) { - return result; - } - } catch (Throwable failure) { - throw stageFailure(index + 1, stage, "deserialize", failure); - } - } - return SerDesStageResult.continueWith(current); - } - - @Override - public boolean requiresDurableContext() { - return stages.stream().anyMatch(SerDesStage::requiresDurableContext); - } - - @Override - public boolean isTerminalPipelineStage() { - return stages.get(stages.size() - 1).isTerminalPipelineStage(); - } - - private static void addFlattened(List> target, SerDesStage stage) { - if (stage instanceof ChainedSerDesStage chained) { - target.addAll(chained.stages); - } else { - target.add(stage); - } - } - - private static RuntimeException stageFailure(int index, SerDesStage stage, String action, Throwable failure) { - if (failure instanceof Error error) { - throw error; - } - var message = String.format( - "SerDes pipeline stage %d (%s) failed to %s", - index, stage.getClass().getName(), action); - if (failure instanceof RetryableSerDesException) { - return new RetryableSerDesException(message, failure); - } - return new SerDesException(message, failure); - } -} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/ComposableSerDes.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/ComposableSerDes.java index 2277c7ed5..5484f8cd8 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/ComposableSerDes.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/ComposableSerDes.java @@ -3,6 +3,7 @@ package software.amazon.lambda.durable.serde; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.Objects; import software.amazon.lambda.durable.TypeToken; @@ -18,15 +19,15 @@ */ public final class ComposableSerDes implements SerDes { private final SerDes valueCodec; - private final List> stages; + private final List stages; - private ComposableSerDes(SerDes valueCodec, List> stages) { + private ComposableSerDes(SerDes valueCodec, List stages) { this.valueCodec = Objects.requireNonNull(valueCodec, "valueCodec cannot be null"); if (!stages.isEmpty() && valueCodec.isTerminalPipelineStage()) { throw terminalStageFailure(0, valueCodec); } for (int index = 0; index < stages.size() - 1; index++) { - if (stages.get(index).isTerminalPipelineStage()) { + if (isTerminal(stages.get(index))) { throw terminalStageFailure(index + 1, stages.get(index)); } } @@ -34,24 +35,24 @@ private ComposableSerDes(SerDes valueCodec, List> stages) { } /** - * Creates a pipeline containing only a value codec. + * Creates a pipeline with a value codec followed by zero or more typed stages. * - * @param valueCodec the value codec + * @param first the value codec + * @param remaining reversible SerDes stages * @return an immutable pipeline */ - public static ComposableSerDes of(SerDes valueCodec) { - return builder(valueCodec).build(); - } - - /** - * Creates a pipeline with a value codec followed by a typed stage or stage chain. - * - * @param valueCodec the value codec - * @param stage the reversible typed stage or stage chain - * @return an immutable pipeline - */ - public static ComposableSerDes of(SerDes valueCodec, SerDesStage stage) { - return builder(valueCodec).then(stage).build(); + public static ComposableSerDes of(SerDes first, SerDes... remaining) { + Objects.requireNonNull(remaining, "remaining stages cannot be null"); + var valueCodec = Objects.requireNonNull(first, "first stage cannot be null"); + var stages = new ArrayList(); + if (valueCodec instanceof ComposableSerDes composable) { + valueCodec = composable.valueCodec; + stages.addAll(composable.stages); + } + Arrays.stream(remaining) + .map(stage -> Objects.requireNonNull(stage, "pipeline stage cannot be null")) + .forEach(stage -> addFlattened(stages, stage)); + return new ComposableSerDes(valueCodec, stages); } /** @@ -71,14 +72,28 @@ public SerDes getValueCodec() { @Override public boolean requiresDurableContext() { - return valueCodec.requiresDurableContext() || stages.stream().anyMatch(SerDesStage::requiresDurableContext); + return valueCodec.requiresDurableContext() || stages.stream().anyMatch(ComposableSerDes::requiresContext); } @Override public boolean isTerminalPipelineStage() { - return stages.isEmpty() - ? valueCodec.isTerminalPipelineStage() - : stages.get(stages.size() - 1).isTerminalPipelineStage(); + return stages.isEmpty() ? valueCodec.isTerminalPipelineStage() : isTerminal(stages.get(stages.size() - 1)); + } + + /** Returns a new pipeline with the supplied stage appended. */ + @Override + public ComposableSerDes then(SerDes stage) { + var combined = new ArrayList<>(stages); + addFlattened(combined, Objects.requireNonNull(stage, "stage cannot be null")); + return new ComposableSerDes(valueCodec, combined); + } + + /** Returns a new pipeline with the supplied typed stage appended. */ + @Override + public ComposableSerDes then(SerDesStage stage) { + var combined = new ArrayList<>(stages); + addFlattened(combined, Objects.requireNonNull(stage, "stage cannot be null")); + return new ComposableSerDes(valueCodec, combined); } @Override @@ -120,9 +135,11 @@ public T deserialize(String data, TypeToken typeToken) { } @SuppressWarnings("unchecked") - private static Object invokeStageSerialize(SerDesStage stage, Object value, int index) { + private static Object invokeStageSerialize(Object stage, Object value, int index) { try { - var result = ((SerDesStage) stage).serialize(value); + var result = stage instanceof SerDes serDes + ? serDes.serialize(value) + : ((SerDesStage) stage).serialize(value); if (result == null) { throw new SerDesException("Stage returned null for a non-null value"); } @@ -133,9 +150,18 @@ private static Object invokeStageSerialize(SerDesStage stage, Object value } @SuppressWarnings("unchecked") - private static SerDesStageResult invokeStageDeserialize(SerDesStage stage, Object data, int index) { + private static SerDesStageResult invokeStageDeserialize(Object stage, Object data, int index) { try { - var result = ((SerDesStage) stage).deserializePipelineStage(data); + SerDesStageResult result; + if (stage instanceof SerDes serDes) { + if (!(data instanceof String stringData)) { + throw new SerDesException("SerDes stage requires String input but received " + + data.getClass().getName()); + } + result = serDes.deserializePipelineStage(stringData); + } else { + result = ((SerDesStage) stage).deserializePipelineStage(data); + } if (result == null) { throw new SerDesException("Stage returned a null pipeline result"); } @@ -184,18 +210,35 @@ private static IllegalArgumentException terminalStageFailure(int index, Object s index, stage.getClass().getName())); } - private static void addFlattened(List> target, SerDesStage stage) { - if (stage instanceof ChainedSerDesStage chained) { - target.addAll(chained.stages()); + private static boolean requiresContext(Object stage) { + return stage instanceof SerDes serDes + ? serDes.requiresDurableContext() + : ((SerDesStage) stage).requiresDurableContext(); + } + + private static boolean isTerminal(Object stage) { + return stage instanceof SerDes serDes + ? serDes.isTerminalPipelineStage() + : ((SerDesStage) stage).isTerminalPipelineStage(); + } + + private static void addFlattened(List target, SerDes stage) { + if (stage instanceof ComposableSerDes composable) { + target.add(composable.valueCodec); + target.addAll(composable.stages); } else { target.add(stage); } } + private static void addFlattened(List target, SerDesStage stage) { + target.add(stage); + } + /** Builder for an immutable {@link ComposableSerDes}. */ public static final class Builder { private SerDes valueCodec; - private final List> stages = new ArrayList<>(); + private final List stages = new ArrayList<>(); private Builder(SerDes valueCodec) { this.valueCodec = Objects.requireNonNull(valueCodec, "valueCodec cannot be null"); @@ -205,6 +248,12 @@ private Builder(SerDes valueCodec) { } } + /** Appends a reversible typed stage. */ + public Builder then(SerDes stage) { + addFlattened(stages, Objects.requireNonNull(stage, "stage cannot be null")); + return this; + } + /** Appends a reversible typed stage. */ public Builder then(SerDesStage stage) { addFlattened(stages, Objects.requireNonNull(stage, "stage cannot be null")); diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java index 043d94eb3..903d50e1b 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java @@ -38,7 +38,7 @@ *

Do not use Lambda's ephemeral {@code /tmp} storage. Use a durable shared mount such as EFS, or S3 Files only when * its synchronization and crash-durability tradeoffs are acceptable for the workload. */ -public final class FileSystemSerDes implements SerDes, SerDesStage { +public final class FileSystemSerDes implements SerDes { private static final String ENVELOPE_MARKER = "__durable_execution_filesystem_serdes"; private static final String PAYLOAD_TYPE_FIELD = "payloadType"; private static final int ENVELOPE_VERSION = 1; @@ -140,17 +140,6 @@ public T deserialize(String data, TypeToken typeToken) { return delegate.deserialize(serializedString, typeToken); } - @Override - public Object deserialize(String data) { - if (data == null) { - return null; - } - if (!stageMode) { - throw new SerDesException("Standalone FileSystemSerDes cannot be used as a pipeline stage"); - } - return resolveSerializedPayload(data, requireContext()).value(); - } - @Override public SerDesStageResult deserializePipelineStage(String data) { if (!stageMode) { diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/RetrySerDes.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/RetrySerDes.java index b4d96bf08..2b0017baf 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/RetrySerDes.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/RetrySerDes.java @@ -18,7 +18,7 @@ *

Only {@link RetryableSerDesException} is retried. Other failures are propagated immediately. Retry delays block * the thread executing the SerDes call: the caller by default or the configured SerDes executor thread. */ -public final class RetrySerDes implements SerDes, SerDesStage { +public final class RetrySerDes implements SerDes { private static final Sleeper DEFAULT_SLEEPER = delay -> { if (delay.getSeconds() > 0) { TimeUnit.SECONDS.sleep(delay.getSeconds()); @@ -58,16 +58,6 @@ public T deserialize(String data, TypeToken typeToken) { return execute("deserialization", () -> delegate.deserialize(data, typeToken)); } - @Override - @SuppressWarnings("unchecked") - public Object deserialize(String data) { - if (delegate instanceof SerDesStage stage) { - return execute( - "pipeline stage deserialization", () -> ((SerDesStage) stage).deserialize(data)); - } - return execute("pipeline stage deserialization", () -> delegate.deserialize(data, TypeToken.get(String.class))); - } - @Override public SerDesStageResult deserializePipelineStage(String data) { return execute("pipeline stage deserialization", () -> delegate.deserializePipelineStage(data)); diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDes.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDes.java index 7c656378e..fa61b83bf 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDes.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDes.java @@ -5,7 +5,12 @@ import software.amazon.lambda.durable.TypeToken; import software.amazon.lambda.durable.exception.SerDesException; -/** Interface for serialization and deserialization of objects at the persisted string boundary. */ +/** + * Interface for serialization and deserialization of objects at the persisted string boundary. + * + *

Implementations can also be used as string-producing stages in a {@link ComposableSerDes}. Use {@link SerDesStage} + * for typed intermediate transformations that produce or consume non-string values. + */ public interface SerDes { /** * Serializes an object to a JSON string. @@ -66,4 +71,28 @@ default boolean requiresDurableContext() { default boolean isTerminalPipelineStage() { return false; } + + /** + * Returns an immutable processing pipeline that invokes this SerDes followed by {@code nextStage} when serializing + * and in reverse order when deserializing. + * + *

This SerDes is the value codec. Intermediate stages may transform values into arbitrary Java types, but the + * final stage must return a string for persistence. + * + * @param nextStage the reversible stage to append + * @return a composable SerDes pipeline + */ + default ComposableSerDes then(SerDes nextStage) { + return ComposableSerDes.of(this, nextStage); + } + + /** + * Returns an immutable processing pipeline with a typed intermediate stage appended. + * + * @param nextStage the reversible typed stage to append + * @return a composable SerDes pipeline + */ + default ComposableSerDes then(SerDesStage nextStage) { + return ComposableSerDes.builder(this).then(nextStage).build(); + } } diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesStage.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesStage.java index cedb9250e..9662e87f3 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesStage.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesStage.java @@ -2,8 +2,6 @@ // SPDX-License-Identifier: Apache-2.0 package software.amazon.lambda.durable.serde; -import java.util.Objects; - /** * A reversible typed stage in a {@link ComposableSerDes} pipeline. * @@ -54,16 +52,4 @@ default boolean requiresDurableContext() { default boolean isTerminalPipelineStage() { return false; } - - /** - * Returns an immutable stage chain that applies this stage followed by {@code nextStage} during serialization and - * reverses them in the opposite order during deserialization. - * - * @param nextStage the reversible stage that consumes this stage's output - * @param the next stage's output type - * @return the composed stage chain - */ - default SerDesStage then(SerDesStage nextStage) { - return ChainedSerDesStage.of(this, Objects.requireNonNull(nextStage, "nextStage cannot be null")); - } } diff --git a/sdk/src/test/java/software/amazon/lambda/durable/serde/ComposableSerDesTest.java b/sdk/src/test/java/software/amazon/lambda/durable/serde/ComposableSerDesTest.java index 62d9a0a0e..1fa9927f4 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/serde/ComposableSerDesTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/serde/ComposableSerDesTest.java @@ -26,7 +26,7 @@ void serializesForwardAndDeserializesInReverse() { var calls = new ArrayList(); var first = stringStage("first", "<", ">", calls); var second = stringStage("second", "[", "]", calls); - var pipeline = ComposableSerDes.of(new JacksonSerDes(), first.then(second)); + var pipeline = new JacksonSerDes().then(first).then(second); var serialized = pipeline.serialize("value"); var deserialized = pipeline.deserialize(serialized, TypeToken.get(String.class)); @@ -65,7 +65,7 @@ public byte[] deserialize(String data) { return Base64.getDecoder().decode(data); } }; - var pipeline = ComposableSerDes.of(new JacksonSerDes(), utf8.then(base64)); + var pipeline = new JacksonSerDes().then(utf8).then(base64); var serialized = pipeline.serialize("value"); var deserialized = pipeline.deserialize(serialized, TypeToken.get(String.class)); @@ -76,14 +76,15 @@ public byte[] deserialize(String data) { } @Test - void factoryBuilderAndThenFlattenNestedStageChains() { + void factoryBuilderAndThenFlattenNestedPipelines() { var calls = new ArrayList(); - var nested = stringStage("one", "1", "1", calls).then(stringStage("two", "2", "2", calls)); - var pipeline = - ComposableSerDes.builder(new JacksonSerDes()).then(nested).build(); + var nested = ComposableSerDes.builder(stringStage("codec", "", "", calls)) + .then(stringStage("one", "1", "1", calls)) + .build(); + var pipeline = ComposableSerDes.of(nested).then(stringStage("two", "2", "2", calls)); - assertEquals("21\"value\"12", pipeline.serialize("value")); - assertEquals(List.of("one-serialize", "two-serialize"), calls); + assertEquals("21value12", pipeline.serialize("value")); + assertEquals(List.of("codec-serialize", "one-serialize", "two-serialize"), calls); } @Test @@ -112,19 +113,20 @@ public T deserialize(String data, TypeToken typeToken) { @Test void valueCodecMayDecodeNonNullRepresentationToNull() { var intermediateCalls = new AtomicInteger(); - var identityStage = new SerDesStage() { + var identityStage = new SerDes() { @Override - public String serialize(String value) { - return value; + public String serialize(Object value) { + return value.toString(); } @Override - public String deserialize(String data) { + @SuppressWarnings("unchecked") + public T deserialize(String data, TypeToken typeToken) { intermediateCalls.incrementAndGet(); - return data; + return (T) data; } }; - var pipeline = ComposableSerDes.of(new JacksonSerDes(), identityStage); + var pipeline = new JacksonSerDes().then(identityStage); assertNull(pipeline.deserialize("null", TypeToken.get(Object.class))); assertEquals(1, intermediateCalls.get()); @@ -133,27 +135,29 @@ public String deserialize(String data) { @Test void stageMayDecodeExternalDataDirectlyWithValueCodec() { var transformDeserializations = new AtomicInteger(); - var transform = new SerDesStage() { + var transform = new SerDes() { @Override - public String serialize(String value) { + public String serialize(Object value) { return "<" + value + ">"; } @Override - public String deserialize(String data) { + @SuppressWarnings("unchecked") + public T deserialize(String data, TypeToken typeToken) { transformDeserializations.incrementAndGet(); - return data.substring(1, data.length() - 1); + return (T) data.substring(1, data.length() - 1); } }; - var externalBoundary = new SerDesStage() { + var externalBoundary = new SerDes() { @Override - public String serialize(String value) { - return value; + public String serialize(Object value) { + return value.toString(); } @Override - public String deserialize(String data) { - return data; + @SuppressWarnings("unchecked") + public T deserialize(String data, TypeToken typeToken) { + return (T) data; } @Override @@ -161,7 +165,7 @@ public SerDesStageResult deserializePipelineStage(String data) { return SerDesStageResult.decodeWithValueCodec(data); } }; - var pipeline = ComposableSerDes.of(new JacksonSerDes(), transform.then(externalBoundary)); + var pipeline = new JacksonSerDes().then(transform).then(externalBoundary); assertEquals("value", pipeline.deserialize("\"value\"", TypeToken.get(String.class))); assertEquals(0, transformDeserializations.get()); @@ -169,15 +173,16 @@ public SerDesStageResult deserializePipelineStage(String data) { @Test void rejectsStagesAfterTerminalStage() { - var terminal = new SerDesStage() { + var terminal = new SerDes() { @Override - public String serialize(String value) { - return value; + public String serialize(Object value) { + return value.toString(); } @Override - public String deserialize(String data) { - return data; + @SuppressWarnings("unchecked") + public T deserialize(String data, TypeToken typeToken) { + return (T) data; } @Override @@ -187,7 +192,8 @@ public boolean isTerminalPipelineStage() { }; var failure = assertThrows( - IllegalArgumentException.class, () -> terminal.then(stringStage("late", "", "", new ArrayList<>()))); + IllegalArgumentException.class, + () -> new JacksonSerDes().then(terminal).then(stringStage("late", "", "", new ArrayList<>()))); assertTrue(failure.getMessage().contains("stage 1")); assertTrue(failure.getMessage().contains("final stage")); @@ -195,19 +201,19 @@ public boolean isTerminalPipelineStage() { @Test void rejectsNullIntermediateAndNonStringBoundaryValues() { - var nullStage = new SerDesStage() { + var nullStage = new SerDes() { @Override - public String serialize(String value) { + public String serialize(Object value) { return null; } @Override - public String deserialize(String data) { + public T deserialize(String data, TypeToken typeToken) { return null; } }; - var nullFailure = assertThrows(SerDesException.class, () -> ComposableSerDes.of(new JacksonSerDes(), nullStage) - .serialize("value")); + var nullFailure = assertThrows( + SerDesException.class, () -> new JacksonSerDes().then(nullStage).serialize("value")); assertTrue(nullFailure.getMessage().contains("stage 1")); assertTrue(nullFailure.getMessage().contains(nullStage.getClass().getName())); @@ -222,9 +228,9 @@ public String deserialize(Integer data) { return "x".repeat(data); } }; - var typeFailure = - assertThrows(SerDesException.class, () -> ComposableSerDes.of(new JacksonSerDes(), nonStringFinalStage) - .serialize("value")); + var typeFailure = assertThrows( + SerDesException.class, + () -> new JacksonSerDes().then(nonStringFinalStage).serialize("value")); assertTrue(typeFailure.getMessage().contains("final stage")); assertTrue(typeFailure.getMessage().contains(Integer.class.getName())); } @@ -243,8 +249,9 @@ public Integer deserialize(String data) { } }; - var failure = assertThrows(SerDesException.class, () -> ComposableSerDes.of(new JacksonSerDes(), integerStage) - .serialize("value")); + var failure = assertThrows( + SerDesException.class, + () -> new JacksonSerDes().then(integerStage).serialize("value")); assertTrue(failure.getMessage().contains("stage 1")); assertInstanceOf(ClassCastException.class, failure.getCause()); @@ -252,21 +259,21 @@ public Integer deserialize(String data) { @Test void preservesRetryabilityWhenDecoratingStageFailures() { - var transientStage = new SerDesStage() { + var transientStage = new SerDes() { @Override - public String serialize(String value) { + public String serialize(Object value) { throw new RetryableSerDesException("retry"); } @Override - public String deserialize(String data) { + public T deserialize(String data, TypeToken typeToken) { return null; } }; var failure = assertThrows( RetryableSerDesException.class, - () -> ComposableSerDes.of(new JacksonSerDes(), transientStage).serialize("value")); + () -> new JacksonSerDes().then(transientStage).serialize("value")); assertInstanceOf(RetryableSerDesException.class, failure.getCause()); assertTrue(failure.getMessage().contains("stage 1")); @@ -275,30 +282,30 @@ public String deserialize(String data) { @Test void preservesFatalErrorsFromEveryPipelineCall() { var serializeError = new OutOfMemoryError("serialize"); - var serializeStage = new SerDesStage() { + var serializeStage = new SerDes() { @Override - public String serialize(String value) { + public String serialize(Object value) { throw serializeError; } @Override - public String deserialize(String data) { + public T deserialize(String data, TypeToken typeToken) { return null; } }; - assertSame(serializeError, assertThrows(OutOfMemoryError.class, () -> ComposableSerDes.of( - new JacksonSerDes(), serializeStage) + assertSame(serializeError, assertThrows(OutOfMemoryError.class, () -> new JacksonSerDes() + .then(serializeStage) .serialize("value"))); var stringStageError = new StackOverflowError("string-stage-deserialize"); - var stringStage = new SerDesStage() { + var stringStage = new SerDes() { @Override - public String serialize(String value) { - return value; + public String serialize(Object value) { + return value.toString(); } @Override - public String deserialize(String data) { + public T deserialize(String data, TypeToken typeToken) { return null; } @@ -307,8 +314,8 @@ public SerDesStageResult deserializePipelineStage(String data) { throw stringStageError; } }; - assertSame(stringStageError, assertThrows(StackOverflowError.class, () -> ComposableSerDes.of( - new JacksonSerDes(), stringStage) + assertSame(stringStageError, assertThrows(StackOverflowError.class, () -> new JacksonSerDes() + .then(stringStage) .deserialize("value", TypeToken.get(String.class)))); var valueCodecError = new AssertionError("value-codec-deserialize"); @@ -327,19 +334,19 @@ public T deserialize(String data, TypeToken typeToken) { .deserialize("value", TypeToken.get(String.class)))); } - private static SerDesStage stringStage( - String name, String prefix, String suffix, List calls) { - return new SerDesStage<>() { + private static SerDes stringStage(String name, String prefix, String suffix, List calls) { + return new SerDes() { @Override - public String serialize(String value) { + public String serialize(Object value) { calls.add(name + "-serialize"); return prefix + value + suffix; } @Override - public String deserialize(String data) { + @SuppressWarnings("unchecked") + public T deserialize(String data, TypeToken typeToken) { calls.add(name + "-deserialize"); - return data.substring(prefix.length(), data.length() - suffix.length()); + return (T) data.substring(prefix.length(), data.length() - suffix.length()); } }; } diff --git a/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java b/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java index 850947f9f..0dfb14900 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java @@ -22,7 +22,6 @@ import software.amazon.lambda.durable.exception.RetryableSerDesException; import software.amazon.lambda.durable.exception.SerDesException; import software.amazon.lambda.durable.model.OperationSubType; -import software.amazon.lambda.durable.retry.RetryStrategies; class FileSystemSerDesTest { private static final String ARN = @@ -54,7 +53,7 @@ void standaloneModeWritesDelegatePayloadAndReplaysIt() throws Exception { @Test void stageModeComposesWithValueCodec() throws Exception { var stage = FileSystemSerDes.stageBuilder(basePath).build(); - var pipeline = ComposableSerDes.of(new JacksonSerDes(), stage); + var pipeline = new JacksonSerDes().then(stage); var runner = new SerDesRunner(null); var envelope = runner.serialize(pipeline, Map.of("id", 42), context()); @@ -70,21 +69,7 @@ void stageModeComposesWithValueCodec() throws Exception { () -> runner.deserialize(stage, envelope, TypeToken.get(Integer.class), context())); assertThrows(IllegalStateException.class, () -> FileSystemSerDes.stageBuilder(basePath) .delegate(new JacksonSerDes())); - assertThrows(IllegalArgumentException.class, () -> stage.then(wrappingStage())); - } - - @Test - void retryDecoratorPreservesFilesystemStageComposition() { - var retryStage = - new RetrySerDes(FileSystemSerDes.stageBuilder(basePath).build(), RetryStrategies.Presets.NO_RETRY); - var pipeline = ComposableSerDes.of(new JacksonSerDes(), retryStage); - var runner = new SerDesRunner(null); - - var envelope = runner.serialize(pipeline, Map.of("id", 42), context()); - - assertEquals( - Map.of("id", 42), - runner.deserialize(pipeline, envelope, new TypeToken>() {}, context())); + assertThrows(IllegalArgumentException.class, () -> pipeline.then(wrappingStage())); } @Test @@ -101,7 +86,7 @@ public String deserialize(byte[] data) { } }; var stage = FileSystemSerDes.stageBuilder(basePath).build(); - var pipeline = ComposableSerDes.of(new JacksonSerDes(), utf8.then(stage)); + var pipeline = new JacksonSerDes().then(utf8).then(stage); var runner = new SerDesRunner(null); var envelope = runner.serialize(pipeline, Map.of("id", 42), context()); @@ -203,7 +188,7 @@ void includesBoundedPreviewWithoutChangingStoredPayload() throws Exception { void acceptsExternallyOriginatedRawPayloadsOnlyForSupportedSources() { var standalone = FileSystemSerDes.builder(basePath).build(); var stage = FileSystemSerDes.stageBuilder(basePath).build(); - var pipeline = ComposableSerDes.of(new JacksonSerDes(), wrappingStage().then(stage)); + var pipeline = new JacksonSerDes().then(wrappingStage()).then(stage); var runner = new SerDesRunner(null); assertEquals( @@ -305,15 +290,17 @@ void overflowFilesystemStageMustRemainTerminal() { .storageMode(FileSystemStorageMode.OVERFLOW) .build(); - var failure = assertThrows(IllegalArgumentException.class, () -> filesystem.then(wrappingStage())); + var failure = assertThrows( + IllegalArgumentException.class, + () -> new JacksonSerDes().then(filesystem).then(wrappingStage())); assertTrue(failure.getMessage().contains("final stage")); } @Test void fileReferencesCrossInvokeInputAndResultBoundaries() { - var serDes = ComposableSerDes.of( - new JacksonSerDes(), FileSystemSerDes.stageBuilder(basePath).build()); + var serDes = + new JacksonSerDes().then(FileSystemSerDes.stageBuilder(basePath).build()); var runner = new SerDesRunner(null); var callerArn = "arn:aws:lambda:us-east-1:123456789012:function:caller:1/durable-execution/caller-execution/caller-invocation"; @@ -488,16 +475,17 @@ private static SerDesContext operationContext(OperationType operationType, Opera ARN, "1", "operation", null, operationType, operationSubType, SerDesPayloadKind.RESULT, null); } - private static SerDesStage wrappingStage() { - return new SerDesStage<>() { + private static SerDes wrappingStage() { + return new SerDes() { @Override - public String serialize(String value) { + public String serialize(Object value) { return "<" + value + ">"; } @Override - public String deserialize(String data) { - return data.substring(1, data.length() - 1); + @SuppressWarnings("unchecked") + public T deserialize(String data, TypeToken typeToken) { + return (T) data.substring(1, data.length() - 1); } }; } From 016b5bbae47f5afcdc35b8547cb7a7c16a5b7051 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Tue, 25 Aug 2026 15:25:29 +0000 Subject: [PATCH 22/56] refactor: compose SerDes through string stages --- docs/adr/005-filesystem-serdes.md | 230 +++++++++------ docs/advanced/configuration.md | 22 +- docs/advanced/filesystem-serdes.md | 24 +- docs/design.md | 9 +- .../FileSystemSerDesIntegrationTest.java | 22 +- .../testing/LocalDurableTestRunnerTest.java | 35 ++- .../serde/Base64StringBinaryCodec.java | 22 ++ .../lambda/durable/serde/BinarySerDes.java | 32 +++ .../serde/ComposableBinarySerDesStage.java | 179 ++++++++++++ .../durable/serde/ComposableSerDes.java | 59 ++-- .../durable/serde/FileSystemSerDes.java | 45 +-- .../amazon/lambda/durable/serde/SerDes.java | 8 +- .../lambda/durable/serde/SerDesStage.java | 27 +- .../durable/serde/SerDesStageResult.java | 4 +- .../durable/serde/StringBinaryCodec.java | 32 +++ .../durable/serde/Utf8StringBinaryCodec.java | 22 ++ .../ComposableBinarySerDesStageTest.java | 271 ++++++++++++++++++ .../durable/serde/ComposableSerDesTest.java | 79 ++--- .../durable/serde/FileSystemSerDesTest.java | 65 +++-- 19 files changed, 886 insertions(+), 301 deletions(-) create mode 100644 sdk/src/main/java/software/amazon/lambda/durable/serde/Base64StringBinaryCodec.java create mode 100644 sdk/src/main/java/software/amazon/lambda/durable/serde/BinarySerDes.java create mode 100644 sdk/src/main/java/software/amazon/lambda/durable/serde/ComposableBinarySerDesStage.java create mode 100644 sdk/src/main/java/software/amazon/lambda/durable/serde/StringBinaryCodec.java create mode 100644 sdk/src/main/java/software/amazon/lambda/durable/serde/Utf8StringBinaryCodec.java create mode 100644 sdk/src/test/java/software/amazon/lambda/durable/serde/ComposableBinarySerDesStageTest.java diff --git a/docs/adr/005-filesystem-serdes.md b/docs/adr/005-filesystem-serdes.md index fc999a198..d05ceda05 100644 --- a/docs/adr/005-filesystem-serdes.md +++ b/docs/adr/005-filesystem-serdes.md @@ -2,7 +2,7 @@ **Status:** Accepted — Approach A with ComposableSerDes pipeline **Date:** 2026-07-02 -**Updated:** 2026-08-25 — Included FileSystemSerDes in core and generalized pipelines to typed intermediate stages. +**Updated:** 2026-08-25 — Included FileSystemSerDes in core and added string and nested binary pipelines. ## Context @@ -31,18 +31,34 @@ There are a few Java-specific constraints: ### Summary -Keep the existing `SerDes` serialization methods source- and binary-compatible, add a typed `SerDesStage` +Keep the existing `SerDes` serialization methods source- and binary-compatible, add a string-to-string `SerDesStage` contract and a core `ComposableSerDes` implementation which together form a processing pipeline, and implement -`FileSystemSerDes` in the core SDK. Existing `SerDes` implementations continue to participate as string-producing -stages without changing their binary contract, while dedicated intermediate stages may exchange arbitrary Java types. -The filesystem stage uses -`SerDesContext.getCurrentContext()` to identify the durable execution and entity being serialized. +`FileSystemSerDes` in the core SDK. Every top-level stage consumes and produces a string, making stage composition +uniform and preventing intermediate type mismatches. + +Binary transformations compose inside one `ComposableBinarySerDesStage`. That outer stage converts strings to bytes +with a configurable starting codec, applies any number of reversible `BinarySerDes` implementations without +intermediate text conversion, and converts the final bytes back to a string with a configurable ending codec. The +filesystem stage uses `SerDesContext.getCurrentContext()` to identify the durable execution and entity being +serialized. ```java -public interface SerDesStage { - O serialize(I value); +public interface SerDesStage { + String serialize(String value); + + String deserialize(String data); +} + +public interface BinarySerDes { + byte[] serialize(byte[] value); - I deserialize(O data); + byte[] deserialize(byte[] data); +} + +public interface StringBinaryCodec { + byte[] toBytes(String value); + + String fromBytes(byte[] data); } public interface SerDes { @@ -54,21 +70,20 @@ public interface SerDes { return ComposableSerDes.of(this, nextStage); } - default ComposableSerDes then(SerDesStage nextStage) { + default ComposableSerDes then(SerDesStage nextStage) { return ComposableSerDes.builder(this).then(nextStage).build(); } } ``` -`ComposableSerDes` treats the first stage as the value codec and every later stage as a reversible typed -transformation. This lets customers compose JSON encoding, binary compression, encryption, filesystem storage, or -other processing without artificial Base64 conversion between every stage. The complete pipeline still returns a -string because checkpoints use the existing `SerDes` boundary. +`ComposableSerDes` treats the first stage as the value codec and every later stage as a reversible string +transformation. This lets customers compose JSON encoding, framed string transformations, binary processing, and +filesystem storage without unsafe heterogeneous top-level stages. A `ComposableBinarySerDesStage` performs UTF-8, +compression, encryption, and similar byte processing internally and encodes the result once at its string boundary. -`FileSystemSerDes` acts as a terminal payload-storage stage. It writes the string or byte array produced by the -previous stage to the filesystem when configured to do so and returns a small checkpoint envelope. For standalone -compatibility, it may still be constructed with a value-encoding delegate; pipeline configuration is the preferred -composition model. +`FileSystemSerDes` acts as a terminal payload-storage stage. It writes the string produced by the previous stage to +the filesystem when configured to do so and returns a small checkpoint envelope. For standalone compatibility, it may +still be constructed with a value-encoding delegate; pipeline configuration is the preferred composition model. Because the existing `SerDes` methods do not accept context parameters, this approach needs a thread-local `SerDesContext` so `FileSystemSerDes` can discover the current payload identity without changing the @@ -139,28 +154,26 @@ public final class ComposableSerDes implements SerDes { public SerDes getValueCodec(); public ComposableSerDes then(SerDes stage); - public ComposableSerDes then(SerDesStage stage); + public ComposableSerDes then(SerDesStage stage); public static final class Builder { public Builder then(SerDes stage); - public Builder then(SerDesStage stage); + public Builder then(SerDesStage stage); public ComposableSerDes build(); } } -public record SerDesStageResult(Object value, boolean skipRemainingStages) { - public static SerDesStageResult continueWith(Object value); +public record SerDesStageResult(String value, boolean skipRemainingStages) { + public static SerDesStageResult continueWith(String value); public static SerDesStageResult decodeWithValueCodec(String value); } ``` The first stage is the **value codec**. It converts the user value to a string and converts the final decoded string -back to the requested `TypeToken`. Every later stage is a typed `SerDesStage`. Adjacent stages must be -compatible: one stage's serialized output becomes the next stage's input, and deserialization applies the inverse -mapping. Runtime stage metadata is preserved in failures because Java type erasure prevents complete validation when -heterogeneous stages are held in one immutable pipeline. +back to the requested `TypeToken`. Every later stage consumes and produces a `String`. Stage composition is valid by +construction; binary or other non-string intermediate representations remain encapsulated inside a string stage. Serialization runs from first to last: @@ -168,9 +181,9 @@ Serialization runs from first to last: Object -> value codec -> String - -> typed stage 1 - -> intermediate type A - -> typed stage 2 + -> string stage 1 + -> String + -> string stage 2 -> ... -> checkpoint String ``` @@ -179,9 +192,9 @@ Deserialization runs in the opposite direction: ```text checkpoint String - -> last typed stage + -> last string stage -> ... - -> first typed stage + -> first string stage -> String -> value codec, deserialized as the requested TypeToken -> T @@ -191,15 +204,15 @@ Equivalent pseudocode: ```java String serialize(Object value) { - Object current = valueCodec.serialize(value); + String current = valueCodec.serialize(value); for (var stage : stages) { current = stage.serialize(current); } - return requireStringCheckpoint(current); + return current; } T deserialize(String data, TypeToken targetType) { - Object current = data; + String current = data; for (int i = stages.size() - 1; i >= 0; i--) { var decoded = stages.get(i).deserializePipelineStage(current); current = decoded.value(); @@ -207,15 +220,14 @@ String serialize(Object value) { break; } } - return valueCodec.deserialize(requireStringValueCodecInput(current), targetType); + return valueCodec.deserialize(current, targetType); } ``` Pipeline rules: -- A pipeline must contain exactly one value codec in the first position and zero or more typed intermediate stages. -- Intermediate values may use any non-null Java type. The complete serialization pipeline must finish with a - `String`, and reverse processing must return a `String` to the value codec. +- A pipeline must contain exactly one value codec in the first position and zero or more string stages. +- Every top-level stage consumes and produces a non-null `String`. This makes all stage orderings type-compatible. - A stage may declare that it requires durable context or that it must be terminal. A terminal stage must be the last stage so later transformations cannot invalidate its checkpoint-size or storage decision. - `SerDes.then(...)`, `ComposableSerDes.of(...)`, and the builder flatten nested `ComposableSerDes` instances while @@ -234,9 +246,9 @@ Pipeline rules: - A boundary stage may return `SerDesStageResult.decodeWithValueCodec(...)` when it receives raw data that did not pass through the configured pipeline. `ComposableSerDes` then skips every earlier intermediate stage and decodes the raw value directly with the value codec. -- Stage order is meaningful. For example, `JSON -> compression -> encryption -> filesystem` writes encrypted, - compressed data to the filesystem. `FileSystemSerDes` is terminal; placing encryption or any expanding - transformation after it is rejected. +- Stage order is meaningful. For example, `JSON -> binary composite -> filesystem` writes the encoded result of the + binary composite to the filesystem. `FileSystemSerDes` is terminal; placing any later transformation after it is + rejected. - The ordered stage list and each stage's configuration are part of the persisted checkpoint format. They must remain replay-compatible for in-flight executions. Reordering, removing, or incompatibly reconfiguring a stage requires a versioned envelope or an explicit migration boundary. @@ -251,12 +263,69 @@ The default `SerDes.then(...)` method and immutable `ComposableSerDes.then(...)` independently reusable processing chains: ```java +var binaryStage = ComposableBinarySerDesStage.builder() + .startWith(Utf8StringBinaryCodec.INSTANCE) + .then(compressionBinarySerDes) + .then(encryptionBinarySerDes) + .endWith(Base64StringBinaryCodec.INSTANCE) + .build(); + var securePayloads = new JacksonSerDes() - .then(compressionSerDes) - .then(encryptionSerDes) + .then(binaryStage) .then(fileSystemStage); ``` +### Composable binary stage + +`ComposableBinarySerDesStage` is one top-level `String -> String` stage containing zero or more `byte[] -> byte[]` +transformations: + +```java +public final class ComposableBinarySerDesStage implements SerDesStage { + public static StartBuilder builder(); + + public interface StartBuilder { + BinaryStagesBuilder startWith(StringBinaryCodec codec); + } + + public interface BinaryStagesBuilder { + BinaryStagesBuilder then(BinarySerDes serDes); + + CompletedBuilder endWith(StringBinaryCodec codec); + } + + public interface CompletedBuilder { + ComposableBinarySerDesStage build(); + } +} +``` + +The builder follows forward serialization order and its staged return types prevent another binary transformation from +being appended after `endWith(...)`. + +```text +serialization: +String + -> startingCodec.toBytes + -> binary SerDes 1 + -> binary SerDes 2 + -> endingCodec.fromBytes + -> String + +deserialization: +String + -> endingCodec.toBytes + -> binary SerDes 2 + -> binary SerDes 1 + -> startingCodec.fromBytes + -> String +``` + +Both boundaries use the same `StringBinaryCodec` contract. The core SDK provides UTF-8 and standard Base64 +implementations, while callers may provide reversible alternatives. Each `BinarySerDes` must include required +metadata, such as a format version or encryption initialization vector, in its output. The composite performs text +conversion only at its two outer boundaries; binary stages pass bytes directly to each other. + ### Retryable SerDes stages Transient failures are explicit. `RetryableSerDesException` extends `SerDesException` and marks a failure that may @@ -287,8 +356,8 @@ Retry rules: checkpoint. Strategies must therefore use short, bounded delays that fit within the Lambda invocation timeout. - If the invocation is interrupted or times out, replay may execute the SerDes pipeline again. Any stage with side effects must use stable addressing and idempotent writes. -- `RetrySerDes` can wrap an individual stage or the complete pipeline. Wrapping the smallest transient stage avoids - repeating deterministic encoding, compression, or encryption work. +- `RetrySerDes` can wrap an individual SerDes stage or the complete pipeline. Wrapping the smallest transient SerDes + avoids repeating deterministic encoding, compression, or encryption work. - Pipeline error decoration must preserve retryability: a stage-level `RetryableSerDesException` must remain that type, with stage metadata added to its message or cause, so an enclosing `RetrySerDes` can recognize it. - Filesystem read and write `IOException`s are retryable. Malformed envelopes, invalid paths, unsupported types, and @@ -311,15 +380,15 @@ Path encodings: Envelope format: ```json -{"__durable_execution_filesystem_serdes":1,"data":"","payloadType":"STRING|BYTES","ownerDurableExecutionArn":"","ownerEntityId":""} -{"__durable_execution_filesystem_serdes":1,"file":"","payloadType":"STRING|BYTES","ownerDurableExecutionArn":"","ownerEntityId":""} -{"__durable_execution_filesystem_serdes":1,"file":"","payloadType":"STRING|BYTES","ownerDurableExecutionArn":"","ownerEntityId":"","preview":{ "...": "..." }} +{"__durable_execution_filesystem_serdes":1,"data":"","payloadType":"STRING","ownerDurableExecutionArn":"","ownerEntityId":""} +{"__durable_execution_filesystem_serdes":1,"file":"","payloadType":"STRING","ownerDurableExecutionArn":"","ownerEntityId":""} +{"__durable_execution_filesystem_serdes":1,"file":"","payloadType":"STRING","ownerDurableExecutionArn":"","ownerEntityId":"","preview":{ "...": "..." }} ``` `FileSystemSerDes` must reject calls when `SerDesContext.getCurrentContext()` is `null` or does not include -`durableExecutionArn` and `entityId`. In pipeline mode it accepts `String` and `byte[]` values, records the payload type -in the envelope, stores byte arrays without text conversion, and restores the same representation during reverse -processing. Standalone mode continues to use its configured string value codec. +`durableExecutionArn` and `entityId`. In pipeline mode it accepts a `String`, records the payload type in the envelope, +and restores that string during reverse processing. A preceding `ComposableBinarySerDesStage` must encode its final +bytes to a string before filesystem storage. Standalone mode continues to use its configured string value codec. The marker and version distinguish filesystem envelopes from raw service-originated JSON. Initial root input, callback results, and standard Lambda invoke results may arrive before this SerDes has processed them. For those external @@ -344,12 +413,12 @@ The final file envelope, including any preview, must remain below the checkpoint rejected rather than producing a checkpoint that the service cannot accept. `FileSystemSerDes` declares itself terminal in every mode. This makes its overflow decision apply to the final -checkpoint representation and prevents a later Base64, encryption, or other expanding stage from pushing an inline -envelope over the service limit. +checkpoint representation and prevents a later encoding or other expanding stage from pushing an inline envelope over +the service limit. -In stage mode, the preview generator receives the `String` or `byte[]` produced by the preceding stage, not the -original domain object. A preview that needs domain fields should either parse that representation, be produced by an -earlier stage, or use standalone compatibility mode where `FileSystemSerDes` receives the original value. +In stage mode, the preview generator receives the `String` produced by the preceding stage, not the original domain +object. A preview that needs domain fields should either parse that representation, be produced by an earlier stage, +or use standalone compatibility mode where `FileSystemSerDes` receives the original value. ### Runtime flow @@ -368,12 +437,11 @@ try { } ``` -On deserialization, `FileSystemSerDes` parses its envelope. If the envelope contains `data`, it restores inline text or -Base64-decoded bytes according to `payloadType`. If the envelope contains `file`, it reads the raw file contents and -restores the same representation. `ComposableSerDes` then passes that value to the preceding typed stage. In standalone -compatibility mode, `FileSystemSerDes` passes the resolved string to its configured value-encoding delegate. Raw -external input, callback results, and standard invoke results skip all intermediate stages and go directly to the -value codec when no versioned filesystem marker is present. +On deserialization, `FileSystemSerDes` parses its envelope. If the envelope contains `data`, it restores the inline +text. If the envelope contains `file`, it reads the stored string. `ComposableSerDes` then passes that value to the +preceding string stage. In standalone compatibility mode, `FileSystemSerDes` passes the resolved string to its +configured value-encoding delegate. Raw external input, callback results, and standard invoke results skip all +intermediate stages and go directly to the value codec when no versioned filesystem marker is present. ### Threading @@ -483,28 +551,30 @@ must not synthesize a durable context and serialize initial input through the fu 2. Add the binary-compatible `SerDes.then(...)` default method and `ComposableSerDes` with immutable stage ordering, forward serialization, reverse deserialization, external-boundary bypass, terminal-stage validation, null short-circuiting, and stage-aware errors. -3. Add `RetryableSerDesException` and `RetrySerDes`, reusing `RetryStrategy` for bounded in-invocation retries. -4. Add `SerDesRunner` with inline execution by default and optional dispatch through +3. Add the string-only `SerDesStage` contract plus `BinarySerDes`, `StringBinaryCodec`, and + `ComposableBinarySerDesStage` for nested binary processing with ordered, configurable boundaries. +4. Add `RetryableSerDesException` and `RetrySerDes`, reusing `RetryStrategy` for bounded in-invocation retries. +5. Add `SerDesRunner` with inline execution by default and optional dispatch through `DurableConfig.withSerDesExecutorService(...)`. Do not create a default SerDes pool. -5. Update root input/output handling in `DurableExecutor` to run user payload SerDes through `SerDesRunner` while +6. Update root input/output handling in `DurableExecutor` to run user payload SerDes through `SerDesRunner` while leaving `DurableInputOutputSerDes` internal. -6. Update `SerializableDurableOperation`, `InvokeOperation`, `StepOperation`, `WaitForConditionOperation`, +7. Update `SerializableDurableOperation`, `InvokeOperation`, `StepOperation`, `WaitForConditionOperation`, `CallbackOperation`, `ChildContextOperation`, `MapOperation`, and test helpers to use `SerDesRunner`. -7. Add bounded, invocation-scoped deserialization caching keyed by SerDes identity, entity, payload kind, type, and +8. Add bounded, invocation-scoped deserialization caching keyed by SerDes identity, entity, payload kind, type, and serialized data hash. -8. Update exception serialization and deserialization paths to set `SerDesPayloadKind.EXCEPTION` in TLS. -9. Implement `FileSystemSerDes` in the core `software.amazon.lambda.durable.serde` package with standalone compatibility and - typed terminal-stage modes, `ALWAYS` and `OVERFLOW` storage, `URI` and `HASH` path encodings, envelope parsing, atomic file +9. Update exception serialization and deserialization paths to set `SerDesPayloadKind.EXCEPTION` in TLS. +10. Implement `FileSystemSerDes` in the core `software.amazon.lambda.durable.serde` package with standalone compatibility and + string terminal-stage modes, `ALWAYS` and `OVERFLOW` storage, `URI` and `HASH` path encodings, envelope parsing, atomic file writes where supported by the filesystem, retryable I/O failures, and clear validation errors for missing context or invalid stage input. -10. Add unit tests for pipeline ordering, reverse processing, nulls, invalid intermediate stage types, stage failures, - retry selection, exhaustion, delay handling, interruption, context construction, TLS scoping and restoration, - inline execution, configured thread-pool isolation, cache hits, cache invalidation, exception reconstruction, - malformed filesystem envelopes, and core-artifact packaging. -11. Add integration tests with `LocalDurableTestRunner` for multi-stage pipelines, step results, wait-for-condition +11. Add unit tests for string and binary pipeline ordering, custom boundary codecs, reverse processing, nulls, stage + failures, retry selection, exhaustion, delay handling, interruption, context construction, TLS scoping and + restoration, inline execution, configured thread-pool isolation, cache hits, cache invalidation, exception + reconstruction, malformed filesystem envelopes, and core-artifact packaging. +12. Add integration tests with `LocalDurableTestRunner` for multi-stage pipelines, step results, wait-for-condition state, invoke payload/result, child context results, map results, repeated `get()`, replay from file pointers, and custom exception types. -12. Update README and advanced configuration docs with pipeline and retry examples, filesystem configuration, and +13. Update README and advanced configuration docs with pipeline and retry examples, filesystem configuration, and warnings about `/tmp`, S3 Files flush behavior, and EFS/S3 Files operational requirements. ### Pros @@ -512,8 +582,8 @@ must not synthesize a durable context and serialize initial input through the fu - Delivers the requested parity feature with the smallest new public API surface. - Uses an extension point customers already understand and can configure per operation. - Preserves inline SerDes execution by default, avoiding new thread-hop overhead for existing applications. -- Makes serialization, compression, encryption, and storage independently composable without adding a storage-specific - core interface. +- Makes string stages safely composable while keeping compression and encryption efficiently composable inside one + binary stage. - Makes filesystem storage available without an additional Maven dependency or release artifact. - Avoids committing the core SDK to a generalized offloading envelope before the storage use cases are proven. - Closest to the current JavaScript `createFileSystemSerdes` model and issue #463 wording. @@ -800,8 +870,8 @@ Negative: - Adds optional executor, context, and caching machinery that must stay deterministic. - Adds storage-specific public API and implementation code to the core SDK artifact. - Approach A requires thread-local SerDes context because the existing `SerDes` methods do not accept context. -- Approach A uses typed stage contracts, but heterogeneous pipeline compatibility is still validated at runtime after - type erasure. +- Top-level stages must encode non-string representations at their boundaries. A composable binary stage avoids + repeated conversion between binary substages, but its final bytes still require one string encoding. - Pipeline ordering and stage configuration must remain compatible with persisted checkpoints. - The inline default means filesystem I/O and retry delays block the caller when customers do not configure a SerDes executor. diff --git a/docs/advanced/configuration.md b/docs/advanced/configuration.md index 552dd1790..9685ca09e 100644 --- a/docs/advanced/configuration.md +++ b/docs/advanced/configuration.md @@ -53,7 +53,7 @@ and uses a bounded weak-reference cache for successful deserialization results d ### Filesystem-backed payload storage -The core SDK provides a reversible terminal stage for storing serialized text or bytes on a shared filesystem: +The core SDK provides a reversible terminal stage for storing serialized strings on a shared filesystem: ```java var fileSystemStage = FileSystemSerDes.stageBuilder(Path.of("/mnt/efs/durable-payloads")) @@ -66,7 +66,16 @@ var resilientFileSystemStage = new RetrySerDes( fileSystemStage, RetryStrategies.fixedDelay(3, Duration.ofSeconds(1))); -var serDes = new JacksonSerDes().then(resilientFileSystemStage); +var binaryStage = ComposableBinarySerDesStage.builder() + .startWith(Utf8StringBinaryCodec.INSTANCE) + .then(compressionBinarySerDes) + .then(encryptionBinarySerDes) + .endWith(Base64StringBinaryCodec.INSTANCE) + .build(); + +var serDes = new JacksonSerDes() + .then(binaryStage) + .then(resilientFileSystemStage); var serDesExecutor = Executors.newFixedThreadPool(4); return DurableConfig.builder() @@ -75,10 +84,11 @@ return DurableConfig.builder() .build(); ``` -Pipelines may exchange non-string intermediate values. For example, a custom -`SerDesStage` can compress the JSON string and feed the resulting bytes directly into -`FileSystemSerDes`; reverse processing restores the bytes to the compression stage. The complete pipeline still -returns a string checkpoint envelope. +Every top-level stage consumes and produces a string, so stages compose without intermediate type mismatches. +`ComposableBinarySerDesStage` contains an ordered chain of `BinarySerDes` implementations for compression, encryption, +or other `byte[]` transformations. Its `startWith(...)`, `then(...)`, and `endWith(...)` calls follow serialization +order; deserialization reverses them. Both boundaries use the customizable `StringBinaryCodec` interface. UTF-8 and +Base64 implementations are included in the core SDK, and conversion occurs only around the complete binary chain. `ALWAYS` stores every non-null payload in a file. `OVERFLOW` keeps payloads inline until the checkpoint envelope approaches the 256 KB service limit. `URI` produces readable escaped paths; `HASH` produces fixed-length SHA-256 path diff --git a/docs/advanced/filesystem-serdes.md b/docs/advanced/filesystem-serdes.md index 201782310..89dc89d27 100644 --- a/docs/advanced/filesystem-serdes.md +++ b/docs/advanced/filesystem-serdes.md @@ -28,7 +28,16 @@ var resilientFileSystemStage = new RetrySerDes( fileSystemStage, RetryStrategies.fixedDelay(3, Duration.ofSeconds(1))); -var serDes = new JacksonSerDes().then(resilientFileSystemStage); +var binaryStage = ComposableBinarySerDesStage.builder() + .startWith(Utf8StringBinaryCodec.INSTANCE) + .then(compressionBinarySerDes) + .then(encryptionBinarySerDes) + .endWith(Base64StringBinaryCodec.INSTANCE) + .build(); + +var serDes = new JacksonSerDes() + .then(binaryStage) + .then(resilientFileSystemStage); var serDesExecutor = Executors.newFixedThreadPool(4); return DurableConfig.builder() @@ -37,18 +46,19 @@ return DurableConfig.builder() .build(); ``` -Serialization runs from `JacksonSerDes` to the filesystem stage. Deserialization runs in reverse. Additional reversible -typed stages such as compression or encryption can be inserted with `then(...)`. A -`SerDesStage` may pass compressed or encrypted bytes directly to `FileSystemSerDes`; no Base64 adapter -is required between those stages. The complete pipeline still produces a string checkpoint envelope. +Serialization follows the declaration order above and deserialization runs in reverse. Every top-level stage consumes +and produces a string. `ComposableBinarySerDesStage` converts the string with its starting codec, passes bytes directly +through each `BinarySerDes`, then converts the final bytes to a string with its ending codec. Both boundaries are +customizable through the same `StringBinaryCodec` interface; the example performs UTF-8 and Base64 conversion once +around the complete compression/encryption chain. - `ALWAYS` writes every non-null payload to a file. - `OVERFLOW` stores small payloads inline and offloads envelopes approaching the 256 KB checkpoint limit. - `URI` uses readable escaped path segments. - `HASH` uses fixed-length SHA-256 path segments. -The preview generator receives the incoming stage value, which may be a `String` or `byte[]`. Its output is included -only in file envelopes and the final envelope must remain below the checkpoint threshold. +The preview generator receives the incoming stage string. Its output is included only in file envelopes and the final +envelope must remain below the checkpoint threshold. For compatibility, `FileSystemSerDes.builder(path)` creates a standalone SerDes with `JacksonSerDes` as its default value codec. A custom standalone codec can be supplied with `.delegate(...)`. diff --git a/docs/design.md b/docs/design.md index 269e98500..200339fd8 100644 --- a/docs/design.md +++ b/docs/design.md @@ -355,8 +355,13 @@ software.amazon.lambda.durable │ ├── serde/ │ ├── SerDes # Interface and pipeline composition entry point -│ ├── SerDesStage # Reversible typed intermediate pipeline stage -│ ├── ComposableSerDes # Immutable ordered value-codec/typed-stage pipeline +│ ├── SerDesStage # Reversible string-to-string pipeline stage +│ ├── ComposableSerDes # Immutable ordered value-codec/string-stage pipeline +│ ├── BinarySerDes # Reversible byte-array transformation +│ ├── StringBinaryCodec # Customizable string/byte boundary conversion +│ ├── Utf8StringBinaryCodec # UTF-8 string/byte conversion +│ ├── Base64StringBinaryCodec # Standard Base64 string/byte conversion +│ ├── ComposableBinarySerDesStage # Ordered binary chain exposed as one string stage │ ├── JacksonSerDes # Jackson impl │ ├── RetrySerDes # Retry decorator for transient SerDes failures │ ├── SerDesRunner # Context, optional executor dispatch, and invocation cache diff --git a/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FileSystemSerDesIntegrationTest.java b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FileSystemSerDesIntegrationTest.java index ee0505fe3..701b34dbf 100644 --- a/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FileSystemSerDesIntegrationTest.java +++ b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FileSystemSerDesIntegrationTest.java @@ -8,7 +8,6 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import com.fasterxml.jackson.databind.ObjectMapper; -import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.time.Duration; @@ -37,13 +36,15 @@ import software.amazon.lambda.durable.retry.JitterStrategy; import software.amazon.lambda.durable.retry.RetryStrategies; import software.amazon.lambda.durable.retry.WaitStrategies; +import software.amazon.lambda.durable.serde.Base64StringBinaryCodec; +import software.amazon.lambda.durable.serde.ComposableBinarySerDesStage; import software.amazon.lambda.durable.serde.FileSystemSerDes; import software.amazon.lambda.durable.serde.JacksonSerDes; import software.amazon.lambda.durable.serde.SerDes; import software.amazon.lambda.durable.serde.SerDesContext; import software.amazon.lambda.durable.serde.SerDesPayloadKind; import software.amazon.lambda.durable.serde.SerDesRunner; -import software.amazon.lambda.durable.serde.SerDesStage; +import software.amazon.lambda.durable.serde.Utf8StringBinaryCodec; import software.amazon.lambda.durable.testing.LocalDurableTestRunner; import software.amazon.lambda.durable.testing.TestResult; import software.amazon.lambda.durable.testing.local.LocalMemoryExecutionClient; @@ -469,19 +470,12 @@ void nestedCallbackFailurePreservesProducerContextAcrossReplay() throws Exceptio } private SerDes filesystemPipeline() { - SerDesStage utf8 = new SerDesStage<>() { - @Override - public byte[] serialize(String value) { - return value.getBytes(StandardCharsets.UTF_8); - } - - @Override - public String deserialize(byte[] data) { - return new String(data, StandardCharsets.UTF_8); - } - }; + var binaryStage = ComposableBinarySerDesStage.builder() + .startWith(Utf8StringBinaryCodec.INSTANCE) + .endWith(Base64StringBinaryCodec.INSTANCE) + .build(); return new JacksonSerDes() - .then(utf8) + .then(binaryStage) .then(FileSystemSerDes.stageBuilder(basePath).build()); } diff --git a/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/LocalDurableTestRunnerTest.java b/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/LocalDurableTestRunnerTest.java index 7cf89f041..2f0507990 100644 --- a/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/LocalDurableTestRunnerTest.java +++ b/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/LocalDurableTestRunnerTest.java @@ -5,7 +5,6 @@ import static org.junit.jupiter.api.Assertions.*; import static software.amazon.lambda.durable.TypeToken.get; -import java.nio.charset.StandardCharsets; import java.nio.file.Path; import java.time.Duration; import java.time.Instant; @@ -20,10 +19,14 @@ import software.amazon.lambda.durable.model.ExecutionStatus; import software.amazon.lambda.durable.plugin.DurableExecutionPlugin; import software.amazon.lambda.durable.plugin.InvocationInfo; +import software.amazon.lambda.durable.serde.Base64StringBinaryCodec; +import software.amazon.lambda.durable.serde.BinarySerDes; +import software.amazon.lambda.durable.serde.ComposableBinarySerDesStage; import software.amazon.lambda.durable.serde.FileSystemSerDes; import software.amazon.lambda.durable.serde.JacksonSerDes; import software.amazon.lambda.durable.serde.SerDes; import software.amazon.lambda.durable.serde.SerDesStage; +import software.amazon.lambda.durable.serde.Utf8StringBinaryCodec; class LocalDurableTestRunnerTest { @@ -207,18 +210,22 @@ public T deserialize(String data, TypeToken typeToken) { }; } - private static SerDesStage bytesStage(AtomicInteger deserializeCalls) { - return new SerDesStage<>() { - @Override - public byte[] serialize(String value) { - return value.getBytes(StandardCharsets.UTF_8); - } - - @Override - public String deserialize(byte[] data) { - deserializeCalls.incrementAndGet(); - return new String(data, StandardCharsets.UTF_8); - } - }; + private static SerDesStage bytesStage(AtomicInteger deserializeCalls) { + return ComposableBinarySerDesStage.builder() + .startWith(Utf8StringBinaryCodec.INSTANCE) + .then(new BinarySerDes() { + @Override + public byte[] serialize(byte[] value) { + return value; + } + + @Override + public byte[] deserialize(byte[] data) { + deserializeCalls.incrementAndGet(); + return data; + } + }) + .endWith(Base64StringBinaryCodec.INSTANCE) + .build(); } } diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/Base64StringBinaryCodec.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/Base64StringBinaryCodec.java new file mode 100644 index 000000000..6b4812c84 --- /dev/null +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/Base64StringBinaryCodec.java @@ -0,0 +1,22 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.serde; + +import java.util.Base64; + +/** Converts bytes to and from standard Base64 strings. */ +public final class Base64StringBinaryCodec implements StringBinaryCodec { + public static final Base64StringBinaryCodec INSTANCE = new Base64StringBinaryCodec(); + + private Base64StringBinaryCodec() {} + + @Override + public byte[] toBytes(String value) { + return Base64.getDecoder().decode(value); + } + + @Override + public String fromBytes(byte[] data) { + return Base64.getEncoder().encodeToString(data); + } +} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/BinarySerDes.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/BinarySerDes.java new file mode 100644 index 000000000..350057ee1 --- /dev/null +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/BinarySerDes.java @@ -0,0 +1,32 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.serde; + +/** + * A reversible binary transformation used inside a {@link ComposableBinarySerDesStage}. + * + *

Implementations must include any metadata needed for deserialization, such as format versions or encryption + * initialization vectors, in the returned bytes. + */ +public interface BinarySerDes { + /** + * Applies this transformation during forward serialization. + * + * @param value the non-null input bytes + * @return the non-null transformed bytes + */ + byte[] serialize(byte[] value); + + /** + * Reverses this transformation during deserialization. + * + * @param data the non-null bytes produced by this transformation + * @return the non-null bytes expected by the preceding transformation + */ + byte[] deserialize(byte[] data); + + /** Returns whether this transformation requires an SDK-managed durable execution context. */ + default boolean requiresDurableContext() { + return false; + } +} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/ComposableBinarySerDesStage.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/ComposableBinarySerDesStage.java new file mode 100644 index 000000000..b096afdd8 --- /dev/null +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/ComposableBinarySerDesStage.java @@ -0,0 +1,179 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.serde; + +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import software.amazon.lambda.durable.exception.RetryableSerDesException; +import software.amazon.lambda.durable.exception.SerDesException; + +/** + * A string SerDes stage containing an ordered chain of binary transformations. + * + *

Serialization converts the input string with the starting codec, applies binary SerDes instances in declaration + * order, and converts the final bytes to a string with the ending codec. Deserialization reverses the complete process. + */ +public final class ComposableBinarySerDesStage implements SerDesStage { + private final StringBinaryCodec startingCodec; + private final List binarySerDes; + private final StringBinaryCodec endingCodec; + + private ComposableBinarySerDesStage( + StringBinaryCodec startingCodec, List binarySerDes, StringBinaryCodec endingCodec) { + this.startingCodec = startingCodec; + this.binarySerDes = List.copyOf(binarySerDes); + this.endingCodec = endingCodec; + } + + /** Creates a builder whose methods follow forward serialization order. */ + public static StartBuilder builder() { + return new Builder(); + } + + @Override + public String serialize(String value) { + Objects.requireNonNull(value, "value cannot be null"); + var current = invokeToBytes(startingCodec, value, "starting codec"); + for (int index = 0; index < binarySerDes.size(); index++) { + current = invokeSerialize(binarySerDes.get(index), current, index); + } + return invokeFromBytes(endingCodec, current, "ending codec"); + } + + @Override + public String deserialize(String data) { + Objects.requireNonNull(data, "data cannot be null"); + var current = invokeToBytes(endingCodec, data, "ending codec"); + for (int index = binarySerDes.size() - 1; index >= 0; index--) { + current = invokeDeserialize(binarySerDes.get(index), current, index); + } + return invokeFromBytes(startingCodec, current, "starting codec"); + } + + @Override + public boolean requiresDurableContext() { + return startingCodec.requiresDurableContext() + || endingCodec.requiresDurableContext() + || binarySerDes.stream().anyMatch(BinarySerDes::requiresDurableContext); + } + + private static byte[] invokeToBytes(StringBinaryCodec codec, String value, String name) { + try { + return requireResult(codec.toBytes(value), name); + } catch (Throwable failure) { + throw componentFailure(name, "convert string to bytes", failure); + } + } + + private static String invokeFromBytes(StringBinaryCodec codec, byte[] data, String name) { + try { + return requireResult(codec.fromBytes(data), name); + } catch (Throwable failure) { + throw componentFailure(name, "convert bytes to string", failure); + } + } + + private static byte[] invokeSerialize(BinarySerDes serDes, byte[] value, int index) { + try { + return requireResult(serDes.serialize(value), binaryStageName(index, serDes)); + } catch (Throwable failure) { + throw componentFailure(binaryStageName(index, serDes), "serialize", failure); + } + } + + private static byte[] invokeDeserialize(BinarySerDes serDes, byte[] data, int index) { + try { + return requireResult(serDes.deserialize(data), binaryStageName(index, serDes)); + } catch (Throwable failure) { + throw componentFailure(binaryStageName(index, serDes), "deserialize", failure); + } + } + + private static T requireResult(T result, String component) { + if (result == null) { + throw new SerDesException(component + " returned null for non-null input"); + } + return result; + } + + private static String binaryStageName(int index, BinarySerDes serDes) { + return String.format("binary stage %d (%s)", index, serDes.getClass().getName()); + } + + private static RuntimeException componentFailure(String component, String action, Throwable failure) { + if (failure instanceof Error error) { + throw error; + } + var message = String.format("Composable binary SerDes %s failed to %s", component, action); + if (failure instanceof RetryableSerDesException) { + return new RetryableSerDesException(message, failure); + } + return new SerDesException(message, failure); + } + + /** Builder stage that requires the starting string/binary codec. */ + public interface StartBuilder { + /** + * Sets the codec that converts the input string to bytes during serialization. + * + * @param codec the starting boundary codec + * @return the binary-stage builder + */ + BinaryStagesBuilder startWith(StringBinaryCodec codec); + } + + /** Builder stage that accepts binary SerDes instances in processing order. */ + public interface BinaryStagesBuilder { + /** + * Appends a binary transformation. + * + * @param serDes the binary SerDes + * @return this builder stage + */ + BinaryStagesBuilder then(BinarySerDes serDes); + + /** + * Sets the codec that converts the final bytes to a string during serialization. + * + * @param codec the ending boundary codec + * @return the completed builder + */ + CompletedBuilder endWith(StringBinaryCodec codec); + } + + /** Builder stage that permits only construction of the completed binary pipeline. */ + public interface CompletedBuilder { + /** Returns the immutable string stage. */ + ComposableBinarySerDesStage build(); + } + + private static final class Builder implements StartBuilder, BinaryStagesBuilder, CompletedBuilder { + private StringBinaryCodec startingCodec; + private final List binarySerDes = new ArrayList<>(); + private StringBinaryCodec endingCodec; + + @Override + public BinaryStagesBuilder startWith(StringBinaryCodec codec) { + startingCodec = Objects.requireNonNull(codec, "starting codec cannot be null"); + return this; + } + + @Override + public BinaryStagesBuilder then(BinarySerDes serDes) { + binarySerDes.add(Objects.requireNonNull(serDes, "binary SerDes cannot be null")); + return this; + } + + @Override + public CompletedBuilder endWith(StringBinaryCodec codec) { + endingCodec = Objects.requireNonNull(codec, "ending codec cannot be null"); + return this; + } + + @Override + public ComposableBinarySerDesStage build() { + return new ComposableBinarySerDesStage(startingCodec, binarySerDes, endingCodec); + } + } +} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/ComposableSerDes.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/ComposableSerDes.java index 5484f8cd8..808f925e5 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/ComposableSerDes.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/ComposableSerDes.java @@ -13,9 +13,8 @@ /** * An immutable SerDes processing pipeline. * - *

The first stage is the value codec. Later {@link SerDesStage} instances may exchange arbitrary intermediate Java - * types. Serialization runs from first to last; deserialization runs from last to first. The final serialized value and - * the value returned to the value codec during deserialization must be strings. + *

The first stage is the value codec. Every later stage consumes and produces a string. Serialization runs from + * first to last; deserialization runs from last to first. */ public final class ComposableSerDes implements SerDes { private final SerDes valueCodec; @@ -35,7 +34,7 @@ private ComposableSerDes(SerDes valueCodec, List stages) { } /** - * Creates a pipeline with a value codec followed by zero or more typed stages. + * Creates a pipeline with a value codec followed by zero or more string stages. * * @param first the value codec * @param remaining reversible SerDes stages @@ -88,9 +87,9 @@ public ComposableSerDes then(SerDes stage) { return new ComposableSerDes(valueCodec, combined); } - /** Returns a new pipeline with the supplied typed stage appended. */ + /** Returns a new pipeline with the supplied string stage appended. */ @Override - public ComposableSerDes then(SerDesStage stage) { + public ComposableSerDes then(SerDesStage stage) { var combined = new ArrayList<>(stages); addFlattened(combined, Objects.requireNonNull(stage, "stage cannot be null")); return new ComposableSerDes(valueCodec, combined); @@ -101,15 +100,11 @@ public String serialize(Object value) { if (value == null) { return null; } - Object current = invokeValueCodecSerialize(valueCodec, value); + String current = invokeValueCodecSerialize(valueCodec, value); for (int index = 0; index < stages.size(); index++) { current = invokeStageSerialize(stages.get(index), current, index + 1); } - if (!(current instanceof String serialized)) { - throw new SerDesException( - "SerDes pipeline final stage returned " + current.getClass().getName() + " instead of String"); - } - return serialized; + return current; } @Override @@ -118,7 +113,7 @@ public T deserialize(String data, TypeToken typeToken) { return null; } Objects.requireNonNull(typeToken, "typeToken cannot be null"); - Object current = data; + String current = data; for (int index = stages.size() - 1; index >= 0; index--) { var decoded = invokeStageDeserialize(stages.get(index), current, index + 1); current = decoded.value(); @@ -126,20 +121,13 @@ public T deserialize(String data, TypeToken typeToken) { break; } } - if (!(current instanceof String valueCodecInput)) { - throw new SerDesException("SerDes pipeline produced " - + current.getClass().getName() - + " instead of String for the value codec"); - } - return invokeValueCodecDeserialize(valueCodec, valueCodecInput, typeToken); + return invokeValueCodecDeserialize(valueCodec, current, typeToken); } - @SuppressWarnings("unchecked") - private static Object invokeStageSerialize(Object stage, Object value, int index) { + private static String invokeStageSerialize(Object stage, String value, int index) { try { - var result = stage instanceof SerDes serDes - ? serDes.serialize(value) - : ((SerDesStage) stage).serialize(value); + var result = + stage instanceof SerDes serDes ? serDes.serialize(value) : ((SerDesStage) stage).serialize(value); if (result == null) { throw new SerDesException("Stage returned null for a non-null value"); } @@ -149,18 +137,13 @@ private static Object invokeStageSerialize(Object stage, Object value, int index } } - @SuppressWarnings("unchecked") - private static SerDesStageResult invokeStageDeserialize(Object stage, Object data, int index) { + private static SerDesStageResult invokeStageDeserialize(Object stage, String data, int index) { try { SerDesStageResult result; if (stage instanceof SerDes serDes) { - if (!(data instanceof String stringData)) { - throw new SerDesException("SerDes stage requires String input but received " - + data.getClass().getName()); - } - result = serDes.deserializePipelineStage(stringData); + result = serDes.deserializePipelineStage(data); } else { - result = ((SerDesStage) stage).deserializePipelineStage(data); + result = ((SerDesStage) stage).deserializePipelineStage(data); } if (result == null) { throw new SerDesException("Stage returned a null pipeline result"); @@ -213,13 +196,13 @@ private static IllegalArgumentException terminalStageFailure(int index, Object s private static boolean requiresContext(Object stage) { return stage instanceof SerDes serDes ? serDes.requiresDurableContext() - : ((SerDesStage) stage).requiresDurableContext(); + : ((SerDesStage) stage).requiresDurableContext(); } private static boolean isTerminal(Object stage) { return stage instanceof SerDes serDes ? serDes.isTerminalPipelineStage() - : ((SerDesStage) stage).isTerminalPipelineStage(); + : ((SerDesStage) stage).isTerminalPipelineStage(); } private static void addFlattened(List target, SerDes stage) { @@ -231,7 +214,7 @@ private static void addFlattened(List target, SerDes stage) { } } - private static void addFlattened(List target, SerDesStage stage) { + private static void addFlattened(List target, SerDesStage stage) { target.add(stage); } @@ -248,14 +231,14 @@ private Builder(SerDes valueCodec) { } } - /** Appends a reversible typed stage. */ + /** Appends a reversible SerDes stage. */ public Builder then(SerDes stage) { addFlattened(stages, Objects.requireNonNull(stage, "stage cannot be null")); return this; } - /** Appends a reversible typed stage. */ - public Builder then(SerDesStage stage) { + /** Appends a reversible string stage. */ + public Builder then(SerDesStage stage) { addFlattened(stages, Objects.requireNonNull(stage, "stage cannot be null")); return this; } diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java index 903d50e1b..d7ca3d490 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java @@ -17,7 +17,6 @@ import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.util.Arrays; -import java.util.Base64; import java.util.HexFormat; import java.util.LinkedHashMap; import java.util.Map; @@ -77,7 +76,8 @@ public static Builder builder(Path basePath) { /** * Creates a filesystem terminal-stage builder for use in a composable SerDes pipeline. * - *

Stage mode accepts {@link String} and {@code byte[]} values. + *

Stage mode accepts strings. Use {@link ComposableBinarySerDesStage} before this stage when binary + * transformations are required. * * @param basePath durable shared filesystem root * @return a terminal-stage builder @@ -126,18 +126,14 @@ public T deserialize(String data, TypeToken typeToken) { var context = requireContext(); var serialized = resolveSerializedPayload(data, context).value(); if (stageMode) { - if ((TypeToken.get(String.class).equals(typeToken) && serialized instanceof String) - || (TypeToken.get(byte[].class).equals(typeToken) && serialized instanceof byte[])) { + if (TypeToken.get(String.class).equals(typeToken)) { @SuppressWarnings("unchecked") var value = (T) serialized; return value; } throw new SerDesException("FileSystemSerDes stage payload type does not match requested type " + typeToken); } - if (!(serialized instanceof String serializedString)) { - throw new SerDesException("Standalone FileSystemSerDes cannot decode a binary stage payload"); - } - return delegate.deserialize(serializedString, typeToken); + return delegate.deserialize(serialized, typeToken); } @Override @@ -148,7 +144,7 @@ public SerDesStageResult deserializePipelineStage(String data) { var context = requireContext(); var resolved = resolveSerializedPayload(data, context); return resolved.external() - ? SerDesStageResult.decodeWithValueCodec((String) resolved.value()) + ? SerDesStageResult.decodeWithValueCodec(resolved.value()) : SerDesStageResult.continueWith(resolved.value()); } @@ -167,10 +163,7 @@ private SerializedPayload serializeValue(Object value) { if (value instanceof String stringValue) { return SerializedPayload.fromString(stringValue); } - if (value instanceof byte[] bytes) { - return SerializedPayload.fromBytes(bytes); - } - throw new SerDesException("FileSystemSerDes stage supports String and byte[] values, but received " + throw new SerDesException("FileSystemSerDes stage supports String values, but received " + value.getClass().getName()); } var serialized = delegate.serialize(value); @@ -596,11 +589,10 @@ private static String sha256(byte[] value) { private record PayloadOwner(String durableExecutionArn, String entityId) {} - private record ResolvedPayload(Object value, boolean external) {} + private record ResolvedPayload(String value, boolean external) {} private enum PayloadType { - STRING, - BYTES + STRING } private record SerializedPayload(PayloadType type, byte[] data) { @@ -613,15 +605,8 @@ private static SerializedPayload fromString(String value) { return new SerializedPayload(PayloadType.STRING, value.getBytes(StandardCharsets.UTF_8)); } - private static SerializedPayload fromBytes(byte[] value) { - return new SerializedPayload(PayloadType.BYTES, value); - } - private static SerializedPayload fromInlineValue(PayloadType type, String value) { - return switch (type) { - case STRING -> fromString(value); - case BYTES -> fromBytes(Base64.getDecoder().decode(value)); - }; + return fromString(value); } @Override @@ -641,20 +626,14 @@ private String inlineValue() { if (data == null) { throw new IllegalStateException("Serialized payload does not contain inline data"); } - return switch (type) { - case STRING -> new String(data, StandardCharsets.UTF_8); - case BYTES -> Base64.getEncoder().encodeToString(data); - }; + return new String(data, StandardCharsets.UTF_8); } - private Object value() { + private String value() { if (data == null) { throw new IllegalStateException("Serialized payload does not contain data"); } - return switch (type) { - case STRING -> new String(data, StandardCharsets.UTF_8); - case BYTES -> data.clone(); - }; + return new String(data, StandardCharsets.UTF_8); } } diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDes.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDes.java index fa61b83bf..a990f3345 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDes.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDes.java @@ -9,7 +9,7 @@ * Interface for serialization and deserialization of objects at the persisted string boundary. * *

Implementations can also be used as string-producing stages in a {@link ComposableSerDes}. Use {@link SerDesStage} - * for typed intermediate transformations that produce or consume non-string values. + * for transformations that consume and produce strings. */ public interface SerDes { /** @@ -87,12 +87,12 @@ default ComposableSerDes then(SerDes nextStage) { } /** - * Returns an immutable processing pipeline with a typed intermediate stage appended. + * Returns an immutable processing pipeline with a string stage appended. * - * @param nextStage the reversible typed stage to append + * @param nextStage the reversible string stage to append * @return a composable SerDes pipeline */ - default ComposableSerDes then(SerDesStage nextStage) { + default ComposableSerDes then(SerDesStage nextStage) { return ComposableSerDes.builder(this).then(nextStage).build(); } } diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesStage.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesStage.java index 9662e87f3..43a91a058 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesStage.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesStage.java @@ -3,31 +3,28 @@ package software.amazon.lambda.durable.serde; /** - * A reversible typed stage in a {@link ComposableSerDes} pipeline. + * A reversible string stage in a {@link ComposableSerDes} pipeline. * - *

Serialization maps {@code I} to {@code O}; deserialization applies the inverse mapping. Intermediate stages may - * use any Java types. The complete pipeline must still produce a {@link String} at its checkpoint boundary because that - * is the persisted representation required by {@link SerDes}. - * - * @param the stage input type during serialization - * @param the stage output type during serialization + *

Every top-level stage consumes and produces a string, so stages can be composed in any order without intermediate + * type mismatches. Use {@link ComposableBinarySerDesStage} to perform an efficient chain of binary transformations + * inside one string stage. */ -public interface SerDesStage { +public interface SerDesStage { /** * Applies this stage during forward serialization. * - * @param value the non-null input value - * @return the non-null transformed value + * @param value the non-null input string + * @return the non-null transformed string */ - O serialize(I value); + String serialize(String value); /** * Reverses this stage during deserialization. * - * @param data the non-null serialized form produced by this stage - * @return the non-null value expected by the preceding stage + * @param data the non-null serialized string produced by this stage + * @return the non-null string expected by the preceding stage */ - I deserialize(O data); + String deserialize(String data); /** * Reverses this stage with control over external-boundary processing. @@ -39,7 +36,7 @@ public interface SerDesStage { * @param data the non-null serialized form produced by this stage * @return the stage result */ - default SerDesStageResult deserializePipelineStage(O data) { + default SerDesStageResult deserializePipelineStage(String data) { return SerDesStageResult.continueWith(deserialize(data)); } diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesStageResult.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesStageResult.java index ed7c3a939..c0aefb4c6 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesStageResult.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesStageResult.java @@ -11,13 +11,13 @@ * @param skipRemainingStages whether deserialization should skip the remaining intermediate stages and decode * {@code value} directly with the pipeline's value codec */ -public record SerDesStageResult(Object value, boolean skipRemainingStages) { +public record SerDesStageResult(String value, boolean skipRemainingStages) { public SerDesStageResult { Objects.requireNonNull(value, "value cannot be null"); } /** Continues reverse processing through the remaining intermediate stages. */ - public static SerDesStageResult continueWith(Object value) { + public static SerDesStageResult continueWith(String value) { return new SerDesStageResult(value, false); } diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/StringBinaryCodec.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/StringBinaryCodec.java new file mode 100644 index 000000000..9f5b96a8a --- /dev/null +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/StringBinaryCodec.java @@ -0,0 +1,32 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.serde; + +/** + * A reversible conversion between strings and bytes at a {@link ComposableBinarySerDesStage} boundary. + * + *

The neutral {@code toBytes}/{@code fromBytes} names allow the same contract to be used at both ends of the binary + * processing chain. + */ +public interface StringBinaryCodec { + /** + * Converts a non-null string to bytes. + * + * @param value the string value + * @return the non-null byte representation + */ + byte[] toBytes(String value); + + /** + * Converts non-null bytes to a string. + * + * @param data the byte representation + * @return the non-null string value + */ + String fromBytes(byte[] data); + + /** Returns whether this codec requires an SDK-managed durable execution context. */ + default boolean requiresDurableContext() { + return false; + } +} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/Utf8StringBinaryCodec.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/Utf8StringBinaryCodec.java new file mode 100644 index 000000000..27af11f98 --- /dev/null +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/Utf8StringBinaryCodec.java @@ -0,0 +1,22 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.serde; + +import java.nio.charset.StandardCharsets; + +/** Converts strings to and from UTF-8 bytes. */ +public final class Utf8StringBinaryCodec implements StringBinaryCodec { + public static final Utf8StringBinaryCodec INSTANCE = new Utf8StringBinaryCodec(); + + private Utf8StringBinaryCodec() {} + + @Override + public byte[] toBytes(String value) { + return value.getBytes(StandardCharsets.UTF_8); + } + + @Override + public String fromBytes(byte[] data) { + return new String(data, StandardCharsets.UTF_8); + } +} diff --git a/sdk/src/test/java/software/amazon/lambda/durable/serde/ComposableBinarySerDesStageTest.java b/sdk/src/test/java/software/amazon/lambda/durable/serde/ComposableBinarySerDesStageTest.java new file mode 100644 index 000000000..1fe3811dc --- /dev/null +++ b/sdk/src/test/java/software/amazon/lambda/durable/serde/ComposableBinarySerDesStageTest.java @@ -0,0 +1,271 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.serde; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Base64; +import java.util.List; +import org.junit.jupiter.api.Test; +import software.amazon.lambda.durable.TypeToken; +import software.amazon.lambda.durable.exception.RetryableSerDesException; +import software.amazon.lambda.durable.exception.SerDesException; + +class ComposableBinarySerDesStageTest { + + @Test + void processesBoundariesAndBinarySerDesInDeclarationOrder() { + var calls = new ArrayList(); + var stage = ComposableBinarySerDesStage.builder() + .startWith(recordingCodec("starting", calls)) + .then(appendingSerDes("first", (byte) 1, calls)) + .then(appendingSerDes("second", (byte) 2, calls)) + .endWith(recordingCodec("ending", calls)) + .build(); + + var serialized = stage.serialize("value"); + var deserialized = stage.deserialize(serialized); + + assertEquals(Base64.getEncoder().encodeToString(new byte[] {'v', 'a', 'l', 'u', 'e', 1, 2}), serialized); + assertEquals("value", deserialized); + assertEquals( + List.of( + "starting-to-bytes", + "first-serialize", + "second-serialize", + "ending-from-bytes", + "ending-to-bytes", + "second-deserialize", + "first-deserialize", + "starting-from-bytes"), + calls); + } + + @Test + void composesWithRootSerDesAsOneStringStage() { + var stage = ComposableBinarySerDesStage.builder() + .startWith(Utf8StringBinaryCodec.INSTANCE) + .then(xorSerDes((byte) 0x5A)) + .endWith(Base64StringBinaryCodec.INSTANCE) + .build(); + var pipeline = new JacksonSerDes().then(stage); + + var serialized = pipeline.serialize("value"); + + assertEquals("value", pipeline.deserialize(serialized, TypeToken.get(String.class))); + } + + @Test + void supportsCustomCodecsAtBothBoundaries() { + var reverseUtf8 = new StringBinaryCodec() { + @Override + public byte[] toBytes(String value) { + var bytes = value.getBytes(StandardCharsets.UTF_8); + reverse(bytes); + return bytes; + } + + @Override + public String fromBytes(byte[] data) { + var copy = Arrays.copyOf(data, data.length); + reverse(copy); + return new String(copy, StandardCharsets.UTF_8); + } + }; + var stage = ComposableBinarySerDesStage.builder() + .startWith(reverseUtf8) + .endWith(Base64StringBinaryCodec.INSTANCE) + .build(); + + var serialized = stage.serialize("value"); + + assertEquals(Base64.getEncoder().encodeToString("eulav".getBytes(StandardCharsets.UTF_8)), serialized); + assertEquals("value", stage.deserialize(serialized)); + } + + @Test + void delegatesDurableContextRequirement() { + var contextCodec = new StringBinaryCodec() { + @Override + public byte[] toBytes(String value) { + return value.getBytes(StandardCharsets.UTF_8); + } + + @Override + public String fromBytes(byte[] data) { + return new String(data, StandardCharsets.UTF_8); + } + + @Override + public boolean requiresDurableContext() { + return true; + } + }; + + var stage = ComposableBinarySerDesStage.builder() + .startWith(contextCodec) + .endWith(Base64StringBinaryCodec.INSTANCE) + .build(); + + assertTrue(stage.requiresDurableContext()); + } + + @Test + void validatesConfigurationAndComponentResults() { + assertThrows(NullPointerException.class, () -> ComposableBinarySerDesStage.builder() + .startWith(null)); + assertThrows(NullPointerException.class, () -> ComposableBinarySerDesStage.builder() + .startWith(Utf8StringBinaryCodec.INSTANCE) + .then(null)); + assertThrows(NullPointerException.class, () -> ComposableBinarySerDesStage.builder() + .startWith(Utf8StringBinaryCodec.INSTANCE) + .endWith(null)); + + var nullStage = ComposableBinarySerDesStage.builder() + .startWith(Utf8StringBinaryCodec.INSTANCE) + .then(new BinarySerDes() { + @Override + public byte[] serialize(byte[] value) { + return null; + } + + @Override + public byte[] deserialize(byte[] data) { + return data; + } + }) + .endWith(Base64StringBinaryCodec.INSTANCE) + .build(); + + var failure = assertThrows(SerDesException.class, () -> nullStage.serialize("value")); + assertTrue(failure.getMessage().contains("binary stage 0")); + } + + @Test + void preservesRetryableFailuresAndFatalErrors() { + var retryableStage = ComposableBinarySerDesStage.builder() + .startWith(Utf8StringBinaryCodec.INSTANCE) + .then(new BinarySerDes() { + @Override + public byte[] serialize(byte[] value) { + throw new RetryableSerDesException("retry"); + } + + @Override + public byte[] deserialize(byte[] data) { + return data; + } + }) + .endWith(Base64StringBinaryCodec.INSTANCE) + .build(); + + var retryable = assertThrows(RetryableSerDesException.class, () -> retryableStage.serialize("value")); + assertTrue(retryable.getMessage().contains("binary stage 0")); + + var fatalError = new AssertionError("fatal"); + var fatalStage = ComposableBinarySerDesStage.builder() + .startWith(new StringBinaryCodec() { + @Override + public byte[] toBytes(String value) { + throw fatalError; + } + + @Override + public String fromBytes(byte[] data) { + return ""; + } + }) + .endWith(Base64StringBinaryCodec.INSTANCE) + .build(); + + assertSame(fatalError, assertThrows(AssertionError.class, () -> fatalStage.serialize("value"))); + } + + @Test + void providedCodecsRoundTrip() { + var value = "hello λ"; + var bytes = Utf8StringBinaryCodec.INSTANCE.toBytes(value); + + assertEquals(value, Utf8StringBinaryCodec.INSTANCE.fromBytes(bytes)); + assertArrayEquals( + bytes, Base64StringBinaryCodec.INSTANCE.toBytes(Base64StringBinaryCodec.INSTANCE.fromBytes(bytes))); + } + + private static StringBinaryCodec recordingCodec(String name, List calls) { + return new StringBinaryCodec() { + @Override + public byte[] toBytes(String value) { + calls.add(name + "-to-bytes"); + return name.equals("ending") + ? Base64.getDecoder().decode(value) + : value.getBytes(StandardCharsets.UTF_8); + } + + @Override + public String fromBytes(byte[] data) { + calls.add(name + "-from-bytes"); + return name.equals("ending") + ? Base64.getEncoder().encodeToString(data) + : new String(data, StandardCharsets.UTF_8); + } + }; + } + + private static BinarySerDes appendingSerDes(String name, byte suffix, List calls) { + return new BinarySerDes() { + @Override + public byte[] serialize(byte[] value) { + calls.add(name + "-serialize"); + var result = Arrays.copyOf(value, value.length + 1); + result[value.length] = suffix; + return result; + } + + @Override + public byte[] deserialize(byte[] data) { + calls.add(name + "-deserialize"); + if (data.length == 0 || data[data.length - 1] != suffix) { + throw new SerDesException("Unexpected suffix"); + } + return Arrays.copyOf(data, data.length - 1); + } + }; + } + + private static BinarySerDes xorSerDes(byte key) { + return new BinarySerDes() { + @Override + public byte[] serialize(byte[] value) { + return xor(value, key); + } + + @Override + public byte[] deserialize(byte[] data) { + return xor(data, key); + } + }; + } + + private static byte[] xor(byte[] value, byte key) { + var result = Arrays.copyOf(value, value.length); + for (int index = 0; index < result.length; index++) { + result[index] ^= key; + } + return result; + } + + private static void reverse(byte[] value) { + for (int left = 0, right = value.length - 1; left < right; left++, right--) { + var temporary = value[left]; + value[left] = value[right]; + value[right] = temporary; + } + } +} diff --git a/sdk/src/test/java/software/amazon/lambda/durable/serde/ComposableSerDesTest.java b/sdk/src/test/java/software/amazon/lambda/durable/serde/ComposableSerDesTest.java index 1fa9927f4..fc9208786 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/serde/ComposableSerDesTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/serde/ComposableSerDesTest.java @@ -9,9 +9,7 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; -import java.nio.charset.StandardCharsets; import java.util.ArrayList; -import java.util.Base64; import java.util.List; import java.util.concurrent.atomic.AtomicInteger; import org.junit.jupiter.api.Test; @@ -37,42 +35,42 @@ void serializesForwardAndDeserializesInReverse() { } @Test - void supportsTypedIntermediateValues() { + void supportsDedicatedStringStages() { var calls = new ArrayList(); - SerDesStage utf8 = new SerDesStage<>() { + var first = new SerDesStage() { @Override - public byte[] serialize(String value) { - calls.add("bytes-serialize"); - return value.getBytes(StandardCharsets.UTF_8); + public String serialize(String value) { + calls.add("first-serialize"); + return "<" + value + ">"; } @Override - public String deserialize(byte[] data) { - calls.add("bytes-deserialize"); - return new String(data, StandardCharsets.UTF_8); + public String deserialize(String data) { + calls.add("first-deserialize"); + return data.substring(1, data.length() - 1); } }; - SerDesStage base64 = new SerDesStage<>() { + var second = new SerDesStage() { @Override - public String serialize(byte[] value) { - calls.add("base64-serialize"); - return Base64.getEncoder().encodeToString(value); + public String serialize(String value) { + calls.add("second-serialize"); + return "[" + value + "]"; } @Override - public byte[] deserialize(String data) { - calls.add("base64-deserialize"); - return Base64.getDecoder().decode(data); + public String deserialize(String data) { + calls.add("second-deserialize"); + return data.substring(1, data.length() - 1); } }; - var pipeline = new JacksonSerDes().then(utf8).then(base64); + var pipeline = new JacksonSerDes().then(first).then(second); var serialized = pipeline.serialize("value"); var deserialized = pipeline.deserialize(serialized, TypeToken.get(String.class)); - assertEquals(Base64.getEncoder().encodeToString("\"value\"".getBytes(StandardCharsets.UTF_8)), serialized); + assertEquals("[<\"value\">]", serialized); assertEquals("value", deserialized); - assertEquals(List.of("bytes-serialize", "base64-serialize", "base64-deserialize", "bytes-deserialize"), calls); + assertEquals(List.of("first-serialize", "second-serialize", "second-deserialize", "first-deserialize"), calls); } @Test @@ -200,7 +198,7 @@ public boolean isTerminalPipelineStage() { } @Test - void rejectsNullIntermediateAndNonStringBoundaryValues() { + void rejectsNullIntermediateValues() { var nullStage = new SerDes() { @Override public String serialize(Object value) { @@ -216,45 +214,6 @@ public T deserialize(String data, TypeToken typeToken) { SerDesException.class, () -> new JacksonSerDes().then(nullStage).serialize("value")); assertTrue(nullFailure.getMessage().contains("stage 1")); assertTrue(nullFailure.getMessage().contains(nullStage.getClass().getName())); - - SerDesStage nonStringFinalStage = new SerDesStage<>() { - @Override - public Integer serialize(String value) { - return value.length(); - } - - @Override - public String deserialize(Integer data) { - return "x".repeat(data); - } - }; - var typeFailure = assertThrows( - SerDesException.class, - () -> new JacksonSerDes().then(nonStringFinalStage).serialize("value")); - assertTrue(typeFailure.getMessage().contains("final stage")); - assertTrue(typeFailure.getMessage().contains(Integer.class.getName())); - } - - @Test - void incompatibleTypedStagesFailWithStageMetadata() { - SerDesStage integerStage = new SerDesStage<>() { - @Override - public String serialize(Integer value) { - return value.toString(); - } - - @Override - public Integer deserialize(String data) { - return Integer.valueOf(data); - } - }; - - var failure = assertThrows( - SerDesException.class, - () -> new JacksonSerDes().then(integerStage).serialize("value")); - - assertTrue(failure.getMessage().contains("stage 1")); - assertInstanceOf(ClassCastException.class, failure.getCause()); } @Test diff --git a/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java b/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java index 0dfb14900..f76a82848 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java @@ -2,7 +2,6 @@ // SPDX-License-Identifier: Apache-2.0 package software.amazon.lambda.durable.serde; -import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertInstanceOf; @@ -14,6 +13,8 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; +import java.util.Arrays; +import java.util.Base64; import java.util.Map; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -73,47 +74,37 @@ void stageModeComposesWithValueCodec() throws Exception { } @Test - void stageModeStoresAndRestoresBinaryIntermediateValues() throws Exception { - SerDesStage utf8 = new SerDesStage<>() { - @Override - public byte[] serialize(String value) { - return value.getBytes(StandardCharsets.UTF_8); - } - - @Override - public String deserialize(byte[] data) { - return new String(data, StandardCharsets.UTF_8); - } - }; + void stageModeStoresAndRestoresComposableBinaryOutput() throws Exception { + var binaryStage = ComposableBinarySerDesStage.builder() + .startWith(Utf8StringBinaryCodec.INSTANCE) + .then(xorBinarySerDes((byte) 0x5A)) + .endWith(Base64StringBinaryCodec.INSTANCE) + .build(); var stage = FileSystemSerDes.stageBuilder(basePath).build(); - var pipeline = new JacksonSerDes().then(utf8).then(stage); + var pipeline = new JacksonSerDes().then(binaryStage).then(stage); var runner = new SerDesRunner(null); var envelope = runner.serialize(pipeline, Map.of("id", 42), context()); var json = MAPPER.readTree(envelope); var file = Path.of(json.get("file").textValue()); - assertEquals("BYTES", json.get("payloadType").textValue()); - assertEquals("{\"id\":42}", new String(Files.readAllBytes(file), StandardCharsets.UTF_8)); + assertEquals("STRING", json.get("payloadType").textValue()); + assertEquals( + Base64.getEncoder().encodeToString(xor("{\"id\":42}".getBytes(StandardCharsets.UTF_8), (byte) 0x5A)), + Files.readString(file)); assertEquals( Map.of("id", 42), runner.deserialize(pipeline, envelope, new TypeToken>() {}, context())); } @Test - void overflowModeKeepsSmallBinaryPayloadsInline() throws Exception { + void stageModeRejectsDirectBinaryValues() { var stage = FileSystemSerDes.stageBuilder(basePath) .storageMode(FileSystemStorageMode.OVERFLOW) .build(); var runner = new SerDesRunner(null); - var value = new byte[] {0, 1, 2, -1}; - var envelope = runner.serialize(stage, value, context()); - var json = MAPPER.readTree(envelope); - - assertEquals("BYTES", json.get("payloadType").textValue()); - assertTrue(json.has("data")); - assertArrayEquals(value, runner.deserialize(stage, envelope, TypeToken.get(byte[].class), context())); + assertThrows(SerDesException.class, () -> runner.serialize(stage, new byte[] {0, 1, 2, -1}, context())); } @Test @@ -251,7 +242,7 @@ void rejectsUnsupportedEnvelopeVersionsAtExternalBoundaries() { } @Test - void rejectsMalformedBinaryInlinePayload() { + void rejectsUnsupportedBinaryPayloadType() { var envelope = "{\"__durable_execution_filesystem_serdes\":1," + "\"ownerDurableExecutionArn\":\"" + ARN @@ -261,7 +252,7 @@ void rejectsMalformedBinaryInlinePayload() { .deserialize( FileSystemSerDes.stageBuilder(basePath).build(), envelope, - TypeToken.get(byte[].class), + TypeToken.get(String.class), context())); } @@ -475,6 +466,28 @@ private static SerDesContext operationContext(OperationType operationType, Opera ARN, "1", "operation", null, operationType, operationSubType, SerDesPayloadKind.RESULT, null); } + private static BinarySerDes xorBinarySerDes(byte key) { + return new BinarySerDes() { + @Override + public byte[] serialize(byte[] value) { + return xor(value, key); + } + + @Override + public byte[] deserialize(byte[] data) { + return xor(data, key); + } + }; + } + + private static byte[] xor(byte[] value, byte key) { + var result = Arrays.copyOf(value, value.length); + for (int index = 0; index < result.length; index++) { + result[index] ^= key; + } + return result; + } + private static SerDes wrappingStage() { return new SerDes() { @Override From 0d561b33bdf24435da71d1cdb05765aebb24795a Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Tue, 25 Aug 2026 17:18:47 +0000 Subject: [PATCH 23/56] fix: address filesystem SerDes review feedback --- .../durable/serde/FileSystemSerDes.java | 29 +++++++++---------- .../durable/serde/Utf8StringBinaryCodec.java | 28 ++++++++++++++++-- .../ComposableBinarySerDesStageTest.java | 8 +++++ .../durable/serde/FileSystemSerDesTest.java | 14 +++++++++ 4 files changed, 61 insertions(+), 18 deletions(-) diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java index d7ca3d490..21b61ba08 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java @@ -7,13 +7,11 @@ import com.fasterxml.jackson.databind.ObjectMapper; import java.io.IOException; import java.nio.charset.StandardCharsets; -import java.nio.file.AtomicMoveNotSupportedException; import java.nio.file.FileAlreadyExistsException; import java.nio.file.Files; import java.nio.file.LinkOption; import java.nio.file.NoSuchFileException; import java.nio.file.Path; -import java.nio.file.StandardCopyOption; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.util.Arrays; @@ -450,17 +448,14 @@ private void writePayload(SerializedPayload payload, Path file) throws IOExcepti throw new SerDesException("Filesystem SerDes directory resolves outside the configured base path"); } if (Files.exists(file, LinkOption.NOFOLLOW_LINKS)) { - var existing = Files.readAllBytes(file); - if (!Arrays.equals(existing, payload.data())) { - throw new SerDesException("Filesystem SerDes content-addressed file contains unexpected data"); - } + validateExistingPayload(file, payload.data()); return; } var temporary = Files.createTempFile(directory, file.getFileName().toString(), ".tmp"); try { Files.write(temporary, payload.data()); - moveWithoutReplacement(temporary, file); + publishWithoutReplacement(temporary, file, payload.data()); } finally { Files.deleteIfExists(temporary); } @@ -514,17 +509,19 @@ private synchronized Path retainCanonicalBasePath(Path currentBasePath) { return canonicalBasePath; } - private static void moveWithoutReplacement(Path temporary, Path file) throws IOException { + private void publishWithoutReplacement(Path temporary, Path file, byte[] expectedData) throws IOException { try { - Files.move(temporary, file, StandardCopyOption.ATOMIC_MOVE); - } catch (AtomicMoveNotSupportedException e) { - try { - Files.move(temporary, file); - } catch (FileAlreadyExistsException ignored) { - // Another thread or invocation already persisted the same content-addressed payload. - } + Files.createLink(file, temporary); } catch (FileAlreadyExistsException ignored) { - // Another thread or invocation already persisted the same content-addressed payload. + validateExistingPayload(file, expectedData); + } + } + + private void validateExistingPayload(Path file, byte[] expectedData) throws IOException { + rejectSymbolicLinks(file); + var existing = Files.readAllBytes(file); + if (!Arrays.equals(existing, expectedData)) { + throw new SerDesException("Filesystem SerDes content-addressed file contains unexpected data"); } } diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/Utf8StringBinaryCodec.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/Utf8StringBinaryCodec.java index 27af11f98..1d78fe42a 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/Utf8StringBinaryCodec.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/Utf8StringBinaryCodec.java @@ -2,7 +2,12 @@ // SPDX-License-Identifier: Apache-2.0 package software.amazon.lambda.durable.serde; +import java.nio.ByteBuffer; +import java.nio.CharBuffer; +import java.nio.charset.CharacterCodingException; +import java.nio.charset.CodingErrorAction; import java.nio.charset.StandardCharsets; +import software.amazon.lambda.durable.exception.SerDesException; /** Converts strings to and from UTF-8 bytes. */ public final class Utf8StringBinaryCodec implements StringBinaryCodec { @@ -12,11 +17,30 @@ private Utf8StringBinaryCodec() {} @Override public byte[] toBytes(String value) { - return value.getBytes(StandardCharsets.UTF_8); + var encoder = StandardCharsets.UTF_8 + .newEncoder() + .onMalformedInput(CodingErrorAction.REPORT) + .onUnmappableCharacter(CodingErrorAction.REPORT); + try { + var encoded = encoder.encode(CharBuffer.wrap(value)); + var result = new byte[encoded.remaining()]; + encoded.get(result); + return result; + } catch (CharacterCodingException e) { + throw new SerDesException("Failed to encode string as UTF-8", e); + } } @Override public String fromBytes(byte[] data) { - return new String(data, StandardCharsets.UTF_8); + var decoder = StandardCharsets.UTF_8 + .newDecoder() + .onMalformedInput(CodingErrorAction.REPORT) + .onUnmappableCharacter(CodingErrorAction.REPORT); + try { + return decoder.decode(ByteBuffer.wrap(data)).toString(); + } catch (CharacterCodingException e) { + throw new SerDesException("Failed to decode UTF-8 bytes", e); + } } } diff --git a/sdk/src/test/java/software/amazon/lambda/durable/serde/ComposableBinarySerDesStageTest.java b/sdk/src/test/java/software/amazon/lambda/durable/serde/ComposableBinarySerDesStageTest.java index 1fe3811dc..a24f5f3f9 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/serde/ComposableBinarySerDesStageTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/serde/ComposableBinarySerDesStageTest.java @@ -198,6 +198,14 @@ void providedCodecsRoundTrip() { bytes, Base64StringBinaryCodec.INSTANCE.toBytes(Base64StringBinaryCodec.INSTANCE.fromBytes(bytes))); } + @Test + void utf8CodecRejectsLossyConversions() { + assertThrows(SerDesException.class, () -> Utf8StringBinaryCodec.INSTANCE.toBytes("lone surrogate \uD800")); + assertThrows( + SerDesException.class, + () -> Utf8StringBinaryCodec.INSTANCE.fromBytes(new byte[] {(byte) 0xC3, (byte) 0x28})); + } + private static StringBinaryCodec recordingCodec(String name, List calls) { return new StringBinaryCodec() { @Override diff --git a/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java b/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java index f76a82848..4fa212d25 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java @@ -139,6 +139,20 @@ void immutableContentAddressedFilesPreservePriorCheckpoints() throws Exception { assertEquals("state-one", runner.deserialize(serDes, firstEnvelope, TypeToken.get(String.class), firstContext)); } + @Test + void rejectsUnexpectedContentInExistingContentAddressedFile() throws Exception { + var serDes = FileSystemSerDes.stageBuilder(basePath).build(); + var runner = new SerDesRunner(null); + var envelope = runner.serialize(serDes, "expected", context()); + var file = payloadFile(envelope); + Files.writeString(file, "unexpected"); + + var failure = assertThrows(SerDesException.class, () -> runner.serialize(serDes, "expected", context())); + + assertTrue(failure.getCause().getMessage().contains("contains unexpected data")); + assertEquals("unexpected", Files.readString(file)); + } + @Test void hashEncodingUsesFixedLengthSegments() throws Exception { var serDes = FileSystemSerDes.builder(basePath) From 4189fee3069533c57c21adb640efd79927456e95 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Tue, 25 Aug 2026 17:35:56 +0000 Subject: [PATCH 24/56] fix: detect decorated input SerDes pipelines --- .../testing/CloudDurableTestRunner.java | 3 +-- .../testing/LocalDurableTestRunner.java | 3 +-- .../testing/CloudDurableTestRunnerTest.java | 18 ++++++++++++++++++ .../testing/LocalDurableTestRunnerTest.java | 18 ++++++++++++++++++ .../lambda/durable/serde/ComposableSerDes.java | 5 +++++ .../lambda/durable/serde/RetrySerDes.java | 5 +++++ .../amazon/lambda/durable/serde/SerDes.java | 9 +++++++++ .../lambda/durable/serde/RetrySerDesTest.java | 7 +++++++ 8 files changed, 64 insertions(+), 4 deletions(-) diff --git a/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/CloudDurableTestRunner.java b/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/CloudDurableTestRunner.java index 9d81a2b22..b556b665b 100644 --- a/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/CloudDurableTestRunner.java +++ b/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/CloudDurableTestRunner.java @@ -10,7 +10,6 @@ import software.amazon.awssdk.services.lambda.model.InvocationType; import software.amazon.awssdk.services.lambda.model.InvokeRequest; import software.amazon.lambda.durable.TypeToken; -import software.amazon.lambda.durable.serde.ComposableSerDes; import software.amazon.lambda.durable.serde.JacksonSerDes; import software.amazon.lambda.durable.serde.SerDes; import software.amazon.lambda.durable.serde.SerDesRunner; @@ -286,7 +285,7 @@ private String serializeInput(I input) { "Initial input SerDes requires a durable execution context; configure a context-free " + "input SerDes with withInputSerDes(...)"); } - if (inputSerDes instanceof ComposableSerDes && serDes.requiresDurableContext()) { + if (!serializer.isValueCodecOnly() && serDes.requiresDurableContext()) { throw new IllegalStateException( "Initial input for a context-dependent persisted SerDes must use a value codec, not a composable " + "pipeline"); diff --git a/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/LocalDurableTestRunner.java b/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/LocalDurableTestRunner.java index 18eb4f9c1..51274b0e3 100644 --- a/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/LocalDurableTestRunner.java +++ b/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/LocalDurableTestRunner.java @@ -23,7 +23,6 @@ import software.amazon.lambda.durable.model.DurableExecutionInput; import software.amazon.lambda.durable.model.ExecutionStatus; import software.amazon.lambda.durable.plugin.DurableExecutionPlugin; -import software.amazon.lambda.durable.serde.ComposableSerDes; import software.amazon.lambda.durable.serde.SerDes; import software.amazon.lambda.durable.serde.SerDesRunner; import software.amazon.lambda.durable.testing.local.LocalMemoryExecutionClient; @@ -406,7 +405,7 @@ private String serializeInput(I input) { "Initial input SerDes requires a durable execution context; configure a context-free " + "input SerDes with withInputSerDes(...)"); } - if (inputSerDes instanceof ComposableSerDes && serDes.requiresDurableContext()) { + if (!serializer.isValueCodecOnly() && serDes.requiresDurableContext()) { throw new IllegalStateException( "Initial input for a context-dependent persisted SerDes must use a value codec, not a composable " + "pipeline"); diff --git a/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/CloudDurableTestRunnerTest.java b/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/CloudDurableTestRunnerTest.java index 9aa79ce4c..485d34378 100644 --- a/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/CloudDurableTestRunnerTest.java +++ b/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/CloudDurableTestRunnerTest.java @@ -15,8 +15,10 @@ import software.amazon.awssdk.services.lambda.model.InvokeRequest; import software.amazon.awssdk.services.lambda.model.InvokeResponse; import software.amazon.lambda.durable.TypeToken; +import software.amazon.lambda.durable.retry.RetryStrategies; import software.amazon.lambda.durable.serde.FileSystemSerDes; import software.amazon.lambda.durable.serde.JacksonSerDes; +import software.amazon.lambda.durable.serde.RetrySerDes; import software.amazon.lambda.durable.serde.SerDes; import software.amazon.lambda.durable.serde.SerDesContext; import software.amazon.lambda.durable.serde.SerDesPayloadKind; @@ -124,6 +126,22 @@ void contextDependentPersistedSerDesRequiresValueCodecInput() { verifyNoInteractions(mockClient); } + @Test + void contextDependentPersistedSerDesRejectsRetryWrappedComposableInput() { + var mockClient = mock(LambdaClient.class); + var inputSerDes = new RetrySerDes(new JacksonSerDes().then(wrappingStage()), RetryStrategies.Presets.NO_RETRY); + var runner = CloudDurableTestRunner.create( + "arn:aws:lambda:us-east-2:123:function:test", String.class, String.class, mockClient) + .withSerDes(new JacksonSerDes().then(contextDependentStage())) + .withInputSerDes(inputSerDes); + + var failure = assertThrows(RuntimeException.class, () -> runner.startAsync("value")); + + assertInstanceOf(IllegalStateException.class, failure.getCause()); + assertTrue(failure.getCause().getMessage().contains("must use a value codec")); + verifyNoInteractions(mockClient); + } + @Test void valueCodecInputRoundTripsThroughFileSystemPipeline(@TempDir Path basePath) { var mockClient = mock(LambdaClient.class); diff --git a/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/LocalDurableTestRunnerTest.java b/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/LocalDurableTestRunnerTest.java index 2f0507990..370674018 100644 --- a/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/LocalDurableTestRunnerTest.java +++ b/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/LocalDurableTestRunnerTest.java @@ -19,11 +19,13 @@ import software.amazon.lambda.durable.model.ExecutionStatus; import software.amazon.lambda.durable.plugin.DurableExecutionPlugin; import software.amazon.lambda.durable.plugin.InvocationInfo; +import software.amazon.lambda.durable.retry.RetryStrategies; import software.amazon.lambda.durable.serde.Base64StringBinaryCodec; import software.amazon.lambda.durable.serde.BinarySerDes; import software.amazon.lambda.durable.serde.ComposableBinarySerDesStage; import software.amazon.lambda.durable.serde.FileSystemSerDes; import software.amazon.lambda.durable.serde.JacksonSerDes; +import software.amazon.lambda.durable.serde.RetrySerDes; import software.amazon.lambda.durable.serde.SerDes; import software.amazon.lambda.durable.serde.SerDesStage; import software.amazon.lambda.durable.serde.Utf8StringBinaryCodec; @@ -176,6 +178,22 @@ void contextDependentPersistedSerDesRequiresValueCodecInput(@TempDir Path basePa assertTrue(failure.getMessage().contains("must use a value codec")); } + @Test + void contextDependentPersistedSerDesRejectsRetryWrappedComposableInput(@TempDir Path basePath) { + var config = DurableConfig.builder() + .withSerDes(new JacksonSerDes() + .then(FileSystemSerDes.stageBuilder(basePath).build())) + .build(); + var inputSerDes = new RetrySerDes( + new JacksonSerDes().then(wrappingStage(new AtomicInteger())), RetryStrategies.Presets.NO_RETRY); + var runner = LocalDurableTestRunner.create(String.class, (input, context) -> input, config) + .withInputSerDes(inputSerDes); + + var failure = assertThrows(IllegalStateException.class, () -> runner.run("value")); + + assertTrue(failure.getMessage().contains("must use a value codec")); + } + @Test void initialInputBypassesStagesBeforeFileSystemSerDes(@TempDir Path basePath) { var deserializeCalls = new AtomicInteger(); diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/ComposableSerDes.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/ComposableSerDes.java index 808f925e5..a5b18cf5d 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/ComposableSerDes.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/ComposableSerDes.java @@ -79,6 +79,11 @@ public boolean isTerminalPipelineStage() { return stages.isEmpty() ? valueCodec.isTerminalPipelineStage() : isTerminal(stages.get(stages.size() - 1)); } + @Override + public boolean isValueCodecOnly() { + return stages.isEmpty() && valueCodec.isValueCodecOnly(); + } + /** Returns a new pipeline with the supplied stage appended. */ @Override public ComposableSerDes then(SerDes stage) { diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/RetrySerDes.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/RetrySerDes.java index 2b0017baf..24a08ca7a 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/RetrySerDes.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/RetrySerDes.java @@ -73,6 +73,11 @@ public boolean isTerminalPipelineStage() { return delegate.isTerminalPipelineStage(); } + @Override + public boolean isValueCodecOnly() { + return delegate.isValueCodecOnly(); + } + private T execute(String action, Supplier operation) { int attempt = 1; while (true) { diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDes.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDes.java index a990f3345..4dd6bde5a 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDes.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDes.java @@ -72,6 +72,15 @@ default boolean isTerminalPipelineStage() { return false; } + /** + * Returns whether this SerDes performs only value-codec processing without additional composable pipeline stages. + * + *

SerDes decorators should delegate this capability to their wrapped SerDes. + */ + default boolean isValueCodecOnly() { + return true; + } + /** * Returns an immutable processing pipeline that invokes this SerDes followed by {@code nextStage} when serializing * and in reverse order when deserializing. diff --git a/sdk/src/test/java/software/amazon/lambda/durable/serde/RetrySerDesTest.java b/sdk/src/test/java/software/amazon/lambda/durable/serde/RetrySerDesTest.java index 4c1cf0bb6..0df0e7ccf 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/serde/RetrySerDesTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/serde/RetrySerDesTest.java @@ -3,6 +3,7 @@ package software.amazon.lambda.durable.serde; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -104,6 +105,11 @@ public boolean requiresDurableContext() { public boolean isTerminalPipelineStage() { return true; } + + @Override + public boolean isValueCodecOnly() { + return false; + } }; var retrySerDes = new RetrySerDes(delegate, (error, attempt) -> RetryDecision.retry(Duration.ZERO), delay -> {}); @@ -115,6 +121,7 @@ public boolean isTerminalPipelineStage() { assertEquals(2, calls.get()); assertTrue(retrySerDes.requiresDurableContext()); assertTrue(retrySerDes.isTerminalPipelineStage()); + assertFalse(retrySerDes.isValueCodecOnly()); } @Test From e6d317c2ff61e407fea51c9404d863c9d49a37a8 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Tue, 25 Aug 2026 17:48:53 +0000 Subject: [PATCH 25/56] fix: harden SerDes pipeline boundaries --- .../durable/serde/ComposableSerDes.java | 15 ++++++++++ .../durable/serde/FileSystemSerDes.java | 13 ++++---- .../durable/serde/ComposableSerDesTest.java | 15 ++++++++++ .../durable/serde/FileSystemSerDesTest.java | 30 +++++++++++++++++++ 4 files changed, 66 insertions(+), 7 deletions(-) diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/ComposableSerDes.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/ComposableSerDes.java index a5b18cf5d..7b7a4191a 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/ComposableSerDes.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/ComposableSerDes.java @@ -22,9 +22,18 @@ public final class ComposableSerDes implements SerDes { private ComposableSerDes(SerDes valueCodec, List stages) { this.valueCodec = Objects.requireNonNull(valueCodec, "valueCodec cannot be null"); + if (!stages.isEmpty() && !valueCodec.isValueCodecOnly()) { + throw nestedPipelineFailure(0, valueCodec); + } if (!stages.isEmpty() && valueCodec.isTerminalPipelineStage()) { throw terminalStageFailure(0, valueCodec); } + for (int index = 0; index < stages.size(); index++) { + var stage = stages.get(index); + if (stage instanceof SerDes serDes && !serDes.isValueCodecOnly()) { + throw nestedPipelineFailure(index + 1, stage); + } + } for (int index = 0; index < stages.size() - 1; index++) { if (isTerminal(stages.get(index))) { throw terminalStageFailure(index + 1, stages.get(index)); @@ -198,6 +207,12 @@ private static IllegalArgumentException terminalStageFailure(int index, Object s index, stage.getClass().getName())); } + private static IllegalArgumentException nestedPipelineFailure(int index, Object stage) { + return new IllegalArgumentException(String.format( + "SerDes pipeline stage %d (%s) must be a value codec or a single reversible stage, not a nested pipeline", + index, stage.getClass().getName())); + } + private static boolean requiresContext(Object stage) { return stage instanceof SerDes serDes ? serDes.requiresDurableContext() diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java index 21b61ba08..45028b1da 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java @@ -6,7 +6,6 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import java.io.IOException; -import java.nio.charset.StandardCharsets; import java.nio.file.FileAlreadyExistsException; import java.nio.file.Files; import java.nio.file.LinkOption; @@ -409,7 +408,7 @@ private Map generatePreview(Object value, SerDesContext context) } private static boolean fitsCheckpoint(String envelope) { - return envelope.getBytes(StandardCharsets.UTF_8).length <= CHECKPOINT_ENVELOPE_LIMIT_BYTES; + return Utf8StringBinaryCodec.INSTANCE.toBytes(envelope).length <= CHECKPOINT_ENVELOPE_LIMIT_BYTES; } private SerDesContext requireContext() { @@ -552,7 +551,7 @@ private String encode(String value) { return sha256(value); } var encoded = new StringBuilder(); - for (byte valueByte : value.getBytes(StandardCharsets.UTF_8)) { + for (byte valueByte : Utf8StringBinaryCodec.INSTANCE.toBytes(value)) { int current = valueByte & 0xff; if (current >= 'a' && current <= 'z' || current >= 'A' && current <= 'Z' @@ -572,7 +571,7 @@ private String encode(String value) { } private static String sha256(String value) { - return sha256(value.getBytes(StandardCharsets.UTF_8)); + return sha256(Utf8StringBinaryCodec.INSTANCE.toBytes(value)); } private static String sha256(byte[] value) { @@ -599,7 +598,7 @@ private record SerializedPayload(PayloadType type, byte[] data) { } private static SerializedPayload fromString(String value) { - return new SerializedPayload(PayloadType.STRING, value.getBytes(StandardCharsets.UTF_8)); + return new SerializedPayload(PayloadType.STRING, Utf8StringBinaryCodec.INSTANCE.toBytes(value)); } private static SerializedPayload fromInlineValue(PayloadType type, String value) { @@ -623,14 +622,14 @@ private String inlineValue() { if (data == null) { throw new IllegalStateException("Serialized payload does not contain inline data"); } - return new String(data, StandardCharsets.UTF_8); + return Utf8StringBinaryCodec.INSTANCE.fromBytes(data); } private String value() { if (data == null) { throw new IllegalStateException("Serialized payload does not contain data"); } - return new String(data, StandardCharsets.UTF_8); + return Utf8StringBinaryCodec.INSTANCE.fromBytes(data); } } diff --git a/sdk/src/test/java/software/amazon/lambda/durable/serde/ComposableSerDesTest.java b/sdk/src/test/java/software/amazon/lambda/durable/serde/ComposableSerDesTest.java index fc9208786..acf220a50 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/serde/ComposableSerDesTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/serde/ComposableSerDesTest.java @@ -16,6 +16,7 @@ import software.amazon.lambda.durable.TypeToken; import software.amazon.lambda.durable.exception.RetryableSerDesException; import software.amazon.lambda.durable.exception.SerDesException; +import software.amazon.lambda.durable.retry.RetryStrategies; class ComposableSerDesTest { @@ -197,6 +198,20 @@ public boolean isTerminalPipelineStage() { assertTrue(failure.getMessage().contains("final stage")); } + @Test + void rejectsDecoratedNestedPipelines() { + var nested = new JacksonSerDes().then(stringStage("nested", "<", ">", new ArrayList<>())); + var decorated = new RetrySerDes(nested, RetryStrategies.Presets.NO_RETRY); + + var rootFailure = assertThrows( + IllegalArgumentException.class, + () -> decorated.then(stringStage("outer", "[", "]", new ArrayList<>()))); + var stageFailure = assertThrows(IllegalArgumentException.class, () -> new JacksonSerDes().then(decorated)); + + assertTrue(rootFailure.getMessage().contains("nested pipeline")); + assertTrue(stageFailure.getMessage().contains("nested pipeline")); + } + @Test void rejectsNullIntermediateValues() { var nullStage = new SerDes() { diff --git a/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java b/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java index 4fa212d25..bf94ecde5 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java @@ -10,11 +10,14 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; +import java.security.MessageDigest; import java.util.Arrays; import java.util.Base64; +import java.util.HexFormat; import java.util.Map; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -153,6 +156,25 @@ void rejectsUnexpectedContentInExistingContentAddressedFile() throws Exception { assertEquals("unexpected", Files.readString(file)); } + @Test + void rejectsMalformedUtf8StringsAndFilePayloads() throws Exception { + var serDes = FileSystemSerDes.stageBuilder(basePath).build(); + var runner = new SerDesRunner(null); + + assertThrows(SerDesException.class, () -> runner.serialize(serDes, "lone surrogate \uD800", context())); + + var envelope = runner.serialize(serDes, "valid", context()); + var malformed = new byte[] {(byte) 0xC3, (byte) 0x28}; + var malformedFile = contentAddressedPath(payloadFile(envelope), malformed); + Files.write(malformedFile, malformed); + var malformedEnvelope = (ObjectNode) MAPPER.readTree(envelope); + malformedEnvelope.put("file", malformedFile.toString()); + + assertThrows( + SerDesException.class, + () -> runner.deserialize(serDes, malformedEnvelope.toString(), TypeToken.get(String.class), context())); + } + @Test void hashEncodingUsesFixedLengthSegments() throws Exception { var serDes = FileSystemSerDes.builder(basePath) @@ -462,6 +484,14 @@ private static Path payloadFile(String envelope) { } } + private static Path contentAddressedPath(Path original, byte[] data) throws Exception { + var name = original.getFileName().toString(); + var suffix = ".payload"; + var hashStart = name.length() - suffix.length() - 64; + var hash = HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(data)); + return original.resolveSibling(name.substring(0, hashStart) + hash + suffix); + } + private static SerDesContext context() { return context(1); } From 2d84fc8fff5fe9a41b867a35ef7ceab08aacfe82 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Tue, 25 Aug 2026 18:09:42 +0000 Subject: [PATCH 26/56] refactor: require SerDesStage pipeline components --- docs/adr/005-filesystem-serdes.md | 50 +++++----- docs/advanced/configuration.md | 8 +- docs/advanced/filesystem-serdes.md | 16 ++-- docs/design.md | 2 +- .../FileSystemSerDesIntegrationTest.java | 17 ++-- .../testing/CloudDurableTestRunnerTest.java | 58 ++++++----- .../testing/LocalDurableTestRunnerTest.java | 12 +-- .../durable/serde/ComposableSerDes.java | 95 ++++++------------- .../durable/serde/FileSystemSerDes.java | 16 +++- .../lambda/durable/serde/RetrySerDes.java | 39 +++++++- .../amazon/lambda/durable/serde/SerDes.java | 42 +------- .../durable/serde/ComposableSerDesTest.java | 95 ++++++++++--------- .../durable/serde/FileSystemSerDesTest.java | 27 ++++-- .../lambda/durable/serde/RetrySerDesTest.java | 15 ++- 14 files changed, 249 insertions(+), 243 deletions(-) diff --git a/docs/adr/005-filesystem-serdes.md b/docs/adr/005-filesystem-serdes.md index d05ceda05..42578d577 100644 --- a/docs/adr/005-filesystem-serdes.md +++ b/docs/adr/005-filesystem-serdes.md @@ -2,7 +2,8 @@ **Status:** Accepted — Approach A with ComposableSerDes pipeline **Date:** 2026-07-02 -**Updated:** 2026-08-25 — Included FileSystemSerDes in core and added string and nested binary pipelines. +**Updated:** 2026-08-25 — Included FileSystemSerDes in core and made every post-codec pipeline component an explicit +string stage. ## Context @@ -66,10 +67,6 @@ public interface SerDes { T deserialize(String data, TypeToken typeToken); - default ComposableSerDes then(SerDes nextStage) { - return ComposableSerDes.of(this, nextStage); - } - default ComposableSerDes then(SerDesStage nextStage) { return ComposableSerDes.builder(this).then(nextStage).build(); } @@ -147,17 +144,15 @@ the existing `serialize` and `deserialize` methods: ```java public final class ComposableSerDes implements SerDes { - public static ComposableSerDes of(SerDes first, SerDes... remaining); + public static ComposableSerDes of(SerDes valueCodec, SerDesStage... stages); public static Builder builder(SerDes valueCodec); public SerDes getValueCodec(); - public ComposableSerDes then(SerDes stage); public ComposableSerDes then(SerDesStage stage); public static final class Builder { - public Builder then(SerDes stage); public Builder then(SerDesStage stage); public ComposableSerDes build(); @@ -228,10 +223,17 @@ Pipeline rules: - A pipeline must contain exactly one value codec in the first position and zero or more string stages. - Every top-level stage consumes and produces a non-null `String`. This makes all stage orderings type-compatible. -- A stage may declare that it requires durable context or that it must be terminal. A terminal stage must be the last - stage so later transformations cannot invalidate its checkpoint-size or storage decision. -- `SerDes.then(...)`, `ComposableSerDes.of(...)`, and the builder flatten nested `ComposableSerDes` instances while - preserving stage order. +- `SerDes.requiresDurableContext()` remains a root capability because the SDK and test runners must validate the + configured SerDes before an initial durable context exists. `ComposableSerDes` aggregates this capability from its + value codec and stages, and decorators delegate it. +- `SerDes.isValueCodecOnly()` remains a root/decorator capability so input validation and nested-pipeline validation + can identify a `ComposableSerDes` even when it is wrapped by `RetrySerDes`. +- Terminality belongs only to `SerDesStage`. A terminal stage must be last so later transformations cannot invalidate + its checkpoint-size or storage decision. +- `SerDes.then(...)`, `ComposableSerDes.then(...)`, and the builder accept only `SerDesStage` after the value codec. + This makes the root `SerDes` versus subsequent string-stage roles explicit in the Java type system. +- If the value-codec argument to `ComposableSerDes.of(...)` or the builder is already a `ComposableSerDes`, its root + codec and string stages are flattened while preserving stage order. - A `null` value at the pipeline boundary short-circuits the entire pipeline: serializing or deserializing `null` returns `null` without invoking any stage. A stage returning `null` for non-null input is an error. The value codec may decode a non-null representation such as the JSON literal `null` to a null domain value. @@ -259,8 +261,8 @@ Pipeline rules: - Invocation-scoped caching wraps the complete pipeline. Cache keys use the final checkpoint string and target type, so cache hits skip every reverse-processing stage, including filesystem reads. -The default `SerDes.then(...)` method and immutable `ComposableSerDes.then(...)` method provide a concise form for -independently reusable processing chains: +The default `SerDes.then(SerDesStage)` method and immutable `ComposableSerDes.then(SerDesStage)` method provide a +concise form for independently reusable processing chains: ```java var binaryStage = ComposableBinarySerDesStage.builder() @@ -329,7 +331,8 @@ conversion only at its two outer boundaries; binary stages pass bytes directly t ### Retryable SerDes stages Transient failures are explicit. `RetryableSerDesException` extends `SerDesException` and marks a failure that may -succeed when attempted again. `RetrySerDes` decorates another SerDes instance and applies an existing `RetryStrategy`: +succeed when attempted again. `RetrySerDes` implements both `SerDes` and `SerDesStage`, decorates another SerDes +instance, and applies an existing `RetryStrategy`: ```java var resilientFileSystemStage = new RetrySerDes( @@ -356,8 +359,9 @@ Retry rules: checkpoint. Strategies must therefore use short, bounded delays that fit within the Lambda invocation timeout. - If the invocation is interrupted or times out, replay may execute the SerDes pipeline again. Any stage with side effects must use stable addressing and idempotent writes. -- `RetrySerDes` can wrap an individual SerDes stage or the complete pipeline. Wrapping the smallest transient SerDes - avoids repeating deterministic encoding, compression, or encryption work. +- `RetrySerDes` can wrap an individual component that implements both `SerDes` and `SerDesStage`, such as + `FileSystemSerDes`, or the complete pipeline. Wrapping the smallest transient component avoids repeating + deterministic encoding, compression, or encryption work. - Pipeline error decoration must preserve retryability: a stage-level `RetryableSerDesException` must remain that type, with stage metadata added to its message or cause, so an enclosing `RetrySerDes` can recognize it. - Filesystem read and write `IOException`s are retryable. Malformed envelopes, invalid paths, unsupported types, and @@ -548,9 +552,9 @@ must not synthesize a durable context and serialize initial input through the fu 1. Add `SerDesContext`, `SerDesPayloadKind`, and package-private TLS setter/clearer support. Leave the existing `SerDes` methods unchanged. -2. Add the binary-compatible `SerDes.then(...)` default method and `ComposableSerDes` with immutable stage ordering, - forward serialization, reverse deserialization, external-boundary bypass, terminal-stage validation, null - short-circuiting, and stage-aware errors. +2. Add the binary-compatible `SerDes.then(SerDesStage)` default method and `ComposableSerDes` with one root `SerDes` + followed only by immutable `SerDesStage` entries, forward serialization, reverse deserialization, + external-boundary bypass, terminal-stage validation, null short-circuiting, and stage-aware errors. 3. Add the string-only `SerDesStage` contract plus `BinarySerDes`, `StringBinaryCodec`, and `ComposableBinarySerDesStage` for nested binary processing with ordered, configurable boundaries. 4. Add `RetryableSerDesException` and `RetrySerDes`, reusing `RetryStrategy` for bounded in-invocation retries. @@ -790,8 +794,8 @@ This approach gives the SDK one consistent policy for root payloads, operation r | Responsibility boundary | Combines value serialization and storage-reference creation in ordered, composable SerDes stages. | Keeps object encoding in `SerDes` and storage movement in a separate offloader. | | User configuration | Users configure one SerDes or a `ComposableSerDes` pipeline. Operation-level SerDes selection replaces the whole pipeline. | Users configure both a SerDes and an offloader. The SDK must define global, per-operation, and per-payload precedence. | | Parity with JS issue | Closest to the current JavaScript `createFileSystemSerdes` model and issue #463 wording. | Diverges from JavaScript naming and shape, though it may be architecturally cleaner for Java. | -| Core SDK changes | Adds the compatible `SerDes.then(...)` default method, `ComposableSerDes`, and `SerDesContext` TLS because the existing serialization methods have no context parameter. | Requires a new core extension point, envelope type, config surface, payload pipeline, and migration story. | -| Applicability | Any compatible SerDes stages can be chained, but storage stages still own their envelope and lifecycle behavior. | Any SerDes output can be offloaded uniformly after serialization. Users can combine Jackson/custom SerDes with any offloader. | +| Core SDK changes | Adds the compatible `SerDes.then(SerDesStage)` default method, `ComposableSerDes`, and `SerDesContext` TLS because the existing serialization methods have no context parameter. | Requires a new core extension point, envelope type, config surface, payload pipeline, and migration story. | +| Applicability | Any compatible `SerDesStage` implementations can be chained, but storage stages still own their envelope and lifecycle behavior. | Any SerDes output can be offloaded uniformly after serialization. Users can combine Jackson/custom SerDes with any offloader. | | Envelope ownership | FileSystemSerDes owns the checkpoint envelope (`data`, `file`, preview), so the SDK treats it as opaque serialized data. | SDK owns the checkpoint/offload envelope and must guarantee it composes with replay, errors, callbacks, and test utilities. | | Caching | SDK can cache deserialized values, but FileSystemSerDes may still do file reads internally unless cache hits happen before SerDes. | SDK can cache at both layers: resolved offloaded payload text and final deserialized object. | | Exception handling | Works if every exception serialization path is routed through SerDes with `SerDesPayloadKind.EXCEPTION`. | Works uniformly because exception `errorData` is another serialized payload that can be offloaded after SerDes. | @@ -859,7 +863,7 @@ Positive: signatures. - Approach A makes filesystem-backed storage available from the core SDK without an additional artifact. - Custom payload implementations get enough context to use external storage safely. -- Customers can compose reusable SerDes stages without creating a bespoke wrapper for each combination. +- Customers can compose reusable `SerDesStage` implementations without creating a bespoke wrapper for each combination. - Blocking payload work can be isolated from user operation threads with an explicitly configured SerDes executor and never runs on the SDK coordination executor. - Repeated file reads and repeated object reconstruction can be reduced within an invocation. diff --git a/docs/advanced/configuration.md b/docs/advanced/configuration.md index 9685ca09e..940d4026e 100644 --- a/docs/advanced/configuration.md +++ b/docs/advanced/configuration.md @@ -95,10 +95,10 @@ approaches the 256 KB service limit. `URI` produces readable escaped paths; `HAS segments. Files are content-addressed and never overwrite data referenced by an earlier checkpoint. References are validated against the current durable execution and entity, and symbolic-link paths are rejected. -`RetrySerDes` retries only failures marked with `RetryableSerDesException`. Filesystem read and write I/O use this -marker; malformed envelopes and codec failures fail immediately. Backoff occurs within the current Lambda invocation, -so use short, bounded retry strategies. Without a configured SerDes executor, filesystem I/O and retry delays block the -calling thread. +`RetrySerDes` implements `SerDesStage` and retries only failures marked with `RetryableSerDesException`. Filesystem +read and write I/O use this marker; malformed envelopes and codec failures fail immediately. Backoff occurs within the +current Lambda invocation, so use short, bounded retry strategies. Without a configured SerDes executor, filesystem +I/O and retry delays block the calling thread. Do not use Lambda's ephemeral `/tmp` directory: replay can run in a different execution environment. Use a durable, shared mount such as EFS. S3 Files can have delayed synchronization, so a runtime crash before the mount flushes may diff --git a/docs/advanced/filesystem-serdes.md b/docs/advanced/filesystem-serdes.md index 89dc89d27..836344c3a 100644 --- a/docs/advanced/filesystem-serdes.md +++ b/docs/advanced/filesystem-serdes.md @@ -15,7 +15,7 @@ envelopes in checkpoints. It is included in the core `aws-durable-execution-sdk- ## Pipeline configuration -The preferred configuration uses `FileSystemSerDes` as a reversible terminal stage after a value codec: +The preferred configuration uses `FileSystemSerDes` as a reversible terminal `SerDesStage` after a value codec: ```java var fileSystemStage = FileSystemSerDes.stageBuilder(Path.of("/mnt/efs/durable-payloads")) @@ -46,11 +46,12 @@ return DurableConfig.builder() .build(); ``` -Serialization follows the declaration order above and deserialization runs in reverse. Every top-level stage consumes -and produces a string. `ComposableBinarySerDesStage` converts the string with its starting codec, passes bytes directly -through each `BinarySerDes`, then converts the final bytes to a string with its ending codec. Both boundaries are -customizable through the same `StringBinaryCodec` interface; the example performs UTF-8 and Base64 conversion once -around the complete compression/encryption chain. +Serialization follows the declaration order above and deserialization runs in reverse. The first component is the +`SerDes` value codec; every component appended with `then(...)` implements `SerDesStage` and consumes and produces a +string. `ComposableBinarySerDesStage` converts the string with its starting codec, passes bytes directly through each +`BinarySerDes`, then converts the final bytes to a string with its ending codec. Both boundaries are customizable +through the same `StringBinaryCodec` interface; the example performs UTF-8 and Base64 conversion once around the +complete compression/encryption chain. - `ALWAYS` writes every non-null payload to a file. - `OVERFLOW` stores small payloads inline and offloads envelopes approaching the 256 KB checkpoint limit. @@ -69,7 +70,8 @@ SerDes runs inline by default. Filesystem access and retry backoff are blocking, provide a dedicated executor with `withSerDesExecutorService(...)`. It must be different from the user-operation executor. -Filesystem read and write I/O failures are reported as `RetryableSerDesException`. `RetrySerDes` retries only that +Filesystem read and write I/O failures are reported as `RetryableSerDesException`. `RetrySerDes` implements +`SerDesStage`, so the retrying filesystem component can be appended directly to the pipeline. It retries only that exception type. Malformed envelopes, invalid paths, unsupported stage types, and codec failures are permanent. Retry delays consume time in the current Lambda invocation, so keep attempts and delays bounded. diff --git a/docs/design.md b/docs/design.md index 200339fd8..7e633ef63 100644 --- a/docs/design.md +++ b/docs/design.md @@ -363,7 +363,7 @@ software.amazon.lambda.durable │ ├── Base64StringBinaryCodec # Standard Base64 string/byte conversion │ ├── ComposableBinarySerDesStage # Ordered binary chain exposed as one string stage │ ├── JacksonSerDes # Jackson impl -│ ├── RetrySerDes # Retry decorator for transient SerDes failures +│ ├── RetrySerDes # Retrying SerDes and string-stage decorator │ ├── SerDesRunner # Context, optional executor dispatch, and invocation cache │ ├── SerDesContext # Read-only durable payload identity │ ├── SerDesPayloadKind # Input/result/state/exception/invoke payload kind diff --git a/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FileSystemSerDesIntegrationTest.java b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FileSystemSerDesIntegrationTest.java index 701b34dbf..312282277 100644 --- a/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FileSystemSerDesIntegrationTest.java +++ b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FileSystemSerDesIntegrationTest.java @@ -44,6 +44,7 @@ import software.amazon.lambda.durable.serde.SerDesContext; import software.amazon.lambda.durable.serde.SerDesPayloadKind; import software.amazon.lambda.durable.serde.SerDesRunner; +import software.amazon.lambda.durable.serde.SerDesStage; import software.amazon.lambda.durable.serde.Utf8StringBinaryCodec; import software.amazon.lambda.durable.testing.LocalDurableTestRunner; import software.amazon.lambda.durable.testing.TestResult; @@ -505,20 +506,18 @@ private static Operation executionOperation(String id, String name, String input .build(); } - private static SerDes identityStage(RecordingFunction recorder) { - return new SerDes() { + private static SerDesStage identityStage(RecordingFunction recorder) { + return new SerDesStage() { @Override - public String serialize(Object value) { - var stringValue = (String) value; - recorder.record("serialize", stringValue); - return stringValue; + public String serialize(String value) { + recorder.record("serialize", value); + return value; } @Override - @SuppressWarnings("unchecked") - public T deserialize(String data, TypeToken typeToken) { + public String deserialize(String data) { recorder.record("deserialize", data); - return (T) data; + return data; } }; } diff --git a/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/CloudDurableTestRunnerTest.java b/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/CloudDurableTestRunnerTest.java index 485d34378..b7857bb05 100644 --- a/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/CloudDurableTestRunnerTest.java +++ b/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/CloudDurableTestRunnerTest.java @@ -23,6 +23,7 @@ import software.amazon.lambda.durable.serde.SerDesContext; import software.amazon.lambda.durable.serde.SerDesPayloadKind; import software.amazon.lambda.durable.serde.SerDesRunner; +import software.amazon.lambda.durable.serde.SerDesStage; class CloudDurableTestRunnerTest { @@ -190,38 +191,49 @@ void replacingPersistedSerDesPreservesExplicitInputSerDes() { assertEquals("<\"value\">", request.getValue().payload().asUtf8String()); } - private static SerDes wrappingStage() { - return new SerDes() { + private static SerDesStage wrappingStage() { + return new SerDesStage() { @Override - public String serialize(Object value) { + public String serialize(String value) { return "<" + value + ">"; } @Override - @SuppressWarnings("unchecked") - public T deserialize(String data, TypeToken typeToken) { - return (T) data.substring(1, data.length() - 1); + public String deserialize(String data) { + return data.substring(1, data.length() - 1); } }; } - private static SerDes contextDependentStage() { - return new SerDes() { - @Override - public String serialize(Object value) { - return value.toString(); - } - - @Override - @SuppressWarnings("unchecked") - public T deserialize(String data, TypeToken typeToken) { - return (T) data; - } + private static ContextDependentSerDesStage contextDependentStage() { + return new ContextDependentSerDesStage(); + } - @Override - public boolean requiresDurableContext() { - return true; - } - }; + private static final class ContextDependentSerDesStage implements SerDes, SerDesStage { + @Override + public String serialize(Object value) { + return value.toString(); + } + + @Override + public String serialize(String value) { + return value; + } + + @Override + @SuppressWarnings("unchecked") + public T deserialize(String data, TypeToken typeToken) { + return (T) data; + } + + @Override + public String deserialize(String data) { + return data; + } + + @Override + public boolean requiresDurableContext() { + return true; + } } } diff --git a/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/LocalDurableTestRunnerTest.java b/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/LocalDurableTestRunnerTest.java index 370674018..da310ed69 100644 --- a/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/LocalDurableTestRunnerTest.java +++ b/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/LocalDurableTestRunnerTest.java @@ -26,7 +26,6 @@ import software.amazon.lambda.durable.serde.FileSystemSerDes; import software.amazon.lambda.durable.serde.JacksonSerDes; import software.amazon.lambda.durable.serde.RetrySerDes; -import software.amazon.lambda.durable.serde.SerDes; import software.amazon.lambda.durable.serde.SerDesStage; import software.amazon.lambda.durable.serde.Utf8StringBinaryCodec; @@ -212,18 +211,17 @@ void initialInputBypassesStagesBeforeFileSystemSerDes(@TempDir Path basePath) { assertEquals("value:0", result.getResult()); } - private static SerDes wrappingStage(AtomicInteger deserializeCalls) { - return new SerDes() { + private static SerDesStage wrappingStage(AtomicInteger deserializeCalls) { + return new SerDesStage() { @Override - public String serialize(Object value) { + public String serialize(String value) { return "<" + value + ">"; } @Override - @SuppressWarnings("unchecked") - public T deserialize(String data, TypeToken typeToken) { + public String deserialize(String data) { deserializeCalls.incrementAndGet(); - return (T) data.substring(1, data.length() - 1); + return data.substring(1, data.length() - 1); } }; } diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/ComposableSerDes.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/ComposableSerDes.java index 7b7a4191a..82ed763d7 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/ComposableSerDes.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/ComposableSerDes.java @@ -3,7 +3,6 @@ package software.amazon.lambda.durable.serde; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import java.util.Objects; import software.amazon.lambda.durable.TypeToken; @@ -13,19 +12,21 @@ /** * An immutable SerDes processing pipeline. * - *

The first stage is the value codec. Every later stage consumes and produces a string. Serialization runs from - * first to last; deserialization runs from last to first. + *

The first component is the value codec. Every later component is a {@link SerDesStage} that consumes and produces + * a string. Serialization runs from first to last; deserialization runs from last to first. */ public final class ComposableSerDes implements SerDes { private final SerDes valueCodec; - private final List stages; + private final List stages; - private ComposableSerDes(SerDes valueCodec, List stages) { + private ComposableSerDes(SerDes valueCodec, List stages) { this.valueCodec = Objects.requireNonNull(valueCodec, "valueCodec cannot be null"); if (!stages.isEmpty() && !valueCodec.isValueCodecOnly()) { throw nestedPipelineFailure(0, valueCodec); } - if (!stages.isEmpty() && valueCodec.isTerminalPipelineStage()) { + if (!stages.isEmpty() + && valueCodec instanceof SerDesStage valueCodecStage + && valueCodecStage.isTerminalPipelineStage()) { throw terminalStageFailure(0, valueCodec); } for (int index = 0; index < stages.size(); index++) { @@ -45,28 +46,28 @@ private ComposableSerDes(SerDes valueCodec, List stages) { /** * Creates a pipeline with a value codec followed by zero or more string stages. * - * @param first the value codec - * @param remaining reversible SerDes stages + * @param valueCodec the value codec + * @param remaining reversible string stages * @return an immutable pipeline */ - public static ComposableSerDes of(SerDes first, SerDes... remaining) { + public static ComposableSerDes of(SerDes valueCodec, SerDesStage... remaining) { Objects.requireNonNull(remaining, "remaining stages cannot be null"); - var valueCodec = Objects.requireNonNull(first, "first stage cannot be null"); - var stages = new ArrayList(); + valueCodec = Objects.requireNonNull(valueCodec, "valueCodec cannot be null"); + var stages = new ArrayList(); if (valueCodec instanceof ComposableSerDes composable) { valueCodec = composable.valueCodec; stages.addAll(composable.stages); } - Arrays.stream(remaining) - .map(stage -> Objects.requireNonNull(stage, "pipeline stage cannot be null")) - .forEach(stage -> addFlattened(stages, stage)); + for (var stage : remaining) { + stages.add(Objects.requireNonNull(stage, "pipeline stage cannot be null")); + } return new ComposableSerDes(valueCodec, stages); } /** * Creates a pipeline builder. * - * @param valueCodec the first stage which converts values to and from strings + * @param valueCodec the value codec which converts values to and from strings * @return a new builder */ public static Builder builder(SerDes valueCodec) { @@ -83,29 +84,16 @@ public boolean requiresDurableContext() { return valueCodec.requiresDurableContext() || stages.stream().anyMatch(ComposableSerDes::requiresContext); } - @Override - public boolean isTerminalPipelineStage() { - return stages.isEmpty() ? valueCodec.isTerminalPipelineStage() : isTerminal(stages.get(stages.size() - 1)); - } - @Override public boolean isValueCodecOnly() { return stages.isEmpty() && valueCodec.isValueCodecOnly(); } - /** Returns a new pipeline with the supplied stage appended. */ - @Override - public ComposableSerDes then(SerDes stage) { - var combined = new ArrayList<>(stages); - addFlattened(combined, Objects.requireNonNull(stage, "stage cannot be null")); - return new ComposableSerDes(valueCodec, combined); - } - /** Returns a new pipeline with the supplied string stage appended. */ @Override public ComposableSerDes then(SerDesStage stage) { var combined = new ArrayList<>(stages); - addFlattened(combined, Objects.requireNonNull(stage, "stage cannot be null")); + combined.add(Objects.requireNonNull(stage, "stage cannot be null")); return new ComposableSerDes(valueCodec, combined); } @@ -138,10 +126,9 @@ public T deserialize(String data, TypeToken typeToken) { return invokeValueCodecDeserialize(valueCodec, current, typeToken); } - private static String invokeStageSerialize(Object stage, String value, int index) { + private static String invokeStageSerialize(SerDesStage stage, String value, int index) { try { - var result = - stage instanceof SerDes serDes ? serDes.serialize(value) : ((SerDesStage) stage).serialize(value); + var result = stage.serialize(value); if (result == null) { throw new SerDesException("Stage returned null for a non-null value"); } @@ -151,14 +138,9 @@ private static String invokeStageSerialize(Object stage, String value, int index } } - private static SerDesStageResult invokeStageDeserialize(Object stage, String data, int index) { + private static SerDesStageResult invokeStageDeserialize(SerDesStage stage, String data, int index) { try { - SerDesStageResult result; - if (stage instanceof SerDes serDes) { - result = serDes.deserializePipelineStage(data); - } else { - result = ((SerDesStage) stage).deserializePipelineStage(data); - } + var result = stage.deserializePipelineStage(data); if (result == null) { throw new SerDesException("Stage returned a null pipeline result"); } @@ -213,35 +195,18 @@ private static IllegalArgumentException nestedPipelineFailure(int index, Object index, stage.getClass().getName())); } - private static boolean requiresContext(Object stage) { - return stage instanceof SerDes serDes - ? serDes.requiresDurableContext() - : ((SerDesStage) stage).requiresDurableContext(); + private static boolean requiresContext(SerDesStage stage) { + return stage.requiresDurableContext(); } - private static boolean isTerminal(Object stage) { - return stage instanceof SerDes serDes - ? serDes.isTerminalPipelineStage() - : ((SerDesStage) stage).isTerminalPipelineStage(); - } - - private static void addFlattened(List target, SerDes stage) { - if (stage instanceof ComposableSerDes composable) { - target.add(composable.valueCodec); - target.addAll(composable.stages); - } else { - target.add(stage); - } - } - - private static void addFlattened(List target, SerDesStage stage) { - target.add(stage); + private static boolean isTerminal(SerDesStage stage) { + return stage.isTerminalPipelineStage(); } /** Builder for an immutable {@link ComposableSerDes}. */ public static final class Builder { private SerDes valueCodec; - private final List stages = new ArrayList<>(); + private final List stages = new ArrayList<>(); private Builder(SerDes valueCodec) { this.valueCodec = Objects.requireNonNull(valueCodec, "valueCodec cannot be null"); @@ -251,15 +216,9 @@ private Builder(SerDes valueCodec) { } } - /** Appends a reversible SerDes stage. */ - public Builder then(SerDes stage) { - addFlattened(stages, Objects.requireNonNull(stage, "stage cannot be null")); - return this; - } - /** Appends a reversible string stage. */ public Builder then(SerDesStage stage) { - addFlattened(stages, Objects.requireNonNull(stage, "stage cannot be null")); + stages.add(Objects.requireNonNull(stage, "stage cannot be null")); return this; } diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java index 45028b1da..d379f12c8 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java @@ -26,7 +26,7 @@ import software.amazon.lambda.durable.exception.SerDesException; /** - * A SerDes that stores payloads on a durable shared filesystem. + * A SerDes and terminal string stage that stores payloads on a durable shared filesystem. * *

Use {@link #stageBuilder(Path)} when composing this implementation after a value codec. The compatibility * {@link #builder(Path)} form includes its own value codec and can be used as a standalone SerDes. @@ -34,7 +34,7 @@ *

Do not use Lambda's ephemeral {@code /tmp} storage. Use a durable shared mount such as EFS, or S3 Files only when * its synchronization and crash-durability tradeoffs are acceptable for the workload. */ -public final class FileSystemSerDes implements SerDes { +public final class FileSystemSerDes implements SerDes, SerDesStage { private static final String ENVELOPE_MARKER = "__durable_execution_filesystem_serdes"; private static final String PAYLOAD_TYPE_FIELD = "payloadType"; private static final int ENVELOPE_VERSION = 1; @@ -114,6 +114,11 @@ public String serialize(Object value) { } } + @Override + public String serialize(String value) { + return serialize((Object) value); + } + @Override public T deserialize(String data, TypeToken typeToken) { if (data == null) { @@ -133,10 +138,15 @@ public T deserialize(String data, TypeToken typeToken) { return delegate.deserialize(serialized, typeToken); } + @Override + public String deserialize(String data) { + return deserialize(data, TypeToken.get(String.class)); + } + @Override public SerDesStageResult deserializePipelineStage(String data) { if (!stageMode) { - return SerDes.super.deserializePipelineStage(data); + return SerDesStage.super.deserializePipelineStage(data); } var context = requireContext(); var resolved = resolveSerializedPayload(data, context); diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/RetrySerDes.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/RetrySerDes.java index 24a08ca7a..f0a7e60a7 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/RetrySerDes.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/RetrySerDes.java @@ -13,12 +13,16 @@ import software.amazon.lambda.durable.retry.RetryStrategy; /** - * A SerDes decorator that retries transient failures from another {@link SerDes}. + * A SerDes and string-stage decorator that retries transient failures from another {@link SerDes}. * *

Only {@link RetryableSerDesException} is retried. Other failures are propagated immediately. Retry delays block * the thread executing the SerDes call: the caller by default or the configured SerDes executor thread. + * + *

When the delegate also implements {@link SerDesStage}, this decorator uses its explicit string-stage behavior when + * appended to a {@link ComposableSerDes}. Otherwise, the delegate is invoked with {@link String} values and a + * {@code TypeToken}. */ -public final class RetrySerDes implements SerDes { +public final class RetrySerDes implements SerDes, SerDesStage { private static final Sleeper DEFAULT_SLEEPER = delay -> { if (delay.getSeconds() > 0) { TimeUnit.SECONDS.sleep(delay.getSeconds()); @@ -29,6 +33,7 @@ public final class RetrySerDes implements SerDes { }; private final SerDes delegate; + private final SerDesStage stageDelegate; private final RetryStrategy retryStrategy; private final Sleeper sleeper; @@ -44,6 +49,7 @@ public RetrySerDes(SerDes delegate, RetryStrategy retryStrategy) { RetrySerDes(SerDes delegate, RetryStrategy retryStrategy, Sleeper sleeper) { this.delegate = Objects.requireNonNull(delegate, "delegate cannot be null"); + this.stageDelegate = delegate instanceof SerDesStage stage ? stage : null; this.retryStrategy = Objects.requireNonNull(retryStrategy, "retryStrategy cannot be null"); this.sleeper = Objects.requireNonNull(sleeper, "sleeper cannot be null"); } @@ -53,14 +59,39 @@ public String serialize(Object value) { return execute("serialization", () -> delegate.serialize(value)); } + @Override + public String serialize(String value) { + return execute( + "pipeline stage serialization", + () -> stageDelegate == null ? delegate.serialize(value) : stageDelegate.serialize(value)); + } + @Override public T deserialize(String data, TypeToken typeToken) { return execute("deserialization", () -> delegate.deserialize(data, typeToken)); } + @Override + public String deserialize(String data) { + return execute( + "pipeline stage deserialization", + () -> stageDelegate == null + ? delegate.deserialize(data, TypeToken.get(String.class)) + : stageDelegate.deserialize(data)); + } + @Override public SerDesStageResult deserializePipelineStage(String data) { - return execute("pipeline stage deserialization", () -> delegate.deserializePipelineStage(data)); + return execute("pipeline stage deserialization", () -> { + if (stageDelegate != null) { + return stageDelegate.deserializePipelineStage(data); + } + var result = delegate.deserialize(data, TypeToken.get(String.class)); + if (result == null) { + throw new SerDesException("Stage returned null for non-null data"); + } + return SerDesStageResult.continueWith(result); + }); } @Override @@ -70,7 +101,7 @@ public boolean requiresDurableContext() { @Override public boolean isTerminalPipelineStage() { - return delegate.isTerminalPipelineStage(); + return stageDelegate != null && stageDelegate.isTerminalPipelineStage(); } @Override diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDes.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDes.java index 4dd6bde5a..958c60078 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDes.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDes.java @@ -3,13 +3,12 @@ package software.amazon.lambda.durable.serde; import software.amazon.lambda.durable.TypeToken; -import software.amazon.lambda.durable.exception.SerDesException; /** * Interface for serialization and deserialization of objects at the persisted string boundary. * - *

Implementations can also be used as string-producing stages in a {@link ComposableSerDes}. Use {@link SerDesStage} - * for transformations that consume and produce strings. + *

A {@link ComposableSerDes} starts with one SerDes value codec and may be followed by {@link SerDesStage} + * transformations that consume and produce strings. */ public interface SerDes { /** @@ -39,19 +38,6 @@ public interface SerDes { */ T deserialize(String data, TypeToken typeToken); - /** - * Deserializes this SerDes when it is used as an intermediate pipeline stage. - * - *

The default uses {@link String} as the intermediate value type, preserving existing SerDes behavior. - */ - default SerDesStageResult deserializePipelineStage(String data) { - var result = deserialize(data, TypeToken.get(String.class)); - if (result == null) { - throw new SerDesException("Stage returned null for non-null data"); - } - return SerDesStageResult.continueWith(result); - } - /** * Returns whether this SerDes requires an SDK-managed durable execution context. * @@ -62,16 +48,6 @@ default boolean requiresDurableContext() { return false; } - /** - * Returns whether this SerDes must be the final stage in a composable pipeline. - * - *

Stages that make size-based storage decisions should normally be terminal so later transformations cannot - * expand their output beyond checkpoint limits. - */ - default boolean isTerminalPipelineStage() { - return false; - } - /** * Returns whether this SerDes performs only value-codec processing without additional composable pipeline stages. * @@ -81,20 +57,6 @@ default boolean isValueCodecOnly() { return true; } - /** - * Returns an immutable processing pipeline that invokes this SerDes followed by {@code nextStage} when serializing - * and in reverse order when deserializing. - * - *

This SerDes is the value codec. Intermediate stages may transform values into arbitrary Java types, but the - * final stage must return a string for persistence. - * - * @param nextStage the reversible stage to append - * @return a composable SerDes pipeline - */ - default ComposableSerDes then(SerDes nextStage) { - return ComposableSerDes.of(this, nextStage); - } - /** * Returns an immutable processing pipeline with a string stage appended. * diff --git a/sdk/src/test/java/software/amazon/lambda/durable/serde/ComposableSerDesTest.java b/sdk/src/test/java/software/amazon/lambda/durable/serde/ComposableSerDesTest.java index acf220a50..3129d9010 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/serde/ComposableSerDesTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/serde/ComposableSerDesTest.java @@ -20,6 +20,14 @@ class ComposableSerDesTest { + @Test + void onlyAcceptsStringStagesAfterTheValueCodec() throws Exception { + assertThrows(NoSuchMethodException.class, () -> SerDes.class.getMethod("then", SerDes.class)); + assertEquals( + ComposableSerDes.class, + SerDes.class.getMethod("then", SerDesStage.class).getReturnType()); + } + @Test void serializesForwardAndDeserializesInReverse() { var calls = new ArrayList(); @@ -75,15 +83,15 @@ public String deserialize(String data) { } @Test - void factoryBuilderAndThenFlattenNestedPipelines() { + void factoryBuilderAndThenFlattenRootPipeline() { var calls = new ArrayList(); - var nested = ComposableSerDes.builder(stringStage("codec", "", "", calls)) + var nested = ComposableSerDes.builder(new JacksonSerDes()) .then(stringStage("one", "1", "1", calls)) .build(); var pipeline = ComposableSerDes.of(nested).then(stringStage("two", "2", "2", calls)); - assertEquals("21value12", pipeline.serialize("value")); - assertEquals(List.of("codec-serialize", "one-serialize", "two-serialize"), calls); + assertEquals("21\"value\"12", pipeline.serialize("value")); + assertEquals(List.of("one-serialize", "two-serialize"), calls); } @Test @@ -112,17 +120,16 @@ public T deserialize(String data, TypeToken typeToken) { @Test void valueCodecMayDecodeNonNullRepresentationToNull() { var intermediateCalls = new AtomicInteger(); - var identityStage = new SerDes() { + var identityStage = new SerDesStage() { @Override - public String serialize(Object value) { - return value.toString(); + public String serialize(String value) { + return value; } @Override - @SuppressWarnings("unchecked") - public T deserialize(String data, TypeToken typeToken) { + public String deserialize(String data) { intermediateCalls.incrementAndGet(); - return (T) data; + return data; } }; var pipeline = new JacksonSerDes().then(identityStage); @@ -134,29 +141,27 @@ public T deserialize(String data, TypeToken typeToken) { @Test void stageMayDecodeExternalDataDirectlyWithValueCodec() { var transformDeserializations = new AtomicInteger(); - var transform = new SerDes() { + var transform = new SerDesStage() { @Override - public String serialize(Object value) { + public String serialize(String value) { return "<" + value + ">"; } @Override - @SuppressWarnings("unchecked") - public T deserialize(String data, TypeToken typeToken) { + public String deserialize(String data) { transformDeserializations.incrementAndGet(); - return (T) data.substring(1, data.length() - 1); + return data.substring(1, data.length() - 1); } }; - var externalBoundary = new SerDes() { + var externalBoundary = new SerDesStage() { @Override - public String serialize(Object value) { - return value.toString(); + public String serialize(String value) { + return value; } @Override - @SuppressWarnings("unchecked") - public T deserialize(String data, TypeToken typeToken) { - return (T) data; + public String deserialize(String data) { + return data; } @Override @@ -172,16 +177,15 @@ public SerDesStageResult deserializePipelineStage(String data) { @Test void rejectsStagesAfterTerminalStage() { - var terminal = new SerDes() { + var terminal = new SerDesStage() { @Override - public String serialize(Object value) { - return value.toString(); + public String serialize(String value) { + return value; } @Override - @SuppressWarnings("unchecked") - public T deserialize(String data, TypeToken typeToken) { - return (T) data; + public String deserialize(String data) { + return data; } @Override @@ -214,14 +218,14 @@ void rejectsDecoratedNestedPipelines() { @Test void rejectsNullIntermediateValues() { - var nullStage = new SerDes() { + var nullStage = new SerDesStage() { @Override - public String serialize(Object value) { + public String serialize(String value) { return null; } @Override - public T deserialize(String data, TypeToken typeToken) { + public String deserialize(String data) { return null; } }; @@ -233,14 +237,14 @@ public T deserialize(String data, TypeToken typeToken) { @Test void preservesRetryabilityWhenDecoratingStageFailures() { - var transientStage = new SerDes() { + var transientStage = new SerDesStage() { @Override - public String serialize(Object value) { + public String serialize(String value) { throw new RetryableSerDesException("retry"); } @Override - public T deserialize(String data, TypeToken typeToken) { + public String deserialize(String data) { return null; } }; @@ -256,14 +260,14 @@ public T deserialize(String data, TypeToken typeToken) { @Test void preservesFatalErrorsFromEveryPipelineCall() { var serializeError = new OutOfMemoryError("serialize"); - var serializeStage = new SerDes() { + var serializeStage = new SerDesStage() { @Override - public String serialize(Object value) { + public String serialize(String value) { throw serializeError; } @Override - public T deserialize(String data, TypeToken typeToken) { + public String deserialize(String data) { return null; } }; @@ -272,14 +276,14 @@ public T deserialize(String data, TypeToken typeToken) { .serialize("value"))); var stringStageError = new StackOverflowError("string-stage-deserialize"); - var stringStage = new SerDes() { + var stringStage = new SerDesStage() { @Override - public String serialize(Object value) { - return value.toString(); + public String serialize(String value) { + return value; } @Override - public T deserialize(String data, TypeToken typeToken) { + public String deserialize(String data) { return null; } @@ -308,19 +312,18 @@ public T deserialize(String data, TypeToken typeToken) { .deserialize("value", TypeToken.get(String.class)))); } - private static SerDes stringStage(String name, String prefix, String suffix, List calls) { - return new SerDes() { + private static SerDesStage stringStage(String name, String prefix, String suffix, List calls) { + return new SerDesStage() { @Override - public String serialize(Object value) { + public String serialize(String value) { calls.add(name + "-serialize"); return prefix + value + suffix; } @Override - @SuppressWarnings("unchecked") - public T deserialize(String data, TypeToken typeToken) { + public String deserialize(String data) { calls.add(name + "-deserialize"); - return (T) data.substring(prefix.length(), data.length() - suffix.length()); + return data.substring(prefix.length(), data.length() - suffix.length()); } }; } diff --git a/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java b/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java index bf94ecde5..d450ec388 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java @@ -26,6 +26,7 @@ import software.amazon.lambda.durable.exception.RetryableSerDesException; import software.amazon.lambda.durable.exception.SerDesException; import software.amazon.lambda.durable.model.OperationSubType; +import software.amazon.lambda.durable.retry.RetryStrategies; class FileSystemSerDesTest { private static final String ARN = @@ -76,6 +77,21 @@ void stageModeComposesWithValueCodec() throws Exception { assertThrows(IllegalArgumentException.class, () -> pipeline.then(wrappingStage())); } + @Test + void retryDecoratorComposesAsAFileSystemStage() { + var stage = FileSystemSerDes.stageBuilder(basePath) + .storageMode(FileSystemStorageMode.OVERFLOW) + .build(); + var pipeline = new JacksonSerDes().then(new RetrySerDes(stage, RetryStrategies.Presets.NO_RETRY)); + var runner = new SerDesRunner(null); + + var envelope = runner.serialize(pipeline, Map.of("id", 42), context()); + + assertEquals( + Map.of("id", 42), + runner.deserialize(pipeline, envelope, new TypeToken>() {}, context())); + } + @Test void stageModeStoresAndRestoresComposableBinaryOutput() throws Exception { var binaryStage = ComposableBinarySerDesStage.builder() @@ -532,17 +548,16 @@ private static byte[] xor(byte[] value, byte key) { return result; } - private static SerDes wrappingStage() { - return new SerDes() { + private static SerDesStage wrappingStage() { + return new SerDesStage() { @Override - public String serialize(Object value) { + public String serialize(String value) { return "<" + value + ">"; } @Override - @SuppressWarnings("unchecked") - public T deserialize(String data, TypeToken typeToken) { - return (T) data.substring(1, data.length() - 1); + public String deserialize(String data) { + return data.substring(1, data.length() - 1); } }; } diff --git a/sdk/src/test/java/software/amazon/lambda/durable/serde/RetrySerDesTest.java b/sdk/src/test/java/software/amazon/lambda/durable/serde/RetrySerDesTest.java index 0df0e7ccf..1beefcee5 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/serde/RetrySerDesTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/serde/RetrySerDesTest.java @@ -77,7 +77,7 @@ public T deserialize(String data, TypeToken typeToken) { @Test void retriesPipelineStageDeserializationAndDelegatesCapabilities() { var calls = new AtomicInteger(); - var delegate = new SerDes() { + class RetryableStage implements SerDes, SerDesStage { @Override public String serialize(Object value) { return value.toString(); @@ -88,6 +88,16 @@ public T deserialize(String data, TypeToken typeToken) { return null; } + @Override + public String serialize(String value) { + return value; + } + + @Override + public String deserialize(String data) { + return data; + } + @Override public SerDesStageResult deserializePipelineStage(String data) { if (calls.incrementAndGet() == 1) { @@ -110,7 +120,8 @@ public boolean isTerminalPipelineStage() { public boolean isValueCodecOnly() { return false; } - }; + } + var delegate = new RetryableStage(); var retrySerDes = new RetrySerDes(delegate, (error, attempt) -> RetryDecision.retry(Duration.ZERO), delay -> {}); From 310ca89d9790f9997229a96c59445484604ab1b2 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Tue, 25 Aug 2026 18:27:45 +0000 Subject: [PATCH 27/56] refactor: make filesystem and retry SerDes stage-only --- docs/adr/005-filesystem-serdes.md | 66 ++++----- docs/advanced/configuration.md | 2 +- docs/advanced/filesystem-serdes.md | 7 +- docs/design.md | 4 +- .../FileSystemSerDesIntegrationTest.java | 8 +- .../testing/CloudDurableTestRunner.java | 3 +- .../testing/LocalDurableTestRunner.java | 3 +- .../testing/CloudDurableTestRunnerTest.java | 20 +-- .../testing/LocalDurableTestRunnerTest.java | 24 +-- .../durable/serde/ComposableSerDes.java | 20 --- .../durable/serde/FileSystemSerDes.java | 104 ++----------- .../lambda/durable/serde/RetrySerDes.java | 59 ++------ .../amazon/lambda/durable/serde/SerDes.java | 9 -- .../SerializableDurableOperationTest.java | 2 +- .../durable/serde/ComposableSerDesTest.java | 15 -- .../durable/serde/FileSystemSerDesTest.java | 137 ++++++++++-------- .../lambda/durable/serde/RetrySerDesTest.java | 89 ++++++------ 17 files changed, 196 insertions(+), 376 deletions(-) diff --git a/docs/adr/005-filesystem-serdes.md b/docs/adr/005-filesystem-serdes.md index 42578d577..8d7b48ba7 100644 --- a/docs/adr/005-filesystem-serdes.md +++ b/docs/adr/005-filesystem-serdes.md @@ -79,8 +79,8 @@ filesystem storage without unsafe heterogeneous top-level stages. A `ComposableB compression, encryption, and similar byte processing internally and encodes the result once at its string boundary. `FileSystemSerDes` acts as a terminal payload-storage stage. It writes the string produced by the previous stage to -the filesystem when configured to do so and returns a small checkpoint envelope. For standalone compatibility, it may -still be constructed with a value-encoding delegate; pipeline configuration is the preferred composition model. +the filesystem when configured to do so and returns a small checkpoint envelope. It is only a `SerDesStage`; a value +codec must precede it in a `ComposableSerDes`. Because the existing `SerDes` methods do not accept context parameters, this approach needs a thread-local `SerDesContext` so `FileSystemSerDes` can discover the current payload identity without changing the @@ -123,7 +123,7 @@ import software.amazon.lambda.durable.serde.FileSystemSerDes; import software.amazon.lambda.durable.serde.FileSystemStorageMode; import software.amazon.lambda.durable.serde.JacksonSerDes; -var fileSystemStage = FileSystemSerDes.stageBuilder(Path.of("/mnt/efs/durable-payloads")) +var fileSystemStage = FileSystemSerDes.builder(Path.of("/mnt/efs/durable-payloads")) .storageMode(FileSystemStorageMode.ALWAYS) .pathEncoding(FileSystemPathEncoding.URI) .previewGenerator(optionalPreviewGenerator) @@ -225,9 +225,9 @@ Pipeline rules: - Every top-level stage consumes and produces a non-null `String`. This makes all stage orderings type-compatible. - `SerDes.requiresDurableContext()` remains a root capability because the SDK and test runners must validate the configured SerDes before an initial durable context exists. `ComposableSerDes` aggregates this capability from its - value codec and stages, and decorators delegate it. -- `SerDes.isValueCodecOnly()` remains a root/decorator capability so input validation and nested-pipeline validation - can identify a `ComposableSerDes` even when it is wrapped by `RetrySerDes`. + value codec and stages. +- The test runners identify a configured input pipeline directly as `ComposableSerDes`; no generic value-codec-only + capability is needed because stage decorators cannot wrap the root SerDes. - Terminality belongs only to `SerDesStage`. A terminal stage must be last so later transformations cannot invalidate its checkpoint-size or storage decision. - `SerDes.then(...)`, `ComposableSerDes.then(...)`, and the builder accept only `SerDesStage` after the value codec. @@ -331,8 +331,8 @@ conversion only at its two outer boundaries; binary stages pass bytes directly t ### Retryable SerDes stages Transient failures are explicit. `RetryableSerDesException` extends `SerDesException` and marks a failure that may -succeed when attempted again. `RetrySerDes` implements both `SerDes` and `SerDesStage`, decorates another SerDes -instance, and applies an existing `RetryStrategy`: +succeed when attempted again. `RetrySerDes` implements `SerDesStage`, decorates another `SerDesStage`, and applies an +existing `RetryStrategy`: ```java var resilientFileSystemStage = new RetrySerDes( @@ -359,13 +359,13 @@ Retry rules: checkpoint. Strategies must therefore use short, bounded delays that fit within the Lambda invocation timeout. - If the invocation is interrupted or times out, replay may execute the SerDes pipeline again. Any stage with side effects must use stable addressing and idempotent writes. -- `RetrySerDes` can wrap an individual component that implements both `SerDes` and `SerDesStage`, such as - `FileSystemSerDes`, or the complete pipeline. Wrapping the smallest transient component avoids repeating - deterministic encoding, compression, or encryption work. +- `RetrySerDes` wraps an individual `SerDesStage`, such as `FileSystemSerDes`. It cannot wrap the value codec or the + complete pipeline. Retrying only the transient component avoids repeating deterministic encoding, compression, or + encryption work. - Pipeline error decoration must preserve retryability: a stage-level `RetryableSerDesException` must remain that type, with stage metadata added to its message or cause, so an enclosing `RetrySerDes` can recognize it. - Filesystem read and write `IOException`s are retryable. Malformed envelopes, invalid paths, unsupported types, and - delegate encoding errors are permanent. + stage transformation errors are permanent. Storage modes: @@ -390,17 +390,17 @@ Envelope format: ``` `FileSystemSerDes` must reject calls when `SerDesContext.getCurrentContext()` is `null` or does not include -`durableExecutionArn` and `entityId`. In pipeline mode it accepts a `String`, records the payload type in the envelope, -and restores that string during reverse processing. A preceding `ComposableBinarySerDesStage` must encode its final -bytes to a string before filesystem storage. Standalone mode continues to use its configured string value codec. +`durableExecutionArn` and `entityId`. It accepts a `String`, records the payload type in the envelope, and restores that +string during reverse processing. A preceding `ComposableBinarySerDesStage` must encode its final bytes to a string +before filesystem storage. The marker and version distinguish filesystem envelopes from raw service-originated JSON. Initial root input, callback -results, and standard Lambda invoke results may arrive before this SerDes has processed them. For those external -payload sources only, an input without the filesystem marker is decoded directly with the pipeline value codec or -standalone delegate. Skipping every intermediate stage is required because raw external data has not been compressed, -encrypted, or otherwise transformed by those stages. Missing or malformed markers on SDK-checkpointed payloads are -permanent errors. The marker name is reserved: malformed marked envelopes and unsupported envelope versions are -rejected at external boundaries rather than being treated as raw user data. +results, and standard Lambda invoke results may arrive before this stage has processed them. For those external +payload sources only, an input without the filesystem marker is decoded directly with the pipeline value codec. +Skipping every intermediate stage is required because raw external data has not been compressed, encrypted, or +otherwise transformed by those stages. Missing or malformed markers on SDK-checkpointed payloads are permanent errors. +The marker name is reserved: malformed marked envelopes and unsupported envelope versions are rejected at external +boundaries rather than being treated as raw user data. Offloaded filenames include a content hash and are immutable. Serializing new state for the same entity creates a new path instead of replacing a file referenced by an earlier checkpoint. Repeating the same serialization may reuse the @@ -416,13 +416,12 @@ must be protected with the same care as the payload it references. The final file envelope, including any preview, must remain below the checkpoint threshold. Oversized previews are rejected rather than producing a checkpoint that the service cannot accept. -`FileSystemSerDes` declares itself terminal in every mode. This makes its overflow decision apply to the final -checkpoint representation and prevents a later encoding or other expanding stage from pushing an inline envelope over -the service limit. +`FileSystemSerDes` declares itself terminal. This makes its overflow decision apply to the final checkpoint +representation and prevents a later encoding or other expanding stage from pushing an inline envelope over the +service limit. -In stage mode, the preview generator receives the `String` produced by the preceding stage, not the original domain -object. A preview that needs domain fields should either parse that representation, be produced by an earlier stage, -or use standalone compatibility mode where `FileSystemSerDes` receives the original value. +The preview generator receives the `String` produced by the preceding stage, not the original domain object. A preview +that needs domain fields should parse that representation or be produced by an earlier stage. ### Runtime flow @@ -443,9 +442,8 @@ try { On deserialization, `FileSystemSerDes` parses its envelope. If the envelope contains `data`, it restores the inline text. If the envelope contains `file`, it reads the stored string. `ComposableSerDes` then passes that value to the -preceding string stage. In standalone compatibility mode, `FileSystemSerDes` passes the resolved string to its -configured value-encoding delegate. Raw external input, callback results, and standard invoke results skip all -intermediate stages and go directly to the value codec when no versioned filesystem marker is present. +preceding string stage. Raw external input, callback results, and standard invoke results skip all intermediate stages +and go directly to the value codec when no versioned filesystem marker is present. ### Threading @@ -567,10 +565,10 @@ must not synthesize a durable context and serialize initial input through the fu 8. Add bounded, invocation-scoped deserialization caching keyed by SerDes identity, entity, payload kind, type, and serialized data hash. 9. Update exception serialization and deserialization paths to set `SerDesPayloadKind.EXCEPTION` in TLS. -10. Implement `FileSystemSerDes` in the core `software.amazon.lambda.durable.serde` package with standalone compatibility and - string terminal-stage modes, `ALWAYS` and `OVERFLOW` storage, `URI` and `HASH` path encodings, envelope parsing, atomic file - writes where supported by the filesystem, retryable I/O failures, and clear validation errors for missing context or - invalid stage input. +10. Implement `FileSystemSerDes` in the core `software.amazon.lambda.durable.serde` package as a string terminal stage + with `ALWAYS` and `OVERFLOW` storage, `URI` and `HASH` path encodings, envelope parsing, atomic file writes where + supported by the filesystem, retryable I/O failures, and clear validation errors for missing context or invalid + stage input. 11. Add unit tests for string and binary pipeline ordering, custom boundary codecs, reverse processing, nulls, stage failures, retry selection, exhaustion, delay handling, interruption, context construction, TLS scoping and restoration, inline execution, configured thread-pool isolation, cache hits, cache invalidation, exception diff --git a/docs/advanced/configuration.md b/docs/advanced/configuration.md index 940d4026e..e19ba7743 100644 --- a/docs/advanced/configuration.md +++ b/docs/advanced/configuration.md @@ -56,7 +56,7 @@ and uses a bounded weak-reference cache for successful deserialization results d The core SDK provides a reversible terminal stage for storing serialized strings on a shared filesystem: ```java -var fileSystemStage = FileSystemSerDes.stageBuilder(Path.of("/mnt/efs/durable-payloads")) +var fileSystemStage = FileSystemSerDes.builder(Path.of("/mnt/efs/durable-payloads")) .storageMode(FileSystemStorageMode.OVERFLOW) .pathEncoding(FileSystemPathEncoding.HASH) .previewGenerator(json -> Map.of("format", "json")) diff --git a/docs/advanced/filesystem-serdes.md b/docs/advanced/filesystem-serdes.md index 836344c3a..0c0bd31b0 100644 --- a/docs/advanced/filesystem-serdes.md +++ b/docs/advanced/filesystem-serdes.md @@ -15,10 +15,10 @@ envelopes in checkpoints. It is included in the core `aws-durable-execution-sdk- ## Pipeline configuration -The preferred configuration uses `FileSystemSerDes` as a reversible terminal `SerDesStage` after a value codec: +`FileSystemSerDes` is a reversible terminal `SerDesStage` and must be configured after a value codec: ```java -var fileSystemStage = FileSystemSerDes.stageBuilder(Path.of("/mnt/efs/durable-payloads")) +var fileSystemStage = FileSystemSerDes.builder(Path.of("/mnt/efs/durable-payloads")) .storageMode(FileSystemStorageMode.ALWAYS) .pathEncoding(FileSystemPathEncoding.URI) .previewGenerator(json -> Map.of("format", "json")) @@ -61,9 +61,6 @@ complete compression/encryption chain. The preview generator receives the incoming stage string. Its output is included only in file envelopes and the final envelope must remain below the checkpoint threshold. -For compatibility, `FileSystemSerDes.builder(path)` creates a standalone SerDes with `JacksonSerDes` as its default -value codec. A custom standalone codec can be supplied with `.delegate(...)`. - ## Execution and retries SerDes runs inline by default. Filesystem access and retry backoff are blocking, so production configurations should diff --git a/docs/design.md b/docs/design.md index 7e633ef63..129e09cfe 100644 --- a/docs/design.md +++ b/docs/design.md @@ -363,7 +363,7 @@ software.amazon.lambda.durable │ ├── Base64StringBinaryCodec # Standard Base64 string/byte conversion │ ├── ComposableBinarySerDesStage # Ordered binary chain exposed as one string stage │ ├── JacksonSerDes # Jackson impl -│ ├── RetrySerDes # Retrying SerDes and string-stage decorator +│ ├── RetrySerDes # Retrying string-stage decorator │ ├── SerDesRunner # Context, optional executor dispatch, and invocation cache │ ├── SerDesContext # Read-only durable payload identity │ ├── SerDesPayloadKind # Input/result/state/exception/invoke payload kind @@ -523,7 +523,7 @@ SuspendExecutionException # Internal: triggers suspension (not | `NonDeterministicExecutionException` | Replay finds different operation than expected | Bug in handler (non-deterministic code) | | `IllegalDurableOperationException` | Illegal operation detected | Bug in handler | | `SerDesException` | Jackson fails to serialize/deserialize | Fix data model or custom SerDes | -| `RetryableSerDesException` | Transient SerDes or payload storage failure | Wrap the SerDes with `RetrySerDes` and a bounded retry strategy | +| `RetryableSerDesException` | Transient stage or payload storage failure | Wrap the failing `SerDesStage` with `RetrySerDes` and a bounded retry strategy | --- diff --git a/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FileSystemSerDesIntegrationTest.java b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FileSystemSerDesIntegrationTest.java index 312282277..57c648120 100644 --- a/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FileSystemSerDesIntegrationTest.java +++ b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FileSystemSerDesIntegrationTest.java @@ -164,7 +164,7 @@ void acceptsRawCallbackAndInvokeResultsAndOffloadsInvokePayload() throws Excepti }); var serDes = new JacksonSerDes() .then(recordingStage) - .then(FileSystemSerDes.stageBuilder(basePath).build()); + .then(FileSystemSerDes.builder(basePath).build()); var config = DurableConfig.builder().withSerDes(serDes).build(); var runner = LocalDurableTestRunner.create( String.class, @@ -282,7 +282,7 @@ void repeatedGetUsesInvocationCacheForTheCompletePipeline() { }); var serDes = new JacksonSerDes() .then(countingStage) - .then(FileSystemSerDes.stageBuilder(basePath).build()); + .then(FileSystemSerDes.builder(basePath).build()); var config = DurableConfig.builder().withSerDes(serDes).build(); var runner = LocalDurableTestRunner.create( String.class, @@ -319,7 +319,7 @@ void successfulRetryUsesTheProducingAttemptForResultSerialization() { }); var serDes = new JacksonSerDes() .then(attemptStage) - .then(FileSystemSerDes.stageBuilder(basePath).build()); + .then(FileSystemSerDes.builder(basePath).build()); var config = DurableConfig.builder().withSerDes(serDes).build(); var stepConfig = StepConfig.builder() .retryStrategy(RetryStrategies.fixedDelay(2, Duration.ofSeconds(1))) @@ -477,7 +477,7 @@ private SerDes filesystemPipeline() { .build(); return new JacksonSerDes() .then(binaryStage) - .then(FileSystemSerDes.stageBuilder(basePath).build()); + .then(FileSystemSerDes.builder(basePath).build()); } private static DurableExecutionInput durableInput( diff --git a/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/CloudDurableTestRunner.java b/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/CloudDurableTestRunner.java index b556b665b..63775a13b 100644 --- a/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/CloudDurableTestRunner.java +++ b/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/CloudDurableTestRunner.java @@ -10,6 +10,7 @@ import software.amazon.awssdk.services.lambda.model.InvocationType; import software.amazon.awssdk.services.lambda.model.InvokeRequest; import software.amazon.lambda.durable.TypeToken; +import software.amazon.lambda.durable.serde.ComposableSerDes; import software.amazon.lambda.durable.serde.JacksonSerDes; import software.amazon.lambda.durable.serde.SerDes; import software.amazon.lambda.durable.serde.SerDesRunner; @@ -285,7 +286,7 @@ private String serializeInput(I input) { "Initial input SerDes requires a durable execution context; configure a context-free " + "input SerDes with withInputSerDes(...)"); } - if (!serializer.isValueCodecOnly() && serDes.requiresDurableContext()) { + if (serializer instanceof ComposableSerDes && serDes.requiresDurableContext()) { throw new IllegalStateException( "Initial input for a context-dependent persisted SerDes must use a value codec, not a composable " + "pipeline"); diff --git a/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/LocalDurableTestRunner.java b/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/LocalDurableTestRunner.java index 51274b0e3..c055b4123 100644 --- a/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/LocalDurableTestRunner.java +++ b/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/LocalDurableTestRunner.java @@ -23,6 +23,7 @@ import software.amazon.lambda.durable.model.DurableExecutionInput; import software.amazon.lambda.durable.model.ExecutionStatus; import software.amazon.lambda.durable.plugin.DurableExecutionPlugin; +import software.amazon.lambda.durable.serde.ComposableSerDes; import software.amazon.lambda.durable.serde.SerDes; import software.amazon.lambda.durable.serde.SerDesRunner; import software.amazon.lambda.durable.testing.local.LocalMemoryExecutionClient; @@ -405,7 +406,7 @@ private String serializeInput(I input) { "Initial input SerDes requires a durable execution context; configure a context-free " + "input SerDes with withInputSerDes(...)"); } - if (!serializer.isValueCodecOnly() && serDes.requiresDurableContext()) { + if (serializer instanceof ComposableSerDes && serDes.requiresDurableContext()) { throw new IllegalStateException( "Initial input for a context-dependent persisted SerDes must use a value codec, not a composable " + "pipeline"); diff --git a/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/CloudDurableTestRunnerTest.java b/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/CloudDurableTestRunnerTest.java index b7857bb05..93d12f73d 100644 --- a/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/CloudDurableTestRunnerTest.java +++ b/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/CloudDurableTestRunnerTest.java @@ -15,10 +15,8 @@ import software.amazon.awssdk.services.lambda.model.InvokeRequest; import software.amazon.awssdk.services.lambda.model.InvokeResponse; import software.amazon.lambda.durable.TypeToken; -import software.amazon.lambda.durable.retry.RetryStrategies; import software.amazon.lambda.durable.serde.FileSystemSerDes; import software.amazon.lambda.durable.serde.JacksonSerDes; -import software.amazon.lambda.durable.serde.RetrySerDes; import software.amazon.lambda.durable.serde.SerDes; import software.amazon.lambda.durable.serde.SerDesContext; import software.amazon.lambda.durable.serde.SerDesPayloadKind; @@ -127,22 +125,6 @@ void contextDependentPersistedSerDesRequiresValueCodecInput() { verifyNoInteractions(mockClient); } - @Test - void contextDependentPersistedSerDesRejectsRetryWrappedComposableInput() { - var mockClient = mock(LambdaClient.class); - var inputSerDes = new RetrySerDes(new JacksonSerDes().then(wrappingStage()), RetryStrategies.Presets.NO_RETRY); - var runner = CloudDurableTestRunner.create( - "arn:aws:lambda:us-east-2:123:function:test", String.class, String.class, mockClient) - .withSerDes(new JacksonSerDes().then(contextDependentStage())) - .withInputSerDes(inputSerDes); - - var failure = assertThrows(RuntimeException.class, () -> runner.startAsync("value")); - - assertInstanceOf(IllegalStateException.class, failure.getCause()); - assertTrue(failure.getCause().getMessage().contains("must use a value codec")); - verifyNoInteractions(mockClient); - } - @Test void valueCodecInputRoundTripsThroughFileSystemPipeline(@TempDir Path basePath) { var mockClient = mock(LambdaClient.class); @@ -152,7 +134,7 @@ void valueCodecInputRoundTripsThroughFileSystemPipeline(@TempDir Path basePath) .durableExecutionArn(executionArn) .build()); var persistedSerDes = - new JacksonSerDes().then(FileSystemSerDes.stageBuilder(basePath).build()); + new JacksonSerDes().then(FileSystemSerDes.builder(basePath).build()); var runner = CloudDurableTestRunner.create( "arn:aws:lambda:us-east-2:123:function:test", String.class, String.class, mockClient) .withSerDes(persistedSerDes) diff --git a/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/LocalDurableTestRunnerTest.java b/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/LocalDurableTestRunnerTest.java index da310ed69..edc0ebdd5 100644 --- a/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/LocalDurableTestRunnerTest.java +++ b/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/LocalDurableTestRunnerTest.java @@ -19,13 +19,11 @@ import software.amazon.lambda.durable.model.ExecutionStatus; import software.amazon.lambda.durable.plugin.DurableExecutionPlugin; import software.amazon.lambda.durable.plugin.InvocationInfo; -import software.amazon.lambda.durable.retry.RetryStrategies; import software.amazon.lambda.durable.serde.Base64StringBinaryCodec; import software.amazon.lambda.durable.serde.BinarySerDes; import software.amazon.lambda.durable.serde.ComposableBinarySerDesStage; import software.amazon.lambda.durable.serde.FileSystemSerDes; import software.amazon.lambda.durable.serde.JacksonSerDes; -import software.amazon.lambda.durable.serde.RetrySerDes; import software.amazon.lambda.durable.serde.SerDesStage; import software.amazon.lambda.durable.serde.Utf8StringBinaryCodec; @@ -154,7 +152,7 @@ void checkpointedLargeOutputReplaysWithoutDuplicateExecutionOperation() { void contextDependentPersistedSerDesRequiresExplicitInputSerDes(@TempDir Path basePath) { var config = DurableConfig.builder() .withSerDes(new JacksonSerDes() - .then(FileSystemSerDes.stageBuilder(basePath).build())) + .then(FileSystemSerDes.builder(basePath).build())) .build(); var runner = LocalDurableTestRunner.create(String.class, (input, context) -> input, config); @@ -167,7 +165,7 @@ void contextDependentPersistedSerDesRequiresExplicitInputSerDes(@TempDir Path ba void contextDependentPersistedSerDesRequiresValueCodecInput(@TempDir Path basePath) { var config = DurableConfig.builder() .withSerDes(new JacksonSerDes() - .then(FileSystemSerDes.stageBuilder(basePath).build())) + .then(FileSystemSerDes.builder(basePath).build())) .build(); var runner = LocalDurableTestRunner.create(String.class, (input, context) -> input, config) .withInputSerDes(new JacksonSerDes().then(wrappingStage(new AtomicInteger()))); @@ -177,28 +175,12 @@ void contextDependentPersistedSerDesRequiresValueCodecInput(@TempDir Path basePa assertTrue(failure.getMessage().contains("must use a value codec")); } - @Test - void contextDependentPersistedSerDesRejectsRetryWrappedComposableInput(@TempDir Path basePath) { - var config = DurableConfig.builder() - .withSerDes(new JacksonSerDes() - .then(FileSystemSerDes.stageBuilder(basePath).build())) - .build(); - var inputSerDes = new RetrySerDes( - new JacksonSerDes().then(wrappingStage(new AtomicInteger())), RetryStrategies.Presets.NO_RETRY); - var runner = LocalDurableTestRunner.create(String.class, (input, context) -> input, config) - .withInputSerDes(inputSerDes); - - var failure = assertThrows(IllegalStateException.class, () -> runner.run("value")); - - assertTrue(failure.getMessage().contains("must use a value codec")); - } - @Test void initialInputBypassesStagesBeforeFileSystemSerDes(@TempDir Path basePath) { var deserializeCalls = new AtomicInteger(); var persistedSerDes = new JacksonSerDes() .then(bytesStage(deserializeCalls)) - .then(FileSystemSerDes.stageBuilder(basePath).build()); + .then(FileSystemSerDes.builder(basePath).build()); var config = DurableConfig.builder().withSerDes(persistedSerDes).build(); var runner = LocalDurableTestRunner.create( String.class, (input, context) -> input + ":" + deserializeCalls.get(), config) diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/ComposableSerDes.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/ComposableSerDes.java index 82ed763d7..6433b9cd3 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/ComposableSerDes.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/ComposableSerDes.java @@ -21,20 +21,11 @@ public final class ComposableSerDes implements SerDes { private ComposableSerDes(SerDes valueCodec, List stages) { this.valueCodec = Objects.requireNonNull(valueCodec, "valueCodec cannot be null"); - if (!stages.isEmpty() && !valueCodec.isValueCodecOnly()) { - throw nestedPipelineFailure(0, valueCodec); - } if (!stages.isEmpty() && valueCodec instanceof SerDesStage valueCodecStage && valueCodecStage.isTerminalPipelineStage()) { throw terminalStageFailure(0, valueCodec); } - for (int index = 0; index < stages.size(); index++) { - var stage = stages.get(index); - if (stage instanceof SerDes serDes && !serDes.isValueCodecOnly()) { - throw nestedPipelineFailure(index + 1, stage); - } - } for (int index = 0; index < stages.size() - 1; index++) { if (isTerminal(stages.get(index))) { throw terminalStageFailure(index + 1, stages.get(index)); @@ -84,11 +75,6 @@ public boolean requiresDurableContext() { return valueCodec.requiresDurableContext() || stages.stream().anyMatch(ComposableSerDes::requiresContext); } - @Override - public boolean isValueCodecOnly() { - return stages.isEmpty() && valueCodec.isValueCodecOnly(); - } - /** Returns a new pipeline with the supplied string stage appended. */ @Override public ComposableSerDes then(SerDesStage stage) { @@ -189,12 +175,6 @@ private static IllegalArgumentException terminalStageFailure(int index, Object s index, stage.getClass().getName())); } - private static IllegalArgumentException nestedPipelineFailure(int index, Object stage) { - return new IllegalArgumentException(String.format( - "SerDes pipeline stage %d (%s) must be a value codec or a single reversible stage, not a nested pipeline", - index, stage.getClass().getName())); - } - private static boolean requiresContext(SerDesStage stage) { return stage.requiresDurableContext(); } diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java index d379f12c8..cada25190 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java @@ -21,20 +21,16 @@ import java.util.function.Function; import java.util.regex.Pattern; import software.amazon.awssdk.services.lambda.model.OperationType; -import software.amazon.lambda.durable.TypeToken; import software.amazon.lambda.durable.exception.RetryableSerDesException; import software.amazon.lambda.durable.exception.SerDesException; /** - * A SerDes and terminal string stage that stores payloads on a durable shared filesystem. - * - *

Use {@link #stageBuilder(Path)} when composing this implementation after a value codec. The compatibility - * {@link #builder(Path)} form includes its own value codec and can be used as a standalone SerDes. + * A terminal string stage that stores payloads on a durable shared filesystem. * *

Do not use Lambda's ephemeral {@code /tmp} storage. Use a durable shared mount such as EFS, or S3 Files only when * its synchronization and crash-durability tradeoffs are acceptable for the workload. */ -public final class FileSystemSerDes implements SerDes, SerDesStage { +public final class FileSystemSerDes implements SerDesStage { private static final String ENVELOPE_MARKER = "__durable_execution_filesystem_serdes"; private static final String PAYLOAD_TYPE_FIELD = "payloadType"; private static final int ENVELOPE_VERSION = 1; @@ -46,50 +42,35 @@ public final class FileSystemSerDes implements SerDes, SerDesStage { private final Path basePath; private final FileSystemStorageMode storageMode; private final FileSystemPathEncoding pathEncoding; - private final SerDes delegate; - private final Function> previewGenerator; - private final boolean stageMode; + private final Function> previewGenerator; private volatile Path canonicalBasePath; private FileSystemSerDes(Builder builder) { basePath = builder.basePath.toAbsolutePath().normalize(); storageMode = builder.storageMode; pathEncoding = builder.pathEncoding; - delegate = builder.delegate; previewGenerator = builder.previewGenerator; - stageMode = builder.stageMode; - } - - /** - * Creates a standalone filesystem SerDes builder with {@link JacksonSerDes} as its default value codec. - * - * @param basePath durable shared filesystem root - * @return a standalone builder - */ - public static Builder builder(Path basePath) { - return new Builder(basePath, false); } /** - * Creates a filesystem terminal-stage builder for use in a composable SerDes pipeline. + * Creates a filesystem terminal-stage builder for use after a value codec in a composable SerDes pipeline. * - *

Stage mode accepts strings. Use {@link ComposableBinarySerDesStage} before this stage when binary - * transformations are required. + *

Use {@link ComposableBinarySerDesStage} before this stage when binary transformations are required. * * @param basePath durable shared filesystem root * @return a terminal-stage builder */ - public static Builder stageBuilder(Path basePath) { - return new Builder(basePath, true); + public static Builder builder(Path basePath) { + return new Builder(basePath); } @Override - public String serialize(Object value) { + public String serialize(String value) { if (value == null) { return null; } var context = requireContext(); - var payload = serializeValue(value); + var payload = SerializedPayload.fromString(value); if (storageMode == FileSystemStorageMode.OVERFLOW) { var inlineEnvelope = encodeEnvelope(payload, null, null, context); if (fitsCheckpoint(inlineEnvelope)) { @@ -115,39 +96,16 @@ public String serialize(Object value) { } @Override - public String serialize(String value) { - return serialize((Object) value); - } - - @Override - public T deserialize(String data, TypeToken typeToken) { + public String deserialize(String data) { if (data == null) { return null; } - Objects.requireNonNull(typeToken, "typeToken cannot be null"); var context = requireContext(); - var serialized = resolveSerializedPayload(data, context).value(); - if (stageMode) { - if (TypeToken.get(String.class).equals(typeToken)) { - @SuppressWarnings("unchecked") - var value = (T) serialized; - return value; - } - throw new SerDesException("FileSystemSerDes stage payload type does not match requested type " + typeToken); - } - return delegate.deserialize(serialized, typeToken); - } - - @Override - public String deserialize(String data) { - return deserialize(data, TypeToken.get(String.class)); + return resolveSerializedPayload(data, context).value(); } @Override public SerDesStageResult deserializePipelineStage(String data) { - if (!stageMode) { - return SerDesStage.super.deserializePipelineStage(data); - } var context = requireContext(); var resolved = resolveSerializedPayload(data, context); return resolved.external() @@ -165,21 +123,6 @@ public boolean isTerminalPipelineStage() { return true; } - private SerializedPayload serializeValue(Object value) { - if (stageMode) { - if (value instanceof String stringValue) { - return SerializedPayload.fromString(stringValue); - } - throw new SerDesException("FileSystemSerDes stage supports String values, but received " - + value.getClass().getName()); - } - var serialized = delegate.serialize(value); - if (serialized == null) { - throw new SerDesException("Delegate SerDes returned null for a non-null value"); - } - return SerializedPayload.fromString(serialized); - } - private ResolvedPayload resolveSerializedPayload(String data, SerDesContext context) { final JsonNode envelope; try { @@ -405,7 +348,7 @@ private String encodeEnvelope( } } - private Map generatePreview(Object value, SerDesContext context) { + private Map generatePreview(String value, SerDesContext context) { if (previewGenerator == null) { return null; } @@ -646,16 +589,12 @@ private String value() { /** Builder for {@link FileSystemSerDes}. */ public static final class Builder { private final Path basePath; - private final boolean stageMode; private FileSystemStorageMode storageMode = FileSystemStorageMode.ALWAYS; private FileSystemPathEncoding pathEncoding = FileSystemPathEncoding.URI; - private SerDes delegate; - private Function> previewGenerator; + private Function> previewGenerator; - private Builder(Path basePath, boolean stageMode) { + private Builder(Path basePath) { this.basePath = Objects.requireNonNull(basePath, "basePath cannot be null"); - this.stageMode = stageMode; - this.delegate = stageMode ? null : new JacksonSerDes(); } public Builder storageMode(FileSystemStorageMode storageMode) { @@ -668,20 +607,7 @@ public Builder pathEncoding(FileSystemPathEncoding pathEncoding) { return this; } - /** - * Sets the value codec used by standalone mode. - * - * @throws IllegalStateException when called on a stage builder - */ - public Builder delegate(SerDes delegate) { - if (stageMode) { - throw new IllegalStateException("FileSystemSerDes stage mode does not use a delegate"); - } - this.delegate = Objects.requireNonNull(delegate, "delegate cannot be null"); - return this; - } - - public Builder previewGenerator(Function> previewGenerator) { + public Builder previewGenerator(Function> previewGenerator) { this.previewGenerator = Objects.requireNonNull(previewGenerator, "previewGenerator cannot be null"); return this; } diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/RetrySerDes.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/RetrySerDes.java index f0a7e60a7..197726aff 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/RetrySerDes.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/RetrySerDes.java @@ -6,23 +6,18 @@ import java.util.Objects; import java.util.concurrent.TimeUnit; import java.util.function.Supplier; -import software.amazon.lambda.durable.TypeToken; import software.amazon.lambda.durable.exception.RetryableSerDesException; import software.amazon.lambda.durable.exception.SerDesException; import software.amazon.lambda.durable.retry.RetryDecision; import software.amazon.lambda.durable.retry.RetryStrategy; /** - * A SerDes and string-stage decorator that retries transient failures from another {@link SerDes}. + * A string-stage decorator that retries transient failures from another {@link SerDesStage}. * *

Only {@link RetryableSerDesException} is retried. Other failures are propagated immediately. Retry delays block * the thread executing the SerDes call: the caller by default or the configured SerDes executor thread. - * - *

When the delegate also implements {@link SerDesStage}, this decorator uses its explicit string-stage behavior when - * appended to a {@link ComposableSerDes}. Otherwise, the delegate is invoked with {@link String} values and a - * {@code TypeToken}. */ -public final class RetrySerDes implements SerDes, SerDesStage { +public final class RetrySerDes implements SerDesStage { private static final Sleeper DEFAULT_SLEEPER = delay -> { if (delay.getSeconds() > 0) { TimeUnit.SECONDS.sleep(delay.getSeconds()); @@ -32,66 +27,39 @@ public final class RetrySerDes implements SerDes, SerDesStage { } }; - private final SerDes delegate; - private final SerDesStage stageDelegate; + private final SerDesStage delegate; private final RetryStrategy retryStrategy; private final Sleeper sleeper; /** - * Creates a retrying SerDes decorator. + * Creates a retrying string-stage decorator. * - * @param delegate the SerDes to invoke + * @param delegate the stage to invoke * @param retryStrategy strategy that controls attempts and delays */ - public RetrySerDes(SerDes delegate, RetryStrategy retryStrategy) { + public RetrySerDes(SerDesStage delegate, RetryStrategy retryStrategy) { this(delegate, retryStrategy, DEFAULT_SLEEPER); } - RetrySerDes(SerDes delegate, RetryStrategy retryStrategy, Sleeper sleeper) { + RetrySerDes(SerDesStage delegate, RetryStrategy retryStrategy, Sleeper sleeper) { this.delegate = Objects.requireNonNull(delegate, "delegate cannot be null"); - this.stageDelegate = delegate instanceof SerDesStage stage ? stage : null; this.retryStrategy = Objects.requireNonNull(retryStrategy, "retryStrategy cannot be null"); this.sleeper = Objects.requireNonNull(sleeper, "sleeper cannot be null"); } - @Override - public String serialize(Object value) { - return execute("serialization", () -> delegate.serialize(value)); - } - @Override public String serialize(String value) { - return execute( - "pipeline stage serialization", - () -> stageDelegate == null ? delegate.serialize(value) : stageDelegate.serialize(value)); - } - - @Override - public T deserialize(String data, TypeToken typeToken) { - return execute("deserialization", () -> delegate.deserialize(data, typeToken)); + return execute("pipeline stage serialization", () -> delegate.serialize(value)); } @Override public String deserialize(String data) { - return execute( - "pipeline stage deserialization", - () -> stageDelegate == null - ? delegate.deserialize(data, TypeToken.get(String.class)) - : stageDelegate.deserialize(data)); + return execute("pipeline stage deserialization", () -> delegate.deserialize(data)); } @Override public SerDesStageResult deserializePipelineStage(String data) { - return execute("pipeline stage deserialization", () -> { - if (stageDelegate != null) { - return stageDelegate.deserializePipelineStage(data); - } - var result = delegate.deserialize(data, TypeToken.get(String.class)); - if (result == null) { - throw new SerDesException("Stage returned null for non-null data"); - } - return SerDesStageResult.continueWith(result); - }); + return execute("pipeline stage deserialization", () -> delegate.deserializePipelineStage(data)); } @Override @@ -101,12 +69,7 @@ public boolean requiresDurableContext() { @Override public boolean isTerminalPipelineStage() { - return stageDelegate != null && stageDelegate.isTerminalPipelineStage(); - } - - @Override - public boolean isValueCodecOnly() { - return delegate.isValueCodecOnly(); + return delegate.isTerminalPipelineStage(); } private T execute(String action, Supplier operation) { diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDes.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDes.java index 958c60078..516753f56 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDes.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDes.java @@ -48,15 +48,6 @@ default boolean requiresDurableContext() { return false; } - /** - * Returns whether this SerDes performs only value-codec processing without additional composable pipeline stages. - * - *

SerDes decorators should delegate this capability to their wrapped SerDes. - */ - default boolean isValueCodecOnly() { - return true; - } - /** * Returns an immutable processing pipeline with a string stage appended. * diff --git a/sdk/src/test/java/software/amazon/lambda/durable/operation/SerializableDurableOperationTest.java b/sdk/src/test/java/software/amazon/lambda/durable/operation/SerializableDurableOperationTest.java index 82e4f0224..a2008d842 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/operation/SerializableDurableOperationTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/operation/SerializableDurableOperationTest.java @@ -543,7 +543,7 @@ void deserializeExceptionPreservesRetryableStorageFailure() { when(executionManager.getDurableExecutionArn()) .thenReturn( "arn:aws:lambda:us-east-1:123456789012:function:test:1/durable-execution/execution/invocation"); - var serDes = FileSystemSerDes.builder(basePath).build(); + var serDes = new JacksonSerDes().then(FileSystemSerDes.builder(basePath).build()); var checkpointedError = new AtomicReference(); SerializableDurableOperation producer = new SerializableDurableOperation<>(OPERATION_IDENTIFIER, RESULT_TYPE, serDes, durableContext) { diff --git a/sdk/src/test/java/software/amazon/lambda/durable/serde/ComposableSerDesTest.java b/sdk/src/test/java/software/amazon/lambda/durable/serde/ComposableSerDesTest.java index 3129d9010..9e3607a42 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/serde/ComposableSerDesTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/serde/ComposableSerDesTest.java @@ -16,7 +16,6 @@ import software.amazon.lambda.durable.TypeToken; import software.amazon.lambda.durable.exception.RetryableSerDesException; import software.amazon.lambda.durable.exception.SerDesException; -import software.amazon.lambda.durable.retry.RetryStrategies; class ComposableSerDesTest { @@ -202,20 +201,6 @@ public boolean isTerminalPipelineStage() { assertTrue(failure.getMessage().contains("final stage")); } - @Test - void rejectsDecoratedNestedPipelines() { - var nested = new JacksonSerDes().then(stringStage("nested", "<", ">", new ArrayList<>())); - var decorated = new RetrySerDes(nested, RetryStrategies.Presets.NO_RETRY); - - var rootFailure = assertThrows( - IllegalArgumentException.class, - () -> decorated.then(stringStage("outer", "[", "]", new ArrayList<>()))); - var stageFailure = assertThrows(IllegalArgumentException.class, () -> new JacksonSerDes().then(decorated)); - - assertTrue(rootFailure.getMessage().contains("nested pipeline")); - assertTrue(stageFailure.getMessage().contains("nested pipeline")); - } - @Test void rejectsNullIntermediateValues() { var nullStage = new SerDesStage() { diff --git a/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java b/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java index d450ec388..9f254c5e8 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java @@ -6,6 +6,7 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -38,8 +39,8 @@ class FileSystemSerDesTest { Path basePath; @Test - void standaloneModeWritesDelegatePayloadAndReplaysIt() throws Exception { - var serDes = FileSystemSerDes.builder(basePath).build(); + void writesValueCodecPayloadAndReplaysIt() throws Exception { + var serDes = new JacksonSerDes().then(FileSystemSerDes.builder(basePath).build()); var runner = new SerDesRunner(null); var envelope = runner.serialize(serDes, Map.of("id", 42), context()); @@ -56,30 +57,19 @@ void standaloneModeWritesDelegatePayloadAndReplaysIt() throws Exception { } @Test - void stageModeComposesWithValueCodec() throws Exception { - var stage = FileSystemSerDes.stageBuilder(basePath).build(); + void isAContextDependentTerminalStage() { + var stage = FileSystemSerDes.builder(basePath).build(); var pipeline = new JacksonSerDes().then(stage); - var runner = new SerDesRunner(null); - - var envelope = runner.serialize(pipeline, Map.of("id", 42), context()); - var file = Path.of(MAPPER.readTree(envelope).get("file").textValue()); - assertEquals("{\"id\":42}", Files.readString(file)); - assertEquals( - Map.of("id", 42), - runner.deserialize(pipeline, envelope, new TypeToken>() {}, context())); - assertThrows(SerDesException.class, () -> runner.serialize(stage, Map.of("id", 42), context())); - assertThrows( - SerDesException.class, - () -> runner.deserialize(stage, envelope, TypeToken.get(Integer.class), context())); - assertThrows(IllegalStateException.class, () -> FileSystemSerDes.stageBuilder(basePath) - .delegate(new JacksonSerDes())); + assertFalse(SerDes.class.isAssignableFrom(FileSystemSerDes.class)); + assertTrue(stage.requiresDurableContext()); + assertTrue(stage.isTerminalPipelineStage()); assertThrows(IllegalArgumentException.class, () -> pipeline.then(wrappingStage())); } @Test void retryDecoratorComposesAsAFileSystemStage() { - var stage = FileSystemSerDes.stageBuilder(basePath) + var stage = FileSystemSerDes.builder(basePath) .storageMode(FileSystemStorageMode.OVERFLOW) .build(); var pipeline = new JacksonSerDes().then(new RetrySerDes(stage, RetryStrategies.Presets.NO_RETRY)); @@ -93,13 +83,13 @@ void retryDecoratorComposesAsAFileSystemStage() { } @Test - void stageModeStoresAndRestoresComposableBinaryOutput() throws Exception { + void storesAndRestoresComposableBinaryOutput() throws Exception { var binaryStage = ComposableBinarySerDesStage.builder() .startWith(Utf8StringBinaryCodec.INSTANCE) .then(xorBinarySerDes((byte) 0x5A)) .endWith(Base64StringBinaryCodec.INSTANCE) .build(); - var stage = FileSystemSerDes.stageBuilder(basePath).build(); + var stage = FileSystemSerDes.builder(basePath).build(); var pipeline = new JacksonSerDes().then(binaryStage).then(stage); var runner = new SerDesRunner(null); @@ -116,21 +106,12 @@ void stageModeStoresAndRestoresComposableBinaryOutput() throws Exception { runner.deserialize(pipeline, envelope, new TypeToken>() {}, context())); } - @Test - void stageModeRejectsDirectBinaryValues() { - var stage = FileSystemSerDes.stageBuilder(basePath) - .storageMode(FileSystemStorageMode.OVERFLOW) - .build(); - var runner = new SerDesRunner(null); - - assertThrows(SerDesException.class, () -> runner.serialize(stage, new byte[] {0, 1, 2, -1}, context())); - } - @Test void overflowModeKeepsSmallPayloadInlineAndWritesLargePayload() throws Exception { - var serDes = FileSystemSerDes.builder(basePath) + var stage = FileSystemSerDes.builder(basePath) .storageMode(FileSystemStorageMode.OVERFLOW) .build(); + var serDes = stringCodec().then(stage); var runner = new SerDesRunner(null); var inline = runner.serialize(serDes, "small", context()); @@ -142,7 +123,7 @@ void overflowModeKeepsSmallPayloadInlineAndWritesLargePayload() throws Exception @Test void immutableContentAddressedFilesPreservePriorCheckpoints() throws Exception { - var serDes = FileSystemSerDes.stageBuilder(basePath).build(); + var serDes = stringCodec().then(FileSystemSerDes.builder(basePath).build()); var runner = new SerDesRunner(null); var firstContext = context(1); @@ -160,7 +141,7 @@ void immutableContentAddressedFilesPreservePriorCheckpoints() throws Exception { @Test void rejectsUnexpectedContentInExistingContentAddressedFile() throws Exception { - var serDes = FileSystemSerDes.stageBuilder(basePath).build(); + var serDes = stringCodec().then(FileSystemSerDes.builder(basePath).build()); var runner = new SerDesRunner(null); var envelope = runner.serialize(serDes, "expected", context()); var file = payloadFile(envelope); @@ -168,13 +149,13 @@ void rejectsUnexpectedContentInExistingContentAddressedFile() throws Exception { var failure = assertThrows(SerDesException.class, () -> runner.serialize(serDes, "expected", context())); - assertTrue(failure.getCause().getMessage().contains("contains unexpected data")); + assertCauseMessage(failure, "contains unexpected data"); assertEquals("unexpected", Files.readString(file)); } @Test void rejectsMalformedUtf8StringsAndFilePayloads() throws Exception { - var serDes = FileSystemSerDes.stageBuilder(basePath).build(); + var serDes = stringCodec().then(FileSystemSerDes.builder(basePath).build()); var runner = new SerDesRunner(null); assertThrows(SerDesException.class, () -> runner.serialize(serDes, "lone surrogate \uD800", context())); @@ -193,9 +174,10 @@ void rejectsMalformedUtf8StringsAndFilePayloads() throws Exception { @Test void hashEncodingUsesFixedLengthSegments() throws Exception { - var serDes = FileSystemSerDes.builder(basePath) + var stage = FileSystemSerDes.builder(basePath) .pathEncoding(FileSystemPathEncoding.HASH) .build(); + var serDes = stringCodec().then(stage); var envelope = new SerDesRunner(null).serialize(serDes, "value", context()); var file = Path.of(MAPPER.readTree(envelope).get("file").textValue()); @@ -207,9 +189,10 @@ void hashEncodingUsesFixedLengthSegments() throws Exception { @Test void includesBoundedPreviewWithoutChangingStoredPayload() throws Exception { - var serDes = FileSystemSerDes.builder(basePath) + var stage = FileSystemSerDes.builder(basePath) .previewGenerator(value -> Map.of("summary", "order")) .build(); + var serDes = new JacksonSerDes().then(stage); var runner = new SerDesRunner(null); var envelope = runner.serialize(serDes, Map.of("secret", "value"), context()); @@ -220,24 +203,25 @@ void includesBoundedPreviewWithoutChangingStoredPayload() throws Exception { "{\"secret\":\"value\"}", Files.readString(Path.of(json.get("file").textValue()))); - var oversizedPreview = FileSystemSerDes.builder(basePath) + var oversizedPreviewStage = FileSystemSerDes.builder(basePath) .previewGenerator(value -> Map.of("summary", "x".repeat(256 * 1024))) .build(); + var oversizedPreview = stringCodec().then(oversizedPreviewStage); var failure = assertThrows(SerDesException.class, () -> runner.serialize(oversizedPreview, "value", context())); - assertTrue(failure.getCause().getMessage().contains("checkpoint payload limit")); + assertCauseMessage(failure, "checkpoint payload limit"); } @Test void acceptsExternallyOriginatedRawPayloadsOnlyForSupportedSources() { - var standalone = FileSystemSerDes.builder(basePath).build(); - var stage = FileSystemSerDes.stageBuilder(basePath).build(); + var stage = FileSystemSerDes.builder(basePath).build(); + var filesystemPipeline = new JacksonSerDes().then(stage); var pipeline = new JacksonSerDes().then(wrappingStage()).then(stage); var runner = new SerDesRunner(null); assertEquals( Map.of("id", 42), runner.deserialize( - standalone, + filesystemPipeline, "{\"id\":42}", new TypeToken>() {}, executionContext(SerDesPayloadKind.INPUT))); @@ -258,26 +242,26 @@ void acceptsExternallyOriginatedRawPayloadsOnlyForSupportedSources() { assertEquals( Map.of("domainMarker", 1, "data", "domain-value"), runner.deserialize( - standalone, + filesystemPipeline, "{\"domainMarker\":1,\"data\":\"domain-value\"}", new TypeToken>() {}, executionContext(SerDesPayloadKind.INPUT))); assertThrows( SerDesException.class, () -> runner.deserialize( - standalone, + filesystemPipeline, "{\"__durable_execution_filesystem_serdes\":1,\"data\":\"domain-value\"}", new TypeToken>() {}, executionContext(SerDesPayloadKind.INPUT))); assertThrows( SerDesException.class, - () -> runner.deserialize(stage, "\"raw-step\"", TypeToken.get(String.class), context())); + () -> runner.deserialize(filesystemPipeline, "\"raw-step\"", TypeToken.get(String.class), context())); } @Test void rejectsUnsupportedEnvelopeVersionsAtExternalBoundaries() { - var serDes = FileSystemSerDes.builder(basePath).build(); + var serDes = stringCodec().then(FileSystemSerDes.builder(basePath).build()); var futureEnvelope = "{\"__durable_execution_filesystem_serdes\":2," + "\"ownerDurableExecutionArn\":\"" + ARN @@ -290,7 +274,7 @@ void rejectsUnsupportedEnvelopeVersionsAtExternalBoundaries() { TypeToken.get(String.class), executionContext(SerDesPayloadKind.INPUT))); - assertTrue(failure.getCause().getMessage().contains("Unsupported filesystem SerDes envelope version 2")); + assertCauseMessage(failure, "Unsupported filesystem SerDes envelope version 2"); } @Test @@ -302,7 +286,7 @@ void rejectsUnsupportedBinaryPayloadType() { assertThrows(SerDesException.class, () -> new SerDesRunner(null) .deserialize( - FileSystemSerDes.stageBuilder(basePath).build(), + stringCodec().then(FileSystemSerDes.builder(basePath).build()), envelope, TypeToken.get(String.class), context())); @@ -310,7 +294,7 @@ void rejectsUnsupportedBinaryPayloadType() { @Test void rejectsOutOfRangeEnvelopeVersionsWithoutTruncation() { - var serDes = FileSystemSerDes.builder(basePath).build(); + var serDes = stringCodec().then(FileSystemSerDes.builder(basePath).build()); var oversizedVersion = "{\"__durable_execution_filesystem_serdes\":4294967297," + "\"ownerDurableExecutionArn\":\"" + ARN @@ -323,13 +307,12 @@ void rejectsOutOfRangeEnvelopeVersionsWithoutTruncation() { TypeToken.get(String.class), executionContext(SerDesPayloadKind.INPUT))); - assertTrue( - failure.getCause().getMessage().contains("Unsupported filesystem SerDes envelope version 4294967297")); + assertCauseMessage(failure, "Unsupported filesystem SerDes envelope version 4294967297"); } @Test void overflowFilesystemStageMustRemainTerminal() { - var filesystem = FileSystemSerDes.stageBuilder(basePath) + var filesystem = FileSystemSerDes.builder(basePath) .storageMode(FileSystemStorageMode.OVERFLOW) .build(); @@ -342,8 +325,7 @@ void overflowFilesystemStageMustRemainTerminal() { @Test void fileReferencesCrossInvokeInputAndResultBoundaries() { - var serDes = - new JacksonSerDes().then(FileSystemSerDes.stageBuilder(basePath).build()); + var serDes = new JacksonSerDes().then(FileSystemSerDes.builder(basePath).build()); var runner = new SerDesRunner(null); var callerArn = "arn:aws:lambda:us-east-1:123456789012:function:caller:1/durable-execution/caller-execution/caller-invocation"; @@ -387,8 +369,9 @@ void fileReferencesCrossInvokeInputAndResultBoundaries() { @Test void rejectsCallsWithoutContextAndMalformedOrUnsafeEnvelopes() throws Exception { - var serDes = FileSystemSerDes.builder(basePath).build(); - assertThrows(SerDesException.class, () -> serDes.serialize("value")); + var stage = FileSystemSerDes.builder(basePath).build(); + var serDes = stringCodec().then(stage); + assertThrows(SerDesException.class, () -> stage.serialize("value")); var runner = new SerDesRunner(null); assertThrows( @@ -409,7 +392,7 @@ void rejectsCallsWithoutContextAndMalformedOrUnsafeEnvelopes() throws Exception @Test void rejectsCrossEntityAndSymbolicLinkReferences() throws Exception { - var serDes = FileSystemSerDes.stageBuilder(basePath).build(); + var serDes = stringCodec().then(FileSystemSerDes.builder(basePath).build()); var envelope = new SerDesRunner(null).serialize(serDes, "payload", context()); var otherEntity = SerDesContext.forOperation( @@ -431,7 +414,7 @@ void rejectsCrossEntityAndSymbolicLinkReferences() throws Exception { void rejectsSymbolicLinkDirectoriesWhenWriting() throws Exception { var outside = Files.createTempDirectory(basePath.getParent(), "outside-payloads-"); Files.createSymbolicLink(basePath.resolve("orders"), outside); - var serDes = FileSystemSerDes.stageBuilder(basePath).build(); + var serDes = stringCodec().then(FileSystemSerDes.builder(basePath).build()); assertThrows(SerDesException.class, () -> new SerDesRunner(null).serialize(serDes, "payload", context())); try (var files = Files.list(outside)) { @@ -445,14 +428,15 @@ void rejectsSymbolicLinkConfiguredBasePathAndAncestors() throws Exception { var linkedRoot = basePath.resolve("linked-root"); Files.createSymbolicLink(linkedRoot, outsideRoot); - var rootSerDes = FileSystemSerDes.stageBuilder(linkedRoot).build(); + var rootSerDes = stringCodec().then(FileSystemSerDes.builder(linkedRoot).build()); assertThrows(SerDesException.class, () -> new SerDesRunner(null).serialize(rootSerDes, "payload", context())); var outsideAncestor = Files.createTempDirectory(basePath.getParent(), "outside-ancestor-"); var linkedAncestor = basePath.resolve("linked-ancestor"); Files.createSymbolicLink(linkedAncestor, outsideAncestor); - var nestedSerDes = FileSystemSerDes.stageBuilder(linkedAncestor.resolve("payloads")) - .build(); + var nestedSerDes = stringCodec() + .then(FileSystemSerDes.builder(linkedAncestor.resolve("payloads")) + .build()); assertThrows(SerDesException.class, () -> new SerDesRunner(null).serialize(nestedSerDes, "payload", context())); assertFalse(Files.exists(outsideAncestor.resolve("payloads"))); @@ -460,7 +444,7 @@ void rejectsSymbolicLinkConfiguredBasePathAndAncestors() throws Exception { @Test void rejectsExecutionPathsOutsideConfiguredBasePath() { - var serDes = FileSystemSerDes.builder(basePath).build(); + var serDes = stringCodec().then(FileSystemSerDes.builder(basePath).build()); var unsafeContext = SerDesContext.forOperation( "arn:aws:lambda:us-east-1:123456789012:function:..:1/durable-execution/../..", "1", @@ -474,6 +458,33 @@ void rejectsExecutionPathsOutsideConfiguredBasePath() { assertThrows(SerDesException.class, () -> new SerDesRunner(null).serialize(serDes, "value", unsafeContext)); } + private static SerDes stringCodec() { + return new SerDes() { + @Override + public String serialize(Object value) { + return (String) value; + } + + @Override + @SuppressWarnings("unchecked") + public T deserialize(String data, TypeToken typeToken) { + if (!TypeToken.get(String.class).equals(typeToken)) { + throw new SerDesException("String codec cannot deserialize " + typeToken); + } + return (T) data; + } + }; + } + + private static void assertCauseMessage(Throwable failure, String expected) { + var current = failure; + while (current != null + && (current.getMessage() == null || !current.getMessage().contains(expected))) { + current = current.getCause(); + } + assertNotNull(current, "Expected exception chain to contain: " + expected); + } + private static String envelopeWithFile(String file) { try { return MAPPER.writeValueAsString(Map.of( diff --git a/sdk/src/test/java/software/amazon/lambda/durable/serde/RetrySerDesTest.java b/sdk/src/test/java/software/amazon/lambda/durable/serde/RetrySerDesTest.java index 1beefcee5..97e543033 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/serde/RetrySerDesTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/serde/RetrySerDesTest.java @@ -14,7 +14,6 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; import org.junit.jupiter.api.Test; -import software.amazon.lambda.durable.TypeToken; import software.amazon.lambda.durable.exception.RetryableSerDesException; import software.amazon.lambda.durable.exception.SerDesException; import software.amazon.lambda.durable.retry.RetryDecision; @@ -27,9 +26,9 @@ void retriesSerializationWithStrategyDelays() { var calls = new AtomicInteger(); var strategyAttempts = new ArrayList(); var delays = new ArrayList(); - var delegate = new SerDes() { + var delegate = new SerDesStage() { @Override - public String serialize(Object value) { + public String serialize(String value) { if (calls.incrementAndGet() < 3) { throw new RetryableSerDesException("transient"); } @@ -37,8 +36,8 @@ public String serialize(Object value) { } @Override - public T deserialize(String data, TypeToken typeToken) { - return null; + public String deserialize(String data) { + return data; } }; var retrySerDes = new RetrySerDes( @@ -58,36 +57,31 @@ public T deserialize(String data, TypeToken typeToken) { @Test void retriesDeserialization() { var calls = new AtomicInteger(); - var delegate = new JacksonSerDes() { + var delegate = new SerDesStage() { + @Override + public String serialize(String value) { + return value; + } + @Override - public T deserialize(String data, TypeToken typeToken) { + public String deserialize(String data) { if (calls.incrementAndGet() == 1) { throw new RetryableSerDesException("transient"); } - return super.deserialize(data, typeToken); + return data; } }; var retrySerDes = new RetrySerDes(delegate, (error, attempt) -> RetryDecision.retry(Duration.ZERO), delay -> {}); - assertEquals("value", retrySerDes.deserialize("\"value\"", TypeToken.get(String.class))); + assertEquals("value", retrySerDes.deserialize("value")); assertEquals(2, calls.get()); } @Test void retriesPipelineStageDeserializationAndDelegatesCapabilities() { var calls = new AtomicInteger(); - class RetryableStage implements SerDes, SerDesStage { - @Override - public String serialize(Object value) { - return value.toString(); - } - - @Override - public T deserialize(String data, TypeToken typeToken) { - return null; - } - + class RetryableStage implements SerDesStage { @Override public String serialize(String value) { return value; @@ -115,11 +109,6 @@ public boolean requiresDurableContext() { public boolean isTerminalPipelineStage() { return true; } - - @Override - public boolean isValueCodecOnly() { - return false; - } } var delegate = new RetryableStage(); var retrySerDes = @@ -132,22 +121,21 @@ public boolean isValueCodecOnly() { assertEquals(2, calls.get()); assertTrue(retrySerDes.requiresDurableContext()); assertTrue(retrySerDes.isTerminalPipelineStage()); - assertFalse(retrySerDes.isValueCodecOnly()); } @Test void doesNotRetryPermanentSerDesFailure() { var calls = new AtomicInteger(); - var delegate = new SerDes() { + var delegate = new SerDesStage() { @Override - public String serialize(Object value) { + public String serialize(String value) { calls.incrementAndGet(); throw new SerDesException("permanent"); } @Override - public T deserialize(String data, TypeToken typeToken) { - return null; + public String deserialize(String data) { + return data; } }; var retrySerDes = new RetrySerDes(delegate, (error, attempt) -> RetryDecision.retry(Duration.ZERO), delay -> { @@ -161,21 +149,22 @@ public T deserialize(String data, TypeToken typeToken) { @Test void rejectsInvalidConfigurationAndRetryDelay() { - var delegate = new JacksonSerDes(); + var delegate = identityStage(); assertThrows(NullPointerException.class, () -> new RetrySerDes(null, RetryStrategies.Presets.NO_RETRY)); assertThrows(NullPointerException.class, () -> new RetrySerDes(delegate, null)); + assertFalse(SerDes.class.isAssignableFrom(RetrySerDes.class)); var retrySerDes = new RetrySerDes( - new SerDes() { + new SerDesStage() { @Override - public String serialize(Object value) { + public String serialize(String value) { throw new RetryableSerDesException("transient"); } @Override - public T deserialize(String data, TypeToken typeToken) { - return null; + public String deserialize(String data) { + return data; } }, (error, attempt) -> RetryDecision.retry(Duration.ofSeconds(-1)), @@ -189,17 +178,17 @@ public T deserialize(String data, TypeToken typeToken) { void rethrowsLastRetryableFailureWhenRetriesAreExhausted() { var calls = new AtomicInteger(); var lastFailure = new AtomicReference(); - var delegate = new SerDes() { + var delegate = new SerDesStage() { @Override - public String serialize(Object value) { + public String serialize(String value) { var failure = new RetryableSerDesException("attempt-" + calls.incrementAndGet()); lastFailure.set(failure); throw failure; } @Override - public T deserialize(String data, TypeToken typeToken) { - return null; + public String deserialize(String data) { + return data; } }; var retrySerDes = new RetrySerDes(delegate, RetryStrategies.fixedDelay(2, Duration.ofSeconds(1)), delay -> {}); @@ -213,15 +202,15 @@ public T deserialize(String data, TypeToken typeToken) { @Test void restoresInterruptStatusWhenBackoffIsInterrupted() { var retryable = new RetryableSerDesException("transient"); - var delegate = new SerDes() { + var delegate = new SerDesStage() { @Override - public String serialize(Object value) { + public String serialize(String value) { throw retryable; } @Override - public T deserialize(String data, TypeToken typeToken) { - return null; + public String deserialize(String data) { + return data; } }; var retrySerDes = @@ -238,4 +227,18 @@ public T deserialize(String data, TypeToken typeToken) { Thread.interrupted(); } } + + private static SerDesStage identityStage() { + return new SerDesStage() { + @Override + public String serialize(String value) { + return value; + } + + @Override + public String deserialize(String data) { + return data; + } + }; + } } From 9d3f22edba979cc187e07e22f63cc17b1611d2fb Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Tue, 25 Aug 2026 18:31:49 +0000 Subject: [PATCH 28/56] refactor: allow stages after filesystem SerDes --- docs/adr/005-filesystem-serdes.md | 27 +++++++++---------- docs/advanced/configuration.md | 12 ++++++--- docs/advanced/filesystem-serdes.md | 12 ++++++--- .../durable/serde/ComposableSerDes.java | 20 -------------- .../durable/serde/FileSystemSerDes.java | 11 +++----- .../lambda/durable/serde/RetrySerDes.java | 5 ---- .../lambda/durable/serde/SerDesStage.java | 5 ---- .../durable/serde/ComposableSerDesTest.java | 27 ------------------- .../durable/serde/FileSystemSerDesTest.java | 23 +++++++++------- .../lambda/durable/serde/RetrySerDesTest.java | 8 +----- 10 files changed, 47 insertions(+), 103 deletions(-) diff --git a/docs/adr/005-filesystem-serdes.md b/docs/adr/005-filesystem-serdes.md index 8d7b48ba7..cc4a68e3b 100644 --- a/docs/adr/005-filesystem-serdes.md +++ b/docs/adr/005-filesystem-serdes.md @@ -78,9 +78,9 @@ transformation. This lets customers compose JSON encoding, framed string transfo filesystem storage without unsafe heterogeneous top-level stages. A `ComposableBinarySerDesStage` performs UTF-8, compression, encryption, and similar byte processing internally and encodes the result once at its string boundary. -`FileSystemSerDes` acts as a terminal payload-storage stage. It writes the string produced by the previous stage to -the filesystem when configured to do so and returns a small checkpoint envelope. It is only a `SerDesStage`; a value -codec must precede it in a `ComposableSerDes`. +`FileSystemSerDes` acts as a payload-storage stage. It writes the string produced by the previous stage to the +filesystem when configured to do so and returns a small checkpoint envelope. It is only a `SerDesStage`; a value codec +must precede it in a `ComposableSerDes`, and other string stages may follow it. Because the existing `SerDes` methods do not accept context parameters, this approach needs a thread-local `SerDesContext` so `FileSystemSerDes` can discover the current payload identity without changing the @@ -228,8 +228,6 @@ Pipeline rules: value codec and stages. - The test runners identify a configured input pipeline directly as `ComposableSerDes`; no generic value-codec-only capability is needed because stage decorators cannot wrap the root SerDes. -- Terminality belongs only to `SerDesStage`. A terminal stage must be last so later transformations cannot invalidate - its checkpoint-size or storage decision. - `SerDes.then(...)`, `ComposableSerDes.then(...)`, and the builder accept only `SerDesStage` after the value codec. This makes the root `SerDes` versus subsequent string-stage roles explicit in the Java type system. - If the value-codec argument to `ComposableSerDes.of(...)` or the builder is already a `ComposableSerDes`, its root @@ -249,8 +247,8 @@ Pipeline rules: through the configured pipeline. `ComposableSerDes` then skips every earlier intermediate stage and decodes the raw value directly with the value codec. - Stage order is meaningful. For example, `JSON -> binary composite -> filesystem` writes the encoded result of the - binary composite to the filesystem. `FileSystemSerDes` is terminal; placing any later transformation after it is - rejected. + binary composite to the filesystem, while `JSON -> filesystem -> signing envelope` signs the filesystem envelope + rather than the offloaded payload. - The ordered stage list and each stage's configuration are part of the persisted checkpoint format. They must remain replay-compatible for in-flight executions. Reordering, removing, or incompatibly reconfiguring a stage requires a versioned envelope or an explicit migration boundary. @@ -416,9 +414,10 @@ must be protected with the same care as the payload it references. The final file envelope, including any preview, must remain below the checkpoint threshold. Oversized previews are rejected rather than producing a checkpoint that the service cannot accept. -`FileSystemSerDes` declares itself terminal. This makes its overflow decision apply to the final checkpoint -representation and prevents a later encoding or other expanding stage from pushing an inline envelope over the -service limit. +Stages may follow `FileSystemSerDes` to transform its inline or file-reference envelope. Its overflow and preview-size +checks apply at the filesystem stage boundary, so configurations must account for any size expansion introduced by +later stages. At raw external payload boundaries, those later stages run first during reverse processing and must +tolerate or explicitly bypass data that has not passed through the configured pipeline. The preview generator receives the `String` produced by the preceding stage, not the original domain object. A preview that needs domain fields should parse that representation or be produced by an earlier stage. @@ -540,7 +539,7 @@ context-free `ComposableSerDes`, it serializes the invocation payload with the c compression, encryption, and other ordinary transformations remain compatible with the deployed function. When the persisted SerDes reports that it requires durable context, the cloud and local runners require a separate context-free input value codec via `withInputSerDes(...)`. That codec must not be a composable pipeline: -an unframed external payload does not identify which input stages ran, while a context-dependent terminal stage must +an unframed external payload does not identify which input stages ran, while a context-dependent filesystem stage must also accept raw service payloads such as callbacks and invoke results. Fluent configuration preserves that explicit input codec when other runner configuration is replaced. `LocalDurableTestRunner` must create the execution operation with this raw external payload and let `DurableExecutor` apply the persisted pipeline's external-boundary behavior; it @@ -552,7 +551,7 @@ must not synthesize a durable context and serialize initial input through the fu `SerDes` methods unchanged. 2. Add the binary-compatible `SerDes.then(SerDesStage)` default method and `ComposableSerDes` with one root `SerDes` followed only by immutable `SerDesStage` entries, forward serialization, reverse deserialization, - external-boundary bypass, terminal-stage validation, null short-circuiting, and stage-aware errors. + external-boundary bypass, null short-circuiting, and stage-aware errors. 3. Add the string-only `SerDesStage` contract plus `BinarySerDes`, `StringBinaryCodec`, and `ComposableBinarySerDesStage` for nested binary processing with ordered, configurable boundaries. 4. Add `RetryableSerDesException` and `RetrySerDes`, reusing `RetryStrategy` for bounded in-invocation retries. @@ -565,8 +564,8 @@ must not synthesize a durable context and serialize initial input through the fu 8. Add bounded, invocation-scoped deserialization caching keyed by SerDes identity, entity, payload kind, type, and serialized data hash. 9. Update exception serialization and deserialization paths to set `SerDesPayloadKind.EXCEPTION` in TLS. -10. Implement `FileSystemSerDes` in the core `software.amazon.lambda.durable.serde` package as a string terminal stage - with `ALWAYS` and `OVERFLOW` storage, `URI` and `HASH` path encodings, envelope parsing, atomic file writes where +10. Implement `FileSystemSerDes` in the core `software.amazon.lambda.durable.serde` package as a string stage with + `ALWAYS` and `OVERFLOW` storage, `URI` and `HASH` path encodings, envelope parsing, atomic file writes where supported by the filesystem, retryable I/O failures, and clear validation errors for missing context or invalid stage input. 11. Add unit tests for string and binary pipeline ordering, custom boundary codecs, reverse processing, nulls, stage diff --git a/docs/advanced/configuration.md b/docs/advanced/configuration.md index e19ba7743..e410f0265 100644 --- a/docs/advanced/configuration.md +++ b/docs/advanced/configuration.md @@ -53,7 +53,7 @@ and uses a bounded weak-reference cache for successful deserialization results d ### Filesystem-backed payload storage -The core SDK provides a reversible terminal stage for storing serialized strings on a shared filesystem: +The core SDK provides a reversible stage for storing serialized strings on a shared filesystem: ```java var fileSystemStage = FileSystemSerDes.builder(Path.of("/mnt/efs/durable-payloads")) @@ -75,7 +75,8 @@ var binaryStage = ComposableBinarySerDesStage.builder() var serDes = new JacksonSerDes() .then(binaryStage) - .then(resilientFileSystemStage); + .then(resilientFileSystemStage) + .then(checkpointEnvelopeStage); var serDesExecutor = Executors.newFixedThreadPool(4); return DurableConfig.builder() @@ -110,8 +111,11 @@ If the persisted SerDes requires durable context, such as `FileSystemSerDes`, ca `withInputSerDes(...)` with a separate context-free input value codec because the durable execution ARN does not exist yet. In that case, the input SerDes must be a value codec rather than a composable pipeline because the external payload does not carry framing that identifies which stages ran. -`FileSystemSerDes` must also be the final pipeline stage so its checkpoint-size decision cannot be invalidated by a -later expanding transformation. + +Stages may follow `FileSystemSerDes` to transform its inline or file-reference envelope. `OVERFLOW` and preview-size +checks apply to the filesystem stage's output; account for any expansion introduced by later stages when staying +within the service checkpoint limit. At external payload boundaries, later stages run before `FileSystemSerDes` during +deserialization, so they must tolerate or explicitly bypass raw payloads that have not passed through the pipeline. ### Dynamic plugin loading diff --git a/docs/advanced/filesystem-serdes.md b/docs/advanced/filesystem-serdes.md index 0c0bd31b0..ac4810177 100644 --- a/docs/advanced/filesystem-serdes.md +++ b/docs/advanced/filesystem-serdes.md @@ -15,7 +15,7 @@ envelopes in checkpoints. It is included in the core `aws-durable-execution-sdk- ## Pipeline configuration -`FileSystemSerDes` is a reversible terminal `SerDesStage` and must be configured after a value codec: +`FileSystemSerDes` is a reversible `SerDesStage` that must be configured after a value codec: ```java var fileSystemStage = FileSystemSerDes.builder(Path.of("/mnt/efs/durable-payloads")) @@ -37,7 +37,8 @@ var binaryStage = ComposableBinarySerDesStage.builder() var serDes = new JacksonSerDes() .then(binaryStage) - .then(resilientFileSystemStage); + .then(resilientFileSystemStage) + .then(checkpointEnvelopeStage); var serDesExecutor = Executors.newFixedThreadPool(4); return DurableConfig.builder() @@ -86,8 +87,11 @@ owner, while invoke input and result boundaries may consume a file owned by the functions use the same shared root and path encoding. Treat file envelopes as capabilities. Content hashes are verified when reading, and symbolic-link paths are rejected. -`FileSystemSerDes` must be the final stage in a pipeline. Its overflow decision is therefore made against the final -checkpoint representation; a later expanding transform cannot push an inline envelope over the service limit. +Stages may follow `FileSystemSerDes` to transform its inline or file-reference envelope. The filesystem stage's +`OVERFLOW` and preview-size checks apply before those later transformations, so account for any expansion when staying +within the service checkpoint limit. During deserialization of a raw external payload, later stages run before +`FileSystemSerDes` can identify the external boundary; those stages must tolerate or explicitly bypass payloads that +have not passed through the pipeline. The cloud and local test runners cannot use `FileSystemSerDes` for the initial Lambda invocation because an execution ARN is not available yet. Configure a separate context-free initial-input value codec with diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/ComposableSerDes.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/ComposableSerDes.java index 6433b9cd3..b8857ad01 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/ComposableSerDes.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/ComposableSerDes.java @@ -21,16 +21,6 @@ public final class ComposableSerDes implements SerDes { private ComposableSerDes(SerDes valueCodec, List stages) { this.valueCodec = Objects.requireNonNull(valueCodec, "valueCodec cannot be null"); - if (!stages.isEmpty() - && valueCodec instanceof SerDesStage valueCodecStage - && valueCodecStage.isTerminalPipelineStage()) { - throw terminalStageFailure(0, valueCodec); - } - for (int index = 0; index < stages.size() - 1; index++) { - if (isTerminal(stages.get(index))) { - throw terminalStageFailure(index + 1, stages.get(index)); - } - } this.stages = List.copyOf(stages); } @@ -169,20 +159,10 @@ private static RuntimeException stageFailure(int index, Object stage, String act return new SerDesException(message, failure); } - private static IllegalArgumentException terminalStageFailure(int index, Object stage) { - return new IllegalArgumentException(String.format( - "SerDes pipeline stage %d (%s) must be the final stage", - index, stage.getClass().getName())); - } - private static boolean requiresContext(SerDesStage stage) { return stage.requiresDurableContext(); } - private static boolean isTerminal(SerDesStage stage) { - return stage.isTerminalPipelineStage(); - } - /** Builder for an immutable {@link ComposableSerDes}. */ public static final class Builder { private SerDes valueCodec; diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java index cada25190..fcd0a8000 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java @@ -25,7 +25,7 @@ import software.amazon.lambda.durable.exception.SerDesException; /** - * A terminal string stage that stores payloads on a durable shared filesystem. + * A string stage that stores payloads on a durable shared filesystem. * *

Do not use Lambda's ephemeral {@code /tmp} storage. Use a durable shared mount such as EFS, or S3 Files only when * its synchronization and crash-durability tradeoffs are acceptable for the workload. @@ -53,12 +53,12 @@ private FileSystemSerDes(Builder builder) { } /** - * Creates a filesystem terminal-stage builder for use after a value codec in a composable SerDes pipeline. + * Creates a filesystem stage builder for use after a value codec in a composable SerDes pipeline. * *

Use {@link ComposableBinarySerDesStage} before this stage when binary transformations are required. * * @param basePath durable shared filesystem root - * @return a terminal-stage builder + * @return a filesystem stage builder */ public static Builder builder(Path basePath) { return new Builder(basePath); @@ -118,11 +118,6 @@ public boolean requiresDurableContext() { return true; } - @Override - public boolean isTerminalPipelineStage() { - return true; - } - private ResolvedPayload resolveSerializedPayload(String data, SerDesContext context) { final JsonNode envelope; try { diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/RetrySerDes.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/RetrySerDes.java index 197726aff..e69815422 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/RetrySerDes.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/RetrySerDes.java @@ -67,11 +67,6 @@ public boolean requiresDurableContext() { return delegate.requiresDurableContext(); } - @Override - public boolean isTerminalPipelineStage() { - return delegate.isTerminalPipelineStage(); - } - private T execute(String action, Supplier operation) { int attempt = 1; while (true) { diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesStage.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesStage.java index 43a91a058..856eabd4b 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesStage.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesStage.java @@ -44,9 +44,4 @@ default SerDesStageResult deserializePipelineStage(String data) { default boolean requiresDurableContext() { return false; } - - /** Returns whether this stage must be the final stage in a composable pipeline. */ - default boolean isTerminalPipelineStage() { - return false; - } } diff --git a/sdk/src/test/java/software/amazon/lambda/durable/serde/ComposableSerDesTest.java b/sdk/src/test/java/software/amazon/lambda/durable/serde/ComposableSerDesTest.java index 9e3607a42..db6ed1ea0 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/serde/ComposableSerDesTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/serde/ComposableSerDesTest.java @@ -174,33 +174,6 @@ public SerDesStageResult deserializePipelineStage(String data) { assertEquals(0, transformDeserializations.get()); } - @Test - void rejectsStagesAfterTerminalStage() { - var terminal = new SerDesStage() { - @Override - public String serialize(String value) { - return value; - } - - @Override - public String deserialize(String data) { - return data; - } - - @Override - public boolean isTerminalPipelineStage() { - return true; - } - }; - - var failure = assertThrows( - IllegalArgumentException.class, - () -> new JacksonSerDes().then(terminal).then(stringStage("late", "", "", new ArrayList<>()))); - - assertTrue(failure.getMessage().contains("stage 1")); - assertTrue(failure.getMessage().contains("final stage")); - } - @Test void rejectsNullIntermediateValues() { var nullStage = new SerDesStage() { diff --git a/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java b/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java index 9f254c5e8..ab08cb204 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java @@ -57,14 +57,18 @@ void writesValueCodecPayloadAndReplaysIt() throws Exception { } @Test - void isAContextDependentTerminalStage() { + void isAContextDependentStageThatCanBeFollowedByOtherStages() { var stage = FileSystemSerDes.builder(basePath).build(); - var pipeline = new JacksonSerDes().then(stage); + var pipeline = new JacksonSerDes().then(stage).then(wrappingStage()); + var runner = new SerDesRunner(null); assertFalse(SerDes.class.isAssignableFrom(FileSystemSerDes.class)); assertTrue(stage.requiresDurableContext()); - assertTrue(stage.isTerminalPipelineStage()); - assertThrows(IllegalArgumentException.class, () -> pipeline.then(wrappingStage())); + var checkpoint = runner.serialize(pipeline, Map.of("id", 42), context()); + assertTrue(checkpoint.startsWith("<")); + assertEquals( + Map.of("id", 42), + runner.deserialize(pipeline, checkpoint, new TypeToken>() {}, context())); } @Test @@ -311,16 +315,17 @@ void rejectsOutOfRangeEnvelopeVersionsWithoutTruncation() { } @Test - void overflowFilesystemStageMustRemainTerminal() { + void overflowFilesystemStageCanBeFollowedByAnotherStage() { var filesystem = FileSystemSerDes.builder(basePath) .storageMode(FileSystemStorageMode.OVERFLOW) .build(); + var pipeline = stringCodec().then(filesystem).then(wrappingStage()); + var runner = new SerDesRunner(null); - var failure = assertThrows( - IllegalArgumentException.class, - () -> new JacksonSerDes().then(filesystem).then(wrappingStage())); + var checkpoint = runner.serialize(pipeline, "small", context()); - assertTrue(failure.getMessage().contains("final stage")); + assertTrue(checkpoint.startsWith("<")); + assertEquals("small", runner.deserialize(pipeline, checkpoint, TypeToken.get(String.class), context())); } @Test diff --git a/sdk/src/test/java/software/amazon/lambda/durable/serde/RetrySerDesTest.java b/sdk/src/test/java/software/amazon/lambda/durable/serde/RetrySerDesTest.java index 97e543033..ae2191d38 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/serde/RetrySerDesTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/serde/RetrySerDesTest.java @@ -79,7 +79,7 @@ public String deserialize(String data) { } @Test - void retriesPipelineStageDeserializationAndDelegatesCapabilities() { + void retriesPipelineStageDeserializationAndDelegatesContextCapability() { var calls = new AtomicInteger(); class RetryableStage implements SerDesStage { @Override @@ -104,11 +104,6 @@ public SerDesStageResult deserializePipelineStage(String data) { public boolean requiresDurableContext() { return true; } - - @Override - public boolean isTerminalPipelineStage() { - return true; - } } var delegate = new RetryableStage(); var retrySerDes = @@ -120,7 +115,6 @@ public boolean isTerminalPipelineStage() { assertTrue(result.skipRemainingStages()); assertEquals(2, calls.get()); assertTrue(retrySerDes.requiresDurableContext()); - assertTrue(retrySerDes.isTerminalPipelineStage()); } @Test From 7e3f005cc41117867016dd6ec47d3195f9e7d8f5 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Tue, 25 Aug 2026 18:39:29 +0000 Subject: [PATCH 29/56] refactor: separate test runner input codecs --- docs/adr/005-filesystem-serdes.md | 29 +++-- docs/advanced/configuration.md | 10 +- docs/advanced/filesystem-serdes.md | 10 +- .../FileSystemSerDesIntegrationTest.java | 7 -- .../testing/CloudDurableTestRunner.java | 30 ++--- .../testing/LocalDurableTestRunner.java | 29 ++--- .../testing/CloudDurableTestRunnerTest.java | 113 +++++------------- .../testing/LocalDurableTestRunnerTest.java | 24 ++-- .../lambda/durable/serde/BinarySerDes.java | 5 - .../serde/ComposableBinarySerDesStage.java | 7 -- .../durable/serde/ComposableSerDes.java | 9 -- .../durable/serde/FileSystemSerDes.java | 5 - .../lambda/durable/serde/RetrySerDes.java | 5 - .../amazon/lambda/durable/serde/SerDes.java | 10 -- .../lambda/durable/serde/SerDesStage.java | 5 - .../durable/serde/StringBinaryCodec.java | 5 - .../ComposableBinarySerDesStageTest.java | 27 ----- .../durable/serde/FileSystemSerDesTest.java | 3 +- .../lambda/durable/serde/RetrySerDesTest.java | 8 +- 19 files changed, 96 insertions(+), 245 deletions(-) diff --git a/docs/adr/005-filesystem-serdes.md b/docs/adr/005-filesystem-serdes.md index cc4a68e3b..475754b8f 100644 --- a/docs/adr/005-filesystem-serdes.md +++ b/docs/adr/005-filesystem-serdes.md @@ -223,11 +223,11 @@ Pipeline rules: - A pipeline must contain exactly one value codec in the first position and zero or more string stages. - Every top-level stage consumes and produces a non-null `String`. This makes all stage orderings type-compatible. -- `SerDes.requiresDurableContext()` remains a root capability because the SDK and test runners must validate the - configured SerDes before an initial durable context exists. `ComposableSerDes` aggregates this capability from its - value codec and stages. -- The test runners identify a configured input pipeline directly as `ComposableSerDes`; no generic value-codec-only - capability is needed because stage decorators cannot wrap the root SerDes. +- Context requirements are stage behavior rather than a capability on `SerDes`, `SerDesStage`, binary transformations, + or codecs. A context-dependent stage accesses `SerDesContext` when it runs and reports a normal SerDes failure when + the required context is unavailable. +- The test runners identify a configured input pipeline directly as `ComposableSerDes` and reject it because initial + input accepts one value codec, not a pipeline. - `SerDes.then(...)`, `ComposableSerDes.then(...)`, and the builder accept only `SerDesStage` after the value codec. This makes the root `SerDes` versus subsequent string-stage roles explicit in the Java type system. - If the value-codec argument to `ComposableSerDes.of(...)` or the builder is already a `ComposableSerDes`, its root @@ -534,16 +534,15 @@ When serializing `errorData`, set `SerDesPayloadKind.EXCEPTION` and use an entit Root user input and output payloads should route through `SerDesRunner` so `FileSystemSerDes` can see `SerDesContext`. The internal `DurableExecutionInput` and `DurableExecutionOutput` envelope stays with `DurableInputOutputSerDes`. -The cloud test runner must send initial Lambda input before it receives a durable execution ARN. When configured with a -context-free `ComposableSerDes`, it serializes the invocation payload with the complete configured pipeline so -compression, encryption, and other ordinary transformations remain compatible with the deployed function. When the -persisted SerDes reports that it requires durable context, the cloud and local runners require a separate context-free -input value codec via `withInputSerDes(...)`. That codec must not be a composable pipeline: -an unframed external payload does not identify which input stages ran, while a context-dependent filesystem stage must -also accept raw service payloads such as callbacks and invoke results. Fluent configuration preserves that explicit -input codec when other runner configuration is replaced. `LocalDurableTestRunner` must create the execution operation -with this raw external payload and let `DurableExecutor` apply the persisted pipeline's external-boundary behavior; it -must not synthesize a durable context and serialize initial input through the full persisted pipeline. +The cloud test runner must send initial Lambda input before it receives a durable execution ARN. The cloud and local +runners therefore always serialize that input with a separate context-free value codec, defaulting to +`JacksonSerDes`. `withInputSerDes(...)` replaces this codec, but rejects `ComposableSerDes`: the runners must never +reuse the persisted composable pipeline for the initial invocation, and an unframed external payload does not identify +which stages ran. A custom input codec must produce data that the persisted pipeline's root value codec can decode at +the external-input boundary. Fluent configuration preserves the input codec when other runner configuration is +replaced. `LocalDurableTestRunner` creates the execution operation with this raw external payload and lets +`DurableExecutor` apply the persisted pipeline's external-boundary behavior; it does not synthesize a durable context +or serialize initial input through the persisted pipeline. ### Implementation plan diff --git a/docs/advanced/configuration.md b/docs/advanced/configuration.md index e410f0265..f379f9faa 100644 --- a/docs/advanced/configuration.md +++ b/docs/advanced/configuration.md @@ -106,11 +106,11 @@ shared mount such as EFS. S3 Files can have delayed synchronization, so a runtim lose recent writes; use it only when that tradeoff is acceptable. The SDK does not delete offloaded files, so configure storage lifecycle and retention separately. -For a context-free `ComposableSerDes`, the test runners apply the complete pipeline to the initial Lambda invocation. -If the persisted SerDes requires durable context, such as `FileSystemSerDes`, call -`withInputSerDes(...)` with a separate context-free input value codec because the durable execution ARN does not exist -yet. In that case, the input SerDes must be a value codec rather than a composable pipeline because the external -payload does not carry framing that identifies which stages ran. +The cloud and local test runners always serialize the initial Lambda invocation with a separate context-free value +codec. This codec defaults to `JacksonSerDes` and can be replaced with `withInputSerDes(...)`. It is independent of the +SerDes used for persisted execution payloads: the runners never apply that persisted composable pipeline to the initial +invocation. The input SerDes must therefore be a value codec rather than a `ComposableSerDes`. A custom input codec must +produce data that the persisted pipeline's root value codec can decode at the external-input boundary. Stages may follow `FileSystemSerDes` to transform its inline or file-reference envelope. `OVERFLOW` and preview-size checks apply to the filesystem stage's output; account for any expansion introduced by later stages when staying diff --git a/docs/advanced/filesystem-serdes.md b/docs/advanced/filesystem-serdes.md index ac4810177..d5b0c8aba 100644 --- a/docs/advanced/filesystem-serdes.md +++ b/docs/advanced/filesystem-serdes.md @@ -93,11 +93,11 @@ within the service checkpoint limit. During deserialization of a raw external pa `FileSystemSerDes` can identify the external boundary; those stages must tolerate or explicitly bypass payloads that have not passed through the pipeline. -The cloud and local test runners cannot use `FileSystemSerDes` for the initial Lambda invocation because an execution -ARN is not available yet. Configure a separate context-free initial-input value codec with -`withInputSerDes(...)`. Do not use a composable pipeline for that input boundary: the unframed external payload does -not identify which stages ran before the context-dependent filesystem stage. Context-free -persisted pipelines still use their complete pipeline for the initial invocation. +The cloud and local test runners always use a separate context-free value codec for the initial Lambda invocation +because an execution ARN is not available yet. The input codec defaults to `JacksonSerDes` and can be replaced with +`withInputSerDes(...)`. The runners never reuse the persisted composable pipeline for this boundary. Do not configure a +`ComposableSerDes` as the input codec: the unframed external payload does not identify which stages ran before the +filesystem stage. A custom input codec must produce data that the persisted pipeline's root value codec can decode. ## Storage requirements diff --git a/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FileSystemSerDesIntegrationTest.java b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FileSystemSerDesIntegrationTest.java index 57c648120..75e249590 100644 --- a/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FileSystemSerDesIntegrationTest.java +++ b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FileSystemSerDesIntegrationTest.java @@ -93,7 +93,6 @@ void pipelineReplaysStepWaitChildAndMapPayloadsFromFilesystem() throws Exception return childResult + "-" + pollResult + "-" + mapResult.results(); }, config) - .withInputSerDes(new JacksonSerDes()) .withOutputType(String.class); var result = runner.runUntilComplete("order"); @@ -175,7 +174,6 @@ void acceptsRawCallbackAndInvokeResultsAndOffloadsInvokePayload() throws Excepti "notify", "target-function", Map.of("approval", approval), String.class); }, config) - .withInputSerDes(new JacksonSerDes()) .withOutputType(String.class); var waitingForCallback = runner.run("input"); @@ -295,7 +293,6 @@ void repeatedGetUsesInvocationCacheForTheCompletePipeline() { return first.value(); }, config) - .withInputSerDes(new JacksonSerDes()) .withOutputType(String.class); var result = runner.runUntilComplete("cached"); @@ -337,7 +334,6 @@ void successfulRetryUsesTheProducingAttemptForResultSerialization() { }, stepConfig), config) - .withInputSerDes(new JacksonSerDes()) .withOutputType(String.class); var result = runner.runUntilComplete("value"); @@ -364,7 +360,6 @@ void customExceptionPayloadsRoundTripThroughFilesystem() throws Exception { }, stepConfig), config) - .withInputSerDes(new JacksonSerDes()) .withOutputType(String.class); var result = runner.runUntilComplete("input"); @@ -395,7 +390,6 @@ void nestedInvokeFailurePreservesProducerContextAcrossReplay() throws Exception } }, config) - .withInputSerDes(new JacksonSerDes()) .withOutputType(String.class); assertEquals(ExecutionStatus.PENDING, runner.run("input").getStatus()); @@ -446,7 +440,6 @@ void nestedCallbackFailurePreservesProducerContextAcrossReplay() throws Exceptio } }, config) - .withInputSerDes(new JacksonSerDes()) .withOutputType(String.class); assertEquals(ExecutionStatus.PENDING, runner.run("input").getStatus()); diff --git a/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/CloudDurableTestRunner.java b/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/CloudDurableTestRunner.java index 63775a13b..69c056db4 100644 --- a/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/CloudDurableTestRunner.java +++ b/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/CloudDurableTestRunner.java @@ -56,7 +56,7 @@ private CloudDurableTestRunner( this.timeout = timeout; this.invocationType = invocationType; this.serDes = Objects.requireNonNullElseGet(serDes, JacksonSerDes::new); - this.inputSerDes = inputSerDes; + this.inputSerDes = inputValueCodec(inputSerDes); } private static LambdaClient createDefaultLambdaClient() { @@ -170,13 +170,10 @@ public CloudDurableTestRunner withSerDes(SerDes serDes) { } /** - * Returns a new runner with a separate SerDes for the initial Lambda invocation payload. + * Returns a new runner with a separate value codec for the initial Lambda invocation payload. * - *

The supplied SerDes is used exactly as configured, including every stage in a composable pipeline. Configure a - * separate context-free input SerDes when the persisted SerDes requires a durable execution context, because that - * context does not exist before the initial Lambda invocation. In that case the input SerDes must be a value codec, - * not a composable pipeline, because context-dependent persisted stages cannot distinguish which input stages - * produced an unframed external payload. + *

The input codec is independent of the SerDes used for persisted execution payloads and must not be a + * {@link ComposableSerDes}. The default is {@link JacksonSerDes}. */ public CloudDurableTestRunner withInputSerDes(SerDes inputSerDes) { return new CloudDurableTestRunner<>( @@ -280,17 +277,14 @@ public TestOperation getOperation(String name) { } private String serializeInput(I input) { - var serializer = inputSerDes != null ? inputSerDes : serDes; - if (serializer.requiresDurableContext()) { - throw new IllegalStateException( - "Initial input SerDes requires a durable execution context; configure a context-free " - + "input SerDes with withInputSerDes(...)"); - } - if (serializer instanceof ComposableSerDes && serDes.requiresDurableContext()) { - throw new IllegalStateException( - "Initial input for a context-dependent persisted SerDes must use a value codec, not a composable " - + "pipeline"); + return inputSerDes.serialize(input); + } + + private static SerDes inputValueCodec(SerDes inputSerDes) { + var codec = Objects.requireNonNullElseGet(inputSerDes, JacksonSerDes::new); + if (codec instanceof ComposableSerDes) { + throw new IllegalArgumentException("inputSerDes must be a value codec, not a composable pipeline"); } - return serializer.serialize(input); + return codec; } } diff --git a/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/LocalDurableTestRunner.java b/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/LocalDurableTestRunner.java index c055b4123..c7c03a0ba 100644 --- a/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/LocalDurableTestRunner.java +++ b/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/LocalDurableTestRunner.java @@ -24,6 +24,7 @@ import software.amazon.lambda.durable.model.ExecutionStatus; import software.amazon.lambda.durable.plugin.DurableExecutionPlugin; import software.amazon.lambda.durable.serde.ComposableSerDes; +import software.amazon.lambda.durable.serde.JacksonSerDes; import software.amazon.lambda.durable.serde.SerDes; import software.amazon.lambda.durable.serde.SerDesRunner; import software.amazon.lambda.durable.testing.local.LocalMemoryExecutionClient; @@ -62,7 +63,7 @@ private LocalDurableTestRunner( this.inputType = inputType; this.outputType = outputType; this.handler = handlerFn; - this.inputSerDes = inputSerDes; + this.inputSerDes = inputValueCodec(inputSerDes); this.storage = new LocalMemoryExecutionClient(); // Create config that uses customer's configuration but overrides the client with in-memory storage @@ -216,11 +217,10 @@ public LocalDurableTestRunner withOutputType(Class outputType) { } /** - * Returns a new runner with a separate SerDes for the initial Lambda invocation payload. + * Returns a new runner with a separate value codec for the initial Lambda invocation payload. * - *

Configure a context-free input value codec when the persisted SerDes requires durable context. This preserves - * production behavior: the initial external payload is decoded directly by the value codec, while the persisted - * pipeline processes payloads after the durable execution context exists. + *

The input codec is independent of the SerDes used for persisted execution payloads and must not be a + * {@link ComposableSerDes}. The default is {@link JacksonSerDes}. */ public LocalDurableTestRunner withInputSerDes(SerDes inputSerDes) { return new LocalDurableTestRunner<>( @@ -400,18 +400,15 @@ private DurableExecutionInput createDurableInput(I input) { } private String serializeInput(I input) { - var serializer = inputSerDes != null ? inputSerDes : serDes; - if (serializer.requiresDurableContext()) { - throw new IllegalStateException( - "Initial input SerDes requires a durable execution context; configure a context-free " - + "input SerDes with withInputSerDes(...)"); - } - if (serializer instanceof ComposableSerDes && serDes.requiresDurableContext()) { - throw new IllegalStateException( - "Initial input for a context-dependent persisted SerDes must use a value codec, not a composable " - + "pipeline"); + return inputSerDes.serialize(input); + } + + private static SerDes inputValueCodec(SerDes inputSerDes) { + var codec = Objects.requireNonNullElseGet(inputSerDes, JacksonSerDes::new); + if (codec instanceof ComposableSerDes) { + throw new IllegalArgumentException("inputSerDes must be a value codec, not a composable pipeline"); } - return serializer.serialize(input); + return codec; } private Context mockLambdaContext() { diff --git a/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/CloudDurableTestRunnerTest.java b/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/CloudDurableTestRunnerTest.java index 93d12f73d..9ffa51cdb 100644 --- a/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/CloudDurableTestRunnerTest.java +++ b/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/CloudDurableTestRunnerTest.java @@ -46,26 +46,20 @@ void testPlaceholderMethods() { } @Test - void explicitComposableInputSerDesUsesTheCompletePipeline() { + void rejectsComposableInputSerDes() { var mockClient = mock(LambdaClient.class); - when(mockClient.invoke(any(InvokeRequest.class))) - .thenReturn(InvokeResponse.builder() - .durableExecutionArn("arn:aws:lambda:us-east-2:123:function:test:1/durable-execution/e/i") - .build()); - var wrappingStage = wrappingStage(); var runner = CloudDurableTestRunner.create( - "arn:aws:lambda:us-east-2:123:function:test", String.class, String.class, mockClient) - .withInputSerDes(new JacksonSerDes().then(wrappingStage)); + "arn:aws:lambda:us-east-2:123:function:test", String.class, String.class, mockClient); - runner.startAsync("value"); + var failure = assertThrows( + IllegalArgumentException.class, + () -> runner.withInputSerDes(new JacksonSerDes().then(wrappingStage()))); - var request = ArgumentCaptor.forClass(InvokeRequest.class); - verify(mockClient).invoke(request.capture()); - assertEquals("<\"value\">", request.getValue().payload().asUtf8String()); + assertTrue(failure.getMessage().contains("value codec")); } @Test - void persistedComposableSerDesUsesCompletePipelineForDefaultInput() { + void persistedComposableSerDesUsesDefaultJacksonInputCodec() { var mockClient = mock(LambdaClient.class); when(mockClient.invoke(any(InvokeRequest.class))) .thenReturn(InvokeResponse.builder() @@ -79,50 +73,25 @@ void persistedComposableSerDesUsesCompletePipelineForDefaultInput() { var request = ArgumentCaptor.forClass(InvokeRequest.class); verify(mockClient).invoke(request.capture()); - assertEquals("<\"value\">", request.getValue().payload().asUtf8String()); + assertEquals("\"value\"", request.getValue().payload().asUtf8String()); } @Test - void contextDependentPersistedSerDesRequiresExplicitInputSerDes() { - var mockClient = mock(LambdaClient.class); - var runner = CloudDurableTestRunner.create( - "arn:aws:lambda:us-east-2:123:function:test", String.class, String.class, mockClient) - .withSerDes(new JacksonSerDes().then(contextDependentStage())); - - var failure = assertThrows(RuntimeException.class, () -> runner.startAsync("value")); - - assertInstanceOf(IllegalStateException.class, failure.getCause()); - assertTrue(failure.getCause().getMessage().contains("withInputSerDes")); - verifyNoInteractions(mockClient); - } - - @Test - void explicitInputSerDesMustBeContextFree() { - var mockClient = mock(LambdaClient.class); - var runner = CloudDurableTestRunner.create( - "arn:aws:lambda:us-east-2:123:function:test", String.class, String.class, mockClient) - .withInputSerDes(contextDependentStage()); - - var failure = assertThrows(RuntimeException.class, () -> runner.startAsync("value")); - - assertInstanceOf(IllegalStateException.class, failure.getCause()); - assertTrue(failure.getCause().getMessage().contains("Initial input SerDes")); - verifyNoInteractions(mockClient); - } - - @Test - void contextDependentPersistedSerDesRequiresValueCodecInput() { + void explicitInputValueCodecIsUsed() { var mockClient = mock(LambdaClient.class); + when(mockClient.invoke(any(InvokeRequest.class))) + .thenReturn(InvokeResponse.builder() + .durableExecutionArn("arn:aws:lambda:us-east-2:123:function:test:1/durable-execution/e/i") + .build()); var runner = CloudDurableTestRunner.create( "arn:aws:lambda:us-east-2:123:function:test", String.class, String.class, mockClient) - .withSerDes(new JacksonSerDes().then(contextDependentStage())) - .withInputSerDes(new JacksonSerDes().then(wrappingStage())); + .withInputSerDes(wrappingValueCodec()); - var failure = assertThrows(RuntimeException.class, () -> runner.startAsync("value")); + runner.startAsync("value"); - assertInstanceOf(IllegalStateException.class, failure.getCause()); - assertTrue(failure.getCause().getMessage().contains("must use a value codec")); - verifyNoInteractions(mockClient); + var request = ArgumentCaptor.forClass(InvokeRequest.class); + verify(mockClient).invoke(request.capture()); + assertEquals("<\"value\">", request.getValue().payload().asUtf8String()); } @Test @@ -137,8 +106,7 @@ void valueCodecInputRoundTripsThroughFileSystemPipeline(@TempDir Path basePath) new JacksonSerDes().then(FileSystemSerDes.builder(basePath).build()); var runner = CloudDurableTestRunner.create( "arn:aws:lambda:us-east-2:123:function:test", String.class, String.class, mockClient) - .withSerDes(persistedSerDes) - .withInputSerDes(new JacksonSerDes()); + .withSerDes(persistedSerDes); runner.startAsync("value"); @@ -163,7 +131,7 @@ void replacingPersistedSerDesPreservesExplicitInputSerDes() { .build()); var runner = CloudDurableTestRunner.create( "arn:aws:lambda:us-east-2:123:function:test", String.class, String.class, mockClient) - .withInputSerDes(new JacksonSerDes().then(wrappingStage())) + .withInputSerDes(wrappingValueCodec()) .withSerDes(new JacksonSerDes()); runner.startAsync("value"); @@ -187,35 +155,18 @@ public String deserialize(String data) { }; } - private static ContextDependentSerDesStage contextDependentStage() { - return new ContextDependentSerDesStage(); - } + private static SerDes wrappingValueCodec() { + var delegate = new JacksonSerDes(); + return new SerDes() { + @Override + public String serialize(Object value) { + return "<" + delegate.serialize(value) + ">"; + } - private static final class ContextDependentSerDesStage implements SerDes, SerDesStage { - @Override - public String serialize(Object value) { - return value.toString(); - } - - @Override - public String serialize(String value) { - return value; - } - - @Override - @SuppressWarnings("unchecked") - public T deserialize(String data, TypeToken typeToken) { - return (T) data; - } - - @Override - public String deserialize(String data) { - return data; - } - - @Override - public boolean requiresDurableContext() { - return true; - } + @Override + public T deserialize(String data, TypeToken typeToken) { + return delegate.deserialize(data.substring(1, data.length() - 1), typeToken); + } + }; } } diff --git a/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/LocalDurableTestRunnerTest.java b/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/LocalDurableTestRunnerTest.java index edc0ebdd5..7b4385269 100644 --- a/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/LocalDurableTestRunnerTest.java +++ b/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/LocalDurableTestRunnerTest.java @@ -149,34 +149,37 @@ void checkpointedLargeOutputReplaysWithoutDuplicateExecutionOperation() { } @Test - void contextDependentPersistedSerDesRequiresExplicitInputSerDes(@TempDir Path basePath) { + void filesystemPersistedSerDesUsesDefaultJacksonInputCodec(@TempDir Path basePath) { var config = DurableConfig.builder() .withSerDes(new JacksonSerDes() .then(FileSystemSerDes.builder(basePath).build())) .build(); - var runner = LocalDurableTestRunner.create(String.class, (input, context) -> input, config); + var runner = LocalDurableTestRunner.create(String.class, (input, context) -> input, config) + .withOutputType(String.class); - var failure = assertThrows(IllegalStateException.class, () -> runner.run("value")); + var result = runner.run("value"); - assertTrue(failure.getMessage().contains("withInputSerDes")); + assertEquals(ExecutionStatus.SUCCEEDED, result.getStatus()); + assertEquals("value", result.getResult()); } @Test - void contextDependentPersistedSerDesRequiresValueCodecInput(@TempDir Path basePath) { + void rejectsComposableInputSerDes(@TempDir Path basePath) { var config = DurableConfig.builder() .withSerDes(new JacksonSerDes() .then(FileSystemSerDes.builder(basePath).build())) .build(); - var runner = LocalDurableTestRunner.create(String.class, (input, context) -> input, config) - .withInputSerDes(new JacksonSerDes().then(wrappingStage(new AtomicInteger()))); + var runner = LocalDurableTestRunner.create(String.class, (input, context) -> input, config); - var failure = assertThrows(IllegalStateException.class, () -> runner.run("value")); + var failure = assertThrows( + IllegalArgumentException.class, + () -> runner.withInputSerDes(new JacksonSerDes().then(wrappingStage(new AtomicInteger())))); - assertTrue(failure.getMessage().contains("must use a value codec")); + assertTrue(failure.getMessage().contains("value codec")); } @Test - void initialInputBypassesStagesBeforeFileSystemSerDes(@TempDir Path basePath) { + void defaultInputCodecBypassesPersistedStagesBeforeFileSystemSerDes(@TempDir Path basePath) { var deserializeCalls = new AtomicInteger(); var persistedSerDes = new JacksonSerDes() .then(bytesStage(deserializeCalls)) @@ -184,7 +187,6 @@ void initialInputBypassesStagesBeforeFileSystemSerDes(@TempDir Path basePath) { var config = DurableConfig.builder().withSerDes(persistedSerDes).build(); var runner = LocalDurableTestRunner.create( String.class, (input, context) -> input + ":" + deserializeCalls.get(), config) - .withInputSerDes(new JacksonSerDes()) .withOutputType(String.class); var result = runner.run("value"); diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/BinarySerDes.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/BinarySerDes.java index 350057ee1..278678166 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/BinarySerDes.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/BinarySerDes.java @@ -24,9 +24,4 @@ public interface BinarySerDes { * @return the non-null bytes expected by the preceding transformation */ byte[] deserialize(byte[] data); - - /** Returns whether this transformation requires an SDK-managed durable execution context. */ - default boolean requiresDurableContext() { - return false; - } } diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/ComposableBinarySerDesStage.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/ComposableBinarySerDesStage.java index b096afdd8..388dfb611 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/ComposableBinarySerDesStage.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/ComposableBinarySerDesStage.java @@ -51,13 +51,6 @@ public String deserialize(String data) { return invokeFromBytes(startingCodec, current, "starting codec"); } - @Override - public boolean requiresDurableContext() { - return startingCodec.requiresDurableContext() - || endingCodec.requiresDurableContext() - || binarySerDes.stream().anyMatch(BinarySerDes::requiresDurableContext); - } - private static byte[] invokeToBytes(StringBinaryCodec codec, String value, String name) { try { return requireResult(codec.toBytes(value), name); diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/ComposableSerDes.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/ComposableSerDes.java index b8857ad01..cad11da14 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/ComposableSerDes.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/ComposableSerDes.java @@ -60,11 +60,6 @@ public SerDes getValueCodec() { return valueCodec; } - @Override - public boolean requiresDurableContext() { - return valueCodec.requiresDurableContext() || stages.stream().anyMatch(ComposableSerDes::requiresContext); - } - /** Returns a new pipeline with the supplied string stage appended. */ @Override public ComposableSerDes then(SerDesStage stage) { @@ -159,10 +154,6 @@ private static RuntimeException stageFailure(int index, Object stage, String act return new SerDesException(message, failure); } - private static boolean requiresContext(SerDesStage stage) { - return stage.requiresDurableContext(); - } - /** Builder for an immutable {@link ComposableSerDes}. */ public static final class Builder { private SerDes valueCodec; diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java index fcd0a8000..2e9272c5d 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java @@ -113,11 +113,6 @@ public SerDesStageResult deserializePipelineStage(String data) { : SerDesStageResult.continueWith(resolved.value()); } - @Override - public boolean requiresDurableContext() { - return true; - } - private ResolvedPayload resolveSerializedPayload(String data, SerDesContext context) { final JsonNode envelope; try { diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/RetrySerDes.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/RetrySerDes.java index e69815422..947a77993 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/RetrySerDes.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/RetrySerDes.java @@ -62,11 +62,6 @@ public SerDesStageResult deserializePipelineStage(String data) { return execute("pipeline stage deserialization", () -> delegate.deserializePipelineStage(data)); } - @Override - public boolean requiresDurableContext() { - return delegate.requiresDurableContext(); - } - private T execute(String action, Supplier operation) { int attempt = 1; while (true) { diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDes.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDes.java index 516753f56..3cdd7b905 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDes.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDes.java @@ -38,16 +38,6 @@ public interface SerDes { */ T deserialize(String data, TypeToken typeToken); - /** - * Returns whether this SerDes requires an SDK-managed durable execution context. - * - *

Context-dependent SerDes implementations cannot process an initial external invocation payload unless a - * separate context-free input SerDes is configured. - */ - default boolean requiresDurableContext() { - return false; - } - /** * Returns an immutable processing pipeline with a string stage appended. * diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesStage.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesStage.java index 856eabd4b..a70c9678c 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesStage.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesStage.java @@ -39,9 +39,4 @@ public interface SerDesStage { default SerDesStageResult deserializePipelineStage(String data) { return SerDesStageResult.continueWith(deserialize(data)); } - - /** Returns whether this stage requires an SDK-managed durable execution context. */ - default boolean requiresDurableContext() { - return false; - } } diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/StringBinaryCodec.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/StringBinaryCodec.java index 9f5b96a8a..92315ccb2 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/StringBinaryCodec.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/StringBinaryCodec.java @@ -24,9 +24,4 @@ public interface StringBinaryCodec { * @return the non-null string value */ String fromBytes(byte[] data); - - /** Returns whether this codec requires an SDK-managed durable execution context. */ - default boolean requiresDurableContext() { - return false; - } } diff --git a/sdk/src/test/java/software/amazon/lambda/durable/serde/ComposableBinarySerDesStageTest.java b/sdk/src/test/java/software/amazon/lambda/durable/serde/ComposableBinarySerDesStageTest.java index a24f5f3f9..ce4a94796 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/serde/ComposableBinarySerDesStageTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/serde/ComposableBinarySerDesStageTest.java @@ -90,33 +90,6 @@ public String fromBytes(byte[] data) { assertEquals("value", stage.deserialize(serialized)); } - @Test - void delegatesDurableContextRequirement() { - var contextCodec = new StringBinaryCodec() { - @Override - public byte[] toBytes(String value) { - return value.getBytes(StandardCharsets.UTF_8); - } - - @Override - public String fromBytes(byte[] data) { - return new String(data, StandardCharsets.UTF_8); - } - - @Override - public boolean requiresDurableContext() { - return true; - } - }; - - var stage = ComposableBinarySerDesStage.builder() - .startWith(contextCodec) - .endWith(Base64StringBinaryCodec.INSTANCE) - .build(); - - assertTrue(stage.requiresDurableContext()); - } - @Test void validatesConfigurationAndComponentResults() { assertThrows(NullPointerException.class, () -> ComposableBinarySerDesStage.builder() diff --git a/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java b/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java index ab08cb204..d0d545c6a 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java @@ -57,13 +57,12 @@ void writesValueCodecPayloadAndReplaysIt() throws Exception { } @Test - void isAContextDependentStageThatCanBeFollowedByOtherStages() { + void isAStageThatCanBeFollowedByOtherStages() { var stage = FileSystemSerDes.builder(basePath).build(); var pipeline = new JacksonSerDes().then(stage).then(wrappingStage()); var runner = new SerDesRunner(null); assertFalse(SerDes.class.isAssignableFrom(FileSystemSerDes.class)); - assertTrue(stage.requiresDurableContext()); var checkpoint = runner.serialize(pipeline, Map.of("id", 42), context()); assertTrue(checkpoint.startsWith("<")); assertEquals( diff --git a/sdk/src/test/java/software/amazon/lambda/durable/serde/RetrySerDesTest.java b/sdk/src/test/java/software/amazon/lambda/durable/serde/RetrySerDesTest.java index ae2191d38..aeaec52cd 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/serde/RetrySerDesTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/serde/RetrySerDesTest.java @@ -79,7 +79,7 @@ public String deserialize(String data) { } @Test - void retriesPipelineStageDeserializationAndDelegatesContextCapability() { + void retriesPipelineStageDeserialization() { var calls = new AtomicInteger(); class RetryableStage implements SerDesStage { @Override @@ -99,11 +99,6 @@ public SerDesStageResult deserializePipelineStage(String data) { } return SerDesStageResult.decodeWithValueCodec(data); } - - @Override - public boolean requiresDurableContext() { - return true; - } } var delegate = new RetryableStage(); var retrySerDes = @@ -114,7 +109,6 @@ public boolean requiresDurableContext() { assertEquals("value", result.value()); assertTrue(result.skipRemainingStages()); assertEquals(2, calls.get()); - assertTrue(retrySerDes.requiresDurableContext()); } @Test From 97296a280572db60a722ed4383e03bed8eabc4ae Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Tue, 25 Aug 2026 18:41:29 +0000 Subject: [PATCH 30/56] fix: support filesystems without hard links --- docs/advanced/filesystem-serdes.md | 10 ++++++---- .../lambda/durable/serde/FileSystemSerDes.java | 15 +++++++++++++++ .../durable/serde/FileSystemSerDesTest.java | 18 ++++++++++++++++++ 3 files changed, 39 insertions(+), 4 deletions(-) diff --git a/docs/advanced/filesystem-serdes.md b/docs/advanced/filesystem-serdes.md index d5b0c8aba..6f3762e81 100644 --- a/docs/advanced/filesystem-serdes.md +++ b/docs/advanced/filesystem-serdes.md @@ -82,10 +82,12 @@ malformed marked envelopes and unsupported versions fail instead of falling back Offloaded files are content-addressed and immutable. Updating wait-for-condition state or retry results creates a new path instead of replacing a file referenced by an earlier checkpoint. Repeating the same write can safely reuse the -same file. File envelopes identify the producing execution and entity. Ordinary checkpoint replay must match that -owner, while invoke input and result boundaries may consume a file owned by the other Lambda execution when both -functions use the same shared root and path encoding. Treat file envelopes as capabilities. Content hashes are -verified when reading, and symbolic-link paths are rejected. +same file. Publication uses an atomic hard-link create-if-absent operation when the provider supports it and falls back +to a create-new copy without replacing an existing target on providers that do not support hard links. File envelopes +identify the producing execution and entity. Ordinary checkpoint replay must match that owner, while invoke input and +result boundaries may consume a file owned by the other Lambda execution when both functions use the same shared root +and path encoding. Treat file envelopes as capabilities. Content hashes are verified when reading, and symbolic-link +paths are rejected. Stages may follow `FileSystemSerDes` to transform its inline or file-reference envelope. The filesystem stage's `OVERFLOW` and preview-size checks apply before those later transformations, so account for any expansion when staying diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java index 2e9272c5d..1b20c1ab3 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java @@ -456,6 +456,21 @@ private void publishWithoutReplacement(Path temporary, Path file, byte[] expecte Files.createLink(file, temporary); } catch (FileAlreadyExistsException ignored) { validateExistingPayload(file, expectedData); + } catch (UnsupportedOperationException | IOException linkFailure) { + publishByCreateNewCopy(temporary, file, expectedData, linkFailure); + } + } + + private void publishByCreateNewCopy(Path temporary, Path file, byte[] expectedData, Exception linkFailure) + throws IOException { + try { + Files.copy(temporary, file); + validateExistingPayload(file, expectedData); + } catch (FileAlreadyExistsException ignored) { + validateExistingPayload(file, expectedData); + } catch (IOException | RuntimeException copyFailure) { + copyFailure.addSuppressed(linkFailure); + throw copyFailure; } } diff --git a/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java b/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java index d0d545c6a..d82ca6c37 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java @@ -12,7 +12,9 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; +import java.net.URI; import java.nio.charset.StandardCharsets; +import java.nio.file.FileSystems; import java.nio.file.Files; import java.nio.file.Path; import java.security.MessageDigest; @@ -156,6 +158,22 @@ void rejectsUnexpectedContentInExistingContentAddressedFile() throws Exception { assertEquals("unexpected", Files.readString(file)); } + @Test + void publishesOnFileSystemsWithoutHardLinkSupport() throws Exception { + var archive = basePath.resolve("payloads.zip"); + try (var fileSystem = + FileSystems.newFileSystem(URI.create("jar:" + archive.toUri()), Map.of("create", "true"))) { + var archiveBasePath = fileSystem.getPath("/payloads"); + var serDes = + stringCodec().then(FileSystemSerDes.builder(archiveBasePath).build()); + + var envelope = new SerDesRunner(null).serialize(serDes, "expected", context()); + var file = fileSystem.getPath(MAPPER.readTree(envelope).get("file").textValue()); + + assertEquals("expected", Files.readString(file)); + } + } + @Test void rejectsMalformedUtf8StringsAndFilePayloads() throws Exception { var serDes = stringCodec().then(FileSystemSerDes.builder(basePath).build()); From f22f4910cc202760df877b81d5bf33d93aa28711 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Tue, 25 Aug 2026 19:03:28 +0000 Subject: [PATCH 31/56] fix: address latest SerDes review feedback --- docs/adr/005-filesystem-serdes.md | 11 ++-- docs/advanced/configuration.md | 9 +-- docs/advanced/filesystem-serdes.md | 24 ++++---- .../testing/CloudDurableTestRunner.java | 34 ++++++++--- .../testing/LocalDurableTestRunner.java | 24 +++++--- .../testing/CloudDurableTestRunnerTest.java | 24 +++++++- .../testing/LocalDurableTestRunnerTest.java | 32 ++++++++++ .../durable/serde/FileSystemSerDes.java | 61 ++++++++++++------- .../durable/serde/FileSystemSerDesTest.java | 12 ++++ 9 files changed, 169 insertions(+), 62 deletions(-) diff --git a/docs/adr/005-filesystem-serdes.md b/docs/adr/005-filesystem-serdes.md index 475754b8f..25eaa4f33 100644 --- a/docs/adr/005-filesystem-serdes.md +++ b/docs/adr/005-filesystem-serdes.md @@ -535,11 +535,12 @@ When serializing `errorData`, set `SerDesPayloadKind.EXCEPTION` and use an entit Root user input and output payloads should route through `SerDesRunner` so `FileSystemSerDes` can see `SerDesContext`. The internal `DurableExecutionInput` and `DurableExecutionOutput` envelope stays with `DurableInputOutputSerDes`. The cloud test runner must send initial Lambda input before it receives a durable execution ARN. The cloud and local -runners therefore always serialize that input with a separate context-free value codec, defaulting to -`JacksonSerDes`. `withInputSerDes(...)` replaces this codec, but rejects `ComposableSerDes`: the runners must never -reuse the persisted composable pipeline for the initial invocation, and an unframed external payload does not identify -which stages ran. A custom input codec must produce data that the persisted pipeline's root value codec can decode at -the external-input boundary. Fluent configuration preserves the input codec when other runner configuration is +runners therefore always serialize that input with a separate context-free value codec. By default, they use the +configured persisted SerDes when it is a plain value codec, or the root value codec when it is composable. +`withInputSerDes(...)` replaces this codec, but rejects `ComposableSerDes`: the runners must never reuse persisted +pipeline stages for the initial invocation, and an unframed external payload does not identify which stages ran. A +custom input codec must produce data that the persisted pipeline's root value codec can decode at the external-input +boundary. Fluent configuration preserves an explicit input-codec override when other runner configuration is replaced. `LocalDurableTestRunner` creates the execution operation with this raw external payload and lets `DurableExecutor` apply the persisted pipeline's external-boundary behavior; it does not synthesize a durable context or serialize initial input through the persisted pipeline. diff --git a/docs/advanced/configuration.md b/docs/advanced/configuration.md index f379f9faa..7bf752c1f 100644 --- a/docs/advanced/configuration.md +++ b/docs/advanced/configuration.md @@ -107,10 +107,11 @@ lose recent writes; use it only when that tradeoff is acceptable. The SDK does n storage lifecycle and retention separately. The cloud and local test runners always serialize the initial Lambda invocation with a separate context-free value -codec. This codec defaults to `JacksonSerDes` and can be replaced with `withInputSerDes(...)`. It is independent of the -SerDes used for persisted execution payloads: the runners never apply that persisted composable pipeline to the initial -invocation. The input SerDes must therefore be a value codec rather than a `ComposableSerDes`. A custom input codec must -produce data that the persisted pipeline's root value codec can decode at the external-input boundary. +codec. By default, this is the configured persisted SerDes when it is a plain value codec, or the root value codec when +the persisted SerDes is a `ComposableSerDes`; `withInputSerDes(...)` provides an explicit override. The runners never +apply persisted pipeline stages to the initial invocation. The explicit input SerDes must therefore be a value codec +rather than a `ComposableSerDes`. A custom input codec must produce data that the persisted pipeline's root value codec +can decode at the external-input boundary. Stages may follow `FileSystemSerDes` to transform its inline or file-reference envelope. `OVERFLOW` and preview-size checks apply to the filesystem stage's output; account for any expansion introduced by later stages when staying diff --git a/docs/advanced/filesystem-serdes.md b/docs/advanced/filesystem-serdes.md index 6f3762e81..4da432264 100644 --- a/docs/advanced/filesystem-serdes.md +++ b/docs/advanced/filesystem-serdes.md @@ -81,13 +81,14 @@ been wrapped by this SerDes. Payloads containing the reserved marker must be val malformed marked envelopes and unsupported versions fail instead of falling back to raw-data decoding. Offloaded files are content-addressed and immutable. Updating wait-for-condition state or retry results creates a new -path instead of replacing a file referenced by an earlier checkpoint. Repeating the same write can safely reuse the -same file. Publication uses an atomic hard-link create-if-absent operation when the provider supports it and falls back -to a create-new copy without replacing an existing target on providers that do not support hard links. File envelopes -identify the producing execution and entity. Ordinary checkpoint replay must match that owner, while invoke input and -result boundaries may consume a file owned by the other Lambda execution when both functions use the same shared root -and path encoding. Treat file envelopes as capabilities. Content hashes are verified when reading, and symbolic-link -paths are rejected. +path instead of replacing a file referenced by an earlier checkpoint. Publication uses an atomic hard-link +create-if-absent operation when the provider supports it, allowing repeated identical writes to reuse one file. On +providers that do not support hard links, it retains the completed, uniquely named content-addressed staging file. The +fallback path is not exposed in an envelope until its write completes, so a crash can leave only an unreferenced orphan +rather than a partial checkpoint target. File envelopes identify the producing execution and entity. Ordinary +checkpoint replay must match that owner, while invoke input and result boundaries may consume a file owned by the +other Lambda execution when both functions use the same shared root and path encoding. Treat file envelopes as +capabilities. Content hashes are verified when reading, and symbolic-link paths are rejected. Stages may follow `FileSystemSerDes` to transform its inline or file-reference envelope. The filesystem stage's `OVERFLOW` and preview-size checks apply before those later transformations, so account for any expansion when staying @@ -96,10 +97,11 @@ within the service checkpoint limit. During deserialization of a raw external pa have not passed through the pipeline. The cloud and local test runners always use a separate context-free value codec for the initial Lambda invocation -because an execution ARN is not available yet. The input codec defaults to `JacksonSerDes` and can be replaced with -`withInputSerDes(...)`. The runners never reuse the persisted composable pipeline for this boundary. Do not configure a -`ComposableSerDes` as the input codec: the unframed external payload does not identify which stages ran before the -filesystem stage. A custom input codec must produce data that the persisted pipeline's root value codec can decode. +because an execution ARN is not available yet. By default, they use the configured persisted SerDes when it is a plain +value codec, or the root value codec when it is a `ComposableSerDes`; `withInputSerDes(...)` provides an explicit +override. The runners never reuse persisted pipeline stages for this boundary. Do not configure a `ComposableSerDes` +as the explicit input codec: the unframed external payload does not identify which stages ran before the filesystem +stage. A custom input codec must produce data that the persisted pipeline's root value codec can decode. ## Storage requirements diff --git a/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/CloudDurableTestRunner.java b/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/CloudDurableTestRunner.java index 69c056db4..c6af9bd00 100644 --- a/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/CloudDurableTestRunner.java +++ b/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/CloudDurableTestRunner.java @@ -33,6 +33,7 @@ public class CloudDurableTestRunner { private final Duration timeout; private final InvocationType invocationType; private final SerDes inputSerDes; + private final SerDes inputSerDesOverride; private final SerDes serDes; // Store last execution result for operation inspection private TestResult lastResult; @@ -56,7 +57,8 @@ private CloudDurableTestRunner( this.timeout = timeout; this.invocationType = invocationType; this.serDes = Objects.requireNonNullElseGet(serDes, JacksonSerDes::new); - this.inputSerDes = inputValueCodec(inputSerDes); + this.inputSerDesOverride = inputSerDes; + this.inputSerDes = inputValueCodec(inputSerDes, this.serDes); } private static LambdaClient createDefaultLambdaClient() { @@ -117,7 +119,7 @@ public CloudDurableTestRunner withLambdaClient(LambdaClient lambdaClient) pollInterval, timeout, invocationType, - inputSerDes, + inputSerDesOverride, serDes); } @@ -131,7 +133,7 @@ public CloudDurableTestRunner withPollInterval(Duration interval) { interval, timeout, invocationType, - inputSerDes, + inputSerDesOverride, serDes); } @@ -145,14 +147,22 @@ public CloudDurableTestRunner withTimeout(Duration timeout) { pollInterval, timeout, invocationType, - inputSerDes, + inputSerDesOverride, serDes); } /** Returns a new runner with the specified Lambda invocation type. */ public CloudDurableTestRunner withInvocationType(InvocationType type) { return new CloudDurableTestRunner<>( - functionArn, inputType, outputType, lambdaClient, pollInterval, timeout, type, inputSerDes, serDes); + functionArn, + inputType, + outputType, + lambdaClient, + pollInterval, + timeout, + type, + inputSerDesOverride, + serDes); } /** Returns a new runner with the specified SerDes for persisted execution payloads. */ @@ -165,7 +175,7 @@ public CloudDurableTestRunner withSerDes(SerDes serDes) { pollInterval, timeout, invocationType, - inputSerDes, + inputSerDesOverride, serDes); } @@ -173,7 +183,8 @@ public CloudDurableTestRunner withSerDes(SerDes serDes) { * Returns a new runner with a separate value codec for the initial Lambda invocation payload. * *

The input codec is independent of the SerDes used for persisted execution payloads and must not be a - * {@link ComposableSerDes}. The default is {@link JacksonSerDes}. + * {@link ComposableSerDes}. By default, the configured persisted SerDes is used when it is a value codec; for a + * composable pipeline, its root value codec is used. */ public CloudDurableTestRunner withInputSerDes(SerDes inputSerDes) { return new CloudDurableTestRunner<>( @@ -280,8 +291,13 @@ private String serializeInput(I input) { return inputSerDes.serialize(input); } - private static SerDes inputValueCodec(SerDes inputSerDes) { - var codec = Objects.requireNonNullElseGet(inputSerDes, JacksonSerDes::new); + private static SerDes inputValueCodec(SerDes inputSerDes, SerDes persistedSerDes) { + var codec = inputSerDes; + if (codec == null) { + codec = persistedSerDes instanceof ComposableSerDes composable + ? composable.getValueCodec() + : persistedSerDes; + } if (codec instanceof ComposableSerDes) { throw new IllegalArgumentException("inputSerDes must be a value codec, not a composable pipeline"); } diff --git a/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/LocalDurableTestRunner.java b/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/LocalDurableTestRunner.java index c7c03a0ba..a208cfb76 100644 --- a/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/LocalDurableTestRunner.java +++ b/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/LocalDurableTestRunner.java @@ -24,7 +24,6 @@ import software.amazon.lambda.durable.model.ExecutionStatus; import software.amazon.lambda.durable.plugin.DurableExecutionPlugin; import software.amazon.lambda.durable.serde.ComposableSerDes; -import software.amazon.lambda.durable.serde.JacksonSerDes; import software.amazon.lambda.durable.serde.SerDes; import software.amazon.lambda.durable.serde.SerDesRunner; import software.amazon.lambda.durable.testing.local.LocalMemoryExecutionClient; @@ -45,6 +44,7 @@ public class LocalDurableTestRunner { private final BiFunction handler; private final LocalMemoryExecutionClient storage; private final SerDes inputSerDes; + private final SerDes inputSerDesOverride; private final SerDes serDes; private final DurableConfig customerConfig; private final Instant executionStartTime = Instant.now(); @@ -63,7 +63,7 @@ private LocalDurableTestRunner( this.inputType = inputType; this.outputType = outputType; this.handler = handlerFn; - this.inputSerDes = inputValueCodec(inputSerDes); + this.inputSerDesOverride = inputSerDes; this.storage = new LocalMemoryExecutionClient(); // Create config that uses customer's configuration but overrides the client with in-memory storage @@ -90,6 +90,7 @@ private LocalDurableTestRunner( DurableConfig.builder().withDurableExecutionClient(storage).build(); } this.serDes = this.customerConfig.getSerDes(); + this.inputSerDes = inputValueCodec(inputSerDes, this.serDes); } /** @@ -203,24 +204,26 @@ public static LocalDurableTestRunner create(Class inputType, Dur * a new runner instance. */ public LocalDurableTestRunner withDurableConfig(DurableConfig config) { - return new LocalDurableTestRunner<>(inputType, outputType, handler, config, inputSerDes); + return new LocalDurableTestRunner<>(inputType, outputType, handler, config, inputSerDesOverride); } /** Overrides the output type for this test runner. */ public LocalDurableTestRunner withOutputType(TypeToken outputType) { - return new LocalDurableTestRunner<>(inputType, outputType, handler, customerConfig, inputSerDes); + return new LocalDurableTestRunner<>(inputType, outputType, handler, customerConfig, inputSerDesOverride); } /** Overrides the output type for this test runner. */ public LocalDurableTestRunner withOutputType(Class outputType) { - return new LocalDurableTestRunner<>(inputType, TypeToken.get(outputType), handler, customerConfig, inputSerDes); + return new LocalDurableTestRunner<>( + inputType, TypeToken.get(outputType), handler, customerConfig, inputSerDesOverride); } /** * Returns a new runner with a separate value codec for the initial Lambda invocation payload. * *

The input codec is independent of the SerDes used for persisted execution payloads and must not be a - * {@link ComposableSerDes}. The default is {@link JacksonSerDes}. + * {@link ComposableSerDes}. By default, the configured persisted SerDes is used when it is a value codec; for a + * composable pipeline, its root value codec is used. */ public LocalDurableTestRunner withInputSerDes(SerDes inputSerDes) { return new LocalDurableTestRunner<>( @@ -403,8 +406,13 @@ private String serializeInput(I input) { return inputSerDes.serialize(input); } - private static SerDes inputValueCodec(SerDes inputSerDes) { - var codec = Objects.requireNonNullElseGet(inputSerDes, JacksonSerDes::new); + private static SerDes inputValueCodec(SerDes inputSerDes, SerDes persistedSerDes) { + var codec = inputSerDes; + if (codec == null) { + codec = persistedSerDes instanceof ComposableSerDes composable + ? composable.getValueCodec() + : persistedSerDes; + } if (codec instanceof ComposableSerDes) { throw new IllegalArgumentException("inputSerDes must be a value codec, not a composable pipeline"); } diff --git a/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/CloudDurableTestRunnerTest.java b/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/CloudDurableTestRunnerTest.java index 9ffa51cdb..61b0fb3bd 100644 --- a/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/CloudDurableTestRunnerTest.java +++ b/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/CloudDurableTestRunnerTest.java @@ -59,7 +59,7 @@ void rejectsComposableInputSerDes() { } @Test - void persistedComposableSerDesUsesDefaultJacksonInputCodec() { + void persistedComposableSerDesUsesRootValueCodec() { var mockClient = mock(LambdaClient.class); when(mockClient.invoke(any(InvokeRequest.class))) .thenReturn(InvokeResponse.builder() @@ -67,13 +67,31 @@ void persistedComposableSerDesUsesDefaultJacksonInputCodec() { .build()); var runner = CloudDurableTestRunner.create( "arn:aws:lambda:us-east-2:123:function:test", String.class, String.class, mockClient) - .withSerDes(new JacksonSerDes().then(wrappingStage())); + .withSerDes(wrappingValueCodec().then(wrappingStage())); runner.startAsync("value"); var request = ArgumentCaptor.forClass(InvokeRequest.class); verify(mockClient).invoke(request.capture()); - assertEquals("\"value\"", request.getValue().payload().asUtf8String()); + assertEquals("<\"value\">", request.getValue().payload().asUtf8String()); + } + + @Test + void plainPersistedSerDesIsUsedAsDefaultInputCodec() { + var mockClient = mock(LambdaClient.class); + when(mockClient.invoke(any(InvokeRequest.class))) + .thenReturn(InvokeResponse.builder() + .durableExecutionArn("arn:aws:lambda:us-east-2:123:function:test:1/durable-execution/e/i") + .build()); + var runner = CloudDurableTestRunner.create( + "arn:aws:lambda:us-east-2:123:function:test", String.class, String.class, mockClient) + .withSerDes(wrappingValueCodec()); + + runner.startAsync("value"); + + var request = ArgumentCaptor.forClass(InvokeRequest.class); + verify(mockClient).invoke(request.capture()); + assertEquals("<\"value\">", request.getValue().payload().asUtf8String()); } @Test diff --git a/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/LocalDurableTestRunnerTest.java b/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/LocalDurableTestRunnerTest.java index 7b4385269..f38b80bf2 100644 --- a/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/LocalDurableTestRunnerTest.java +++ b/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/LocalDurableTestRunnerTest.java @@ -16,6 +16,7 @@ import org.junit.jupiter.api.io.TempDir; import software.amazon.lambda.durable.DurableConfig; import software.amazon.lambda.durable.TypeToken; +import software.amazon.lambda.durable.exception.SerDesException; import software.amazon.lambda.durable.model.ExecutionStatus; import software.amazon.lambda.durable.plugin.DurableExecutionPlugin; import software.amazon.lambda.durable.plugin.InvocationInfo; @@ -24,6 +25,7 @@ import software.amazon.lambda.durable.serde.ComposableBinarySerDesStage; import software.amazon.lambda.durable.serde.FileSystemSerDes; import software.amazon.lambda.durable.serde.JacksonSerDes; +import software.amazon.lambda.durable.serde.SerDes; import software.amazon.lambda.durable.serde.SerDesStage; import software.amazon.lambda.durable.serde.Utf8StringBinaryCodec; @@ -163,6 +165,18 @@ void filesystemPersistedSerDesUsesDefaultJacksonInputCodec(@TempDir Path basePat assertEquals("value", result.getResult()); } + @Test + void plainPersistedSerDesIsUsedAsDefaultInputCodec() { + var config = DurableConfig.builder().withSerDes(prefixedStringSerDes()).build(); + var runner = LocalDurableTestRunner.create(String.class, (input, context) -> input, config) + .withOutputType(String.class); + + var result = runner.run("value"); + + assertEquals(ExecutionStatus.SUCCEEDED, result.getStatus()); + assertEquals("value", result.getResult()); + } + @Test void rejectsComposableInputSerDes(@TempDir Path basePath) { var config = DurableConfig.builder() @@ -228,4 +242,22 @@ public byte[] deserialize(byte[] data) { .endWith(Base64StringBinaryCodec.INSTANCE) .build(); } + + private static SerDes prefixedStringSerDes() { + return new SerDes() { + @Override + public String serialize(Object value) { + return "custom:" + value; + } + + @Override + @SuppressWarnings("unchecked") + public T deserialize(String data, TypeToken typeToken) { + if (!TypeToken.get(String.class).equals(typeToken) || !data.startsWith("custom:")) { + throw new SerDesException("Invalid custom string payload"); + } + return (T) data.substring("custom:".length()); + } + }; + } } diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java index 1b20c1ab3..cb9644208 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java @@ -87,8 +87,17 @@ public String serialize(String value) { + "'"); } try { - writePayload(payload, file); - return fileEnvelope; + var publishedFile = writePayload(payload, file); + if (publishedFile.equals(file)) { + return fileEnvelope; + } + var fallbackEnvelope = encodeEnvelope(payload.withoutData(), publishedFile, preview, context); + if (!fitsCheckpoint(fallbackEnvelope)) { + throw new SerDesException("Filesystem SerDes envelope exceeds the checkpoint payload limit for entity '" + + context.entityId() + + "'"); + } + return fallbackEnvelope; } catch (IOException e) { throw new RetryableSerDesException( "Failed to store filesystem payload for entity '" + context.entityId() + "'", e); @@ -178,7 +187,7 @@ private SerializedPayload readPayload( } var serialized = new SerializedPayload(payloadType, Files.readAllBytes(realFile)); var expectedFileName = payloadFileName(serialized, owner.entityId()); - if (!realFile.getFileName().toString().equals(expectedFileName)) { + if (!matchesPublishedPayloadFileName(realFile.getFileName().toString(), expectedFileName)) { throw new SerDesException("Filesystem SerDes file content does not match its content-addressed path"); } return serialized; @@ -194,7 +203,9 @@ private void validatePayloadPath(Path file, PayloadOwner owner) { if (fileName == null || file.getParent() == null || !file.getParent().equals(expectedDirectory) - || !fileName.toString().matches(Pattern.quote(encode(owner.entityId())) + "-[0-9a-f]{64}\\.payload")) { + || !fileName.toString() + .matches(Pattern.quote(encode(owner.entityId())) + + "-[0-9a-f]{64}(?:-[A-Za-z0-9_-]+)?\\.payload")) { throw new SerDesException("Filesystem SerDes file is not valid for its declared durable entity"); } } @@ -381,7 +392,7 @@ private String payloadFileName(SerializedPayload payload, String entityId) { return encode(entityId) + "-" + sha256(payload.data()) + ".payload"; } - private void writePayload(SerializedPayload payload, Path file) throws IOException { + private Path writePayload(SerializedPayload payload, Path file) throws IOException { var directory = file.getParent(); var realBasePath = createDirectoriesWithoutSymbolicLinks(directory); rejectSymbolicLinks(file); @@ -391,15 +402,22 @@ private void writePayload(SerializedPayload payload, Path file) throws IOExcepti } if (Files.exists(file, LinkOption.NOFOLLOW_LINKS)) { validateExistingPayload(file, payload.data()); - return; + return file; } - var temporary = Files.createTempFile(directory, file.getFileName().toString(), ".tmp"); + var fileName = file.getFileName().toString(); + var temporary = Files.createTempFile( + directory, fileName.substring(0, fileName.length() - ".payload".length()) + "-", ".payload"); + var retainTemporary = false; try { Files.write(temporary, payload.data()); - publishWithoutReplacement(temporary, file, payload.data()); + var publishedFile = publishWithoutReplacement(temporary, file, payload.data()); + retainTemporary = publishedFile.equals(temporary); + return publishedFile; } finally { - Files.deleteIfExists(temporary); + if (!retainTemporary) { + Files.deleteIfExists(temporary); + } } } @@ -451,27 +469,26 @@ private synchronized Path retainCanonicalBasePath(Path currentBasePath) { return canonicalBasePath; } - private void publishWithoutReplacement(Path temporary, Path file, byte[] expectedData) throws IOException { + private Path publishWithoutReplacement(Path temporary, Path file, byte[] expectedData) throws IOException { try { Files.createLink(file, temporary); + return file; } catch (FileAlreadyExistsException ignored) { validateExistingPayload(file, expectedData); - } catch (UnsupportedOperationException | IOException linkFailure) { - publishByCreateNewCopy(temporary, file, expectedData, linkFailure); + return file; + } catch (UnsupportedOperationException | IOException ignored) { + validateExistingPayload(temporary, expectedData); + return temporary; } } - private void publishByCreateNewCopy(Path temporary, Path file, byte[] expectedData, Exception linkFailure) - throws IOException { - try { - Files.copy(temporary, file); - validateExistingPayload(file, expectedData); - } catch (FileAlreadyExistsException ignored) { - validateExistingPayload(file, expectedData); - } catch (IOException | RuntimeException copyFailure) { - copyFailure.addSuppressed(linkFailure); - throw copyFailure; + private static boolean matchesPublishedPayloadFileName(String actualFileName, String expectedFileName) { + if (actualFileName.equals(expectedFileName)) { + return true; } + var suffix = ".payload"; + var expectedPrefix = expectedFileName.substring(0, expectedFileName.length() - suffix.length()); + return actualFileName.startsWith(expectedPrefix + "-") && actualFileName.endsWith(suffix); } private void validateExistingPayload(Path file, byte[] expectedData) throws IOException { diff --git a/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java b/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java index d82ca6c37..9f7763409 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java @@ -169,8 +169,20 @@ void publishesOnFileSystemsWithoutHardLinkSupport() throws Exception { var envelope = new SerDesRunner(null).serialize(serDes, "expected", context()); var file = fileSystem.getPath(MAPPER.readTree(envelope).get("file").textValue()); + var hash = HexFormat.of() + .formatHex( + MessageDigest.getInstance("SHA-256").digest("expected".getBytes(StandardCharsets.UTF_8))); + var fileName = file.getFileName().toString(); + assertTrue(fileName.contains(hash)); + var hashEnd = fileName.indexOf(hash) + hash.length(); + var deterministicFile = file.resolveSibling(fileName.substring(0, hashEnd) + ".payload"); assertEquals("expected", Files.readString(file)); + assertNotEquals(deterministicFile, file); + assertFalse(Files.exists(deterministicFile)); + assertTrue(fileName.startsWith( + deterministicFile.getFileName().toString().replace(".payload", "-"))); + assertTrue(fileName.endsWith(".payload")); } } From 9eb06a55313568cecc55ab58b4a1a0695e760dfb Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Tue, 25 Aug 2026 19:11:30 +0000 Subject: [PATCH 32/56] docs: record binary SerDes pipeline alternative --- docs/adr/005-filesystem-serdes.md | 57 +++++++++++++++++++++++++++++-- 1 file changed, 55 insertions(+), 2 deletions(-) diff --git a/docs/adr/005-filesystem-serdes.md b/docs/adr/005-filesystem-serdes.md index 25eaa4f33..d555dbf21 100644 --- a/docs/adr/005-filesystem-serdes.md +++ b/docs/adr/005-filesystem-serdes.md @@ -2,8 +2,8 @@ **Status:** Accepted — Approach A with ComposableSerDes pipeline **Date:** 2026-07-02 -**Updated:** 2026-08-25 — Included FileSystemSerDes in core and made every post-codec pipeline component an explicit -string stage. +**Updated:** 2026-08-25 — Included FileSystemSerDes in core, made every post-codec pipeline component an explicit +string stage, and documented the rejected binary-only top-level pipeline. ## Context @@ -810,6 +810,59 @@ that requires SDK-owned storage envelopes and lifecycle policy. ## Other Alternatives Considered +### Use a binary-only top-level SerDes pipeline + +Considered replacing the two-level string/binary composition model with one binary pipeline: + +```java +public interface BinarySerDes { + byte[] serialize(Object value); + + T deserialize(byte[] data, TypeToken typeToken); + + BinarySerDes then(SerDesStage stage); + + SerDes then(StringBinaryCodec terminalCodec); +} + +public interface SerDesStage { + byte[] serialize(byte[] value); + + byte[] deserialize(byte[] data); +} +``` + +In this alternative, `SerDes` keeps only its existing object-to-string methods for backward compatibility. +Composition starts from a new object-to-bytes implementation such as `JacksonBinarySerDes`, every intermediate stage +uses `byte[]`, and the chain ends with one binary-to-string codec. `JacksonSerDes` could appear to be implemented as: + +```java +var jacksonSerDes = new JacksonBinarySerDes() + .then(Utf8StringBinaryCodec.INSTANCE); +``` + +This has an attractive single-level type model and is potentially more efficient when compression, encryption, or +another binary stage always follows Jackson. Jackson can produce bytes directly, and those bytes can pass through all +intermediate stages without first creating a JSON string and encoding that string back to UTF-8. It also eliminates +the need for `ComposableBinarySerDesStage`. + +Rejected for the current design because it makes the common default `JacksonSerDes` path less efficient. Implementing +the existing string-facing API through `JacksonBinarySerDes` would serialize as object -> UTF-8 `byte[]` -> `String` +and deserialize as `String` -> UTF-8 `byte[]` -> object. Compared with Jackson's direct `writeValueAsString(...)` and +`readValue(String, ...)` paths, that requires an additional full-payload conversion and byte-array allocation in both +directions. Special-casing `JacksonBinarySerDes` plus the UTF-8 terminal codec to bypass those conversions would restore +performance, but it would introduce a second execution path and undermine the uniform pipeline model. + +The alternative also moves all existing string-oriented stages and custom `SerDes` implementations behind adapters or +requires binary replacements, while removing `SerDes.then(...)` from the composable API. The accepted design preserves +direct string SerDes performance and compatibility, and its nested binary composite already avoids conversions between +individual binary transformations: it converts to bytes once before the binary chain and back to a string once after +the chain. + +Revisit a binary-root pipeline in a future major version if profiling shows that JSON string creation before +binary-heavy pipelines is a material bottleneck and the benefit outweighs the default-path allocation cost and +migration burden. + ### Add FileSystemSerDes without SerDesContext Rejected. A filesystem-backed implementation needs stable operation identity. Without context, it cannot choose a safe file name, distinguish result and exception payloads for the same operation, or avoid collisions across durable executions. From f491d7f871f4fa58f18233adfb5bb18a4e2e20f9 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Tue, 25 Aug 2026 19:26:36 +0000 Subject: [PATCH 33/56] fix: preserve SerDes pipeline context boundaries --- docs/adr/005-filesystem-serdes.md | 31 ++++++++-------- docs/advanced/configuration.md | 13 +++---- docs/advanced/filesystem-serdes.md | 12 +++---- .../testing/CloudDurableTestRunner.java | 18 +++++----- .../testing/LocalDurableTestRunner.java | 18 +++++----- .../testing/CloudDurableTestRunnerTest.java | 29 ++++++++++++--- .../testing/LocalDurableTestRunnerTest.java | 35 +++++++++++++++++-- .../durable/serde/ComposableSerDes.java | 5 +++ .../durable/serde/FileSystemSerDes.java | 7 +++- .../lambda/durable/serde/RetrySerDes.java | 5 +++ .../lambda/durable/serde/SerDesStage.java | 13 +++++++ .../durable/serde/ComposableSerDesTest.java | 30 ++++++++++++++++ .../durable/serde/FileSystemSerDesTest.java | 4 +++ .../lambda/durable/serde/RetrySerDesTest.java | 23 ++++++++++++ 14 files changed, 191 insertions(+), 52 deletions(-) diff --git a/docs/adr/005-filesystem-serdes.md b/docs/adr/005-filesystem-serdes.md index d555dbf21..51188dbd2 100644 --- a/docs/adr/005-filesystem-serdes.md +++ b/docs/adr/005-filesystem-serdes.md @@ -45,6 +45,10 @@ serialized. ```java public interface SerDesStage { + default boolean requiresDurableContext() { + return false; + } + String serialize(String value); String deserialize(String data); @@ -223,11 +227,11 @@ Pipeline rules: - A pipeline must contain exactly one value codec in the first position and zero or more string stages. - Every top-level stage consumes and produces a non-null `String`. This makes all stage orderings type-compatible. -- Context requirements are stage behavior rather than a capability on `SerDes`, `SerDesStage`, binary transformations, - or codecs. A context-dependent stage accesses `SerDesContext` when it runs and reports a normal SerDes failure when - the required context is unavailable. -- The test runners identify a configured input pipeline directly as `ComposableSerDes` and reject it because initial - input accepts one value codec, not a pipeline. +- `SerDesStage.requiresDurableContext()` reports whether a stage needs `SerDesContext`. It defaults to `false`; + context-dependent stages return `true`, and decorators preserve their delegate's requirement. +- `ComposableSerDes.requiresDurableContext()` reports whether any contained stage requires context. The test runners + can use a context-free pipeline for initial input, but use only the root value codec when the persisted pipeline + requires context. - `SerDes.then(...)`, `ComposableSerDes.then(...)`, and the builder accept only `SerDesStage` after the value codec. This makes the root `SerDes` versus subsequent string-stage roles explicit in the Java type system. - If the value-codec argument to `ComposableSerDes.of(...)` or the builder is already a `ComposableSerDes`, its root @@ -535,15 +539,14 @@ When serializing `errorData`, set `SerDesPayloadKind.EXCEPTION` and use an entit Root user input and output payloads should route through `SerDesRunner` so `FileSystemSerDes` can see `SerDesContext`. The internal `DurableExecutionInput` and `DurableExecutionOutput` envelope stays with `DurableInputOutputSerDes`. The cloud test runner must send initial Lambda input before it receives a durable execution ARN. The cloud and local -runners therefore always serialize that input with a separate context-free value codec. By default, they use the -configured persisted SerDes when it is a plain value codec, or the root value codec when it is composable. -`withInputSerDes(...)` replaces this codec, but rejects `ComposableSerDes`: the runners must never reuse persisted -pipeline stages for the initial invocation, and an unframed external payload does not identify which stages ran. A -custom input codec must produce data that the persisted pipeline's root value codec can decode at the external-input -boundary. Fluent configuration preserves an explicit input-codec override when other runner configuration is -replaced. `LocalDurableTestRunner` creates the execution operation with this raw external payload and lets -`DurableExecutor` apply the persisted pipeline's external-boundary behavior; it does not synthesize a durable context -or serialize initial input through the persisted pipeline. +runners therefore serialize that input with a context-free SerDes. By default, they use the entire configured +persisted SerDes when it is plain or when every stage in a composable pipeline is context-free. When the persisted +pipeline contains a context-dependent stage, they use only its root value codec. `withInputSerDes(...)` accepts plain +and context-free composable overrides but rejects a pipeline whose stages require durable context. A custom input +SerDes must produce data that the persisted pipeline can decode at the external-input boundary. Fluent configuration +preserves an explicit input-SerDes override when other runner configuration is replaced. +`LocalDurableTestRunner` creates the execution operation with this raw external payload and lets `DurableExecutor` +apply the persisted pipeline's external-boundary behavior; it does not synthesize a durable context for initial input. ### Implementation plan diff --git a/docs/advanced/configuration.md b/docs/advanced/configuration.md index 7bf752c1f..b2a481c0d 100644 --- a/docs/advanced/configuration.md +++ b/docs/advanced/configuration.md @@ -106,12 +106,13 @@ shared mount such as EFS. S3 Files can have delayed synchronization, so a runtim lose recent writes; use it only when that tradeoff is acceptable. The SDK does not delete offloaded files, so configure storage lifecycle and retention separately. -The cloud and local test runners always serialize the initial Lambda invocation with a separate context-free value -codec. By default, this is the configured persisted SerDes when it is a plain value codec, or the root value codec when -the persisted SerDes is a `ComposableSerDes`; `withInputSerDes(...)` provides an explicit override. The runners never -apply persisted pipeline stages to the initial invocation. The explicit input SerDes must therefore be a value codec -rather than a `ComposableSerDes`. A custom input codec must produce data that the persisted pipeline's root value codec -can decode at the external-input boundary. +The cloud and local test runners always serialize the initial Lambda invocation with a context-free SerDes. By default, +they use the entire configured persisted SerDes when it is plain or when every stage in a `ComposableSerDes` is +context-free. When a composable pipeline contains a context-dependent stage such as `FileSystemSerDes`, the runners +use its root value codec because a durable execution ARN is not available yet. `withInputSerDes(...)` provides an +explicit override and accepts context-free composable pipelines, but rejects pipelines containing a stage that requires +durable context. A custom input SerDes must produce data that the persisted pipeline can decode at the external-input +boundary. Stages may follow `FileSystemSerDes` to transform its inline or file-reference envelope. `OVERFLOW` and preview-size checks apply to the filesystem stage's output; account for any expansion introduced by later stages when staying diff --git a/docs/advanced/filesystem-serdes.md b/docs/advanced/filesystem-serdes.md index 4da432264..920c8a7d7 100644 --- a/docs/advanced/filesystem-serdes.md +++ b/docs/advanced/filesystem-serdes.md @@ -96,12 +96,12 @@ within the service checkpoint limit. During deserialization of a raw external pa `FileSystemSerDes` can identify the external boundary; those stages must tolerate or explicitly bypass payloads that have not passed through the pipeline. -The cloud and local test runners always use a separate context-free value codec for the initial Lambda invocation -because an execution ARN is not available yet. By default, they use the configured persisted SerDes when it is a plain -value codec, or the root value codec when it is a `ComposableSerDes`; `withInputSerDes(...)` provides an explicit -override. The runners never reuse persisted pipeline stages for this boundary. Do not configure a `ComposableSerDes` -as the explicit input codec: the unframed external payload does not identify which stages ran before the filesystem -stage. A custom input codec must produce data that the persisted pipeline's root value codec can decode. +The cloud and local test runners always use a context-free SerDes for the initial Lambda invocation because an +execution ARN is not available yet. They use an entire plain or context-free composable persisted SerDes by default. +For a pipeline containing `FileSystemSerDes`, they use the root value codec and bypass every stage because the +filesystem stage requires durable context. `withInputSerDes(...)` accepts plain and context-free composable overrides +but rejects pipelines containing a context-dependent stage. A custom input SerDes must produce data that the persisted +pipeline can decode at the external-input boundary. ## Storage requirements diff --git a/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/CloudDurableTestRunner.java b/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/CloudDurableTestRunner.java index c6af9bd00..5a721195a 100644 --- a/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/CloudDurableTestRunner.java +++ b/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/CloudDurableTestRunner.java @@ -58,7 +58,7 @@ private CloudDurableTestRunner( this.invocationType = invocationType; this.serDes = Objects.requireNonNullElseGet(serDes, JacksonSerDes::new); this.inputSerDesOverride = inputSerDes; - this.inputSerDes = inputValueCodec(inputSerDes, this.serDes); + this.inputSerDes = selectInputSerDes(inputSerDes, this.serDes); } private static LambdaClient createDefaultLambdaClient() { @@ -180,11 +180,11 @@ public CloudDurableTestRunner withSerDes(SerDes serDes) { } /** - * Returns a new runner with a separate value codec for the initial Lambda invocation payload. + * Returns a new runner with a separate SerDes for the initial Lambda invocation payload. * - *

The input codec is independent of the SerDes used for persisted execution payloads and must not be a - * {@link ComposableSerDes}. By default, the configured persisted SerDes is used when it is a value codec; for a - * composable pipeline, its root value codec is used. + *

The input SerDes is independent of the SerDes used for persisted execution payloads and must not require a + * durable execution context. By default, the configured persisted SerDes is used unless it is a composable pipeline + * that requires context, in which case its root value codec is used. */ public CloudDurableTestRunner withInputSerDes(SerDes inputSerDes) { return new CloudDurableTestRunner<>( @@ -291,15 +291,15 @@ private String serializeInput(I input) { return inputSerDes.serialize(input); } - private static SerDes inputValueCodec(SerDes inputSerDes, SerDes persistedSerDes) { + private static SerDes selectInputSerDes(SerDes inputSerDes, SerDes persistedSerDes) { var codec = inputSerDes; if (codec == null) { - codec = persistedSerDes instanceof ComposableSerDes composable + codec = persistedSerDes instanceof ComposableSerDes composable && composable.requiresDurableContext() ? composable.getValueCodec() : persistedSerDes; } - if (codec instanceof ComposableSerDes) { - throw new IllegalArgumentException("inputSerDes must be a value codec, not a composable pipeline"); + if (codec instanceof ComposableSerDes composable && composable.requiresDurableContext()) { + throw new IllegalArgumentException("inputSerDes must not require a durable execution context"); } return codec; } diff --git a/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/LocalDurableTestRunner.java b/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/LocalDurableTestRunner.java index a208cfb76..5f787ebb1 100644 --- a/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/LocalDurableTestRunner.java +++ b/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/LocalDurableTestRunner.java @@ -90,7 +90,7 @@ private LocalDurableTestRunner( DurableConfig.builder().withDurableExecutionClient(storage).build(); } this.serDes = this.customerConfig.getSerDes(); - this.inputSerDes = inputValueCodec(inputSerDes, this.serDes); + this.inputSerDes = selectInputSerDes(inputSerDes, this.serDes); } /** @@ -219,11 +219,11 @@ public LocalDurableTestRunner withOutputType(Class outputType) { } /** - * Returns a new runner with a separate value codec for the initial Lambda invocation payload. + * Returns a new runner with a separate SerDes for the initial Lambda invocation payload. * - *

The input codec is independent of the SerDes used for persisted execution payloads and must not be a - * {@link ComposableSerDes}. By default, the configured persisted SerDes is used when it is a value codec; for a - * composable pipeline, its root value codec is used. + *

The input SerDes is independent of the SerDes used for persisted execution payloads and must not require a + * durable execution context. By default, the configured persisted SerDes is used unless it is a composable pipeline + * that requires context, in which case its root value codec is used. */ public LocalDurableTestRunner withInputSerDes(SerDes inputSerDes) { return new LocalDurableTestRunner<>( @@ -406,15 +406,15 @@ private String serializeInput(I input) { return inputSerDes.serialize(input); } - private static SerDes inputValueCodec(SerDes inputSerDes, SerDes persistedSerDes) { + private static SerDes selectInputSerDes(SerDes inputSerDes, SerDes persistedSerDes) { var codec = inputSerDes; if (codec == null) { - codec = persistedSerDes instanceof ComposableSerDes composable + codec = persistedSerDes instanceof ComposableSerDes composable && composable.requiresDurableContext() ? composable.getValueCodec() : persistedSerDes; } - if (codec instanceof ComposableSerDes) { - throw new IllegalArgumentException("inputSerDes must be a value codec, not a composable pipeline"); + if (codec instanceof ComposableSerDes composable && composable.requiresDurableContext()) { + throw new IllegalArgumentException("inputSerDes must not require a durable execution context"); } return codec; } diff --git a/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/CloudDurableTestRunnerTest.java b/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/CloudDurableTestRunnerTest.java index 61b0fb3bd..10282d5b1 100644 --- a/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/CloudDurableTestRunnerTest.java +++ b/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/CloudDurableTestRunnerTest.java @@ -46,20 +46,21 @@ void testPlaceholderMethods() { } @Test - void rejectsComposableInputSerDes() { + void rejectsContextDependentComposableInputSerDes(@TempDir Path basePath) { var mockClient = mock(LambdaClient.class); var runner = CloudDurableTestRunner.create( "arn:aws:lambda:us-east-2:123:function:test", String.class, String.class, mockClient); var failure = assertThrows( IllegalArgumentException.class, - () -> runner.withInputSerDes(new JacksonSerDes().then(wrappingStage()))); + () -> runner.withInputSerDes(new JacksonSerDes() + .then(FileSystemSerDes.builder(basePath).build()))); - assertTrue(failure.getMessage().contains("value codec")); + assertTrue(failure.getMessage().contains("durable execution context")); } @Test - void persistedComposableSerDesUsesRootValueCodec() { + void contextFreePersistedComposableSerDesUsesFullPipeline() { var mockClient = mock(LambdaClient.class); when(mockClient.invoke(any(InvokeRequest.class))) .thenReturn(InvokeResponse.builder() @@ -71,6 +72,26 @@ void persistedComposableSerDesUsesRootValueCodec() { runner.startAsync("value"); + var request = ArgumentCaptor.forClass(InvokeRequest.class); + verify(mockClient).invoke(request.capture()); + assertEquals("<<\"value\">>", request.getValue().payload().asUtf8String()); + } + + @Test + void explicitContextFreeComposableInputSerDesIsUsed() { + var mockClient = mock(LambdaClient.class); + when(mockClient.invoke(any(InvokeRequest.class))) + .thenReturn(InvokeResponse.builder() + .durableExecutionArn("arn:aws:lambda:us-east-2:123:function:test:1/durable-execution/e/i") + .build()); + var pipeline = new JacksonSerDes().then(wrappingStage()); + var runner = CloudDurableTestRunner.create( + "arn:aws:lambda:us-east-2:123:function:test", String.class, String.class, mockClient) + .withSerDes(pipeline) + .withInputSerDes(pipeline); + + runner.startAsync("value"); + var request = ArgumentCaptor.forClass(InvokeRequest.class); verify(mockClient).invoke(request.capture()); assertEquals("<\"value\">", request.getValue().payload().asUtf8String()); diff --git a/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/LocalDurableTestRunnerTest.java b/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/LocalDurableTestRunnerTest.java index f38b80bf2..344d85799 100644 --- a/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/LocalDurableTestRunnerTest.java +++ b/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/LocalDurableTestRunnerTest.java @@ -178,7 +178,35 @@ void plainPersistedSerDesIsUsedAsDefaultInputCodec() { } @Test - void rejectsComposableInputSerDes(@TempDir Path basePath) { + void contextFreePersistedComposableSerDesUsesFullPipeline() { + var config = DurableConfig.builder() + .withSerDes(new JacksonSerDes().then(wrappingStage(new AtomicInteger()))) + .build(); + var runner = LocalDurableTestRunner.create(String.class, (input, context) -> input, config) + .withOutputType(String.class); + + var result = runner.run("value"); + + assertEquals(ExecutionStatus.SUCCEEDED, result.getStatus()); + assertEquals("value", result.getResult()); + } + + @Test + void explicitContextFreeComposableInputSerDesIsUsed() { + var pipeline = new JacksonSerDes().then(wrappingStage(new AtomicInteger())); + var config = DurableConfig.builder().withSerDes(pipeline).build(); + var runner = LocalDurableTestRunner.create(String.class, (input, context) -> input, config) + .withInputSerDes(pipeline) + .withOutputType(String.class); + + var result = runner.run("value"); + + assertEquals(ExecutionStatus.SUCCEEDED, result.getStatus()); + assertEquals("value", result.getResult()); + } + + @Test + void rejectsContextDependentComposableInputSerDes(@TempDir Path basePath) { var config = DurableConfig.builder() .withSerDes(new JacksonSerDes() .then(FileSystemSerDes.builder(basePath).build())) @@ -187,9 +215,10 @@ void rejectsComposableInputSerDes(@TempDir Path basePath) { var failure = assertThrows( IllegalArgumentException.class, - () -> runner.withInputSerDes(new JacksonSerDes().then(wrappingStage(new AtomicInteger())))); + () -> runner.withInputSerDes(new JacksonSerDes() + .then(FileSystemSerDes.builder(basePath).build()))); - assertTrue(failure.getMessage().contains("value codec")); + assertTrue(failure.getMessage().contains("durable execution context")); } @Test diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/ComposableSerDes.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/ComposableSerDes.java index cad11da14..2d3ba8b92 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/ComposableSerDes.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/ComposableSerDes.java @@ -60,6 +60,11 @@ public SerDes getValueCodec() { return valueCodec; } + /** Returns whether any stage in this pipeline requires a durable execution context. */ + public boolean requiresDurableContext() { + return stages.stream().anyMatch(SerDesStage::requiresDurableContext); + } + /** Returns a new pipeline with the supplied string stage appended. */ @Override public ComposableSerDes then(SerDesStage stage) { diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java index cb9644208..7f8f5f9e7 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java @@ -64,6 +64,11 @@ public static Builder builder(Path basePath) { return new Builder(basePath); } + @Override + public boolean requiresDurableContext() { + return true; + } + @Override public String serialize(String value) { if (value == null) { @@ -173,7 +178,7 @@ private ResolvedPayload resolveSerializedPayload(String data, SerDesContext cont private SerializedPayload readPayload( String fileValue, PayloadType payloadType, PayloadOwner owner, SerDesContext context) { - var file = Path.of(fileValue).toAbsolutePath().normalize(); + var file = basePath.getFileSystem().getPath(fileValue).toAbsolutePath().normalize(); validatePayloadPath(file, owner); try { var realBasePath = validateBasePath(false); diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/RetrySerDes.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/RetrySerDes.java index 947a77993..e88c929fc 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/RetrySerDes.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/RetrySerDes.java @@ -47,6 +47,11 @@ public RetrySerDes(SerDesStage delegate, RetryStrategy retryStrategy) { this.sleeper = Objects.requireNonNull(sleeper, "sleeper cannot be null"); } + @Override + public boolean requiresDurableContext() { + return delegate.requiresDurableContext(); + } + @Override public String serialize(String value) { return execute("pipeline stage serialization", () -> delegate.serialize(value)); diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesStage.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesStage.java index a70c9678c..fae6d11dc 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesStage.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesStage.java @@ -10,6 +10,19 @@ * inside one string stage. */ public interface SerDesStage { + /** + * Returns whether this stage requires a durable execution context when it runs. + * + *

Context-free stages may be used to serialize the initial Lambda invocation before a durable execution ARN is + * available. Stages that access {@link SerDesContext} must override this method and return {@code true}. Decorators + * must preserve the requirement of the stage they wrap. + * + * @return {@code true} when this stage requires a durable execution context + */ + default boolean requiresDurableContext() { + return false; + } + /** * Applies this stage during forward serialization. * diff --git a/sdk/src/test/java/software/amazon/lambda/durable/serde/ComposableSerDesTest.java b/sdk/src/test/java/software/amazon/lambda/durable/serde/ComposableSerDesTest.java index db6ed1ea0..dfb2d1592 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/serde/ComposableSerDesTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/serde/ComposableSerDesTest.java @@ -3,6 +3,7 @@ package software.amazon.lambda.durable.serde; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertSame; @@ -93,6 +94,35 @@ void factoryBuilderAndThenFlattenRootPipeline() { assertEquals(List.of("one-serialize", "two-serialize"), calls); } + @Test + void reportsWhetherAnyStageRequiresDurableContext() { + var contextDependentStage = new SerDesStage() { + @Override + public boolean requiresDurableContext() { + return true; + } + + @Override + public String serialize(String value) { + return value; + } + + @Override + public String deserialize(String data) { + return data; + } + }; + + assertFalse(ComposableSerDes.of(new JacksonSerDes()).requiresDurableContext()); + assertFalse(new JacksonSerDes() + .then(stringStage("context-free", "<", ">", new ArrayList<>())) + .requiresDurableContext()); + assertTrue(new JacksonSerDes() + .then(stringStage("context-free", "<", ">", new ArrayList<>())) + .then(contextDependentStage) + .requiresDurableContext()); + } + @Test void nullBoundarySkipsEveryStage() { var calls = new AtomicInteger(); diff --git a/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java b/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java index 9f7763409..51870c37b 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java @@ -65,6 +65,7 @@ void isAStageThatCanBeFollowedByOtherStages() { var runner = new SerDesRunner(null); assertFalse(SerDes.class.isAssignableFrom(FileSystemSerDes.class)); + assertTrue(stage.requiresDurableContext()); var checkpoint = runner.serialize(pipeline, Map.of("id", 42), context()); assertTrue(checkpoint.startsWith("<")); assertEquals( @@ -183,6 +184,9 @@ void publishesOnFileSystemsWithoutHardLinkSupport() throws Exception { assertTrue(fileName.startsWith( deterministicFile.getFileName().toString().replace(".payload", "-"))); assertTrue(fileName.endsWith(".payload")); + assertEquals( + "expected", + new SerDesRunner(null).deserialize(serDes, envelope, TypeToken.get(String.class), context())); } } diff --git a/sdk/src/test/java/software/amazon/lambda/durable/serde/RetrySerDesTest.java b/sdk/src/test/java/software/amazon/lambda/durable/serde/RetrySerDesTest.java index aeaec52cd..db506fe70 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/serde/RetrySerDesTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/serde/RetrySerDesTest.java @@ -111,6 +111,29 @@ public SerDesStageResult deserializePipelineStage(String data) { assertEquals(2, calls.get()); } + @Test + void preservesDelegateContextRequirement() { + var contextDependentStage = new SerDesStage() { + @Override + public boolean requiresDurableContext() { + return true; + } + + @Override + public String serialize(String value) { + return value; + } + + @Override + public String deserialize(String data) { + return data; + } + }; + + assertFalse(new RetrySerDes(identityStage(), RetryStrategies.Presets.NO_RETRY).requiresDurableContext()); + assertTrue(new RetrySerDes(contextDependentStage, RetryStrategies.Presets.NO_RETRY).requiresDurableContext()); + } + @Test void doesNotRetryPermanentSerDesFailure() { var calls = new AtomicInteger(); From cdf37529d7bf1d106d5f814bb4ed57edd573b26e Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Tue, 25 Aug 2026 19:38:07 +0000 Subject: [PATCH 34/56] refactor: make SerDes stages self-identifying --- docs/adr/005-filesystem-serdes.md | 98 ++++++++++--------- docs/advanced/configuration.md | 28 +++--- docs/advanced/filesystem-serdes.md | 36 ++++--- docs/design.md | 2 +- .../testing/CloudDurableTestRunnerTest.java | 7 ++ .../testing/LocalDurableTestRunnerTest.java | 12 ++- ...narySerDes.java => BinarySerDesStage.java} | 4 +- .../serde/ComposableBinarySerDesStage.java | 64 +++++++----- .../durable/serde/ComposableSerDes.java | 15 ++- .../durable/serde/FileSystemSerDes.java | 52 +++------- .../lambda/durable/serde/RetrySerDes.java | 5 - .../lambda/durable/serde/SerDesStage.java | 33 ++++--- .../durable/serde/SerDesStageResult.java | 28 ------ .../ComposableBinarySerDesStageTest.java | 42 +++++--- .../durable/serde/ComposableSerDesTest.java | 56 ++++------- .../durable/serde/FileSystemSerDesTest.java | 38 +++++-- .../lambda/durable/serde/RetrySerDesTest.java | 33 ------- 17 files changed, 264 insertions(+), 289 deletions(-) rename sdk/src/main/java/software/amazon/lambda/durable/serde/{BinarySerDes.java => BinarySerDesStage.java} (87%) delete mode 100644 sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesStageResult.java diff --git a/docs/adr/005-filesystem-serdes.md b/docs/adr/005-filesystem-serdes.md index d555dbf21..a63b182fa 100644 --- a/docs/adr/005-filesystem-serdes.md +++ b/docs/adr/005-filesystem-serdes.md @@ -38,7 +38,7 @@ contract and a core `ComposableSerDes` implementation which together form a proc uniform and preventing intermediate type mismatches. Binary transformations compose inside one `ComposableBinarySerDesStage`. That outer stage converts strings to bytes -with a configurable starting codec, applies any number of reversible `BinarySerDes` implementations without +with a configurable starting codec, applies any number of reversible `BinarySerDesStage` implementations without intermediate text conversion, and converts the final bytes back to a string with a configurable ending codec. The filesystem stage uses `SerDesContext.getCurrentContext()` to identify the durable execution and entity being serialized. @@ -50,7 +50,7 @@ public interface SerDesStage { String deserialize(String data); } -public interface BinarySerDes { +public interface BinarySerDesStage { byte[] serialize(byte[] value); byte[] deserialize(byte[] data); @@ -158,12 +158,6 @@ public final class ComposableSerDes implements SerDes { public ComposableSerDes build(); } } - -public record SerDesStageResult(String value, boolean skipRemainingStages) { - public static SerDesStageResult continueWith(String value); - - public static SerDesStageResult decodeWithValueCodec(String value); -} ``` The first stage is the **value codec**. It converts the user value to a string and converts the final decoded string @@ -209,11 +203,7 @@ String serialize(Object value) { T deserialize(String data, TypeToken targetType) { String current = data; for (int i = stages.size() - 1; i >= 0; i--) { - var decoded = stages.get(i).deserializePipelineStage(current); - current = decoded.value(); - if (decoded.skipRemainingStages()) { - break; - } + current = stages.get(i).deserialize(current); } return valueCodec.deserialize(current, targetType); } @@ -243,9 +233,25 @@ Pipeline rules: entity and payload-kind metadata around the pipeline failure. - A stage must be reversible. Compression, encryption, signing envelopes, and external payload storage are suitable stages; lossy redaction is not. -- A boundary stage may return `SerDesStageResult.decodeWithValueCodec(...)` when it receives raw data that did not pass - through the configured pipeline. `ComposableSerDes` then skips every earlier intermediate stage and decodes the raw value - directly with the value codec. +- A stage must serialize into a self-identifying format, normally using a reserved marker and explicit version. + During deserialization it reverses recognized valid input, rejects recognized malformed or unsupported input, and + returns unrecognized input unchanged. Recognition must inspect the marker rather than attempt decoding and infer the + format from success or failure. +- The pass-through rule lets raw external payloads traverse every stage in reverse order and reach the root value codec + without filesystem-specific pipeline control flow. + +Equivalent stage pseudocode: + +```java +String deserialize(String data) { + if (!hasStageMarker(data)) { + return data; + } + validateSupportedEnvelope(data); + return decodeEnvelope(data); +} +``` + - Stage order is meaningful. For example, `JSON -> binary composite -> filesystem` writes the encoded result of the binary composite to the filesystem, while `JSON -> filesystem -> signing envelope` signs the filesystem envelope rather than the offloaded payload. @@ -265,8 +271,8 @@ concise form for independently reusable processing chains: ```java var binaryStage = ComposableBinarySerDesStage.builder() .startWith(Utf8StringBinaryCodec.INSTANCE) - .then(compressionBinarySerDes) - .then(encryptionBinarySerDes) + .then(compressionBinaryStage) + .then(encryptionBinaryStage) .endWith(Base64StringBinaryCodec.INSTANCE) .build(); @@ -278,7 +284,8 @@ var securePayloads = new JacksonSerDes() ### Composable binary stage `ComposableBinarySerDesStage` is one top-level `String -> String` stage containing zero or more `byte[] -> byte[]` -transformations: +transformations. It wraps the ending codec's string in its own reserved, versioned frame so it can distinguish its +serialized output from raw external input: ```java public final class ComposableBinarySerDesStage implements SerDesStage { @@ -289,7 +296,7 @@ public final class ComposableBinarySerDesStage implements SerDesStage { } public interface BinaryStagesBuilder { - BinaryStagesBuilder then(BinarySerDes serDes); + BinaryStagesBuilder then(BinarySerDesStage stage); CompletedBuilder endWith(StringBinaryCodec codec); } @@ -322,7 +329,7 @@ String ``` Both boundaries use the same `StringBinaryCodec` contract. The core SDK provides UTF-8 and standard Base64 -implementations, while callers may provide reversible alternatives. Each `BinarySerDes` must include required +implementations, while callers may provide reversible alternatives. Each `BinarySerDesStage` must include required metadata, such as a format version or encryption initialization vector, in its output. The composite performs text conversion only at its two outer boundaries; binary stages pass bytes directly to each other. @@ -392,13 +399,11 @@ Envelope format: string during reverse processing. A preceding `ComposableBinarySerDesStage` must encode its final bytes to a string before filesystem storage. -The marker and version distinguish filesystem envelopes from raw service-originated JSON. Initial root input, callback -results, and standard Lambda invoke results may arrive before this stage has processed them. For those external -payload sources only, an input without the filesystem marker is decoded directly with the pipeline value codec. -Skipping every intermediate stage is required because raw external data has not been compressed, encrypted, or -otherwise transformed by those stages. Missing or malformed markers on SDK-checkpointed payloads are permanent errors. -The marker name is reserved: malformed marked envelopes and unsupported envelope versions are rejected at external -boundaries rather than being treated as raw user data. +The marker and version distinguish filesystem envelopes from strings that were not produced by this stage. +`FileSystemSerDes.deserialize(...)` returns input without the filesystem marker unchanged, regardless of payload +source. If the marker is present, the value is recognized as filesystem data and must be a valid supported envelope; +malformed marked envelopes and unsupported versions fail rather than falling back to pass-through behavior. A +recognized filesystem envelope also requires an SDK-managed `SerDesContext`, while unrecognized input does not. Offloaded filenames include a content hash and are immutable. Serializing new state for the same entity creates a new path instead of replacing a file referenced by an earlier checkpoint. Repeating the same serialization may reuse the @@ -416,8 +421,9 @@ rejected rather than producing a checkpoint that the service cannot accept. Stages may follow `FileSystemSerDes` to transform its inline or file-reference envelope. Its overflow and preview-size checks apply at the filesystem stage boundary, so configurations must account for any size expansion introduced by -later stages. At raw external payload boundaries, those later stages run first during reverse processing and must -tolerate or explicitly bypass data that has not passed through the configured pipeline. +later stages. During deserialization, each stage validates its own reserved format and passes unrecognized input +through unchanged. Therefore raw external data can traverse stages on either side of `FileSystemSerDes` without being +decoded as a pipeline value. The preview generator receives the `String` produced by the preceding stage, not the original domain object. A preview that needs domain fields should parse that representation or be produced by an earlier stage. @@ -439,10 +445,10 @@ try { } ``` -On deserialization, `FileSystemSerDes` parses its envelope. If the envelope contains `data`, it restores the inline -text. If the envelope contains `file`, it reads the stored string. `ComposableSerDes` then passes that value to the -preceding string stage. Raw external input, callback results, and standard invoke results skip all intermediate stages -and go directly to the value codec when no versioned filesystem marker is present. +On deserialization, `FileSystemSerDes` first checks for its reserved marker. Unmarked input is returned unchanged. If +the marked envelope contains `data`, it restores the inline text; if it contains `file`, it reads the stored string. +`ComposableSerDes` then passes that value to the preceding string stage. Raw external input, callback results, and +standard invoke results pass unchanged through every stage whose marker is absent until they reach the value codec. ### Threading @@ -537,13 +543,13 @@ Root user input and output payloads should route through `SerDesRunner` so `File The cloud test runner must send initial Lambda input before it receives a durable execution ARN. The cloud and local runners therefore always serialize that input with a separate context-free value codec. By default, they use the configured persisted SerDes when it is a plain value codec, or the root value codec when it is composable. -`withInputSerDes(...)` replaces this codec, but rejects `ComposableSerDes`: the runners must never reuse persisted -pipeline stages for the initial invocation, and an unframed external payload does not identify which stages ran. A -custom input codec must produce data that the persisted pipeline's root value codec can decode at the external-input -boundary. Fluent configuration preserves an explicit input-codec override when other runner configuration is -replaced. `LocalDurableTestRunner` creates the execution operation with this raw external payload and lets -`DurableExecutor` apply the persisted pipeline's external-boundary behavior; it does not synthesize a durable context -or serialize initial input through the persisted pipeline. +`withInputSerDes(...)` replaces this codec, but rejects `ComposableSerDes`: persisted stages are never used as the +initial invocation's input encoder. A custom input codec must produce data that the persisted pipeline's root value +codec can decode after the stages pass the unrecognized input through. Fluent configuration preserves an explicit +input-codec override when other runner configuration is replaced. `LocalDurableTestRunner` creates the execution +operation with this raw external payload and lets `DurableExecutor` apply the persisted pipeline's pass-through +behavior during deserialization; it does not synthesize a durable context or serialize initial input through the +persisted pipeline. ### Implementation plan @@ -551,9 +557,9 @@ or serialize initial input through the persisted pipeline. `SerDes` methods unchanged. 2. Add the binary-compatible `SerDes.then(SerDesStage)` default method and `ComposableSerDes` with one root `SerDes` followed only by immutable `SerDesStage` entries, forward serialization, reverse deserialization, - external-boundary bypass, null short-circuiting, and stage-aware errors. -3. Add the string-only `SerDesStage` contract plus `BinarySerDes`, `StringBinaryCodec`, and - `ComposableBinarySerDesStage` for nested binary processing with ordered, configurable boundaries. + self-identifying stage pass-through, null short-circuiting, and stage-aware errors. +3. Add the string-only `SerDesStage` contract plus `BinarySerDesStage`, `StringBinaryCodec`, and + a version-framed `ComposableBinarySerDesStage` for nested binary processing with ordered, configurable boundaries. 4. Add `RetryableSerDesException` and `RetrySerDes`, reusing `RetryStrategy` for bounded in-invocation retries. 5. Add `SerDesRunner` with inline execution by default and optional dispatch through `DurableConfig.withSerDesExecutorService(...)`. Do not create a default SerDes pool. @@ -566,8 +572,8 @@ or serialize initial input through the persisted pipeline. 9. Update exception serialization and deserialization paths to set `SerDesPayloadKind.EXCEPTION` in TLS. 10. Implement `FileSystemSerDes` in the core `software.amazon.lambda.durable.serde` package as a string stage with `ALWAYS` and `OVERFLOW` storage, `URI` and `HASH` path encodings, envelope parsing, atomic file writes where - supported by the filesystem, retryable I/O failures, and clear validation errors for missing context or invalid - stage input. + supported by the filesystem, retryable I/O failures, unrecognized-input pass-through, and clear validation errors + for recognized malformed input. 11. Add unit tests for string and binary pipeline ordering, custom boundary codecs, reverse processing, nulls, stage failures, retry selection, exhaustion, delay handling, interruption, context construction, TLS scoping and restoration, inline execution, configured thread-pool isolation, cache hits, cache invalidation, exception diff --git a/docs/advanced/configuration.md b/docs/advanced/configuration.md index 7bf752c1f..b98f7ce74 100644 --- a/docs/advanced/configuration.md +++ b/docs/advanced/configuration.md @@ -68,8 +68,8 @@ var resilientFileSystemStage = new RetrySerDes( var binaryStage = ComposableBinarySerDesStage.builder() .startWith(Utf8StringBinaryCodec.INSTANCE) - .then(compressionBinarySerDes) - .then(encryptionBinarySerDes) + .then(compressionBinaryStage) + .then(encryptionBinaryStage) .endWith(Base64StringBinaryCodec.INSTANCE) .build(); @@ -86,10 +86,14 @@ return DurableConfig.builder() ``` Every top-level stage consumes and produces a string, so stages compose without intermediate type mismatches. -`ComposableBinarySerDesStage` contains an ordered chain of `BinarySerDes` implementations for compression, encryption, -or other `byte[]` transformations. Its `startWith(...)`, `then(...)`, and `endWith(...)` calls follow serialization -order; deserialization reverses them. Both boundaries use the customizable `StringBinaryCodec` interface. UTF-8 and -Base64 implementations are included in the core SDK, and conversion occurs only around the complete binary chain. +Every stage must emit a self-identifying, normally versioned representation. On deserialization it reverses recognized +valid input, rejects recognized malformed or unsupported input, and returns unrecognized input unchanged. This lets +raw external payloads pass through the configured stages and reach the root value codec. +`ComposableBinarySerDesStage` contains an ordered chain of `BinarySerDesStage` implementations for compression, +encryption, or other `byte[]` transformations. Its `startWith(...)`, `then(...)`, and `endWith(...)` calls follow +serialization order; deserialization reverses them. Both boundaries use the customizable `StringBinaryCodec` +interface. UTF-8 and Base64 implementations are included in the core SDK, conversion occurs only around the complete +binary chain, and the outer stage adds a reserved versioned frame for reliable format recognition. `ALWAYS` stores every non-null payload in a file. `OVERFLOW` keeps payloads inline until the checkpoint envelope approaches the 256 KB service limit. `URI` produces readable escaped paths; `HASH` produces fixed-length SHA-256 path @@ -109,14 +113,16 @@ storage lifecycle and retention separately. The cloud and local test runners always serialize the initial Lambda invocation with a separate context-free value codec. By default, this is the configured persisted SerDes when it is a plain value codec, or the root value codec when the persisted SerDes is a `ComposableSerDes`; `withInputSerDes(...)` provides an explicit override. The runners never -apply persisted pipeline stages to the initial invocation. The explicit input SerDes must therefore be a value codec -rather than a `ComposableSerDes`. A custom input codec must produce data that the persisted pipeline's root value codec -can decode at the external-input boundary. +use persisted pipeline stages to encode the initial invocation. The explicit input SerDes must therefore be a value +codec rather than a `ComposableSerDes`. When the runtime later deserializes that raw input, each persisted stage passes +it through unless its self-identifying format is present. A custom input codec must produce data that the persisted +pipeline's root value codec can decode. Stages may follow `FileSystemSerDes` to transform its inline or file-reference envelope. `OVERFLOW` and preview-size checks apply to the filesystem stage's output; account for any expansion introduced by later stages when staying -within the service checkpoint limit. At external payload boundaries, later stages run before `FileSystemSerDes` during -deserialization, so they must tolerate or explicitly bypass raw payloads that have not passed through the pipeline. +within the service checkpoint limit. On deserialization, every stage passes input through unchanged when its own +self-identifying format is absent, so raw external payloads can safely traverse stages on either side of the +filesystem stage. ### Dynamic plugin loading diff --git a/docs/advanced/filesystem-serdes.md b/docs/advanced/filesystem-serdes.md index 4da432264..983a8c411 100644 --- a/docs/advanced/filesystem-serdes.md +++ b/docs/advanced/filesystem-serdes.md @@ -30,8 +30,8 @@ var resilientFileSystemStage = new RetrySerDes( var binaryStage = ComposableBinarySerDesStage.builder() .startWith(Utf8StringBinaryCodec.INSTANCE) - .then(compressionBinarySerDes) - .then(encryptionBinarySerDes) + .then(compressionBinaryStage) + .then(encryptionBinaryStage) .endWith(Base64StringBinaryCodec.INSTANCE) .build(); @@ -49,10 +49,12 @@ return DurableConfig.builder() Serialization follows the declaration order above and deserialization runs in reverse. The first component is the `SerDes` value codec; every component appended with `then(...)` implements `SerDesStage` and consumes and produces a -string. `ComposableBinarySerDesStage` converts the string with its starting codec, passes bytes directly through each -`BinarySerDes`, then converts the final bytes to a string with its ending codec. Both boundaries are customizable -through the same `StringBinaryCodec` interface; the example performs UTF-8 and Base64 conversion once around the -complete compression/encryption chain. +string. Each stage must use a self-identifying format: it reverses recognized valid input, rejects recognized malformed +or unsupported input, and returns unrecognized input unchanged. `ComposableBinarySerDesStage` converts the string +with its starting codec, passes bytes directly through each `BinarySerDesStage`, converts the final bytes to a string +with its ending codec, and adds a reserved versioned frame. Both boundaries are customizable through the same +`StringBinaryCodec` interface; the example performs UTF-8 and Base64 conversion once around the complete +compression/encryption chain. - `ALWAYS` writes every non-null payload to a file. - `OVERFLOW` stores small payloads inline and offloads envelopes approaching the 256 KB checkpoint limit. @@ -75,10 +77,11 @@ delays consume time in the current Lambda invocation, so keep attempts and delay ## Replay and envelope behavior -Filesystem envelopes include a reserved version marker. Raw root input, callback results, and standard Lambda invoke -results bypass every intermediate stage and decode directly with the pipeline value codec when they have not yet -been wrapped by this SerDes. Payloads containing the reserved marker must be valid supported filesystem envelopes; -malformed marked envelopes and unsupported versions fail instead of falling back to raw-data decoding. +Filesystem envelopes include a reserved version marker. `FileSystemSerDes` returns input without that marker unchanged, +allowing raw root input, callback results, and standard Lambda invoke results to continue through the remaining stages +to the pipeline value codec. Payloads containing the reserved marker must be valid supported filesystem envelopes; +malformed marked envelopes and unsupported versions fail instead of falling back to pass-through behavior. An +unrecognized value does not require `SerDesContext`; a recognized filesystem envelope does. Offloaded files are content-addressed and immutable. Updating wait-for-condition state or retry results creates a new path instead of replacing a file referenced by an earlier checkpoint. Publication uses an atomic hard-link @@ -92,16 +95,17 @@ capabilities. Content hashes are verified when reading, and symbolic-link paths Stages may follow `FileSystemSerDes` to transform its inline or file-reference envelope. The filesystem stage's `OVERFLOW` and preview-size checks apply before those later transformations, so account for any expansion when staying -within the service checkpoint limit. During deserialization of a raw external payload, later stages run before -`FileSystemSerDes` can identify the external boundary; those stages must tolerate or explicitly bypass payloads that -have not passed through the pipeline. +within the service checkpoint limit. During deserialization, every stage checks its own marker and returns +unrecognized input unchanged, so raw external payloads can traverse later and earlier stages without being decoded by +them. The cloud and local test runners always use a separate context-free value codec for the initial Lambda invocation because an execution ARN is not available yet. By default, they use the configured persisted SerDes when it is a plain value codec, or the root value codec when it is a `ComposableSerDes`; `withInputSerDes(...)` provides an explicit -override. The runners never reuse persisted pipeline stages for this boundary. Do not configure a `ComposableSerDes` -as the explicit input codec: the unframed external payload does not identify which stages ran before the filesystem -stage. A custom input codec must produce data that the persisted pipeline's root value codec can decode. +override. The runners never use persisted pipeline stages to encode this boundary. Do not configure a +`ComposableSerDes` as the explicit input codec. When the runtime later deserializes the raw input, each persisted stage +passes it through unless its self-identifying format is present. A custom input codec must produce data that the +persisted pipeline's root value codec can decode. ## Storage requirements diff --git a/docs/design.md b/docs/design.md index 129e09cfe..937d0b611 100644 --- a/docs/design.md +++ b/docs/design.md @@ -357,7 +357,7 @@ software.amazon.lambda.durable │ ├── SerDes # Interface and pipeline composition entry point │ ├── SerDesStage # Reversible string-to-string pipeline stage │ ├── ComposableSerDes # Immutable ordered value-codec/string-stage pipeline -│ ├── BinarySerDes # Reversible byte-array transformation +│ ├── BinarySerDesStage # Reversible byte-array transformation │ ├── StringBinaryCodec # Customizable string/byte boundary conversion │ ├── Utf8StringBinaryCodec # UTF-8 string/byte conversion │ ├── Base64StringBinaryCodec # Standard Base64 string/byte conversion diff --git a/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/CloudDurableTestRunnerTest.java b/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/CloudDurableTestRunnerTest.java index 61b0fb3bd..014b7f5f7 100644 --- a/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/CloudDurableTestRunnerTest.java +++ b/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/CloudDurableTestRunnerTest.java @@ -15,6 +15,7 @@ import software.amazon.awssdk.services.lambda.model.InvokeRequest; import software.amazon.awssdk.services.lambda.model.InvokeResponse; import software.amazon.lambda.durable.TypeToken; +import software.amazon.lambda.durable.exception.SerDesException; import software.amazon.lambda.durable.serde.FileSystemSerDes; import software.amazon.lambda.durable.serde.JacksonSerDes; import software.amazon.lambda.durable.serde.SerDes; @@ -168,6 +169,12 @@ public String serialize(String value) { @Override public String deserialize(String data) { + if (!data.startsWith("<")) { + return data; + } + if (!data.endsWith(">")) { + throw new SerDesException("Malformed wrapping stage value"); + } return data.substring(1, data.length() - 1); } }; diff --git a/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/LocalDurableTestRunnerTest.java b/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/LocalDurableTestRunnerTest.java index f38b80bf2..12bfd10c4 100644 --- a/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/LocalDurableTestRunnerTest.java +++ b/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/LocalDurableTestRunnerTest.java @@ -21,7 +21,7 @@ import software.amazon.lambda.durable.plugin.DurableExecutionPlugin; import software.amazon.lambda.durable.plugin.InvocationInfo; import software.amazon.lambda.durable.serde.Base64StringBinaryCodec; -import software.amazon.lambda.durable.serde.BinarySerDes; +import software.amazon.lambda.durable.serde.BinarySerDesStage; import software.amazon.lambda.durable.serde.ComposableBinarySerDesStage; import software.amazon.lambda.durable.serde.FileSystemSerDes; import software.amazon.lambda.durable.serde.JacksonSerDes; @@ -193,7 +193,7 @@ void rejectsComposableInputSerDes(@TempDir Path basePath) { } @Test - void defaultInputCodecBypassesPersistedStagesBeforeFileSystemSerDes(@TempDir Path basePath) { + void rawInputPassesThroughPersistedStagesBeforeFileSystemSerDes(@TempDir Path basePath) { var deserializeCalls = new AtomicInteger(); var persistedSerDes = new JacksonSerDes() .then(bytesStage(deserializeCalls)) @@ -219,6 +219,12 @@ public String serialize(String value) { @Override public String deserialize(String data) { deserializeCalls.incrementAndGet(); + if (!data.startsWith("<")) { + return data; + } + if (!data.endsWith(">")) { + throw new SerDesException("Malformed wrapping stage value"); + } return data.substring(1, data.length() - 1); } }; @@ -227,7 +233,7 @@ public String deserialize(String data) { private static SerDesStage bytesStage(AtomicInteger deserializeCalls) { return ComposableBinarySerDesStage.builder() .startWith(Utf8StringBinaryCodec.INSTANCE) - .then(new BinarySerDes() { + .then(new BinarySerDesStage() { @Override public byte[] serialize(byte[] value) { return value; diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/BinarySerDes.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/BinarySerDesStage.java similarity index 87% rename from sdk/src/main/java/software/amazon/lambda/durable/serde/BinarySerDes.java rename to sdk/src/main/java/software/amazon/lambda/durable/serde/BinarySerDesStage.java index 278678166..75f69740f 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/BinarySerDes.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/BinarySerDesStage.java @@ -3,12 +3,12 @@ package software.amazon.lambda.durable.serde; /** - * A reversible binary transformation used inside a {@link ComposableBinarySerDesStage}. + * A reversible binary stage used inside a {@link ComposableBinarySerDesStage}. * *

Implementations must include any metadata needed for deserialization, such as format versions or encryption * initialization vectors, in the returned bytes. */ -public interface BinarySerDes { +public interface BinarySerDesStage { /** * Applies this transformation during forward serialization. * diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/ComposableBinarySerDesStage.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/ComposableBinarySerDesStage.java index 388dfb611..99ca30f35 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/ComposableBinarySerDesStage.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/ComposableBinarySerDesStage.java @@ -11,18 +11,22 @@ /** * A string SerDes stage containing an ordered chain of binary transformations. * - *

Serialization converts the input string with the starting codec, applies binary SerDes instances in declaration - * order, and converts the final bytes to a string with the ending codec. Deserialization reverses the complete process. + *

Serialization converts the input string with the starting codec, applies binary stages in declaration order, + * converts the final bytes to a string with the ending codec, and adds a versioned frame. Deserialization reverses the + * complete process when that frame is present and passes unrecognized input through unchanged. */ public final class ComposableBinarySerDesStage implements SerDesStage { + private static final String FRAME_MARKER = "__durable_execution_composable_binary_serdes:"; + private static final String FRAME_PREFIX = FRAME_MARKER + "1:"; + private final StringBinaryCodec startingCodec; - private final List binarySerDes; + private final List binaryStages; private final StringBinaryCodec endingCodec; private ComposableBinarySerDesStage( - StringBinaryCodec startingCodec, List binarySerDes, StringBinaryCodec endingCodec) { + StringBinaryCodec startingCodec, List binaryStages, StringBinaryCodec endingCodec) { this.startingCodec = startingCodec; - this.binarySerDes = List.copyOf(binarySerDes); + this.binaryStages = List.copyOf(binaryStages); this.endingCodec = endingCodec; } @@ -35,18 +39,24 @@ public static StartBuilder builder() { public String serialize(String value) { Objects.requireNonNull(value, "value cannot be null"); var current = invokeToBytes(startingCodec, value, "starting codec"); - for (int index = 0; index < binarySerDes.size(); index++) { - current = invokeSerialize(binarySerDes.get(index), current, index); + for (int index = 0; index < binaryStages.size(); index++) { + current = invokeSerialize(binaryStages.get(index), current, index); } - return invokeFromBytes(endingCodec, current, "ending codec"); + return FRAME_PREFIX + invokeFromBytes(endingCodec, current, "ending codec"); } @Override public String deserialize(String data) { Objects.requireNonNull(data, "data cannot be null"); - var current = invokeToBytes(endingCodec, data, "ending codec"); - for (int index = binarySerDes.size() - 1; index >= 0; index--) { - current = invokeDeserialize(binarySerDes.get(index), current, index); + if (!data.startsWith(FRAME_MARKER)) { + return data; + } + if (!data.startsWith(FRAME_PREFIX)) { + throw new SerDesException("Unsupported or malformed composable binary SerDes frame"); + } + var current = invokeToBytes(endingCodec, data.substring(FRAME_PREFIX.length()), "ending codec"); + for (int index = binaryStages.size() - 1; index >= 0; index--) { + current = invokeDeserialize(binaryStages.get(index), current, index); } return invokeFromBytes(startingCodec, current, "starting codec"); } @@ -67,19 +77,19 @@ private static String invokeFromBytes(StringBinaryCodec codec, byte[] data, Stri } } - private static byte[] invokeSerialize(BinarySerDes serDes, byte[] value, int index) { + private static byte[] invokeSerialize(BinarySerDesStage stage, byte[] value, int index) { try { - return requireResult(serDes.serialize(value), binaryStageName(index, serDes)); + return requireResult(stage.serialize(value), binaryStageName(index, stage)); } catch (Throwable failure) { - throw componentFailure(binaryStageName(index, serDes), "serialize", failure); + throw componentFailure(binaryStageName(index, stage), "serialize", failure); } } - private static byte[] invokeDeserialize(BinarySerDes serDes, byte[] data, int index) { + private static byte[] invokeDeserialize(BinarySerDesStage stage, byte[] data, int index) { try { - return requireResult(serDes.deserialize(data), binaryStageName(index, serDes)); + return requireResult(stage.deserialize(data), binaryStageName(index, stage)); } catch (Throwable failure) { - throw componentFailure(binaryStageName(index, serDes), "deserialize", failure); + throw componentFailure(binaryStageName(index, stage), "deserialize", failure); } } @@ -90,15 +100,15 @@ private static T requireResult(T result, String component) { return result; } - private static String binaryStageName(int index, BinarySerDes serDes) { - return String.format("binary stage %d (%s)", index, serDes.getClass().getName()); + private static String binaryStageName(int index, BinarySerDesStage stage) { + return String.format("binary stage %d (%s)", index, stage.getClass().getName()); } private static RuntimeException componentFailure(String component, String action, Throwable failure) { if (failure instanceof Error error) { throw error; } - var message = String.format("Composable binary SerDes %s failed to %s", component, action); + var message = String.format("Composable binary SerDes stage %s failed to %s", component, action); if (failure instanceof RetryableSerDesException) { return new RetryableSerDesException(message, failure); } @@ -116,15 +126,15 @@ public interface StartBuilder { BinaryStagesBuilder startWith(StringBinaryCodec codec); } - /** Builder stage that accepts binary SerDes instances in processing order. */ + /** Builder stage that accepts binary stages in processing order. */ public interface BinaryStagesBuilder { /** * Appends a binary transformation. * - * @param serDes the binary SerDes + * @param stage the binary stage * @return this builder stage */ - BinaryStagesBuilder then(BinarySerDes serDes); + BinaryStagesBuilder then(BinarySerDesStage stage); /** * Sets the codec that converts the final bytes to a string during serialization. @@ -143,7 +153,7 @@ public interface CompletedBuilder { private static final class Builder implements StartBuilder, BinaryStagesBuilder, CompletedBuilder { private StringBinaryCodec startingCodec; - private final List binarySerDes = new ArrayList<>(); + private final List binaryStages = new ArrayList<>(); private StringBinaryCodec endingCodec; @Override @@ -153,8 +163,8 @@ public BinaryStagesBuilder startWith(StringBinaryCodec codec) { } @Override - public BinaryStagesBuilder then(BinarySerDes serDes) { - binarySerDes.add(Objects.requireNonNull(serDes, "binary SerDes cannot be null")); + public BinaryStagesBuilder then(BinarySerDesStage stage) { + binaryStages.add(Objects.requireNonNull(stage, "binary stage cannot be null")); return this; } @@ -166,7 +176,7 @@ public CompletedBuilder endWith(StringBinaryCodec codec) { @Override public ComposableBinarySerDesStage build() { - return new ComposableBinarySerDesStage(startingCodec, binarySerDes, endingCodec); + return new ComposableBinarySerDesStage(startingCodec, binaryStages, endingCodec); } } } diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/ComposableSerDes.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/ComposableSerDes.java index cad11da14..11e74e836 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/ComposableSerDes.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/ComposableSerDes.java @@ -13,7 +13,8 @@ * An immutable SerDes processing pipeline. * *

The first component is the value codec. Every later component is a {@link SerDesStage} that consumes and produces - * a string. Serialization runs from first to last; deserialization runs from last to first. + * a string. Serialization runs from first to last; deserialization runs from last to first. Each stage returns + * unrecognized input unchanged, allowing raw values to pass through to the root value codec. */ public final class ComposableSerDes implements SerDes { private final SerDes valueCodec; @@ -88,11 +89,7 @@ public T deserialize(String data, TypeToken typeToken) { Objects.requireNonNull(typeToken, "typeToken cannot be null"); String current = data; for (int index = stages.size() - 1; index >= 0; index--) { - var decoded = invokeStageDeserialize(stages.get(index), current, index + 1); - current = decoded.value(); - if (decoded.skipRemainingStages()) { - break; - } + current = invokeStageDeserialize(stages.get(index), current, index + 1); } return invokeValueCodecDeserialize(valueCodec, current, typeToken); } @@ -109,11 +106,11 @@ private static String invokeStageSerialize(SerDesStage stage, String value, int } } - private static SerDesStageResult invokeStageDeserialize(SerDesStage stage, String data, int index) { + private static String invokeStageDeserialize(SerDesStage stage, String data, int index) { try { - var result = stage.deserializePipelineStage(data); + var result = stage.deserialize(data); if (result == null) { - throw new SerDesException("Stage returned a null pipeline result"); + throw new SerDesException("Stage returned null for non-null input"); } return result; } catch (Throwable failure) { diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java index cb9644208..633b97831 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java @@ -29,9 +29,13 @@ * *

Do not use Lambda's ephemeral {@code /tmp} storage. Use a durable shared mount such as EFS, or S3 Files only when * its synchronization and crash-durability tradeoffs are acceptable for the workload. + * + *

Deserialization recognizes the reserved filesystem envelope marker. Input without that marker is returned + * unchanged; input with the marker must be a valid supported envelope. */ public final class FileSystemSerDes implements SerDesStage { private static final String ENVELOPE_MARKER = "__durable_execution_filesystem_serdes"; + private static final String ENVELOPE_PREFIX = "{\"" + ENVELOPE_MARKER + "\":"; private static final String PAYLOAD_TYPE_FIELD = "payloadType"; private static final int ENVELOPE_VERSION = 1; private static final int CHECKPOINT_ENVELOPE_LIMIT_BYTES = 256 * 1024 - 1024; @@ -109,36 +113,24 @@ public String deserialize(String data) { if (data == null) { return null; } - var context = requireContext(); - return resolveSerializedPayload(data, context).value(); - } - - @Override - public SerDesStageResult deserializePipelineStage(String data) { - var context = requireContext(); - var resolved = resolveSerializedPayload(data, context); - return resolved.external() - ? SerDesStageResult.decodeWithValueCodec(resolved.value()) - : SerDesStageResult.continueWith(resolved.value()); + return resolveSerializedPayload(data); } - private ResolvedPayload resolveSerializedPayload(String data, SerDesContext context) { + private String resolveSerializedPayload(String data) { final JsonNode envelope; try { envelope = ENVELOPE_MAPPER.readTree(data); } catch (JsonProcessingException e) { - if (acceptsExternalPayload(context)) { - return new ResolvedPayload(data, true); + if (data.startsWith(ENVELOPE_PREFIX)) { + throw malformedEnvelope(requireContext(), e); } - throw malformedEnvelope(context, e); + return data; } if (!hasFilesystemMarker(envelope)) { - if (acceptsExternalPayload(context)) { - return new ResolvedPayload(data, true); - } - throw malformedEnvelope(context, null); + return data; } + var context = requireContext(); var marker = envelope.get(ENVELOPE_MARKER); if (!marker.isIntegralNumber()) { throw malformedEnvelope(context, null); @@ -156,19 +148,15 @@ private ResolvedPayload resolveSerializedPayload(String data, SerDesContext cont var owner = payloadOwner(envelope, context); if (hasData) { try { - return new ResolvedPayload( - SerializedPayload.fromInlineValue( - payloadType, envelope.get("data").textValue()) - .value(), - false); + return SerializedPayload.fromInlineValue( + payloadType, envelope.get("data").textValue()) + .value(); } catch (IllegalArgumentException e) { throw malformedEnvelope(context, e); } } - return new ResolvedPayload( - readPayload(envelope.get("file").textValue(), payloadType, owner, context) - .value(), - false); + return readPayload(envelope.get("file").textValue(), payloadType, owner, context) + .value(); } private SerializedPayload readPayload( @@ -307,12 +295,6 @@ private static boolean hasFilesystemMarker(JsonNode envelope) { return envelope != null && envelope.isObject() && envelope.has(ENVELOPE_MARKER); } - private static boolean acceptsExternalPayload(SerDesContext context) { - return context.payloadKind() == SerDesPayloadKind.INPUT - || context.operationType() == OperationType.CALLBACK - || context.operationType() == OperationType.CHAINED_INVOKE; - } - private static SerDesException malformedEnvelope(SerDesContext context, Throwable cause) { var message = "Invalid filesystem SerDes envelope for entity '" + context.entityId() + "'"; return cause == null ? new SerDesException(message) : new SerDesException(message, cause); @@ -560,8 +542,6 @@ private static String sha256(byte[] value) { private record PayloadOwner(String durableExecutionArn, String entityId) {} - private record ResolvedPayload(String value, boolean external) {} - private enum PayloadType { STRING } diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/RetrySerDes.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/RetrySerDes.java index 947a77993..8a2c8b573 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/RetrySerDes.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/RetrySerDes.java @@ -57,11 +57,6 @@ public String deserialize(String data) { return execute("pipeline stage deserialization", () -> delegate.deserialize(data)); } - @Override - public SerDesStageResult deserializePipelineStage(String data) { - return execute("pipeline stage deserialization", () -> delegate.deserializePipelineStage(data)); - } - private T execute(String action, Supplier operation) { int attempt = 1; while (true) { diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesStage.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesStage.java index a70c9678c..20b567d20 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesStage.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesStage.java @@ -8,6 +8,18 @@ *

Every top-level stage consumes and produces a string, so stages can be composed in any order without intermediate * type mismatches. Use {@link ComposableBinarySerDesStage} to perform an efficient chain of binary transformations * inside one string stage. + * + *

A stage must serialize values into a self-identifying format. During deserialization, it must: + * + *

    + *
  • reverse input that is in its recognized, valid format; + *
  • throw a SerDes failure when input identifies itself as this stage's format but is malformed or unsupported; + *
  • return input unchanged when it is not in this stage's format. + *
+ * + *

This pass-through behavior allows raw external payloads to traverse a configured pipeline and reach its root value + * codec without special pipeline control flow. Implementations should inspect an explicit marker or versioned envelope + * before decoding rather than treating any successfully decodable value as recognized. */ public interface SerDesStage { /** @@ -21,22 +33,11 @@ public interface SerDesStage { /** * Reverses this stage during deserialization. * - * @param data the non-null serialized string produced by this stage - * @return the non-null string expected by the preceding stage - */ - String deserialize(String data); - - /** - * Reverses this stage with control over external-boundary processing. + *

If {@code data} is not in this stage's self-identifying format, implementations must return it unchanged. If + * it identifies this stage's format but is malformed or unsupported, implementations must throw a SerDes failure. * - *

Most stages should use the default result. Boundary stages may return - * {@link SerDesStageResult#decodeWithValueCodec(String)} when the input originated outside the configured pipeline - * and should bypass the remaining intermediate stages. - * - * @param data the non-null serialized form produced by this stage - * @return the stage result + * @param data the non-null input string + * @return the non-null string expected by the preceding stage, or {@code data} unchanged when unrecognized */ - default SerDesStageResult deserializePipelineStage(String data) { - return SerDesStageResult.continueWith(deserialize(data)); - } + String deserialize(String data); } diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesStageResult.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesStageResult.java deleted file mode 100644 index c0aefb4c6..000000000 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesStageResult.java +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 -package software.amazon.lambda.durable.serde; - -import java.util.Objects; - -/** - * Result returned when a {@link SerDesStage} is reversed in a {@link ComposableSerDes}. - * - * @param value the non-null value produced by the stage - * @param skipRemainingStages whether deserialization should skip the remaining intermediate stages and decode - * {@code value} directly with the pipeline's value codec - */ -public record SerDesStageResult(String value, boolean skipRemainingStages) { - public SerDesStageResult { - Objects.requireNonNull(value, "value cannot be null"); - } - - /** Continues reverse processing through the remaining intermediate stages. */ - public static SerDesStageResult continueWith(String value) { - return new SerDesStageResult(value, false); - } - - /** Skips the remaining intermediate stages and decodes the value directly with the pipeline's value codec. */ - public static SerDesStageResult decodeWithValueCodec(String value) { - return new SerDesStageResult(value, true); - } -} diff --git a/sdk/src/test/java/software/amazon/lambda/durable/serde/ComposableBinarySerDesStageTest.java b/sdk/src/test/java/software/amazon/lambda/durable/serde/ComposableBinarySerDesStageTest.java index ce4a94796..461d93b75 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/serde/ComposableBinarySerDesStageTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/serde/ComposableBinarySerDesStageTest.java @@ -19,21 +19,25 @@ import software.amazon.lambda.durable.exception.SerDesException; class ComposableBinarySerDesStageTest { + private static final String BINARY_FRAME_MARKER = "__durable_execution_composable_binary_serdes:"; + private static final String BINARY_FRAME_PREFIX = BINARY_FRAME_MARKER + "1:"; @Test - void processesBoundariesAndBinarySerDesInDeclarationOrder() { + void processesBoundariesAndBinaryStagesInDeclarationOrder() { var calls = new ArrayList(); var stage = ComposableBinarySerDesStage.builder() .startWith(recordingCodec("starting", calls)) - .then(appendingSerDes("first", (byte) 1, calls)) - .then(appendingSerDes("second", (byte) 2, calls)) + .then(appendingStage("first", (byte) 1, calls)) + .then(appendingStage("second", (byte) 2, calls)) .endWith(recordingCodec("ending", calls)) .build(); var serialized = stage.serialize("value"); var deserialized = stage.deserialize(serialized); - assertEquals(Base64.getEncoder().encodeToString(new byte[] {'v', 'a', 'l', 'u', 'e', 1, 2}), serialized); + assertEquals( + BINARY_FRAME_PREFIX + Base64.getEncoder().encodeToString(new byte[] {'v', 'a', 'l', 'u', 'e', 1, 2}), + serialized); assertEquals("value", deserialized); assertEquals( List.of( @@ -52,7 +56,7 @@ void processesBoundariesAndBinarySerDesInDeclarationOrder() { void composesWithRootSerDesAsOneStringStage() { var stage = ComposableBinarySerDesStage.builder() .startWith(Utf8StringBinaryCodec.INSTANCE) - .then(xorSerDes((byte) 0x5A)) + .then(xorStage((byte) 0x5A)) .endWith(Base64StringBinaryCodec.INSTANCE) .build(); var pipeline = new JacksonSerDes().then(stage); @@ -86,10 +90,24 @@ public String fromBytes(byte[] data) { var serialized = stage.serialize("value"); - assertEquals(Base64.getEncoder().encodeToString("eulav".getBytes(StandardCharsets.UTF_8)), serialized); + assertEquals( + BINARY_FRAME_PREFIX + Base64.getEncoder().encodeToString("eulav".getBytes(StandardCharsets.UTF_8)), + serialized); assertEquals("value", stage.deserialize(serialized)); } + @Test + void passesThroughUnrecognizedInputAndRejectsInvalidFrames() { + var stage = ComposableBinarySerDesStage.builder() + .startWith(Utf8StringBinaryCodec.INSTANCE) + .endWith(Base64StringBinaryCodec.INSTANCE) + .build(); + + assertEquals("\"external\"", stage.deserialize("\"external\"")); + assertThrows(SerDesException.class, () -> stage.deserialize(BINARY_FRAME_MARKER + "2:value")); + assertThrows(SerDesException.class, () -> stage.deserialize(BINARY_FRAME_PREFIX + "not-base64!")); + } + @Test void validatesConfigurationAndComponentResults() { assertThrows(NullPointerException.class, () -> ComposableBinarySerDesStage.builder() @@ -103,7 +121,7 @@ void validatesConfigurationAndComponentResults() { var nullStage = ComposableBinarySerDesStage.builder() .startWith(Utf8StringBinaryCodec.INSTANCE) - .then(new BinarySerDes() { + .then(new BinarySerDesStage() { @Override public byte[] serialize(byte[] value) { return null; @@ -125,7 +143,7 @@ public byte[] deserialize(byte[] data) { void preservesRetryableFailuresAndFatalErrors() { var retryableStage = ComposableBinarySerDesStage.builder() .startWith(Utf8StringBinaryCodec.INSTANCE) - .then(new BinarySerDes() { + .then(new BinarySerDesStage() { @Override public byte[] serialize(byte[] value) { throw new RetryableSerDesException("retry"); @@ -199,8 +217,8 @@ public String fromBytes(byte[] data) { }; } - private static BinarySerDes appendingSerDes(String name, byte suffix, List calls) { - return new BinarySerDes() { + private static BinarySerDesStage appendingStage(String name, byte suffix, List calls) { + return new BinarySerDesStage() { @Override public byte[] serialize(byte[] value) { calls.add(name + "-serialize"); @@ -220,8 +238,8 @@ public byte[] deserialize(byte[] data) { }; } - private static BinarySerDes xorSerDes(byte key) { - return new BinarySerDes() { + private static BinarySerDesStage xorStage(byte key) { + return new BinarySerDesStage() { @Override public byte[] serialize(byte[] value) { return xor(value, key); diff --git a/sdk/src/test/java/software/amazon/lambda/durable/serde/ComposableSerDesTest.java b/sdk/src/test/java/software/amazon/lambda/durable/serde/ComposableSerDesTest.java index db6ed1ea0..03c927185 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/serde/ComposableSerDesTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/serde/ComposableSerDesTest.java @@ -138,40 +138,25 @@ public String deserialize(String data) { } @Test - void stageMayDecodeExternalDataDirectlyWithValueCodec() { - var transformDeserializations = new AtomicInteger(); - var transform = new SerDesStage() { - @Override - public String serialize(String value) { - return "<" + value + ">"; - } + void unrecognizedInputPassesThroughEveryStage() { + var calls = new ArrayList(); + var pipeline = new JacksonSerDes() + .then(stringStage("first", "<", ">", calls)) + .then(stringStage("second", "[", "]", calls)); - @Override - public String deserialize(String data) { - transformDeserializations.incrementAndGet(); - return data.substring(1, data.length() - 1); - } - }; - var externalBoundary = new SerDesStage() { - @Override - public String serialize(String value) { - return value; - } + assertEquals("value", pipeline.deserialize("\"value\"", TypeToken.get(String.class))); + assertEquals(List.of("second-deserialize", "first-deserialize"), calls); + } - @Override - public String deserialize(String data) { - return data; - } + @Test + void recognizedMalformedInputFailsAtTheOwningStage() { + var pipeline = new JacksonSerDes().then(stringStage("framed", "<", ">", new ArrayList<>())); - @Override - public SerDesStageResult deserializePipelineStage(String data) { - return SerDesStageResult.decodeWithValueCodec(data); - } - }; - var pipeline = new JacksonSerDes().then(transform).then(externalBoundary); + var failure = assertThrows( + SerDesException.class, () -> pipeline.deserialize("<\"value\"", TypeToken.get(String.class))); - assertEquals("value", pipeline.deserialize("\"value\"", TypeToken.get(String.class))); - assertEquals(0, transformDeserializations.get()); + assertTrue(failure.getMessage().contains("stage 1")); + assertTrue(failure.getCause().getMessage().contains("Malformed framed stage value")); } @Test @@ -242,11 +227,6 @@ public String serialize(String value) { @Override public String deserialize(String data) { - return null; - } - - @Override - public SerDesStageResult deserializePipelineStage(String data) { throw stringStageError; } }; @@ -281,6 +261,12 @@ public String serialize(String value) { @Override public String deserialize(String data) { calls.add(name + "-deserialize"); + if (!data.startsWith(prefix)) { + return data; + } + if (!data.endsWith(suffix) || data.length() < prefix.length() + suffix.length()) { + throw new SerDesException("Malformed " + name + " stage value"); + } return data.substring(prefix.length(), data.length() - suffix.length()); } }; diff --git a/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java b/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java index 9f7763409..3936290c4 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java @@ -91,7 +91,7 @@ void retryDecoratorComposesAsAFileSystemStage() { void storesAndRestoresComposableBinaryOutput() throws Exception { var binaryStage = ComposableBinarySerDesStage.builder() .startWith(Utf8StringBinaryCodec.INSTANCE) - .then(xorBinarySerDes((byte) 0x5A)) + .then(xorBinaryStage((byte) 0x5A)) .endWith(Base64StringBinaryCodec.INSTANCE) .build(); var stage = FileSystemSerDes.builder(basePath).build(); @@ -104,7 +104,9 @@ void storesAndRestoresComposableBinaryOutput() throws Exception { assertEquals("STRING", json.get("payloadType").textValue()); assertEquals( - Base64.getEncoder().encodeToString(xor("{\"id\":42}".getBytes(StandardCharsets.UTF_8), (byte) 0x5A)), + "__durable_execution_composable_binary_serdes:1:" + + Base64.getEncoder() + .encodeToString(xor("{\"id\":42}".getBytes(StandardCharsets.UTF_8), (byte) 0x5A)), Files.readString(file)); assertEquals( Map.of("id", 42), @@ -245,10 +247,11 @@ void includesBoundedPreviewWithoutChangingStoredPayload() throws Exception { } @Test - void acceptsExternallyOriginatedRawPayloadsOnlyForSupportedSources() { + void passesUnrecognizedPayloadsThroughAtEverySource() { var stage = FileSystemSerDes.builder(basePath).build(); var filesystemPipeline = new JacksonSerDes().then(stage); var pipeline = new JacksonSerDes().then(wrappingStage()).then(stage); + var pipelineWithStageAfterFilesystem = new JacksonSerDes().then(stage).then(wrappingStage()); var runner = new SerDesRunner(null); assertEquals( @@ -272,6 +275,13 @@ void acceptsExternallyOriginatedRawPayloadsOnlyForSupportedSources() { "\"invoke-result\"", TypeToken.get(String.class), operationContext(OperationType.CHAINED_INVOKE, OperationSubType.CHAINED_INVOKE))); + assertEquals( + Map.of("id", 42), + runner.deserialize( + pipelineWithStageAfterFilesystem, + "{\"id\":42}", + new TypeToken>() {}, + executionContext(SerDesPayloadKind.INPUT))); assertEquals( Map.of("domainMarker", 1, "data", "domain-value"), runner.deserialize( @@ -287,9 +297,9 @@ void acceptsExternallyOriginatedRawPayloadsOnlyForSupportedSources() { new TypeToken>() {}, executionContext(SerDesPayloadKind.INPUT))); - assertThrows( - SerDesException.class, - () -> runner.deserialize(filesystemPipeline, "\"raw-step\"", TypeToken.get(String.class), context())); + assertEquals( + "raw-step", + runner.deserialize(filesystemPipeline, "\"raw-step\"", TypeToken.get(String.class), context())); } @Test @@ -406,10 +416,14 @@ void rejectsCallsWithoutContextAndMalformedOrUnsafeEnvelopes() throws Exception var stage = FileSystemSerDes.builder(basePath).build(); var serDes = stringCodec().then(stage); assertThrows(SerDesException.class, () -> stage.serialize("value")); + assertEquals("value", stage.deserialize("value")); var runner = new SerDesRunner(null); + assertEquals("{}", runner.deserialize(serDes, "{}", TypeToken.get(String.class), context())); assertThrows( - SerDesException.class, () -> runner.deserialize(serDes, "{}", TypeToken.get(String.class), context())); + SerDesException.class, + () -> runner.deserialize( + serDes, "{\"__durable_execution_filesystem_serdes\":", TypeToken.get(String.class), context())); assertThrows( SerDesException.class, () -> runner.deserialize( @@ -571,8 +585,8 @@ private static SerDesContext operationContext(OperationType operationType, Opera ARN, "1", "operation", null, operationType, operationSubType, SerDesPayloadKind.RESULT, null); } - private static BinarySerDes xorBinarySerDes(byte key) { - return new BinarySerDes() { + private static BinarySerDesStage xorBinaryStage(byte key) { + return new BinarySerDesStage() { @Override public byte[] serialize(byte[] value) { return xor(value, key); @@ -602,6 +616,12 @@ public String serialize(String value) { @Override public String deserialize(String data) { + if (!data.startsWith("<")) { + return data; + } + if (!data.endsWith(">")) { + throw new SerDesException("Malformed wrapping stage value"); + } return data.substring(1, data.length() - 1); } }; diff --git a/sdk/src/test/java/software/amazon/lambda/durable/serde/RetrySerDesTest.java b/sdk/src/test/java/software/amazon/lambda/durable/serde/RetrySerDesTest.java index aeaec52cd..9a9c3dc04 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/serde/RetrySerDesTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/serde/RetrySerDesTest.java @@ -78,39 +78,6 @@ public String deserialize(String data) { assertEquals(2, calls.get()); } - @Test - void retriesPipelineStageDeserialization() { - var calls = new AtomicInteger(); - class RetryableStage implements SerDesStage { - @Override - public String serialize(String value) { - return value; - } - - @Override - public String deserialize(String data) { - return data; - } - - @Override - public SerDesStageResult deserializePipelineStage(String data) { - if (calls.incrementAndGet() == 1) { - throw new RetryableSerDesException("transient"); - } - return SerDesStageResult.decodeWithValueCodec(data); - } - } - var delegate = new RetryableStage(); - var retrySerDes = - new RetrySerDes(delegate, (error, attempt) -> RetryDecision.retry(Duration.ZERO), delay -> {}); - - var result = retrySerDes.deserializePipelineStage("value"); - - assertEquals("value", result.value()); - assertTrue(result.skipRemainingStages()); - assertEquals(2, calls.get()); - } - @Test void doesNotRetryPermanentSerDesFailure() { var calls = new AtomicInteger(); From c93d8016f5a03c0c8ceaccb31527e57f3d4802a6 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Tue, 25 Aug 2026 20:00:18 +0000 Subject: [PATCH 35/56] fix: complete filesystem SerDes parity --- docs/adr/005-filesystem-serdes.md | 21 ++- docs/advanced/configuration.md | 29 ++- docs/advanced/filesystem-serdes.md | 52 ++++-- docs/design.md | 5 + .../lambda/durable/serde/FieldMatchMode.java | 12 ++ .../durable/serde/FileSystemSerDes.java | 92 ++++----- .../lambda/durable/serde/PreviewConfig.java | 113 +++++++++++ .../lambda/durable/serde/PreviewField.java | 36 ++++ .../lambda/durable/serde/PreviewMode.java | 12 ++ .../lambda/durable/serde/SerDesPreview.java | 176 ++++++++++++++++++ .../durable/serde/FileSystemSerDesTest.java | 80 +++++--- .../durable/serde/SerDesPreviewTest.java | 152 +++++++++++++++ 12 files changed, 679 insertions(+), 101 deletions(-) create mode 100644 sdk/src/main/java/software/amazon/lambda/durable/serde/FieldMatchMode.java create mode 100644 sdk/src/main/java/software/amazon/lambda/durable/serde/PreviewConfig.java create mode 100644 sdk/src/main/java/software/amazon/lambda/durable/serde/PreviewField.java create mode 100644 sdk/src/main/java/software/amazon/lambda/durable/serde/PreviewMode.java create mode 100644 sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesPreview.java create mode 100644 sdk/src/test/java/software/amazon/lambda/durable/serde/SerDesPreviewTest.java diff --git a/docs/adr/005-filesystem-serdes.md b/docs/adr/005-filesystem-serdes.md index a63b182fa..28f06fa78 100644 --- a/docs/adr/005-filesystem-serdes.md +++ b/docs/adr/005-filesystem-serdes.md @@ -405,14 +405,14 @@ source. If the marker is present, the value is recognized as filesystem data and malformed marked envelopes and unsupported versions fail rather than falling back to pass-through behavior. A recognized filesystem envelope also requires an SDK-managed `SerDesContext`, while unrecognized input does not. -Offloaded filenames include a content hash and are immutable. Serializing new state for the same entity creates a new -path instead of replacing a file referenced by an earlier checkpoint. Repeating the same serialization may reuse the -same content-addressed file. +Offloaded filenames include the entity identity, content hash, and a UUID. Each serialization publishes a new immutable +file with one `CREATE_NEW` write. It does not require hard links or renames, making the write path compatible with S3 +Files as well as EFS. Serializing new state never replaces a file referenced by an earlier checkpoint. File envelopes identify the execution ARN and entity that produced the content. Normal checkpoint replay requires that owner to match the current context. Initial input and chained-invoke result boundaries may consume a reference owned by the other Lambda execution, allowing two functions configured with the same durable filesystem root and path -encoding to exchange offloaded invoke payloads and results. The declared owner must still match the content-addressed +encoding to exchange offloaded invoke payloads and results. The declared owner must still match the content-hashed path, and the resolved file must remain beneath the configured root. The file envelope is therefore a capability and must be protected with the same care as the payload it references. @@ -425,8 +425,11 @@ later stages. During deserialization, each stage validates its own reserved form through unchanged. Therefore raw external data can traverse stages on either side of `FileSystemSerDes` without being decoded as a pipeline value. -The preview generator receives the `String` produced by the preceding stage, not the original domain object. A preview -that needs domain fields should parse that representation or be produced by an earlier stage. +For JSON pipelines, `previewConfig(...)` parses the `String` produced by the preceding stage and provides the same +structured preview controls as the Python and TypeScript SDKs: include-all or exclude-all mode, include/exclude/mask +selectors, field-name or exact-path matching, a configurable mask string, and a default 4 KB byte budget. The +standalone `SerDesPreview` utility exposes the same builder for customer-managed values. `previewGenerator(...)` +remains available for non-JSON stage values and fully custom preview logic. ### Runtime flow @@ -571,9 +574,9 @@ persisted pipeline. serialized data hash. 9. Update exception serialization and deserialization paths to set `SerDesPayloadKind.EXCEPTION` in TLS. 10. Implement `FileSystemSerDes` in the core `software.amazon.lambda.durable.serde` package as a string stage with - `ALWAYS` and `OVERFLOW` storage, `URI` and `HASH` path encodings, envelope parsing, atomic file writes where - supported by the filesystem, retryable I/O failures, unrecognized-input pass-through, and clear validation errors - for recognized malformed input. + `ALWAYS` and `OVERFLOW` storage, `URI` and `HASH` path encodings, envelope parsing, immutable `CREATE_NEW` writes + compatible with EFS and S3 Files, retryable I/O failures, unrecognized-input pass-through, structured preview + generation, and clear validation errors for recognized malformed input. 11. Add unit tests for string and binary pipeline ordering, custom boundary codecs, reverse processing, nulls, stage failures, retry selection, exhaustion, delay handling, interruption, context construction, TLS scoping and restoration, inline execution, configured thread-pool isolation, cache hits, cache invalidation, exception diff --git a/docs/advanced/configuration.md b/docs/advanced/configuration.md index b98f7ce74..3e46867c4 100644 --- a/docs/advanced/configuration.md +++ b/docs/advanced/configuration.md @@ -59,7 +59,6 @@ The core SDK provides a reversible stage for storing serialized strings on a sha var fileSystemStage = FileSystemSerDes.builder(Path.of("/mnt/efs/durable-payloads")) .storageMode(FileSystemStorageMode.OVERFLOW) .pathEncoding(FileSystemPathEncoding.HASH) - .previewGenerator(json -> Map.of("format", "json")) .build(); var resilientFileSystemStage = new RetrySerDes( @@ -97,18 +96,38 @@ binary chain, and the outer stage adds a reserved versioned frame for reliable f `ALWAYS` stores every non-null payload in a file. `OVERFLOW` keeps payloads inline until the checkpoint envelope approaches the 256 KB service limit. `URI` produces readable escaped paths; `HASH` produces fixed-length SHA-256 path -segments. Files are content-addressed and never overwrite data referenced by an earlier checkpoint. References are +segments. Files include a content hash and never overwrite data referenced by an earlier checkpoint. References are validated against the current durable execution and entity, and symbolic-link paths are rejected. +For a `JacksonSerDes -> FileSystemSerDes` pipeline, structured preview configuration provides the same field +selection, masking, exact-path matching, and default 4 KB preview budget as the Python and TypeScript SDKs: + +```java +var previewConfig = PreviewConfig.builder(PreviewMode.EXCLUDE_ALL) + .include(PreviewField.anywhere("id"), PreviewField.path("customer.status")) + .mask(PreviewField.anywhere("email")) + .build(); + +var fileSystemStage = FileSystemSerDes.builder(Path.of("/mnt/s3/durable-payloads")) + .previewConfig(previewConfig) + .build(); + +var serDes = new JacksonSerDes().then(fileSystemStage); +``` + +The built-in preview configuration parses the string produced by the preceding stage as JSON. Use +`previewGenerator(...)` when the preceding stage produces another format or when fully custom preview logic is needed. + `RetrySerDes` implements `SerDesStage` and retries only failures marked with `RetryableSerDesException`. Filesystem read and write I/O use this marker; malformed envelopes and codec failures fail immediately. Backoff occurs within the current Lambda invocation, so use short, bounded retry strategies. Without a configured SerDes executor, filesystem I/O and retry delays block the calling thread. Do not use Lambda's ephemeral `/tmp` directory: replay can run in a different execution environment. Use a durable, -shared mount such as EFS. S3 Files can have delayed synchronization, so a runtime crash before the mount flushes may -lose recent writes; use it only when that tradeoff is acceptable. The SDK does not delete offloaded files, so configure -storage lifecycle and retention separately. +shared mount such as EFS or S3 Files. Payloads are published with one immutable `CREATE_NEW` write, without hard links +or renames, so the write path is compatible with S3 Files. S3 Files can have delayed synchronization, so a runtime +crash before the mount flushes may lose recent writes; use it only when that tradeoff is acceptable. The SDK does not +delete offloaded files, so configure storage lifecycle and retention separately. The cloud and local test runners always serialize the initial Lambda invocation with a separate context-free value codec. By default, this is the configured persisted SerDes when it is a plain value codec, or the root value codec when diff --git a/docs/advanced/filesystem-serdes.md b/docs/advanced/filesystem-serdes.md index 983a8c411..01a3150a7 100644 --- a/docs/advanced/filesystem-serdes.md +++ b/docs/advanced/filesystem-serdes.md @@ -21,7 +21,6 @@ envelopes in checkpoints. It is included in the core `aws-durable-execution-sdk- var fileSystemStage = FileSystemSerDes.builder(Path.of("/mnt/efs/durable-payloads")) .storageMode(FileSystemStorageMode.ALWAYS) .pathEncoding(FileSystemPathEncoding.URI) - .previewGenerator(json -> Map.of("format", "json")) .build(); var resilientFileSystemStage = new RetrySerDes( @@ -61,8 +60,33 @@ compression/encryption chain. - `URI` uses readable escaped path segments. - `HASH` uses fixed-length SHA-256 path segments. -The preview generator receives the incoming stage string. Its output is included only in file envelopes and the final -envelope must remain below the checkpoint threshold. +## Structured previews + +Java includes the same structured preview controls as the Python and TypeScript SDKs: + +```java +var previewConfig = PreviewConfig.builder(PreviewMode.EXCLUDE_ALL) + .include(PreviewField.anywhere("id"), PreviewField.path("customer.status")) + .exclude(PreviewField.anywhere("internal")) + .mask(PreviewField.anywhere("email")) + .maskString("***") + .maxPreviewBytes(4096) + .build(); + +var fileSystemStage = FileSystemSerDes.builder(Path.of("/mnt/s3/durable-payloads")) + .previewConfig(previewConfig) + .build(); + +var serDes = new JacksonSerDes().then(fileSystemStage); +``` + +`INCLUDE_ALL` starts with every leaf visible and applies exclude and mask rules. `EXCLUDE_ALL` starts with no fields +visible; include and mask rules make selected fields visible. `ANYWHERE` matches a field name at any depth, while +`PATH` matches an exact dot-separated path. Exclude rules win over mask rules, and masking implies visibility. + +The built-in `previewConfig(...)` parses the incoming stage string as JSON. Use `previewGenerator(...)` for non-JSON +stage values or fully custom logic. Previews are included only in file envelopes. The structured builder defaults to a +4 KB preview budget, and the complete file envelope must still remain below the checkpoint threshold. ## Execution and retries @@ -83,15 +107,14 @@ to the pipeline value codec. Payloads containing the reserved marker must be val malformed marked envelopes and unsupported versions fail instead of falling back to pass-through behavior. An unrecognized value does not require `SerDesContext`; a recognized filesystem envelope does. -Offloaded files are content-addressed and immutable. Updating wait-for-condition state or retry results creates a new -path instead of replacing a file referenced by an earlier checkpoint. Publication uses an atomic hard-link -create-if-absent operation when the provider supports it, allowing repeated identical writes to reuse one file. On -providers that do not support hard links, it retains the completed, uniquely named content-addressed staging file. The -fallback path is not exposed in an envelope until its write completes, so a crash can leave only an unreferenced orphan -rather than a partial checkpoint target. File envelopes identify the producing execution and entity. Ordinary -checkpoint replay must match that owner, while invoke input and result boundaries may consume a file owned by the -other Lambda execution when both functions use the same shared root and path encoding. Treat file envelopes as -capabilities. Content hashes are verified when reading, and symbolic-link paths are rejected. +Offloaded files are content-hashed and immutable. Every serialization uses a unique filename containing the entity +identity, content hash, and UUID, and publishes it with one `CREATE_NEW` write. Existing files are never overwritten, +and publication does not require hard links or renames, making it compatible with both EFS and S3 Files. A failed write +can leave only an unreferenced orphan rather than replacing data referenced by an earlier checkpoint. File envelopes +identify the producing execution and entity. Ordinary checkpoint replay must match that owner, while invoke input and +result boundaries may consume a file owned by the other Lambda execution when both functions use the same shared root +and path encoding. Treat file envelopes as capabilities. Content hashes are verified when reading, and symbolic-link +paths are rejected. Stages may follow `FileSystemSerDes` to transform its inline or file-reference envelope. The filesystem stage's `OVERFLOW` and preview-size checks apply before those later transformations, so account for any expansion when staying @@ -112,7 +135,8 @@ persisted pipeline's root value codec can decode. Do not use Lambda's ephemeral `/tmp` directory. Durable replay may run in another execution environment where that file does not exist. -Use a durable shared mount such as EFS. S3 Files can synchronize writes asynchronously, so a runtime crash before a -flush can lose recent data; use it only when that durability tradeoff is acceptable. +Use a durable shared mount such as EFS or S3 Files. The SDK does not rely on hard links or renames, which S3 Files does +not support. S3 Files can synchronize writes asynchronously, so a runtime crash before a flush can lose recent data; +use it only when that durability tradeoff is acceptable. The SDK does not delete payload files. Configure an appropriate retention or lifecycle policy for the backing storage. diff --git a/docs/design.md b/docs/design.md index 937d0b611..7755d8cbd 100644 --- a/docs/design.md +++ b/docs/design.md @@ -364,6 +364,11 @@ software.amazon.lambda.durable │ ├── ComposableBinarySerDesStage # Ordered binary chain exposed as one string stage │ ├── JacksonSerDes # Jackson impl │ ├── RetrySerDes # Retrying string-stage decorator +│ ├── SerDesPreview # Structured preview builder +│ ├── PreviewConfig # Preview selection, masking, and size configuration +│ ├── PreviewField # Field-name or exact-path preview selector +│ ├── PreviewMode # Include-all or exclude-all preview default +│ ├── FieldMatchMode # Anywhere or exact-path field matching │ ├── SerDesRunner # Context, optional executor dispatch, and invocation cache │ ├── SerDesContext # Read-only durable payload identity │ ├── SerDesPayloadKind # Input/result/state/exception/invoke payload kind diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/FieldMatchMode.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/FieldMatchMode.java new file mode 100644 index 000000000..36cf1e726 --- /dev/null +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/FieldMatchMode.java @@ -0,0 +1,12 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.serde; + +/** Controls how a {@link PreviewField} matches a field in a structured value. */ +public enum FieldMatchMode { + /** Matches the field name at any depth in the object tree. */ + ANYWHERE, + + /** Matches the exact dot-separated path from the root object. */ + PATH +} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java index e47a4e7ea..59fd0b4ed 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java @@ -11,13 +11,14 @@ import java.nio.file.LinkOption; import java.nio.file.NoSuchFileException; import java.nio.file.Path; +import java.nio.file.StandardOpenOption; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; -import java.util.Arrays; import java.util.HexFormat; import java.util.LinkedHashMap; import java.util.Map; import java.util.Objects; +import java.util.UUID; import java.util.function.Function; import java.util.regex.Pattern; import software.amazon.awssdk.services.lambda.model.OperationType; @@ -30,6 +31,9 @@ *

Do not use Lambda's ephemeral {@code /tmp} storage. Use a durable shared mount such as EFS, or S3 Files only when * its synchronization and crash-durability tradeoffs are acceptable for the workload. * + *

Payload files are immutable and created with a single {@code CREATE_NEW} write. Publication does not require hard + * links or renames, so the write path is compatible with S3 Files. + * *

Deserialization recognizes the reserved filesystem envelope marker. Input without that marker is returned * unchanged; input with the marker must be a valid supported envelope. */ @@ -91,17 +95,8 @@ public String serialize(String value) { + "'"); } try { - var publishedFile = writePayload(payload, file); - if (publishedFile.equals(file)) { - return fileEnvelope; - } - var fallbackEnvelope = encodeEnvelope(payload.withoutData(), publishedFile, preview, context); - if (!fitsCheckpoint(fallbackEnvelope)) { - throw new SerDesException("Filesystem SerDes envelope exceeds the checkpoint payload limit for entity '" - + context.entityId() - + "'"); - } - return fallbackEnvelope; + writePayload(payload, file); + return fileEnvelope; } catch (IOException e) { throw new RetryableSerDesException( "Failed to store filesystem payload for entity '" + context.entityId() + "'", e); @@ -176,7 +171,7 @@ private SerializedPayload readPayload( var serialized = new SerializedPayload(payloadType, Files.readAllBytes(realFile)); var expectedFileName = payloadFileName(serialized, owner.entityId()); if (!matchesPublishedPayloadFileName(realFile.getFileName().toString(), expectedFileName)) { - throw new SerDesException("Filesystem SerDes file content does not match its content-addressed path"); + throw new SerDesException("Filesystem SerDes file content hash does not match its path"); } return serialized; } catch (IOException e) { @@ -362,7 +357,12 @@ private SerDesContext requireContext() { private Path resolvePayloadPath(SerializedPayload payload, SerDesContext context) { var directory = resolveExecutionDirectory(context.durableExecutionArn()); - var fileName = payloadFileName(payload, context.entityId()); + var deterministicName = payloadFileName(payload, context.entityId()); + var suffix = ".payload"; + var fileName = deterministicName.substring(0, deterministicName.length() - suffix.length()) + + "-" + + UUID.randomUUID() + + suffix; var file = directory.resolve(fileName).normalize(); if (!file.startsWith(directory)) { throw new SerDesException("Resolved filesystem payload path is outside the execution directory"); @@ -374,7 +374,7 @@ private String payloadFileName(SerializedPayload payload, String entityId) { return encode(entityId) + "-" + sha256(payload.data()) + ".payload"; } - private Path writePayload(SerializedPayload payload, Path file) throws IOException { + private void writePayload(SerializedPayload payload, Path file) throws IOException { var directory = file.getParent(); var realBasePath = createDirectoriesWithoutSymbolicLinks(directory); rejectSymbolicLinks(file); @@ -382,24 +382,17 @@ private Path writePayload(SerializedPayload payload, Path file) throws IOExcepti if (!realDirectory.startsWith(realBasePath)) { throw new SerDesException("Filesystem SerDes directory resolves outside the configured base path"); } - if (Files.exists(file, LinkOption.NOFOLLOW_LINKS)) { - validateExistingPayload(file, payload.data()); - return file; - } - - var fileName = file.getFileName().toString(); - var temporary = Files.createTempFile( - directory, fileName.substring(0, fileName.length() - ".payload".length()) + "-", ".payload"); - var retainTemporary = false; try { - Files.write(temporary, payload.data()); - var publishedFile = publishWithoutReplacement(temporary, file, payload.data()); - retainTemporary = publishedFile.equals(temporary); - return publishedFile; - } finally { - if (!retainTemporary) { - Files.deleteIfExists(temporary); + Files.write(file, payload.data(), StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE); + } catch (FileAlreadyExistsException failure) { + throw failure; + } catch (IOException failure) { + try { + Files.deleteIfExists(file); + } catch (IOException cleanupFailure) { + failure.addSuppressed(cleanupFailure); } + throw failure; } } @@ -451,19 +444,6 @@ private synchronized Path retainCanonicalBasePath(Path currentBasePath) { return canonicalBasePath; } - private Path publishWithoutReplacement(Path temporary, Path file, byte[] expectedData) throws IOException { - try { - Files.createLink(file, temporary); - return file; - } catch (FileAlreadyExistsException ignored) { - validateExistingPayload(file, expectedData); - return file; - } catch (UnsupportedOperationException | IOException ignored) { - validateExistingPayload(temporary, expectedData); - return temporary; - } - } - private static boolean matchesPublishedPayloadFileName(String actualFileName, String expectedFileName) { if (actualFileName.equals(expectedFileName)) { return true; @@ -473,14 +453,6 @@ private static boolean matchesPublishedPayloadFileName(String actualFileName, St return actualFileName.startsWith(expectedPrefix + "-") && actualFileName.endsWith(suffix); } - private void validateExistingPayload(Path file, byte[] expectedData) throws IOException { - rejectSymbolicLinks(file); - var existing = Files.readAllBytes(file); - if (!Arrays.equals(existing, expectedData)) { - throw new SerDesException("Filesystem SerDes content-addressed file contains unexpected data"); - } - } - private Path resolveExecutionDirectory(String durableExecutionArn) { Path directory; if (pathEncoding == FileSystemPathEncoding.URI) { @@ -609,11 +581,27 @@ public Builder pathEncoding(FileSystemPathEncoding pathEncoding) { return this; } + /** + * Configures a custom preview generator that receives the string produced by the preceding pipeline stage. + * + *

The returned preview is included only when the payload is stored in a file. + */ public Builder previewGenerator(Function> previewGenerator) { this.previewGenerator = Objects.requireNonNull(previewGenerator, "previewGenerator cannot be null"); return this; } + /** + * Configures structured preview generation for JSON produced by the preceding stage. + * + *

Use {@link #previewGenerator(Function)} for non-JSON stage values or fully custom preview logic. + */ + public Builder previewConfig(PreviewConfig previewConfig) { + Objects.requireNonNull(previewConfig, "previewConfig cannot be null"); + this.previewGenerator = value -> SerDesPreview.buildPreviewFromJson(value, previewConfig); + return this; + } + public FileSystemSerDes build() { return new FileSystemSerDes(this); } diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/PreviewConfig.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/PreviewConfig.java new file mode 100644 index 000000000..7d2f8ebf4 --- /dev/null +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/PreviewConfig.java @@ -0,0 +1,113 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.serde; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; + +/** + * Configuration for {@link SerDesPreview#buildPreview(Object, PreviewConfig)}. + * + * @param mode whether fields are included or excluded by default + * @param include fields made visible in {@link PreviewMode#EXCLUDE_ALL} mode + * @param exclude fields hidden from the preview + * @param mask fields made visible with their values replaced by {@code maskString} + * @param maskString replacement for masked field values + * @param maxPreviewBytes maximum estimated UTF-8 size of accepted preview entries + */ +public record PreviewConfig( + PreviewMode mode, + List include, + List exclude, + List mask, + String maskString, + int maxPreviewBytes) { + public static final String DEFAULT_MASK_STRING = "***"; + public static final int DEFAULT_MAX_PREVIEW_BYTES = 4096; + + public PreviewConfig { + Objects.requireNonNull(mode, "mode cannot be null"); + include = immutableFields(include, "include"); + exclude = immutableFields(exclude, "exclude"); + mask = immutableFields(mask, "mask"); + Objects.requireNonNull(maskString, "maskString cannot be null"); + if (maxPreviewBytes < 0) { + throw new IllegalArgumentException("maxPreviewBytes cannot be negative"); + } + } + + /** Creates a preview configuration builder. */ + public static Builder builder(PreviewMode mode) { + return new Builder(mode); + } + + private static List immutableFields(List fields, String name) { + Objects.requireNonNull(fields, name + " cannot be null"); + if (fields.stream().anyMatch(Objects::isNull)) { + throw new NullPointerException(name + " cannot contain null"); + } + return List.copyOf(fields); + } + + /** Builder for {@link PreviewConfig}. */ + public static final class Builder { + private final PreviewMode mode; + private final List include = new ArrayList<>(); + private final List exclude = new ArrayList<>(); + private final List mask = new ArrayList<>(); + private String maskString = DEFAULT_MASK_STRING; + private int maxPreviewBytes = DEFAULT_MAX_PREVIEW_BYTES; + + private Builder(PreviewMode mode) { + this.mode = Objects.requireNonNull(mode, "mode cannot be null"); + } + + /** Adds fields that should be visible. */ + public Builder include(PreviewField... fields) { + include.addAll(validFields(fields, "include")); + return this; + } + + /** Adds fields that should be hidden. */ + public Builder exclude(PreviewField... fields) { + exclude.addAll(validFields(fields, "exclude")); + return this; + } + + /** Adds fields whose values should be masked. */ + public Builder mask(PreviewField... fields) { + mask.addAll(validFields(fields, "mask")); + return this; + } + + /** Sets the value used for masked fields. */ + public Builder maskString(String maskString) { + this.maskString = Objects.requireNonNull(maskString, "maskString cannot be null"); + return this; + } + + /** Sets the maximum estimated UTF-8 preview size. */ + public Builder maxPreviewBytes(int maxPreviewBytes) { + if (maxPreviewBytes < 0) { + throw new IllegalArgumentException("maxPreviewBytes cannot be negative"); + } + this.maxPreviewBytes = maxPreviewBytes; + return this; + } + + /** Returns the immutable preview configuration. */ + public PreviewConfig build() { + return new PreviewConfig(mode, include, exclude, mask, maskString, maxPreviewBytes); + } + + private static List validFields(PreviewField[] fields, String name) { + Objects.requireNonNull(fields, name + " cannot be null"); + if (Arrays.stream(fields).anyMatch(Objects::isNull)) { + throw new NullPointerException(name + " cannot contain null"); + } + return List.of(fields); + } + } +} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/PreviewField.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/PreviewField.java new file mode 100644 index 000000000..49bd84304 --- /dev/null +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/PreviewField.java @@ -0,0 +1,36 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.serde; + +import java.util.Objects; + +/** + * A field selector used by {@link PreviewConfig}. + * + * @param name a field name for {@link FieldMatchMode#ANYWHERE}, or a dot-separated path for {@link FieldMatchMode#PATH} + * @param match how the selector is matched + */ +public record PreviewField(String name, FieldMatchMode match) { + public PreviewField { + Objects.requireNonNull(name, "name cannot be null"); + Objects.requireNonNull(match, "match cannot be null"); + if (name.isBlank()) { + throw new IllegalArgumentException("name cannot be blank"); + } + } + + /** Creates a selector that matches this field name at any depth. */ + public PreviewField(String name) { + this(name, FieldMatchMode.ANYWHERE); + } + + /** Creates a selector that matches this field name at any depth. */ + public static PreviewField anywhere(String name) { + return new PreviewField(name, FieldMatchMode.ANYWHERE); + } + + /** Creates a selector that matches an exact dot-separated path. */ + public static PreviewField path(String name) { + return new PreviewField(name, FieldMatchMode.PATH); + } +} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/PreviewMode.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/PreviewMode.java new file mode 100644 index 000000000..8dd0f4851 --- /dev/null +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/PreviewMode.java @@ -0,0 +1,12 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.serde; + +/** Controls which fields are visible by default in a structured payload preview. */ +public enum PreviewMode { + /** Includes every field unless an exclude rule removes it. */ + INCLUDE_ALL, + + /** Excludes every field unless an include or mask rule selects it. */ + EXCLUDE_ALL +} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesPreview.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesPreview.java new file mode 100644 index 000000000..9bcd64ca5 --- /dev/null +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesPreview.java @@ -0,0 +1,176 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.serde; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import software.amazon.lambda.durable.exception.SerDesException; + +/** Utilities for building compact structured previews for externally stored SerDes payloads. */ +public final class SerDesPreview { + private static final ObjectMapper MAPPER = new ObjectMapper(); + + private SerDesPreview() {} + + /** + * Builds a preview from an object using include, exclude, mask, path-matching, and byte-budget rules. + * + *

Object fields are traversed in their Jackson serialization order. Arrays are flattened into their containing + * path, matching the Python and TypeScript preview behavior. Fields whose names contain dots are skipped because + * they cannot be distinguished from dot-separated paths. + * + * @return a nested preview map, or {@code null} when no fields are visible + */ + public static Map buildPreview(Object value, PreviewConfig config) { + Objects.requireNonNull(config, "config cannot be null"); + final JsonNode root; + try { + root = MAPPER.valueToTree(value); + } catch (IllegalArgumentException e) { + throw new SerDesException("Failed to convert value for preview generation", e); + } + return buildPreview(root, config); + } + + /** + * Builds a preview from a JSON string. + * + *

This is used by {@link FileSystemSerDes.Builder#previewConfig(PreviewConfig)}, because a pipeline stage + * receives the serialized string produced by the preceding stage. + * + * @return a nested preview map, or {@code null} when no fields are visible + */ + public static Map buildPreviewFromJson(String value, PreviewConfig config) { + Objects.requireNonNull(value, "value cannot be null"); + Objects.requireNonNull(config, "config cannot be null"); + try { + return buildPreview(MAPPER.readTree(value), config); + } catch (JsonProcessingException e) { + throw new SerDesException("Built-in preview generation requires a JSON stage value", e); + } + } + + private static Map buildPreview(JsonNode root, PreviewConfig config) { + if (root == null || !root.isObject()) { + return null; + } + + var pairs = new ArrayList(); + collect(root, "", config, pairs); + if (pairs.isEmpty()) { + return null; + } + + var accepted = new ArrayList(); + int estimatedSize = 2; + for (var pair : pairs) { + int entrySize = previewEntrySize(pair); + if (estimatedSize + entrySize > config.maxPreviewBytes()) { + break; + } + accepted.add(pair); + estimatedSize += entrySize; + } + if (accepted.isEmpty()) { + return null; + } + + Map result = new LinkedHashMap<>(); + for (var pair : accepted) { + insert(result, pair.path(), pair.value()); + } + return result; + } + + private static void collect(JsonNode node, String pathPrefix, PreviewConfig config, List pairs) { + if (node == null || node.isNull()) { + return; + } + if (node.isArray()) { + for (var item : node) { + collect(item, pathPrefix, config, pairs); + } + return; + } + if (!node.isObject()) { + return; + } + + for (var field : node.properties()) { + var name = field.getKey(); + if (name.contains(".")) { + continue; + } + var path = pathPrefix.isEmpty() ? name : pathPrefix + "." + name; + var masked = isMatched(path, config.mask()); + var excluded = isMatched(path, config.exclude()); + var visible = !excluded + && (masked || config.mode() == PreviewMode.INCLUDE_ALL || isMatched(path, config.include())); + + if (!visible) { + if (!excluded) { + collect(field.getValue(), path, config, pairs); + } + continue; + } + if (masked) { + pairs.add(new PreviewEntry(path, config.maskString())); + } else if (field.getValue().isContainerNode()) { + collect(field.getValue(), path, config, pairs); + } else { + pairs.add(new PreviewEntry(path, MAPPER.convertValue(field.getValue(), Object.class))); + } + } + } + + private static boolean isMatched(String path, List fields) { + for (var field : fields) { + if (field.match() == FieldMatchMode.PATH) { + if (path.equals(field.name())) { + return true; + } + } else { + for (var segment : path.split("\\.")) { + if (segment.equals(field.name())) { + return true; + } + } + } + } + return false; + } + + private static int previewEntrySize(PreviewEntry entry) { + try { + var serialized = + MAPPER.writeValueAsString(entry.path()) + ":" + MAPPER.writeValueAsString(entry.value()) + ","; + return serialized.getBytes(StandardCharsets.UTF_8).length; + } catch (JsonProcessingException e) { + throw new SerDesException("Failed to estimate preview size", e); + } + } + + @SuppressWarnings("unchecked") + private static void insert(Map result, String path, Object value) { + var parts = path.split("\\."); + Map current = result; + for (int index = 0; index < parts.length - 1; index++) { + var existing = current.get(parts[index]); + if (!(existing instanceof Map)) { + existing = new LinkedHashMap(); + current.put(parts[index], existing); + } + current = (Map) existing; + } + current.put(parts[parts.length - 1], value); + } + + private record PreviewEntry(String path, Object value) {} +} diff --git a/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java b/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java index 9bd46fd36..ca4789159 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java @@ -147,21 +147,21 @@ void immutableContentAddressedFilesPreservePriorCheckpoints() throws Exception { } @Test - void rejectsUnexpectedContentInExistingContentAddressedFile() throws Exception { + void repeatedPayloadsUseDistinctImmutableFiles() throws Exception { var serDes = stringCodec().then(FileSystemSerDes.builder(basePath).build()); var runner = new SerDesRunner(null); - var envelope = runner.serialize(serDes, "expected", context()); - var file = payloadFile(envelope); - Files.writeString(file, "unexpected"); - - var failure = assertThrows(SerDesException.class, () -> runner.serialize(serDes, "expected", context())); + var firstEnvelope = runner.serialize(serDes, "expected", context()); + var secondEnvelope = runner.serialize(serDes, "expected", context()); + var firstFile = payloadFile(firstEnvelope); + var secondFile = payloadFile(secondEnvelope); - assertCauseMessage(failure, "contains unexpected data"); - assertEquals("unexpected", Files.readString(file)); + assertNotEquals(firstFile, secondFile); + assertEquals("expected", Files.readString(firstFile)); + assertEquals("expected", Files.readString(secondFile)); } @Test - void publishesOnFileSystemsWithoutHardLinkSupport() throws Exception { + void writesOnFileSystemsWithoutHardLinkSupport() throws Exception { var archive = basePath.resolve("payloads.zip"); try (var fileSystem = FileSystems.newFileSystem(URI.create("jar:" + archive.toUri()), Map.of("create", "true"))) { @@ -176,14 +176,9 @@ void publishesOnFileSystemsWithoutHardLinkSupport() throws Exception { MessageDigest.getInstance("SHA-256").digest("expected".getBytes(StandardCharsets.UTF_8))); var fileName = file.getFileName().toString(); assertTrue(fileName.contains(hash)); - var hashEnd = fileName.indexOf(hash) + hash.length(); - var deterministicFile = file.resolveSibling(fileName.substring(0, hashEnd) + ".payload"); assertEquals("expected", Files.readString(file)); - assertNotEquals(deterministicFile, file); - assertFalse(Files.exists(deterministicFile)); - assertTrue(fileName.startsWith( - deterministicFile.getFileName().toString().replace(".payload", "-"))); + assertTrue(fileName.matches(".*-" + hash + "-[0-9a-f-]{36}\\.payload")); assertTrue(fileName.endsWith(".payload")); assertEquals( "expected", @@ -200,7 +195,7 @@ void rejectsMalformedUtf8StringsAndFilePayloads() throws Exception { var envelope = runner.serialize(serDes, "valid", context()); var malformed = new byte[] {(byte) 0xC3, (byte) 0x28}; - var malformedFile = contentAddressedPath(payloadFile(envelope), malformed); + var malformedFile = contentHashedPath(payloadFile(envelope), malformed); Files.write(malformedFile, malformed); var malformedEnvelope = (ObjectNode) MAPPER.readTree(envelope); malformedEnvelope.put("file", malformedFile.toString()); @@ -221,7 +216,8 @@ void hashEncodingUsesFixedLengthSegments() throws Exception { var file = Path.of(MAPPER.readTree(envelope).get("file").textValue()); assertEquals(64, file.getParent().getFileName().toString().length()); - assertEquals(137, file.getFileName().toString().length()); + assertEquals(174, file.getFileName().toString().length()); + assertTrue(file.getFileName().toString().matches("[0-9a-f]{64}-[0-9a-f]{64}-[0-9a-f-]{36}\\.payload")); assertFalse(file.toString().contains("operation")); } @@ -249,6 +245,48 @@ void includesBoundedPreviewWithoutChangingStoredPayload() throws Exception { assertCauseMessage(failure, "checkpoint payload limit"); } + @Test + void structuredPreviewConfigSelectsAndMasksJsonFields() throws Exception { + var stage = FileSystemSerDes.builder(basePath) + .previewConfig(PreviewConfig.builder(PreviewMode.EXCLUDE_ALL) + .include(PreviewField.anywhere("id"), PreviewField.path("customer.status")) + .mask(PreviewField.anywhere("email")) + .build()) + .build(); + var serDes = new JacksonSerDes().then(stage); + var runner = new SerDesRunner(null); + + var value = Map.of( + "id", + "order-1", + "email", + "root@example.com", + "customer", + Map.of("status", "ready", "email", "customer@example.com", "secret", "hidden")); + var envelope = runner.serialize(serDes, value, context()); + var preview = MAPPER.readTree(envelope).get("preview"); + + assertEquals("order-1", preview.get("id").textValue()); + assertEquals("***", preview.get("email").textValue()); + assertEquals("ready", preview.get("customer").get("status").textValue()); + assertEquals("***", preview.get("customer").get("email").textValue()); + assertFalse(preview.get("customer").has("secret")); + assertEquals(value, runner.deserialize(serDes, envelope, new TypeToken>() {}, context())); + } + + @Test + void structuredPreviewConfigRequiresJsonStageValue() { + var stage = FileSystemSerDes.builder(basePath) + .previewConfig(PreviewConfig.builder(PreviewMode.INCLUDE_ALL).build()) + .build(); + var runner = new SerDesRunner(null); + + var failure = assertThrows( + SerDesException.class, () -> runner.serialize(stringCodec().then(stage), "not-json", context())); + + assertCauseMessage(failure, "requires a JSON stage value"); + } + @Test void passesUnrecognizedPayloadsThroughAtEverySource() { var stage = FileSystemSerDes.builder(basePath).build(); @@ -562,12 +600,12 @@ private static Path payloadFile(String envelope) { } } - private static Path contentAddressedPath(Path original, byte[] data) throws Exception { + private static Path contentHashedPath(Path original, byte[] data) throws Exception { var name = original.getFileName().toString(); - var suffix = ".payload"; - var hashStart = name.length() - suffix.length() - 64; + var existingHash = + HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(Files.readAllBytes(original))); var hash = HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(data)); - return original.resolveSibling(name.substring(0, hashStart) + hash + suffix); + return original.resolveSibling(name.replace(existingHash, hash)); } private static SerDesContext context() { diff --git a/sdk/src/test/java/software/amazon/lambda/durable/serde/SerDesPreviewTest.java b/sdk/src/test/java/software/amazon/lambda/durable/serde/SerDesPreviewTest.java new file mode 100644 index 000000000..4cc029c82 --- /dev/null +++ b/sdk/src/test/java/software/amazon/lambda/durable/serde/SerDesPreviewTest.java @@ -0,0 +1,152 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.serde; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.Test; +import software.amazon.lambda.durable.exception.SerDesException; + +class SerDesPreviewTest { + + @Test + void includeAllAppliesExcludeAndMaskRules() { + var value = Map.of( + "id", + "123", + "email", + "alice@example.com", + "ssn", + "000-00-0000", + "user", + Map.of("name", "Alice", "role", "admin")); + var config = PreviewConfig.builder(PreviewMode.INCLUDE_ALL) + .exclude(PreviewField.anywhere("role")) + .mask(PreviewField.anywhere("ssn")) + .build(); + + var preview = SerDesPreview.buildPreview(value, config); + + assertEquals("123", preview.get("id")); + assertEquals("***", preview.get("ssn")); + assertFalse(nested(preview, "user").containsKey("role")); + assertEquals("Alice", nested(preview, "user").get("name")); + } + + @Test + void excludeAllIncludesSelectedAndMaskedFields() { + var value = Map.of("id", "123", "email", "alice@example.com", "ssn", "000-00-0000"); + var config = PreviewConfig.builder(PreviewMode.EXCLUDE_ALL) + .include(PreviewField.anywhere("id")) + .mask(PreviewField.anywhere("ssn")) + .build(); + + var preview = SerDesPreview.buildPreview(value, config); + + assertEquals(Map.of("id", "123", "ssn", "***"), preview); + } + + @Test + void excludeWinsOverMask() { + var config = PreviewConfig.builder(PreviewMode.INCLUDE_ALL) + .exclude(PreviewField.anywhere("ssn")) + .mask(PreviewField.anywhere("ssn")) + .build(); + + var preview = SerDesPreview.buildPreview(Map.of("id", "123", "ssn", "secret"), config); + + assertEquals(Map.of("id", "123"), preview); + } + + @Test + void pathAndAnywhereMatchingHaveDifferentScopes() { + var value = Map.of("email", "root@example.com", "user", Map.of("email", "nested@example.com", "id", "user-1")); + var pathConfig = PreviewConfig.builder(PreviewMode.EXCLUDE_ALL) + .include(PreviewField.path("email")) + .build(); + var anywhereConfig = PreviewConfig.builder(PreviewMode.EXCLUDE_ALL) + .include(PreviewField.anywhere("email")) + .build(); + + var pathPreview = SerDesPreview.buildPreview(value, pathConfig); + var anywherePreview = SerDesPreview.buildPreview(value, anywhereConfig); + + assertEquals(Map.of("email", "root@example.com"), pathPreview); + assertEquals("root@example.com", anywherePreview.get("email")); + assertEquals("nested@example.com", nested(anywherePreview, "user").get("email")); + } + + @Test + void arraysMergeFieldsAtTheirContainingPath() { + var value = Map.of("items", List.of(Map.of("id", "first"), Map.of("email", "second@example.com"))); + var config = PreviewConfig.builder(PreviewMode.INCLUDE_ALL).build(); + + var preview = SerDesPreview.buildPreview(value, config); + + assertEquals(Map.of("id", "first", "email", "second@example.com"), nested(preview, "items")); + } + + @Test + void customMaskStringAndByteBudgetAreApplied() { + var value = new LinkedHashMap(); + value.put("first", "one"); + value.put("second", "two"); + value.put("secret", "hidden"); + var config = PreviewConfig.builder(PreviewMode.INCLUDE_ALL) + .mask(PreviewField.anywhere("secret")) + .maskString("[REDACTED]") + .maxPreviewBytes(18) + .build(); + + var preview = SerDesPreview.buildPreview(value, config); + + assertEquals(1, preview.size()); + assertTrue(preview.containsKey("first")); + } + + @Test + void returnsNullWhenNoFieldsAreVisibleOrValueIsNotAnObject() { + var config = PreviewConfig.builder(PreviewMode.EXCLUDE_ALL).build(); + + assertNull(SerDesPreview.buildPreview(Map.of("id", "123"), config)); + assertNull(SerDesPreview.buildPreview("value", config)); + assertNull(SerDesPreview.buildPreview(List.of(Map.of("id", "123")), config)); + } + + @Test + void jsonPreviewRejectsMalformedJsonAndSkipsDottedFieldNames() { + var config = PreviewConfig.builder(PreviewMode.INCLUDE_ALL).build(); + + assertThrows(SerDesException.class, () -> SerDesPreview.buildPreviewFromJson("not-json", config)); + assertEquals( + Map.of("safe", "value"), + SerDesPreview.buildPreviewFromJson("{\"safe\":\"value\",\"not.addressable\":\"secret\"}", config)); + } + + @Test + void validatesConfiguration() { + assertThrows(NullPointerException.class, () -> PreviewConfig.builder(null)); + assertNull(SerDesPreview.buildPreview( + Map.of("id", "123"), + PreviewConfig.builder(PreviewMode.INCLUDE_ALL) + .maxPreviewBytes(0) + .build())); + assertThrows(IllegalArgumentException.class, () -> PreviewConfig.builder(PreviewMode.INCLUDE_ALL) + .maxPreviewBytes(-1)); + assertThrows(IllegalArgumentException.class, () -> new PreviewField(" ")); + assertThrows(NullPointerException.class, () -> PreviewConfig.builder(PreviewMode.INCLUDE_ALL) + .include((PreviewField) null)); + } + + @SuppressWarnings("unchecked") + private static Map nested(Map value, String field) { + return (Map) value.get(field); + } +} From 82a5695b89d6967076e8899286ace695a195c944 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Tue, 25 Aug 2026 20:13:27 +0000 Subject: [PATCH 36/56] feat: pass context explicitly to SerDes stages --- docs/adr/005-filesystem-serdes.md | 96 ++++++++++--------- docs/advanced/configuration.md | 8 +- docs/advanced/filesystem-serdes.md | 6 +- .../FileSystemSerDesIntegrationTest.java | 4 +- .../testing/CloudDurableTestRunnerTest.java | 4 +- .../testing/LocalDurableTestRunnerTest.java | 9 +- .../durable/serde/BinarySerDesStage.java | 9 +- .../serde/ComposableBinarySerDesStage.java | 19 ++-- .../durable/serde/ComposableSerDes.java | 23 +++-- .../durable/serde/FileSystemSerDes.java | 20 ++-- .../lambda/durable/serde/RetrySerDes.java | 11 ++- .../lambda/durable/serde/SerDesContext.java | 13 ++- .../lambda/durable/serde/SerDesRunner.java | 19 +++- .../lambda/durable/serde/SerDesStage.java | 9 +- .../ComposableBinarySerDesStageTest.java | 68 +++++++++---- .../durable/serde/ComposableSerDesTest.java | 61 ++++++++---- .../durable/serde/FileSystemSerDesTest.java | 12 +-- .../lambda/durable/serde/RetrySerDesTest.java | 44 +++++---- 18 files changed, 276 insertions(+), 159 deletions(-) diff --git a/docs/adr/005-filesystem-serdes.md b/docs/adr/005-filesystem-serdes.md index 28f06fa78..27a70408a 100644 --- a/docs/adr/005-filesystem-serdes.md +++ b/docs/adr/005-filesystem-serdes.md @@ -40,20 +40,19 @@ uniform and preventing intermediate type mismatches. Binary transformations compose inside one `ComposableBinarySerDesStage`. That outer stage converts strings to bytes with a configurable starting codec, applies any number of reversible `BinarySerDesStage` implementations without intermediate text conversion, and converts the final bytes back to a string with a configurable ending codec. The -filesystem stage uses `SerDesContext.getCurrentContext()` to identify the durable execution and entity being -serialized. +filesystem stage receives `SerDesContext` explicitly to identify the durable execution and entity being serialized. ```java public interface SerDesStage { - String serialize(String value); + String serialize(String value, SerDesContext context); - String deserialize(String data); + String deserialize(String data, SerDesContext context); } public interface BinarySerDesStage { - byte[] serialize(byte[] value); + byte[] serialize(byte[] value, SerDesContext context); - byte[] deserialize(byte[] data); + byte[] deserialize(byte[] data, SerDesContext context); } public interface StringBinaryCodec { @@ -82,9 +81,10 @@ compression, encryption, and similar byte processing internally and encodes the filesystem when configured to do so and returns a small checkpoint envelope. It is only a `SerDesStage`; a value codec must precede it in a `ComposableSerDes`, and other string stages may follow it. -Because the existing `SerDes` methods do not accept context parameters, this approach needs a thread-local -`SerDesContext` so `FileSystemSerDes` can discover the current payload identity without changing the -`serialize`/`deserialize` signatures. +The existing `SerDes` methods remain unchanged. `SerDesRunner` passes `SerDesContext` directly into +`ComposableSerDes`, which forwards the same instance to every `SerDesStage` and nested `BinarySerDesStage` call. +The runner also installs the context in a thread-local compatibility accessor for existing root value codecs and +custom `SerDes` implementations whose backward-compatible methods cannot accept it explicitly. ```java public record SerDesContext( @@ -103,7 +103,10 @@ public record SerDesContext( } ``` -The SDK owns setting and clearing this thread-local value around SDK-managed SerDes calls. The setter should not be part of the public customer API; customers only read the current context. If SerDes is called directly by customer code outside the SDK, `getCurrentContext()` returns `null`. +The SDK owns setting and clearing this thread-local value around SDK-managed SerDes calls. The setter is not part of +the public customer API. Stage implementations use their context parameter; existing value codecs may read the +compatibility accessor. If SerDes is called directly by customer code outside the SDK, `getCurrentContext()` returns +`null`, and a stage invoked through that direct call receives `null`. ### Package @@ -192,18 +195,18 @@ checkpoint String Equivalent pseudocode: ```java -String serialize(Object value) { +String serialize(Object value, SerDesContext context) { String current = valueCodec.serialize(value); for (var stage : stages) { - current = stage.serialize(current); + current = stage.serialize(current, context); } return current; } - T deserialize(String data, TypeToken targetType) { + T deserialize(String data, TypeToken targetType, SerDesContext context) { String current = data; for (int i = stages.size() - 1; i >= 0; i--) { - current = stages.get(i).deserialize(current); + current = stages.get(i).deserialize(current, context); } return valueCodec.deserialize(current, targetType); } @@ -213,9 +216,9 @@ Pipeline rules: - A pipeline must contain exactly one value codec in the first position and zero or more string stages. - Every top-level stage consumes and produces a non-null `String`. This makes all stage orderings type-compatible. -- Context requirements are stage behavior rather than a capability on `SerDes`, `SerDesStage`, binary transformations, - or codecs. A context-dependent stage accesses `SerDesContext` when it runs and reports a normal SerDes failure when - the required context is unavailable. +- `SerDesRunner` passes the same read-only `SerDesContext` explicitly to every top-level and binary stage call. + A context-dependent stage validates the supplied parameter and reports a normal SerDes failure when it is + unavailable or incomplete. - The test runners identify a configured input pipeline directly as `ComposableSerDes` and reject it because initial input accepts one value codec, not a pipeline. - `SerDes.then(...)`, `ComposableSerDes.then(...)`, and the builder accept only `SerDesStage` after the value codec. @@ -225,7 +228,7 @@ Pipeline rules: - A `null` value at the pipeline boundary short-circuits the entire pipeline: serializing or deserializing `null` returns `null` without invoking any stage. A stage returning `null` for non-null input is an error. The value codec may decode a non-null representation such as the JSON literal `null` to a null domain value. -- All stages execute within the same `SerDesRunner` invocation and observe the same read-only `SerDesContext`, whether +- All stages execute within the same `SerDesRunner` invocation and receive the same read-only `SerDesContext`, whether the runner executes inline or dispatches to a configured executor. - `ComposableSerDes` is immutable. It is safe for concurrent use only when every contained stage is also safe for concurrent use, matching the existing `SerDes` requirement. @@ -243,7 +246,7 @@ Pipeline rules: Equivalent stage pseudocode: ```java -String deserialize(String data) { +String deserialize(String data, SerDesContext context) { if (!hasStageMarker(data)) { return data; } @@ -357,7 +360,7 @@ Retry rules: - Only `RetryableSerDesException` is retried. Ordinary `SerDesException` and other failures propagate immediately. - `RetryStrategy.makeRetryDecision(error, attempt)` receives the transient failure and a 1-based attempt number. - When the strategy returns `fail`, `RetrySerDes` rethrows the last `RetryableSerDesException`. -- The same read-only `SerDesContext` remains installed for every attempt because retrying happens inside the original +- The same read-only `SerDesContext` parameter is passed to every attempt because retrying happens inside the original `SerDesRunner` task. - A retry delay blocks the thread executing the SerDes call. This is the caller thread by default or a SerDes executor thread when one is explicitly configured. It is an in-invocation infrastructure retry, not a durable wait or @@ -394,16 +397,17 @@ Envelope format: {"__durable_execution_filesystem_serdes":1,"file":"","payloadType":"STRING","ownerDurableExecutionArn":"","ownerEntityId":"","preview":{ "...": "..." }} ``` -`FileSystemSerDes` must reject calls when `SerDesContext.getCurrentContext()` is `null` or does not include -`durableExecutionArn` and `entityId`. It accepts a `String`, records the payload type in the envelope, and restores that -string during reverse processing. A preceding `ComposableBinarySerDesStage` must encode its final bytes to a string -before filesystem storage. +`FileSystemSerDes` must reject recognized filesystem operations when its `SerDesContext` parameter is `null` or does +not include `durableExecutionArn` and `entityId`. It accepts a `String`, records the payload type in the envelope, and +restores that string during reverse processing. A preceding `ComposableBinarySerDesStage` must encode its final bytes +to a string before filesystem storage. The marker and version distinguish filesystem envelopes from strings that were not produced by this stage. `FileSystemSerDes.deserialize(...)` returns input without the filesystem marker unchanged, regardless of payload source. If the marker is present, the value is recognized as filesystem data and must be a valid supported envelope; -malformed marked envelopes and unsupported versions fail rather than falling back to pass-through behavior. A -recognized filesystem envelope also requires an SDK-managed `SerDesContext`, while unrecognized input does not. +malformed marked envelopes and unsupported versions fail rather than falling back to pass-through behavior. The +context parameter is explicit on every call, but a recognized filesystem envelope requires a non-null SDK-managed +context while unrecognized input passes through even if that parameter is `null`. Offloaded filenames include the entity identity, content hash, and a UUID. Each serialization publishes a new immutable file with one `CREATE_NEW` write. It does not require hard links or renames, making the write path compatible with S3 @@ -434,20 +438,14 @@ remains available for non-JSON stage values and fully custom preview logic. ### Runtime flow ```java -var previousContext = SerDesContextHolder.get(); -SerDesContextHolder.set(context); -try { - var checkpointPayload = composableSerDes.serialize(value); - sendCheckpoint(checkpointPayload); -} finally { - if (previousContext == null) { - SerDesContextHolder.clear(); - } else { - SerDesContextHolder.set(previousContext); - } -} +var checkpointPayload = serDesRunner.serialize(composableSerDes, value, context); +sendCheckpoint(checkpointPayload); ``` +Inside `SerDesRunner`, a composable pipeline receives `context` directly and forwards it to every stage. The runner +also installs the same value in `SerDesContextHolder` around the call so the root `SerDes` value codec remains +backward-compatible. + On deserialization, `FileSystemSerDes` first checks for its reserved marker. Unmarked input is returned unchanged. If the marked envelope contains `data`, it restores the inline text; if it contains `file`, it reads the stored string. `ComposableSerDes` then passes that value to the preceding string stage. Raw external input, callback results, and @@ -478,8 +476,10 @@ The core SDK should route user payload SerDes calls through a helper, tentativel - Builds the correct `SerDesContext`. - Executes inline when no SerDes executor is configured. - Dispatches to the configured executor only when one is present. -- Sets `SerDesContext` in TLS on the thread that actually invokes the SerDes. -- Invokes the existing `SerDes.serialize` and `SerDes.deserialize` methods. +- Passes `SerDesContext` explicitly to every stage in a `ComposableSerDes`. +- Sets the same `SerDesContext` in TLS on the thread that invokes the root `SerDes`, preserving compatibility for + existing value codecs. +- Invokes the existing `SerDes.serialize` and `SerDes.deserialize` methods for non-composable implementations. - Restores the previous TLS value after each SerDes call, or clears it when there was no previous value. - Wraps failures in `SerDesException` with operation and payload kind metadata. @@ -800,7 +800,7 @@ This approach gives the SDK one consistent policy for root payloads, operation r | Responsibility boundary | Combines value serialization and storage-reference creation in ordered, composable SerDes stages. | Keeps object encoding in `SerDes` and storage movement in a separate offloader. | | User configuration | Users configure one SerDes or a `ComposableSerDes` pipeline. Operation-level SerDes selection replaces the whole pipeline. | Users configure both a SerDes and an offloader. The SDK must define global, per-operation, and per-payload precedence. | | Parity with JS issue | Closest to the current JavaScript `createFileSystemSerdes` model and issue #463 wording. | Diverges from JavaScript naming and shape, though it may be architecturally cleaner for Java. | -| Core SDK changes | Adds the compatible `SerDes.then(SerDesStage)` default method, `ComposableSerDes`, and `SerDesContext` TLS because the existing serialization methods have no context parameter. | Requires a new core extension point, envelope type, config surface, payload pipeline, and migration story. | +| Core SDK changes | Adds the compatible `SerDes.then(SerDesStage)` default method, `ComposableSerDes`, explicit stage context parameters, and `SerDesContext` TLS compatibility for existing root codecs because `SerDes` methods have no context parameter. | Requires a new core extension point, envelope type, config surface, payload pipeline, and migration story. | | Applicability | Any compatible `SerDesStage` implementations can be chained, but storage stages still own their envelope and lifecycle behavior. | Any SerDes output can be offloaded uniformly after serialization. Users can combine Jackson/custom SerDes with any offloader. | | Envelope ownership | FileSystemSerDes owns the checkpoint envelope (`data`, `file`, preview), so the SDK treats it as opaque serialized data. | SDK owns the checkpoint/offload envelope and must guarantee it composes with replay, errors, callbacks, and test utilities. | | Caching | SDK can cache deserialized values, but FileSystemSerDes may still do file reads internally unless cache hits happen before SerDes. | SDK can cache at both layers: resolved offloaded payload text and final deserialized object. | @@ -835,9 +835,9 @@ public interface BinarySerDes { } public interface SerDesStage { - byte[] serialize(byte[] value); + byte[] serialize(byte[] value, SerDesContext context); - byte[] deserialize(byte[] data); + byte[] deserialize(byte[] data, SerDesContext context); } ``` @@ -884,9 +884,10 @@ publishing, documentation, and dependency-management overhead without isolating ### Add context-aware SerDes overloads -Rejected for Approach A. Explicit overloads are more discoverable, but they force context into every custom -implementation's serialization method surface. Approach A uses `SerDesContext` TLS to keep the existing -`serialize`/`deserialize` signatures unchanged. Approach B does not need SerDes TLS because `PayloadOffloader` receives +Rejected for the root `SerDes` interface in Approach A. Context is explicit on the new stage interfaces, where it does +not affect compatibility. Adding it to `SerDes` itself would force every existing custom value codec to change its +serialization method surface. Approach A keeps those signatures unchanged and retains `SerDesContext` TLS only as a +compatibility bridge for root codecs. Approach B does not need SerDes TLS because `PayloadOffloader` receives `PayloadOffloadContext` explicitly. ### Make SerDes async @@ -932,7 +933,8 @@ Negative: - Adds optional executor, context, and caching machinery that must stay deterministic. - Adds storage-specific public API and implementation code to the core SDK artifact. -- Approach A requires thread-local SerDes context because the existing `SerDes` methods do not accept context. +- Approach A retains thread-local SerDes context for backward-compatible root codecs because the existing `SerDes` + methods do not accept context; pipeline stages receive it explicitly. - Top-level stages must encode non-string representations at their boundaries. A composable binary stage avoids repeated conversion between binary substages, but its final bytes still require one string encoding. - Pipeline ordering and stage configuration must remain compatible with persisted checkpoints. diff --git a/docs/advanced/configuration.md b/docs/advanced/configuration.md index 3e46867c4..25b9e7bed 100644 --- a/docs/advanced/configuration.md +++ b/docs/advanced/configuration.md @@ -48,8 +48,10 @@ By default, SerDes runs synchronously on the calling thread to preserve existing SerDes executor must be different from the user-operation executor to avoid deadlock when the operation pool is saturated. -The SDK installs `SerDesContext` on whichever thread performs the call, restores any previous nested context afterward, -and uses a bounded weak-reference cache for successful deserialization results during the current invocation. +The SDK passes `SerDesContext` explicitly to every `SerDesStage` and nested `BinarySerDesStage`. It also installs the +same context on whichever thread performs the root `SerDes` call for backward compatibility with existing value +codecs, restores any previous nested context afterward, and uses a bounded weak-reference cache for successful +deserialization results during the current invocation. ### Filesystem-backed payload storage @@ -85,6 +87,8 @@ return DurableConfig.builder() ``` Every top-level stage consumes and produces a string, so stages compose without intermediate type mismatches. +Every stage method also receives the current read-only `SerDesContext`; the same instance is propagated through the +complete pipeline and through every retry attempt. Every stage must emit a self-identifying, normally versioned representation. On deserialization it reverses recognized valid input, rejects recognized malformed or unsupported input, and returns unrecognized input unchanged. This lets raw external payloads pass through the configured stages and reach the root value codec. diff --git a/docs/advanced/filesystem-serdes.md b/docs/advanced/filesystem-serdes.md index 01a3150a7..c57c9a36b 100644 --- a/docs/advanced/filesystem-serdes.md +++ b/docs/advanced/filesystem-serdes.md @@ -49,7 +49,8 @@ return DurableConfig.builder() Serialization follows the declaration order above and deserialization runs in reverse. The first component is the `SerDes` value codec; every component appended with `then(...)` implements `SerDesStage` and consumes and produces a string. Each stage must use a self-identifying format: it reverses recognized valid input, rejects recognized malformed -or unsupported input, and returns unrecognized input unchanged. `ComposableBinarySerDesStage` converts the string +or unsupported input, and returns unrecognized input unchanged. The runner passes the same read-only +`SerDesContext` explicitly to every string stage and binary substage. `ComposableBinarySerDesStage` converts the string with its starting codec, passes bytes directly through each `BinarySerDesStage`, converts the final bytes to a string with its ending codec, and adds a reserved versioned frame. Both boundaries are customizable through the same `StringBinaryCodec` interface; the example performs UTF-8 and Base64 conversion once around the complete @@ -105,7 +106,8 @@ Filesystem envelopes include a reserved version marker. `FileSystemSerDes` retur allowing raw root input, callback results, and standard Lambda invoke results to continue through the remaining stages to the pipeline value codec. Payloads containing the reserved marker must be valid supported filesystem envelopes; malformed marked envelopes and unsupported versions fail instead of falling back to pass-through behavior. An -unrecognized value does not require `SerDesContext`; a recognized filesystem envelope does. +unrecognized value does not require the explicit `SerDesContext` parameter to be non-null; a recognized filesystem +envelope does. Offloaded files are content-hashed and immutable. Every serialization uses a unique filename containing the entity identity, content hash, and UUID, and publishes it with one `CREATE_NEW` write. Existing files are never overwritten, diff --git a/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FileSystemSerDesIntegrationTest.java b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FileSystemSerDesIntegrationTest.java index 75e249590..ce44c1175 100644 --- a/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FileSystemSerDesIntegrationTest.java +++ b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FileSystemSerDesIntegrationTest.java @@ -502,13 +502,13 @@ private static Operation executionOperation(String id, String name, String input private static SerDesStage identityStage(RecordingFunction recorder) { return new SerDesStage() { @Override - public String serialize(String value) { + public String serialize(String value, SerDesContext context) { recorder.record("serialize", value); return value; } @Override - public String deserialize(String data) { + public String deserialize(String data, SerDesContext context) { recorder.record("deserialize", data); return data; } diff --git a/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/CloudDurableTestRunnerTest.java b/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/CloudDurableTestRunnerTest.java index 014b7f5f7..586f519c9 100644 --- a/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/CloudDurableTestRunnerTest.java +++ b/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/CloudDurableTestRunnerTest.java @@ -163,12 +163,12 @@ void replacingPersistedSerDesPreservesExplicitInputSerDes() { private static SerDesStage wrappingStage() { return new SerDesStage() { @Override - public String serialize(String value) { + public String serialize(String value, SerDesContext context) { return "<" + value + ">"; } @Override - public String deserialize(String data) { + public String deserialize(String data, SerDesContext context) { if (!data.startsWith("<")) { return data; } diff --git a/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/LocalDurableTestRunnerTest.java b/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/LocalDurableTestRunnerTest.java index 12bfd10c4..5eaaaf5f9 100644 --- a/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/LocalDurableTestRunnerTest.java +++ b/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/LocalDurableTestRunnerTest.java @@ -26,6 +26,7 @@ import software.amazon.lambda.durable.serde.FileSystemSerDes; import software.amazon.lambda.durable.serde.JacksonSerDes; import software.amazon.lambda.durable.serde.SerDes; +import software.amazon.lambda.durable.serde.SerDesContext; import software.amazon.lambda.durable.serde.SerDesStage; import software.amazon.lambda.durable.serde.Utf8StringBinaryCodec; @@ -212,12 +213,12 @@ void rawInputPassesThroughPersistedStagesBeforeFileSystemSerDes(@TempDir Path ba private static SerDesStage wrappingStage(AtomicInteger deserializeCalls) { return new SerDesStage() { @Override - public String serialize(String value) { + public String serialize(String value, SerDesContext context) { return "<" + value + ">"; } @Override - public String deserialize(String data) { + public String deserialize(String data, SerDesContext context) { deserializeCalls.incrementAndGet(); if (!data.startsWith("<")) { return data; @@ -235,12 +236,12 @@ private static SerDesStage bytesStage(AtomicInteger deserializeCalls) { .startWith(Utf8StringBinaryCodec.INSTANCE) .then(new BinarySerDesStage() { @Override - public byte[] serialize(byte[] value) { + public byte[] serialize(byte[] value, SerDesContext context) { return value; } @Override - public byte[] deserialize(byte[] data) { + public byte[] deserialize(byte[] data, SerDesContext context) { deserializeCalls.incrementAndGet(); return data; } diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/BinarySerDesStage.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/BinarySerDesStage.java index 75f69740f..171829319 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/BinarySerDesStage.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/BinarySerDesStage.java @@ -7,21 +7,26 @@ * *

Implementations must include any metadata needed for deserialization, such as format versions or encryption * initialization vectors, in the returned bytes. + * + *

The enclosing composable stage passes the same durable payload context to each binary stage. The context may be + * {@code null} only when the stage is invoked outside an SDK-managed SerDes call. */ public interface BinarySerDesStage { /** * Applies this transformation during forward serialization. * * @param value the non-null input bytes + * @param context the current durable payload context, or {@code null} outside SDK-managed calls * @return the non-null transformed bytes */ - byte[] serialize(byte[] value); + byte[] serialize(byte[] value, SerDesContext context); /** * Reverses this transformation during deserialization. * * @param data the non-null bytes produced by this transformation + * @param context the current durable payload context, or {@code null} outside SDK-managed calls * @return the non-null bytes expected by the preceding transformation */ - byte[] deserialize(byte[] data); + byte[] deserialize(byte[] data, SerDesContext context); } diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/ComposableBinarySerDesStage.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/ComposableBinarySerDesStage.java index 99ca30f35..bd4ef868f 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/ComposableBinarySerDesStage.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/ComposableBinarySerDesStage.java @@ -13,7 +13,8 @@ * *

Serialization converts the input string with the starting codec, applies binary stages in declaration order, * converts the final bytes to a string with the ending codec, and adds a versioned frame. Deserialization reverses the - * complete process when that frame is present and passes unrecognized input through unchanged. + * complete process when that frame is present and passes unrecognized input through unchanged. The context supplied to + * this string stage is forwarded unchanged to every binary stage. */ public final class ComposableBinarySerDesStage implements SerDesStage { private static final String FRAME_MARKER = "__durable_execution_composable_binary_serdes:"; @@ -36,17 +37,17 @@ public static StartBuilder builder() { } @Override - public String serialize(String value) { + public String serialize(String value, SerDesContext context) { Objects.requireNonNull(value, "value cannot be null"); var current = invokeToBytes(startingCodec, value, "starting codec"); for (int index = 0; index < binaryStages.size(); index++) { - current = invokeSerialize(binaryStages.get(index), current, index); + current = invokeSerialize(binaryStages.get(index), current, context, index); } return FRAME_PREFIX + invokeFromBytes(endingCodec, current, "ending codec"); } @Override - public String deserialize(String data) { + public String deserialize(String data, SerDesContext context) { Objects.requireNonNull(data, "data cannot be null"); if (!data.startsWith(FRAME_MARKER)) { return data; @@ -56,7 +57,7 @@ public String deserialize(String data) { } var current = invokeToBytes(endingCodec, data.substring(FRAME_PREFIX.length()), "ending codec"); for (int index = binaryStages.size() - 1; index >= 0; index--) { - current = invokeDeserialize(binaryStages.get(index), current, index); + current = invokeDeserialize(binaryStages.get(index), current, context, index); } return invokeFromBytes(startingCodec, current, "starting codec"); } @@ -77,17 +78,17 @@ private static String invokeFromBytes(StringBinaryCodec codec, byte[] data, Stri } } - private static byte[] invokeSerialize(BinarySerDesStage stage, byte[] value, int index) { + private static byte[] invokeSerialize(BinarySerDesStage stage, byte[] value, SerDesContext context, int index) { try { - return requireResult(stage.serialize(value), binaryStageName(index, stage)); + return requireResult(stage.serialize(value, context), binaryStageName(index, stage)); } catch (Throwable failure) { throw componentFailure(binaryStageName(index, stage), "serialize", failure); } } - private static byte[] invokeDeserialize(BinarySerDesStage stage, byte[] data, int index) { + private static byte[] invokeDeserialize(BinarySerDesStage stage, byte[] data, SerDesContext context, int index) { try { - return requireResult(stage.deserialize(data), binaryStageName(index, stage)); + return requireResult(stage.deserialize(data, context), binaryStageName(index, stage)); } catch (Throwable failure) { throw componentFailure(binaryStageName(index, stage), "deserialize", failure); } diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/ComposableSerDes.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/ComposableSerDes.java index 11e74e836..d9a002bb6 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/ComposableSerDes.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/ComposableSerDes.java @@ -14,7 +14,8 @@ * *

The first component is the value codec. Every later component is a {@link SerDesStage} that consumes and produces * a string. Serialization runs from first to last; deserialization runs from last to first. Each stage returns - * unrecognized input unchanged, allowing raw values to pass through to the root value codec. + * unrecognized input unchanged, allowing raw values to pass through to the root value codec. SDK-managed calls pass the + * same {@link SerDesContext} explicitly to every stage. */ public final class ComposableSerDes implements SerDes { private final SerDes valueCodec; @@ -71,32 +72,40 @@ public ComposableSerDes then(SerDesStage stage) { @Override public String serialize(Object value) { + return serialize(value, SerDesContext.getCurrentContext()); + } + + String serialize(Object value, SerDesContext context) { if (value == null) { return null; } String current = invokeValueCodecSerialize(valueCodec, value); for (int index = 0; index < stages.size(); index++) { - current = invokeStageSerialize(stages.get(index), current, index + 1); + current = invokeStageSerialize(stages.get(index), current, context, index + 1); } return current; } @Override public T deserialize(String data, TypeToken typeToken) { + return deserialize(data, typeToken, SerDesContext.getCurrentContext()); + } + + T deserialize(String data, TypeToken typeToken, SerDesContext context) { if (data == null) { return null; } Objects.requireNonNull(typeToken, "typeToken cannot be null"); String current = data; for (int index = stages.size() - 1; index >= 0; index--) { - current = invokeStageDeserialize(stages.get(index), current, index + 1); + current = invokeStageDeserialize(stages.get(index), current, context, index + 1); } return invokeValueCodecDeserialize(valueCodec, current, typeToken); } - private static String invokeStageSerialize(SerDesStage stage, String value, int index) { + private static String invokeStageSerialize(SerDesStage stage, String value, SerDesContext context, int index) { try { - var result = stage.serialize(value); + var result = stage.serialize(value, context); if (result == null) { throw new SerDesException("Stage returned null for a non-null value"); } @@ -106,9 +115,9 @@ private static String invokeStageSerialize(SerDesStage stage, String value, int } } - private static String invokeStageDeserialize(SerDesStage stage, String data, int index) { + private static String invokeStageDeserialize(SerDesStage stage, String data, SerDesContext context, int index) { try { - var result = stage.deserialize(data); + var result = stage.deserialize(data, context); if (result == null) { throw new SerDesException("Stage returned null for non-null input"); } diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java index 59fd0b4ed..897daebcb 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java @@ -35,7 +35,8 @@ * links or renames, so the write path is compatible with S3 Files. * *

Deserialization recognizes the reserved filesystem envelope marker. Input without that marker is returned - * unchanged; input with the marker must be a valid supported envelope. + * unchanged; input with the marker must be a valid supported envelope. Filesystem operations use the explicit + * {@link SerDesContext} stage parameter for durable payload identity. */ public final class FileSystemSerDes implements SerDesStage { private static final String ENVELOPE_MARKER = "__durable_execution_filesystem_serdes"; @@ -73,11 +74,11 @@ public static Builder builder(Path basePath) { } @Override - public String serialize(String value) { + public String serialize(String value, SerDesContext context) { if (value == null) { return null; } - var context = requireContext(); + context = requireContext(context); var payload = SerializedPayload.fromString(value); if (storageMode == FileSystemStorageMode.OVERFLOW) { var inlineEnvelope = encodeEnvelope(payload, null, null, context); @@ -104,20 +105,20 @@ public String serialize(String value) { } @Override - public String deserialize(String data) { + public String deserialize(String data, SerDesContext context) { if (data == null) { return null; } - return resolveSerializedPayload(data); + return resolveSerializedPayload(data, context); } - private String resolveSerializedPayload(String data) { + private String resolveSerializedPayload(String data, SerDesContext context) { final JsonNode envelope; try { envelope = ENVELOPE_MAPPER.readTree(data); } catch (JsonProcessingException e) { if (data.startsWith(ENVELOPE_PREFIX)) { - throw malformedEnvelope(requireContext(), e); + throw malformedEnvelope(requireContext(context), e); } return data; } @@ -125,7 +126,7 @@ private String resolveSerializedPayload(String data) { if (!hasFilesystemMarker(envelope)) { return data; } - var context = requireContext(); + context = requireContext(context); var marker = envelope.get(ENVELOPE_MARKER); if (!marker.isIntegralNumber()) { throw malformedEnvelope(context, null); @@ -342,8 +343,7 @@ private static boolean fitsCheckpoint(String envelope) { return Utf8StringBinaryCodec.INSTANCE.toBytes(envelope).length <= CHECKPOINT_ENVELOPE_LIMIT_BYTES; } - private SerDesContext requireContext() { - var context = SerDesContext.getCurrentContext(); + private SerDesContext requireContext(SerDesContext context) { if (context == null || context.durableExecutionArn() == null || context.durableExecutionArn().isBlank() diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/RetrySerDes.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/RetrySerDes.java index 8a2c8b573..cde1c0cdc 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/RetrySerDes.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/RetrySerDes.java @@ -15,7 +15,8 @@ * A string-stage decorator that retries transient failures from another {@link SerDesStage}. * *

Only {@link RetryableSerDesException} is retried. Other failures are propagated immediately. Retry delays block - * the thread executing the SerDes call: the caller by default or the configured SerDes executor thread. + * the thread executing the SerDes call: the caller by default or the configured SerDes executor thread. Every attempt + * receives the same {@link SerDesContext} supplied to this decorator. */ public final class RetrySerDes implements SerDesStage { private static final Sleeper DEFAULT_SLEEPER = delay -> { @@ -48,13 +49,13 @@ public RetrySerDes(SerDesStage delegate, RetryStrategy retryStrategy) { } @Override - public String serialize(String value) { - return execute("pipeline stage serialization", () -> delegate.serialize(value)); + public String serialize(String value, SerDesContext context) { + return execute("pipeline stage serialization", () -> delegate.serialize(value, context)); } @Override - public String deserialize(String data) { - return execute("pipeline stage deserialization", () -> delegate.deserialize(data)); + public String deserialize(String data, SerDesContext context) { + return execute("pipeline stage deserialization", () -> delegate.deserialize(data, context)); } private T execute(String action, Supplier operation) { diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesContext.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesContext.java index 26d09006c..16ca69326 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesContext.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesContext.java @@ -8,8 +8,10 @@ /** * Describes the durable payload currently being processed by a {@link SerDes}. * - *

The SDK sets this context only while invoking a configured SerDes. Direct customer calls to SerDes methods do not - * have a current context. + *

The SDK passes this context explicitly to {@link SerDesStage} and {@link BinarySerDesStage} methods. It is also + * installed for the duration of the configured {@link SerDes} call so existing value codecs can read it without + * changing the backward-compatible SerDes interface. Direct customer calls to SerDes methods do not have a current + * context. */ public record SerDesContext( String durableExecutionArn, @@ -22,7 +24,12 @@ public record SerDesContext( OperationSubType operationSubType, Integer attempt) { - /** Returns the context for the SerDes call on the current thread, or {@code null} outside SDK-managed calls. */ + /** + * Returns the context for the SerDes call on the current thread, or {@code null} outside SDK-managed calls. + * + *

Pipeline stages should use the context parameter passed to their methods instead of this compatibility + * accessor. + */ public static SerDesContext getCurrentContext() { return SerDesContextHolder.get(); } diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesRunner.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesRunner.java index fc1a83a79..513dcf3fc 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesRunner.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesRunner.java @@ -54,7 +54,7 @@ public SerDesRunner(ExecutorService executorService) { /** Serializes a value with the supplied durable payload context. */ public String serialize(SerDes serDes, Object value, SerDesContext context) { Objects.requireNonNull(serDes, "serDes cannot be null"); - return run("serialize", context, () -> serDes.serialize(value)); + return run("serialize", context, () -> serializeWithContext(serDes, value, context)); } /** Deserializes a value with invocation-scoped caching. */ @@ -91,7 +91,7 @@ public T deserialize(SerDes serDes, String data, TypeToken typeToken, Ser return (T) unmaskNull(cached); } - T value = run("deserialize", context, () -> serDes.deserialize(data, typeToken)); + T value = run("deserialize", context, () -> deserializeWithContext(serDes, data, typeToken, context)); var cacheValue = maskNull(value); putCompleted(key, cacheValue); pending.complete(cacheValue); @@ -132,6 +132,21 @@ private static Object unmaskNull(Object value) { return value == NULL_VALUE ? null : value; } + private static String serializeWithContext(SerDes serDes, Object value, SerDesContext context) { + if (serDes instanceof ComposableSerDes composable) { + return composable.serialize(value, context); + } + return serDes.serialize(value); + } + + private static T deserializeWithContext( + SerDes serDes, String data, TypeToken typeToken, SerDesContext context) { + if (serDes instanceof ComposableSerDes composable) { + return composable.deserialize(data, typeToken, context); + } + return serDes.deserialize(data, typeToken); + } + private T run(String action, SerDesContext context, Supplier supplier) { Objects.requireNonNull(supplier, "supplier cannot be null"); Objects.requireNonNull(context, "SerDesContext cannot be null"); diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesStage.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesStage.java index 20b567d20..64700487c 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesStage.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesStage.java @@ -20,15 +20,19 @@ *

This pass-through behavior allows raw external payloads to traverse a configured pipeline and reach its root value * codec without special pipeline control flow. Implementations should inspect an explicit marker or versioned envelope * before decoding rather than treating any successfully decodable value as recognized. + * + *

The SDK passes the durable payload context explicitly to every stage invocation. The context may be {@code null} + * only when a pipeline or stage is invoked directly outside an SDK-managed SerDes call. */ public interface SerDesStage { /** * Applies this stage during forward serialization. * * @param value the non-null input string + * @param context the current durable payload context, or {@code null} outside SDK-managed calls * @return the non-null transformed string */ - String serialize(String value); + String serialize(String value, SerDesContext context); /** * Reverses this stage during deserialization. @@ -37,7 +41,8 @@ public interface SerDesStage { * it identifies this stage's format but is malformed or unsupported, implementations must throw a SerDes failure. * * @param data the non-null input string + * @param context the current durable payload context, or {@code null} outside SDK-managed calls * @return the non-null string expected by the preceding stage, or {@code data} unchanged when unrecognized */ - String deserialize(String data); + String deserialize(String data, SerDesContext context); } diff --git a/sdk/src/test/java/software/amazon/lambda/durable/serde/ComposableBinarySerDesStageTest.java b/sdk/src/test/java/software/amazon/lambda/durable/serde/ComposableBinarySerDesStageTest.java index 461d93b75..b738c4e07 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/serde/ComposableBinarySerDesStageTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/serde/ComposableBinarySerDesStageTest.java @@ -13,6 +13,7 @@ import java.util.Arrays; import java.util.Base64; import java.util.List; +import java.util.concurrent.atomic.AtomicReference; import org.junit.jupiter.api.Test; import software.amazon.lambda.durable.TypeToken; import software.amazon.lambda.durable.exception.RetryableSerDesException; @@ -21,6 +22,8 @@ class ComposableBinarySerDesStageTest { private static final String BINARY_FRAME_MARKER = "__durable_execution_composable_binary_serdes:"; private static final String BINARY_FRAME_PREFIX = BINARY_FRAME_MARKER + "1:"; + private static final SerDesContext CONTEXT = + SerDesContext.forExecution("arn", "invocation", "execution", SerDesPayloadKind.RESULT); @Test void processesBoundariesAndBinaryStagesInDeclarationOrder() { @@ -32,8 +35,8 @@ void processesBoundariesAndBinaryStagesInDeclarationOrder() { .endWith(recordingCodec("ending", calls)) .build(); - var serialized = stage.serialize("value"); - var deserialized = stage.deserialize(serialized); + var serialized = stage.serialize("value", CONTEXT); + var deserialized = stage.deserialize(serialized, CONTEXT); assertEquals( BINARY_FRAME_PREFIX + Base64.getEncoder().encodeToString(new byte[] {'v', 'a', 'l', 'u', 'e', 1, 2}), @@ -52,6 +55,35 @@ void processesBoundariesAndBinaryStagesInDeclarationOrder() { calls); } + @Test + void passesTheSameContextToEveryBinaryStageCall() { + var serializeContext = new AtomicReference(); + var deserializeContext = new AtomicReference(); + var stage = ComposableBinarySerDesStage.builder() + .startWith(Utf8StringBinaryCodec.INSTANCE) + .then(new BinarySerDesStage() { + @Override + public byte[] serialize(byte[] value, SerDesContext context) { + serializeContext.set(context); + return value; + } + + @Override + public byte[] deserialize(byte[] data, SerDesContext context) { + deserializeContext.set(context); + return data; + } + }) + .endWith(Base64StringBinaryCodec.INSTANCE) + .build(); + + var serialized = stage.serialize("value", CONTEXT); + stage.deserialize(serialized, CONTEXT); + + assertSame(CONTEXT, serializeContext.get()); + assertSame(CONTEXT, deserializeContext.get()); + } + @Test void composesWithRootSerDesAsOneStringStage() { var stage = ComposableBinarySerDesStage.builder() @@ -88,12 +120,12 @@ public String fromBytes(byte[] data) { .endWith(Base64StringBinaryCodec.INSTANCE) .build(); - var serialized = stage.serialize("value"); + var serialized = stage.serialize("value", CONTEXT); assertEquals( BINARY_FRAME_PREFIX + Base64.getEncoder().encodeToString("eulav".getBytes(StandardCharsets.UTF_8)), serialized); - assertEquals("value", stage.deserialize(serialized)); + assertEquals("value", stage.deserialize(serialized, CONTEXT)); } @Test @@ -103,9 +135,9 @@ void passesThroughUnrecognizedInputAndRejectsInvalidFrames() { .endWith(Base64StringBinaryCodec.INSTANCE) .build(); - assertEquals("\"external\"", stage.deserialize("\"external\"")); - assertThrows(SerDesException.class, () -> stage.deserialize(BINARY_FRAME_MARKER + "2:value")); - assertThrows(SerDesException.class, () -> stage.deserialize(BINARY_FRAME_PREFIX + "not-base64!")); + assertEquals("\"external\"", stage.deserialize("\"external\"", CONTEXT)); + assertThrows(SerDesException.class, () -> stage.deserialize(BINARY_FRAME_MARKER + "2:value", CONTEXT)); + assertThrows(SerDesException.class, () -> stage.deserialize(BINARY_FRAME_PREFIX + "not-base64!", CONTEXT)); } @Test @@ -123,19 +155,19 @@ void validatesConfigurationAndComponentResults() { .startWith(Utf8StringBinaryCodec.INSTANCE) .then(new BinarySerDesStage() { @Override - public byte[] serialize(byte[] value) { + public byte[] serialize(byte[] value, SerDesContext context) { return null; } @Override - public byte[] deserialize(byte[] data) { + public byte[] deserialize(byte[] data, SerDesContext context) { return data; } }) .endWith(Base64StringBinaryCodec.INSTANCE) .build(); - var failure = assertThrows(SerDesException.class, () -> nullStage.serialize("value")); + var failure = assertThrows(SerDesException.class, () -> nullStage.serialize("value", CONTEXT)); assertTrue(failure.getMessage().contains("binary stage 0")); } @@ -145,19 +177,19 @@ void preservesRetryableFailuresAndFatalErrors() { .startWith(Utf8StringBinaryCodec.INSTANCE) .then(new BinarySerDesStage() { @Override - public byte[] serialize(byte[] value) { + public byte[] serialize(byte[] value, SerDesContext context) { throw new RetryableSerDesException("retry"); } @Override - public byte[] deserialize(byte[] data) { + public byte[] deserialize(byte[] data, SerDesContext context) { return data; } }) .endWith(Base64StringBinaryCodec.INSTANCE) .build(); - var retryable = assertThrows(RetryableSerDesException.class, () -> retryableStage.serialize("value")); + var retryable = assertThrows(RetryableSerDesException.class, () -> retryableStage.serialize("value", CONTEXT)); assertTrue(retryable.getMessage().contains("binary stage 0")); var fatalError = new AssertionError("fatal"); @@ -176,7 +208,7 @@ public String fromBytes(byte[] data) { .endWith(Base64StringBinaryCodec.INSTANCE) .build(); - assertSame(fatalError, assertThrows(AssertionError.class, () -> fatalStage.serialize("value"))); + assertSame(fatalError, assertThrows(AssertionError.class, () -> fatalStage.serialize("value", CONTEXT))); } @Test @@ -220,7 +252,7 @@ public String fromBytes(byte[] data) { private static BinarySerDesStage appendingStage(String name, byte suffix, List calls) { return new BinarySerDesStage() { @Override - public byte[] serialize(byte[] value) { + public byte[] serialize(byte[] value, SerDesContext context) { calls.add(name + "-serialize"); var result = Arrays.copyOf(value, value.length + 1); result[value.length] = suffix; @@ -228,7 +260,7 @@ public byte[] serialize(byte[] value) { } @Override - public byte[] deserialize(byte[] data) { + public byte[] deserialize(byte[] data, SerDesContext context) { calls.add(name + "-deserialize"); if (data.length == 0 || data[data.length - 1] != suffix) { throw new SerDesException("Unexpected suffix"); @@ -241,12 +273,12 @@ public byte[] deserialize(byte[] data) { private static BinarySerDesStage xorStage(byte key) { return new BinarySerDesStage() { @Override - public byte[] serialize(byte[] value) { + public byte[] serialize(byte[] value, SerDesContext context) { return xor(value, key); } @Override - public byte[] deserialize(byte[] data) { + public byte[] deserialize(byte[] data, SerDesContext context) { return xor(data, key); } }; diff --git a/sdk/src/test/java/software/amazon/lambda/durable/serde/ComposableSerDesTest.java b/sdk/src/test/java/software/amazon/lambda/durable/serde/ComposableSerDesTest.java index 03c927185..e564e42b8 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/serde/ComposableSerDesTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/serde/ComposableSerDesTest.java @@ -12,6 +12,7 @@ import java.util.ArrayList; import java.util.List; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; import org.junit.jupiter.api.Test; import software.amazon.lambda.durable.TypeToken; import software.amazon.lambda.durable.exception.RetryableSerDesException; @@ -42,31 +43,59 @@ void serializesForwardAndDeserializesInReverse() { assertEquals(List.of("first-serialize", "second-serialize", "second-deserialize", "first-deserialize"), calls); } + @Test + void passesTheRunnerContextExplicitlyToEveryStageCall() { + var observedSerializeContext = new AtomicReference(); + var observedDeserializeContext = new AtomicReference(); + var stage = new SerDesStage() { + @Override + public String serialize(String value, SerDesContext context) { + observedSerializeContext.set(context); + return value; + } + + @Override + public String deserialize(String data, SerDesContext context) { + observedDeserializeContext.set(context); + return data; + } + }; + var context = SerDesContext.forExecution("arn", "invocation", "execution", SerDesPayloadKind.RESULT); + var pipeline = new JacksonSerDes().then(stage); + var runner = new SerDesRunner(null); + + var serialized = runner.serialize(pipeline, "value", context); + runner.deserialize(pipeline, serialized, TypeToken.get(String.class), context); + + assertSame(context, observedSerializeContext.get()); + assertSame(context, observedDeserializeContext.get()); + } + @Test void supportsDedicatedStringStages() { var calls = new ArrayList(); var first = new SerDesStage() { @Override - public String serialize(String value) { + public String serialize(String value, SerDesContext context) { calls.add("first-serialize"); return "<" + value + ">"; } @Override - public String deserialize(String data) { + public String deserialize(String data, SerDesContext context) { calls.add("first-deserialize"); return data.substring(1, data.length() - 1); } }; var second = new SerDesStage() { @Override - public String serialize(String value) { + public String serialize(String value, SerDesContext context) { calls.add("second-serialize"); return "[" + value + "]"; } @Override - public String deserialize(String data) { + public String deserialize(String data, SerDesContext context) { calls.add("second-deserialize"); return data.substring(1, data.length() - 1); } @@ -121,12 +150,12 @@ void valueCodecMayDecodeNonNullRepresentationToNull() { var intermediateCalls = new AtomicInteger(); var identityStage = new SerDesStage() { @Override - public String serialize(String value) { + public String serialize(String value, SerDesContext context) { return value; } @Override - public String deserialize(String data) { + public String deserialize(String data, SerDesContext context) { intermediateCalls.incrementAndGet(); return data; } @@ -163,12 +192,12 @@ void recognizedMalformedInputFailsAtTheOwningStage() { void rejectsNullIntermediateValues() { var nullStage = new SerDesStage() { @Override - public String serialize(String value) { + public String serialize(String value, SerDesContext context) { return null; } @Override - public String deserialize(String data) { + public String deserialize(String data, SerDesContext context) { return null; } }; @@ -182,12 +211,12 @@ public String deserialize(String data) { void preservesRetryabilityWhenDecoratingStageFailures() { var transientStage = new SerDesStage() { @Override - public String serialize(String value) { + public String serialize(String value, SerDesContext context) { throw new RetryableSerDesException("retry"); } @Override - public String deserialize(String data) { + public String deserialize(String data, SerDesContext context) { return null; } }; @@ -205,12 +234,12 @@ void preservesFatalErrorsFromEveryPipelineCall() { var serializeError = new OutOfMemoryError("serialize"); var serializeStage = new SerDesStage() { @Override - public String serialize(String value) { + public String serialize(String value, SerDesContext context) { throw serializeError; } @Override - public String deserialize(String data) { + public String deserialize(String data, SerDesContext context) { return null; } }; @@ -221,12 +250,12 @@ public String deserialize(String data) { var stringStageError = new StackOverflowError("string-stage-deserialize"); var stringStage = new SerDesStage() { @Override - public String serialize(String value) { + public String serialize(String value, SerDesContext context) { return value; } @Override - public String deserialize(String data) { + public String deserialize(String data, SerDesContext context) { throw stringStageError; } }; @@ -253,13 +282,13 @@ public T deserialize(String data, TypeToken typeToken) { private static SerDesStage stringStage(String name, String prefix, String suffix, List calls) { return new SerDesStage() { @Override - public String serialize(String value) { + public String serialize(String value, SerDesContext context) { calls.add(name + "-serialize"); return prefix + value + suffix; } @Override - public String deserialize(String data) { + public String deserialize(String data, SerDesContext context) { calls.add(name + "-deserialize"); if (!data.startsWith(prefix)) { return data; diff --git a/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java b/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java index ca4789159..727b1d941 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java @@ -456,8 +456,8 @@ void fileReferencesCrossInvokeInputAndResultBoundaries() { void rejectsCallsWithoutContextAndMalformedOrUnsafeEnvelopes() throws Exception { var stage = FileSystemSerDes.builder(basePath).build(); var serDes = stringCodec().then(stage); - assertThrows(SerDesException.class, () -> stage.serialize("value")); - assertEquals("value", stage.deserialize("value")); + assertThrows(SerDesException.class, () -> stage.serialize("value", null)); + assertEquals("value", stage.deserialize("value", null)); var runner = new SerDesRunner(null); assertEquals("{}", runner.deserialize(serDes, "{}", TypeToken.get(String.class), context())); @@ -629,12 +629,12 @@ private static SerDesContext operationContext(OperationType operationType, Opera private static BinarySerDesStage xorBinaryStage(byte key) { return new BinarySerDesStage() { @Override - public byte[] serialize(byte[] value) { + public byte[] serialize(byte[] value, SerDesContext context) { return xor(value, key); } @Override - public byte[] deserialize(byte[] data) { + public byte[] deserialize(byte[] data, SerDesContext context) { return xor(data, key); } }; @@ -651,12 +651,12 @@ private static byte[] xor(byte[] value, byte key) { private static SerDesStage wrappingStage() { return new SerDesStage() { @Override - public String serialize(String value) { + public String serialize(String value, SerDesContext context) { return "<" + value + ">"; } @Override - public String deserialize(String data) { + public String deserialize(String data, SerDesContext context) { if (!data.startsWith("<")) { return data; } diff --git a/sdk/src/test/java/software/amazon/lambda/durable/serde/RetrySerDesTest.java b/sdk/src/test/java/software/amazon/lambda/durable/serde/RetrySerDesTest.java index 9a9c3dc04..e98c69244 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/serde/RetrySerDesTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/serde/RetrySerDesTest.java @@ -20,6 +20,8 @@ import software.amazon.lambda.durable.retry.RetryStrategies; class RetrySerDesTest { + private static final SerDesContext CONTEXT = + SerDesContext.forExecution("arn", "invocation", "execution", SerDesPayloadKind.RESULT); @Test void retriesSerializationWithStrategyDelays() { @@ -28,7 +30,8 @@ void retriesSerializationWithStrategyDelays() { var delays = new ArrayList(); var delegate = new SerDesStage() { @Override - public String serialize(String value) { + public String serialize(String value, SerDesContext context) { + assertSame(CONTEXT, context); if (calls.incrementAndGet() < 3) { throw new RetryableSerDesException("transient"); } @@ -36,7 +39,7 @@ public String serialize(String value) { } @Override - public String deserialize(String data) { + public String deserialize(String data, SerDesContext context) { return data; } }; @@ -48,7 +51,7 @@ public String deserialize(String data) { }, delays::add); - assertEquals("serialized", retrySerDes.serialize("value")); + assertEquals("serialized", retrySerDes.serialize("value", CONTEXT)); assertEquals(3, calls.get()); assertEquals(List.of(1, 2), strategyAttempts); assertEquals(List.of(Duration.ofMillis(1), Duration.ofMillis(2)), delays); @@ -59,12 +62,13 @@ void retriesDeserialization() { var calls = new AtomicInteger(); var delegate = new SerDesStage() { @Override - public String serialize(String value) { + public String serialize(String value, SerDesContext context) { return value; } @Override - public String deserialize(String data) { + public String deserialize(String data, SerDesContext context) { + assertSame(CONTEXT, context); if (calls.incrementAndGet() == 1) { throw new RetryableSerDesException("transient"); } @@ -74,7 +78,7 @@ public String deserialize(String data) { var retrySerDes = new RetrySerDes(delegate, (error, attempt) -> RetryDecision.retry(Duration.ZERO), delay -> {}); - assertEquals("value", retrySerDes.deserialize("value")); + assertEquals("value", retrySerDes.deserialize("value", CONTEXT)); assertEquals(2, calls.get()); } @@ -83,13 +87,13 @@ void doesNotRetryPermanentSerDesFailure() { var calls = new AtomicInteger(); var delegate = new SerDesStage() { @Override - public String serialize(String value) { + public String serialize(String value, SerDesContext context) { calls.incrementAndGet(); throw new SerDesException("permanent"); } @Override - public String deserialize(String data) { + public String deserialize(String data, SerDesContext context) { return data; } }; @@ -97,7 +101,7 @@ public String deserialize(String data) { throw new AssertionError("permanent failures must not sleep"); }); - var failure = assertThrows(SerDesException.class, () -> retrySerDes.serialize("value")); + var failure = assertThrows(SerDesException.class, () -> retrySerDes.serialize("value", CONTEXT)); assertEquals("permanent", failure.getMessage()); assertEquals(1, calls.get()); } @@ -113,19 +117,19 @@ void rejectsInvalidConfigurationAndRetryDelay() { var retrySerDes = new RetrySerDes( new SerDesStage() { @Override - public String serialize(String value) { + public String serialize(String value, SerDesContext context) { throw new RetryableSerDesException("transient"); } @Override - public String deserialize(String data) { + public String deserialize(String data, SerDesContext context) { return data; } }, (error, attempt) -> RetryDecision.retry(Duration.ofSeconds(-1)), delay -> {}); - var failure = assertThrows(SerDesException.class, () -> retrySerDes.serialize("value")); + var failure = assertThrows(SerDesException.class, () -> retrySerDes.serialize("value", CONTEXT)); assertTrue(failure.getMessage().contains("invalid delay")); } @@ -135,20 +139,20 @@ void rethrowsLastRetryableFailureWhenRetriesAreExhausted() { var lastFailure = new AtomicReference(); var delegate = new SerDesStage() { @Override - public String serialize(String value) { + public String serialize(String value, SerDesContext context) { var failure = new RetryableSerDesException("attempt-" + calls.incrementAndGet()); lastFailure.set(failure); throw failure; } @Override - public String deserialize(String data) { + public String deserialize(String data, SerDesContext context) { return data; } }; var retrySerDes = new RetrySerDes(delegate, RetryStrategies.fixedDelay(2, Duration.ofSeconds(1)), delay -> {}); - var thrown = assertThrows(RetryableSerDesException.class, () -> retrySerDes.serialize("value")); + var thrown = assertThrows(RetryableSerDesException.class, () -> retrySerDes.serialize("value", CONTEXT)); assertSame(lastFailure.get(), thrown); assertEquals("attempt-2", thrown.getMessage()); assertEquals(2, calls.get()); @@ -159,12 +163,12 @@ void restoresInterruptStatusWhenBackoffIsInterrupted() { var retryable = new RetryableSerDesException("transient"); var delegate = new SerDesStage() { @Override - public String serialize(String value) { + public String serialize(String value, SerDesContext context) { throw retryable; } @Override - public String deserialize(String data) { + public String deserialize(String data, SerDesContext context) { return data; } }; @@ -174,7 +178,7 @@ public String deserialize(String data) { }); try { - var thrown = assertThrows(SerDesException.class, () -> retrySerDes.serialize("value")); + var thrown = assertThrows(SerDesException.class, () -> retrySerDes.serialize("value", CONTEXT)); assertTrue(thrown.getMessage().contains("Interrupted")); assertTrue(Thread.currentThread().isInterrupted()); assertSame(retryable, thrown.getSuppressed()[0]); @@ -186,12 +190,12 @@ public String deserialize(String data) { private static SerDesStage identityStage() { return new SerDesStage() { @Override - public String serialize(String value) { + public String serialize(String value, SerDesContext context) { return value; } @Override - public String deserialize(String data) { + public String deserialize(String data, SerDesContext context) { return data; } }; From dd624fce405fb091614ceba86e2013fbe310ee7b Mon Sep 17 00:00:00 2001 From: Frank Chen <65260095+zhongkechen@users.noreply.github.com> Date: Tue, 25 Aug 2026 14:41:52 -0700 Subject: [PATCH 37/56] Update sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- .../software/amazon/lambda/durable/serde/FileSystemSerDes.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java index 897daebcb..15ca2078d 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java @@ -117,7 +117,7 @@ private String resolveSerializedPayload(String data, SerDesContext context) { try { envelope = ENVELOPE_MAPPER.readTree(data); } catch (JsonProcessingException e) { - if (data.startsWith(ENVELOPE_PREFIX)) { + if (data.stripLeading().startsWith(ENVELOPE_PREFIX)) { throw malformedEnvelope(requireContext(context), e); } return data; From e3bf9e4cb6a98627e71c9ee18b138df1398d6f68 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Tue, 25 Aug 2026 21:49:05 +0000 Subject: [PATCH 38/56] fix: rebind terminal invoke errors --- .../exception/InvokeStoppedException.java | 4 + .../exception/InvokeTimedOutException.java | 4 + .../durable/operation/InvokeOperation.java | 9 +- .../operation/InvokeOperationTest.java | 95 +++++++++++++++++++ 4 files changed, 107 insertions(+), 5 deletions(-) diff --git a/sdk/src/main/java/software/amazon/lambda/durable/exception/InvokeStoppedException.java b/sdk/src/main/java/software/amazon/lambda/durable/exception/InvokeStoppedException.java index 01dd3e22c..7786e00f7 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/exception/InvokeStoppedException.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/exception/InvokeStoppedException.java @@ -10,4 +10,8 @@ public class InvokeStoppedException extends InvokeException { public InvokeStoppedException(Operation operation) { super(operation); } + + public InvokeStoppedException(Operation operation, Throwable deserializedError) { + super(operation, deserializedError); + } } diff --git a/sdk/src/main/java/software/amazon/lambda/durable/exception/InvokeTimedOutException.java b/sdk/src/main/java/software/amazon/lambda/durable/exception/InvokeTimedOutException.java index a0c36c623..df2241924 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/exception/InvokeTimedOutException.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/exception/InvokeTimedOutException.java @@ -10,4 +10,8 @@ public class InvokeTimedOutException extends InvokeException { public InvokeTimedOutException(Operation operation) { super(operation); } + + public InvokeTimedOutException(Operation operation, Throwable deserializedError) { + super(operation, deserializedError); + } } diff --git a/sdk/src/main/java/software/amazon/lambda/durable/operation/InvokeOperation.java b/sdk/src/main/java/software/amazon/lambda/durable/operation/InvokeOperation.java index 7ad3a834b..9a389daeb 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/operation/InvokeOperation.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/operation/InvokeOperation.java @@ -90,13 +90,12 @@ public T get() { var op = waitForOperationCompletion(); var invokeDetails = op.chainedInvokeDetails(); var result = invokeDetails != null ? invokeDetails.result() : null; + var error = invokeDetails != null ? invokeDetails.error() : null; return switch (op.status()) { case SUCCEEDED -> deserializeResult(result); - case FAILED -> - throw new InvokeFailedException( - op, deserializeException(invokeDetails != null ? invokeDetails.error() : null)); - case TIMED_OUT -> throw new InvokeTimedOutException(op); - case STOPPED -> throw new InvokeStoppedException(op); + case FAILED -> throw new InvokeFailedException(op, deserializeException(error)); + case TIMED_OUT -> throw new InvokeTimedOutException(op, deserializeException(error)); + case STOPPED -> throw new InvokeStoppedException(op, deserializeException(error)); // Unexpected status which should not happen. This is added for forward-compatibility. default -> throw new InvokeException(op); }; diff --git a/sdk/src/test/java/software/amazon/lambda/durable/operation/InvokeOperationTest.java b/sdk/src/test/java/software/amazon/lambda/durable/operation/InvokeOperationTest.java index 21363148b..3bdeca694 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/operation/InvokeOperationTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/operation/InvokeOperationTest.java @@ -9,8 +9,12 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; +import java.nio.file.Path; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; import software.amazon.awssdk.services.lambda.model.ChainedInvokeDetails; import software.amazon.awssdk.services.lambda.model.ErrorObject; import software.amazon.awssdk.services.lambda.model.Operation; @@ -18,6 +22,7 @@ import software.amazon.lambda.durable.TypeToken; import software.amazon.lambda.durable.config.InvokeConfig; import software.amazon.lambda.durable.context.DurableContextImpl; +import software.amazon.lambda.durable.exception.DurableOperationException; import software.amazon.lambda.durable.exception.InvokeException; import software.amazon.lambda.durable.exception.InvokeFailedException; import software.amazon.lambda.durable.exception.InvokeStoppedException; @@ -27,7 +32,12 @@ import software.amazon.lambda.durable.execution.ThreadType; import software.amazon.lambda.durable.model.OperationIdentifier; import software.amazon.lambda.durable.model.OperationSubType; +import software.amazon.lambda.durable.serde.FileSystemSerDes; import software.amazon.lambda.durable.serde.JacksonSerDes; +import software.amazon.lambda.durable.serde.SerDes; +import software.amazon.lambda.durable.serde.SerDesContext; +import software.amazon.lambda.durable.serde.SerDesPayloadKind; +import software.amazon.lambda.durable.serde.SerDesRunner; class InvokeOperationTest { private static final String OPERATION_ID = "2"; @@ -38,6 +48,9 @@ class InvokeOperationTest { private ExecutionManager executionManager; private DurableContextImpl durableContext; + @TempDir + Path basePath; + @BeforeEach void setUp() { executionManager = mock(ExecutionManager.class); @@ -192,6 +205,59 @@ void getInvokeStoppedExceptionWhenInvocationTimedOut() { assertEquals("errorMessage", ex.getMessage()); } + @ParameterizedTest + @EnumSource( + value = OperationStatus.class, + names = {"TIMED_OUT", "STOPPED"}) + void nestedTerminalInvokeRebindsFilesystemErrorForReplay(OperationStatus status) { + var callerArn = "arn:aws:lambda:us-east-1:123456789012:function:caller/durable-execution/caller/invocation"; + var calleeArn = "arn:aws:lambda:us-east-1:123456789012:function:callee/durable-execution/callee/invocation"; + when(executionManager.getDurableExecutionArn()).thenReturn(callerArn); + + var serDes = new JacksonSerDes().then(FileSystemSerDes.builder(basePath).build()); + var original = new IllegalStateException("callee failed"); + var errorData = new SerDesRunner(null) + .serialize( + serDes, + original, + SerDesContext.forExecution( + calleeArn, "callee-invocation", "callee-execution", SerDesPayloadKind.EXCEPTION)); + var op = Operation.builder() + .id(OPERATION_ID) + .name(OPERATION_NAME) + .status(status) + .chainedInvokeDetails(ChainedInvokeDetails.builder() + .error(ErrorObject.builder() + .errorType(original.getClass().getName()) + .errorMessage(original.getMessage()) + .errorData(errorData) + .build()) + .build()) + .build(); + when(executionManager.getOperationAndUpdateReplayState(OPERATION_ID)).thenReturn(op); + + var invoke = new InvokeOperation<>( + OPERATION_IDENTIFIER, + "test-function", + "{}", + TypeToken.get(String.class), + InvokeConfig.builder().serDes(serDes).build(), + durableContext); + invoke.onCheckpointComplete(op); + + DurableOperationException forwarded = status == OperationStatus.TIMED_OUT + ? assertThrows(InvokeTimedOutException.class, invoke::get) + : assertThrows(InvokeStoppedException.class, invoke::get); + assertInstanceOf(IllegalStateException.class, forwarded.deserializedError()); + + var child = new ChildContextRebindingOperation(serDes, durableContext); + var rebound = child.rebind(forwarded); + var replayed = child.deserialize(rebound); + + assertInstanceOf(IllegalStateException.class, replayed); + assertEquals("callee failed", replayed.getMessage()); + } + @Test void getInvokeFailedExceptionWhenInvocationEndedUnexpectedly() { var op = Operation.builder() @@ -219,4 +285,33 @@ void getInvokeFailedExceptionWhenInvocationEndedUnexpectedly() { assertThrows(InvokeException.class, () -> operation.get()); } + + private static final class ChildContextRebindingOperation extends SerializableDurableOperation { + private ChildContextRebindingOperation(SerDes serDes, DurableContextImpl durableContext) { + super( + OperationIdentifier.of("child", "child", OperationSubType.RUN_IN_CHILD_CONTEXT), + TypeToken.get(String.class), + serDes, + durableContext); + } + + private ErrorObject rebind(DurableOperationException exception) { + return rebindForwardedException(exception); + } + + private Throwable deserialize(ErrorObject error) { + return deserializeException(error); + } + + @Override + protected void start() {} + + @Override + protected void replay(Operation existing) {} + + @Override + public String get() { + return null; + } + } } From 46a409860a1bd8c9d9a9a51e6060c89a697be166 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Tue, 25 Aug 2026 22:13:10 +0000 Subject: [PATCH 39/56] refactor: scope SerDes context to stages --- docs/adr/005-filesystem-serdes.md | 102 ++++++------- docs/advanced/configuration.md | 24 +-- docs/advanced/error-handling.md | 2 +- docs/advanced/filesystem-serdes.md | 27 ++-- docs/design.md | 9 +- .../FileSystemSerDesIntegrationTest.java | 15 +- .../durable/testing/TestOperationTest.java | 18 ++- .../cloud/HistoryEventProcessorTest.java | 18 ++- .../exception/RetryableSerDesException.java | 3 +- .../durable/serde/BinarySerDesStage.java | 6 +- .../durable/serde/ComposableSerDes.java | 13 +- .../durable/serde/FileSystemSerDes.java | 21 +-- .../durable/serde/RetryBinarySerDesStage.java | 45 ++++++ .../durable/serde/RetrySerDesStage.java | 44 ++++++ .../lambda/durable/serde/SerDesContext.java | 43 +++--- .../durable/serde/SerDesContextHolder.java | 21 --- ...rySerDes.java => SerDesRetryExecutor.java} | 37 +---- .../lambda/durable/serde/SerDesRunner.java | 21 +-- .../lambda/durable/serde/SerDesStage.java | 12 +- .../durable/operation/StepOperationTest.java | 17 ++- .../ComposableBinarySerDesStageTest.java | 28 ++++ .../durable/serde/ComposableSerDesTest.java | 7 +- .../durable/serde/FileSystemSerDesTest.java | 19 ++- .../serde/RetryBinarySerDesStageTest.java | 141 ++++++++++++++++++ ...DesTest.java => RetrySerDesStageTest.java} | 35 +++-- .../durable/serde/SerDesRunnerTest.java | 67 +++------ 26 files changed, 523 insertions(+), 272 deletions(-) create mode 100644 sdk/src/main/java/software/amazon/lambda/durable/serde/RetryBinarySerDesStage.java create mode 100644 sdk/src/main/java/software/amazon/lambda/durable/serde/RetrySerDesStage.java delete mode 100644 sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesContextHolder.java rename sdk/src/main/java/software/amazon/lambda/durable/serde/{RetrySerDes.java => SerDesRetryExecutor.java} (67%) create mode 100644 sdk/src/test/java/software/amazon/lambda/durable/serde/RetryBinarySerDesStageTest.java rename sdk/src/test/java/software/amazon/lambda/durable/serde/{RetrySerDesTest.java => RetrySerDesStageTest.java} (84%) diff --git a/docs/adr/005-filesystem-serdes.md b/docs/adr/005-filesystem-serdes.md index 27a70408a..56bb571c0 100644 --- a/docs/adr/005-filesystem-serdes.md +++ b/docs/adr/005-filesystem-serdes.md @@ -81,10 +81,11 @@ compression, encryption, and similar byte processing internally and encodes the filesystem when configured to do so and returns a small checkpoint envelope. It is only a `SerDesStage`; a value codec must precede it in a `ComposableSerDes`, and other string stages may follow it. -The existing `SerDes` methods remain unchanged. `SerDesRunner` passes `SerDesContext` directly into -`ComposableSerDes`, which forwards the same instance to every `SerDesStage` and nested `BinarySerDesStage` call. -The runner also installs the context in a thread-local compatibility accessor for existing root value codecs and -custom `SerDes` implementations whose backward-compatible methods cannot accept it explicitly. +The existing `SerDes` methods remain unchanged and context-free. `SerDesRunner` passes a base `SerDesContext` directly +into `ComposableSerDes`. During serialization, the pipeline derives one stage context whose `originalValue` is the +object supplied to the root value codec and forwards it to every `SerDesStage` and nested `BinarySerDesStage`. During +deserialization, the forwarded stage context has a `null` `originalValue`. No SerDes-specific thread-local storage is +required. ```java public record SerDesContext( @@ -96,17 +97,13 @@ public record SerDesContext( String parentId, OperationType operationType, OperationSubType operationSubType, - Integer attempt) { - public static SerDesContext getCurrentContext() { - return SerDesContextHolder.get(); - } -} + Integer attempt, + Object originalValue) {} ``` -The SDK owns setting and clearing this thread-local value around SDK-managed SerDes calls. The setter is not part of -the public customer API. Stage implementations use their context parameter; existing value codecs may read the -compatibility accessor. If SerDes is called directly by customer code outside the SDK, `getCurrentContext()` returns -`null`, and a stage invoked through that direct call receives `null`. +Only stage implementations receive this context. Root value codecs continue using the backward-compatible `SerDes` +methods without context. If a composable SerDes or stage is invoked directly by customer code outside the SDK, its +stage context is `null`. ### Package @@ -197,16 +194,18 @@ Equivalent pseudocode: ```java String serialize(Object value, SerDesContext context) { String current = valueCodec.serialize(value); + SerDesContext stageContext = context.withOriginalValue(value); for (var stage : stages) { - current = stage.serialize(current, context); + current = stage.serialize(current, stageContext); } return current; } T deserialize(String data, TypeToken targetType, SerDesContext context) { String current = data; + SerDesContext stageContext = context.withOriginalValue(null); for (int i = stages.size() - 1; i >= 0; i--) { - current = stages.get(i).deserialize(current, context); + current = stages.get(i).deserialize(current, stageContext); } return valueCodec.deserialize(current, targetType); } @@ -216,9 +215,10 @@ Pipeline rules: - A pipeline must contain exactly one value codec in the first position and zero or more string stages. - Every top-level stage consumes and produces a non-null `String`. This makes all stage orderings type-compatible. -- `SerDesRunner` passes the same read-only `SerDesContext` explicitly to every top-level and binary stage call. - A context-dependent stage validates the supplied parameter and reports a normal SerDes failure when it is - unavailable or incomplete. +- `SerDesRunner` passes a read-only `SerDesContext` explicitly to every top-level and binary stage call. During + serialization the same derived stage context exposes the original object through `originalValue`; during + deserialization `originalValue` is `null`. A context-dependent stage validates the supplied parameter and reports a + normal SerDes failure when it is unavailable or incomplete. - The test runners identify a configured input pipeline directly as `ComposableSerDes` and reject it because initial input accepts one value codec, not a pipeline. - `SerDes.then(...)`, `ComposableSerDes.then(...)`, and the builder accept only `SerDesStage` after the value codec. @@ -228,7 +228,7 @@ Pipeline rules: - A `null` value at the pipeline boundary short-circuits the entire pipeline: serializing or deserializing `null` returns `null` without invoking any stage. A stage returning `null` for non-null input is an error. The value codec may decode a non-null representation such as the JSON literal `null` to a null domain value. -- All stages execute within the same `SerDesRunner` invocation and receive the same read-only `SerDesContext`, whether +- All stages execute within the same `SerDesRunner` invocation and receive the same read-only stage context, whether the runner executes inline or dispatches to a configured executor. - `ComposableSerDes` is immutable. It is safe for concurrent use only when every contained stage is also safe for concurrent use, matching the existing `SerDes` requirement. @@ -339,11 +339,11 @@ conversion only at its two outer boundaries; binary stages pass bytes directly t ### Retryable SerDes stages Transient failures are explicit. `RetryableSerDesException` extends `SerDesException` and marks a failure that may -succeed when attempted again. `RetrySerDes` implements `SerDesStage`, decorates another `SerDesStage`, and applies an -existing `RetryStrategy`: +succeed when attempted again. `RetrySerDesStage` decorates a `SerDesStage`, and `RetryBinarySerDesStage` decorates a +`BinarySerDesStage`. Both apply an existing `RetryStrategy`: ```java -var resilientFileSystemStage = new RetrySerDes( +var resilientFileSystemStage = new RetrySerDesStage( fileSystemStage, RetryStrategies.exponentialBackoff( 3, @@ -353,13 +353,17 @@ var resilientFileSystemStage = new RetrySerDes( JitterStrategy.FULL)); var serDes = new JacksonSerDes().then(resilientFileSystemStage); + +var resilientEncryptionStage = new RetryBinarySerDesStage( + encryptionStage, + RetryStrategies.fixedDelay(3, Duration.ofMillis(100))); ``` Retry rules: - Only `RetryableSerDesException` is retried. Ordinary `SerDesException` and other failures propagate immediately. - `RetryStrategy.makeRetryDecision(error, attempt)` receives the transient failure and a 1-based attempt number. -- When the strategy returns `fail`, `RetrySerDes` rethrows the last `RetryableSerDesException`. +- When the strategy returns `fail`, the retry wrapper rethrows the last `RetryableSerDesException`. - The same read-only `SerDesContext` parameter is passed to every attempt because retrying happens inside the original `SerDesRunner` task. - A retry delay blocks the thread executing the SerDes call. This is the caller thread by default or a SerDes executor @@ -367,11 +371,12 @@ Retry rules: checkpoint. Strategies must therefore use short, bounded delays that fit within the Lambda invocation timeout. - If the invocation is interrupted or times out, replay may execute the SerDes pipeline again. Any stage with side effects must use stable addressing and idempotent writes. -- `RetrySerDes` wraps an individual `SerDesStage`, such as `FileSystemSerDes`. It cannot wrap the value codec or the - complete pipeline. Retrying only the transient component avoids repeating deterministic encoding, compression, or - encryption work. +- `RetrySerDesStage` wraps an individual `SerDesStage`, such as `FileSystemSerDes`. + `RetryBinarySerDesStage` wraps an individual binary stage inside a `ComposableBinarySerDesStage`. Neither can wrap + the value codec or complete pipeline. Retrying only the transient component avoids repeating unrelated deterministic + work. - Pipeline error decoration must preserve retryability: a stage-level `RetryableSerDesException` must remain that type, - with stage metadata added to its message or cause, so an enclosing `RetrySerDes` can recognize it. + with stage metadata added to its message or cause, so an enclosing retry wrapper can recognize it. - Filesystem read and write `IOException`s are retryable. Malformed envelopes, invalid paths, unsupported types, and stage transformation errors are permanent. @@ -433,7 +438,8 @@ For JSON pipelines, `previewConfig(...)` parses the `String` produced by the pre structured preview controls as the Python and TypeScript SDKs: include-all or exclude-all mode, include/exclude/mask selectors, field-name or exact-path matching, a configurable mask string, and a default 4 KB byte budget. The standalone `SerDesPreview` utility exposes the same builder for customer-managed values. `previewGenerator(...)` -remains available for non-JSON stage values and fully custom preview logic. +remains available for non-JSON stage values and fully custom preview logic; it receives both the stage string and the +serialization `SerDesContext`, whose `originalValue` exposes the object supplied to the root value codec. ### Runtime flow @@ -442,9 +448,9 @@ var checkpointPayload = serDesRunner.serialize(composableSerDes, value, context) sendCheckpoint(checkpointPayload); ``` -Inside `SerDesRunner`, a composable pipeline receives `context` directly and forwards it to every stage. The runner -also installs the same value in `SerDesContextHolder` around the call so the root `SerDes` value codec remains -backward-compatible. +Inside `SerDesRunner`, a composable pipeline receives `context` directly. The pipeline invokes the root value codec +without context, derives a serialization stage context containing the original object, and forwards that derived +context to every stage. Deserialization stages receive a context with no original value. On deserialization, `FileSystemSerDes` first checks for its reserved marker. Unmarked input is returned unchanged. If the marked envelope contains `data`, it restores the inline text; if it contains `file`, it reads the stored string. @@ -477,10 +483,8 @@ The core SDK should route user payload SerDes calls through a helper, tentativel - Executes inline when no SerDes executor is configured. - Dispatches to the configured executor only when one is present. - Passes `SerDesContext` explicitly to every stage in a `ComposableSerDes`. -- Sets the same `SerDesContext` in TLS on the thread that invokes the root `SerDes`, preserving compatibility for - existing value codecs. +- Adds the original object to the context passed to serialization stages and clears it for deserialization stages. - Invokes the existing `SerDes.serialize` and `SerDes.deserialize` methods for non-composable implementations. -- Restores the previous TLS value after each SerDes call, or clears it when there was no previous value. - Wraps failures in `SerDesException` with operation and payload kind metadata. Equivalent execution flow: @@ -488,21 +492,18 @@ Equivalent execution flow: ```java T run(SerDesContext context, Supplier operation) { if (serDesExecutorService == null) { - return runWithContext(context, operation); + return operation.get(); } - return CompletableFuture - .supplyAsync(() -> runWithContext(context, operation), serDesExecutorService) - .join(); + return CompletableFuture.supplyAsync(operation, serDesExecutorService).join(); } ``` -Because TLS is bound to a single Java thread, `SerDesRunner` must install the context on whichever thread executes the -operation. It must not rely on inheritable thread-local propagation. Restoring the previous value also makes nested -SDK-managed SerDes calls safe in inline mode. +Context propagation does not depend on thread-local state. `SerDesRunner` passes context into the composable pipeline +as an ordinary argument, so inline and executor-backed calls have the same stage semantics. Inline execution is the compatibility and low-overhead default, not a recommendation to perform blocking storage work on operation threads. Documentation and filesystem examples should configure a SerDes executor whenever -`FileSystemSerDes`, delayed `RetrySerDes`, or another blocking stage is used. If it is omitted, I/O and retry delays +`FileSystemSerDes`, a delayed retry stage, or another blocking stage is used. If it is omitted, I/O and retry delays block the calling thread. ### Caching @@ -556,14 +557,14 @@ persisted pipeline. ### Implementation plan -1. Add `SerDesContext`, `SerDesPayloadKind`, and package-private TLS setter/clearer support. Leave the existing - `SerDes` methods unchanged. +1. Add `SerDesContext` and `SerDesPayloadKind`. Leave the existing context-free `SerDes` methods unchanged. 2. Add the binary-compatible `SerDes.then(SerDesStage)` default method and `ComposableSerDes` with one root `SerDes` followed only by immutable `SerDesStage` entries, forward serialization, reverse deserialization, self-identifying stage pass-through, null short-circuiting, and stage-aware errors. 3. Add the string-only `SerDesStage` contract plus `BinarySerDesStage`, `StringBinaryCodec`, and a version-framed `ComposableBinarySerDesStage` for nested binary processing with ordered, configurable boundaries. -4. Add `RetryableSerDesException` and `RetrySerDes`, reusing `RetryStrategy` for bounded in-invocation retries. +4. Add `RetryableSerDesException`, `RetrySerDesStage`, and `RetryBinarySerDesStage`, reusing `RetryStrategy` for + bounded in-invocation retries. 5. Add `SerDesRunner` with inline execution by default and optional dispatch through `DurableConfig.withSerDesExecutorService(...)`. Do not create a default SerDes pool. 6. Update root input/output handling in `DurableExecutor` to run user payload SerDes through `SerDesRunner` while @@ -800,7 +801,7 @@ This approach gives the SDK one consistent policy for root payloads, operation r | Responsibility boundary | Combines value serialization and storage-reference creation in ordered, composable SerDes stages. | Keeps object encoding in `SerDes` and storage movement in a separate offloader. | | User configuration | Users configure one SerDes or a `ComposableSerDes` pipeline. Operation-level SerDes selection replaces the whole pipeline. | Users configure both a SerDes and an offloader. The SDK must define global, per-operation, and per-payload precedence. | | Parity with JS issue | Closest to the current JavaScript `createFileSystemSerdes` model and issue #463 wording. | Diverges from JavaScript naming and shape, though it may be architecturally cleaner for Java. | -| Core SDK changes | Adds the compatible `SerDes.then(SerDesStage)` default method, `ComposableSerDes`, explicit stage context parameters, and `SerDesContext` TLS compatibility for existing root codecs because `SerDes` methods have no context parameter. | Requires a new core extension point, envelope type, config surface, payload pipeline, and migration story. | +| Core SDK changes | Adds the compatible `SerDes.then(SerDesStage)` default method, `ComposableSerDes`, and explicit stage context parameters while root codecs remain context-free. | Requires a new core extension point, envelope type, config surface, payload pipeline, and migration story. | | Applicability | Any compatible `SerDesStage` implementations can be chained, but storage stages still own their envelope and lifecycle behavior. | Any SerDes output can be offloaded uniformly after serialization. Users can combine Jackson/custom SerDes with any offloader. | | Envelope ownership | FileSystemSerDes owns the checkpoint envelope (`data`, `file`, preview), so the SDK treats it as opaque serialized data. | SDK owns the checkpoint/offload envelope and must guarantee it composes with replay, errors, callbacks, and test utilities. | | Caching | SDK can cache deserialized values, but FileSystemSerDes may still do file reads internally unless cache hits happen before SerDes. | SDK can cache at both layers: resolved offloaded payload text and final deserialized object. | @@ -886,9 +887,9 @@ publishing, documentation, and dependency-management overhead without isolating Rejected for the root `SerDes` interface in Approach A. Context is explicit on the new stage interfaces, where it does not affect compatibility. Adding it to `SerDes` itself would force every existing custom value codec to change its -serialization method surface. Approach A keeps those signatures unchanged and retains `SerDesContext` TLS only as a -compatibility bridge for root codecs. Approach B does not need SerDes TLS because `PayloadOffloader` receives -`PayloadOffloadContext` explicitly. +serialization method surface. Approach A keeps those signatures unchanged and root value codecs context-free. The +pipeline enriches serialization-stage context with the original object instead. Approach B likewise passes +`PayloadOffloadContext` explicitly to its offloader. ### Make SerDes async @@ -933,8 +934,7 @@ Negative: - Adds optional executor, context, and caching machinery that must stay deterministic. - Adds storage-specific public API and implementation code to the core SDK artifact. -- Approach A retains thread-local SerDes context for backward-compatible root codecs because the existing `SerDes` - methods do not accept context; pipeline stages receive it explicitly. +- Root value codecs do not receive durable payload context; only pipeline stages receive it explicitly. - Top-level stages must encode non-string representations at their boundaries. A composable binary stage avoids repeated conversion between binary substages, but its final bytes still require one string encoding. - Pipeline ordering and stage configuration must remain compatible with persisted checkpoints. diff --git a/docs/advanced/configuration.md b/docs/advanced/configuration.md index 25b9e7bed..0f3726c17 100644 --- a/docs/advanced/configuration.md +++ b/docs/advanced/configuration.md @@ -48,9 +48,9 @@ By default, SerDes runs synchronously on the calling thread to preserve existing SerDes executor must be different from the user-operation executor to avoid deadlock when the operation pool is saturated. -The SDK passes `SerDesContext` explicitly to every `SerDesStage` and nested `BinarySerDesStage`. It also installs the -same context on whichever thread performs the root `SerDes` call for backward compatibility with existing value -codecs, restores any previous nested context afterward, and uses a bounded weak-reference cache for successful +The SDK passes `SerDesContext` explicitly only to every `SerDesStage` and nested `BinarySerDesStage`; root `SerDes` +value codecs remain context-free. During serialization, `context.originalValue()` contains the object supplied to the +root value codec. During deserialization it is `null`. The SDK also uses a bounded weak-reference cache for successful deserialization results during the current invocation. ### Filesystem-backed payload storage @@ -63,7 +63,7 @@ var fileSystemStage = FileSystemSerDes.builder(Path.of("/mnt/efs/durable-payload .pathEncoding(FileSystemPathEncoding.HASH) .build(); -var resilientFileSystemStage = new RetrySerDes( +var resilientFileSystemStage = new RetrySerDesStage( fileSystemStage, RetryStrategies.fixedDelay(3, Duration.ofSeconds(1))); @@ -87,8 +87,10 @@ return DurableConfig.builder() ``` Every top-level stage consumes and produces a string, so stages compose without intermediate type mismatches. -Every stage method also receives the current read-only `SerDesContext`; the same instance is propagated through the -complete pipeline and through every retry attempt. +Every stage method also receives the current read-only `SerDesContext`. Serialization stages receive one derived +context whose `originalValue()` is the object supplied to the root value codec; deserialization stages receive a +context whose `originalValue()` is `null`. The same stage context is propagated through the complete pipeline and +through every retry attempt. Every stage must emit a self-identifying, normally versioned representation. On deserialization it reverses recognized valid input, rejects recognized malformed or unsupported input, and returns unrecognized input unchanged. This lets raw external payloads pass through the configured stages and reach the root value codec. @@ -121,11 +123,13 @@ var serDes = new JacksonSerDes().then(fileSystemStage); The built-in preview configuration parses the string produced by the preceding stage as JSON. Use `previewGenerator(...)` when the preceding stage produces another format or when fully custom preview logic is needed. +The custom generator receives both that string and `SerDesContext`, so it can use `originalValue()` when the preview +should be derived from the pre-serialization object. Custom generators must avoid exposing sensitive fields. -`RetrySerDes` implements `SerDesStage` and retries only failures marked with `RetryableSerDesException`. Filesystem -read and write I/O use this marker; malformed envelopes and codec failures fail immediately. Backoff occurs within the -current Lambda invocation, so use short, bounded retry strategies. Without a configured SerDes executor, filesystem -I/O and retry delays block the calling thread. +`RetrySerDesStage` wraps a `SerDesStage`, while `RetryBinarySerDesStage` wraps a `BinarySerDesStage`. Both retry only +failures marked with `RetryableSerDesException`. Filesystem read and write I/O use this marker; malformed envelopes and +codec failures fail immediately. Backoff occurs within the current Lambda invocation, so use short, bounded retry +strategies. Without a configured SerDes executor, filesystem I/O and retry delays block the calling thread. Do not use Lambda's ephemeral `/tmp` directory: replay can run in a different execution environment. Use a durable, shared mount such as EFS or S3 Files. Payloads are published with one immutable `CREATE_NEW` write, without hard links diff --git a/docs/advanced/error-handling.md b/docs/advanced/error-handling.md index ddd7552b4..f2a28873e 100644 --- a/docs/advanced/error-handling.md +++ b/docs/advanced/error-handling.md @@ -12,7 +12,7 @@ Error RuntimeException └── DurableExecutionException - General durable exception ├── SerDesException - Serialization and deserialization exception. - │ └── RetryableSerDesException - Transient SerDes failure eligible for RetrySerDes. + │ └── RetryableSerDesException - Transient SerDes failure eligible for a retry stage wrapper. ├── UnrecoverableDurableExecutionException - Execution cannot be recovered. The durable execution will be immediately terminated. │ ├── NonDeterministicExecutionException - Code changed between original execution and replay. Fix code to maintain determinism; don't change step order/names. │ └── IllegalDurableOperationException - An illegal operation was detected. The execution will be immediately terminated. diff --git a/docs/advanced/filesystem-serdes.md b/docs/advanced/filesystem-serdes.md index c57c9a36b..cb447f4ca 100644 --- a/docs/advanced/filesystem-serdes.md +++ b/docs/advanced/filesystem-serdes.md @@ -23,7 +23,7 @@ var fileSystemStage = FileSystemSerDes.builder(Path.of("/mnt/efs/durable-payload .pathEncoding(FileSystemPathEncoding.URI) .build(); -var resilientFileSystemStage = new RetrySerDes( +var resilientFileSystemStage = new RetrySerDesStage( fileSystemStage, RetryStrategies.fixedDelay(3, Duration.ofSeconds(1))); @@ -50,10 +50,11 @@ Serialization follows the declaration order above and deserialization runs in re `SerDes` value codec; every component appended with `then(...)` implements `SerDesStage` and consumes and produces a string. Each stage must use a self-identifying format: it reverses recognized valid input, rejects recognized malformed or unsupported input, and returns unrecognized input unchanged. The runner passes the same read-only -`SerDesContext` explicitly to every string stage and binary substage. `ComposableBinarySerDesStage` converts the string -with its starting codec, passes bytes directly through each `BinarySerDesStage`, converts the final bytes to a string -with its ending codec, and adds a reserved versioned frame. Both boundaries are customizable through the same -`StringBinaryCodec` interface; the example performs UTF-8 and Base64 conversion once around the complete +`SerDesContext` explicitly to every string stage and binary substage. During serialization, `originalValue()` exposes +the object supplied to the root value codec; during deserialization it is `null`. `ComposableBinarySerDesStage` +converts the string with its starting codec, passes bytes directly through each `BinarySerDesStage`, converts the final +bytes to a string with its ending codec, and adds a reserved versioned frame. Both boundaries are customizable through +the same `StringBinaryCodec` interface; the example performs UTF-8 and Base64 conversion once around the complete compression/encryption chain. - `ALWAYS` writes every non-null payload to a file. @@ -86,8 +87,10 @@ visible; include and mask rules make selected fields visible. `ANYWHERE` matches `PATH` matches an exact dot-separated path. Exclude rules win over mask rules, and masking implies visibility. The built-in `previewConfig(...)` parses the incoming stage string as JSON. Use `previewGenerator(...)` for non-JSON -stage values or fully custom logic. Previews are included only in file envelopes. The structured builder defaults to a -4 KB preview budget, and the complete file envelope must still remain below the checkpoint threshold. +stage values or fully custom logic. The custom generator receives the stage string and `SerDesContext`, including the +pre-serialization object in `originalValue()`. Previews are included only in file envelopes. The structured builder +defaults to a 4 KB preview budget, and the complete file envelope must still remain below the checkpoint threshold. +Custom generators must avoid exposing sensitive fields. ## Execution and retries @@ -95,10 +98,12 @@ SerDes runs inline by default. Filesystem access and retry backoff are blocking, provide a dedicated executor with `withSerDesExecutorService(...)`. It must be different from the user-operation executor. -Filesystem read and write I/O failures are reported as `RetryableSerDesException`. `RetrySerDes` implements -`SerDesStage`, so the retrying filesystem component can be appended directly to the pipeline. It retries only that -exception type. Malformed envelopes, invalid paths, unsupported stage types, and codec failures are permanent. Retry -delays consume time in the current Lambda invocation, so keep attempts and delays bounded. +Filesystem read and write I/O failures are reported as `RetryableSerDesException`. `RetrySerDesStage` wraps a +`SerDesStage`, so the retrying filesystem component can be appended directly to the pipeline. +`RetryBinarySerDesStage` provides the same behavior for a `BinarySerDesStage` inside +`ComposableBinarySerDesStage`. Both retry only that exception type. Malformed envelopes, invalid paths, unsupported +stage types, and codec failures are permanent. Retry delays consume time in the current Lambda invocation, so keep +attempts and delays bounded. ## Replay and envelope behavior diff --git a/docs/design.md b/docs/design.md index 7755d8cbd..f6cf311c6 100644 --- a/docs/design.md +++ b/docs/design.md @@ -363,14 +363,15 @@ software.amazon.lambda.durable │ ├── Base64StringBinaryCodec # Standard Base64 string/byte conversion │ ├── ComposableBinarySerDesStage # Ordered binary chain exposed as one string stage │ ├── JacksonSerDes # Jackson impl -│ ├── RetrySerDes # Retrying string-stage decorator +│ ├── RetrySerDesStage # Retrying string-stage decorator +│ ├── RetryBinarySerDesStage # Retrying binary-stage decorator │ ├── SerDesPreview # Structured preview builder │ ├── PreviewConfig # Preview selection, masking, and size configuration │ ├── PreviewField # Field-name or exact-path preview selector │ ├── PreviewMode # Include-all or exclude-all preview default │ ├── FieldMatchMode # Anywhere or exact-path field matching -│ ├── SerDesRunner # Context, optional executor dispatch, and invocation cache -│ ├── SerDesContext # Read-only durable payload identity +│ ├── SerDesRunner # Stage context, optional executor dispatch, and invocation cache +│ ├── SerDesContext # Read-only durable payload identity and serialization source value │ ├── SerDesPayloadKind # Input/result/state/exception/invoke payload kind │ └── AwsSdkV2Module # SDK type support │ @@ -528,7 +529,7 @@ SuspendExecutionException # Internal: triggers suspension (not | `NonDeterministicExecutionException` | Replay finds different operation than expected | Bug in handler (non-deterministic code) | | `IllegalDurableOperationException` | Illegal operation detected | Bug in handler | | `SerDesException` | Jackson fails to serialize/deserialize | Fix data model or custom SerDes | -| `RetryableSerDesException` | Transient stage or payload storage failure | Wrap the failing `SerDesStage` with `RetrySerDes` and a bounded retry strategy | +| `RetryableSerDesException` | Transient stage or payload storage failure | Wrap the failing string or binary stage with `RetrySerDesStage` or `RetryBinarySerDesStage` and a bounded retry strategy | --- diff --git a/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FileSystemSerDesIntegrationTest.java b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FileSystemSerDesIntegrationTest.java index ce44c1175..53843a4d2 100644 --- a/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FileSystemSerDesIntegrationTest.java +++ b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FileSystemSerDesIntegrationTest.java @@ -155,8 +155,7 @@ void durableExecutorAcceptsRawServiceInputBeforeFilesystemEnvelopeExists() { @Test void acceptsRawCallbackAndInvokeResultsAndOffloadsInvokePayload() throws Exception { var invokePayload = new AtomicReference(); - var recordingStage = identityStage((action, value) -> { - var context = SerDesContext.getCurrentContext(); + var recordingStage = identityStage((action, value, context) -> { if ("serialize".equals(action) && context.payloadKind() == SerDesPayloadKind.INVOKE_PAYLOAD) { invokePayload.set(value); } @@ -270,8 +269,7 @@ void callerAndCalleeExchangeOffloadedInvokePayloadAndResult() throws Exception { @Test void repeatedGetUsesInvocationCacheForTheCompletePipeline() { var resultDeserializations = new AtomicInteger(); - var countingStage = identityStage((action, value) -> { - var context = SerDesContext.getCurrentContext(); + var countingStage = identityStage((action, value, context) -> { if ("deserialize".equals(action) && context.payloadKind() == SerDesPayloadKind.RESULT && "cached-step".equals(context.operationName())) { @@ -306,8 +304,7 @@ void repeatedGetUsesInvocationCacheForTheCompletePipeline() { void successfulRetryUsesTheProducingAttemptForResultSerialization() { var executions = new AtomicInteger(); var resultAttempts = new ArrayList(); - var attemptStage = identityStage((action, value) -> { - var context = SerDesContext.getCurrentContext(); + var attemptStage = identityStage((action, value, context) -> { if ("serialize".equals(action) && context.payloadKind() == SerDesPayloadKind.RESULT && "retry-step".equals(context.operationName())) { @@ -503,13 +500,13 @@ private static SerDesStage identityStage(RecordingFunction recorder) { return new SerDesStage() { @Override public String serialize(String value, SerDesContext context) { - recorder.record("serialize", value); + recorder.record("serialize", value, context); return value; } @Override public String deserialize(String data, SerDesContext context) { - recorder.record("deserialize", data); + recorder.record("deserialize", data, context); return data; } }; @@ -532,7 +529,7 @@ private void assertForwardedErrorOwnedByChild(TestResult result, String @FunctionalInterface private interface RecordingFunction { - void record(String action, String value); + void record(String action, String value, SerDesContext context); } record Payload(String value) {} diff --git a/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/TestOperationTest.java b/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/TestOperationTest.java index 42a663691..ba0d29a3e 100644 --- a/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/TestOperationTest.java +++ b/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/TestOperationTest.java @@ -3,6 +3,7 @@ package software.amazon.lambda.durable.testing; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; import java.util.List; import java.util.concurrent.atomic.AtomicReference; @@ -15,6 +16,7 @@ import software.amazon.lambda.durable.serde.SerDes; import software.amazon.lambda.durable.serde.SerDesContext; import software.amazon.lambda.durable.serde.SerDesRunner; +import software.amazon.lambda.durable.serde.SerDesStage; class TestOperationTest { private static final String EXECUTION_ARN = "arn:aws:lambda:us-east-1:123456789012:function:test:$LATEST" @@ -23,7 +25,7 @@ class TestOperationTest { @Test void failedWaitForConditionReadsStateFromPreviousAttempt() { var observedContext = new AtomicReference(); - var serDes = new SerDes() { + var valueCodec = new SerDes() { @Override public String serialize(Object value) { return value.toString(); @@ -32,10 +34,21 @@ public String serialize(Object value) { @Override @SuppressWarnings("unchecked") public T deserialize(String data, TypeToken typeToken) { - observedContext.set(SerDesContext.getCurrentContext()); return (T) data; } }; + var serDes = valueCodec.then(new SerDesStage() { + @Override + public String serialize(String value, SerDesContext context) { + return value; + } + + @Override + public String deserialize(String data, SerDesContext context) { + observedContext.set(context); + return data; + } + }); var operation = Operation.builder() .id("wait-id") .name("wait-condition") @@ -52,5 +65,6 @@ public T deserialize(String data, TypeToken typeToken) { assertEquals("retained-state", testOperation.getStepResult(String.class)); assertEquals(2, observedContext.get().attempt()); assertEquals("operation/wait-id/state/attempt-2", observedContext.get().entityId()); + assertNull(observedContext.get().originalValue()); } } diff --git a/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/cloud/HistoryEventProcessorTest.java b/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/cloud/HistoryEventProcessorTest.java index 633d52459..326be662c 100644 --- a/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/cloud/HistoryEventProcessorTest.java +++ b/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/cloud/HistoryEventProcessorTest.java @@ -3,6 +3,7 @@ package software.amazon.lambda.durable.testing.cloud; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; import java.time.Duration; import java.time.Instant; @@ -26,6 +27,7 @@ import software.amazon.lambda.durable.serde.SerDesContext; import software.amazon.lambda.durable.serde.SerDesPayloadKind; import software.amazon.lambda.durable.serde.SerDesRunner; +import software.amazon.lambda.durable.serde.SerDesStage; class HistoryEventProcessorTest { private static final String EXECUTION_ARN = "arn:aws:lambda:us-east-1:123456789012:function:test:$LATEST" @@ -115,6 +117,7 @@ void deserializesCloudResultsWithDurablePayloadContext() { assertEquals(OperationType.EXECUTION, outputContext.operationType()); assertEquals(SerDesPayloadKind.OUTPUT, outputContext.payloadKind()); assertEquals("execution/invocation-id/output", outputContext.entityId()); + assertNull(outputContext.originalValue()); var stepContext = observedContexts.get(1); assertEquals(OperationType.STEP, stepContext.operationType()); @@ -124,7 +127,7 @@ void deserializesCloudResultsWithDurablePayloadContext() { } private static SerDes recordingStringSerDes(List observedContexts) { - return new SerDes() { + SerDes valueCodec = new SerDes() { @Override public String serialize(Object value) { return value.toString(); @@ -133,9 +136,20 @@ public String serialize(Object value) { @Override @SuppressWarnings("unchecked") public T deserialize(String data, TypeToken typeToken) { - observedContexts.add(SerDesContext.getCurrentContext()); return (T) data; } }; + return valueCodec.then(new SerDesStage() { + @Override + public String serialize(String value, SerDesContext context) { + return value; + } + + @Override + public String deserialize(String data, SerDesContext context) { + observedContexts.add(context); + return data; + } + }); } } diff --git a/sdk/src/main/java/software/amazon/lambda/durable/exception/RetryableSerDesException.java b/sdk/src/main/java/software/amazon/lambda/durable/exception/RetryableSerDesException.java index be06a762f..d40ad067a 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/exception/RetryableSerDesException.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/exception/RetryableSerDesException.java @@ -5,7 +5,8 @@ /** * Indicates a transient serialization or deserialization failure that may succeed when retried. * - *

{@link software.amazon.lambda.durable.serde.RetrySerDes} retries only this exception type. Other + *

{@link software.amazon.lambda.durable.serde.RetrySerDesStage} and + * {@link software.amazon.lambda.durable.serde.RetryBinarySerDesStage} retry only this exception type. Other * {@link SerDesException} instances are treated as permanent failures. */ public class RetryableSerDesException extends SerDesException { diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/BinarySerDesStage.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/BinarySerDesStage.java index 171829319..4baac5d31 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/BinarySerDesStage.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/BinarySerDesStage.java @@ -8,8 +8,10 @@ *

Implementations must include any metadata needed for deserialization, such as format versions or encryption * initialization vectors, in the returned bytes. * - *

The enclosing composable stage passes the same durable payload context to each binary stage. The context may be - * {@code null} only when the stage is invoked outside an SDK-managed SerDes call. + *

The enclosing composable stage passes the same durable payload context to each binary stage. During serialization, + * {@link SerDesContext#originalValue()} is the object supplied to the root value codec. During deserialization it is + * {@code null}. Stages must treat the original value as read-only. The context itself may be {@code null} only when the + * stage is invoked outside an SDK-managed SerDes call. */ public interface BinarySerDesStage { /** diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/ComposableSerDes.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/ComposableSerDes.java index d9a002bb6..0a9bccd29 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/ComposableSerDes.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/ComposableSerDes.java @@ -15,7 +15,8 @@ *

The first component is the value codec. Every later component is a {@link SerDesStage} that consumes and produces * a string. Serialization runs from first to last; deserialization runs from last to first. Each stage returns * unrecognized input unchanged, allowing raw values to pass through to the root value codec. SDK-managed calls pass the - * same {@link SerDesContext} explicitly to every stage. + * same {@link SerDesContext} explicitly to every stage. During serialization that context also exposes the original + * object supplied to the value codec. */ public final class ComposableSerDes implements SerDes { private final SerDes valueCodec; @@ -72,23 +73,24 @@ public ComposableSerDes then(SerDesStage stage) { @Override public String serialize(Object value) { - return serialize(value, SerDesContext.getCurrentContext()); + return serialize(value, null); } String serialize(Object value, SerDesContext context) { if (value == null) { return null; } + var stageContext = context == null ? null : context.withOriginalValue(value); String current = invokeValueCodecSerialize(valueCodec, value); for (int index = 0; index < stages.size(); index++) { - current = invokeStageSerialize(stages.get(index), current, context, index + 1); + current = invokeStageSerialize(stages.get(index), current, stageContext, index + 1); } return current; } @Override public T deserialize(String data, TypeToken typeToken) { - return deserialize(data, typeToken, SerDesContext.getCurrentContext()); + return deserialize(data, typeToken, null); } T deserialize(String data, TypeToken typeToken, SerDesContext context) { @@ -96,9 +98,10 @@ T deserialize(String data, TypeToken typeToken, SerDesContext context) { return null; } Objects.requireNonNull(typeToken, "typeToken cannot be null"); + var stageContext = context == null ? null : context.withOriginalValue(null); String current = data; for (int index = stages.size() - 1; index >= 0; index--) { - current = invokeStageDeserialize(stages.get(index), current, context, index + 1); + current = invokeStageDeserialize(stages.get(index), current, stageContext, index + 1); } return invokeValueCodecDeserialize(valueCodec, current, typeToken); } diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java index 15ca2078d..ce267c7b8 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java @@ -19,7 +19,7 @@ import java.util.Map; import java.util.Objects; import java.util.UUID; -import java.util.function.Function; +import java.util.function.BiFunction; import java.util.regex.Pattern; import software.amazon.awssdk.services.lambda.model.OperationType; import software.amazon.lambda.durable.exception.RetryableSerDesException; @@ -51,7 +51,7 @@ public final class FileSystemSerDes implements SerDesStage { private final Path basePath; private final FileSystemStorageMode storageMode; private final FileSystemPathEncoding pathEncoding; - private final Function> previewGenerator; + private final BiFunction> previewGenerator; private volatile Path canonicalBasePath; private FileSystemSerDes(Builder builder) { @@ -332,7 +332,7 @@ private Map generatePreview(String value, SerDesContext context) return null; } try { - return previewGenerator.apply(value); + return previewGenerator.apply(value, context); } catch (RuntimeException e) { throw new SerDesException( "Failed to generate filesystem payload preview for entity '" + context.entityId() + "'", e); @@ -565,7 +565,7 @@ public static final class Builder { private final Path basePath; private FileSystemStorageMode storageMode = FileSystemStorageMode.ALWAYS; private FileSystemPathEncoding pathEncoding = FileSystemPathEncoding.URI; - private Function> previewGenerator; + private BiFunction> previewGenerator; private Builder(Path basePath) { this.basePath = Objects.requireNonNull(basePath, "basePath cannot be null"); @@ -582,11 +582,14 @@ public Builder pathEncoding(FileSystemPathEncoding pathEncoding) { } /** - * Configures a custom preview generator that receives the string produced by the preceding pipeline stage. + * Configures a custom preview generator that receives the string produced by the preceding pipeline stage and + * its serialization context. * - *

The returned preview is included only when the payload is stored in a file. + *

{@link SerDesContext#originalValue()} contains the object supplied to the pipeline's root value codec. The + * returned preview is included only when the payload is stored in a file. Preview generators are responsible + * for avoiding disclosure of sensitive fields from either input. */ - public Builder previewGenerator(Function> previewGenerator) { + public Builder previewGenerator(BiFunction> previewGenerator) { this.previewGenerator = Objects.requireNonNull(previewGenerator, "previewGenerator cannot be null"); return this; } @@ -594,11 +597,11 @@ public Builder previewGenerator(Function> previewGen /** * Configures structured preview generation for JSON produced by the preceding stage. * - *

Use {@link #previewGenerator(Function)} for non-JSON stage values or fully custom preview logic. + *

Use {@link #previewGenerator(BiFunction)} for non-JSON stage values or fully custom preview logic. */ public Builder previewConfig(PreviewConfig previewConfig) { Objects.requireNonNull(previewConfig, "previewConfig cannot be null"); - this.previewGenerator = value -> SerDesPreview.buildPreviewFromJson(value, previewConfig); + this.previewGenerator = (value, context) -> SerDesPreview.buildPreviewFromJson(value, previewConfig); return this; } diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/RetryBinarySerDesStage.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/RetryBinarySerDesStage.java new file mode 100644 index 000000000..842c08910 --- /dev/null +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/RetryBinarySerDesStage.java @@ -0,0 +1,45 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.serde; + +import java.util.Objects; +import software.amazon.lambda.durable.exception.RetryableSerDesException; +import software.amazon.lambda.durable.retry.RetryStrategy; + +/** + * A binary-stage decorator that retries transient failures from another {@link BinarySerDesStage}. + * + *

Only {@link RetryableSerDesException} is retried. Other failures are propagated immediately. Retry delays block + * the thread executing the SerDes call: the caller by default or the configured SerDes executor thread. Every attempt + * receives the same {@link SerDesContext} supplied to this decorator. + */ +public final class RetryBinarySerDesStage implements BinarySerDesStage { + private final BinarySerDesStage delegate; + private final SerDesRetryExecutor retryExecutor; + + /** + * Creates a retrying binary-stage decorator. + * + * @param delegate the stage to invoke + * @param retryStrategy strategy that controls attempts and delays + */ + public RetryBinarySerDesStage(BinarySerDesStage delegate, RetryStrategy retryStrategy) { + this(delegate, retryStrategy, SerDesRetryExecutor.DEFAULT_SLEEPER); + } + + RetryBinarySerDesStage( + BinarySerDesStage delegate, RetryStrategy retryStrategy, SerDesRetryExecutor.Sleeper sleeper) { + this.delegate = Objects.requireNonNull(delegate, "delegate cannot be null"); + retryExecutor = new SerDesRetryExecutor(retryStrategy, sleeper); + } + + @Override + public byte[] serialize(byte[] value, SerDesContext context) { + return retryExecutor.execute("binary stage serialization", () -> delegate.serialize(value, context)); + } + + @Override + public byte[] deserialize(byte[] data, SerDesContext context) { + return retryExecutor.execute("binary stage deserialization", () -> delegate.deserialize(data, context)); + } +} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/RetrySerDesStage.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/RetrySerDesStage.java new file mode 100644 index 000000000..82698e901 --- /dev/null +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/RetrySerDesStage.java @@ -0,0 +1,44 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.serde; + +import java.util.Objects; +import software.amazon.lambda.durable.exception.RetryableSerDesException; +import software.amazon.lambda.durable.retry.RetryStrategy; + +/** + * A string-stage decorator that retries transient failures from another {@link SerDesStage}. + * + *

Only {@link RetryableSerDesException} is retried. Other failures are propagated immediately. Retry delays block + * the thread executing the SerDes call: the caller by default or the configured SerDes executor thread. Every attempt + * receives the same {@link SerDesContext} supplied to this decorator. + */ +public final class RetrySerDesStage implements SerDesStage { + private final SerDesStage delegate; + private final SerDesRetryExecutor retryExecutor; + + /** + * Creates a retrying string-stage decorator. + * + * @param delegate the stage to invoke + * @param retryStrategy strategy that controls attempts and delays + */ + public RetrySerDesStage(SerDesStage delegate, RetryStrategy retryStrategy) { + this(delegate, retryStrategy, SerDesRetryExecutor.DEFAULT_SLEEPER); + } + + RetrySerDesStage(SerDesStage delegate, RetryStrategy retryStrategy, SerDesRetryExecutor.Sleeper sleeper) { + this.delegate = Objects.requireNonNull(delegate, "delegate cannot be null"); + retryExecutor = new SerDesRetryExecutor(retryStrategy, sleeper); + } + + @Override + public String serialize(String value, SerDesContext context) { + return retryExecutor.execute("string stage serialization", () -> delegate.serialize(value, context)); + } + + @Override + public String deserialize(String data, SerDesContext context) { + return retryExecutor.execute("string stage deserialization", () -> delegate.deserialize(data, context)); + } +} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesContext.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesContext.java index 16ca69326..fdb371b3f 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesContext.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesContext.java @@ -6,12 +6,11 @@ import software.amazon.lambda.durable.model.OperationSubType; /** - * Describes the durable payload currently being processed by a {@link SerDes}. + * Describes the durable payload currently being processed by a {@link SerDesStage} or {@link BinarySerDesStage}. * - *

The SDK passes this context explicitly to {@link SerDesStage} and {@link BinarySerDesStage} methods. It is also - * installed for the duration of the configured {@link SerDes} call so existing value codecs can read it without - * changing the backward-compatible SerDes interface. Direct customer calls to SerDes methods do not have a current - * context. + *

The SDK passes this context explicitly only to pipeline stages. Root {@link SerDes} value codecs remain + * context-free. During serialization, {@link #originalValue()} contains the object supplied to the root value codec; + * during deserialization it is {@code null}. Stages must treat the original value as read-only. */ public record SerDesContext( String durableExecutionArn, @@ -22,17 +21,8 @@ public record SerDesContext( String parentId, OperationType operationType, OperationSubType operationSubType, - Integer attempt) { - - /** - * Returns the context for the SerDes call on the current thread, or {@code null} outside SDK-managed calls. - * - *

Pipeline stages should use the context parameter passed to their methods instead of this compatibility - * accessor. - */ - public static SerDesContext getCurrentContext() { - return SerDesContextHolder.get(); - } + Integer attempt, + Object originalValue) { /** Creates context for a root execution payload. */ public static SerDesContext forExecution( @@ -49,6 +39,7 @@ public static SerDesContext forExecution( null, OperationType.EXECUTION, null, + null, null); } @@ -75,6 +66,24 @@ public static SerDesContext forOperation( parentId, operationType, operationSubType, - attempt); + attempt, + null); + } + + SerDesContext withOriginalValue(Object originalValue) { + if (this.originalValue == originalValue) { + return this; + } + return new SerDesContext( + durableExecutionArn, + entityId, + payloadKind, + operationId, + operationName, + parentId, + operationType, + operationSubType, + attempt, + originalValue); } } diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesContextHolder.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesContextHolder.java deleted file mode 100644 index 6d31cd897..000000000 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesContextHolder.java +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 -package software.amazon.lambda.durable.serde; - -final class SerDesContextHolder { - private static final ThreadLocal CURRENT = new ThreadLocal<>(); - - private SerDesContextHolder() {} - - static SerDesContext get() { - return CURRENT.get(); - } - - static void set(SerDesContext context) { - CURRENT.set(context); - } - - static void clear() { - CURRENT.remove(); - } -} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/RetrySerDes.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesRetryExecutor.java similarity index 67% rename from sdk/src/main/java/software/amazon/lambda/durable/serde/RetrySerDes.java rename to sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesRetryExecutor.java index cde1c0cdc..aaf014ff3 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/RetrySerDes.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesRetryExecutor.java @@ -11,15 +11,8 @@ import software.amazon.lambda.durable.retry.RetryDecision; import software.amazon.lambda.durable.retry.RetryStrategy; -/** - * A string-stage decorator that retries transient failures from another {@link SerDesStage}. - * - *

Only {@link RetryableSerDesException} is retried. Other failures are propagated immediately. Retry delays block - * the thread executing the SerDes call: the caller by default or the configured SerDes executor thread. Every attempt - * receives the same {@link SerDesContext} supplied to this decorator. - */ -public final class RetrySerDes implements SerDesStage { - private static final Sleeper DEFAULT_SLEEPER = delay -> { +final class SerDesRetryExecutor { + static final Sleeper DEFAULT_SLEEPER = delay -> { if (delay.getSeconds() > 0) { TimeUnit.SECONDS.sleep(delay.getSeconds()); } @@ -28,37 +21,15 @@ public final class RetrySerDes implements SerDesStage { } }; - private final SerDesStage delegate; private final RetryStrategy retryStrategy; private final Sleeper sleeper; - /** - * Creates a retrying string-stage decorator. - * - * @param delegate the stage to invoke - * @param retryStrategy strategy that controls attempts and delays - */ - public RetrySerDes(SerDesStage delegate, RetryStrategy retryStrategy) { - this(delegate, retryStrategy, DEFAULT_SLEEPER); - } - - RetrySerDes(SerDesStage delegate, RetryStrategy retryStrategy, Sleeper sleeper) { - this.delegate = Objects.requireNonNull(delegate, "delegate cannot be null"); + SerDesRetryExecutor(RetryStrategy retryStrategy, Sleeper sleeper) { this.retryStrategy = Objects.requireNonNull(retryStrategy, "retryStrategy cannot be null"); this.sleeper = Objects.requireNonNull(sleeper, "sleeper cannot be null"); } - @Override - public String serialize(String value, SerDesContext context) { - return execute("pipeline stage serialization", () -> delegate.serialize(value, context)); - } - - @Override - public String deserialize(String data, SerDesContext context) { - return execute("pipeline stage deserialization", () -> delegate.deserialize(data, context)); - } - - private T execute(String action, Supplier operation) { + T execute(String action, Supplier operation) { int attempt = 1; while (true) { try { diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesRunner.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesRunner.java index 513dcf3fc..c1f55b959 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesRunner.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesRunner.java @@ -21,7 +21,7 @@ import software.amazon.lambda.durable.util.ExceptionHelper; /** - * Runs customer SerDes calls with the correct {@link SerDesContext}. + * Runs customer SerDes calls and passes {@link SerDesContext} explicitly to composable pipeline stages. * *

Calls execute inline unless an executor is configured. Instances are invocation-scoped so successful * deserialization results are cached only for one Lambda invocation. Completed values use a bounded weak-reference @@ -152,10 +152,9 @@ private T run(String action, SerDesContext context, Supplier supplier) { Objects.requireNonNull(context, "SerDesContext cannot be null"); try { if (executorService == null) { - return runWithContext(context, supplier); + return supplier.get(); } - return CompletableFuture.supplyAsync(() -> runWithContext(context, supplier), executorService) - .join(); + return CompletableFuture.supplyAsync(supplier, executorService).join(); } catch (Throwable throwable) { var cause = ExceptionHelper.unwrapCompletableFuture(throwable); if (cause instanceof Error error) { @@ -170,20 +169,6 @@ private T run(String action, SerDesContext context, Supplier supplier) { } } - private static T runWithContext(SerDesContext context, Supplier supplier) { - var previous = SerDesContextHolder.get(); - SerDesContextHolder.set(context); - try { - return supplier.get(); - } finally { - if (previous == null) { - SerDesContextHolder.clear(); - } else { - SerDesContextHolder.set(previous); - } - } - } - private static Object join(CompletableFuture future) { try { return future.join(); diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesStage.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesStage.java index 64700487c..fc4dcedfc 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesStage.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesStage.java @@ -21,15 +21,18 @@ * codec without special pipeline control flow. Implementations should inspect an explicit marker or versioned envelope * before decoding rather than treating any successfully decodable value as recognized. * - *

The SDK passes the durable payload context explicitly to every stage invocation. The context may be {@code null} - * only when a pipeline or stage is invoked directly outside an SDK-managed SerDes call. + *

The SDK passes the durable payload context explicitly to every stage invocation. During serialization, + * {@link SerDesContext#originalValue()} is the object supplied to the root value codec. During deserialization it is + * {@code null}. Stages must treat the original value as read-only. The context itself may be {@code null} only when a + * pipeline or stage is invoked directly outside an SDK-managed SerDes call. */ public interface SerDesStage { /** * Applies this stage during forward serialization. * * @param value the non-null input string - * @param context the current durable payload context, or {@code null} outside SDK-managed calls + * @param context the current durable payload context including the original value, or {@code null} outside + * SDK-managed calls * @return the non-null transformed string */ String serialize(String value, SerDesContext context); @@ -41,7 +44,8 @@ public interface SerDesStage { * it identifies this stage's format but is malformed or unsupported, implementations must throw a SerDes failure. * * @param data the non-null input string - * @param context the current durable payload context, or {@code null} outside SDK-managed calls + * @param context the current durable payload context with a {@code null} original value, or {@code null} outside + * SDK-managed calls * @return the non-null string expected by the preceding stage, or {@code data} unchanged when unrecognized */ String deserialize(String data, SerDesContext context); diff --git a/sdk/src/test/java/software/amazon/lambda/durable/operation/StepOperationTest.java b/sdk/src/test/java/software/amazon/lambda/durable/operation/StepOperationTest.java index 1d8df81b3..669a38714 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/operation/StepOperationTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/operation/StepOperationTest.java @@ -27,6 +27,7 @@ import software.amazon.lambda.durable.model.OperationSubType; import software.amazon.lambda.durable.serde.JacksonSerDes; import software.amazon.lambda.durable.serde.SerDesContext; +import software.amazon.lambda.durable.serde.SerDesStage; class StepOperationTest { @@ -99,13 +100,18 @@ void getDoesNotThrowWhenCalledFromHandlerContext() { @Test void successfulReplayUsesCheckpointedAttemptInSerDesContext() { var observedContext = new AtomicReference(); - var serDes = new JacksonSerDes() { + var serDes = new JacksonSerDes().then(new SerDesStage() { @Override - public T deserialize(String data, TypeToken typeToken) { - observedContext.set(SerDesContext.getCurrentContext()); - return super.deserialize(data, typeToken); + public String serialize(String value, SerDesContext context) { + return value; } - }; + + @Override + public String deserialize(String data, SerDesContext context) { + observedContext.set(context); + return data; + } + }); var op = Operation.builder() .id(OPERATION_ID) .name(OPERATION_NAME) @@ -127,6 +133,7 @@ public T deserialize(String data, TypeToken typeToken) { assertEquals("cached-result", operation.get()); assertEquals(3, observedContext.get().attempt()); + assertNull(observedContext.get().originalValue()); } @Test diff --git a/sdk/src/test/java/software/amazon/lambda/durable/serde/ComposableBinarySerDesStageTest.java b/sdk/src/test/java/software/amazon/lambda/durable/serde/ComposableBinarySerDesStageTest.java index b738c4e07..1c3b6dc2f 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/serde/ComposableBinarySerDesStageTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/serde/ComposableBinarySerDesStageTest.java @@ -13,6 +13,7 @@ import java.util.Arrays; import java.util.Base64; import java.util.List; +import java.util.Map; import java.util.concurrent.atomic.AtomicReference; import org.junit.jupiter.api.Test; import software.amazon.lambda.durable.TypeToken; @@ -84,6 +85,33 @@ public byte[] deserialize(byte[] data, SerDesContext context) { assertSame(CONTEXT, deserializeContext.get()); } + @Test + void receivesTheOriginalValueFromTheRootPipeline() { + var serializeContext = new AtomicReference(); + var binaryStage = ComposableBinarySerDesStage.builder() + .startWith(Utf8StringBinaryCodec.INSTANCE) + .then(new BinarySerDesStage() { + @Override + public byte[] serialize(byte[] value, SerDesContext context) { + serializeContext.set(context); + return value; + } + + @Override + public byte[] deserialize(byte[] data, SerDesContext context) { + return data; + } + }) + .endWith(Base64StringBinaryCodec.INSTANCE) + .build(); + var pipeline = new JacksonSerDes().then(binaryStage); + var originalValue = Map.of("id", 42); + + new SerDesRunner(null).serialize(pipeline, originalValue, CONTEXT); + + assertSame(originalValue, serializeContext.get().originalValue()); + } + @Test void composesWithRootSerDesAsOneStringStage() { var stage = ComposableBinarySerDesStage.builder() diff --git a/sdk/src/test/java/software/amazon/lambda/durable/serde/ComposableSerDesTest.java b/sdk/src/test/java/software/amazon/lambda/durable/serde/ComposableSerDesTest.java index e564e42b8..2e170c6d0 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/serde/ComposableSerDesTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/serde/ComposableSerDesTest.java @@ -63,12 +63,15 @@ public String deserialize(String data, SerDesContext context) { var context = SerDesContext.forExecution("arn", "invocation", "execution", SerDesPayloadKind.RESULT); var pipeline = new JacksonSerDes().then(stage); var runner = new SerDesRunner(null); + var originalValue = new String("value"); - var serialized = runner.serialize(pipeline, "value", context); + var serialized = runner.serialize(pipeline, originalValue, context); runner.deserialize(pipeline, serialized, TypeToken.get(String.class), context); - assertSame(context, observedSerializeContext.get()); + assertSame(originalValue, observedSerializeContext.get().originalValue()); + assertEquals(context.entityId(), observedSerializeContext.get().entityId()); assertSame(context, observedDeserializeContext.get()); + assertNull(observedDeserializeContext.get().originalValue()); } @Test diff --git a/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java b/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java index 727b1d941..4fed8d194 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java @@ -7,6 +7,7 @@ import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -22,6 +23,7 @@ import java.util.Base64; import java.util.HexFormat; import java.util.Map; +import java.util.concurrent.atomic.AtomicReference; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; import software.amazon.awssdk.services.lambda.model.OperationType; @@ -77,7 +79,7 @@ void retryDecoratorComposesAsAFileSystemStage() { var stage = FileSystemSerDes.builder(basePath) .storageMode(FileSystemStorageMode.OVERFLOW) .build(); - var pipeline = new JacksonSerDes().then(new RetrySerDes(stage, RetryStrategies.Presets.NO_RETRY)); + var pipeline = new JacksonSerDes().then(new RetrySerDesStage(stage, RetryStrategies.Presets.NO_RETRY)); var runner = new SerDesRunner(null); var envelope = runner.serialize(pipeline, Map.of("id", 42), context()); @@ -223,22 +225,31 @@ void hashEncodingUsesFixedLengthSegments() throws Exception { @Test void includesBoundedPreviewWithoutChangingStoredPayload() throws Exception { + var previewValue = new AtomicReference(); + var previewContext = new AtomicReference(); var stage = FileSystemSerDes.builder(basePath) - .previewGenerator(value -> Map.of("summary", "order")) + .previewGenerator((value, context) -> { + previewValue.set(value); + previewContext.set(context); + return Map.of("summary", "order"); + }) .build(); var serDes = new JacksonSerDes().then(stage); var runner = new SerDesRunner(null); + var originalValue = Map.of("secret", "value"); - var envelope = runner.serialize(serDes, Map.of("secret", "value"), context()); + var envelope = runner.serialize(serDes, originalValue, context()); var json = MAPPER.readTree(envelope); assertEquals("order", json.get("preview").get("summary").textValue()); + assertEquals("{\"secret\":\"value\"}", previewValue.get()); + assertSame(originalValue, previewContext.get().originalValue()); assertEquals( "{\"secret\":\"value\"}", Files.readString(Path.of(json.get("file").textValue()))); var oversizedPreviewStage = FileSystemSerDes.builder(basePath) - .previewGenerator(value -> Map.of("summary", "x".repeat(256 * 1024))) + .previewGenerator((value, context) -> Map.of("summary", "x".repeat(256 * 1024))) .build(); var oversizedPreview = stringCodec().then(oversizedPreviewStage); var failure = assertThrows(SerDesException.class, () -> runner.serialize(oversizedPreview, "value", context())); diff --git a/sdk/src/test/java/software/amazon/lambda/durable/serde/RetryBinarySerDesStageTest.java b/sdk/src/test/java/software/amazon/lambda/durable/serde/RetryBinarySerDesStageTest.java new file mode 100644 index 000000000..1f6069ad7 --- /dev/null +++ b/sdk/src/test/java/software/amazon/lambda/durable/serde/RetryBinarySerDesStageTest.java @@ -0,0 +1,141 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.serde; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.time.Duration; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import org.junit.jupiter.api.Test; +import software.amazon.lambda.durable.exception.RetryableSerDesException; +import software.amazon.lambda.durable.exception.SerDesException; +import software.amazon.lambda.durable.retry.RetryDecision; +import software.amazon.lambda.durable.retry.RetryStrategies; + +class RetryBinarySerDesStageTest { + private static final Object ORIGINAL_VALUE = new Object(); + private static final SerDesContext CONTEXT = SerDesContext.forExecution( + "arn", "invocation", "execution", SerDesPayloadKind.RESULT) + .withOriginalValue(ORIGINAL_VALUE); + + @Test + void retriesSerializationAndPreservesContext() { + var calls = new AtomicInteger(); + var observedContext = new AtomicReference(); + var delegate = new BinarySerDesStage() { + @Override + public byte[] serialize(byte[] value, SerDesContext context) { + observedContext.set(context); + if (calls.incrementAndGet() == 1) { + throw new RetryableSerDesException("transient"); + } + return value; + } + + @Override + public byte[] deserialize(byte[] data, SerDesContext context) { + return data; + } + }; + var stage = new RetryBinarySerDesStage( + delegate, (error, attempt) -> RetryDecision.retry(Duration.ZERO), delay -> {}); + var value = new byte[] {1, 2, 3}; + + assertArrayEquals(value, stage.serialize(value, CONTEXT)); + assertSame(CONTEXT, observedContext.get()); + assertSame(ORIGINAL_VALUE, observedContext.get().originalValue()); + } + + @Test + void retriesDeserialization() { + var calls = new AtomicInteger(); + var delegate = new BinarySerDesStage() { + @Override + public byte[] serialize(byte[] value, SerDesContext context) { + return value; + } + + @Override + public byte[] deserialize(byte[] data, SerDesContext context) { + if (calls.incrementAndGet() == 1) { + throw new RetryableSerDesException("transient"); + } + return data; + } + }; + var stage = new RetryBinarySerDesStage( + delegate, (error, attempt) -> RetryDecision.retry(Duration.ZERO), delay -> {}); + var value = new byte[] {1, 2, 3}; + + assertArrayEquals(value, stage.deserialize(value, CONTEXT)); + } + + @Test + void doesNotRetryPermanentFailures() { + var calls = new AtomicInteger(); + var delegate = new BinarySerDesStage() { + @Override + public byte[] serialize(byte[] value, SerDesContext context) { + calls.incrementAndGet(); + throw new SerDesException("permanent"); + } + + @Override + public byte[] deserialize(byte[] data, SerDesContext context) { + return data; + } + }; + var stage = + new RetryBinarySerDesStage(delegate, (error, attempt) -> RetryDecision.retry(Duration.ZERO), delay -> { + throw new AssertionError("permanent failures must not sleep"); + }); + + assertThrows(SerDesException.class, () -> stage.serialize(new byte[0], CONTEXT)); + assertEquals(1, calls.get()); + } + + @Test + void validatesConfigurationAndRethrowsExhaustedFailure() { + var delegate = identityStage(); + assertThrows( + NullPointerException.class, () -> new RetryBinarySerDesStage(null, RetryStrategies.Presets.NO_RETRY)); + assertThrows(NullPointerException.class, () -> new RetryBinarySerDesStage(delegate, null)); + assertFalse(SerDesStage.class.isAssignableFrom(RetryBinarySerDesStage.class)); + + var retryable = new RetryableSerDesException("transient"); + var failingDelegate = new BinarySerDesStage() { + @Override + public byte[] serialize(byte[] value, SerDesContext context) { + throw retryable; + } + + @Override + public byte[] deserialize(byte[] data, SerDesContext context) { + return data; + } + }; + var stage = new RetryBinarySerDesStage(failingDelegate, RetryStrategies.Presets.NO_RETRY, delay -> {}); + + assertSame( + retryable, assertThrows(RetryableSerDesException.class, () -> stage.serialize(new byte[0], CONTEXT))); + } + + private static BinarySerDesStage identityStage() { + return new BinarySerDesStage() { + @Override + public byte[] serialize(byte[] value, SerDesContext context) { + return value; + } + + @Override + public byte[] deserialize(byte[] data, SerDesContext context) { + return data; + } + }; + } +} diff --git a/sdk/src/test/java/software/amazon/lambda/durable/serde/RetrySerDesTest.java b/sdk/src/test/java/software/amazon/lambda/durable/serde/RetrySerDesStageTest.java similarity index 84% rename from sdk/src/test/java/software/amazon/lambda/durable/serde/RetrySerDesTest.java rename to sdk/src/test/java/software/amazon/lambda/durable/serde/RetrySerDesStageTest.java index e98c69244..588a6be12 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/serde/RetrySerDesTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/serde/RetrySerDesStageTest.java @@ -19,9 +19,11 @@ import software.amazon.lambda.durable.retry.RetryDecision; import software.amazon.lambda.durable.retry.RetryStrategies; -class RetrySerDesTest { - private static final SerDesContext CONTEXT = - SerDesContext.forExecution("arn", "invocation", "execution", SerDesPayloadKind.RESULT); +class RetrySerDesStageTest { + private static final Object ORIGINAL_VALUE = new Object(); + private static final SerDesContext CONTEXT = SerDesContext.forExecution( + "arn", "invocation", "execution", SerDesPayloadKind.RESULT) + .withOriginalValue(ORIGINAL_VALUE); @Test void retriesSerializationWithStrategyDelays() { @@ -32,6 +34,7 @@ void retriesSerializationWithStrategyDelays() { @Override public String serialize(String value, SerDesContext context) { assertSame(CONTEXT, context); + assertSame(ORIGINAL_VALUE, context.originalValue()); if (calls.incrementAndGet() < 3) { throw new RetryableSerDesException("transient"); } @@ -43,7 +46,7 @@ public String deserialize(String data, SerDesContext context) { return data; } }; - var retrySerDes = new RetrySerDes( + var retrySerDes = new RetrySerDesStage( delegate, (error, attempt) -> { strategyAttempts.add(attempt); @@ -76,7 +79,7 @@ public String deserialize(String data, SerDesContext context) { } }; var retrySerDes = - new RetrySerDes(delegate, (error, attempt) -> RetryDecision.retry(Duration.ZERO), delay -> {}); + new RetrySerDesStage(delegate, (error, attempt) -> RetryDecision.retry(Duration.ZERO), delay -> {}); assertEquals("value", retrySerDes.deserialize("value", CONTEXT)); assertEquals(2, calls.get()); @@ -97,9 +100,10 @@ public String deserialize(String data, SerDesContext context) { return data; } }; - var retrySerDes = new RetrySerDes(delegate, (error, attempt) -> RetryDecision.retry(Duration.ZERO), delay -> { - throw new AssertionError("permanent failures must not sleep"); - }); + var retrySerDes = + new RetrySerDesStage(delegate, (error, attempt) -> RetryDecision.retry(Duration.ZERO), delay -> { + throw new AssertionError("permanent failures must not sleep"); + }); var failure = assertThrows(SerDesException.class, () -> retrySerDes.serialize("value", CONTEXT)); assertEquals("permanent", failure.getMessage()); @@ -110,11 +114,11 @@ public String deserialize(String data, SerDesContext context) { void rejectsInvalidConfigurationAndRetryDelay() { var delegate = identityStage(); - assertThrows(NullPointerException.class, () -> new RetrySerDes(null, RetryStrategies.Presets.NO_RETRY)); - assertThrows(NullPointerException.class, () -> new RetrySerDes(delegate, null)); - assertFalse(SerDes.class.isAssignableFrom(RetrySerDes.class)); + assertThrows(NullPointerException.class, () -> new RetrySerDesStage(null, RetryStrategies.Presets.NO_RETRY)); + assertThrows(NullPointerException.class, () -> new RetrySerDesStage(delegate, null)); + assertFalse(SerDes.class.isAssignableFrom(RetrySerDesStage.class)); - var retrySerDes = new RetrySerDes( + var retrySerDes = new RetrySerDesStage( new SerDesStage() { @Override public String serialize(String value, SerDesContext context) { @@ -150,7 +154,8 @@ public String deserialize(String data, SerDesContext context) { return data; } }; - var retrySerDes = new RetrySerDes(delegate, RetryStrategies.fixedDelay(2, Duration.ofSeconds(1)), delay -> {}); + var retrySerDes = + new RetrySerDesStage(delegate, RetryStrategies.fixedDelay(2, Duration.ofSeconds(1)), delay -> {}); var thrown = assertThrows(RetryableSerDesException.class, () -> retrySerDes.serialize("value", CONTEXT)); assertSame(lastFailure.get(), thrown); @@ -172,8 +177,8 @@ public String deserialize(String data, SerDesContext context) { return data; } }; - var retrySerDes = - new RetrySerDes(delegate, (error, attempt) -> RetryDecision.retry(Duration.ofSeconds(1)), delay -> { + var retrySerDes = new RetrySerDesStage( + delegate, (error, attempt) -> RetryDecision.retry(Duration.ofSeconds(1)), delay -> { throw new InterruptedException("stop"); }); diff --git a/sdk/src/test/java/software/amazon/lambda/durable/serde/SerDesRunnerTest.java b/sdk/src/test/java/software/amazon/lambda/durable/serde/SerDesRunnerTest.java index 35187aaa4..db53d4282 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/serde/SerDesRunnerTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/serde/SerDesRunnerTest.java @@ -4,7 +4,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertInstanceOf; -import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertNotSame; import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -36,25 +36,30 @@ void tearDown() { } @Test - void setsContextInsideExecutorAndClearsItAfterCall() throws Exception { + void passesContextToStagesInsideExecutor() { var observedContext = new AtomicReference(); var observedThread = new AtomicReference(); - var serDes = new JacksonSerDes() { + var stage = new SerDesStage() { @Override - public String serialize(Object value) { - observedContext.set(SerDesContext.getCurrentContext()); + public String serialize(String value, SerDesContext context) { + observedContext.set(context); observedThread.set(Thread.currentThread().getName()); - return super.serialize(value); + return value; + } + + @Override + public String deserialize(String data, SerDesContext context) { + return data; } }; var context = context("operation/1/result"); - new SerDesRunner(executor).serialize(serDes, "value", context); + new SerDesRunner(executor).serialize(new JacksonSerDes().then(stage), "value", context); - assertSame(context, observedContext.get()); + assertNotSame(context, observedContext.get()); + assertEquals(context.entityId(), observedContext.get().entityId()); + assertEquals("value", observedContext.get().originalValue()); assertEquals("test-serdes", observedThread.get()); - assertNull(executor.submit(SerDesContext::getCurrentContext).get()); - assertNull(SerDesContext.getCurrentContext()); } @Test @@ -74,40 +79,8 @@ public String serialize(Object value) { } @Test - void restoresPreviousContextAcrossNestedInlineCalls() { - var runner = new SerDesRunner(null); - var previous = context("previous"); - var outer = context("outer"); - var inner = context("inner"); - var duringOuter = new AtomicReference(); - var afterInner = new AtomicReference(); - var innerSerDes = new JacksonSerDes() { - @Override - public String serialize(Object value) { - assertSame(inner, SerDesContext.getCurrentContext()); - return super.serialize(value); - } - }; - var outerSerDes = new JacksonSerDes() { - @Override - public String serialize(Object value) { - duringOuter.set(SerDesContext.getCurrentContext()); - runner.serialize(innerSerDes, value, inner); - afterInner.set(SerDesContext.getCurrentContext()); - return super.serialize(value); - } - }; - - SerDesContextHolder.set(previous); - try { - runner.serialize(outerSerDes, "value", outer); - assertSame(previous, SerDesContext.getCurrentContext()); - } finally { - SerDesContextHolder.clear(); - } - - assertSame(outer, duringOuter.get()); - assertSame(outer, afterInner.get()); + void doesNotExposeThreadLocalContextAccessor() { + assertThrows(NoSuchMethodException.class, () -> SerDesContext.class.getMethod("getCurrentContext")); } @Test @@ -141,7 +114,8 @@ public T deserialize(String data, TypeToken typeToken) { context.parentId(), context.operationType(), context.operationSubType(), - 2); + 2, + null); assertEquals("one", runner.deserialize(serDes, "\"one\"", TypeToken.get(String.class), nextAttempt)); assertEquals(3, count.get()); @@ -349,6 +323,7 @@ private static SerDesContext context(String entityId) { null, OperationType.STEP, OperationSubType.STEP, - 1); + 1, + null); } } From d25801c2b91ec6fb1900d5b14ae9a28ee4b4343d Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Tue, 25 Aug 2026 22:24:47 +0000 Subject: [PATCH 40/56] refactor: return SerDes from pipeline composition --- docs/adr/005-filesystem-serdes.md | 4 ++-- .../amazon/lambda/durable/serde/ComposableSerDes.java | 2 +- .../java/software/amazon/lambda/durable/serde/SerDes.java | 4 ++-- .../amazon/lambda/durable/serde/ComposableSerDesTest.java | 8 +++++--- 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/docs/adr/005-filesystem-serdes.md b/docs/adr/005-filesystem-serdes.md index 56bb571c0..5d0ac5542 100644 --- a/docs/adr/005-filesystem-serdes.md +++ b/docs/adr/005-filesystem-serdes.md @@ -66,7 +66,7 @@ public interface SerDes { T deserialize(String data, TypeToken typeToken); - default ComposableSerDes then(SerDesStage nextStage) { + default SerDes then(SerDesStage nextStage) { return ComposableSerDes.builder(this).then(nextStage).build(); } } @@ -150,7 +150,7 @@ public final class ComposableSerDes implements SerDes { public SerDes getValueCodec(); - public ComposableSerDes then(SerDesStage stage); + public SerDes then(SerDesStage stage); public static final class Builder { public Builder then(SerDesStage stage); diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/ComposableSerDes.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/ComposableSerDes.java index 0a9bccd29..385ae8bd5 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/ComposableSerDes.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/ComposableSerDes.java @@ -65,7 +65,7 @@ public SerDes getValueCodec() { /** Returns a new pipeline with the supplied string stage appended. */ @Override - public ComposableSerDes then(SerDesStage stage) { + public SerDes then(SerDesStage stage) { var combined = new ArrayList<>(stages); combined.add(Objects.requireNonNull(stage, "stage cannot be null")); return new ComposableSerDes(valueCodec, combined); diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDes.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDes.java index 3cdd7b905..4d7c9f14b 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDes.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDes.java @@ -42,9 +42,9 @@ public interface SerDes { * Returns an immutable processing pipeline with a string stage appended. * * @param nextStage the reversible string stage to append - * @return a composable SerDes pipeline + * @return a SerDes backed by an immutable processing pipeline */ - default ComposableSerDes then(SerDesStage nextStage) { + default SerDes then(SerDesStage nextStage) { return ComposableSerDes.builder(this).then(nextStage).build(); } } diff --git a/sdk/src/test/java/software/amazon/lambda/durable/serde/ComposableSerDesTest.java b/sdk/src/test/java/software/amazon/lambda/durable/serde/ComposableSerDesTest.java index 2e170c6d0..7b0972580 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/serde/ComposableSerDesTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/serde/ComposableSerDesTest.java @@ -21,11 +21,13 @@ class ComposableSerDesTest { @Test - void onlyAcceptsStringStagesAfterTheValueCodec() throws Exception { + void exposesStringStageCompositionThroughTheSerDesInterface() throws Exception { assertThrows(NoSuchMethodException.class, () -> SerDes.class.getMethod("then", SerDes.class)); assertEquals( - ComposableSerDes.class, - SerDes.class.getMethod("then", SerDesStage.class).getReturnType()); + SerDes.class, SerDes.class.getMethod("then", SerDesStage.class).getReturnType()); + assertEquals( + SerDes.class, + ComposableSerDes.class.getMethod("then", SerDesStage.class).getReturnType()); } @Test From 9a4a3a165a13f31ffc327186be8b1f873a708977 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Tue, 25 Aug 2026 22:35:20 +0000 Subject: [PATCH 41/56] refactor: package filesystem SerDes stage --- docs/adr/005-filesystem-serdes.md | 52 +++++++------ docs/advanced/configuration.md | 11 +-- docs/advanced/filesystem-serdes.md | 15 ++-- docs/design.md | 18 +++-- ...FileSystemSerDesStageIntegrationTest.java} | 12 +-- .../testing/CloudDurableTestRunnerTest.java | 4 +- .../testing/LocalDurableTestRunnerTest.java | 10 +-- .../{ => filesystem}/FieldMatchMode.java | 2 +- .../FileSystemPathEncoding.java | 2 +- .../FileSystemSerDesStage.java} | 19 +++-- .../FileSystemStorageMode.java | 2 +- .../serde/{ => filesystem}/PreviewConfig.java | 2 +- .../serde/{ => filesystem}/PreviewField.java | 2 +- .../serde/{ => filesystem}/PreviewMode.java | 2 +- .../serde/{ => filesystem}/SerDesPreview.java | 4 +- .../operation/InvokeOperationTest.java | 5 +- .../SerializableDurableOperationTest.java | 5 +- .../FileSystemSerDesStageTest.java} | 77 +++++++++++-------- .../{ => filesystem}/SerDesPreviewTest.java | 2 +- 19 files changed, 138 insertions(+), 108 deletions(-) rename sdk-integration-tests/src/test/java/software/amazon/lambda/durable/{FileSystemSerDesIntegrationTest.java => FileSystemSerDesStageIntegrationTest.java} (98%) rename sdk/src/main/java/software/amazon/lambda/durable/serde/{ => filesystem}/FieldMatchMode.java (87%) rename sdk/src/main/java/software/amazon/lambda/durable/serde/{ => filesystem}/FileSystemPathEncoding.java (82%) rename sdk/src/main/java/software/amazon/lambda/durable/serde/{FileSystemSerDes.java => filesystem/FileSystemSerDesStage.java} (97%) rename sdk/src/main/java/software/amazon/lambda/durable/serde/{ => filesystem}/FileSystemStorageMode.java (81%) rename sdk/src/main/java/software/amazon/lambda/durable/serde/{ => filesystem}/PreviewConfig.java (98%) rename sdk/src/main/java/software/amazon/lambda/durable/serde/{ => filesystem}/PreviewField.java (95%) rename sdk/src/main/java/software/amazon/lambda/durable/serde/{ => filesystem}/PreviewMode.java (87%) rename sdk/src/main/java/software/amazon/lambda/durable/serde/{ => filesystem}/SerDesPreview.java (97%) rename sdk/src/test/java/software/amazon/lambda/durable/serde/{FileSystemSerDesTest.java => filesystem/FileSystemSerDesStageTest.java} (90%) rename sdk/src/test/java/software/amazon/lambda/durable/serde/{ => filesystem}/SerDesPreviewTest.java (99%) diff --git a/docs/adr/005-filesystem-serdes.md b/docs/adr/005-filesystem-serdes.md index 5d0ac5542..7d3f53352 100644 --- a/docs/adr/005-filesystem-serdes.md +++ b/docs/adr/005-filesystem-serdes.md @@ -2,7 +2,7 @@ **Status:** Accepted — Approach A with ComposableSerDes pipeline **Date:** 2026-07-02 -**Updated:** 2026-08-25 — Included FileSystemSerDes in core, made every post-codec pipeline component an explicit +**Updated:** 2026-08-25 — Included FileSystemSerDesStage in core, made every post-codec pipeline component an explicit string stage, and documented the rejected binary-only top-level pipeline. ## Context @@ -34,8 +34,9 @@ There are a few Java-specific constraints: Keep the existing `SerDes` serialization methods source- and binary-compatible, add a string-to-string `SerDesStage` contract and a core `ComposableSerDes` implementation which together form a processing pipeline, and implement -`FileSystemSerDes` in the core SDK. Every top-level stage consumes and produces a string, making stage composition -uniform and preventing intermediate type mismatches. +`FileSystemSerDesStage` in the core SDK's dedicated `software.amazon.lambda.durable.serde.filesystem` Java package. +Every top-level stage consumes and produces a string, making stage composition uniform and preventing intermediate +type mismatches. Binary transformations compose inside one `ComposableBinarySerDesStage`. That outer stage converts strings to bytes with a configurable starting codec, applies any number of reversible `BinarySerDesStage` implementations without @@ -77,7 +78,7 @@ transformation. This lets customers compose JSON encoding, framed string transfo filesystem storage without unsafe heterogeneous top-level stages. A `ComposableBinarySerDesStage` performs UTF-8, compression, encryption, and similar byte processing internally and encodes the result once at its string boundary. -`FileSystemSerDes` acts as a payload-storage stage. It writes the string produced by the previous stage to the +`FileSystemSerDesStage` acts as a payload-storage stage. It writes the string produced by the previous stage to the filesystem when configured to do so and returns a small checkpoint envelope. It is only a `SerDesStage`; a value codec must precede it in a `ComposableSerDes`, and other string stages may follow it. @@ -112,18 +113,18 @@ stage context is `null`. | Maven module directory | `sdk` | | Maven artifact ID | `aws-durable-execution-sdk-java` | | Maven group ID | `software.amazon.lambda.durable` | -| Java package | `software.amazon.lambda.durable.serde` | +| Java package | `software.amazon.lambda.durable.serde.filesystem` | | Dependency impact | No additional artifact or production dependency is required. | ### Configuration ```java -import software.amazon.lambda.durable.serde.FileSystemPathEncoding; -import software.amazon.lambda.durable.serde.FileSystemSerDes; -import software.amazon.lambda.durable.serde.FileSystemStorageMode; +import software.amazon.lambda.durable.serde.filesystem.FileSystemPathEncoding; +import software.amazon.lambda.durable.serde.filesystem.FileSystemSerDesStage; +import software.amazon.lambda.durable.serde.filesystem.FileSystemStorageMode; import software.amazon.lambda.durable.serde.JacksonSerDes; -var fileSystemStage = FileSystemSerDes.builder(Path.of("/mnt/efs/durable-payloads")) +var fileSystemStage = FileSystemSerDesStage.builder(Path.of("/mnt/efs/durable-payloads")) .storageMode(FileSystemStorageMode.ALWAYS) .pathEncoding(FileSystemPathEncoding.URI) .previewGenerator(optionalPreviewGenerator) @@ -371,7 +372,7 @@ Retry rules: checkpoint. Strategies must therefore use short, bounded delays that fit within the Lambda invocation timeout. - If the invocation is interrupted or times out, replay may execute the SerDes pipeline again. Any stage with side effects must use stable addressing and idempotent writes. -- `RetrySerDesStage` wraps an individual `SerDesStage`, such as `FileSystemSerDes`. +- `RetrySerDesStage` wraps an individual `SerDesStage`, such as `FileSystemSerDesStage`. `RetryBinarySerDesStage` wraps an individual binary stage inside a `ComposableBinarySerDesStage`. Neither can wrap the value codec or complete pipeline. Retrying only the transient component avoids repeating unrelated deterministic work. @@ -402,13 +403,13 @@ Envelope format: {"__durable_execution_filesystem_serdes":1,"file":"","payloadType":"STRING","ownerDurableExecutionArn":"","ownerEntityId":"","preview":{ "...": "..." }} ``` -`FileSystemSerDes` must reject recognized filesystem operations when its `SerDesContext` parameter is `null` or does +`FileSystemSerDesStage` must reject recognized filesystem operations when its `SerDesContext` parameter is `null` or does not include `durableExecutionArn` and `entityId`. It accepts a `String`, records the payload type in the envelope, and restores that string during reverse processing. A preceding `ComposableBinarySerDesStage` must encode its final bytes to a string before filesystem storage. The marker and version distinguish filesystem envelopes from strings that were not produced by this stage. -`FileSystemSerDes.deserialize(...)` returns input without the filesystem marker unchanged, regardless of payload +`FileSystemSerDesStage.deserialize(...)` returns input without the filesystem marker unchanged, regardless of payload source. If the marker is present, the value is recognized as filesystem data and must be a valid supported envelope; malformed marked envelopes and unsupported versions fail rather than falling back to pass-through behavior. The context parameter is explicit on every call, but a recognized filesystem envelope requires a non-null SDK-managed @@ -428,10 +429,10 @@ must be protected with the same care as the payload it references. The final file envelope, including any preview, must remain below the checkpoint threshold. Oversized previews are rejected rather than producing a checkpoint that the service cannot accept. -Stages may follow `FileSystemSerDes` to transform its inline or file-reference envelope. Its overflow and preview-size +Stages may follow `FileSystemSerDesStage` to transform its inline or file-reference envelope. Its overflow and preview-size checks apply at the filesystem stage boundary, so configurations must account for any size expansion introduced by later stages. During deserialization, each stage validates its own reserved format and passes unrecognized input -through unchanged. Therefore raw external data can traverse stages on either side of `FileSystemSerDes` without being +through unchanged. Therefore raw external data can traverse stages on either side of `FileSystemSerDesStage` without being decoded as a pipeline value. For JSON pipelines, `previewConfig(...)` parses the `String` produced by the preceding stage and provides the same @@ -452,7 +453,7 @@ Inside `SerDesRunner`, a composable pipeline receives `context` directly. The pi without context, derives a serialization stage context containing the original object, and forwards that derived context to every stage. Deserialization stages receive a context with no original value. -On deserialization, `FileSystemSerDes` first checks for its reserved marker. Unmarked input is returned unchanged. If +On deserialization, `FileSystemSerDesStage` first checks for its reserved marker. Unmarked input is returned unchanged. If the marked envelope contains `data`, it restores the inline text; if it contains `file`, it reads the stored string. `ComposableSerDes` then passes that value to the preceding string stage. Raw external input, callback results, and standard invoke results pass unchanged through every stage whose marker is absent until they reach the value codec. @@ -503,7 +504,7 @@ as an ordinary argument, so inline and executor-backed calls have the same stage Inline execution is the compatibility and low-overhead default, not a recommendation to perform blocking storage work on operation threads. Documentation and filesystem examples should configure a SerDes executor whenever -`FileSystemSerDes`, a delayed retry stage, or another blocking stage is used. If it is omitted, I/O and retry delays +`FileSystemSerDesStage`, a delayed retry stage, or another blocking stage is used. If it is omitted, I/O and retry delays block the calling thread. ### Caching @@ -525,7 +526,7 @@ so a large replay does not retain every materialized object. Cache entries live and are discarded when `ExecutionManager` closes. Cloud test polling creates a fresh cache for each history snapshot and retains that cache only with the corresponding `TestResult`. -With this approach, SDK caching can avoid repeated calls to `FileSystemSerDes.deserialize`. If a cache miss occurs, `FileSystemSerDes` may perform a file read internally. +With this approach, SDK caching can avoid repeated calls to `FileSystemSerDesStage.deserialize`. If a cache miss occurs, `FileSystemSerDesStage` may perform a file read internally. ### Exceptions @@ -538,11 +539,11 @@ Keep the current `ErrorObject` shape: When serializing `errorData`, set `SerDesPayloadKind.EXCEPTION` and use an entity ID distinct from the operation result. When deserializing, continue to load `Class.forName(errorType)` and call SerDes with `TypeToken.get(exceptionClass.asSubclass(Throwable.class))`. -`FileSystemSerDes` does not own exception type reconstruction. It only stores and loads the exception JSON or file pointer. Reconstruction remains in `SerializableDurableOperation.deserializeException` and `DurableExecutor.buildErrorObject`. +`FileSystemSerDesStage` does not own exception type reconstruction. It only stores and loads the exception JSON or file pointer. Reconstruction remains in `SerializableDurableOperation.deserializeException` and `DurableExecutor.buildErrorObject`. ### Input and output -Root user input and output payloads should route through `SerDesRunner` so `FileSystemSerDes` can see `SerDesContext`. The internal `DurableExecutionInput` and `DurableExecutionOutput` envelope stays with `DurableInputOutputSerDes`. +Root user input and output payloads should route through `SerDesRunner` so `FileSystemSerDesStage` can see `SerDesContext`. The internal `DurableExecutionInput` and `DurableExecutionOutput` envelope stays with `DurableInputOutputSerDes`. The cloud test runner must send initial Lambda input before it receives a durable execution ARN. The cloud and local runners therefore always serialize that input with a separate context-free value codec. By default, they use the @@ -574,7 +575,8 @@ persisted pipeline. 8. Add bounded, invocation-scoped deserialization caching keyed by SerDes identity, entity, payload kind, type, and serialized data hash. 9. Update exception serialization and deserialization paths to set `SerDesPayloadKind.EXCEPTION` in TLS. -10. Implement `FileSystemSerDes` in the core `software.amazon.lambda.durable.serde` package as a string stage with +10. Implement `FileSystemSerDesStage` in the core `software.amazon.lambda.durable.serde.filesystem` package as a string + stage with `ALWAYS` and `OVERFLOW` storage, `URI` and `HASH` path encodings, envelope parsing, immutable `CREATE_NEW` writes compatible with EFS and S3 Files, retryable I/O failures, unrecognized-input pass-through, structured preview generation, and clear validation errors for recognized malformed input. @@ -790,7 +792,7 @@ This approach gives the SDK one consistent policy for root payloads, operation r - Requires a new core SDK extension point and configuration model. - Needs careful interaction rules with operation-level SerDes, payload SerDes, callback deserializers, test helpers, and error serialization. - Requires a migration story for existing custom SerDes implementations that already return external references. -- Slows direct FileSystemSerDes parity while the broader offloading API is designed and stabilized. +- Slows direct FileSystemSerDesStage parity while the broader offloading API is designed and stabilized. - Diverges from the JavaScript `createFileSystemSerdes` naming and shape, even if the behavior is similar. - Adds more core SDK responsibility because the runtime now owns the offload envelope. @@ -803,8 +805,8 @@ This approach gives the SDK one consistent policy for root payloads, operation r | Parity with JS issue | Closest to the current JavaScript `createFileSystemSerdes` model and issue #463 wording. | Diverges from JavaScript naming and shape, though it may be architecturally cleaner for Java. | | Core SDK changes | Adds the compatible `SerDes.then(SerDesStage)` default method, `ComposableSerDes`, and explicit stage context parameters while root codecs remain context-free. | Requires a new core extension point, envelope type, config surface, payload pipeline, and migration story. | | Applicability | Any compatible `SerDesStage` implementations can be chained, but storage stages still own their envelope and lifecycle behavior. | Any SerDes output can be offloaded uniformly after serialization. Users can combine Jackson/custom SerDes with any offloader. | -| Envelope ownership | FileSystemSerDes owns the checkpoint envelope (`data`, `file`, preview), so the SDK treats it as opaque serialized data. | SDK owns the checkpoint/offload envelope and must guarantee it composes with replay, errors, callbacks, and test utilities. | -| Caching | SDK can cache deserialized values, but FileSystemSerDes may still do file reads internally unless cache hits happen before SerDes. | SDK can cache at both layers: resolved offloaded payload text and final deserialized object. | +| Envelope ownership | FileSystemSerDesStage owns the checkpoint envelope (`data`, `file`, preview), so the SDK treats it as opaque serialized data. | SDK owns the checkpoint/offload envelope and must guarantee it composes with replay, errors, callbacks, and test utilities. | +| Caching | SDK can cache deserialized values, but FileSystemSerDesStage may still do file reads internally unless cache hits happen before SerDes. | SDK can cache at both layers: resolved offloaded payload text and final deserialized object. | | Exception handling | Works if every exception serialization path is routed through SerDes with `SerDesPayloadKind.EXCEPTION`. | Works uniformly because exception `errorData` is another serialized payload that can be offloaded after SerDes. | | Third-party storage | S3, DynamoDB, and other backends can be implemented as additional reversible SerDes stages, either in core or separate artifacts based on their dependencies and support model. | Natural home for multiple storage backends: filesystem, S3, DynamoDB, EFS, S3 Files, or custom customer storage. | | Immediate delivery risk | Lower. Builds on existing customization point. | Higher. Requires new API and more runtime integration. | @@ -873,7 +875,7 @@ Revisit a binary-root pipeline in a future major version if profiling shows that binary-heavy pipelines is a material bottleneck and the benefit outweighs the default-path allocation cost and migration burden. -### Add FileSystemSerDes without SerDesContext +### Add FileSystemSerDesStage without SerDesContext Rejected. A filesystem-backed implementation needs stable operation identity. Without context, it cannot choose a safe file name, distinguish result and exception payloads for the same operation, or avoid collisions across durable executions. @@ -986,7 +988,7 @@ substantial provider-specific dependencies or has an independent support and rel | Feature | Artifact ID | Java package | |---------|-------------|--------------| -| Filesystem payload storage, Approach A | `aws-durable-execution-sdk-java` | `software.amazon.lambda.durable.serde` | +| Filesystem payload storage, Approach A | `aws-durable-execution-sdk-java` | `software.amazon.lambda.durable.serde.filesystem` | | Filesystem payload storage, Approach B | `aws-durable-execution-sdk-java-extra-filesystem-offloader` | `software.amazon.lambda.durable.extra.filesystem` | | Event deserialization helpers | `aws-durable-execution-sdk-java-extra-event-deserialization` | `software.amazon.lambda.durable.extra.eventdeserialization` | | Virtual thread executor helpers | `aws-durable-execution-sdk-java-extra-virtual-thread-pool` | `software.amazon.lambda.durable.extra.virtualthreads` | diff --git a/docs/advanced/configuration.md b/docs/advanced/configuration.md index 0f3726c17..b7d8b0785 100644 --- a/docs/advanced/configuration.md +++ b/docs/advanced/configuration.md @@ -55,10 +55,11 @@ deserialization results during the current invocation. ### Filesystem-backed payload storage -The core SDK provides a reversible stage for storing serialized strings on a shared filesystem: +The core SDK provides a reversible stage for storing serialized strings on a shared filesystem. Its stage, storage, +path, and preview APIs live in `software.amazon.lambda.durable.serde.filesystem`: ```java -var fileSystemStage = FileSystemSerDes.builder(Path.of("/mnt/efs/durable-payloads")) +var fileSystemStage = FileSystemSerDesStage.builder(Path.of("/mnt/efs/durable-payloads")) .storageMode(FileSystemStorageMode.OVERFLOW) .pathEncoding(FileSystemPathEncoding.HASH) .build(); @@ -105,7 +106,7 @@ approaches the 256 KB service limit. `URI` produces readable escaped paths; `HAS segments. Files include a content hash and never overwrite data referenced by an earlier checkpoint. References are validated against the current durable execution and entity, and symbolic-link paths are rejected. -For a `JacksonSerDes -> FileSystemSerDes` pipeline, structured preview configuration provides the same field +For a `JacksonSerDes -> FileSystemSerDesStage` pipeline, structured preview configuration provides the same field selection, masking, exact-path matching, and default 4 KB preview budget as the Python and TypeScript SDKs: ```java @@ -114,7 +115,7 @@ var previewConfig = PreviewConfig.builder(PreviewMode.EXCLUDE_ALL) .mask(PreviewField.anywhere("email")) .build(); -var fileSystemStage = FileSystemSerDes.builder(Path.of("/mnt/s3/durable-payloads")) +var fileSystemStage = FileSystemSerDesStage.builder(Path.of("/mnt/s3/durable-payloads")) .previewConfig(previewConfig) .build(); @@ -145,7 +146,7 @@ codec rather than a `ComposableSerDes`. When the runtime later deserializes that it through unless its self-identifying format is present. A custom input codec must produce data that the persisted pipeline's root value codec can decode. -Stages may follow `FileSystemSerDes` to transform its inline or file-reference envelope. `OVERFLOW` and preview-size +Stages may follow `FileSystemSerDesStage` to transform its inline or file-reference envelope. `OVERFLOW` and preview-size checks apply to the filesystem stage's output; account for any expansion introduced by later stages when staying within the service checkpoint limit. On deserialization, every stage passes input through unchanged when its own self-identifying format is absent, so raw external payloads can safely traverse stages on either side of the diff --git a/docs/advanced/filesystem-serdes.md b/docs/advanced/filesystem-serdes.md index cb447f4ca..5abb855af 100644 --- a/docs/advanced/filesystem-serdes.md +++ b/docs/advanced/filesystem-serdes.md @@ -1,7 +1,8 @@ # Filesystem SerDes -`FileSystemSerDes` stores durable user payloads on a shared filesystem while keeping small, versioned file-reference -envelopes in checkpoints. It is included in the core `aws-durable-execution-sdk-java` artifact. +`FileSystemSerDesStage` stores durable user payloads on a shared filesystem while keeping small, versioned file-reference +envelopes in checkpoints. It is included in the core `aws-durable-execution-sdk-java` artifact under the +`software.amazon.lambda.durable.serde.filesystem` Java package. ## Installation @@ -15,10 +16,10 @@ envelopes in checkpoints. It is included in the core `aws-durable-execution-sdk- ## Pipeline configuration -`FileSystemSerDes` is a reversible `SerDesStage` that must be configured after a value codec: +`FileSystemSerDesStage` is a reversible `SerDesStage` that must be configured after a value codec: ```java -var fileSystemStage = FileSystemSerDes.builder(Path.of("/mnt/efs/durable-payloads")) +var fileSystemStage = FileSystemSerDesStage.builder(Path.of("/mnt/efs/durable-payloads")) .storageMode(FileSystemStorageMode.ALWAYS) .pathEncoding(FileSystemPathEncoding.URI) .build(); @@ -75,7 +76,7 @@ var previewConfig = PreviewConfig.builder(PreviewMode.EXCLUDE_ALL) .maxPreviewBytes(4096) .build(); -var fileSystemStage = FileSystemSerDes.builder(Path.of("/mnt/s3/durable-payloads")) +var fileSystemStage = FileSystemSerDesStage.builder(Path.of("/mnt/s3/durable-payloads")) .previewConfig(previewConfig) .build(); @@ -107,7 +108,7 @@ attempts and delays bounded. ## Replay and envelope behavior -Filesystem envelopes include a reserved version marker. `FileSystemSerDes` returns input without that marker unchanged, +Filesystem envelopes include a reserved version marker. `FileSystemSerDesStage` returns input without that marker unchanged, allowing raw root input, callback results, and standard Lambda invoke results to continue through the remaining stages to the pipeline value codec. Payloads containing the reserved marker must be valid supported filesystem envelopes; malformed marked envelopes and unsupported versions fail instead of falling back to pass-through behavior. An @@ -123,7 +124,7 @@ result boundaries may consume a file owned by the other Lambda execution when bo and path encoding. Treat file envelopes as capabilities. Content hashes are verified when reading, and symbolic-link paths are rejected. -Stages may follow `FileSystemSerDes` to transform its inline or file-reference envelope. The filesystem stage's +Stages may follow `FileSystemSerDesStage` to transform its inline or file-reference envelope. The filesystem stage's `OVERFLOW` and preview-size checks apply before those later transformations, so account for any expansion when staying within the service checkpoint limit. During deserialization, every stage checks its own marker and returns unrecognized input unchanged, so raw external payloads can traverse later and earlier stages without being decoded by diff --git a/docs/design.md b/docs/design.md index f6cf311c6..ca34cf437 100644 --- a/docs/design.md +++ b/docs/design.md @@ -18,7 +18,7 @@ aws-durable-execution-sdk-java/ | Module | Purpose | Key Classes | |--------|---------|-------------| -| `sdk` | Core runtime - extend `DurableHandler`, use `DurableContext` for durable operations, and configure composable or filesystem-backed SerDes | `DurableHandler`, `DurableContext`, `DurableExecutor`, `ExecutionManager`, `FileSystemSerDes` | +| `sdk` | Core runtime - extend `DurableHandler`, use `DurableContext` for durable operations, and configure composable or filesystem-backed SerDes | `DurableHandler`, `DurableContext`, `DurableExecutor`, `ExecutionManager`, `FileSystemSerDesStage` | | `sdk-testing` | Test utilities: `LocalDurableTestRunner` (in-memory, simulates re-invocations and time-skipping) and `CloudDurableTestRunner` (executes against deployed Lambda) | `LocalDurableTestRunner`, `CloudDurableTestRunner`, `LocalMemoryExecutionClient`, `TestResult` | | `sdk-integration-tests` | Dogfooding tests - validates the SDK using its own test utilities. Separate module keeps dependencies acyclic: `sdk` → `sdk-testing` → `sdk-integration-tests`. | Test classes only | | `examples` | Real-world usage patterns as customers would implement them, with local and cloud tests | Example handlers, `CloudBasedIntegrationTest` | @@ -365,15 +365,19 @@ software.amazon.lambda.durable │ ├── JacksonSerDes # Jackson impl │ ├── RetrySerDesStage # Retrying string-stage decorator │ ├── RetryBinarySerDesStage # Retrying binary-stage decorator -│ ├── SerDesPreview # Structured preview builder -│ ├── PreviewConfig # Preview selection, masking, and size configuration -│ ├── PreviewField # Field-name or exact-path preview selector -│ ├── PreviewMode # Include-all or exclude-all preview default -│ ├── FieldMatchMode # Anywhere or exact-path field matching │ ├── SerDesRunner # Stage context, optional executor dispatch, and invocation cache │ ├── SerDesContext # Read-only durable payload identity and serialization source value │ ├── SerDesPayloadKind # Input/result/state/exception/invoke payload kind -│ └── AwsSdkV2Module # SDK type support +│ ├── AwsSdkV2Module # SDK type support +│ └── filesystem/ +│ ├── FileSystemSerDesStage # Filesystem-backed string stage +│ ├── FileSystemStorageMode # Always or overflow-only storage +│ ├── FileSystemPathEncoding # URI or hashed path encoding +│ ├── SerDesPreview # Structured preview builder +│ ├── PreviewConfig # Preview selection, masking, and size configuration +│ ├── PreviewField # Field-name or exact-path preview selector +│ ├── PreviewMode # Include-all or exclude-all preview default +│ └── FieldMatchMode # Anywhere or exact-path field matching │ └── exception/ ├── DurableExecutionException diff --git a/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FileSystemSerDesIntegrationTest.java b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FileSystemSerDesStageIntegrationTest.java similarity index 98% rename from sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FileSystemSerDesIntegrationTest.java rename to sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FileSystemSerDesStageIntegrationTest.java index 53843a4d2..ba1a48eef 100644 --- a/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FileSystemSerDesIntegrationTest.java +++ b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FileSystemSerDesStageIntegrationTest.java @@ -38,7 +38,6 @@ import software.amazon.lambda.durable.retry.WaitStrategies; import software.amazon.lambda.durable.serde.Base64StringBinaryCodec; import software.amazon.lambda.durable.serde.ComposableBinarySerDesStage; -import software.amazon.lambda.durable.serde.FileSystemSerDes; import software.amazon.lambda.durable.serde.JacksonSerDes; import software.amazon.lambda.durable.serde.SerDes; import software.amazon.lambda.durable.serde.SerDesContext; @@ -46,12 +45,13 @@ import software.amazon.lambda.durable.serde.SerDesRunner; import software.amazon.lambda.durable.serde.SerDesStage; import software.amazon.lambda.durable.serde.Utf8StringBinaryCodec; +import software.amazon.lambda.durable.serde.filesystem.FileSystemSerDesStage; import software.amazon.lambda.durable.testing.LocalDurableTestRunner; import software.amazon.lambda.durable.testing.TestResult; import software.amazon.lambda.durable.testing.local.LocalMemoryExecutionClient; import software.amazon.lambda.durable.testing.local.OperationResult; -class FileSystemSerDesIntegrationTest { +class FileSystemSerDesStageIntegrationTest { private static final ObjectMapper MAPPER = new ObjectMapper(); @TempDir @@ -162,7 +162,7 @@ void acceptsRawCallbackAndInvokeResultsAndOffloadsInvokePayload() throws Excepti }); var serDes = new JacksonSerDes() .then(recordingStage) - .then(FileSystemSerDes.builder(basePath).build()); + .then(FileSystemSerDesStage.builder(basePath).build()); var config = DurableConfig.builder().withSerDes(serDes).build(); var runner = LocalDurableTestRunner.create( String.class, @@ -278,7 +278,7 @@ void repeatedGetUsesInvocationCacheForTheCompletePipeline() { }); var serDes = new JacksonSerDes() .then(countingStage) - .then(FileSystemSerDes.builder(basePath).build()); + .then(FileSystemSerDesStage.builder(basePath).build()); var config = DurableConfig.builder().withSerDes(serDes).build(); var runner = LocalDurableTestRunner.create( String.class, @@ -313,7 +313,7 @@ void successfulRetryUsesTheProducingAttemptForResultSerialization() { }); var serDes = new JacksonSerDes() .then(attemptStage) - .then(FileSystemSerDes.builder(basePath).build()); + .then(FileSystemSerDesStage.builder(basePath).build()); var config = DurableConfig.builder().withSerDes(serDes).build(); var stepConfig = StepConfig.builder() .retryStrategy(RetryStrategies.fixedDelay(2, Duration.ofSeconds(1))) @@ -467,7 +467,7 @@ private SerDes filesystemPipeline() { .build(); return new JacksonSerDes() .then(binaryStage) - .then(FileSystemSerDes.builder(basePath).build()); + .then(FileSystemSerDesStage.builder(basePath).build()); } private static DurableExecutionInput durableInput( diff --git a/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/CloudDurableTestRunnerTest.java b/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/CloudDurableTestRunnerTest.java index 586f519c9..a3d418a9d 100644 --- a/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/CloudDurableTestRunnerTest.java +++ b/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/CloudDurableTestRunnerTest.java @@ -16,13 +16,13 @@ import software.amazon.awssdk.services.lambda.model.InvokeResponse; import software.amazon.lambda.durable.TypeToken; import software.amazon.lambda.durable.exception.SerDesException; -import software.amazon.lambda.durable.serde.FileSystemSerDes; import software.amazon.lambda.durable.serde.JacksonSerDes; import software.amazon.lambda.durable.serde.SerDes; import software.amazon.lambda.durable.serde.SerDesContext; import software.amazon.lambda.durable.serde.SerDesPayloadKind; import software.amazon.lambda.durable.serde.SerDesRunner; import software.amazon.lambda.durable.serde.SerDesStage; +import software.amazon.lambda.durable.serde.filesystem.FileSystemSerDesStage; class CloudDurableTestRunnerTest { @@ -122,7 +122,7 @@ void valueCodecInputRoundTripsThroughFileSystemPipeline(@TempDir Path basePath) .durableExecutionArn(executionArn) .build()); var persistedSerDes = - new JacksonSerDes().then(FileSystemSerDes.builder(basePath).build()); + new JacksonSerDes().then(FileSystemSerDesStage.builder(basePath).build()); var runner = CloudDurableTestRunner.create( "arn:aws:lambda:us-east-2:123:function:test", String.class, String.class, mockClient) .withSerDes(persistedSerDes); diff --git a/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/LocalDurableTestRunnerTest.java b/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/LocalDurableTestRunnerTest.java index 5eaaaf5f9..d9edc741d 100644 --- a/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/LocalDurableTestRunnerTest.java +++ b/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/LocalDurableTestRunnerTest.java @@ -23,12 +23,12 @@ import software.amazon.lambda.durable.serde.Base64StringBinaryCodec; import software.amazon.lambda.durable.serde.BinarySerDesStage; import software.amazon.lambda.durable.serde.ComposableBinarySerDesStage; -import software.amazon.lambda.durable.serde.FileSystemSerDes; import software.amazon.lambda.durable.serde.JacksonSerDes; import software.amazon.lambda.durable.serde.SerDes; import software.amazon.lambda.durable.serde.SerDesContext; import software.amazon.lambda.durable.serde.SerDesStage; import software.amazon.lambda.durable.serde.Utf8StringBinaryCodec; +import software.amazon.lambda.durable.serde.filesystem.FileSystemSerDesStage; class LocalDurableTestRunnerTest { @@ -155,7 +155,7 @@ void checkpointedLargeOutputReplaysWithoutDuplicateExecutionOperation() { void filesystemPersistedSerDesUsesDefaultJacksonInputCodec(@TempDir Path basePath) { var config = DurableConfig.builder() .withSerDes(new JacksonSerDes() - .then(FileSystemSerDes.builder(basePath).build())) + .then(FileSystemSerDesStage.builder(basePath).build())) .build(); var runner = LocalDurableTestRunner.create(String.class, (input, context) -> input, config) .withOutputType(String.class); @@ -182,7 +182,7 @@ void plainPersistedSerDesIsUsedAsDefaultInputCodec() { void rejectsComposableInputSerDes(@TempDir Path basePath) { var config = DurableConfig.builder() .withSerDes(new JacksonSerDes() - .then(FileSystemSerDes.builder(basePath).build())) + .then(FileSystemSerDesStage.builder(basePath).build())) .build(); var runner = LocalDurableTestRunner.create(String.class, (input, context) -> input, config); @@ -194,11 +194,11 @@ void rejectsComposableInputSerDes(@TempDir Path basePath) { } @Test - void rawInputPassesThroughPersistedStagesBeforeFileSystemSerDes(@TempDir Path basePath) { + void rawInputPassesThroughPersistedStagesBeforeFileSystemSerDesStage(@TempDir Path basePath) { var deserializeCalls = new AtomicInteger(); var persistedSerDes = new JacksonSerDes() .then(bytesStage(deserializeCalls)) - .then(FileSystemSerDes.builder(basePath).build()); + .then(FileSystemSerDesStage.builder(basePath).build()); var config = DurableConfig.builder().withSerDes(persistedSerDes).build(); var runner = LocalDurableTestRunner.create( String.class, (input, context) -> input + ":" + deserializeCalls.get(), config) diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/FieldMatchMode.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/filesystem/FieldMatchMode.java similarity index 87% rename from sdk/src/main/java/software/amazon/lambda/durable/serde/FieldMatchMode.java rename to sdk/src/main/java/software/amazon/lambda/durable/serde/filesystem/FieldMatchMode.java index 36cf1e726..ec07cf52c 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/FieldMatchMode.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/filesystem/FieldMatchMode.java @@ -1,6 +1,6 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -package software.amazon.lambda.durable.serde; +package software.amazon.lambda.durable.serde.filesystem; /** Controls how a {@link PreviewField} matches a field in a structured value. */ public enum FieldMatchMode { diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemPathEncoding.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/filesystem/FileSystemPathEncoding.java similarity index 82% rename from sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemPathEncoding.java rename to sdk/src/main/java/software/amazon/lambda/durable/serde/filesystem/FileSystemPathEncoding.java index 76d53032c..4e4c4e41b 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemPathEncoding.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/filesystem/FileSystemPathEncoding.java @@ -1,6 +1,6 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -package software.amazon.lambda.durable.serde; +package software.amazon.lambda.durable.serde.filesystem; /** Controls how durable execution and entity identifiers are encoded as filesystem paths. */ public enum FileSystemPathEncoding { diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/filesystem/FileSystemSerDesStage.java similarity index 97% rename from sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java rename to sdk/src/main/java/software/amazon/lambda/durable/serde/filesystem/FileSystemSerDesStage.java index ce267c7b8..ef4124e03 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemSerDes.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/filesystem/FileSystemSerDesStage.java @@ -1,6 +1,6 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -package software.amazon.lambda.durable.serde; +package software.amazon.lambda.durable.serde.filesystem; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; @@ -24,6 +24,11 @@ import software.amazon.awssdk.services.lambda.model.OperationType; import software.amazon.lambda.durable.exception.RetryableSerDesException; import software.amazon.lambda.durable.exception.SerDesException; +import software.amazon.lambda.durable.serde.ComposableBinarySerDesStage; +import software.amazon.lambda.durable.serde.SerDesContext; +import software.amazon.lambda.durable.serde.SerDesPayloadKind; +import software.amazon.lambda.durable.serde.SerDesStage; +import software.amazon.lambda.durable.serde.Utf8StringBinaryCodec; /** * A string stage that stores payloads on a durable shared filesystem. @@ -38,7 +43,7 @@ * unchanged; input with the marker must be a valid supported envelope. Filesystem operations use the explicit * {@link SerDesContext} stage parameter for durable payload identity. */ -public final class FileSystemSerDes implements SerDesStage { +public final class FileSystemSerDesStage implements SerDesStage { private static final String ENVELOPE_MARKER = "__durable_execution_filesystem_serdes"; private static final String ENVELOPE_PREFIX = "{\"" + ENVELOPE_MARKER + "\":"; private static final String PAYLOAD_TYPE_FIELD = "payloadType"; @@ -54,7 +59,7 @@ public final class FileSystemSerDes implements SerDesStage { private final BiFunction> previewGenerator; private volatile Path canonicalBasePath; - private FileSystemSerDes(Builder builder) { + private FileSystemSerDesStage(Builder builder) { basePath = builder.basePath.toAbsolutePath().normalize(); storageMode = builder.storageMode; pathEncoding = builder.pathEncoding; @@ -350,7 +355,7 @@ private SerDesContext requireContext(SerDesContext context) { || context.entityId() == null || context.entityId().isBlank()) { throw new SerDesException( - "FileSystemSerDes requires an SDK-managed SerDesContext with durableExecutionArn and entityId"); + "FileSystemSerDesStage requires an SDK-managed SerDesContext with durableExecutionArn and entityId"); } return context; } @@ -560,7 +565,7 @@ private String value() { } } - /** Builder for {@link FileSystemSerDes}. */ + /** Builder for {@link FileSystemSerDesStage}. */ public static final class Builder { private final Path basePath; private FileSystemStorageMode storageMode = FileSystemStorageMode.ALWAYS; @@ -605,8 +610,8 @@ public Builder previewConfig(PreviewConfig previewConfig) { return this; } - public FileSystemSerDes build() { - return new FileSystemSerDes(this); + public FileSystemSerDesStage build() { + return new FileSystemSerDesStage(this); } } } diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemStorageMode.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/filesystem/FileSystemStorageMode.java similarity index 81% rename from sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemStorageMode.java rename to sdk/src/main/java/software/amazon/lambda/durable/serde/filesystem/FileSystemStorageMode.java index 02ca68b98..c5df70af9 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/FileSystemStorageMode.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/filesystem/FileSystemStorageMode.java @@ -1,6 +1,6 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -package software.amazon.lambda.durable.serde; +package software.amazon.lambda.durable.serde.filesystem; /** Controls when serialized payloads are written to the filesystem. */ public enum FileSystemStorageMode { diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/PreviewConfig.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/filesystem/PreviewConfig.java similarity index 98% rename from sdk/src/main/java/software/amazon/lambda/durable/serde/PreviewConfig.java rename to sdk/src/main/java/software/amazon/lambda/durable/serde/filesystem/PreviewConfig.java index 7d2f8ebf4..7f546a6c4 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/PreviewConfig.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/filesystem/PreviewConfig.java @@ -1,6 +1,6 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -package software.amazon.lambda.durable.serde; +package software.amazon.lambda.durable.serde.filesystem; import java.util.ArrayList; import java.util.Arrays; diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/PreviewField.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/filesystem/PreviewField.java similarity index 95% rename from sdk/src/main/java/software/amazon/lambda/durable/serde/PreviewField.java rename to sdk/src/main/java/software/amazon/lambda/durable/serde/filesystem/PreviewField.java index 49bd84304..171e055aa 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/PreviewField.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/filesystem/PreviewField.java @@ -1,6 +1,6 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -package software.amazon.lambda.durable.serde; +package software.amazon.lambda.durable.serde.filesystem; import java.util.Objects; diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/PreviewMode.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/filesystem/PreviewMode.java similarity index 87% rename from sdk/src/main/java/software/amazon/lambda/durable/serde/PreviewMode.java rename to sdk/src/main/java/software/amazon/lambda/durable/serde/filesystem/PreviewMode.java index 8dd0f4851..e855f7d4a 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/PreviewMode.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/filesystem/PreviewMode.java @@ -1,6 +1,6 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -package software.amazon.lambda.durable.serde; +package software.amazon.lambda.durable.serde.filesystem; /** Controls which fields are visible by default in a structured payload preview. */ public enum PreviewMode { diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesPreview.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/filesystem/SerDesPreview.java similarity index 97% rename from sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesPreview.java rename to sdk/src/main/java/software/amazon/lambda/durable/serde/filesystem/SerDesPreview.java index 9bcd64ca5..68259feee 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/SerDesPreview.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/filesystem/SerDesPreview.java @@ -1,6 +1,6 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -package software.amazon.lambda.durable.serde; +package software.amazon.lambda.durable.serde.filesystem; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; @@ -42,7 +42,7 @@ public static Map buildPreview(Object value, PreviewConfig confi /** * Builds a preview from a JSON string. * - *

This is used by {@link FileSystemSerDes.Builder#previewConfig(PreviewConfig)}, because a pipeline stage + *

This is used by {@link FileSystemSerDesStage.Builder#previewConfig(PreviewConfig)}, because a pipeline stage * receives the serialized string produced by the preceding stage. * * @return a nested preview map, or {@code null} when no fields are visible diff --git a/sdk/src/test/java/software/amazon/lambda/durable/operation/InvokeOperationTest.java b/sdk/src/test/java/software/amazon/lambda/durable/operation/InvokeOperationTest.java index 3bdeca694..14dd8e71c 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/operation/InvokeOperationTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/operation/InvokeOperationTest.java @@ -32,12 +32,12 @@ import software.amazon.lambda.durable.execution.ThreadType; import software.amazon.lambda.durable.model.OperationIdentifier; import software.amazon.lambda.durable.model.OperationSubType; -import software.amazon.lambda.durable.serde.FileSystemSerDes; import software.amazon.lambda.durable.serde.JacksonSerDes; import software.amazon.lambda.durable.serde.SerDes; import software.amazon.lambda.durable.serde.SerDesContext; import software.amazon.lambda.durable.serde.SerDesPayloadKind; import software.amazon.lambda.durable.serde.SerDesRunner; +import software.amazon.lambda.durable.serde.filesystem.FileSystemSerDesStage; class InvokeOperationTest { private static final String OPERATION_ID = "2"; @@ -214,7 +214,8 @@ void nestedTerminalInvokeRebindsFilesystemErrorForReplay(OperationStatus status) var calleeArn = "arn:aws:lambda:us-east-1:123456789012:function:callee/durable-execution/callee/invocation"; when(executionManager.getDurableExecutionArn()).thenReturn(callerArn); - var serDes = new JacksonSerDes().then(FileSystemSerDes.builder(basePath).build()); + var serDes = + new JacksonSerDes().then(FileSystemSerDesStage.builder(basePath).build()); var original = new IllegalStateException("callee failed"); var errorData = new SerDesRunner(null) .serialize( diff --git a/sdk/src/test/java/software/amazon/lambda/durable/operation/SerializableDurableOperationTest.java b/sdk/src/test/java/software/amazon/lambda/durable/operation/SerializableDurableOperationTest.java index a2008d842..af0c921d2 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/operation/SerializableDurableOperationTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/operation/SerializableDurableOperationTest.java @@ -49,9 +49,9 @@ import software.amazon.lambda.durable.execution.ThreadType; import software.amazon.lambda.durable.model.OperationIdentifier; import software.amazon.lambda.durable.model.OperationSubType; -import software.amazon.lambda.durable.serde.FileSystemSerDes; import software.amazon.lambda.durable.serde.JacksonSerDes; import software.amazon.lambda.durable.serde.SerDes; +import software.amazon.lambda.durable.serde.filesystem.FileSystemSerDesStage; class SerializableDurableOperationTest { @@ -543,7 +543,8 @@ void deserializeExceptionPreservesRetryableStorageFailure() { when(executionManager.getDurableExecutionArn()) .thenReturn( "arn:aws:lambda:us-east-1:123456789012:function:test:1/durable-execution/execution/invocation"); - var serDes = new JacksonSerDes().then(FileSystemSerDes.builder(basePath).build()); + var serDes = + new JacksonSerDes().then(FileSystemSerDesStage.builder(basePath).build()); var checkpointedError = new AtomicReference(); SerializableDurableOperation producer = new SerializableDurableOperation<>(OPERATION_IDENTIFIER, RESULT_TYPE, serDes, durableContext) { diff --git a/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java b/sdk/src/test/java/software/amazon/lambda/durable/serde/filesystem/FileSystemSerDesStageTest.java similarity index 90% rename from sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java rename to sdk/src/test/java/software/amazon/lambda/durable/serde/filesystem/FileSystemSerDesStageTest.java index 4fed8d194..d6b2ceb37 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/serde/FileSystemSerDesTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/serde/filesystem/FileSystemSerDesStageTest.java @@ -1,6 +1,6 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -package software.amazon.lambda.durable.serde; +package software.amazon.lambda.durable.serde.filesystem; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -32,8 +32,19 @@ import software.amazon.lambda.durable.exception.SerDesException; import software.amazon.lambda.durable.model.OperationSubType; import software.amazon.lambda.durable.retry.RetryStrategies; - -class FileSystemSerDesTest { +import software.amazon.lambda.durable.serde.Base64StringBinaryCodec; +import software.amazon.lambda.durable.serde.BinarySerDesStage; +import software.amazon.lambda.durable.serde.ComposableBinarySerDesStage; +import software.amazon.lambda.durable.serde.JacksonSerDes; +import software.amazon.lambda.durable.serde.RetrySerDesStage; +import software.amazon.lambda.durable.serde.SerDes; +import software.amazon.lambda.durable.serde.SerDesContext; +import software.amazon.lambda.durable.serde.SerDesPayloadKind; +import software.amazon.lambda.durable.serde.SerDesRunner; +import software.amazon.lambda.durable.serde.SerDesStage; +import software.amazon.lambda.durable.serde.Utf8StringBinaryCodec; + +class FileSystemSerDesStageTest { private static final String ARN = "arn:aws:lambda:us-east-1:123456789012:function:orders:1/durable-execution/execution-1/invocation-1"; private static final String ENVELOPE_MARKER = "__durable_execution_filesystem_serdes"; @@ -44,7 +55,8 @@ class FileSystemSerDesTest { @Test void writesValueCodecPayloadAndReplaysIt() throws Exception { - var serDes = new JacksonSerDes().then(FileSystemSerDes.builder(basePath).build()); + var serDes = + new JacksonSerDes().then(FileSystemSerDesStage.builder(basePath).build()); var runner = new SerDesRunner(null); var envelope = runner.serialize(serDes, Map.of("id", 42), context()); @@ -62,11 +74,11 @@ void writesValueCodecPayloadAndReplaysIt() throws Exception { @Test void isAStageThatCanBeFollowedByOtherStages() { - var stage = FileSystemSerDes.builder(basePath).build(); + var stage = FileSystemSerDesStage.builder(basePath).build(); var pipeline = new JacksonSerDes().then(stage).then(wrappingStage()); var runner = new SerDesRunner(null); - assertFalse(SerDes.class.isAssignableFrom(FileSystemSerDes.class)); + assertFalse(SerDes.class.isAssignableFrom(FileSystemSerDesStage.class)); var checkpoint = runner.serialize(pipeline, Map.of("id", 42), context()); assertTrue(checkpoint.startsWith("<")); assertEquals( @@ -76,7 +88,7 @@ void isAStageThatCanBeFollowedByOtherStages() { @Test void retryDecoratorComposesAsAFileSystemStage() { - var stage = FileSystemSerDes.builder(basePath) + var stage = FileSystemSerDesStage.builder(basePath) .storageMode(FileSystemStorageMode.OVERFLOW) .build(); var pipeline = new JacksonSerDes().then(new RetrySerDesStage(stage, RetryStrategies.Presets.NO_RETRY)); @@ -96,7 +108,7 @@ void storesAndRestoresComposableBinaryOutput() throws Exception { .then(xorBinaryStage((byte) 0x5A)) .endWith(Base64StringBinaryCodec.INSTANCE) .build(); - var stage = FileSystemSerDes.builder(basePath).build(); + var stage = FileSystemSerDesStage.builder(basePath).build(); var pipeline = new JacksonSerDes().then(binaryStage).then(stage); var runner = new SerDesRunner(null); @@ -117,7 +129,7 @@ void storesAndRestoresComposableBinaryOutput() throws Exception { @Test void overflowModeKeepsSmallPayloadInlineAndWritesLargePayload() throws Exception { - var stage = FileSystemSerDes.builder(basePath) + var stage = FileSystemSerDesStage.builder(basePath) .storageMode(FileSystemStorageMode.OVERFLOW) .build(); var serDes = stringCodec().then(stage); @@ -132,7 +144,7 @@ void overflowModeKeepsSmallPayloadInlineAndWritesLargePayload() throws Exception @Test void immutableContentAddressedFilesPreservePriorCheckpoints() throws Exception { - var serDes = stringCodec().then(FileSystemSerDes.builder(basePath).build()); + var serDes = stringCodec().then(FileSystemSerDesStage.builder(basePath).build()); var runner = new SerDesRunner(null); var firstContext = context(1); @@ -150,7 +162,7 @@ void immutableContentAddressedFilesPreservePriorCheckpoints() throws Exception { @Test void repeatedPayloadsUseDistinctImmutableFiles() throws Exception { - var serDes = stringCodec().then(FileSystemSerDes.builder(basePath).build()); + var serDes = stringCodec().then(FileSystemSerDesStage.builder(basePath).build()); var runner = new SerDesRunner(null); var firstEnvelope = runner.serialize(serDes, "expected", context()); var secondEnvelope = runner.serialize(serDes, "expected", context()); @@ -168,8 +180,8 @@ void writesOnFileSystemsWithoutHardLinkSupport() throws Exception { try (var fileSystem = FileSystems.newFileSystem(URI.create("jar:" + archive.toUri()), Map.of("create", "true"))) { var archiveBasePath = fileSystem.getPath("/payloads"); - var serDes = - stringCodec().then(FileSystemSerDes.builder(archiveBasePath).build()); + var serDes = stringCodec() + .then(FileSystemSerDesStage.builder(archiveBasePath).build()); var envelope = new SerDesRunner(null).serialize(serDes, "expected", context()); var file = fileSystem.getPath(MAPPER.readTree(envelope).get("file").textValue()); @@ -190,7 +202,7 @@ void writesOnFileSystemsWithoutHardLinkSupport() throws Exception { @Test void rejectsMalformedUtf8StringsAndFilePayloads() throws Exception { - var serDes = stringCodec().then(FileSystemSerDes.builder(basePath).build()); + var serDes = stringCodec().then(FileSystemSerDesStage.builder(basePath).build()); var runner = new SerDesRunner(null); assertThrows(SerDesException.class, () -> runner.serialize(serDes, "lone surrogate \uD800", context())); @@ -209,7 +221,7 @@ void rejectsMalformedUtf8StringsAndFilePayloads() throws Exception { @Test void hashEncodingUsesFixedLengthSegments() throws Exception { - var stage = FileSystemSerDes.builder(basePath) + var stage = FileSystemSerDesStage.builder(basePath) .pathEncoding(FileSystemPathEncoding.HASH) .build(); var serDes = stringCodec().then(stage); @@ -227,7 +239,7 @@ void hashEncodingUsesFixedLengthSegments() throws Exception { void includesBoundedPreviewWithoutChangingStoredPayload() throws Exception { var previewValue = new AtomicReference(); var previewContext = new AtomicReference(); - var stage = FileSystemSerDes.builder(basePath) + var stage = FileSystemSerDesStage.builder(basePath) .previewGenerator((value, context) -> { previewValue.set(value); previewContext.set(context); @@ -248,7 +260,7 @@ void includesBoundedPreviewWithoutChangingStoredPayload() throws Exception { "{\"secret\":\"value\"}", Files.readString(Path.of(json.get("file").textValue()))); - var oversizedPreviewStage = FileSystemSerDes.builder(basePath) + var oversizedPreviewStage = FileSystemSerDesStage.builder(basePath) .previewGenerator((value, context) -> Map.of("summary", "x".repeat(256 * 1024))) .build(); var oversizedPreview = stringCodec().then(oversizedPreviewStage); @@ -258,7 +270,7 @@ void includesBoundedPreviewWithoutChangingStoredPayload() throws Exception { @Test void structuredPreviewConfigSelectsAndMasksJsonFields() throws Exception { - var stage = FileSystemSerDes.builder(basePath) + var stage = FileSystemSerDesStage.builder(basePath) .previewConfig(PreviewConfig.builder(PreviewMode.EXCLUDE_ALL) .include(PreviewField.anywhere("id"), PreviewField.path("customer.status")) .mask(PreviewField.anywhere("email")) @@ -287,7 +299,7 @@ void structuredPreviewConfigSelectsAndMasksJsonFields() throws Exception { @Test void structuredPreviewConfigRequiresJsonStageValue() { - var stage = FileSystemSerDes.builder(basePath) + var stage = FileSystemSerDesStage.builder(basePath) .previewConfig(PreviewConfig.builder(PreviewMode.INCLUDE_ALL).build()) .build(); var runner = new SerDesRunner(null); @@ -300,7 +312,7 @@ void structuredPreviewConfigRequiresJsonStageValue() { @Test void passesUnrecognizedPayloadsThroughAtEverySource() { - var stage = FileSystemSerDes.builder(basePath).build(); + var stage = FileSystemSerDesStage.builder(basePath).build(); var filesystemPipeline = new JacksonSerDes().then(stage); var pipeline = new JacksonSerDes().then(wrappingStage()).then(stage); var pipelineWithStageAfterFilesystem = new JacksonSerDes().then(stage).then(wrappingStage()); @@ -356,7 +368,7 @@ void passesUnrecognizedPayloadsThroughAtEverySource() { @Test void rejectsUnsupportedEnvelopeVersionsAtExternalBoundaries() { - var serDes = stringCodec().then(FileSystemSerDes.builder(basePath).build()); + var serDes = stringCodec().then(FileSystemSerDesStage.builder(basePath).build()); var futureEnvelope = "{\"__durable_execution_filesystem_serdes\":2," + "\"ownerDurableExecutionArn\":\"" + ARN @@ -381,7 +393,8 @@ void rejectsUnsupportedBinaryPayloadType() { assertThrows(SerDesException.class, () -> new SerDesRunner(null) .deserialize( - stringCodec().then(FileSystemSerDes.builder(basePath).build()), + stringCodec() + .then(FileSystemSerDesStage.builder(basePath).build()), envelope, TypeToken.get(String.class), context())); @@ -389,7 +402,7 @@ void rejectsUnsupportedBinaryPayloadType() { @Test void rejectsOutOfRangeEnvelopeVersionsWithoutTruncation() { - var serDes = stringCodec().then(FileSystemSerDes.builder(basePath).build()); + var serDes = stringCodec().then(FileSystemSerDesStage.builder(basePath).build()); var oversizedVersion = "{\"__durable_execution_filesystem_serdes\":4294967297," + "\"ownerDurableExecutionArn\":\"" + ARN @@ -407,7 +420,7 @@ void rejectsOutOfRangeEnvelopeVersionsWithoutTruncation() { @Test void overflowFilesystemStageCanBeFollowedByAnotherStage() { - var filesystem = FileSystemSerDes.builder(basePath) + var filesystem = FileSystemSerDesStage.builder(basePath) .storageMode(FileSystemStorageMode.OVERFLOW) .build(); var pipeline = stringCodec().then(filesystem).then(wrappingStage()); @@ -421,7 +434,8 @@ void overflowFilesystemStageCanBeFollowedByAnotherStage() { @Test void fileReferencesCrossInvokeInputAndResultBoundaries() { - var serDes = new JacksonSerDes().then(FileSystemSerDes.builder(basePath).build()); + var serDes = + new JacksonSerDes().then(FileSystemSerDesStage.builder(basePath).build()); var runner = new SerDesRunner(null); var callerArn = "arn:aws:lambda:us-east-1:123456789012:function:caller:1/durable-execution/caller-execution/caller-invocation"; @@ -465,7 +479,7 @@ void fileReferencesCrossInvokeInputAndResultBoundaries() { @Test void rejectsCallsWithoutContextAndMalformedOrUnsafeEnvelopes() throws Exception { - var stage = FileSystemSerDes.builder(basePath).build(); + var stage = FileSystemSerDesStage.builder(basePath).build(); var serDes = stringCodec().then(stage); assertThrows(SerDesException.class, () -> stage.serialize("value", null)); assertEquals("value", stage.deserialize("value", null)); @@ -492,7 +506,7 @@ void rejectsCallsWithoutContextAndMalformedOrUnsafeEnvelopes() throws Exception @Test void rejectsCrossEntityAndSymbolicLinkReferences() throws Exception { - var serDes = stringCodec().then(FileSystemSerDes.builder(basePath).build()); + var serDes = stringCodec().then(FileSystemSerDesStage.builder(basePath).build()); var envelope = new SerDesRunner(null).serialize(serDes, "payload", context()); var otherEntity = SerDesContext.forOperation( @@ -514,7 +528,7 @@ void rejectsCrossEntityAndSymbolicLinkReferences() throws Exception { void rejectsSymbolicLinkDirectoriesWhenWriting() throws Exception { var outside = Files.createTempDirectory(basePath.getParent(), "outside-payloads-"); Files.createSymbolicLink(basePath.resolve("orders"), outside); - var serDes = stringCodec().then(FileSystemSerDes.builder(basePath).build()); + var serDes = stringCodec().then(FileSystemSerDesStage.builder(basePath).build()); assertThrows(SerDesException.class, () -> new SerDesRunner(null).serialize(serDes, "payload", context())); try (var files = Files.list(outside)) { @@ -528,14 +542,15 @@ void rejectsSymbolicLinkConfiguredBasePathAndAncestors() throws Exception { var linkedRoot = basePath.resolve("linked-root"); Files.createSymbolicLink(linkedRoot, outsideRoot); - var rootSerDes = stringCodec().then(FileSystemSerDes.builder(linkedRoot).build()); + var rootSerDes = + stringCodec().then(FileSystemSerDesStage.builder(linkedRoot).build()); assertThrows(SerDesException.class, () -> new SerDesRunner(null).serialize(rootSerDes, "payload", context())); var outsideAncestor = Files.createTempDirectory(basePath.getParent(), "outside-ancestor-"); var linkedAncestor = basePath.resolve("linked-ancestor"); Files.createSymbolicLink(linkedAncestor, outsideAncestor); var nestedSerDes = stringCodec() - .then(FileSystemSerDes.builder(linkedAncestor.resolve("payloads")) + .then(FileSystemSerDesStage.builder(linkedAncestor.resolve("payloads")) .build()); assertThrows(SerDesException.class, () -> new SerDesRunner(null).serialize(nestedSerDes, "payload", context())); @@ -544,7 +559,7 @@ void rejectsSymbolicLinkConfiguredBasePathAndAncestors() throws Exception { @Test void rejectsExecutionPathsOutsideConfiguredBasePath() { - var serDes = stringCodec().then(FileSystemSerDes.builder(basePath).build()); + var serDes = stringCodec().then(FileSystemSerDesStage.builder(basePath).build()); var unsafeContext = SerDesContext.forOperation( "arn:aws:lambda:us-east-1:123456789012:function:..:1/durable-execution/../..", "1", diff --git a/sdk/src/test/java/software/amazon/lambda/durable/serde/SerDesPreviewTest.java b/sdk/src/test/java/software/amazon/lambda/durable/serde/filesystem/SerDesPreviewTest.java similarity index 99% rename from sdk/src/test/java/software/amazon/lambda/durable/serde/SerDesPreviewTest.java rename to sdk/src/test/java/software/amazon/lambda/durable/serde/filesystem/SerDesPreviewTest.java index 4cc029c82..e2e06a63e 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/serde/SerDesPreviewTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/serde/filesystem/SerDesPreviewTest.java @@ -1,6 +1,6 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -package software.amazon.lambda.durable.serde; +package software.amazon.lambda.durable.serde.filesystem; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; From e4cb4029e8b67dac16f9db2c8764ab9529c09dfb Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Tue, 25 Aug 2026 22:44:12 +0000 Subject: [PATCH 42/56] fix: reject malformed filesystem envelopes --- .../filesystem/FileSystemSerDesStage.java | 71 ++++++++++++++++++- .../filesystem/FileSystemSerDesStageTest.java | 36 ++++++++++ 2 files changed, 104 insertions(+), 3 deletions(-) diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/filesystem/FileSystemSerDesStage.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/filesystem/FileSystemSerDesStage.java index ef4124e03..92d2cb6d6 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/filesystem/FileSystemSerDesStage.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/filesystem/FileSystemSerDesStage.java @@ -3,8 +3,10 @@ package software.amazon.lambda.durable.serde.filesystem; import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectReader; import java.io.IOException; import java.nio.file.FileAlreadyExistsException; import java.nio.file.Files; @@ -45,11 +47,12 @@ */ public final class FileSystemSerDesStage implements SerDesStage { private static final String ENVELOPE_MARKER = "__durable_execution_filesystem_serdes"; - private static final String ENVELOPE_PREFIX = "{\"" + ENVELOPE_MARKER + "\":"; private static final String PAYLOAD_TYPE_FIELD = "payloadType"; private static final int ENVELOPE_VERSION = 1; private static final int CHECKPOINT_ENVELOPE_LIMIT_BYTES = 256 * 1024 - 1024; private static final ObjectMapper ENVELOPE_MAPPER = new ObjectMapper(); + private static final ObjectReader ENVELOPE_READER = + ENVELOPE_MAPPER.reader().with(DeserializationFeature.FAIL_ON_TRAILING_TOKENS); private static final Pattern DURABLE_EXECUTION_ARN_PATTERN = Pattern.compile( "^arn:[^:]*:lambda:[^:]*:[^:]*:function:([^:/]+):[^:/]+/durable-execution/([^/]+)/([^/]+)$"); @@ -120,9 +123,9 @@ public String deserialize(String data, SerDesContext context) { private String resolveSerializedPayload(String data, SerDesContext context) { final JsonNode envelope; try { - envelope = ENVELOPE_MAPPER.readTree(data); + envelope = ENVELOPE_READER.readTree(data); } catch (JsonProcessingException e) { - if (data.stripLeading().startsWith(ENVELOPE_PREFIX)) { + if (containsFilesystemMarkerField(data)) { throw malformedEnvelope(requireContext(context), e); } return data; @@ -296,6 +299,68 @@ private static boolean hasFilesystemMarker(JsonNode envelope) { return envelope != null && envelope.isObject() && envelope.has(ENVELOPE_MARKER); } + private static boolean containsFilesystemMarkerField(String data) { + var index = 0; + while (index < data.length() && Character.isWhitespace(data.charAt(index))) { + index++; + } + if (index == data.length() || data.charAt(index) != '{') { + return false; + } + + var containerDepth = 1; + for (index++; index < data.length() && containerDepth > 0; index++) { + var current = data.charAt(index); + if (current == '{' || current == '[') { + containerDepth++; + } else if (current == '}' || current == ']') { + containerDepth--; + } else if (current == '"') { + var literalStart = index; + var valueStart = index + 1; + var escaped = false; + while (++index < data.length()) { + current = data.charAt(index); + if (escaped) { + escaped = false; + } else if (current == '\\') { + escaped = true; + } else if (current == '"') { + break; + } + } + if (index == data.length()) { + return false; + } + + var delimiter = index + 1; + while (delimiter < data.length() && Character.isWhitespace(data.charAt(delimiter))) { + delimiter++; + } + if (containerDepth == 1 + && delimiter < data.length() + && data.charAt(delimiter) == ':' + && isFilesystemMarkerLiteral(data, literalStart, valueStart, index)) { + return true; + } + } + } + return false; + } + + private static boolean isFilesystemMarkerLiteral(String data, int literalStart, int valueStart, int literalEnd) { + if (literalEnd - valueStart == ENVELOPE_MARKER.length() + && data.regionMatches(valueStart, ENVELOPE_MARKER, 0, ENVELOPE_MARKER.length())) { + return true; + } + try { + return ENVELOPE_MARKER.equals( + ENVELOPE_MAPPER.readValue(data.substring(literalStart, literalEnd + 1), String.class)); + } catch (JsonProcessingException ignored) { + return false; + } + } + private static SerDesException malformedEnvelope(SerDesContext context, Throwable cause) { var message = "Invalid filesystem SerDes envelope for entity '" + context.entityId() + "'"; return cause == null ? new SerDesException(message) : new SerDesException(message, cause); diff --git a/sdk/src/test/java/software/amazon/lambda/durable/serde/filesystem/FileSystemSerDesStageTest.java b/sdk/src/test/java/software/amazon/lambda/durable/serde/filesystem/FileSystemSerDesStageTest.java index d6b2ceb37..a439bfc72 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/serde/filesystem/FileSystemSerDesStageTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/serde/filesystem/FileSystemSerDesStageTest.java @@ -22,6 +22,7 @@ import java.util.Arrays; import java.util.Base64; import java.util.HexFormat; +import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicReference; import org.junit.jupiter.api.Test; @@ -366,6 +367,41 @@ void passesUnrecognizedPayloadsThroughAtEverySource() { runner.deserialize(filesystemPipeline, "\"raw-step\"", TypeToken.get(String.class), context())); } + @Test + void rejectsTrailingTokensAfterFilesystemEnvelope() { + var stage = FileSystemSerDesStage.builder(basePath).build(); + var envelope = "{\"__durable_execution_filesystem_serdes\":1," + + "\"ownerDurableExecutionArn\":\"" + + ARN + + "\",\"ownerEntityId\":\"1\",\"payloadType\":\"STRING\",\"data\":\"value\"}"; + + var failure = assertThrows(SerDesException.class, () -> stage.deserialize(envelope + " true", context())); + + assertCauseMessage(failure, "Invalid filesystem SerDes envelope"); + } + + @Test + void recognizesMalformedFilesystemMarkerRegardlessOfWhitespaceOrFieldOrder() { + var stage = FileSystemSerDesStage.builder(basePath).build(); + var malformedEnvelopes = List.of( + "{ \n \"__durable_execution_filesystem_serdes\" : 1", + "{\"precedingField\":true,\n \"__durable_execution_filesystem_serdes\" : 1", + "{\"\\u005f_durable_execution_filesystem_serdes\" : 1"); + + for (var envelope : malformedEnvelopes) { + var failure = assertThrows(SerDesException.class, () -> stage.deserialize(envelope, context())); + assertCauseMessage(failure, "Invalid filesystem SerDes envelope"); + } + } + + @Test + void doesNotTreatFilesystemMarkerTextInsideAStringAsAnEnvelope() { + var stage = FileSystemSerDesStage.builder(basePath).build(); + var value = "{\"message\":\"__durable_execution_filesystem_serdes\"} trailing"; + + assertEquals(value, stage.deserialize(value, null)); + } + @Test void rejectsUnsupportedEnvelopeVersionsAtExternalBoundaries() { var serDes = stringCodec().then(FileSystemSerDesStage.builder(basePath).build()); From 48bb682337dfc7196427ccc25dc52c25e6f9c8de Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Tue, 25 Aug 2026 22:54:56 +0000 Subject: [PATCH 43/56] test: add filesystem SerDes end-to-end coverage --- .github/workflows/e2e-tests.yml | 28 +++- .gitignore | 1 + examples/generate-template.py | 139 +++++++++++++++++- .../durable/examples/ExampleTemplate.java | 2 + .../general/FileSystemSerDesExample.java | 81 ++++++++++ .../examples/CloudBasedIntegrationTest.java | 78 ++++++++++ 6 files changed, 322 insertions(+), 7 deletions(-) create mode 100644 examples/src/main/java/software/amazon/lambda/durable/examples/general/FileSystemSerDesExample.java diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index 8d9f9d4cd..16335bd40 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -69,7 +69,9 @@ jobs: - name: Build locally run: mvn -B -q -Dmaven.test.skip=true install --file pom.xml - name: Generate SAM template - run: python3 generate-template.py + run: | + python3 generate-template.py + python3 generate-template.py --file-system-only --output filesystem-template.yaml working-directory: ./examples - name: sam build env: @@ -78,6 +80,14 @@ jobs: sam build --debug --parameter-overrides \ 'ParameterKey=Architecture,ParameterValue=x86_64 ParameterKey=JavaVersion,ParameterValue=java${{ matrix.java }} ParameterKey=FunctionNamePrefix,ParameterValue=Java${{ matrix.java }}- ParameterKey=RoleArn,ParameterValue=${{ secrets.TEST_LAMBDA_EXECUTION_ROLE_ARN }}' working-directory: ./examples + - name: sam build filesystem SerDes E2E stack + env: + MAVEN_OPTS: -DskipTests=true -Dmaven.test.skip=true + run: | + sam build --debug --template-file filesystem-template.yaml --build-dir .aws-sam-filesystem \ + --parameter-overrides \ + 'ParameterKey=Architecture,ParameterValue=x86_64 ParameterKey=JavaVersion,ParameterValue=java${{ matrix.java }} ParameterKey=FunctionNamePrefix,ParameterValue=Java${{ matrix.java }}- ParameterKey=RoleArn,ParameterValue=${{ secrets.TEST_LAMBDA_EXECUTION_ROLE_ARN }}' + working-directory: ./examples - name: Clean up unmanaged Lambda log groups run: | # TODO: Remove this one-time migration cleanup after existing e2e stacks adopt managed log groups. @@ -89,6 +99,13 @@ jobs: --resolve-image-repos --resolve-s3 --parameter-overrides \ 'ParameterKey=Architecture,ParameterValue=x86_64 ParameterKey=JavaVersion,ParameterValue=java${{ matrix.java }} ParameterKey=FunctionNamePrefix,ParameterValue=Java${{ matrix.java }}- ParameterKey=RoleArn,ParameterValue=${{ secrets.TEST_LAMBDA_EXECUTION_ROLE_ARN }}' working-directory: ./examples + - name: sam deploy filesystem SerDes E2E stack + run: | + sam deploy --template-file .aws-sam-filesystem/template.yaml \ + --stack-name Java${{ matrix.java }}-JavaSDKFileSystemSerDesE2EStack \ + --resolve-s3 --parameter-overrides \ + 'ParameterKey=Architecture,ParameterValue=x86_64 ParameterKey=JavaVersion,ParameterValue=java${{ matrix.java }} ParameterKey=FunctionNamePrefix,ParameterValue=Java${{ matrix.java }}- ParameterKey=RoleArn,ParameterValue=${{ secrets.TEST_LAMBDA_EXECUTION_ROLE_ARN }}' + working-directory: ./examples - name: Cloud Based Integration Tests run: | mvn clean test -B \ @@ -102,6 +119,15 @@ jobs: -Djunit.jupiter.execution.parallel.config.strategy=fixed \ -Djunit.jupiter.execution.parallel.config.fixed.parallelism=${{ env.E2E_TEST_PARALLELISM }} working-directory: ./examples + - name: Delete filesystem SerDes E2E stack + if: always() + run: | + if aws cloudformation describe-stacks \ + --stack-name Java${{ matrix.java }}-JavaSDKFileSystemSerDesE2EStack >/dev/null 2>&1; then + sam delete --no-prompts \ + --stack-name Java${{ matrix.java }}-JavaSDKFileSystemSerDesE2EStack + fi + working-directory: ./examples - name: Publish test case summary if: always() env: diff --git a/.gitignore b/.gitignore index 378d24717..1addbb95d 100644 --- a/.gitignore +++ b/.gitignore @@ -42,6 +42,7 @@ __pycache__/ # SAM .aws-sam/ examples/template.yaml +examples/filesystem-template.yaml samconfig.toml samconfig.toml.bak diff --git a/examples/generate-template.py b/examples/generate-template.py index 2ffcd5d70..8d510bf4c 100755 --- a/examples/generate-template.py +++ b/examples/generate-template.py @@ -21,6 +21,7 @@ class ExampleFunction: package_name: str suffix: str condition: str | None + file_system: bool @property def logical_id(self) -> str: @@ -56,21 +57,23 @@ def is_top_level_durable_handler(source: str, class_name: str) -> bool: return bool(match and "extends DurableHandler" in match.group("header")) -def read_template_condition(source: str, class_name: str) -> str | None: +def read_template_metadata(source: str, class_name: str) -> tuple[str | None, bool]: class_match = re.search(rf"public\s+(?:final\s+)?class\s+{class_name}\b", source) if not class_match: - return None + return None, False prefix = source[: class_match.start()] matches = list( re.finditer(rf"@(?:[A-Za-z_][\w.]*\.)?{TEMPLATE_ANNOTATION}\s*(?:\((?P.*?)\))?", prefix, re.DOTALL) ) if not matches: - return None + return None, False body = matches[-1].group("body") or "" condition_match = re.search(r'condition\s*=\s*"([^"]+)"', body) - return condition_match.group(1) if condition_match else None + condition = condition_match.group(1) if condition_match else None + file_system = bool(re.search(r"\bfileSystem\s*=\s*true\b", body)) + return condition, file_system def discover_examples() -> list[ExampleFunction]: @@ -81,7 +84,7 @@ def discover_examples() -> list[ExampleFunction]: if not is_top_level_durable_handler(source, class_name): continue - condition = read_template_condition(source, class_name) + condition, file_system = read_template_metadata(source, class_name) package_name = read_package(source, path) examples.append( ExampleFunction( @@ -89,6 +92,7 @@ def discover_examples() -> list[ExampleFunction]: package_name=package_name, suffix=kebab_case(class_name), condition=condition, + file_system=file_system, ) ) return examples @@ -103,7 +107,16 @@ def emit_function(lines: list[str], example: ExampleFunction) -> None: ) if example.condition: lines.append(f" Condition: {example.condition}") - lines.append(f" DependsOn: {example.log_group_logical_id}") + if example.file_system: + lines.extend( + [ + " DependsOn:", + f" - {example.log_group_logical_id}", + " - FileSystemMountTarget", + ] + ) + else: + lines.append(f" DependsOn: {example.log_group_logical_id}") lines.extend( [ " Properties:", @@ -112,6 +125,22 @@ def emit_function(lines: list[str], example: ExampleFunction) -> None: " Role: !Ref RoleArn", ] ) + if example.file_system: + lines.extend( + [ + " VpcConfig:", + " SecurityGroupIds:", + " - !Ref FileSystemLambdaSecurityGroup", + " SubnetIds:", + " - !Ref FileSystemSubnet", + " FileSystemConfigs:", + " - Arn: !GetAtt FileSystemAccessPoint.Arn", + " LocalMountPath: /mnt/efs", + " Environment:", + " Variables:", + " FILESYSTEM_SERDES_PATH: /mnt/efs/durable-payloads", + ] + ) lines.append("") @@ -134,6 +163,95 @@ def emit_log_group(lines: list[str], example: ExampleFunction) -> None: ) +def emit_file_system_resources(lines: list[str]) -> None: + lines.extend( + [ + " FileSystemVpc:", + " Type: AWS::EC2::VPC", + " Properties:", + " CidrBlock: 10.0.0.0/24", + " EnableDnsHostnames: true", + " EnableDnsSupport: true", + "", + " FileSystemSubnet:", + " Type: AWS::EC2::Subnet", + " Properties:", + " CidrBlock: 10.0.0.0/26", + " VpcId: !Ref FileSystemVpc", + "", + " FileSystemLambdaSecurityGroup:", + " Type: AWS::EC2::SecurityGroup", + " Properties:", + " GroupDescription: Lambda access to EFS and the Lambda API endpoint", + " VpcId: !Ref FileSystemVpc", + "", + " FileSystemMountSecurityGroup:", + " Type: AWS::EC2::SecurityGroup", + " Properties:", + " GroupDescription: EFS mount access from Lambda", + " VpcId: !Ref FileSystemVpc", + " SecurityGroupIngress:", + " - IpProtocol: tcp", + " FromPort: 2049", + " ToPort: 2049", + " SourceSecurityGroupId: !Ref FileSystemLambdaSecurityGroup", + "", + " FileSystemEndpointSecurityGroup:", + " Type: AWS::EC2::SecurityGroup", + " Properties:", + " GroupDescription: Lambda API endpoint access from Lambda", + " VpcId: !Ref FileSystemVpc", + " SecurityGroupIngress:", + " - IpProtocol: tcp", + " FromPort: 443", + " ToPort: 443", + " SourceSecurityGroupId: !Ref FileSystemLambdaSecurityGroup", + "", + " FileSystemLambdaEndpoint:", + " Type: AWS::EC2::VPCEndpoint", + " Properties:", + " PrivateDnsEnabled: true", + " SecurityGroupIds:", + " - !Ref FileSystemEndpointSecurityGroup", + ' ServiceName: !Sub "com.amazonaws.${AWS::Region}.lambda"', + " SubnetIds:", + " - !Ref FileSystemSubnet", + " VpcEndpointType: Interface", + " VpcId: !Ref FileSystemVpc", + "", + " FileSystem:", + " Type: AWS::EFS::FileSystem", + " Properties:", + " Encrypted: true", + " PerformanceMode: generalPurpose", + " ThroughputMode: bursting", + "", + " FileSystemMountTarget:", + " Type: AWS::EFS::MountTarget", + " Properties:", + " FileSystemId: !Ref FileSystem", + " SecurityGroups:", + " - !Ref FileSystemMountSecurityGroup", + " SubnetId: !Ref FileSystemSubnet", + "", + " FileSystemAccessPoint:", + " Type: AWS::EFS::AccessPoint", + " Properties:", + " FileSystemId: !Ref FileSystem", + " PosixUser:", + ' Gid: "1000"', + ' Uid: "1000"', + " RootDirectory:", + " CreationInfo:", + ' OwnerGid: "1000"', + ' OwnerUid: "1000"', + ' Permissions: "0777"', + " Path: /durable-serdes", + "", + ] + ) + + def render_template(examples: list[ExampleFunction]) -> str: lines = [ "# This file is generated by examples/generate-template.py. Do not edit it by hand.", @@ -184,6 +302,9 @@ def render_template(examples: list[ExampleFunction]) -> str: "Resources:", ] + if any(example.file_system for example in examples): + emit_file_system_resources(lines) + for example in examples: emit_log_group(lines, example) emit_function(lines, example) @@ -208,9 +329,15 @@ def render_template(examples: list[ExampleFunction]) -> str: def main() -> None: parser = argparse.ArgumentParser(description="Generate the examples SAM template from Java example handlers.") parser.add_argument("--output", type=Path, default=DEFAULT_OUTPUT, help="Path to write the generated template.") + parser.add_argument( + "--file-system-only", + action="store_true", + help="Generate the short-lived EFS-backed filesystem SerDes E2E stack.", + ) args = parser.parse_args() examples = discover_examples() + examples = [example for example in examples if example.file_system == args.file_system_only] if not examples: raise RuntimeError("No DurableHandler examples found") diff --git a/examples/src/main/java/software/amazon/lambda/durable/examples/ExampleTemplate.java b/examples/src/main/java/software/amazon/lambda/durable/examples/ExampleTemplate.java index de93e95ca..55eab7934 100644 --- a/examples/src/main/java/software/amazon/lambda/durable/examples/ExampleTemplate.java +++ b/examples/src/main/java/software/amazon/lambda/durable/examples/ExampleTemplate.java @@ -12,4 +12,6 @@ @Target(ElementType.TYPE) public @interface ExampleTemplate { String condition() default ""; + + boolean fileSystem() default false; } diff --git a/examples/src/main/java/software/amazon/lambda/durable/examples/general/FileSystemSerDesExample.java b/examples/src/main/java/software/amazon/lambda/durable/examples/general/FileSystemSerDesExample.java new file mode 100644 index 000000000..23c11a466 --- /dev/null +++ b/examples/src/main/java/software/amazon/lambda/durable/examples/general/FileSystemSerDesExample.java @@ -0,0 +1,81 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.examples.general; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.time.Duration; +import java.util.HexFormat; +import java.util.LinkedHashMap; +import java.util.Map; +import software.amazon.lambda.durable.DurableConfig; +import software.amazon.lambda.durable.DurableContext; +import software.amazon.lambda.durable.DurableHandler; +import software.amazon.lambda.durable.examples.ExampleTemplate; +import software.amazon.lambda.durable.serde.JacksonSerDes; +import software.amazon.lambda.durable.serde.SerDesContext; +import software.amazon.lambda.durable.serde.filesystem.FileSystemSerDesStage; + +/** E2E fixture that offloads durable payloads to an EFS mount and reads them after reinvocation. */ +@ExampleTemplate(fileSystem = true) +public class FileSystemSerDesExample + extends DurableHandler { + private static final String FILE_SYSTEM_PATH_ENV = "FILESYSTEM_SERDES_PATH"; + + @Override + protected DurableConfig createConfiguration() { + var path = System.getenv(FILE_SYSTEM_PATH_ENV); + if (path == null || path.isBlank()) { + throw new IllegalStateException(FILE_SYSTEM_PATH_ENV + " must identify the mounted durable filesystem"); + } + var fileSystemStage = FileSystemSerDesStage.builder(Path.of(path)) + .previewGenerator(FileSystemSerDesExample::preview) + .build(); + return DurableConfig.builder() + .withSerDes(new JacksonSerDes().then(fileSystemStage)) + .build(); + } + + @Override + public Output handleRequest(Input input, DurableContext context) { + var stored = + context.step("store-payload", Payload.class, stepContext -> new Payload(input.id(), input.value())); + context.wait("force-filesystem-replay", Duration.ofSeconds(1)); + return context.step( + "verify-payload", + Output.class, + stepContext -> new Output(stored.id(), stored.value().length(), sha256(stored.value()))); + } + + private static Map preview(String value, SerDesContext context) { + var preview = new LinkedHashMap(); + preview.put("payloadKind", context.payloadKind().name()); + preview.put("operationName", context.operationName()); + if (context.originalValue() instanceof Payload payload) { + preview.put("id", payload.id()); + preview.put("length", payload.value().length()); + } else if (context.originalValue() instanceof Output output) { + preview.put("id", output.id()); + preview.put("length", output.length()); + preview.put("checksum", output.checksum()); + } + return preview; + } + + private static String sha256(String value) { + try { + return HexFormat.of() + .formatHex(MessageDigest.getInstance("SHA-256").digest(value.getBytes(StandardCharsets.UTF_8))); + } catch (NoSuchAlgorithmException e) { + throw new IllegalStateException("SHA-256 is unavailable", e); + } + } + + public record Input(String id, String value) {} + + public record Payload(String id, String value) {} + + public record Output(String id, int length, String checksum) {} +} diff --git a/examples/src/test/java/software/amazon/lambda/durable/examples/CloudBasedIntegrationTest.java b/examples/src/test/java/software/amazon/lambda/durable/examples/CloudBasedIntegrationTest.java index 31d5af673..619e72fbd 100644 --- a/examples/src/test/java/software/amazon/lambda/durable/examples/CloudBasedIntegrationTest.java +++ b/examples/src/test/java/software/amazon/lambda/durable/examples/CloudBasedIntegrationTest.java @@ -5,8 +5,14 @@ import static org.junit.jupiter.api.Assertions.*; import static software.amazon.lambda.durable.TypeToken.get; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; import java.time.Duration; import java.util.HashMap; +import java.util.HexFormat; import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; @@ -21,9 +27,11 @@ import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.lambda.LambdaClient; import software.amazon.awssdk.services.lambda.model.ErrorObject; +import software.amazon.awssdk.services.lambda.model.EventType; import software.amazon.awssdk.services.lambda.model.OperationStatus; import software.amazon.awssdk.services.sts.StsClient; import software.amazon.lambda.durable.TypeToken; +import software.amazon.lambda.durable.examples.general.FileSystemSerDesExample; import software.amazon.lambda.durable.examples.general.GenericTypesExample; import software.amazon.lambda.durable.examples.types.ApprovalRequest; import software.amazon.lambda.durable.examples.types.GreetingRequest; @@ -38,6 +46,8 @@ @EnabledIf("isEnabled") class CloudBasedIntegrationTest { private static final int PERFORMANCE_TEST_REPEAT = 3; + private static final String FILE_SYSTEM_ENVELOPE_MARKER = "__durable_execution_filesystem_serdes"; + private static final ObjectMapper MAPPER = new ObjectMapper(); private static String account; private static String region; @@ -363,6 +373,43 @@ void testCustomConfigExample() { assertTrue(stepResult.contains("email_address")); } + @Test + void testFileSystemSerDesExample() throws Exception { + var value = "filesystem-e2e-".repeat(24 * 1024); + var input = new FileSystemSerDesExample.Input("payload-1", value); + var expectedChecksum = sha256(value); + var runner = CloudDurableTestRunner.create( + arn("file-system-ser-des-example"), + FileSystemSerDesExample.Input.class, + FileSystemSerDesExample.Output.class, + lambdaClient); + + var result = runner.run(input); + + assertEquals(ExecutionStatus.SUCCEEDED, result.getStatus()); + assertTrue(result.getHistoryEvents().stream() + .filter(event -> event.eventType() == EventType.INVOCATION_COMPLETED) + .count() + >= 2); + assertNotNull(result.getOperation("force-filesystem-replay")); + + var storedEnvelope = assertFileSystemEnvelope( + result.getOperation("store-payload").getStepDetails().result()); + assertPreview(storedEnvelope, "RESULT", "store-payload", input.id(), value.length(), null); + + var verifiedEnvelope = assertFileSystemEnvelope( + result.getOperation("verify-payload").getStepDetails().result()); + assertPreview(verifiedEnvelope, "RESULT", "verify-payload", input.id(), value.length(), expectedChecksum); + + var outputPayload = result.getHistoryEvents().stream() + .filter(event -> event.eventType() == EventType.EXECUTION_SUCCEEDED) + .map(event -> event.executionSucceededDetails().result().payload()) + .findFirst() + .orElseThrow(); + var outputEnvelope = assertFileSystemEnvelope(outputPayload); + assertPreview(outputEnvelope, "OUTPUT", null, input.id(), value.length(), expectedChecksum); + } + @Test void testErrorHandlingExample() { var runner = @@ -853,4 +900,35 @@ void testPluginExample() { assertNotNull(runner.getOperation("create-greeting")); assertNotNull(runner.getOperation("transform")); } + + private static JsonNode assertFileSystemEnvelope(String value) throws Exception { + var envelope = MAPPER.readTree(value); + assertEquals(1, envelope.get(FILE_SYSTEM_ENVELOPE_MARKER).intValue()); + assertEquals("STRING", envelope.get("payloadType").textValue()); + assertTrue(envelope.get("file").textValue().startsWith("/mnt/efs/durable-payloads/")); + return envelope; + } + + private static void assertPreview( + JsonNode envelope, String payloadKind, String operationName, String id, int length, String checksum) { + var preview = envelope.get("preview"); + assertEquals(payloadKind, preview.get("payloadKind").textValue()); + if (operationName != null) { + assertEquals(operationName, preview.get("operationName").textValue()); + } + assertEquals(id, preview.get("id").textValue()); + assertEquals(length, preview.get("length").intValue()); + if (checksum != null) { + assertEquals(checksum, preview.get("checksum").textValue()); + } + } + + private static String sha256(String value) { + try { + return HexFormat.of() + .formatHex(MessageDigest.getInstance("SHA-256").digest(value.getBytes(StandardCharsets.UTF_8))); + } catch (NoSuchAlgorithmException e) { + throw new IllegalStateException("SHA-256 is unavailable", e); + } + } } From f0a3e0bf9c15961725a8f0bac432e755362ba9bc Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Tue, 25 Aug 2026 23:07:33 +0000 Subject: [PATCH 44/56] feat: configure filesystem envelope limit --- docs/adr/005-filesystem-serdes.md | 9 ++-- docs/advanced/configuration.md | 8 ++-- docs/advanced/filesystem-serdes.md | 9 +++- .../filesystem/FileSystemSerDesStage.java | 28 ++++++++++-- .../filesystem/FileSystemSerDesStageTest.java | 44 +++++++++++++++++++ 5 files changed, 87 insertions(+), 11 deletions(-) diff --git a/docs/adr/005-filesystem-serdes.md b/docs/adr/005-filesystem-serdes.md index 7d3f53352..a4fd219cc 100644 --- a/docs/adr/005-filesystem-serdes.md +++ b/docs/adr/005-filesystem-serdes.md @@ -127,6 +127,7 @@ import software.amazon.lambda.durable.serde.JacksonSerDes; var fileSystemStage = FileSystemSerDesStage.builder(Path.of("/mnt/efs/durable-payloads")) .storageMode(FileSystemStorageMode.ALWAYS) .pathEncoding(FileSystemPathEncoding.URI) + .checkpointEnvelopeLimitBytes(configuredCheckpointEnvelopeLimitBytes) .previewGenerator(optionalPreviewGenerator) .build(); @@ -386,7 +387,7 @@ Storage modes: | Mode | Behavior | |------|----------| | `ALWAYS` | Always write the incoming stage representation to a file and return a file envelope. | -| `OVERFLOW` | Return an inline envelope until it approaches the service payload limit, then write the incoming stage representation to a file. | +| `OVERFLOW` | Return an inline envelope until it approaches the configured checkpoint-envelope limit, then write the incoming stage representation to a file. | Path encodings: @@ -426,8 +427,10 @@ encoding to exchange offloaded invoke payloads and results. The declared owner m path, and the resolved file must remain beneath the configured root. The file envelope is therefore a capability and must be protected with the same care as the payload it references. -The final file envelope, including any preview, must remain below the checkpoint threshold. Oversized previews are -rejected rather than producing a checkpoint that the service cannot accept. +The final file envelope, including any preview, must remain below the configured checkpoint-envelope threshold. +`FileSystemSerDesStage` retains the current 255 KiB value as its default while allowing applications to increase it as +the service adds support for larger payloads. Oversized previews are rejected rather than producing a checkpoint that +the service cannot accept. Stages may follow `FileSystemSerDesStage` to transform its inline or file-reference envelope. Its overflow and preview-size checks apply at the filesystem stage boundary, so configurations must account for any size expansion introduced by diff --git a/docs/advanced/configuration.md b/docs/advanced/configuration.md index b7d8b0785..76065a084 100644 --- a/docs/advanced/configuration.md +++ b/docs/advanced/configuration.md @@ -62,6 +62,7 @@ path, and preview APIs live in `software.amazon.lambda.durable.serde.filesystem` var fileSystemStage = FileSystemSerDesStage.builder(Path.of("/mnt/efs/durable-payloads")) .storageMode(FileSystemStorageMode.OVERFLOW) .pathEncoding(FileSystemPathEncoding.HASH) + .checkpointEnvelopeLimitBytes(configuredCheckpointEnvelopeLimitBytes) .build(); var resilientFileSystemStage = new RetrySerDesStage( @@ -102,9 +103,10 @@ interface. UTF-8 and Base64 implementations are included in the core SDK, conver binary chain, and the outer stage adds a reserved versioned frame for reliable format recognition. `ALWAYS` stores every non-null payload in a file. `OVERFLOW` keeps payloads inline until the checkpoint envelope -approaches the 256 KB service limit. `URI` produces readable escaped paths; `HASH` produces fixed-length SHA-256 path -segments. Files include a content hash and never overwrite data referenced by an earlier checkpoint. References are -validated against the current durable execution and entity, and symbolic-link paths are rejected. +approaches the configured limit. The limit defaults to 255 KiB and can be increased when the service accepts larger +payloads. `URI` produces readable escaped paths; `HASH` produces fixed-length SHA-256 path segments. Files include a +content hash and never overwrite data referenced by an earlier checkpoint. References are validated against the +current durable execution and entity, and symbolic-link paths are rejected. For a `JacksonSerDes -> FileSystemSerDesStage` pipeline, structured preview configuration provides the same field selection, masking, exact-path matching, and default 4 KB preview budget as the Python and TypeScript SDKs: diff --git a/docs/advanced/filesystem-serdes.md b/docs/advanced/filesystem-serdes.md index 5abb855af..6088ac17e 100644 --- a/docs/advanced/filesystem-serdes.md +++ b/docs/advanced/filesystem-serdes.md @@ -22,6 +22,7 @@ envelopes in checkpoints. It is included in the core `aws-durable-execution-sdk- var fileSystemStage = FileSystemSerDesStage.builder(Path.of("/mnt/efs/durable-payloads")) .storageMode(FileSystemStorageMode.ALWAYS) .pathEncoding(FileSystemPathEncoding.URI) + .checkpointEnvelopeLimitBytes(configuredCheckpointEnvelopeLimitBytes) .build(); var resilientFileSystemStage = new RetrySerDesStage( @@ -59,10 +60,13 @@ the same `StringBinaryCodec` interface; the example performs UTF-8 and Base64 co compression/encryption chain. - `ALWAYS` writes every non-null payload to a file. -- `OVERFLOW` stores small payloads inline and offloads envelopes approaching the 256 KB checkpoint limit. +- `OVERFLOW` stores small payloads inline and offloads envelopes approaching the configured checkpoint-envelope limit. - `URI` uses readable escaped path segments. - `HASH` uses fixed-length SHA-256 path segments. +`checkpointEnvelopeLimitBytes(...)` controls the maximum UTF-8 size accepted for both inline and file envelopes. It +defaults to 255 KiB and can be increased when the durable execution service supports a larger payload limit. + ## Structured previews Java includes the same structured preview controls as the Python and TypeScript SDKs: @@ -90,7 +94,8 @@ visible; include and mask rules make selected fields visible. `ANYWHERE` matches The built-in `previewConfig(...)` parses the incoming stage string as JSON. Use `previewGenerator(...)` for non-JSON stage values or fully custom logic. The custom generator receives the stage string and `SerDesContext`, including the pre-serialization object in `originalValue()`. Previews are included only in file envelopes. The structured builder -defaults to a 4 KB preview budget, and the complete file envelope must still remain below the checkpoint threshold. +defaults to a 4 KB preview budget, and the complete file envelope must still remain below the configured checkpoint +threshold. Custom generators must avoid exposing sensitive fields. ## Execution and retries diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/filesystem/FileSystemSerDesStage.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/filesystem/FileSystemSerDesStage.java index 92d2cb6d6..db336b66c 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/filesystem/FileSystemSerDesStage.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/filesystem/FileSystemSerDesStage.java @@ -49,7 +49,7 @@ public final class FileSystemSerDesStage implements SerDesStage { private static final String ENVELOPE_MARKER = "__durable_execution_filesystem_serdes"; private static final String PAYLOAD_TYPE_FIELD = "payloadType"; private static final int ENVELOPE_VERSION = 1; - private static final int CHECKPOINT_ENVELOPE_LIMIT_BYTES = 256 * 1024 - 1024; + private static final int DEFAULT_CHECKPOINT_ENVELOPE_LIMIT_BYTES = 256 * 1024 - 1024; private static final ObjectMapper ENVELOPE_MAPPER = new ObjectMapper(); private static final ObjectReader ENVELOPE_READER = ENVELOPE_MAPPER.reader().with(DeserializationFeature.FAIL_ON_TRAILING_TOKENS); @@ -59,6 +59,7 @@ public final class FileSystemSerDesStage implements SerDesStage { private final Path basePath; private final FileSystemStorageMode storageMode; private final FileSystemPathEncoding pathEncoding; + private final int checkpointEnvelopeLimitBytes; private final BiFunction> previewGenerator; private volatile Path canonicalBasePath; @@ -66,6 +67,7 @@ private FileSystemSerDesStage(Builder builder) { basePath = builder.basePath.toAbsolutePath().normalize(); storageMode = builder.storageMode; pathEncoding = builder.pathEncoding; + checkpointEnvelopeLimitBytes = builder.checkpointEnvelopeLimitBytes; previewGenerator = builder.previewGenerator; } @@ -409,8 +411,8 @@ private Map generatePreview(String value, SerDesContext context) } } - private static boolean fitsCheckpoint(String envelope) { - return Utf8StringBinaryCodec.INSTANCE.toBytes(envelope).length <= CHECKPOINT_ENVELOPE_LIMIT_BYTES; + private boolean fitsCheckpoint(String envelope) { + return Utf8StringBinaryCodec.INSTANCE.toBytes(envelope).length <= checkpointEnvelopeLimitBytes; } private SerDesContext requireContext(SerDesContext context) { @@ -635,6 +637,7 @@ public static final class Builder { private final Path basePath; private FileSystemStorageMode storageMode = FileSystemStorageMode.ALWAYS; private FileSystemPathEncoding pathEncoding = FileSystemPathEncoding.URI; + private int checkpointEnvelopeLimitBytes = DEFAULT_CHECKPOINT_ENVELOPE_LIMIT_BYTES; private BiFunction> previewGenerator; private Builder(Path basePath) { @@ -651,6 +654,25 @@ public Builder pathEncoding(FileSystemPathEncoding pathEncoding) { return this; } + /** + * Configures the maximum UTF-8 size of an inline or file checkpoint envelope. + * + *

{@link FileSystemStorageMode#OVERFLOW} offloads an inline envelope that exceeds this limit. A final file + * envelope that exceeds the limit is rejected. The configured value should not exceed the payload limit + * accepted by the durable execution service. The default is 255 KiB. + * + * @param checkpointEnvelopeLimitBytes positive envelope limit in bytes + * @return this builder + * @throws IllegalArgumentException if {@code checkpointEnvelopeLimitBytes} is not positive + */ + public Builder checkpointEnvelopeLimitBytes(int checkpointEnvelopeLimitBytes) { + if (checkpointEnvelopeLimitBytes <= 0) { + throw new IllegalArgumentException("checkpointEnvelopeLimitBytes must be positive"); + } + this.checkpointEnvelopeLimitBytes = checkpointEnvelopeLimitBytes; + return this; + } + /** * Configures a custom preview generator that receives the string produced by the preceding pipeline stage and * its serialization context. diff --git a/sdk/src/test/java/software/amazon/lambda/durable/serde/filesystem/FileSystemSerDesStageTest.java b/sdk/src/test/java/software/amazon/lambda/durable/serde/filesystem/FileSystemSerDesStageTest.java index a439bfc72..590a6c34d 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/serde/filesystem/FileSystemSerDesStageTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/serde/filesystem/FileSystemSerDesStageTest.java @@ -143,6 +143,50 @@ void overflowModeKeepsSmallPayloadInlineAndWritesLargePayload() throws Exception assertTrue(MAPPER.readTree(overflow).has("file")); } + @Test + void checkpointEnvelopeLimitCanBeIncreasedForLargerInlinePayloads() throws Exception { + var value = "x".repeat(300 * 1024); + var runner = new SerDesRunner(null); + var defaultPipeline = stringCodec() + .then(FileSystemSerDesStage.builder(basePath) + .storageMode(FileSystemStorageMode.OVERFLOW) + .build()); + var largerEnvelopePipeline = stringCodec() + .then(FileSystemSerDesStage.builder(basePath) + .storageMode(FileSystemStorageMode.OVERFLOW) + .checkpointEnvelopeLimitBytes(512 * 1024) + .build()); + + assertTrue(MAPPER.readTree(runner.serialize(defaultPipeline, value, context())) + .has("file")); + assertTrue(MAPPER.readTree(runner.serialize(largerEnvelopePipeline, value, context())) + .has("data")); + } + + @Test + void checkpointEnvelopeLimitMustBePositive() { + var zeroFailure = assertThrows(IllegalArgumentException.class, () -> FileSystemSerDesStage.builder(basePath) + .checkpointEnvelopeLimitBytes(0)); + var negativeFailure = assertThrows(IllegalArgumentException.class, () -> FileSystemSerDesStage.builder(basePath) + .checkpointEnvelopeLimitBytes(-1)); + + assertEquals("checkpointEnvelopeLimitBytes must be positive", zeroFailure.getMessage()); + assertEquals("checkpointEnvelopeLimitBytes must be positive", negativeFailure.getMessage()); + } + + @Test + void checkpointEnvelopeLimitAlsoAppliesToFileEnvelopes() { + var pipeline = stringCodec() + .then(FileSystemSerDesStage.builder(basePath) + .checkpointEnvelopeLimitBytes(1) + .build()); + + var failure = assertThrows( + SerDesException.class, () -> new SerDesRunner(null).serialize(pipeline, "value", context())); + + assertCauseMessage(failure, "checkpoint payload limit"); + } + @Test void immutableContentAddressedFilesPreservePriorCheckpoints() throws Exception { var serDes = stringCodec().then(FileSystemSerDesStage.builder(basePath).build()); From 627f68229edfc1db2a00dc7d8e9aed40e81698b1 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Tue, 25 Aug 2026 23:22:27 +0000 Subject: [PATCH 45/56] docs: consolidate SerDes documentation --- README.md | 5 +- docs/advanced/configuration.md | 108 +---------- docs/advanced/filesystem-serdes.md | 155 ---------------- docs/advanced/serdes.md | 288 +++++++++++++++++++++++++++++ 4 files changed, 292 insertions(+), 264 deletions(-) delete mode 100644 docs/advanced/filesystem-serdes.md create mode 100644 docs/advanced/serdes.md diff --git a/README.md b/README.md index 306c0d8e3..278691071 100644 --- a/README.md +++ b/README.md @@ -50,9 +50,6 @@ Your durable function extends `DurableHandler` and implements `handleReque ``` -Filesystem-backed payload storage is included in the core SDK. See -[Filesystem SerDes](docs/advanced/filesystem-serdes.md) for configuration and durability requirements. - ### Your First Durable Function ```java @@ -114,7 +111,7 @@ See [Deploy Lambda durable functions with Infrastructure as Code](https://docs.a **Advanced Topics** - [Configuration](docs/advanced/configuration.md) - Customize SDK behaviour -- [Filesystem SerDes](docs/advanced/filesystem-serdes.md) - Store durable payloads on a shared filesystem +- [Serialization and SerDes Pipelines](docs/advanced/serdes.md) - Configure value codecs, processing pipelines, and filesystem storage - [Error Handling](docs/advanced/error-handling.md) - SDK exceptions for handling failures - [Logging](docs/advanced/logging.md) - How to use DurableLogger - [Migrating from 1.x to 2.x](docs/migration-1.x-to-2.x.md) - Upgrade guide for breaking changes since `v1.2.1` diff --git a/docs/advanced/configuration.md b/docs/advanced/configuration.md index 76065a084..a336af3c8 100644 --- a/docs/advanced/configuration.md +++ b/docs/advanced/configuration.md @@ -48,111 +48,9 @@ By default, SerDes runs synchronously on the calling thread to preserve existing SerDes executor must be different from the user-operation executor to avoid deadlock when the operation pool is saturated. -The SDK passes `SerDesContext` explicitly only to every `SerDesStage` and nested `BinarySerDesStage`; root `SerDes` -value codecs remain context-free. During serialization, `context.originalValue()` contains the object supplied to the -root value codec. During deserialization it is `null`. The SDK also uses a bounded weak-reference cache for successful -deserialization results during the current invocation. - -### Filesystem-backed payload storage - -The core SDK provides a reversible stage for storing serialized strings on a shared filesystem. Its stage, storage, -path, and preview APIs live in `software.amazon.lambda.durable.serde.filesystem`: - -```java -var fileSystemStage = FileSystemSerDesStage.builder(Path.of("/mnt/efs/durable-payloads")) - .storageMode(FileSystemStorageMode.OVERFLOW) - .pathEncoding(FileSystemPathEncoding.HASH) - .checkpointEnvelopeLimitBytes(configuredCheckpointEnvelopeLimitBytes) - .build(); - -var resilientFileSystemStage = new RetrySerDesStage( - fileSystemStage, - RetryStrategies.fixedDelay(3, Duration.ofSeconds(1))); - -var binaryStage = ComposableBinarySerDesStage.builder() - .startWith(Utf8StringBinaryCodec.INSTANCE) - .then(compressionBinaryStage) - .then(encryptionBinaryStage) - .endWith(Base64StringBinaryCodec.INSTANCE) - .build(); - -var serDes = new JacksonSerDes() - .then(binaryStage) - .then(resilientFileSystemStage) - .then(checkpointEnvelopeStage); -var serDesExecutor = Executors.newFixedThreadPool(4); - -return DurableConfig.builder() - .withSerDes(serDes) - .withSerDesExecutorService(serDesExecutor) - .build(); -``` - -Every top-level stage consumes and produces a string, so stages compose without intermediate type mismatches. -Every stage method also receives the current read-only `SerDesContext`. Serialization stages receive one derived -context whose `originalValue()` is the object supplied to the root value codec; deserialization stages receive a -context whose `originalValue()` is `null`. The same stage context is propagated through the complete pipeline and -through every retry attempt. -Every stage must emit a self-identifying, normally versioned representation. On deserialization it reverses recognized -valid input, rejects recognized malformed or unsupported input, and returns unrecognized input unchanged. This lets -raw external payloads pass through the configured stages and reach the root value codec. -`ComposableBinarySerDesStage` contains an ordered chain of `BinarySerDesStage` implementations for compression, -encryption, or other `byte[]` transformations. Its `startWith(...)`, `then(...)`, and `endWith(...)` calls follow -serialization order; deserialization reverses them. Both boundaries use the customizable `StringBinaryCodec` -interface. UTF-8 and Base64 implementations are included in the core SDK, conversion occurs only around the complete -binary chain, and the outer stage adds a reserved versioned frame for reliable format recognition. - -`ALWAYS` stores every non-null payload in a file. `OVERFLOW` keeps payloads inline until the checkpoint envelope -approaches the configured limit. The limit defaults to 255 KiB and can be increased when the service accepts larger -payloads. `URI` produces readable escaped paths; `HASH` produces fixed-length SHA-256 path segments. Files include a -content hash and never overwrite data referenced by an earlier checkpoint. References are validated against the -current durable execution and entity, and symbolic-link paths are rejected. - -For a `JacksonSerDes -> FileSystemSerDesStage` pipeline, structured preview configuration provides the same field -selection, masking, exact-path matching, and default 4 KB preview budget as the Python and TypeScript SDKs: - -```java -var previewConfig = PreviewConfig.builder(PreviewMode.EXCLUDE_ALL) - .include(PreviewField.anywhere("id"), PreviewField.path("customer.status")) - .mask(PreviewField.anywhere("email")) - .build(); - -var fileSystemStage = FileSystemSerDesStage.builder(Path.of("/mnt/s3/durable-payloads")) - .previewConfig(previewConfig) - .build(); - -var serDes = new JacksonSerDes().then(fileSystemStage); -``` - -The built-in preview configuration parses the string produced by the preceding stage as JSON. Use -`previewGenerator(...)` when the preceding stage produces another format or when fully custom preview logic is needed. -The custom generator receives both that string and `SerDesContext`, so it can use `originalValue()` when the preview -should be derived from the pre-serialization object. Custom generators must avoid exposing sensitive fields. - -`RetrySerDesStage` wraps a `SerDesStage`, while `RetryBinarySerDesStage` wraps a `BinarySerDesStage`. Both retry only -failures marked with `RetryableSerDesException`. Filesystem read and write I/O use this marker; malformed envelopes and -codec failures fail immediately. Backoff occurs within the current Lambda invocation, so use short, bounded retry -strategies. Without a configured SerDes executor, filesystem I/O and retry delays block the calling thread. - -Do not use Lambda's ephemeral `/tmp` directory: replay can run in a different execution environment. Use a durable, -shared mount such as EFS or S3 Files. Payloads are published with one immutable `CREATE_NEW` write, without hard links -or renames, so the write path is compatible with S3 Files. S3 Files can have delayed synchronization, so a runtime -crash before the mount flushes may lose recent writes; use it only when that tradeoff is acceptable. The SDK does not -delete offloaded files, so configure storage lifecycle and retention separately. - -The cloud and local test runners always serialize the initial Lambda invocation with a separate context-free value -codec. By default, this is the configured persisted SerDes when it is a plain value codec, or the root value codec when -the persisted SerDes is a `ComposableSerDes`; `withInputSerDes(...)` provides an explicit override. The runners never -use persisted pipeline stages to encode the initial invocation. The explicit input SerDes must therefore be a value -codec rather than a `ComposableSerDes`. When the runtime later deserializes that raw input, each persisted stage passes -it through unless its self-identifying format is present. A custom input codec must produce data that the persisted -pipeline's root value codec can decode. - -Stages may follow `FileSystemSerDesStage` to transform its inline or file-reference envelope. `OVERFLOW` and preview-size -checks apply to the filesystem stage's output; account for any expansion introduced by later stages when staying -within the service checkpoint limit. On deserialization, every stage passes input through unchanged when its own -self-identifying format is absent, so raw external payloads can safely traverse stages on either side of the -filesystem stage. +For value codecs, composable string and binary stages, stage context, retry and executor behavior, test-runner input +codecs, and filesystem-backed payload storage, see +[Serialization and SerDes pipelines](serdes.md). ### Dynamic plugin loading diff --git a/docs/advanced/filesystem-serdes.md b/docs/advanced/filesystem-serdes.md deleted file mode 100644 index 6088ac17e..000000000 --- a/docs/advanced/filesystem-serdes.md +++ /dev/null @@ -1,155 +0,0 @@ -# Filesystem SerDes - -`FileSystemSerDesStage` stores durable user payloads on a shared filesystem while keeping small, versioned file-reference -envelopes in checkpoints. It is included in the core `aws-durable-execution-sdk-java` artifact under the -`software.amazon.lambda.durable.serde.filesystem` Java package. - -## Installation - -```xml - - software.amazon.lambda.durable - aws-durable-execution-sdk-java - VERSION - -``` - -## Pipeline configuration - -`FileSystemSerDesStage` is a reversible `SerDesStage` that must be configured after a value codec: - -```java -var fileSystemStage = FileSystemSerDesStage.builder(Path.of("/mnt/efs/durable-payloads")) - .storageMode(FileSystemStorageMode.ALWAYS) - .pathEncoding(FileSystemPathEncoding.URI) - .checkpointEnvelopeLimitBytes(configuredCheckpointEnvelopeLimitBytes) - .build(); - -var resilientFileSystemStage = new RetrySerDesStage( - fileSystemStage, - RetryStrategies.fixedDelay(3, Duration.ofSeconds(1))); - -var binaryStage = ComposableBinarySerDesStage.builder() - .startWith(Utf8StringBinaryCodec.INSTANCE) - .then(compressionBinaryStage) - .then(encryptionBinaryStage) - .endWith(Base64StringBinaryCodec.INSTANCE) - .build(); - -var serDes = new JacksonSerDes() - .then(binaryStage) - .then(resilientFileSystemStage) - .then(checkpointEnvelopeStage); -var serDesExecutor = Executors.newFixedThreadPool(4); - -return DurableConfig.builder() - .withSerDes(serDes) - .withSerDesExecutorService(serDesExecutor) - .build(); -``` - -Serialization follows the declaration order above and deserialization runs in reverse. The first component is the -`SerDes` value codec; every component appended with `then(...)` implements `SerDesStage` and consumes and produces a -string. Each stage must use a self-identifying format: it reverses recognized valid input, rejects recognized malformed -or unsupported input, and returns unrecognized input unchanged. The runner passes the same read-only -`SerDesContext` explicitly to every string stage and binary substage. During serialization, `originalValue()` exposes -the object supplied to the root value codec; during deserialization it is `null`. `ComposableBinarySerDesStage` -converts the string with its starting codec, passes bytes directly through each `BinarySerDesStage`, converts the final -bytes to a string with its ending codec, and adds a reserved versioned frame. Both boundaries are customizable through -the same `StringBinaryCodec` interface; the example performs UTF-8 and Base64 conversion once around the complete -compression/encryption chain. - -- `ALWAYS` writes every non-null payload to a file. -- `OVERFLOW` stores small payloads inline and offloads envelopes approaching the configured checkpoint-envelope limit. -- `URI` uses readable escaped path segments. -- `HASH` uses fixed-length SHA-256 path segments. - -`checkpointEnvelopeLimitBytes(...)` controls the maximum UTF-8 size accepted for both inline and file envelopes. It -defaults to 255 KiB and can be increased when the durable execution service supports a larger payload limit. - -## Structured previews - -Java includes the same structured preview controls as the Python and TypeScript SDKs: - -```java -var previewConfig = PreviewConfig.builder(PreviewMode.EXCLUDE_ALL) - .include(PreviewField.anywhere("id"), PreviewField.path("customer.status")) - .exclude(PreviewField.anywhere("internal")) - .mask(PreviewField.anywhere("email")) - .maskString("***") - .maxPreviewBytes(4096) - .build(); - -var fileSystemStage = FileSystemSerDesStage.builder(Path.of("/mnt/s3/durable-payloads")) - .previewConfig(previewConfig) - .build(); - -var serDes = new JacksonSerDes().then(fileSystemStage); -``` - -`INCLUDE_ALL` starts with every leaf visible and applies exclude and mask rules. `EXCLUDE_ALL` starts with no fields -visible; include and mask rules make selected fields visible. `ANYWHERE` matches a field name at any depth, while -`PATH` matches an exact dot-separated path. Exclude rules win over mask rules, and masking implies visibility. - -The built-in `previewConfig(...)` parses the incoming stage string as JSON. Use `previewGenerator(...)` for non-JSON -stage values or fully custom logic. The custom generator receives the stage string and `SerDesContext`, including the -pre-serialization object in `originalValue()`. Previews are included only in file envelopes. The structured builder -defaults to a 4 KB preview budget, and the complete file envelope must still remain below the configured checkpoint -threshold. -Custom generators must avoid exposing sensitive fields. - -## Execution and retries - -SerDes runs inline by default. Filesystem access and retry backoff are blocking, so production configurations should -provide a dedicated executor with `withSerDesExecutorService(...)`. It must be different from the user-operation -executor. - -Filesystem read and write I/O failures are reported as `RetryableSerDesException`. `RetrySerDesStage` wraps a -`SerDesStage`, so the retrying filesystem component can be appended directly to the pipeline. -`RetryBinarySerDesStage` provides the same behavior for a `BinarySerDesStage` inside -`ComposableBinarySerDesStage`. Both retry only that exception type. Malformed envelopes, invalid paths, unsupported -stage types, and codec failures are permanent. Retry delays consume time in the current Lambda invocation, so keep -attempts and delays bounded. - -## Replay and envelope behavior - -Filesystem envelopes include a reserved version marker. `FileSystemSerDesStage` returns input without that marker unchanged, -allowing raw root input, callback results, and standard Lambda invoke results to continue through the remaining stages -to the pipeline value codec. Payloads containing the reserved marker must be valid supported filesystem envelopes; -malformed marked envelopes and unsupported versions fail instead of falling back to pass-through behavior. An -unrecognized value does not require the explicit `SerDesContext` parameter to be non-null; a recognized filesystem -envelope does. - -Offloaded files are content-hashed and immutable. Every serialization uses a unique filename containing the entity -identity, content hash, and UUID, and publishes it with one `CREATE_NEW` write. Existing files are never overwritten, -and publication does not require hard links or renames, making it compatible with both EFS and S3 Files. A failed write -can leave only an unreferenced orphan rather than replacing data referenced by an earlier checkpoint. File envelopes -identify the producing execution and entity. Ordinary checkpoint replay must match that owner, while invoke input and -result boundaries may consume a file owned by the other Lambda execution when both functions use the same shared root -and path encoding. Treat file envelopes as capabilities. Content hashes are verified when reading, and symbolic-link -paths are rejected. - -Stages may follow `FileSystemSerDesStage` to transform its inline or file-reference envelope. The filesystem stage's -`OVERFLOW` and preview-size checks apply before those later transformations, so account for any expansion when staying -within the service checkpoint limit. During deserialization, every stage checks its own marker and returns -unrecognized input unchanged, so raw external payloads can traverse later and earlier stages without being decoded by -them. - -The cloud and local test runners always use a separate context-free value codec for the initial Lambda invocation -because an execution ARN is not available yet. By default, they use the configured persisted SerDes when it is a plain -value codec, or the root value codec when it is a `ComposableSerDes`; `withInputSerDes(...)` provides an explicit -override. The runners never use persisted pipeline stages to encode this boundary. Do not configure a -`ComposableSerDes` as the explicit input codec. When the runtime later deserializes the raw input, each persisted stage -passes it through unless its self-identifying format is present. A custom input codec must produce data that the -persisted pipeline's root value codec can decode. - -## Storage requirements - -Do not use Lambda's ephemeral `/tmp` directory. Durable replay may run in another execution environment where that file -does not exist. - -Use a durable shared mount such as EFS or S3 Files. The SDK does not rely on hard links or renames, which S3 Files does -not support. S3 Files can synchronize writes asynchronously, so a runtime crash before a flush can lose recent data; -use it only when that durability tradeoff is acceptable. - -The SDK does not delete payload files. Configure an appropriate retention or lifecycle policy for the backing storage. diff --git a/docs/advanced/serdes.md b/docs/advanced/serdes.md new file mode 100644 index 000000000..ab860567c --- /dev/null +++ b/docs/advanced/serdes.md @@ -0,0 +1,288 @@ +# Serialization and SerDes pipelines + +The SDK uses a `SerDes` to convert durable values between Java objects and the strings stored in checkpoints. The +default is `JacksonSerDes`, so most applications do not need to configure serialization. + +The core SDK also supports composable pipelines for transforming the serialized string before it is persisted. A +pipeline has one context-free value codec followed by zero or more reversible `SerDesStage` instances: + +```text +serialization: Object -> SerDes -> String -> stage 1 -> stage 2 -> String +deserialization: Object <- SerDes <- String <- stage 1 <- stage 2 <- String +``` + +Serialization runs stages in declaration order. Deserialization runs them in reverse order. + +## Value codecs + +`SerDes` is the object-to-string boundary: + +```java +public interface SerDes { + String serialize(Object value); + + T deserialize(String data, TypeToken typeToken); +} +``` + +The `TypeToken` preserves runtime type information needed to deserialize generic Java types. `JacksonSerDes` implements +this interface and can be constructed with a custom Jackson `ObjectMapper`: + +```java +var objectMapper = JsonMapper.builder() + .findAndAddModules() + .build(); + +var serDes = new JacksonSerDes(objectMapper); + +return DurableConfig.builder() + .withSerDes(serDes) + .build(); +``` + +Existing custom `SerDes` implementations remain valid. Pipeline behavior is opt-in: calling `then(...)` creates a +composable `SerDes`, while using a value codec by itself retains the existing object-to-string behavior. + +## Selecting a SerDes + +`DurableConfig.Builder.withSerDes(...)` sets the default SerDes for persisted values: + +```java +return DurableConfig.builder() + .withSerDes(new JacksonSerDes()) + .build(); +``` + +Operation configuration builders also support `serDes(...)` for overriding the complete SerDes used by that operation. +This is a replacement, not an additional stage appended to the global pipeline. `InvokeConfig` additionally supports +`payloadSerDes(...)` for the invoked function's payload; `serDes(...)` controls the invoke result. + +## Composable pipelines + +Every top-level pipeline stage implements `SerDesStage` and transforms a string: + +```java +public interface SerDesStage { + String serialize(String value, SerDesContext context); + + String deserialize(String data, SerDesContext context); +} +``` + +Append stages to a value codec with `SerDes.then(...)`: + +```java +SerDes serDes = new JacksonSerDes() + .then(compressionStage) + .then(encryptionStage) + .then(storageStage); +``` + +The returned value is a `SerDes`, so applications can configure or pass a pipeline anywhere a regular `SerDes` is +accepted. Additional `then(...)` calls continue the same immutable pipeline. + +Stages must use a self-identifying, normally versioned representation. During deserialization, a stage must: + +- reverse input that is valid and uses its format; +- throw an exception for input that identifies itself as the stage's format but is malformed or uses an unsupported + version; and +- return input unchanged when it does not use the stage's format. + +The pass-through rule allows raw invocation payloads, callback results, and standard Lambda invoke results to traverse +the pipeline and reach its value codec. It also allows stages to be added without making older checkpoint values +unreadable. Each stage owns the compatibility policy for its format; the pipeline does not add a shared outer +envelope. + +The pipeline short-circuits a `null` value at its boundary. A stage must not return `null` for non-null input. + +## Binary transformations + +Compression, encryption, and similar transformations are usually easier to implement on bytes. A +`BinarySerDesStage` transforms `byte[]` values: + +```java +public interface BinarySerDesStage { + byte[] serialize(byte[] value, SerDesContext context); + + byte[] deserialize(byte[] data, SerDesContext context); +} +``` + +Use `ComposableBinarySerDesStage` to expose several binary stages as one string-to-string `SerDesStage`: + +```java +var binaryStage = ComposableBinarySerDesStage.builder() + .startWith(Utf8StringBinaryCodec.INSTANCE) + .then(compressionBinaryStage) + .then(encryptionBinaryStage) + .endWith(Base64StringBinaryCodec.INSTANCE) + .build(); + +SerDes serDes = new JacksonSerDes() + .then(binaryStage); +``` + +The builder follows serialization processing order: + +1. `startWith(...)` converts the incoming string to bytes. +2. Each `then(...)` stage transforms those bytes. +3. `endWith(...)` converts the final bytes back to a string. + +Deserialization reverses that order. Both boundaries implement the same `StringBinaryCodec` interface and are +customizable. The core SDK includes UTF-8 and Base64 codecs. + +`ComposableBinarySerDesStage` adds a reserved, versioned frame to its output. It decodes recognized frames, rejects +malformed or unsupported frames, and passes unrecognized strings through unchanged. + +## Stage context + +The SDK passes a read-only `SerDesContext` explicitly to every `SerDesStage` and nested `BinarySerDesStage`. The +context describes the durable execution, entity, operation, attempt, and payload kind being processed. + +During serialization, `originalValue()` contains the object supplied to the root value codec. This lets a stage derive +metadata such as a preview from the original object even though its direct input is a string. During deserialization, +`originalValue()` is `null`. + +Root `SerDes` value codecs do not receive `SerDesContext`; they remain usable outside a durable execution. A context can +also be `null` when an application invokes a stage directly outside the SDK, so stages should document whether their +recognized format requires durable context. + +The SDK caches successful deserialization results for the current Lambda invocation in a bounded, weak-reference +cache. Concurrent requests for the same persisted value share one in-flight deserialization. Results are not cached +across invocations. + +## Retries and execution + +`RetrySerDesStage` wraps a `SerDesStage`, and `RetryBinarySerDesStage` wraps a `BinarySerDesStage`: + +```java +var resilientStorageStage = new RetrySerDesStage( + storageStage, + RetryStrategies.fixedDelay(3, Duration.ofSeconds(1))); +``` + +These wrappers retry only `RetryableSerDesException`. Permanent errors, such as malformed envelopes or codec failures, +are not retried. Retry delays consume time in the current Lambda invocation, so keep strategies short and bounded. + +SerDes runs inline on the calling thread by default, preserving the existing no-thread-pool behavior and avoiding a +thread hop for in-memory serialization. Blocking stages, including filesystem access and retry backoff, can use a +dedicated executor: + +```java +return DurableConfig.builder() + .withSerDes(serDes) + .withSerDesExecutorService(Executors.newFixedThreadPool(4)) + .build(); +``` + +The SerDes executor must be different from the user-operation executor to avoid deadlock when the operation pool is +saturated. + +## Initial invocation payloads in test runners + +`CloudDurableTestRunner` and `LocalDurableTestRunner` serialize the initial Lambda invocation with a separate, +context-free value codec because no durable execution context exists yet. They do not run the persisted pipeline's +stages at this boundary. + +By default, a runner uses the configured SerDes if it is a plain value codec, or the root value codec if it is a +`ComposableSerDes`. Use `withInputSerDes(...)` to select another input codec. The explicit input codec must not be a +`ComposableSerDes`, and it must produce a string that the persisted pipeline's root codec can deserialize. + +## Filesystem-backed payload storage + +`FileSystemSerDesStage` stores serialized payloads on a durable shared filesystem and leaves small, versioned +file-reference envelopes in checkpoints. It is included in the core `aws-durable-execution-sdk-java` artifact under +the `software.amazon.lambda.durable.serde.filesystem` Java package. + +Configure it after a value codec: + +```java +var fileSystemStage = FileSystemSerDesStage.builder(Path.of("/mnt/efs/durable-payloads")) + .storageMode(FileSystemStorageMode.OVERFLOW) + .pathEncoding(FileSystemPathEncoding.HASH) + .checkpointEnvelopeLimitBytes(configuredCheckpointEnvelopeLimitBytes) + .build(); + +var resilientFileSystemStage = new RetrySerDesStage( + fileSystemStage, + RetryStrategies.fixedDelay(3, Duration.ofSeconds(1))); + +SerDes serDes = new JacksonSerDes() + .then(resilientFileSystemStage); + +return DurableConfig.builder() + .withSerDes(serDes) + .withSerDesExecutorService(Executors.newFixedThreadPool(4)) + .build(); +``` + +The storage modes are: + +- `ALWAYS`: store every non-null payload in a file. +- `OVERFLOW`: keep payloads inline until the checkpoint envelope approaches the configured size limit. + +The path encodings are: + +- `URI`: use readable escaped path segments. +- `HASH`: use fixed-length SHA-256 path segments. + +`checkpointEnvelopeLimitBytes(...)` controls the maximum UTF-8 size accepted for both inline and file envelopes. It +defaults to 255 KiB and can be increased when the durable execution service supports a larger payload limit. + +### Structured previews + +File envelopes can include a structured preview: + +```java +var previewConfig = PreviewConfig.builder(PreviewMode.EXCLUDE_ALL) + .include(PreviewField.anywhere("id"), PreviewField.path("customer.status")) + .exclude(PreviewField.anywhere("internal")) + .mask(PreviewField.anywhere("email")) + .maskString("***") + .maxPreviewBytes(4096) + .build(); + +var fileSystemStage = FileSystemSerDesStage.builder(Path.of("/mnt/efs/durable-payloads")) + .previewConfig(previewConfig) + .build(); +``` + +`INCLUDE_ALL` starts with every leaf visible and then applies exclude and mask rules. `EXCLUDE_ALL` starts with no +fields visible; include and mask rules make selected fields visible. `PreviewField.anywhere(...)` matches a field name +at any depth, while `PreviewField.path(...)` matches an exact dot-separated path. Exclude rules win over mask rules, +and masking implies visibility. + +The built-in `previewConfig(...)` parses the incoming stage string as JSON. Use `previewGenerator(...)` for non-JSON +stage values or custom logic. The generator receives the stage string and `SerDesContext`, including +`originalValue()` during serialization. Custom generators must avoid exposing sensitive fields. + +Previews are included only in file envelopes. The structured builder defaults to a 4 KiB preview budget, and the +complete envelope must remain below the configured checkpoint limit. + +### Replay and envelope behavior + +Filesystem envelopes contain a reserved version marker. The stage passes strings without this marker through +unchanged. A string containing the marker must be a valid, supported filesystem envelope; malformed marked envelopes +and unsupported versions fail rather than falling back to pass-through behavior. + +Payload files are content-hashed and immutable. Each serialization publishes a unique filename with a single +`CREATE_NEW` write. Existing files are never overwritten, and publication does not require hard links or renames. +The stage verifies content hashes, rejects symbolic-link paths, and validates that ordinary checkpoint replay matches +the execution and entity that produced the reference. Invoke input and result boundaries can consume a file owned by +the other Lambda execution when both functions use the same shared root and path encoding. + +Stages may follow `FileSystemSerDesStage` to transform its inline or file-reference envelope. `OVERFLOW` and preview +size checks occur before those later stages, so account for any expansion when staying within the service checkpoint +limit. + +### Storage requirements + +Do not use Lambda's ephemeral `/tmp` directory. Durable replay may run in another execution environment where the file +does not exist. + +Use a durable shared mount such as EFS or S3 Files. The stage does not rely on hard links or renames, which S3 Files +does not support. S3 Files can synchronize writes asynchronously, so a runtime crash before a flush can lose recent +data; use it only when that durability tradeoff is acceptable. + +The SDK does not delete payload files. Configure an appropriate retention or lifecycle policy for the backing +storage. From c9d4390dc22c7b853fbe09541bc18e29629e5aa9 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Tue, 25 Aug 2026 23:48:13 +0000 Subject: [PATCH 46/56] fix: address remaining SerDes review comments --- .github/workflows/e2e-tests.yml | 14 +++++--- examples/README.md | 6 ++++ .../examples/CloudBasedIntegrationTest.java | 2 ++ .../filesystem/FileSystemSerDesStage.java | 2 ++ .../operation/CallbackOperationTest.java | 32 +++++++++++++++++++ .../operation/InvokeOperationTest.java | 30 +++++++++++++++++ .../filesystem/FileSystemSerDesStageTest.java | 27 ++++++++++++++++ 7 files changed, 109 insertions(+), 4 deletions(-) diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index 16335bd40..38702faff 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -37,6 +37,7 @@ jobs: env: AWS_REGION: us-west-2 E2E_TEST_PARALLELISM: 4 + FILESYSTEM_SERDES_E2E_ENABLED: ${{ vars.FILESYSTEM_SERDES_E2E_ENABLED == 'true' }} runs-on: ubuntu-latest strategy: fail-fast: false @@ -69,9 +70,11 @@ jobs: - name: Build locally run: mvn -B -q -Dmaven.test.skip=true install --file pom.xml - name: Generate SAM template - run: | - python3 generate-template.py - python3 generate-template.py --file-system-only --output filesystem-template.yaml + run: python3 generate-template.py + working-directory: ./examples + - name: Generate filesystem SerDes E2E SAM template + if: env.FILESYSTEM_SERDES_E2E_ENABLED == 'true' + run: python3 generate-template.py --file-system-only --output filesystem-template.yaml working-directory: ./examples - name: sam build env: @@ -81,6 +84,7 @@ jobs: 'ParameterKey=Architecture,ParameterValue=x86_64 ParameterKey=JavaVersion,ParameterValue=java${{ matrix.java }} ParameterKey=FunctionNamePrefix,ParameterValue=Java${{ matrix.java }}- ParameterKey=RoleArn,ParameterValue=${{ secrets.TEST_LAMBDA_EXECUTION_ROLE_ARN }}' working-directory: ./examples - name: sam build filesystem SerDes E2E stack + if: env.FILESYSTEM_SERDES_E2E_ENABLED == 'true' env: MAVEN_OPTS: -DskipTests=true -Dmaven.test.skip=true run: | @@ -100,6 +104,7 @@ jobs: 'ParameterKey=Architecture,ParameterValue=x86_64 ParameterKey=JavaVersion,ParameterValue=java${{ matrix.java }} ParameterKey=FunctionNamePrefix,ParameterValue=Java${{ matrix.java }}- ParameterKey=RoleArn,ParameterValue=${{ secrets.TEST_LAMBDA_EXECUTION_ROLE_ARN }}' working-directory: ./examples - name: sam deploy filesystem SerDes E2E stack + if: env.FILESYSTEM_SERDES_E2E_ENABLED == 'true' run: | sam deploy --template-file .aws-sam-filesystem/template.yaml \ --stack-name Java${{ matrix.java }}-JavaSDKFileSystemSerDesE2EStack \ @@ -110,6 +115,7 @@ jobs: run: | mvn clean test -B \ -Dtest.cloud.enabled=true \ + -Dtest.filesystem.enabled='${{ env.FILESYSTEM_SERDES_E2E_ENABLED }}' \ -Dtest.aws.account='${{ secrets.TEST_ACCOUNT_ID }}' \ -Dtest=CloudBasedIntegrationTest \ -Dtest.function.name.prefix='Java${{ matrix.java }}-' \ @@ -120,7 +126,7 @@ jobs: -Djunit.jupiter.execution.parallel.config.fixed.parallelism=${{ env.E2E_TEST_PARALLELISM }} working-directory: ./examples - name: Delete filesystem SerDes E2E stack - if: always() + if: ${{ always() && env.FILESYSTEM_SERDES_E2E_ENABLED == 'true' }} run: | if aws cloudformation describe-stacks \ --stack-name Java${{ matrix.java }}-JavaSDKFileSystemSerDesE2EStack >/dev/null 2>&1; then diff --git a/examples/README.md b/examples/README.md index c17750835..c923329be 100644 --- a/examples/README.md +++ b/examples/README.md @@ -77,6 +77,12 @@ mvn test -Dtest=CloudBasedIntegrationTest \ -Dtest.aws.region=us-east-1 ``` +The filesystem SerDes cloud test is disabled by default because it requires a separate VPC and EFS-backed stack. +Generate and deploy that stack with `generate-template.py --file-system-only`, then include +`-Dtest.filesystem.enabled=true` when running `CloudBasedIntegrationTest`. In GitHub Actions, set the repository +variable `FILESYSTEM_SERDES_E2E_ENABLED` to `true` after the test role has permission to manage the required EC2 and +EFS resources. + ## Examples | Example | Description | diff --git a/examples/src/test/java/software/amazon/lambda/durable/examples/CloudBasedIntegrationTest.java b/examples/src/test/java/software/amazon/lambda/durable/examples/CloudBasedIntegrationTest.java index 619e72fbd..f9052d78e 100644 --- a/examples/src/test/java/software/amazon/lambda/durable/examples/CloudBasedIntegrationTest.java +++ b/examples/src/test/java/software/amazon/lambda/durable/examples/CloudBasedIntegrationTest.java @@ -20,6 +20,7 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.condition.EnabledForJreRange; import org.junit.jupiter.api.condition.EnabledIf; +import org.junit.jupiter.api.condition.EnabledIfSystemProperty; import org.junit.jupiter.api.condition.JRE; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.CsvSource; @@ -374,6 +375,7 @@ void testCustomConfigExample() { } @Test + @EnabledIfSystemProperty(named = "test.filesystem.enabled", matches = "true") void testFileSystemSerDesExample() throws Exception { var value = "filesystem-e2e-".repeat(24 * 1024); var input = new FileSystemSerDesExample.Input("payload-1", value); diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/filesystem/FileSystemSerDesStage.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/filesystem/FileSystemSerDesStage.java index db336b66c..b31a1e889 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/filesystem/FileSystemSerDesStage.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/filesystem/FileSystemSerDesStage.java @@ -405,6 +405,8 @@ private Map generatePreview(String value, SerDesContext context) } try { return previewGenerator.apply(value, context); + } catch (RetryableSerDesException e) { + throw e; } catch (RuntimeException e) { throw new SerDesException( "Failed to generate filesystem payload preview for entity '" + context.entityId() + "'", e); diff --git a/sdk/src/test/java/software/amazon/lambda/durable/operation/CallbackOperationTest.java b/sdk/src/test/java/software/amazon/lambda/durable/operation/CallbackOperationTest.java index b1f23d30d..b772ee4ad 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/operation/CallbackOperationTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/operation/CallbackOperationTest.java @@ -223,6 +223,38 @@ void getThrowsCallbackExceptionWhenFailed() { assertTrue(exception.getMessage().contains("ValidationError")); } + @Test + void getThrowsCallbackFailedExceptionWhenErrorTypeIsMissing() { + var existingCallback = Operation.builder() + .id(OPERATION_ID) + .name(OPERATION_NAME) + .type(OperationType.CALLBACK) + .subType(OperationSubType.CALLBACK.getValue()) + .status(OperationStatus.FAILED) + .callbackDetails(CallbackDetails.builder() + .callbackId("callback-id") + .error(ErrorObject.builder() + .errorMessage("untyped callback failure") + .errorData("not-json") + .build()) + .build()) + .build(); + var executionManager = createExecutionManager(List.of(existingCallback)); + when(durableContext.getExecutionManager()).thenReturn(executionManager); + + var operation = new CallbackOperation<>( + OPERATION_IDENTIFIER, + TypeToken.get(String.class), + CallbackConfig.builder().serDes(new JacksonSerDes()).build(), + durableContext); + operation.execute(); + + var exception = assertThrows(CallbackFailedException.class, operation::get); + assertEquals("untyped callback failure", exception.getMessage()); + assertNull(exception.getErrorObject().errorType()); + assertNull(exception.deserializedError()); + } + @Test void getThrowsCallbackTimeoutExceptionWhenTimedOut() { var existingCallback = Operation.builder() diff --git a/sdk/src/test/java/software/amazon/lambda/durable/operation/InvokeOperationTest.java b/sdk/src/test/java/software/amazon/lambda/durable/operation/InvokeOperationTest.java index 14dd8e71c..c3a1b2dfc 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/operation/InvokeOperationTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/operation/InvokeOperationTest.java @@ -143,6 +143,36 @@ void getInvokeFailedExceptionWhenInvocationDetailsAreMissing() { assertNull(exception.deserializedError()); } + @Test + void getInvokeFailedExceptionWhenErrorTypeIsMissing() { + var op = Operation.builder() + .id(OPERATION_ID) + .name(OPERATION_NAME) + .status(OperationStatus.FAILED) + .chainedInvokeDetails(ChainedInvokeDetails.builder() + .error(ErrorObject.builder() + .errorMessage("untyped failure") + .errorData("not-json") + .build()) + .build()) + .build(); + when(executionManager.getOperationAndUpdateReplayState(OPERATION_ID)).thenReturn(op); + + var operation = new InvokeOperation<>( + OPERATION_IDENTIFIER, + "test-function", + "{}", + TypeToken.get(String.class), + InvokeConfig.builder().serDes(new JacksonSerDes()).build(), + durableContext); + operation.onCheckpointComplete(op); + + var exception = assertThrows(InvokeFailedException.class, operation::get); + assertEquals("untyped failure", exception.getMessage()); + assertNull(exception.getErrorObject().errorType()); + assertNull(exception.deserializedError()); + } + @Test void getInvokeTimedOutExceptionWhenInvocationTimedOut() { var op = Operation.builder() diff --git a/sdk/src/test/java/software/amazon/lambda/durable/serde/filesystem/FileSystemSerDesStageTest.java b/sdk/src/test/java/software/amazon/lambda/durable/serde/filesystem/FileSystemSerDesStageTest.java index 590a6c34d..1fe44a965 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/serde/filesystem/FileSystemSerDesStageTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/serde/filesystem/FileSystemSerDesStageTest.java @@ -19,11 +19,13 @@ import java.nio.file.Files; import java.nio.file.Path; import java.security.MessageDigest; +import java.time.Duration; import java.util.Arrays; import java.util.Base64; import java.util.HexFormat; import java.util.List; import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -32,6 +34,7 @@ import software.amazon.lambda.durable.exception.RetryableSerDesException; import software.amazon.lambda.durable.exception.SerDesException; import software.amazon.lambda.durable.model.OperationSubType; +import software.amazon.lambda.durable.retry.RetryDecision; import software.amazon.lambda.durable.retry.RetryStrategies; import software.amazon.lambda.durable.serde.Base64StringBinaryCodec; import software.amazon.lambda.durable.serde.BinarySerDesStage; @@ -313,6 +316,30 @@ void includesBoundedPreviewWithoutChangingStoredPayload() throws Exception { assertCauseMessage(failure, "checkpoint payload limit"); } + @Test + void retryablePreviewFailureCanBeRetried() throws Exception { + var attempts = new AtomicInteger(); + var fileSystemStage = FileSystemSerDesStage.builder(basePath) + .previewGenerator((value, context) -> { + if (attempts.incrementAndGet() == 1) { + throw new RetryableSerDesException("preview service unavailable"); + } + return Map.of("summary", "order"); + }) + .build(); + var pipeline = stringCodec() + .then(new RetrySerDesStage( + fileSystemStage, + (failure, attempt) -> + attempt == 1 ? RetryDecision.retry(Duration.ZERO) : RetryDecision.fail())); + + var envelope = new SerDesRunner(null).serialize(pipeline, "value", context()); + + assertEquals(2, attempts.get()); + assertEquals( + "order", MAPPER.readTree(envelope).get("preview").get("summary").textValue()); + } + @Test void structuredPreviewConfigSelectsAndMasksJsonFields() throws Exception { var stage = FileSystemSerDesStage.builder(basePath) From 97f0e10a439cd36d21b4b521d55b9a338da241cb Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Tue, 25 Aug 2026 23:57:53 +0000 Subject: [PATCH 47/56] test: enable filesystem SerDes E2E --- .github/workflows/e2e-tests.yml | 8 ++------ examples/README.md | 9 ++++----- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index 38702faff..f07582636 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -37,7 +37,6 @@ jobs: env: AWS_REGION: us-west-2 E2E_TEST_PARALLELISM: 4 - FILESYSTEM_SERDES_E2E_ENABLED: ${{ vars.FILESYSTEM_SERDES_E2E_ENABLED == 'true' }} runs-on: ubuntu-latest strategy: fail-fast: false @@ -73,7 +72,6 @@ jobs: run: python3 generate-template.py working-directory: ./examples - name: Generate filesystem SerDes E2E SAM template - if: env.FILESYSTEM_SERDES_E2E_ENABLED == 'true' run: python3 generate-template.py --file-system-only --output filesystem-template.yaml working-directory: ./examples - name: sam build @@ -84,7 +82,6 @@ jobs: 'ParameterKey=Architecture,ParameterValue=x86_64 ParameterKey=JavaVersion,ParameterValue=java${{ matrix.java }} ParameterKey=FunctionNamePrefix,ParameterValue=Java${{ matrix.java }}- ParameterKey=RoleArn,ParameterValue=${{ secrets.TEST_LAMBDA_EXECUTION_ROLE_ARN }}' working-directory: ./examples - name: sam build filesystem SerDes E2E stack - if: env.FILESYSTEM_SERDES_E2E_ENABLED == 'true' env: MAVEN_OPTS: -DskipTests=true -Dmaven.test.skip=true run: | @@ -104,7 +101,6 @@ jobs: 'ParameterKey=Architecture,ParameterValue=x86_64 ParameterKey=JavaVersion,ParameterValue=java${{ matrix.java }} ParameterKey=FunctionNamePrefix,ParameterValue=Java${{ matrix.java }}- ParameterKey=RoleArn,ParameterValue=${{ secrets.TEST_LAMBDA_EXECUTION_ROLE_ARN }}' working-directory: ./examples - name: sam deploy filesystem SerDes E2E stack - if: env.FILESYSTEM_SERDES_E2E_ENABLED == 'true' run: | sam deploy --template-file .aws-sam-filesystem/template.yaml \ --stack-name Java${{ matrix.java }}-JavaSDKFileSystemSerDesE2EStack \ @@ -115,7 +111,7 @@ jobs: run: | mvn clean test -B \ -Dtest.cloud.enabled=true \ - -Dtest.filesystem.enabled='${{ env.FILESYSTEM_SERDES_E2E_ENABLED }}' \ + -Dtest.filesystem.enabled=true \ -Dtest.aws.account='${{ secrets.TEST_ACCOUNT_ID }}' \ -Dtest=CloudBasedIntegrationTest \ -Dtest.function.name.prefix='Java${{ matrix.java }}-' \ @@ -126,7 +122,7 @@ jobs: -Djunit.jupiter.execution.parallel.config.fixed.parallelism=${{ env.E2E_TEST_PARALLELISM }} working-directory: ./examples - name: Delete filesystem SerDes E2E stack - if: ${{ always() && env.FILESYSTEM_SERDES_E2E_ENABLED == 'true' }} + if: always() run: | if aws cloudformation describe-stacks \ --stack-name Java${{ matrix.java }}-JavaSDKFileSystemSerDesE2EStack >/dev/null 2>&1; then diff --git a/examples/README.md b/examples/README.md index c923329be..9dd5d5d48 100644 --- a/examples/README.md +++ b/examples/README.md @@ -77,11 +77,10 @@ mvn test -Dtest=CloudBasedIntegrationTest \ -Dtest.aws.region=us-east-1 ``` -The filesystem SerDes cloud test is disabled by default because it requires a separate VPC and EFS-backed stack. -Generate and deploy that stack with `generate-template.py --file-system-only`, then include -`-Dtest.filesystem.enabled=true` when running `CloudBasedIntegrationTest`. In GitHub Actions, set the repository -variable `FILESYSTEM_SERDES_E2E_ENABLED` to `true` after the test role has permission to manage the required EC2 and -EFS resources. +For manually run cloud tests, the filesystem SerDes test is disabled by default because it requires a separate VPC +and EFS-backed stack. Generate and deploy that stack with `generate-template.py --file-system-only`, then include +`-Dtest.filesystem.enabled=true` when running `CloudBasedIntegrationTest`. GitHub Actions provisions this stack and +runs the filesystem test in every E2E matrix job. ## Examples From 6a813eb0bde61a26877b0fdf7c0675da85966d9c Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Wed, 26 Aug 2026 00:04:19 +0000 Subject: [PATCH 48/56] feat: verify filesystem payload digests --- docs/adr/005-filesystem-serdes.md | 16 ++- docs/advanced/serdes.md | 8 +- .../examples/CloudBasedIntegrationTest.java | 1 + .../filesystem/FileSystemSerDesStage.java | 79 ++++++++++---- .../filesystem/FileSystemSerDesStageTest.java | 101 +++++++++++++++++- 5 files changed, 173 insertions(+), 32 deletions(-) diff --git a/docs/adr/005-filesystem-serdes.md b/docs/adr/005-filesystem-serdes.md index a4fd219cc..a9999f01a 100644 --- a/docs/adr/005-filesystem-serdes.md +++ b/docs/adr/005-filesystem-serdes.md @@ -2,8 +2,9 @@ **Status:** Accepted — Approach A with ComposableSerDes pipeline **Date:** 2026-07-02 -**Updated:** 2026-08-25 — Included FileSystemSerDesStage in core, made every post-codec pipeline component an explicit -string stage, and documented the rejected binary-only top-level pipeline. +**Updated:** 2026-08-26 — Included FileSystemSerDesStage in core, made every post-codec pipeline component an explicit +string stage, documented the rejected binary-only top-level pipeline, and added envelope payload digests for integrity +verification. ## Context @@ -399,9 +400,9 @@ Path encodings: Envelope format: ```json -{"__durable_execution_filesystem_serdes":1,"data":"","payloadType":"STRING","ownerDurableExecutionArn":"","ownerEntityId":""} -{"__durable_execution_filesystem_serdes":1,"file":"","payloadType":"STRING","ownerDurableExecutionArn":"","ownerEntityId":""} -{"__durable_execution_filesystem_serdes":1,"file":"","payloadType":"STRING","ownerDurableExecutionArn":"","ownerEntityId":"","preview":{ "...": "..." }} +{"__durable_execution_filesystem_serdes":1,"data":"","payloadType":"STRING","payloadDigest":"","ownerDurableExecutionArn":"","ownerEntityId":""} +{"__durable_execution_filesystem_serdes":1,"file":"","payloadType":"STRING","payloadDigest":"","ownerDurableExecutionArn":"","ownerEntityId":""} +{"__durable_execution_filesystem_serdes":1,"file":"","payloadType":"STRING","payloadDigest":"","ownerDurableExecutionArn":"","ownerEntityId":"","preview":{ "...": "..." }} ``` `FileSystemSerDesStage` must reject recognized filesystem operations when its `SerDesContext` parameter is `null` or does @@ -420,6 +421,11 @@ Offloaded filenames include the entity identity, content hash, and a UUID. Each file with one `CREATE_NEW` write. It does not require hard links or renames, making the write path compatible with S3 Files as well as EFS. Serializing new state never replaces a file referenced by an earlier checkpoint. +Every envelope includes the SHA-256 digest of the serialized payload. Deserialization verifies inline data and loaded +file bytes against that digest. File references must additionally use a content-addressed filename consistent with the +envelope digest, so changing both a file's contents and its filename cannot bypass integrity validation without also +changing the checkpoint envelope. + File envelopes identify the execution ARN and entity that produced the content. Normal checkpoint replay requires that owner to match the current context. Initial input and chained-invoke result boundaries may consume a reference owned by the other Lambda execution, allowing two functions configured with the same durable filesystem root and path diff --git a/docs/advanced/serdes.md b/docs/advanced/serdes.md index ab860567c..d3621d182 100644 --- a/docs/advanced/serdes.md +++ b/docs/advanced/serdes.md @@ -267,9 +267,11 @@ and unsupported versions fail rather than falling back to pass-through behavior. Payload files are content-hashed and immutable. Each serialization publishes a unique filename with a single `CREATE_NEW` write. Existing files are never overwritten, and publication does not require hard links or renames. -The stage verifies content hashes, rejects symbolic-link paths, and validates that ordinary checkpoint replay matches -the execution and entity that produced the reference. Invoke input and result boundaries can consume a file owned by -the other Lambda execution when both functions use the same shared root and path encoding. +Every inline and file envelope records the payload's SHA-256 digest. During deserialization, the stage verifies the +restored bytes against that envelope digest; file payloads must also have a content-addressed filename consistent with +the digest. The stage rejects symbolic-link paths and validates that ordinary checkpoint replay matches the execution +and entity that produced the reference. Invoke input and result boundaries can consume a file owned by the other +Lambda execution when both functions use the same shared root and path encoding. Stages may follow `FileSystemSerDesStage` to transform its inline or file-reference envelope. `OVERFLOW` and preview size checks occur before those later stages, so account for any expansion when staying within the service checkpoint diff --git a/examples/src/test/java/software/amazon/lambda/durable/examples/CloudBasedIntegrationTest.java b/examples/src/test/java/software/amazon/lambda/durable/examples/CloudBasedIntegrationTest.java index f9052d78e..05a96d1db 100644 --- a/examples/src/test/java/software/amazon/lambda/durable/examples/CloudBasedIntegrationTest.java +++ b/examples/src/test/java/software/amazon/lambda/durable/examples/CloudBasedIntegrationTest.java @@ -907,6 +907,7 @@ private static JsonNode assertFileSystemEnvelope(String value) throws Exception var envelope = MAPPER.readTree(value); assertEquals(1, envelope.get(FILE_SYSTEM_ENVELOPE_MARKER).intValue()); assertEquals("STRING", envelope.get("payloadType").textValue()); + assertTrue(envelope.get("payloadDigest").textValue().matches("[0-9a-f]{64}")); assertTrue(envelope.get("file").textValue().startsWith("/mnt/efs/durable-payloads/")); return envelope; } diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/filesystem/FileSystemSerDesStage.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/filesystem/FileSystemSerDesStage.java index b31a1e889..505ca803a 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/filesystem/FileSystemSerDesStage.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/filesystem/FileSystemSerDesStage.java @@ -41,12 +41,16 @@ *

Payload files are immutable and created with a single {@code CREATE_NEW} write. Publication does not require hard * links or renames, so the write path is compatible with S3 Files. * + *

Every filesystem envelope includes a SHA-256 payload digest. Deserialization verifies inline values and file + * contents against that digest, and file paths must contain the same digest. + * *

Deserialization recognizes the reserved filesystem envelope marker. Input without that marker is returned * unchanged; input with the marker must be a valid supported envelope. Filesystem operations use the explicit * {@link SerDesContext} stage parameter for durable payload identity. */ public final class FileSystemSerDesStage implements SerDesStage { private static final String ENVELOPE_MARKER = "__durable_execution_filesystem_serdes"; + private static final String PAYLOAD_DIGEST_FIELD = "payloadDigest"; private static final String PAYLOAD_TYPE_FIELD = "payloadType"; private static final int ENVELOPE_VERSION = 1; private static final int DEFAULT_CHECKPOINT_ENVELOPE_LIMIT_BYTES = 256 * 1024 - 1024; @@ -55,6 +59,7 @@ public final class FileSystemSerDesStage implements SerDesStage { ENVELOPE_MAPPER.reader().with(DeserializationFeature.FAIL_ON_TRAILING_TOKENS); private static final Pattern DURABLE_EXECUTION_ARN_PATTERN = Pattern.compile( "^arn:[^:]*:lambda:[^:]*:[^:]*:function:([^:/]+):[^:/]+/durable-execution/([^/]+)/([^/]+)$"); + private static final Pattern SHA_256_DIGEST_PATTERN = Pattern.compile("[0-9a-f]{64}"); private final Path basePath; private final FileSystemStorageMode storageMode; @@ -90,16 +95,17 @@ public String serialize(String value, SerDesContext context) { } context = requireContext(context); var payload = SerializedPayload.fromString(value); + var payloadDigest = sha256(payload.data()); if (storageMode == FileSystemStorageMode.OVERFLOW) { - var inlineEnvelope = encodeEnvelope(payload, null, null, context); + var inlineEnvelope = encodeEnvelope(payload, payloadDigest, null, null, context); if (fitsCheckpoint(inlineEnvelope)) { return inlineEnvelope; } } - var file = resolvePayloadPath(payload, context); + var file = resolvePayloadPath(payloadDigest, context); var preview = generatePreview(value, context); - var fileEnvelope = encodeEnvelope(payload.withoutData(), file, preview, context); + var fileEnvelope = encodeEnvelope(payload.withoutData(), payloadDigest, file, preview, context); if (!fitsCheckpoint(fileEnvelope)) { throw new SerDesException("Filesystem SerDes envelope exceeds the checkpoint payload limit for entity '" + context.entityId() @@ -151,24 +157,34 @@ private String resolveSerializedPayload(String data, SerDesContext context) { var hasData = envelope.has("data") && envelope.get("data").isTextual(); var hasFile = envelope.has("file") && envelope.get("file").isTextual(); var payloadType = payloadType(envelope, context); + var payloadDigest = payloadDigest(envelope, context); var owner = payloadOwner(envelope, context); if (hasData) { try { - return SerializedPayload.fromInlineValue( - payloadType, envelope.get("data").textValue()) - .value(); + var payload = SerializedPayload.fromInlineValue( + payloadType, envelope.get("data").textValue()); + verifyPayloadDigest(payload, payloadDigest, context); + return payload.value(); } catch (IllegalArgumentException e) { throw malformedEnvelope(context, e); } } - return readPayload(envelope.get("file").textValue(), payloadType, owner, context) + return readPayload(envelope.get("file").textValue(), payloadType, payloadDigest, owner, context) .value(); } private SerializedPayload readPayload( - String fileValue, PayloadType payloadType, PayloadOwner owner, SerDesContext context) { + String fileValue, + PayloadType payloadType, + String payloadDigest, + PayloadOwner owner, + SerDesContext context) { var file = basePath.getFileSystem().getPath(fileValue).toAbsolutePath().normalize(); validatePayloadPath(file, owner); + var expectedFileName = payloadFileName(payloadDigest, owner.entityId()); + if (!matchesPublishedPayloadFileName(file.getFileName().toString(), expectedFileName)) { + throw new SerDesException("Filesystem SerDes file path does not match its payload digest"); + } try { var realBasePath = validateBasePath(false); rejectSymbolicLinks(file); @@ -180,10 +196,7 @@ private SerializedPayload readPayload( throw new SerDesException("Filesystem SerDes file does not resolve to the expected payload path"); } var serialized = new SerializedPayload(payloadType, Files.readAllBytes(realFile)); - var expectedFileName = payloadFileName(serialized, owner.entityId()); - if (!matchesPublishedPayloadFileName(realFile.getFileName().toString(), expectedFileName)) { - throw new SerDesException("Filesystem SerDes file content hash does not match its path"); - } + verifyPayloadDigest(serialized, payloadDigest, context); return serialized; } catch (IOException e) { throw new RetryableSerDesException( @@ -216,6 +229,24 @@ private static PayloadType payloadType(JsonNode envelope, SerDesContext context) } } + private static String payloadDigest(JsonNode envelope, SerDesContext context) { + var node = envelope.get(PAYLOAD_DIGEST_FIELD); + if (node == null + || !node.isTextual() + || !SHA_256_DIGEST_PATTERN.matcher(node.textValue()).matches()) { + throw malformedEnvelope(context, null); + } + return node.textValue(); + } + + private static void verifyPayloadDigest(SerializedPayload payload, String expectedDigest, SerDesContext context) { + if (!sha256(payload.data()).equals(expectedDigest)) { + throw new SerDesException("Filesystem SerDes payload digest does not match stored content for entity '" + + context.entityId() + + "'"); + } + } + private static PayloadOwner payloadOwner(JsonNode envelope, SerDesContext context) { var hasOwnerArn = envelope.has("ownerDurableExecutionArn") && envelope.get("ownerDurableExecutionArn").isTextual(); @@ -271,7 +302,12 @@ private static boolean isFilesystemEnvelope(JsonNode envelope) { || envelope.get("ownerEntityId").textValue().isBlank() || !envelope.has(PAYLOAD_TYPE_FIELD) || !envelope.get(PAYLOAD_TYPE_FIELD).isTextual() - || !isPayloadType(envelope.get(PAYLOAD_TYPE_FIELD).textValue())) { + || !isPayloadType(envelope.get(PAYLOAD_TYPE_FIELD).textValue()) + || !envelope.has(PAYLOAD_DIGEST_FIELD) + || !envelope.get(PAYLOAD_DIGEST_FIELD).isTextual() + || !SHA_256_DIGEST_PATTERN + .matcher(envelope.get(PAYLOAD_DIGEST_FIELD).textValue()) + .matches()) { return false; } @@ -285,7 +321,7 @@ private static boolean isFilesystemEnvelope(JsonNode envelope) { if (hasPreview && (hasData || !envelope.get("preview").isObject())) { return false; } - return envelope.size() == (hasPreview ? 6 : 5); + return envelope.size() == (hasPreview ? 7 : 6); } private static boolean isPayloadType(String value) { @@ -377,12 +413,17 @@ private static SerDesException unsupportedEnvelopeVersion(SerDesContext context, } private String encodeEnvelope( - SerializedPayload payload, Path file, Map preview, SerDesContext context) { + SerializedPayload payload, + String payloadDigest, + Path file, + Map preview, + SerDesContext context) { var envelope = new LinkedHashMap(); envelope.put(ENVELOPE_MARKER, ENVELOPE_VERSION); envelope.put("ownerDurableExecutionArn", context.durableExecutionArn()); envelope.put("ownerEntityId", context.entityId()); envelope.put(PAYLOAD_TYPE_FIELD, payload.type().name()); + envelope.put(PAYLOAD_DIGEST_FIELD, payloadDigest); if (payload.hasData()) { envelope.put("data", payload.inlineValue()); } else { @@ -429,9 +470,9 @@ private SerDesContext requireContext(SerDesContext context) { return context; } - private Path resolvePayloadPath(SerializedPayload payload, SerDesContext context) { + private Path resolvePayloadPath(String payloadDigest, SerDesContext context) { var directory = resolveExecutionDirectory(context.durableExecutionArn()); - var deterministicName = payloadFileName(payload, context.entityId()); + var deterministicName = payloadFileName(payloadDigest, context.entityId()); var suffix = ".payload"; var fileName = deterministicName.substring(0, deterministicName.length() - suffix.length()) + "-" @@ -444,8 +485,8 @@ private Path resolvePayloadPath(SerializedPayload payload, SerDesContext context return file; } - private String payloadFileName(SerializedPayload payload, String entityId) { - return encode(entityId) + "-" + sha256(payload.data()) + ".payload"; + private String payloadFileName(String payloadDigest, String entityId) { + return encode(entityId) + "-" + payloadDigest + ".payload"; } private void writePayload(SerializedPayload payload, Path file) throws IOException { diff --git a/sdk/src/test/java/software/amazon/lambda/durable/serde/filesystem/FileSystemSerDesStageTest.java b/sdk/src/test/java/software/amazon/lambda/durable/serde/filesystem/FileSystemSerDesStageTest.java index 1fe44a965..f6a212a87 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/serde/filesystem/FileSystemSerDesStageTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/serde/filesystem/FileSystemSerDesStageTest.java @@ -69,6 +69,9 @@ void writesValueCodecPayloadAndReplaysIt() throws Exception { assertEquals(1, json.get(ENVELOPE_MARKER).intValue()); assertEquals("STRING", json.get("payloadType").textValue()); + assertEquals( + sha256("{\"id\":42}".getBytes(StandardCharsets.UTF_8)), + json.get("payloadDigest").textValue()); assertTrue(file.startsWith(basePath.resolve("orders/execution-1/invocation-1"))); assertEquals("{\"id\":42}", Files.readString(file)); assertEquals( @@ -222,6 +225,81 @@ void repeatedPayloadsUseDistinctImmutableFiles() throws Exception { assertEquals("expected", Files.readString(secondFile)); } + @Test + void includesPayloadDigestAndVerifiesFileIntegrity() throws Exception { + var serDes = stringCodec().then(FileSystemSerDesStage.builder(basePath).build()); + var runner = new SerDesRunner(null); + var envelope = runner.serialize(serDes, "expected", context()); + var json = (ObjectNode) MAPPER.readTree(envelope); + var expectedDigest = sha256("expected".getBytes(StandardCharsets.UTF_8)); + + assertEquals(expectedDigest, json.get("payloadDigest").textValue()); + + var tampered = "tampered".getBytes(StandardCharsets.UTF_8); + Files.write(payloadFile(envelope), tampered); + + var failure = assertThrows( + SerDesException.class, + () -> runner.deserialize(serDes, json.toString(), TypeToken.get(String.class), context())); + assertCauseMessage(failure, "payload digest does not match stored content"); + } + + @Test + void verifiesFilePathUsesEnvelopePayloadDigest() throws Exception { + var serDes = stringCodec().then(FileSystemSerDesStage.builder(basePath).build()); + var runner = new SerDesRunner(null); + var envelope = runner.serialize(serDes, "expected", context()); + var json = (ObjectNode) MAPPER.readTree(envelope); + var tampered = "tampered".getBytes(StandardCharsets.UTF_8); + var tamperedFile = contentHashedPath(payloadFile(envelope), tampered); + Files.write(tamperedFile, tampered); + json.put("file", tamperedFile.toString()); + + var failure = assertThrows( + SerDesException.class, + () -> runner.deserialize(serDes, json.toString(), TypeToken.get(String.class), context())); + assertCauseMessage(failure, "file path does not match its payload digest"); + } + + @Test + void includesPayloadDigestAndVerifiesInlineIntegrity() throws Exception { + var serDes = stringCodec() + .then(FileSystemSerDesStage.builder(basePath) + .storageMode(FileSystemStorageMode.OVERFLOW) + .build()); + var runner = new SerDesRunner(null); + var envelope = (ObjectNode) MAPPER.readTree(runner.serialize(serDes, "expected", context())); + + assertEquals( + sha256("expected".getBytes(StandardCharsets.UTF_8)), + envelope.get("payloadDigest").textValue()); + + envelope.put("data", "tampered"); + var failure = assertThrows( + SerDesException.class, + () -> runner.deserialize(serDes, envelope.toString(), TypeToken.get(String.class), context())); + assertCauseMessage(failure, "payload digest does not match stored content"); + } + + @Test + void rejectsMissingOrMalformedPayloadDigest() throws Exception { + var serDes = stringCodec().then(FileSystemSerDesStage.builder(basePath).build()); + var runner = new SerDesRunner(null); + var envelope = (ObjectNode) MAPPER.readTree(runner.serialize(serDes, "expected", context())); + + envelope.remove("payloadDigest"); + var missingFailure = assertThrows( + SerDesException.class, + () -> runner.deserialize(serDes, envelope.toString(), TypeToken.get(String.class), context())); + assertCauseMessage(missingFailure, "Invalid filesystem SerDes envelope"); + + envelope.put("payloadDigest", "not-a-sha-256-digest"); + var malformedFailure = assertThrows( + SerDesException.class, + () -> runner.deserialize(serDes, envelope.toString(), TypeToken.get(String.class), context())); + assertCauseMessage(malformedFailure, "Invalid filesystem SerDes envelope"); + } + @Test void writesOnFileSystemsWithoutHardLinkSupport() throws Exception { var archive = basePath.resolve("payloads.zip"); @@ -261,6 +339,7 @@ void rejectsMalformedUtf8StringsAndFilePayloads() throws Exception { Files.write(malformedFile, malformed); var malformedEnvelope = (ObjectNode) MAPPER.readTree(envelope); malformedEnvelope.put("file", malformedFile.toString()); + malformedEnvelope.put("payloadDigest", sha256(malformed)); assertThrows( SerDesException.class, @@ -444,7 +523,10 @@ void rejectsTrailingTokensAfterFilesystemEnvelope() { var envelope = "{\"__durable_execution_filesystem_serdes\":1," + "\"ownerDurableExecutionArn\":\"" + ARN - + "\",\"ownerEntityId\":\"1\",\"payloadType\":\"STRING\",\"data\":\"value\"}"; + + "\",\"ownerEntityId\":\"1\",\"payloadType\":\"STRING\"," + + "\"payloadDigest\":\"" + + sha256("value".getBytes(StandardCharsets.UTF_8)) + + "\",\"data\":\"value\"}"; var failure = assertThrows(SerDesException.class, () -> stage.deserialize(envelope + " true", context())); @@ -719,7 +801,9 @@ private static String envelopeWithFile(String file) { "ownerEntityId", "1", "payloadType", - "STRING")); + "STRING", + "payloadDigest", + "0".repeat(64))); } catch (Exception e) { throw new AssertionError(e); } @@ -735,12 +819,19 @@ private static Path payloadFile(String envelope) { private static Path contentHashedPath(Path original, byte[] data) throws Exception { var name = original.getFileName().toString(); - var existingHash = - HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(Files.readAllBytes(original))); - var hash = HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(data)); + var existingHash = sha256(Files.readAllBytes(original)); + var hash = sha256(data); return original.resolveSibling(name.replace(existingHash, hash)); } + private static String sha256(byte[] data) { + try { + return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(data)); + } catch (Exception e) { + throw new AssertionError(e); + } + } + private static SerDesContext context() { return context(1); } From 4164e46b2db709c2b60bdadbd900e9690cc25a86 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Wed, 26 Aug 2026 00:23:57 +0000 Subject: [PATCH 49/56] Persist filesystem SerDes E2E infrastructure --- .github/workflows/e2e-tests.yml | 23 +++++- examples/README.md | 22 +++++- examples/generate-template.py | 121 ++++++++++++++++++++++------- examples/test_generate_template.py | 53 +++++++++++++ 4 files changed, 182 insertions(+), 37 deletions(-) create mode 100644 examples/test_generate_template.py diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index f07582636..1715cf68d 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -68,12 +68,29 @@ jobs: cache: maven - name: Build locally run: mvn -B -q -Dmaven.test.skip=true install --file pom.xml + - name: Test SAM template generator + run: python3 -m unittest test_generate_template.py + working-directory: ./examples - name: Generate SAM template run: python3 generate-template.py working-directory: ./examples - name: Generate filesystem SerDes E2E SAM template run: python3 generate-template.py --file-system-only --output filesystem-template.yaml working-directory: ./examples + - name: Generate persistent filesystem SerDes E2E infrastructure template + run: | + python3 generate-template.py \ + --file-system-infrastructure-only \ + --output filesystem-infrastructure-template.yaml + working-directory: ./examples + - name: Ensure persistent filesystem SerDes E2E infrastructure + run: | + aws cloudformation deploy \ + --template-file filesystem-infrastructure-template.yaml \ + --stack-name Java${{ matrix.java }}-JavaSDKFileSystemSerDesE2EInfrastructureStack \ + --no-fail-on-empty-changeset \ + --tags Purpose=JavaSDKFileSystemSerDesE2E JavaVersion=${{ matrix.java }} + working-directory: ./examples - name: sam build env: MAVEN_OPTS: -DskipTests=true -Dmaven.test.skip=true @@ -87,7 +104,7 @@ jobs: run: | sam build --debug --template-file filesystem-template.yaml --build-dir .aws-sam-filesystem \ --parameter-overrides \ - 'ParameterKey=Architecture,ParameterValue=x86_64 ParameterKey=JavaVersion,ParameterValue=java${{ matrix.java }} ParameterKey=FunctionNamePrefix,ParameterValue=Java${{ matrix.java }}- ParameterKey=RoleArn,ParameterValue=${{ secrets.TEST_LAMBDA_EXECUTION_ROLE_ARN }}' + 'ParameterKey=Architecture,ParameterValue=x86_64 ParameterKey=JavaVersion,ParameterValue=java${{ matrix.java }} ParameterKey=FunctionNamePrefix,ParameterValue=Java${{ matrix.java }}- ParameterKey=RoleArn,ParameterValue=${{ secrets.TEST_LAMBDA_EXECUTION_ROLE_ARN }} ParameterKey=FileSystemInfrastructureStackName,ParameterValue=Java${{ matrix.java }}-JavaSDKFileSystemSerDesE2EInfrastructureStack' working-directory: ./examples - name: Clean up unmanaged Lambda log groups run: | @@ -105,7 +122,7 @@ jobs: sam deploy --template-file .aws-sam-filesystem/template.yaml \ --stack-name Java${{ matrix.java }}-JavaSDKFileSystemSerDesE2EStack \ --resolve-s3 --parameter-overrides \ - 'ParameterKey=Architecture,ParameterValue=x86_64 ParameterKey=JavaVersion,ParameterValue=java${{ matrix.java }} ParameterKey=FunctionNamePrefix,ParameterValue=Java${{ matrix.java }}- ParameterKey=RoleArn,ParameterValue=${{ secrets.TEST_LAMBDA_EXECUTION_ROLE_ARN }}' + 'ParameterKey=Architecture,ParameterValue=x86_64 ParameterKey=JavaVersion,ParameterValue=java${{ matrix.java }} ParameterKey=FunctionNamePrefix,ParameterValue=Java${{ matrix.java }}- ParameterKey=RoleArn,ParameterValue=${{ secrets.TEST_LAMBDA_EXECUTION_ROLE_ARN }} ParameterKey=FileSystemInfrastructureStackName,ParameterValue=Java${{ matrix.java }}-JavaSDKFileSystemSerDesE2EInfrastructureStack' working-directory: ./examples - name: Cloud Based Integration Tests run: | @@ -121,7 +138,7 @@ jobs: -Djunit.jupiter.execution.parallel.config.strategy=fixed \ -Djunit.jupiter.execution.parallel.config.fixed.parallelism=${{ env.E2E_TEST_PARALLELISM }} working-directory: ./examples - - name: Delete filesystem SerDes E2E stack + - name: Delete filesystem SerDes E2E Lambda stack if: always() run: | if aws cloudformation describe-stacks \ diff --git a/examples/README.md b/examples/README.md index 9dd5d5d48..257a11ac7 100644 --- a/examples/README.md +++ b/examples/README.md @@ -77,10 +77,24 @@ mvn test -Dtest=CloudBasedIntegrationTest \ -Dtest.aws.region=us-east-1 ``` -For manually run cloud tests, the filesystem SerDes test is disabled by default because it requires a separate VPC -and EFS-backed stack. Generate and deploy that stack with `generate-template.py --file-system-only`, then include -`-Dtest.filesystem.enabled=true` when running `CloudBasedIntegrationTest`. GitHub Actions provisions this stack and -runs the filesystem test in every E2E matrix job. +For manually run cloud tests, the filesystem SerDes test is disabled by default because it requires VPC and EFS +infrastructure. Create the persistent infrastructure stack once: + +```bash +python3 generate-template.py \ + --file-system-infrastructure-only \ + --output filesystem-infrastructure-template.yaml +aws cloudformation deploy \ + --template-file filesystem-infrastructure-template.yaml \ + --stack-name Java17-JavaSDKFileSystemSerDesE2EInfrastructureStack +``` + +Then generate, build, and deploy the filesystem Lambda stack with +`FileSystemInfrastructureStackName=Java17-JavaSDKFileSystemSerDesE2EInfrastructureStack`, and include +`-Dtest.filesystem.enabled=true` when running `CloudBasedIntegrationTest`. + +GitHub Actions maintains one persistent infrastructure stack per Java version. Each E2E matrix job reuses that +stack, deploys the filesystem Lambda separately, runs the test, and deletes only the Lambda stack. ## Examples diff --git a/examples/generate-template.py b/examples/generate-template.py index 8d510bf4c..fdbd8c776 100755 --- a/examples/generate-template.py +++ b/examples/generate-template.py @@ -112,7 +112,6 @@ def emit_function(lines: list[str], example: ExampleFunction) -> None: [ " DependsOn:", f" - {example.log_group_logical_id}", - " - FileSystemMountTarget", ] ) else: @@ -130,11 +129,15 @@ def emit_function(lines: list[str], example: ExampleFunction) -> None: [ " VpcConfig:", " SecurityGroupIds:", - " - !Ref FileSystemLambdaSecurityGroup", + " - Fn::ImportValue:", + ' Fn::Sub: "${FileSystemInfrastructureStackName}-LambdaSecurityGroupId"', " SubnetIds:", - " - !Ref FileSystemSubnet", + " - Fn::ImportValue:", + ' Fn::Sub: "${FileSystemInfrastructureStackName}-SubnetId"', " FileSystemConfigs:", - " - Arn: !GetAtt FileSystemAccessPoint.Arn", + " - Arn:", + " Fn::ImportValue:", + ' Fn::Sub: "${FileSystemInfrastructureStackName}-AccessPointArn"', " LocalMountPath: /mnt/efs", " Environment:", " Variables:", @@ -252,6 +255,44 @@ def emit_file_system_resources(lines: list[str]) -> None: ) +def emit_file_system_outputs(lines: list[str]) -> None: + lines.extend( + [ + "Outputs:", + " SubnetId:", + " Description: Subnet used by the filesystem SerDes E2E Lambda function", + " Value: !Ref FileSystemSubnet", + " Export:", + ' Name: !Sub "${AWS::StackName}-SubnetId"', + "", + " LambdaSecurityGroupId:", + " Description: Security group used by the filesystem SerDes E2E Lambda function", + " Value: !Ref FileSystemLambdaSecurityGroup", + " Export:", + ' Name: !Sub "${AWS::StackName}-LambdaSecurityGroupId"', + "", + " AccessPointArn:", + " Description: EFS access point mounted by the filesystem SerDes E2E Lambda function", + " Value: !GetAtt FileSystemAccessPoint.Arn", + " Export:", + ' Name: !Sub "${AWS::StackName}-AccessPointArn"', + ] + ) + + +def render_file_system_infrastructure_template() -> str: + lines = [ + "# This file is generated by examples/generate-template.py. Do not edit it by hand.", + 'AWSTemplateFormatVersion: "2010-09-09"', + "Description: Persistent EFS infrastructure for filesystem SerDes E2E tests", + "", + "Resources:", + ] + emit_file_system_resources(lines) + emit_file_system_outputs(lines) + return "\n".join(lines) + "\n" + + def render_template(examples: list[ExampleFunction]) -> str: lines = [ "# This file is generated by examples/generate-template.py. Do not edit it by hand.", @@ -278,32 +319,41 @@ def render_template(examples: list[ExampleFunction]) -> str: " RoleArn:", " Type: String", " Description: IAM Role ARN for Lambda function execution", - "", - "Conditions:", - " IsJava21OrLater:", - " !Or", - " - !Equals [!Ref JavaVersion, 'java21']", - " - !Equals [!Ref JavaVersion, 'java25']", - "", - "Globals:", - " Function:", - " Timeout: 900", - " MemorySize: 512", - " Architectures:", - " - !Ref Architecture", - " DurableConfig:", - " ExecutionTimeout: 300", - " RetentionPeriodInDays: 7", - " Runtime: !Ref JavaVersion", - " Environment:", - " Variables:", - " FUNCTION_NAME_PREFIX: !Ref FunctionNamePrefix", - "", - "Resources:", ] - if any(example.file_system for example in examples): - emit_file_system_resources(lines) + lines.extend( + [ + " FileSystemInfrastructureStackName:", + " Type: String", + " Description: Name of the persistent filesystem SerDes E2E infrastructure stack", + ] + ) + lines.extend( + [ + "", + "Conditions:", + " IsJava21OrLater:", + " !Or", + " - !Equals [!Ref JavaVersion, 'java21']", + " - !Equals [!Ref JavaVersion, 'java25']", + "", + "Globals:", + " Function:", + " Timeout: 900", + " MemorySize: 512", + " Architectures:", + " - !Ref Architecture", + " DurableConfig:", + " ExecutionTimeout: 300", + " RetentionPeriodInDays: 7", + " Runtime: !Ref JavaVersion", + " Environment:", + " Variables:", + " FUNCTION_NAME_PREFIX: !Ref FunctionNamePrefix", + "", + "Resources:", + ] + ) for example in examples: emit_log_group(lines, example) @@ -329,13 +379,24 @@ def render_template(examples: list[ExampleFunction]) -> str: def main() -> None: parser = argparse.ArgumentParser(description="Generate the examples SAM template from Java example handlers.") parser.add_argument("--output", type=Path, default=DEFAULT_OUTPUT, help="Path to write the generated template.") - parser.add_argument( + template_selection = parser.add_mutually_exclusive_group() + template_selection.add_argument( "--file-system-only", action="store_true", - help="Generate the short-lived EFS-backed filesystem SerDes E2E stack.", + help="Generate the filesystem SerDes E2E Lambda stack.", + ) + template_selection.add_argument( + "--file-system-infrastructure-only", + action="store_true", + help="Generate the persistent EFS infrastructure stack used by filesystem SerDes E2E tests.", ) args = parser.parse_args() + if args.file_system_infrastructure_only: + args.output.write_text(render_file_system_infrastructure_template(), encoding="utf-8") + print(f"Generated persistent filesystem infrastructure template at {args.output}.") + return + examples = discover_examples() examples = [example for example in examples if example.file_system == args.file_system_only] if not examples: diff --git a/examples/test_generate_template.py b/examples/test_generate_template.py new file mode 100644 index 000000000..9223fec57 --- /dev/null +++ b/examples/test_generate_template.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import importlib.util +import sys +import unittest +from pathlib import Path + + +GENERATOR_PATH = Path(__file__).with_name("generate-template.py") +SPEC = importlib.util.spec_from_file_location("generate_template", GENERATOR_PATH) +if SPEC is None or SPEC.loader is None: + raise RuntimeError(f"Unable to load {GENERATOR_PATH}") +generate_template = importlib.util.module_from_spec(SPEC) +sys.modules[SPEC.name] = generate_template +SPEC.loader.exec_module(generate_template) + + +class GenerateTemplateTest(unittest.TestCase): + def test_default_template_does_not_include_file_system_infrastructure(self) -> None: + examples = [example for example in generate_template.discover_examples() if not example.file_system] + + template = generate_template.render_template(examples) + + self.assertNotIn("FileSystemInfrastructureStackName", template) + self.assertNotIn("AWS::EFS::FileSystem", template) + + def test_file_system_lambda_template_imports_persistent_infrastructure(self) -> None: + examples = [example for example in generate_template.discover_examples() if example.file_system] + + template = generate_template.render_template(examples) + + self.assertIn("FileSystemInfrastructureStackName:", template) + self.assertIn("${FileSystemInfrastructureStackName}-SubnetId", template) + self.assertIn("${FileSystemInfrastructureStackName}-LambdaSecurityGroupId", template) + self.assertIn("${FileSystemInfrastructureStackName}-AccessPointArn", template) + self.assertNotIn("AWS::EFS::FileSystem", template) + self.assertNotIn("FileSystemMountTarget", template) + + def test_file_system_infrastructure_template_exports_shared_resources(self) -> None: + template = generate_template.render_file_system_infrastructure_template() + + self.assertIn("AWS::EFS::FileSystem", template) + self.assertIn("AWS::EFS::MountTarget", template) + self.assertIn("${AWS::StackName}-SubnetId", template) + self.assertIn("${AWS::StackName}-LambdaSecurityGroupId", template) + self.assertIn("${AWS::StackName}-AccessPointArn", template) + self.assertNotIn("AWS::Serverless::Function", template) + + +if __name__ == "__main__": + unittest.main() From fd92ee9686f4fca45b4614cbc873fab729e4465a Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Wed, 26 Aug 2026 00:28:44 +0000 Subject: [PATCH 50/56] Share filesystem E2E infrastructure across Java versions --- .github/workflows/e2e-tests.yml | 57 +++++++++++++++++++++------------ examples/README.md | 8 ++--- examples/generate-template.py | 6 ++-- 3 files changed, 44 insertions(+), 27 deletions(-) diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index 1715cf68d..7e89b580d 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -32,10 +32,44 @@ permissions: id-token: write # This is required for requesting the JWT contents: read # This is required for actions/checkout +env: + AWS_REGION: us-west-2 + FILESYSTEM_INFRASTRUCTURE_STACK_NAME: JavaSDKFileSystemSerDesE2EInfrastructureStack + jobs: + filesystem-infrastructure: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v7 + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6.2.3 + with: + role-to-assume: "${{ secrets.TEST_ROLE_ARN }}" + role-session-name: java-language-sdk-test-infrastructure + aws-region: ${{ env.AWS_REGION }} + allowed-account-ids: ${{ secrets.TEST_ACCOUNT_ID }} + - name: Test SAM template generator + run: python3 -m unittest test_generate_template.py + working-directory: ./examples + - name: Generate persistent filesystem SerDes E2E infrastructure template + run: | + python3 generate-template.py \ + --file-system-infrastructure-only \ + --output filesystem-infrastructure-template.yaml + working-directory: ./examples + - name: Ensure persistent filesystem SerDes E2E infrastructure + run: | + aws cloudformation deploy \ + --template-file filesystem-infrastructure-template.yaml \ + --stack-name ${{ env.FILESYSTEM_INFRASTRUCTURE_STACK_NAME }} \ + --no-fail-on-empty-changeset \ + --tags Purpose=JavaSDKFileSystemSerDesE2E + working-directory: ./examples + e2e-tests: + needs: filesystem-infrastructure env: - AWS_REGION: us-west-2 E2E_TEST_PARALLELISM: 4 runs-on: ubuntu-latest strategy: @@ -68,29 +102,12 @@ jobs: cache: maven - name: Build locally run: mvn -B -q -Dmaven.test.skip=true install --file pom.xml - - name: Test SAM template generator - run: python3 -m unittest test_generate_template.py - working-directory: ./examples - name: Generate SAM template run: python3 generate-template.py working-directory: ./examples - name: Generate filesystem SerDes E2E SAM template run: python3 generate-template.py --file-system-only --output filesystem-template.yaml working-directory: ./examples - - name: Generate persistent filesystem SerDes E2E infrastructure template - run: | - python3 generate-template.py \ - --file-system-infrastructure-only \ - --output filesystem-infrastructure-template.yaml - working-directory: ./examples - - name: Ensure persistent filesystem SerDes E2E infrastructure - run: | - aws cloudformation deploy \ - --template-file filesystem-infrastructure-template.yaml \ - --stack-name Java${{ matrix.java }}-JavaSDKFileSystemSerDesE2EInfrastructureStack \ - --no-fail-on-empty-changeset \ - --tags Purpose=JavaSDKFileSystemSerDesE2E JavaVersion=${{ matrix.java }} - working-directory: ./examples - name: sam build env: MAVEN_OPTS: -DskipTests=true -Dmaven.test.skip=true @@ -104,7 +121,7 @@ jobs: run: | sam build --debug --template-file filesystem-template.yaml --build-dir .aws-sam-filesystem \ --parameter-overrides \ - 'ParameterKey=Architecture,ParameterValue=x86_64 ParameterKey=JavaVersion,ParameterValue=java${{ matrix.java }} ParameterKey=FunctionNamePrefix,ParameterValue=Java${{ matrix.java }}- ParameterKey=RoleArn,ParameterValue=${{ secrets.TEST_LAMBDA_EXECUTION_ROLE_ARN }} ParameterKey=FileSystemInfrastructureStackName,ParameterValue=Java${{ matrix.java }}-JavaSDKFileSystemSerDesE2EInfrastructureStack' + 'ParameterKey=Architecture,ParameterValue=x86_64 ParameterKey=JavaVersion,ParameterValue=java${{ matrix.java }} ParameterKey=FunctionNamePrefix,ParameterValue=Java${{ matrix.java }}- ParameterKey=RoleArn,ParameterValue=${{ secrets.TEST_LAMBDA_EXECUTION_ROLE_ARN }} ParameterKey=FileSystemInfrastructureStackName,ParameterValue=${{ env.FILESYSTEM_INFRASTRUCTURE_STACK_NAME }}' working-directory: ./examples - name: Clean up unmanaged Lambda log groups run: | @@ -122,7 +139,7 @@ jobs: sam deploy --template-file .aws-sam-filesystem/template.yaml \ --stack-name Java${{ matrix.java }}-JavaSDKFileSystemSerDesE2EStack \ --resolve-s3 --parameter-overrides \ - 'ParameterKey=Architecture,ParameterValue=x86_64 ParameterKey=JavaVersion,ParameterValue=java${{ matrix.java }} ParameterKey=FunctionNamePrefix,ParameterValue=Java${{ matrix.java }}- ParameterKey=RoleArn,ParameterValue=${{ secrets.TEST_LAMBDA_EXECUTION_ROLE_ARN }} ParameterKey=FileSystemInfrastructureStackName,ParameterValue=Java${{ matrix.java }}-JavaSDKFileSystemSerDesE2EInfrastructureStack' + 'ParameterKey=Architecture,ParameterValue=x86_64 ParameterKey=JavaVersion,ParameterValue=java${{ matrix.java }} ParameterKey=FunctionNamePrefix,ParameterValue=Java${{ matrix.java }}- ParameterKey=RoleArn,ParameterValue=${{ secrets.TEST_LAMBDA_EXECUTION_ROLE_ARN }} ParameterKey=FileSystemInfrastructureStackName,ParameterValue=${{ env.FILESYSTEM_INFRASTRUCTURE_STACK_NAME }}' working-directory: ./examples - name: Cloud Based Integration Tests run: | diff --git a/examples/README.md b/examples/README.md index 257a11ac7..c554fc271 100644 --- a/examples/README.md +++ b/examples/README.md @@ -86,15 +86,15 @@ python3 generate-template.py \ --output filesystem-infrastructure-template.yaml aws cloudformation deploy \ --template-file filesystem-infrastructure-template.yaml \ - --stack-name Java17-JavaSDKFileSystemSerDesE2EInfrastructureStack + --stack-name JavaSDKFileSystemSerDesE2EInfrastructureStack ``` Then generate, build, and deploy the filesystem Lambda stack with -`FileSystemInfrastructureStackName=Java17-JavaSDKFileSystemSerDesE2EInfrastructureStack`, and include +`FileSystemInfrastructureStackName=JavaSDKFileSystemSerDesE2EInfrastructureStack`, and include `-Dtest.filesystem.enabled=true` when running `CloudBasedIntegrationTest`. -GitHub Actions maintains one persistent infrastructure stack per Java version. Each E2E matrix job reuses that -stack, deploys the filesystem Lambda separately, runs the test, and deletes only the Lambda stack. +GitHub Actions maintains one persistent infrastructure stack shared by every Java version. Each E2E matrix job +deploys its filesystem Lambda against that stack, runs the test, and deletes only its Lambda stack. ## Examples diff --git a/examples/generate-template.py b/examples/generate-template.py index fdbd8c776..9cb380568 100755 --- a/examples/generate-template.py +++ b/examples/generate-template.py @@ -284,7 +284,7 @@ def render_file_system_infrastructure_template() -> str: lines = [ "# This file is generated by examples/generate-template.py. Do not edit it by hand.", 'AWSTemplateFormatVersion: "2010-09-09"', - "Description: Persistent EFS infrastructure for filesystem SerDes E2E tests", + "Description: Persistent shared EFS infrastructure for filesystem SerDes E2E tests", "", "Resources:", ] @@ -325,7 +325,7 @@ def render_template(examples: list[ExampleFunction]) -> str: [ " FileSystemInfrastructureStackName:", " Type: String", - " Description: Name of the persistent filesystem SerDes E2E infrastructure stack", + " Description: Name of the shared persistent filesystem SerDes E2E infrastructure stack", ] ) lines.extend( @@ -388,7 +388,7 @@ def main() -> None: template_selection.add_argument( "--file-system-infrastructure-only", action="store_true", - help="Generate the persistent EFS infrastructure stack used by filesystem SerDes E2E tests.", + help="Generate the shared persistent EFS infrastructure stack used by filesystem SerDes E2E tests.", ) args = parser.parse_args() From 7f09c77e84bf9e4afcaeba4f008b47c7b1735ea9 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Wed, 26 Aug 2026 00:43:11 +0000 Subject: [PATCH 51/56] Persist filesystem SerDes E2E Lambda stacks --- .github/workflows/e2e-tests.yml | 9 --------- examples/README.md | 4 ++-- 2 files changed, 2 insertions(+), 11 deletions(-) diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index 7e89b580d..94f206e9f 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -155,15 +155,6 @@ jobs: -Djunit.jupiter.execution.parallel.config.strategy=fixed \ -Djunit.jupiter.execution.parallel.config.fixed.parallelism=${{ env.E2E_TEST_PARALLELISM }} working-directory: ./examples - - name: Delete filesystem SerDes E2E Lambda stack - if: always() - run: | - if aws cloudformation describe-stacks \ - --stack-name Java${{ matrix.java }}-JavaSDKFileSystemSerDesE2EStack >/dev/null 2>&1; then - sam delete --no-prompts \ - --stack-name Java${{ matrix.java }}-JavaSDKFileSystemSerDesE2EStack - fi - working-directory: ./examples - name: Publish test case summary if: always() env: diff --git a/examples/README.md b/examples/README.md index c554fc271..ab4770876 100644 --- a/examples/README.md +++ b/examples/README.md @@ -93,8 +93,8 @@ Then generate, build, and deploy the filesystem Lambda stack with `FileSystemInfrastructureStackName=JavaSDKFileSystemSerDesE2EInfrastructureStack`, and include `-Dtest.filesystem.enabled=true` when running `CloudBasedIntegrationTest`. -GitHub Actions maintains one persistent infrastructure stack shared by every Java version. Each E2E matrix job -deploys its filesystem Lambda against that stack, runs the test, and deletes only its Lambda stack. +GitHub Actions maintains one persistent infrastructure stack shared by every Java version and one persistent +filesystem Lambda stack per Java version. Each E2E matrix job updates its Lambda stack in place and runs the test. ## Examples From daa3b20b457a5be8a0076d02a33310ca161007a5 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Wed, 26 Aug 2026 03:20:01 +0000 Subject: [PATCH 52/56] Fix initial input SerDes decoding --- docs/adr/005-filesystem-serdes.md | 22 ++++---- docs/advanced/configuration.md | 4 +- docs/advanced/serdes.md | 26 ++++++--- .../testing/CloudDurableTestRunner.java | 3 +- .../testing/LocalDurableTestRunner.java | 29 ++++------ .../testing/LocalDurableTestRunnerTest.java | 45 ++++++++++++++++ .../amazon/lambda/durable/DurableConfig.java | 54 +++++++++++++++++++ .../durable/execution/DurableExecutor.java | 2 +- .../lambda/durable/DurableConfigTest.java | 45 ++++++++++++++++ 9 files changed, 190 insertions(+), 40 deletions(-) diff --git a/docs/adr/005-filesystem-serdes.md b/docs/adr/005-filesystem-serdes.md index a9999f01a..696866540 100644 --- a/docs/adr/005-filesystem-serdes.md +++ b/docs/adr/005-filesystem-serdes.md @@ -554,16 +554,18 @@ When serializing `errorData`, set `SerDesPayloadKind.EXCEPTION` and use an entit Root user input and output payloads should route through `SerDesRunner` so `FileSystemSerDesStage` can see `SerDesContext`. The internal `DurableExecutionInput` and `DurableExecutionOutput` envelope stays with `DurableInputOutputSerDes`. -The cloud test runner must send initial Lambda input before it receives a durable execution ARN. The cloud and local -runners therefore always serialize that input with a separate context-free value codec. By default, they use the -configured persisted SerDes when it is a plain value codec, or the root value codec when it is composable. -`withInputSerDes(...)` replaces this codec, but rejects `ComposableSerDes`: persisted stages are never used as the -initial invocation's input encoder. A custom input codec must produce data that the persisted pipeline's root value -codec can decode after the stages pass the unrecognized input through. Fluent configuration preserves an explicit -input-codec override when other runner configuration is replaced. `LocalDurableTestRunner` creates the execution -operation with this raw external payload and lets `DurableExecutor` apply the persisted pipeline's pass-through -behavior during deserialization; it does not synthesize a durable context or serialize initial input through the -persisted pipeline. +The cloud test runner must send initial Lambda input before it receives a durable execution ARN. Initial input +therefore uses a separate context-free value codec configured by `DurableConfig.withInputSerDes(...)`. By default, +`DurableConfig` uses the configured persisted SerDes when it is a plain value codec, or the root value codec when it is +composable. An explicit input codec rejects `ComposableSerDes`: persisted stages are never used at the external +invocation boundary. + +`DurableExecutor` deserializes the execution operation's initial input directly with the configured input codec rather +than the persisted pipeline. The codecs do not need to be wire-compatible, and an external payload that resembles a +persisted stage frame cannot be consumed by that stage accidentally. `LocalDurableTestRunner.withInputSerDes(...)` +updates both its encoder and the copied runtime configuration. `CloudDurableTestRunner.withInputSerDes(...)` updates +the client-side encoder, so the deployed handler must configure the same input codec through `DurableConfig`. Fluent +runner configuration preserves an explicit input-codec override when other runner configuration is replaced. ### Implementation plan diff --git a/docs/advanced/configuration.md b/docs/advanced/configuration.md index a336af3c8..67ec11e79 100644 --- a/docs/advanced/configuration.md +++ b/docs/advanced/configuration.md @@ -17,6 +17,7 @@ public class OrderProcessor extends DurableHandler { return DurableConfig.builder() .withLambdaClientBuilder(lambdaClientBuilder) .withSerDes(new MyCustomSerDes()) // Custom serialization + .withInputSerDes(new MyInputSerDes()) // Optional initial invocation codec .withExecutorService(Executors.newFixedThreadPool(10)) // Custom thread pool .withSerDesExecutorService(Executors.newFixedThreadPool(4)) // Optional SerDes/payload I/O pool .withLoggerConfig(LoggerConfig.withReplayLogging()) // Enable replay logs @@ -33,7 +34,8 @@ public class OrderProcessor extends DurableHandler { | Option | Description | Default | |-----------------------------|-----------------------------------------|-------------------------------| | `withLambdaClientBuilder()` | Custom AWS Lambda client | Auto-configured Lambda client | -| `withSerDes()` | Serializer for step results | Jackson with default settings | +| `withSerDes()` | Serializer for persisted execution values | Jackson with default settings | +| `withInputSerDes()` | Context-free codec for initial invocation input | Persisted value codec, or pipeline root | | `withExecutorService()` | Thread pool for user-defined operations | Cached daemon thread pool | | `withSerDesExecutorService()` | Optional thread pool for SerDes and payload storage I/O | Inline on the calling thread | | `withLoggerConfig()` | Logger behavior configuration | Suppress logs during replay | diff --git a/docs/advanced/serdes.md b/docs/advanced/serdes.md index d3621d182..760387ef6 100644 --- a/docs/advanced/serdes.md +++ b/docs/advanced/serdes.md @@ -178,15 +178,27 @@ return DurableConfig.builder() The SerDes executor must be different from the user-operation executor to avoid deadlock when the operation pool is saturated. -## Initial invocation payloads in test runners +## Initial invocation payloads -`CloudDurableTestRunner` and `LocalDurableTestRunner` serialize the initial Lambda invocation with a separate, -context-free value codec because no durable execution context exists yet. They do not run the persisted pipeline's -stages at this boundary. +The initial Lambda invocation uses a separate, context-free value codec because no durable execution context exists +yet. `DurableExecutor` deserializes this payload directly with `DurableConfig.getInputSerDes()` and does not run the +persisted pipeline's stages at this boundary. This also prevents an external payload from being mistaken for a +persisted stage frame. -By default, a runner uses the configured SerDes if it is a plain value codec, or the root value codec if it is a -`ComposableSerDes`. Use `withInputSerDes(...)` to select another input codec. The explicit input codec must not be a -`ComposableSerDes`, and it must produce a string that the persisted pipeline's root codec can deserialize. +By default, `DurableConfig` uses the configured SerDes if it is a plain value codec, or the root value codec if it is a +`ComposableSerDes`. Use `DurableConfig.Builder.withInputSerDes(...)` to select another input codec. The explicit input +codec must not be a `ComposableSerDes`, but it does not need to be wire-compatible with the persisted pipeline: + +```java +var config = DurableConfig.builder() + .withSerDes(persistedPipeline) + .withInputSerDes(externalInputCodec) + .build(); +``` + +`LocalDurableTestRunner.withInputSerDes(...)` updates both sides of the local boundary: the runner serializes with the +selected codec and the local runtime deserializes with it. `CloudDurableTestRunner.withInputSerDes(...)` controls the +invocation sent to AWS; the deployed handler must configure the same codec through `DurableConfig`. ## Filesystem-backed payload storage diff --git a/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/CloudDurableTestRunner.java b/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/CloudDurableTestRunner.java index c6af9bd00..ad280a800 100644 --- a/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/CloudDurableTestRunner.java +++ b/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/CloudDurableTestRunner.java @@ -184,7 +184,8 @@ public CloudDurableTestRunner withSerDes(SerDes serDes) { * *

The input codec is independent of the SerDes used for persisted execution payloads and must not be a * {@link ComposableSerDes}. By default, the configured persisted SerDes is used when it is a value codec; for a - * composable pipeline, its root value codec is used. + * composable pipeline, its root value codec is used. The deployed handler must configure the same codec with + * {@link software.amazon.lambda.durable.DurableConfig.Builder#withInputSerDes(SerDes)}. */ public CloudDurableTestRunner withInputSerDes(SerDes inputSerDes) { return new CloudDurableTestRunner<>( diff --git a/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/LocalDurableTestRunner.java b/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/LocalDurableTestRunner.java index a208cfb76..a7d3196eb 100644 --- a/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/LocalDurableTestRunner.java +++ b/sdk-testing/src/main/java/software/amazon/lambda/durable/testing/LocalDurableTestRunner.java @@ -23,7 +23,6 @@ import software.amazon.lambda.durable.model.DurableExecutionInput; import software.amazon.lambda.durable.model.ExecutionStatus; import software.amazon.lambda.durable.plugin.DurableExecutionPlugin; -import software.amazon.lambda.durable.serde.ComposableSerDes; import software.amazon.lambda.durable.serde.SerDes; import software.amazon.lambda.durable.serde.SerDesRunner; import software.amazon.lambda.durable.testing.local.LocalMemoryExecutionClient; @@ -80,17 +79,21 @@ private LocalDurableTestRunner( .withCheckpointEmptyMap(customerConfig.shouldCheckpointEmptyMap()) .withDeserializeAfterSerialization(customerConfig.shouldDeserializeAfterSerialization()) .withPlugins(customerConfig.getPluginRunner().getPlugins().toArray(new DurableExecutionPlugin[0])); + configBuilder.withInputSerDes(inputSerDes != null ? inputSerDes : customerConfig.getInputSerDes()); if (customerConfig.getSerDesExecutorService() != null) { configBuilder.withSerDesExecutorService(customerConfig.getSerDesExecutorService()); } this.customerConfig = configBuilder.build(); } else { // Fallback to default config with in-memory client - this.customerConfig = - DurableConfig.builder().withDurableExecutionClient(storage).build(); + var configBuilder = DurableConfig.builder().withDurableExecutionClient(storage); + if (inputSerDes != null) { + configBuilder.withInputSerDes(inputSerDes); + } + this.customerConfig = configBuilder.build(); } this.serDes = this.customerConfig.getSerDes(); - this.inputSerDes = inputValueCodec(inputSerDes, this.serDes); + this.inputSerDes = this.customerConfig.getInputSerDes(); } /** @@ -221,9 +224,8 @@ public LocalDurableTestRunner withOutputType(Class outputType) { /** * Returns a new runner with a separate value codec for the initial Lambda invocation payload. * - *

The input codec is independent of the SerDes used for persisted execution payloads and must not be a - * {@link ComposableSerDes}. By default, the configured persisted SerDes is used when it is a value codec; for a - * composable pipeline, its root value codec is used. + *

The returned runner uses this codec both to serialize the external payload and to configure + * {@link DurableExecutor} to deserialize it. Persisted pipeline stages are not invoked at this boundary. */ public LocalDurableTestRunner withInputSerDes(SerDes inputSerDes) { return new LocalDurableTestRunner<>( @@ -406,19 +408,6 @@ private String serializeInput(I input) { return inputSerDes.serialize(input); } - private static SerDes inputValueCodec(SerDes inputSerDes, SerDes persistedSerDes) { - var codec = inputSerDes; - if (codec == null) { - codec = persistedSerDes instanceof ComposableSerDes composable - ? composable.getValueCodec() - : persistedSerDes; - } - if (codec instanceof ComposableSerDes) { - throw new IllegalArgumentException("inputSerDes must be a value codec, not a composable pipeline"); - } - return codec; - } - private Context mockLambdaContext() { return null; // Minimal - tests don't need real Lambda context } diff --git a/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/LocalDurableTestRunnerTest.java b/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/LocalDurableTestRunnerTest.java index d9edc741d..e6b6d58ac 100644 --- a/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/LocalDurableTestRunnerTest.java +++ b/sdk-testing/src/test/java/software/amazon/lambda/durable/testing/LocalDurableTestRunnerTest.java @@ -178,6 +178,51 @@ void plainPersistedSerDesIsUsedAsDefaultInputCodec() { assertEquals("value", result.getResult()); } + @Test + void explicitInputSerDesIsUsedByRunnerAndRuntime() { + var config = DurableConfig.builder().withSerDes(new JacksonSerDes()).build(); + var runner = LocalDurableTestRunner.create(String.class, (input, context) -> input, config) + .withInputSerDes(prefixedStringSerDes()) + .withOutputType(String.class); + + var result = runner.run("value"); + + assertEquals(ExecutionStatus.SUCCEEDED, result.getStatus()); + assertEquals("value", result.getResult()); + } + + @Test + void initialInputBypassesPersistedPipelineStages() { + var deserializeCalls = new AtomicInteger(); + var persistedSerDes = new JacksonSerDes().then(new SerDesStage() { + @Override + public String serialize(String value, SerDesContext context) { + return "persisted:" + value; + } + + @Override + public String deserialize(String data, SerDesContext context) { + deserializeCalls.incrementAndGet(); + if (data.startsWith("custom:")) { + throw new SerDesException("Initial input collided with a persisted stage frame"); + } + return data.startsWith("persisted:") ? data.substring("persisted:".length()) : data; + } + }); + var config = DurableConfig.builder() + .withSerDes(persistedSerDes) + .withInputSerDes(prefixedStringSerDes()) + .build(); + var runner = LocalDurableTestRunner.create( + String.class, (input, context) -> input + ":" + deserializeCalls.get(), config) + .withOutputType(String.class); + + var result = runner.run("value"); + + assertEquals(ExecutionStatus.SUCCEEDED, result.getStatus()); + assertEquals("value:0", result.getResult()); + } + @Test void rejectsComposableInputSerDes(@TempDir Path basePath) { var config = DurableConfig.builder() diff --git a/sdk/src/main/java/software/amazon/lambda/durable/DurableConfig.java b/sdk/src/main/java/software/amazon/lambda/durable/DurableConfig.java index 3bd0ad7c6..e5aa1711d 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/DurableConfig.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/DurableConfig.java @@ -27,6 +27,7 @@ import software.amazon.lambda.durable.plugin.PluginRunner; import software.amazon.lambda.durable.retry.PollingStrategies; import software.amazon.lambda.durable.retry.PollingStrategy; +import software.amazon.lambda.durable.serde.ComposableSerDes; import software.amazon.lambda.durable.serde.JacksonSerDes; import software.amazon.lambda.durable.serde.SerDes; @@ -94,6 +95,7 @@ public final class DurableConfig { private final DurableExecutionClient durableExecutionClient; private final SerDes serDes; + private final SerDes inputSerDes; private final ExecutorService executorService; private final ExecutorService serDesExecutorService; private final LoggerConfig loggerConfig; @@ -108,6 +110,7 @@ private DurableConfig(Builder builder) { this.durableExecutionClient = Objects.requireNonNullElseGet( builder.durableExecutionClient, DurableConfig::createDefaultDurableExecutionClient); this.serDes = Objects.requireNonNullElseGet(builder.serDes, JacksonSerDes::new); + this.inputSerDes = inputValueCodec(builder.inputSerDes, this.serDes); this.executorService = Objects.requireNonNullElseGet(builder.executorService, DurableConfig::createDefaultExecutor); this.serDesExecutorService = builder.serDesExecutorService; @@ -157,6 +160,19 @@ public SerDes getSerDes() { return serDes; } + /** + * Gets the context-free value codec used to deserialize the initial Lambda invocation payload. + * + *

This codec is separate from the persisted SerDes pipeline because the initial payload is received before a + * durable execution context exists. If it is not explicitly configured, a plain persisted SerDes is reused, while a + * composable persisted SerDes contributes only its root value codec. + * + * @return the initial invocation value codec + */ + public SerDes getInputSerDes() { + return inputSerDes; + } + /** * Gets the configured ExecutorService. * @@ -243,6 +259,9 @@ public void validateConfiguration() { if (getSerDes() == null) { throw new IllegalStateException("SerDes configuration failed"); } + if (getInputSerDes() == null) { + throw new IllegalStateException("Input SerDes configuration failed"); + } if (getExecutorService() == null) { throw new IllegalStateException("ExecutorService configuration failed"); } @@ -326,10 +345,24 @@ private static ExecutorService createDefaultExecutor() { return DEFAULT_USER_THREAD_POOL; } + private static SerDes inputValueCodec(SerDes inputSerDes, SerDes persistedSerDes) { + var codec = inputSerDes; + if (codec == null) { + codec = persistedSerDes instanceof ComposableSerDes composable + ? composable.getValueCodec() + : persistedSerDes; + } + if (codec instanceof ComposableSerDes) { + throw new IllegalArgumentException("inputSerDes must be a value codec, not a composable pipeline"); + } + return codec; + } + /** Builder for DurableConfig. Provides fluent API for configuring SDK components. */ public static final class Builder { private DurableExecutionClient durableExecutionClient; private SerDes serDes; + private SerDes inputSerDes; private ExecutorService executorService; private ExecutorService serDesExecutorService; private LoggerConfig loggerConfig; @@ -397,6 +430,27 @@ public Builder withSerDes(SerDes serDes) { return this; } + /** + * Sets the context-free value codec used to deserialize the initial Lambda invocation payload. + * + *

The initial input codec is independent of the SerDes used for persisted execution payloads and must not be + * a {@link ComposableSerDes}. If not set, a plain persisted SerDes is reused, while a composable persisted + * SerDes contributes only its root value codec. + * + * @param inputSerDes initial invocation value codec + * @return this builder + * @throws NullPointerException if inputSerDes is null + * @throws IllegalArgumentException if inputSerDes is a composable pipeline + */ + public Builder withInputSerDes(SerDes inputSerDes) { + inputSerDes = Objects.requireNonNull(inputSerDes, "inputSerDes cannot be null"); + if (inputSerDes instanceof ComposableSerDes) { + throw new IllegalArgumentException("inputSerDes must be a value codec, not a composable pipeline"); + } + this.inputSerDes = inputSerDes; + return this; + } + /** * Sets a custom ExecutorService for running user-defined operations. If not set, a default cached thread pool * will be created. diff --git a/sdk/src/main/java/software/amazon/lambda/durable/execution/DurableExecutor.java b/sdk/src/main/java/software/amazon/lambda/durable/execution/DurableExecutor.java index 44970d954..864465dea 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/execution/DurableExecutor.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/execution/DurableExecutor.java @@ -77,7 +77,7 @@ public static DurableExecutionOutput execute( I userInput = null; Throwable inputFailure = null; try { - userInput = extractUserInput(executionManager, config.getSerDes(), inputType); + userInput = extractUserInput(executionManager, config.getInputSerDes(), inputType); } catch (Throwable t) { inputFailure = t; } diff --git a/sdk/src/test/java/software/amazon/lambda/durable/DurableConfigTest.java b/sdk/src/test/java/software/amazon/lambda/durable/DurableConfigTest.java index 0843ac179..667349981 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/DurableConfigTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/DurableConfigTest.java @@ -28,6 +28,7 @@ import software.amazon.lambda.durable.retry.PollingStrategies; import software.amazon.lambda.durable.serde.JacksonSerDes; import software.amazon.lambda.durable.serde.SerDes; +import software.amazon.lambda.durable.serde.SerDesStage; class DurableConfigTest { @@ -53,6 +54,7 @@ void testDefaultConfig_CreatesWithDefaults() { assertInstanceOf(LambdaDurableFunctionsClient.class, config.getDurableExecutionClient()); assertNotNull(config.getSerDes()); assertInstanceOf(JacksonSerDes.class, config.getSerDes()); + assertSame(config.getSerDes(), config.getInputSerDes()); assertNotNull(config.getExecutorService()); assertInstanceOf(ExecutorService.class, config.getExecutorService()); assertNull(config.getSerDesExecutorService()); @@ -78,9 +80,42 @@ void testBuilder_WithCustomSerDes() { assertNotNull(config); assertNotNull(config.getDurableExecutionClient()); assertEquals(mockSerDes, config.getSerDes()); + assertSame(mockSerDes, config.getInputSerDes()); assertNotNull(config.getExecutorService()); } + @Test + void testBuilder_ComposableSerDesDefaultsInputToRootValueCodec() { + var valueCodec = new JacksonSerDes(); + var config = DurableConfig.builder() + .withSerDes(valueCodec.then(mock(SerDesStage.class))) + .build(); + + assertSame(valueCodec, config.getInputSerDes()); + } + + @Test + void testBuilder_WithCustomInputSerDes() { + var inputSerDes = mock(SerDes.class); + var config = DurableConfig.builder() + .withSerDes(mockSerDes) + .withInputSerDes(inputSerDes) + .build(); + + assertSame(inputSerDes, config.getInputSerDes()); + assertSame(mockSerDes, config.getSerDes()); + } + + @Test + void testBuilder_RejectsComposableInputSerDes() { + var inputPipeline = new JacksonSerDes().then(mock(SerDesStage.class)); + + var exception = assertThrows( + IllegalArgumentException.class, () -> DurableConfig.builder().withInputSerDes(inputPipeline)); + + assertTrue(exception.getMessage().contains("value codec")); + } + @Test void testBuilder_WithCustomExecutorService() { var config = DurableConfig.builder().withExecutorService(mockExecutor).build(); @@ -186,6 +221,15 @@ void testBuilder_NullSerDes_ThrowsException() { assertEquals("SerDes cannot be null", exception.getMessage()); } + @Test + void testBuilder_NullInputSerDes_ThrowsException() { + var builder = DurableConfig.builder(); + + var exception = assertThrows(NullPointerException.class, () -> builder.withInputSerDes(null)); + + assertEquals("inputSerDes cannot be null", exception.getMessage()); + } + @Test void testBuilder_NullSerDesExecutorService_ThrowsException() { var builder = DurableConfig.builder(); @@ -202,6 +246,7 @@ void testBuilder_FluentAPI() { // Verify fluent API returns builder assertSame(builder, builder.withDurableExecutionClient(mockClient)); assertSame(builder, builder.withSerDes(mockSerDes)); + assertSame(builder, builder.withInputSerDes(mock(SerDes.class))); assertSame(builder, builder.withExecutorService(mockExecutor)); assertSame(builder, builder.withSerDesExecutorService(mockSerDesExecutor)); assertSame(builder, builder.withDeserializeAfterSerialization(false)); From 98cc2f323de9c40546744cfba62431224683d55f Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Wed, 26 Aug 2026 03:42:15 +0000 Subject: [PATCH 53/56] Address SerDes pipeline review findings --- docs/adr/005-filesystem-serdes.md | 9 ++++ docs/advanced/serdes.md | 10 +++- .../FileSystemSerDesStageIntegrationTest.java | 3 +- .../lambda/durable/config/InvokeConfig.java | 3 ++ .../durable/execution/DurableExecutor.java | 11 +++- .../durable/operation/InvokeOperation.java | 9 ++-- .../durable/serde/RetryBinarySerDesStage.java | 9 ++-- .../serde/filesystem/SerDesPreview.java | 8 ++- .../internal/ChainedInvokePayloadFrame.java | 36 +++++++++++++ .../serde/RetryBinarySerDesStageTest.java | 52 +++++++++++++++++++ .../serde/filesystem/SerDesPreviewTest.java | 20 +++++++ .../ChainedInvokePayloadFrameTest.java | 40 ++++++++++++++ 12 files changed, 197 insertions(+), 13 deletions(-) create mode 100644 sdk/src/main/java/software/amazon/lambda/durable/serde/internal/ChainedInvokePayloadFrame.java create mode 100644 sdk/src/test/java/software/amazon/lambda/durable/serde/internal/ChainedInvokePayloadFrameTest.java diff --git a/docs/adr/005-filesystem-serdes.md b/docs/adr/005-filesystem-serdes.md index 696866540..e326299e6 100644 --- a/docs/adr/005-filesystem-serdes.md +++ b/docs/adr/005-filesystem-serdes.md @@ -369,6 +369,8 @@ Retry rules: - When the strategy returns `fail`, the retry wrapper rethrows the last `RetryableSerDesException`. - The same read-only `SerDesContext` parameter is passed to every attempt because retrying happens inside the original `SerDesRunner` task. +- `RetryBinarySerDesStage` snapshots its input and passes a fresh byte-array clone to every attempt in both directions. + A delegate may therefore mutate its attempt-local bytes without contaminating a later retry or the caller's input. - A retry delay blocks the thread executing the SerDes call. This is the caller thread by default or a SerDes executor thread when one is explicitly configured. It is an in-invocation infrastructure retry, not a durable wait or checkpoint. Strategies must therefore use short, bounded delays that fit within the Lambda invocation timeout. @@ -567,6 +569,13 @@ updates both its encoder and the copied runtime configuration. `CloudDurableTest the client-side encoder, so the deployed handler must configure the same input codec through `DurableConfig`. Fluent runner configuration preserves an explicit input-codec override when other runner configuration is replaced. +Chained invokes retain full persisted-pipeline processing. After serializing an invoke payload, +`InvokeOperation` adds a reserved, versioned source frame outside the pipeline output. The target +`DurableExecutor` recognizes and removes that frame before deserializing with its persisted SerDes. Unframed execution +input always uses the context-free input codec. This separates the two sources without inspecting a filesystem, +compression, encryption, or other stage format and prevents an external payload from colliding with those formats. A +custom per-invoke payload SerDes must be compatible with the target handler's persisted SerDes. + ### Implementation plan 1. Add `SerDesContext` and `SerDesPayloadKind`. Leave the existing context-free `SerDes` methods unchanged. diff --git a/docs/advanced/serdes.md b/docs/advanced/serdes.md index 760387ef6..69737cd46 100644 --- a/docs/advanced/serdes.md +++ b/docs/advanced/serdes.md @@ -162,7 +162,9 @@ var resilientStorageStage = new RetrySerDesStage( ``` These wrappers retry only `RetryableSerDesException`. Permanent errors, such as malformed envelopes or codec failures, -are not retried. Retry delays consume time in the current Lambda invocation, so keep strategies short and bounded. +are not retried. `RetryBinarySerDesStage` snapshots its input and gives every attempt a fresh byte array, so a failed +delegate cannot leak in-place mutations into the next attempt. Retry delays consume time in the current Lambda +invocation, so keep strategies short and bounded. SerDes runs inline on the calling thread by default, preserving the existing no-thread-pool behavior and avoiding a thread hop for in-memory serialization. Blocking stages, including filesystem access and retry backoff, can use a @@ -200,6 +202,12 @@ var config = DurableConfig.builder() selected codec and the local runtime deserializes with it. `CloudDurableTestRunner.withInputSerDes(...)` controls the invocation sent to AWS; the deployed handler must configure the same codec through `DurableConfig`. +Chained invokes use the persisted pipeline instead. `InvokeOperation` adds a reserved, versioned SDK source frame +outside the serialized invoke payload. `DurableExecutor` removes that frame and deserializes the enclosed value with +the configured persisted SerDes. Unframed execution input always uses the context-free input codec, so an external +payload cannot accidentally activate a persisted stage merely by resembling its format. A custom +`InvokeConfig.payloadSerDes(...)` must remain compatible with the target handler's persisted SerDes. + ## Filesystem-backed payload storage `FileSystemSerDesStage` stores serialized payloads on a durable shared filesystem and leaves small, versioned diff --git a/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FileSystemSerDesStageIntegrationTest.java b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FileSystemSerDesStageIntegrationTest.java index ba1a48eef..ed3bccb24 100644 --- a/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FileSystemSerDesStageIntegrationTest.java +++ b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FileSystemSerDesStageIntegrationTest.java @@ -46,6 +46,7 @@ import software.amazon.lambda.durable.serde.SerDesStage; import software.amazon.lambda.durable.serde.Utf8StringBinaryCodec; import software.amazon.lambda.durable.serde.filesystem.FileSystemSerDesStage; +import software.amazon.lambda.durable.serde.internal.ChainedInvokePayloadFrame; import software.amazon.lambda.durable.testing.LocalDurableTestRunner; import software.amazon.lambda.durable.testing.TestResult; import software.amazon.lambda.durable.testing.local.LocalMemoryExecutionClient; @@ -224,7 +225,7 @@ void callerAndCalleeExchangeOffloadedInvokePayloadAndResult() throws Exception { .findFirst() .orElseThrow() .payload(); - assertEnvelopePointsToFile(invokePayload); + assertEnvelopePointsToFile(ChainedInvokePayloadFrame.decode(invokePayload)); var calleeClient = new LocalMemoryExecutionClient(); var calleeConfig = DurableConfig.builder() diff --git a/sdk/src/main/java/software/amazon/lambda/durable/config/InvokeConfig.java b/sdk/src/main/java/software/amazon/lambda/durable/config/InvokeConfig.java index e9dc7af24..b2a5fd9fb 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/config/InvokeConfig.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/config/InvokeConfig.java @@ -73,6 +73,9 @@ public Builder tenantId(String tenantId) { * per-invoke customization of serialization behavior, useful for invoke operations that need special handling * (e.g., custom date formats, encryption, compression). * + *

The target handler deserializes the framed chained-invoke payload with its configured persisted SerDes, so + * a custom payload SerDes must use a compatible wire format. + * * @param payloadSerDes the custom serializer to use, or null to use the default * @return this builder for method chaining */ diff --git a/sdk/src/main/java/software/amazon/lambda/durable/execution/DurableExecutor.java b/sdk/src/main/java/software/amazon/lambda/durable/execution/DurableExecutor.java index 864465dea..f7928942b 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/execution/DurableExecutor.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/execution/DurableExecutor.java @@ -32,6 +32,7 @@ import software.amazon.lambda.durable.serde.SerDes; import software.amazon.lambda.durable.serde.SerDesContext; import software.amazon.lambda.durable.serde.SerDesPayloadKind; +import software.amazon.lambda.durable.serde.internal.ChainedInvokePayloadFrame; import software.amazon.lambda.durable.util.ExceptionHelper; /** @@ -77,7 +78,7 @@ public static DurableExecutionOutput execute( I userInput = null; Throwable inputFailure = null; try { - userInput = extractUserInput(executionManager, config.getInputSerDes(), inputType); + userInput = extractUserInput(executionManager, config, inputType); } catch (Throwable t) { inputFailure = t; } @@ -253,13 +254,19 @@ private static ErrorObject buildErrorObject(Throwable e, ExecutionManager execut .build(); } - private static I extractUserInput(ExecutionManager executionManager, SerDes serDes, TypeToken inputType) { + private static I extractUserInput( + ExecutionManager executionManager, DurableConfig config, TypeToken inputType) { var executionOp = executionManager.getExecutionOperation(); if (executionOp.executionDetails() == null) { throw new IllegalDurableOperationException("EXECUTION operation missing executionDetails"); } var inputPayload = executionOp.executionDetails().inputPayload(); + var serDes = config.getInputSerDes(); + if (ChainedInvokePayloadFrame.isFramed(inputPayload)) { + inputPayload = ChainedInvokePayloadFrame.decode(inputPayload); + serDes = config.getSerDes(); + } return executionManager .getSerDesRunner() .deserialize( diff --git a/sdk/src/main/java/software/amazon/lambda/durable/operation/InvokeOperation.java b/sdk/src/main/java/software/amazon/lambda/durable/operation/InvokeOperation.java index 9a389daeb..dd7f03b2c 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/operation/InvokeOperation.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/operation/InvokeOperation.java @@ -16,6 +16,7 @@ import software.amazon.lambda.durable.model.OperationIdentifier; import software.amazon.lambda.durable.serde.SerDes; import software.amazon.lambda.durable.serde.SerDesPayloadKind; +import software.amazon.lambda.durable.serde.internal.ChainedInvokePayloadFrame; /** * Durable operation that invokes another Lambda function and waits for its result. @@ -65,17 +66,15 @@ protected void replay(Operation existing) { } private void startInvocation() { + var serializedPayload = getSerDesRunner() + .serialize(payloadSerDes, this.payload, createSerDesContext(SerDesPayloadKind.INVOKE_PAYLOAD, null)); var update = OperationUpdate.builder() .action(OperationAction.START) .chainedInvokeOptions(ChainedInvokeOptions.builder() .functionName(functionName) .tenantId(invokeConfig.tenantId()) .build()) - .payload(getSerDesRunner() - .serialize( - payloadSerDes, - this.payload, - createSerDesContext(SerDesPayloadKind.INVOKE_PAYLOAD, null))); + .payload(ChainedInvokePayloadFrame.encode(serializedPayload)); sendOperationUpdate(update); } diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/RetryBinarySerDesStage.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/RetryBinarySerDesStage.java index 842c08910..301791ce9 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/RetryBinarySerDesStage.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/RetryBinarySerDesStage.java @@ -11,7 +11,7 @@ * *

Only {@link RetryableSerDesException} is retried. Other failures are propagated immediately. Retry delays block * the thread executing the SerDes call: the caller by default or the configured SerDes executor thread. Every attempt - * receives the same {@link SerDesContext} supplied to this decorator. + * receives the same {@link SerDesContext} supplied to this decorator and a fresh copy of the original input bytes. */ public final class RetryBinarySerDesStage implements BinarySerDesStage { private final BinarySerDesStage delegate; @@ -35,11 +35,14 @@ public RetryBinarySerDesStage(BinarySerDesStage delegate, RetryStrategy retryStr @Override public byte[] serialize(byte[] value, SerDesContext context) { - return retryExecutor.execute("binary stage serialization", () -> delegate.serialize(value, context)); + var snapshot = Objects.requireNonNull(value, "value cannot be null").clone(); + return retryExecutor.execute("binary stage serialization", () -> delegate.serialize(snapshot.clone(), context)); } @Override public byte[] deserialize(byte[] data, SerDesContext context) { - return retryExecutor.execute("binary stage deserialization", () -> delegate.deserialize(data, context)); + var snapshot = Objects.requireNonNull(data, "data cannot be null").clone(); + return retryExecutor.execute( + "binary stage deserialization", () -> delegate.deserialize(snapshot.clone(), context)); } } diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/filesystem/SerDesPreview.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/filesystem/SerDesPreview.java index 68259feee..96ccdc237 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/filesystem/SerDesPreview.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/filesystem/SerDesPreview.java @@ -3,8 +3,11 @@ package software.amazon.lambda.durable.serde.filesystem; import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.LinkedHashMap; @@ -15,7 +18,10 @@ /** Utilities for building compact structured previews for externally stored SerDes payloads. */ public final class SerDesPreview { - private static final ObjectMapper MAPPER = new ObjectMapper(); + private static final ObjectMapper MAPPER = new ObjectMapper() + .registerModule(new JavaTimeModule()) + .disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS) + .disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES); private SerDesPreview() {} diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/internal/ChainedInvokePayloadFrame.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/internal/ChainedInvokePayloadFrame.java new file mode 100644 index 000000000..66a711173 --- /dev/null +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/internal/ChainedInvokePayloadFrame.java @@ -0,0 +1,36 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.serde.internal; + +import software.amazon.lambda.durable.exception.SerDesException; + +/** + * SDK-internal framing that identifies an execution input as the output of a chained-invoke SerDes pipeline. + * + *

The frame sits outside the serialized payload so the callee can distinguish it from an external invocation and + * select the persisted SerDes pipeline without inspecting or altering the pipeline's own format. + */ +public final class ChainedInvokePayloadFrame { + private static final String FRAME_MARKER = "__durable_execution_chained_invoke_payload:"; + private static final String FRAME_PREFIX = FRAME_MARKER + "1:"; + + private ChainedInvokePayloadFrame() {} + + /** Adds the current chained-invoke frame to a non-null serialized payload. */ + public static String encode(String payload) { + return payload == null ? null : FRAME_PREFIX + payload; + } + + /** Returns whether the payload uses the reserved chained-invoke frame marker. */ + public static boolean isFramed(String payload) { + return payload != null && payload.startsWith(FRAME_MARKER); + } + + /** Removes and validates the current chained-invoke frame. */ + public static String decode(String payload) { + if (payload == null || !payload.startsWith(FRAME_PREFIX)) { + throw new SerDesException("Unsupported or malformed chained-invoke payload frame"); + } + return payload.substring(FRAME_PREFIX.length()); + } +} diff --git a/sdk/src/test/java/software/amazon/lambda/durable/serde/RetryBinarySerDesStageTest.java b/sdk/src/test/java/software/amazon/lambda/durable/serde/RetryBinarySerDesStageTest.java index 1f6069ad7..2739c61c6 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/serde/RetryBinarySerDesStageTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/serde/RetryBinarySerDesStageTest.java @@ -75,6 +75,58 @@ public byte[] deserialize(byte[] data, SerDesContext context) { assertArrayEquals(value, stage.deserialize(value, CONTEXT)); } + @Test + void retriesSerializationWithFreshInputForEveryAttempt() { + var calls = new AtomicInteger(); + var delegate = new BinarySerDesStage() { + @Override + public byte[] serialize(byte[] value, SerDesContext context) { + value[0]++; + if (calls.incrementAndGet() == 1) { + throw new RetryableSerDesException("transient"); + } + return value; + } + + @Override + public byte[] deserialize(byte[] data, SerDesContext context) { + return data; + } + }; + var stage = new RetryBinarySerDesStage( + delegate, (error, attempt) -> RetryDecision.retry(Duration.ZERO), delay -> {}); + var value = new byte[] {1, 2, 3}; + + assertArrayEquals(new byte[] {2, 2, 3}, stage.serialize(value, CONTEXT)); + assertArrayEquals(new byte[] {1, 2, 3}, value); + } + + @Test + void retriesDeserializationWithFreshInputForEveryAttempt() { + var calls = new AtomicInteger(); + var delegate = new BinarySerDesStage() { + @Override + public byte[] serialize(byte[] value, SerDesContext context) { + return value; + } + + @Override + public byte[] deserialize(byte[] data, SerDesContext context) { + data[0]--; + if (calls.incrementAndGet() == 1) { + throw new RetryableSerDesException("transient"); + } + return data; + } + }; + var stage = new RetryBinarySerDesStage( + delegate, (error, attempt) -> RetryDecision.retry(Duration.ZERO), delay -> {}); + var value = new byte[] {3, 2, 1}; + + assertArrayEquals(new byte[] {2, 2, 1}, stage.deserialize(value, CONTEXT)); + assertArrayEquals(new byte[] {3, 2, 1}, value); + } + @Test void doesNotRetryPermanentFailures() { var calls = new AtomicInteger(); diff --git a/sdk/src/test/java/software/amazon/lambda/durable/serde/filesystem/SerDesPreviewTest.java b/sdk/src/test/java/software/amazon/lambda/durable/serde/filesystem/SerDesPreviewTest.java index e2e06a63e..2e874f9c3 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/serde/filesystem/SerDesPreviewTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/serde/filesystem/SerDesPreviewTest.java @@ -8,6 +8,10 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import java.math.BigDecimal; +import java.time.Duration; +import java.time.Instant; +import java.time.LocalDateTime; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -120,6 +124,20 @@ void returnsNullWhenNoFieldsAreVisibleOrValueIsNotAnObject() { assertNull(SerDesPreview.buildPreview(List.of(Map.of("id", "123")), config)); } + @Test + void objectPreviewUsesJacksonSerDesTimeFormats() { + var instant = Instant.parse("2026-08-26T03:30:00Z"); + var duration = Duration.ofMinutes(5); + var localDateTime = LocalDateTime.parse("2026-08-26T03:30:00"); + var config = PreviewConfig.builder(PreviewMode.INCLUDE_ALL).build(); + + var preview = SerDesPreview.buildPreview(new TemporalPayload(instant, duration, localDateTime), config); + + assertEquals("2026-08-26T03:30:00Z", preview.get("instant")); + assertEquals(0, new BigDecimal("300").compareTo((BigDecimal) preview.get("duration"))); + assertEquals("2026-08-26T03:30:00", preview.get("localDateTime")); + } + @Test void jsonPreviewRejectsMalformedJsonAndSkipsDottedFieldNames() { var config = PreviewConfig.builder(PreviewMode.INCLUDE_ALL).build(); @@ -149,4 +167,6 @@ void validatesConfiguration() { private static Map nested(Map value, String field) { return (Map) value.get(field); } + + private record TemporalPayload(Instant instant, Duration duration, LocalDateTime localDateTime) {} } diff --git a/sdk/src/test/java/software/amazon/lambda/durable/serde/internal/ChainedInvokePayloadFrameTest.java b/sdk/src/test/java/software/amazon/lambda/durable/serde/internal/ChainedInvokePayloadFrameTest.java new file mode 100644 index 000000000..c76747f69 --- /dev/null +++ b/sdk/src/test/java/software/amazon/lambda/durable/serde/internal/ChainedInvokePayloadFrameTest.java @@ -0,0 +1,40 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package software.amazon.lambda.durable.serde.internal; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.Test; +import software.amazon.lambda.durable.exception.SerDesException; + +class ChainedInvokePayloadFrameTest { + + @Test + void roundTripsSerializedPayloadWithoutReencodingIt() { + var payload = "{\"file\":\"/mnt/efs/payload\"}"; + + var framed = ChainedInvokePayloadFrame.encode(payload); + + assertTrue(ChainedInvokePayloadFrame.isFramed(framed)); + assertEquals(payload, ChainedInvokePayloadFrame.decode(framed)); + } + + @Test + void nullAndExternalPayloadsAreNotFramed() { + assertNull(ChainedInvokePayloadFrame.encode(null)); + assertFalse(ChainedInvokePayloadFrame.isFramed(null)); + assertFalse(ChainedInvokePayloadFrame.isFramed("{\"external\":true}")); + } + + @Test + void rejectsUnsupportedOrMalformedReservedFrames() { + assertThrows( + SerDesException.class, + () -> ChainedInvokePayloadFrame.decode("__durable_execution_chained_invoke_payload:2:value")); + assertThrows(SerDesException.class, () -> ChainedInvokePayloadFrame.decode("external")); + } +} From 0d9e3c9b282ebda86303448b8d2b093152d9b28f Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Wed, 26 Aug 2026 04:34:11 +0000 Subject: [PATCH 54/56] Address invoke and filesystem review findings --- docs/adr/005-filesystem-serdes.md | 20 +- docs/advanced/serdes.md | 40 +++- docs/core/invoke.md | 6 + .../FileSystemSerDesStageIntegrationTest.java | 51 ++++- .../lambda/durable/config/InvokeConfig.java | 47 +++- .../durable/context/DurableContextImpl.java | 7 +- .../durable/operation/InvokeOperation.java | 5 +- .../filesystem/FileSystemSerDesStage.java | 201 +++++++++++------- .../serde/filesystem/SerDesPreview.java | 40 ++-- .../filesystem/FileSystemSerDesStageTest.java | 36 ++-- .../serde/filesystem/SerDesPreviewTest.java | 19 ++ 11 files changed, 332 insertions(+), 140 deletions(-) diff --git a/docs/adr/005-filesystem-serdes.md b/docs/adr/005-filesystem-serdes.md index e326299e6..c93e004ab 100644 --- a/docs/adr/005-filesystem-serdes.md +++ b/docs/adr/005-filesystem-serdes.md @@ -469,6 +469,11 @@ the marked envelope contains `data`, it restores the inline text; if it contains `ComposableSerDes` then passes that value to the preceding string stage. Raw external input, callback results, and standard invoke results pass unchanged through every stage whose marker is absent until they reach the value codec. +Filesystem path validation and access are atomic with respect to path replacement. The implementation traverses from +the filesystem root through relative `SecureDirectoryStream` handles with `NOFOLLOW_LINKS`, retains those handles +through the payload read or `CREATE_NEW` write, and performs failed-write cleanup relative to the same held directory. +It fails closed when the mounted provider does not support `SecureDirectoryStream`. + ### Threading Preserve the current SDK behavior by executing SerDes inline on the calling thread by default. Do not create a default @@ -569,12 +574,15 @@ updates both its encoder and the copied runtime configuration. `CloudDurableTest the client-side encoder, so the deployed handler must configure the same input codec through `DurableConfig`. Fluent runner configuration preserves an explicit input-codec override when other runner configuration is replaced. -Chained invokes retain full persisted-pipeline processing. After serializing an invoke payload, -`InvokeOperation` adds a reserved, versioned source frame outside the pipeline output. The target -`DurableExecutor` recognizes and removes that frame before deserializing with its persisted SerDes. Unframed execution -input always uses the context-free input codec. This separates the two sources without inspecting a filesystem, -compression, encryption, or other stage format and prevents an external payload from colliding with those formats. A -custom per-invoke payload SerDes must be compatible with the target handler's persisted SerDes. +Chained invokes preserve their existing wire contract by default. Unless an explicit payload SerDes is configured, the +caller uses its context-free input codec and sends the serialized value unchanged. This keeps standard Lambda targets, +non-Java durable targets, and older Java SDK versions compatible. + +`InvokeConfig.usePersistedSerDesForPayload(true)` is an explicit target-capability opt-in for compatible Java durable +handlers. In this mode the caller uses its persisted SerDes by default and `InvokeOperation` adds a reserved, versioned +source frame outside the pipeline output. The target `DurableExecutor` recognizes and removes that frame before +deserializing with its persisted SerDes. Unframed execution input always uses the context-free input codec. A custom +per-invoke payload SerDes in this mode must be compatible with the target handler's persisted SerDes. ### Implementation plan diff --git a/docs/advanced/serdes.md b/docs/advanced/serdes.md index 69737cd46..baaf36858 100644 --- a/docs/advanced/serdes.md +++ b/docs/advanced/serdes.md @@ -202,11 +202,27 @@ var config = DurableConfig.builder() selected codec and the local runtime deserializes with it. `CloudDurableTestRunner.withInputSerDes(...)` controls the invocation sent to AWS; the deployed handler must configure the same codec through `DurableConfig`. -Chained invokes use the persisted pipeline instead. `InvokeOperation` adds a reserved, versioned SDK source frame -outside the serialized invoke payload. `DurableExecutor` removes that frame and deserializes the enclosed value with -the configured persisted SerDes. Unframed execution input always uses the context-free input codec, so an external -payload cannot accidentally activate a persisted stage merely by resembling its format. A custom -`InvokeConfig.payloadSerDes(...)` must remain compatible with the target handler's persisted SerDes. +Chained invokes preserve the existing standard-Lambda wire contract by default: the caller uses its context-free input +codec unless `InvokeConfig.payloadSerDes(...)` is set, and sends that serialized value unchanged. This remains +compatible with standard Lambda functions, non-Java durable functions, and older Java SDK versions. + +To offload or otherwise process an invoke payload through a persisted pipeline, both Java durable handlers must +configure compatible pipelines and the caller must opt in: + +```java +var result = context.invoke( + "invoke-compatible-handler", + targetFunction, + payload, + Result.class, + InvokeConfig.builder() + .usePersistedSerDesForPayload(true) + .build()); +``` + +The opt-in adds a reserved, versioned SDK source frame outside the serialized payload. A compatible target removes the +frame and deserializes the enclosed value with its persisted SerDes. Unframed execution input always uses the +context-free input codec. ## Filesystem-backed payload storage @@ -289,9 +305,12 @@ Payload files are content-hashed and immutable. Each serialization publishes a u `CREATE_NEW` write. Existing files are never overwritten, and publication does not require hard links or renames. Every inline and file envelope records the payload's SHA-256 digest. During deserialization, the stage verifies the restored bytes against that envelope digest; file payloads must also have a content-addressed filename consistent with -the digest. The stage rejects symbolic-link paths and validates that ordinary checkpoint replay matches the execution -and entity that produced the reference. Invoke input and result boundaries can consume a file owned by the other -Lambda execution when both functions use the same shared root and path encoding. +the digest. The stage traverses directories with `SecureDirectoryStream`, disables symbolic-link following, and holds +the relative directory handles through each file read or write. This makes path validation and access one safe +operation even if another process changes names on the shared filesystem. Providers without +`SecureDirectoryStream` support are rejected. The stage also validates that ordinary checkpoint replay matches the +execution and entity that produced the reference. Invoke input and result boundaries can consume a file owned by the +other Lambda execution when both functions use the same shared root and path encoding. Stages may follow `FileSystemSerDesStage` to transform its inline or file-reference envelope. `OVERFLOW` and preview size checks occur before those later stages, so account for any expansion when staying within the service checkpoint @@ -303,8 +322,9 @@ Do not use Lambda's ephemeral `/tmp` directory. Durable replay may run in anothe does not exist. Use a durable shared mount such as EFS or S3 Files. The stage does not rely on hard links or renames, which S3 Files -does not support. S3 Files can synchronize writes asynchronously, so a runtime crash before a flush can lose recent -data; use it only when that durability tradeoff is acceptable. +does not support. The mounted Java filesystem provider must expose `SecureDirectoryStream`; ordinary Linux EFS and S3 +Files mounts use the default provider that supplies it. S3 Files can synchronize writes asynchronously, so a runtime +crash before a flush can lose recent data; use it only when that durability tradeoff is acceptable. The SDK does not delete payload files. Configure an appropriate retention or lifecycle policy for the backing storage. diff --git a/docs/core/invoke.md b/docs/core/invoke.md index e79c99cf0..2d6922975 100644 --- a/docs/core/invoke.md +++ b/docs/core/invoke.md @@ -16,8 +16,14 @@ var result = ctx.invoke("invoke-function", InvokeConfig.builder() .payloadSerDes(...) // payload serializer .serDes(...) // result deserializer + .usePersistedSerDesForPayload(true) // compatible Java durable targets only .tenantId(...) // Lambda tenantId .build() ); ``` + +Invoke payloads use the caller's context-free input codec by default and are sent without SDK framing. This preserves +compatibility with standard Lambda functions, non-Java durable functions, and older Java SDK versions. Enable +`usePersistedSerDesForPayload(true)` only when the target is a compatible Java durable handler with the same persisted +SerDes pipeline—for example, when both handlers use `FileSystemSerDesStage` with a shared filesystem. diff --git a/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FileSystemSerDesStageIntegrationTest.java b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FileSystemSerDesStageIntegrationTest.java index ed3bccb24..1c4049581 100644 --- a/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FileSystemSerDesStageIntegrationTest.java +++ b/sdk-integration-tests/src/test/java/software/amazon/lambda/durable/FileSystemSerDesStageIntegrationTest.java @@ -27,6 +27,7 @@ import software.amazon.awssdk.services.lambda.model.OperationAction; import software.amazon.awssdk.services.lambda.model.OperationStatus; import software.amazon.awssdk.services.lambda.model.OperationType; +import software.amazon.lambda.durable.config.InvokeConfig; import software.amazon.lambda.durable.config.StepConfig; import software.amazon.lambda.durable.config.WaitForConditionConfig; import software.amazon.lambda.durable.execution.DurableExecutor; @@ -171,7 +172,13 @@ void acceptsRawCallbackAndInvokeResultsAndOffloadsInvokePayload() throws Excepti var callback = context.createCallback("approval", String.class); var approval = callback.get(); return context.invoke( - "notify", "target-function", Map.of("approval", approval), String.class); + "notify", + "target-function", + Map.of("approval", approval), + String.class, + InvokeConfig.builder() + .usePersistedSerDesForPayload(true) + .build()); }, config) .withOutputType(String.class); @@ -206,8 +213,12 @@ void callerAndCalleeExchangeOffloadedInvokePayloadAndResult() throws Exception { .withDurableExecutionClient(callerClient) .withSerDes(serDes) .build(); - BiFunction callerHandler = (input, context) -> - context.invoke("call-callee", "callee", new CrossInvokeRequest(input), CrossInvokeResponse.class); + BiFunction callerHandler = (input, context) -> context.invoke( + "call-callee", + "callee", + new CrossInvokeRequest(input), + CrossInvokeResponse.class, + InvokeConfig.builder().usePersistedSerDesForPayload(true).build()); var callerExecution = executionOperation("caller-invocation", "caller-execution", "\"request\"", OperationStatus.STARTED); @@ -267,6 +278,40 @@ void callerAndCalleeExchangeOffloadedInvokePayloadAndResult() throws Exception { assertEquals(new CrossInvokeResponse("reply:request"), result); } + @Test + void defaultInvokePayloadPreservesStandardAndLegacyJavaWireContracts() { + var callerArn = + "arn:aws:lambda:us-east-1:123456789012:function:caller:1/durable-execution/caller-execution/caller-invocation"; + var callerClient = new LocalMemoryExecutionClient(); + var callerConfig = DurableConfig.builder() + .withDurableExecutionClient(callerClient) + .withSerDes(filesystemPipeline()) + .build(); + BiFunction callerHandler = (input, context) -> + context.invoke("call-standard", "standard", new CrossInvokeRequest(input), String.class); + var callerExecution = + executionOperation("caller-invocation", "caller-execution", "\"request\"", OperationStatus.STARTED); + + var pending = DurableExecutor.execute( + durableInput(callerArn, callerExecution, List.of(), List.of()), + null, + TypeToken.get(String.class), + callerHandler, + callerConfig); + + assertEquals(ExecutionStatus.PENDING, pending.status()); + var invokePayload = callerClient.getOperationUpdates().stream() + .filter(update -> + update.type() == OperationType.CHAINED_INVOKE && update.action() == OperationAction.START) + .findFirst() + .orElseThrow() + .payload(); + assertEquals("{\"value\":\"request\"}", invokePayload); + assertEquals( + new CrossInvokeRequest("request"), + new JacksonSerDes().deserialize(invokePayload, TypeToken.get(CrossInvokeRequest.class))); + } + @Test void repeatedGetUsesInvocationCacheForTheCompletePipeline() { var resultDeserializations = new AtomicInteger(); diff --git a/sdk/src/main/java/software/amazon/lambda/durable/config/InvokeConfig.java b/sdk/src/main/java/software/amazon/lambda/durable/config/InvokeConfig.java index b2a5fd9fb..5eeaf9ee4 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/config/InvokeConfig.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/config/InvokeConfig.java @@ -13,11 +13,13 @@ public class InvokeConfig { private final SerDes payloadSerDes; private final SerDes resultSerDes; private final String tenantId; + private final boolean usePersistedSerDesForPayload; public InvokeConfig(Builder builder) { this.payloadSerDes = builder.payloadSerDes; this.resultSerDes = builder.resultSerDes; this.tenantId = builder.tenantId; + this.usePersistedSerDesForPayload = builder.usePersistedSerDesForPayload; } public SerDes payloadSerDes() { @@ -32,12 +34,17 @@ public String tenantId() { return tenantId; } + /** Returns whether the target should decode this invoke payload with its persisted SerDes pipeline. */ + public boolean usePersistedSerDesForPayload() { + return usePersistedSerDesForPayload; + } + public static Builder builder() { - return new Builder(null, null, null); + return new Builder(null, null, null, false); } public Builder toBuilder() { - return new Builder(payloadSerDes, resultSerDes, tenantId); + return new Builder(payloadSerDes, resultSerDes, tenantId, usePersistedSerDesForPayload); } /** Builder for creating InvokeConfig instances. */ @@ -45,11 +52,14 @@ public static class Builder { private SerDes payloadSerDes; private SerDes resultSerDes; private String tenantId; + private boolean usePersistedSerDesForPayload; - private Builder(SerDes payloadSerDes, SerDes resultSerDes, String tenantId) { + private Builder( + SerDes payloadSerDes, SerDes resultSerDes, String tenantId, boolean usePersistedSerDesForPayload) { this.payloadSerDes = payloadSerDes; this.resultSerDes = resultSerDes; this.tenantId = tenantId; + this.usePersistedSerDesForPayload = usePersistedSerDesForPayload; } /** @@ -69,12 +79,13 @@ public Builder tenantId(String tenantId) { /** * Sets a custom serializer for the invoke operation payload. * - *

If not specified, the invoke operation will use the default SerDes configured for the handler. This allows - * per-invoke customization of serialization behavior, useful for invoke operations that need special handling - * (e.g., custom date formats, encryption, compression). + *

If not specified, the invoke operation uses the handler's context-free input codec by default, or its + * persisted SerDes when {@link #usePersistedSerDesForPayload(boolean)} is enabled. This method allows + * per-invoke customization of serialization behavior, useful for invoke operations that need special handling. * - *

The target handler deserializes the framed chained-invoke payload with its configured persisted SerDes, so - * a custom payload SerDes must use a compatible wire format. + *

By default, the serialized value is sent unchanged and must match the target function's ordinary input + * wire format. When {@link #usePersistedSerDesForPayload(boolean)} is enabled, it must instead be compatible + * with the target durable handler's persisted SerDes. * * @param payloadSerDes the custom serializer to use, or null to use the default * @return this builder for method chaining @@ -84,6 +95,26 @@ public Builder payloadSerDes(SerDes payloadSerDes) { return this; } + /** + * Selects whether a compatible durable target should deserialize the invoke payload with its persisted SerDes + * pipeline. + * + *

This is disabled by default so standard Lambda functions, non-Java durable functions, and older Java SDK + * versions continue to receive the configured serialized payload unchanged. Enable it only when the target is a + * Java durable handler that supports the SDK's chained-invoke payload frame and configures a compatible + * persisted SerDes pipeline. + * + *

When enabled without an explicit {@link #payloadSerDes(SerDes)}, the caller's persisted SerDes is used. + * Otherwise, the caller's context-free input codec is the default payload serializer. + * + * @param enabled whether the compatible target should use its persisted SerDes pipeline + * @return this builder for method chaining + */ + public Builder usePersistedSerDesForPayload(boolean enabled) { + this.usePersistedSerDesForPayload = enabled; + return this; + } + /** * Sets a custom serializer for the invoke result. * diff --git a/sdk/src/main/java/software/amazon/lambda/durable/context/DurableContextImpl.java b/sdk/src/main/java/software/amazon/lambda/durable/context/DurableContextImpl.java index 0c79165ec..a5b19480f 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/context/DurableContextImpl.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/context/DurableContextImpl.java @@ -177,9 +177,10 @@ public DurableFuture invokeAsync( config = config.toBuilder().serDes(getDurableConfig().getSerDes()).build(); } if (config.payloadSerDes() == null) { - config = config.toBuilder() - .payloadSerDes(getDurableConfig().getSerDes()) - .build(); + var payloadSerDes = config.usePersistedSerDesForPayload() + ? getDurableConfig().getSerDes() + : getDurableConfig().getInputSerDes(); + config = config.toBuilder().payloadSerDes(payloadSerDes).build(); } var operationId = nextOperationId(); diff --git a/sdk/src/main/java/software/amazon/lambda/durable/operation/InvokeOperation.java b/sdk/src/main/java/software/amazon/lambda/durable/operation/InvokeOperation.java index dd7f03b2c..9d60609e9 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/operation/InvokeOperation.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/operation/InvokeOperation.java @@ -74,7 +74,10 @@ private void startInvocation() { .functionName(functionName) .tenantId(invokeConfig.tenantId()) .build()) - .payload(ChainedInvokePayloadFrame.encode(serializedPayload)); + .payload( + invokeConfig.usePersistedSerDesForPayload() + ? ChainedInvokePayloadFrame.encode(serializedPayload) + : serializedPayload); sendOperationUpdate(update); } diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/filesystem/FileSystemSerDesStage.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/filesystem/FileSystemSerDesStage.java index 505ca803a..de5e05df7 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/filesystem/FileSystemSerDesStage.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/filesystem/FileSystemSerDesStage.java @@ -8,18 +8,25 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectReader; import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.Channels; +import java.nio.file.DirectoryStream; import java.nio.file.FileAlreadyExistsException; import java.nio.file.Files; import java.nio.file.LinkOption; import java.nio.file.NoSuchFileException; import java.nio.file.Path; +import java.nio.file.SecureDirectoryStream; import java.nio.file.StandardOpenOption; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; import java.util.HexFormat; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Set; import java.util.UUID; import java.util.function.BiFunction; import java.util.regex.Pattern; @@ -41,6 +48,10 @@ *

Payload files are immutable and created with a single {@code CREATE_NEW} write. Publication does not require hard * links or renames, so the write path is compatible with S3 Files. * + *

The mounted filesystem provider must support {@link SecureDirectoryStream}. The stage traverses relative directory + * handles with symbolic-link following disabled and keeps those handles open through file I/O, preventing a checked + * path from being redirected between validation and access. + * *

Every filesystem envelope includes a SHA-256 payload digest. Deserialization verifies inline values and file * contents against that digest, and file paths must contain the same digest. * @@ -66,7 +77,6 @@ public final class FileSystemSerDesStage implements SerDesStage { private final FileSystemPathEncoding pathEncoding; private final int checkpointEnvelopeLimitBytes; private final BiFunction> previewGenerator; - private volatile Path canonicalBasePath; private FileSystemSerDesStage(Builder builder) { basePath = builder.basePath.toAbsolutePath().normalize(); @@ -186,16 +196,16 @@ private SerializedPayload readPayload( throw new SerDesException("Filesystem SerDes file path does not match its payload digest"); } try { - var realBasePath = validateBasePath(false); - rejectSymbolicLinks(file); - var realDirectory = file.getParent().toRealPath(); - var realFile = file.toRealPath(); - if (!realDirectory.startsWith(realBasePath) - || !realFile.getParent().equals(realDirectory) - || !realFile.equals(file.toRealPath(LinkOption.NOFOLLOW_LINKS))) { - throw new SerDesException("Filesystem SerDes file does not resolve to the expected payload path"); + byte[] storedData; + try (var directory = openSecureDirectory(file.getParent(), false); + var channel = directory + .directory() + .newByteChannel( + file.getFileName(), Set.of(StandardOpenOption.READ, LinkOption.NOFOLLOW_LINKS)); + var input = Channels.newInputStream(channel)) { + storedData = input.readAllBytes(); } - var serialized = new SerializedPayload(payloadType, Files.readAllBytes(realFile)); + var serialized = new SerializedPayload(payloadType, storedData); verifyPayloadDigest(serialized, payloadDigest, context); return serialized; } catch (IOException e) { @@ -276,17 +286,6 @@ private static boolean acceptsCrossExecutionReference(SerDesContext context) { || context.operationType() == OperationType.CHAINED_INVOKE; } - private void rejectSymbolicLinks(Path file) throws IOException { - validateBasePath(false); - var current = basePath; - for (var component : basePath.relativize(file)) { - current = current.resolve(component); - if (Files.isSymbolicLink(current)) { - throw new SerDesException("Filesystem SerDes payload path must not contain symbolic links"); - } - } - } - private static boolean isFilesystemEnvelope(JsonNode envelope) { if (envelope == null || !envelope.isObject() @@ -491,72 +490,94 @@ private String payloadFileName(String payloadDigest, String entityId) { private void writePayload(SerializedPayload payload, Path file) throws IOException { var directory = file.getParent(); - var realBasePath = createDirectoriesWithoutSymbolicLinks(directory); - rejectSymbolicLinks(file); - var realDirectory = directory.toRealPath(); - if (!realDirectory.startsWith(realBasePath)) { - throw new SerDesException("Filesystem SerDes directory resolves outside the configured base path"); - } - try { - Files.write(file, payload.data(), StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE); - } catch (FileAlreadyExistsException failure) { - throw failure; - } catch (IOException failure) { - try { - Files.deleteIfExists(file); - } catch (IOException cleanupFailure) { - failure.addSuppressed(cleanupFailure); + try (var secureDirectory = openSecureDirectory(directory, true)) { + var created = false; + try (var channel = secureDirectory + .directory() + .newByteChannel( + file.getFileName(), + Set.of( + StandardOpenOption.CREATE_NEW, + StandardOpenOption.WRITE, + LinkOption.NOFOLLOW_LINKS))) { + created = true; + var buffer = ByteBuffer.wrap(payload.data()); + while (buffer.hasRemaining()) { + channel.write(buffer); + } + } catch (FileAlreadyExistsException failure) { + throw failure; + } catch (IOException failure) { + if (created) { + try { + secureDirectory.directory().deleteFile(file.getFileName()); + } catch (IOException cleanupFailure) { + failure.addSuppressed(cleanupFailure); + } + } + throw failure; } - throw failure; } } - private Path createDirectoriesWithoutSymbolicLinks(Path directory) throws IOException { - var realBasePath = validateBasePath(true); - var current = basePath; - for (var component : basePath.relativize(directory)) { - current = current.resolve(component); - ensureRealDirectory(current, true); + private SecureDirectoryHandle openSecureDirectory(Path directory, boolean createMissing) throws IOException { + if (directory == null || !directory.startsWith(basePath)) { + throw new SerDesException("Filesystem SerDes directory is outside the configured base path"); } - return realBasePath; - } - - private Path validateBasePath(boolean createMissing) throws IOException { - var current = basePath.getRoot(); - if (current == null) { + var root = basePath.getRoot(); + if (root == null) { throw new SerDesException("Filesystem SerDes base path must be absolute"); } - ensureRealDirectory(current, false); - for (var component : basePath) { - current = current.resolve(component); - ensureRealDirectory(current, createMissing); - } - return retainCanonicalBasePath(basePath.toRealPath()); - } - private static void ensureRealDirectory(Path directory, boolean createMissing) throws IOException { - if (!Files.exists(directory, LinkOption.NOFOLLOW_LINKS)) { - if (!createMissing) { - throw new NoSuchFileException(directory.toString()); - } - try { - Files.createDirectory(directory); - } catch (FileAlreadyExistsException ignored) { - // Validate the entry created by another writer below. + var openedStreams = new ArrayList>(); + try { + var current = requireSecureDirectoryStream(Files.newDirectoryStream(root), openedStreams); + var currentPath = root; + for (var component : root.relativize(directory)) { + var nextPath = currentPath.resolve(component); + DirectoryStream next; + try { + next = current.newDirectoryStream(component, LinkOption.NOFOLLOW_LINKS); + } catch (NoSuchFileException missing) { + if (!createMissing) { + throw missing; + } + try { + Files.createDirectory(nextPath); + } catch (FileAlreadyExistsException ignored) { + // Validate and open the entry relative to the held parent directory below. + } + next = current.newDirectoryStream(component, LinkOption.NOFOLLOW_LINKS); + } + current = requireSecureDirectoryStream(next, openedStreams); + currentPath = nextPath; } + return new SecureDirectoryHandle(current, openedStreams); + } catch (IOException | RuntimeException failure) { + closeDirectoryStreams(openedStreams, failure); + throw failure; } - if (Files.isSymbolicLink(directory) || !Files.isDirectory(directory, LinkOption.NOFOLLOW_LINKS)) { - throw new SerDesException("Filesystem SerDes directory path must contain only real directories"); + } + + @SuppressWarnings("unchecked") + private static SecureDirectoryStream requireSecureDirectoryStream( + DirectoryStream stream, List> openedStreams) { + openedStreams.add(stream); + if (stream instanceof SecureDirectoryStream secureStream) { + return (SecureDirectoryStream) secureStream; } + throw new SerDesException( + "FileSystemSerDesStage requires a filesystem provider with SecureDirectoryStream support"); } - private synchronized Path retainCanonicalBasePath(Path currentBasePath) { - if (canonicalBasePath == null) { - canonicalBasePath = currentBasePath; - } else if (!canonicalBasePath.equals(currentBasePath)) { - throw new SerDesException("Filesystem SerDes base path changed after validation"); + private static void closeDirectoryStreams(List> streams, Throwable failure) { + for (int index = streams.size() - 1; index >= 0; index--) { + try { + streams.get(index).close(); + } catch (IOException closeFailure) { + failure.addSuppressed(closeFailure); + } } - return canonicalBasePath; } private static boolean matchesPublishedPayloadFileName(String actualFileName, String expectedFileName) { @@ -629,6 +650,40 @@ private static String sha256(byte[] value) { private record PayloadOwner(String durableExecutionArn, String entityId) {} + private static final class SecureDirectoryHandle implements AutoCloseable { + private final SecureDirectoryStream directory; + private final List> openedStreams; + + private SecureDirectoryHandle( + SecureDirectoryStream directory, List> openedStreams) { + this.directory = directory; + this.openedStreams = List.copyOf(openedStreams); + } + + private SecureDirectoryStream directory() { + return directory; + } + + @Override + public void close() throws IOException { + IOException failure = null; + for (int index = openedStreams.size() - 1; index >= 0; index--) { + try { + openedStreams.get(index).close(); + } catch (IOException closeFailure) { + if (failure == null) { + failure = closeFailure; + } else { + failure.addSuppressed(closeFailure); + } + } + } + if (failure != null) { + throw failure; + } + } + } + private enum PayloadType { STRING } diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/filesystem/SerDesPreview.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/filesystem/SerDesPreview.java index 96ccdc237..138c4e0bf 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/filesystem/SerDesPreview.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/filesystem/SerDesPreview.java @@ -8,7 +8,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.SerializationFeature; import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; -import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; @@ -74,25 +73,16 @@ private static Map buildPreview(JsonNode root, PreviewConfig con return null; } - var accepted = new ArrayList(); - int estimatedSize = 2; + Map result = new LinkedHashMap<>(); for (var pair : pairs) { - int entrySize = previewEntrySize(pair); - if (estimatedSize + entrySize > config.maxPreviewBytes()) { + var candidate = copy(result); + insert(candidate, pair.path(), pair.value()); + if (serializedSize(candidate) > config.maxPreviewBytes()) { break; } - accepted.add(pair); - estimatedSize += entrySize; - } - if (accepted.isEmpty()) { - return null; - } - - Map result = new LinkedHashMap<>(); - for (var pair : accepted) { - insert(result, pair.path(), pair.value()); + result = candidate; } - return result; + return result.isEmpty() ? null : result; } private static void collect(JsonNode node, String pathPrefix, PreviewConfig config, List pairs) { @@ -153,14 +143,22 @@ private static boolean isMatched(String path, List fields) { return false; } - private static int previewEntrySize(PreviewEntry entry) { + private static int serializedSize(Map preview) { try { - var serialized = - MAPPER.writeValueAsString(entry.path()) + ":" + MAPPER.writeValueAsString(entry.value()) + ","; - return serialized.getBytes(StandardCharsets.UTF_8).length; + return MAPPER.writeValueAsBytes(preview).length; } catch (JsonProcessingException e) { - throw new SerDesException("Failed to estimate preview size", e); + throw new SerDesException("Failed to measure preview size", e); + } + } + + @SuppressWarnings("unchecked") + private static Map copy(Map source) { + var copy = new LinkedHashMap(); + for (var entry : source.entrySet()) { + var value = entry.getValue(); + copy.put(entry.getKey(), value instanceof Map nested ? copy((Map) nested) : value); } + return copy; } @SuppressWarnings("unchecked") diff --git a/sdk/src/test/java/software/amazon/lambda/durable/serde/filesystem/FileSystemSerDesStageTest.java b/sdk/src/test/java/software/amazon/lambda/durable/serde/filesystem/FileSystemSerDesStageTest.java index f6a212a87..9475a78bd 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/serde/filesystem/FileSystemSerDesStageTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/serde/filesystem/FileSystemSerDesStageTest.java @@ -301,7 +301,7 @@ void rejectsMissingOrMalformedPayloadDigest() throws Exception { } @Test - void writesOnFileSystemsWithoutHardLinkSupport() throws Exception { + void failsClosedWhenTheFileSystemProviderLacksSecureDirectoryStreams() throws Exception { var archive = basePath.resolve("payloads.zip"); try (var fileSystem = FileSystems.newFileSystem(URI.create("jar:" + archive.toUri()), Map.of("create", "true"))) { @@ -309,20 +309,10 @@ void writesOnFileSystemsWithoutHardLinkSupport() throws Exception { var serDes = stringCodec() .then(FileSystemSerDesStage.builder(archiveBasePath).build()); - var envelope = new SerDesRunner(null).serialize(serDes, "expected", context()); - var file = fileSystem.getPath(MAPPER.readTree(envelope).get("file").textValue()); - var hash = HexFormat.of() - .formatHex( - MessageDigest.getInstance("SHA-256").digest("expected".getBytes(StandardCharsets.UTF_8))); - var fileName = file.getFileName().toString(); - assertTrue(fileName.contains(hash)); - - assertEquals("expected", Files.readString(file)); - assertTrue(fileName.matches(".*-" + hash + "-[0-9a-f-]{36}\\.payload")); - assertTrue(fileName.endsWith(".payload")); - assertEquals( - "expected", - new SerDesRunner(null).deserialize(serDes, envelope, TypeToken.get(String.class), context())); + var failure = assertThrows( + SerDesException.class, () -> new SerDesRunner(null).serialize(serDes, "expected", context())); + + assertCauseMessage(failure, "SecureDirectoryStream support"); } } @@ -725,6 +715,22 @@ void rejectsSymbolicLinkDirectoriesWhenWriting() throws Exception { } } + @Test + void rejectsSymbolicLinkDirectoriesWhenReading() throws Exception { + var serDes = stringCodec().then(FileSystemSerDesStage.builder(basePath).build()); + var runner = new SerDesRunner(null); + var envelope = runner.serialize(serDes, "payload", context()); + var orders = basePath.resolve("orders"); + var outside = Files.createTempDirectory(basePath.getParent(), "outside-payloads-"); + var outsideOrders = outside.resolve("orders"); + Files.move(orders, outsideOrders); + Files.createSymbolicLink(orders, outsideOrders); + + assertThrows( + SerDesException.class, + () -> runner.deserialize(serDes, envelope, TypeToken.get(String.class), context())); + } + @Test void rejectsSymbolicLinkConfiguredBasePathAndAncestors() throws Exception { var outsideRoot = Files.createTempDirectory(basePath.getParent(), "outside-root-"); diff --git a/sdk/src/test/java/software/amazon/lambda/durable/serde/filesystem/SerDesPreviewTest.java b/sdk/src/test/java/software/amazon/lambda/durable/serde/filesystem/SerDesPreviewTest.java index 2e874f9c3..a633594a4 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/serde/filesystem/SerDesPreviewTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/serde/filesystem/SerDesPreviewTest.java @@ -115,6 +115,25 @@ void customMaskStringAndByteBudgetAreApplied() { assertTrue(preview.containsKey("first")); } + @Test + void nestedPreviewUsesTheExactSerializedByteBudget() { + var value = Map.of("a", Map.of("b", "x")); + + var tooSmall = SerDesPreview.buildPreview( + value, + PreviewConfig.builder(PreviewMode.INCLUDE_ALL) + .maxPreviewBytes(14) + .build()); + var exactFit = SerDesPreview.buildPreview( + value, + PreviewConfig.builder(PreviewMode.INCLUDE_ALL) + .maxPreviewBytes(15) + .build()); + + assertNull(tooSmall); + assertEquals(value, exactFit); + } + @Test void returnsNullWhenNoFieldsAreVisibleOrValueIsNotAnObject() { var config = PreviewConfig.builder(PreviewMode.EXCLUDE_ALL).build(); From b78cc4fd78536e183c2100b4b6b47c4c2e04ffae Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Wed, 26 Aug 2026 04:53:02 +0000 Subject: [PATCH 55/56] Rebind forwarded step exceptions by attempt --- .../SerializableDurableOperation.java | 14 +- .../durable/operation/StepOperation.java | 2 +- .../operation/WaitForConditionOperation.java | 2 +- .../durable/operation/StepOperationTest.java | 125 ++++++++++++++++++ .../WaitForConditionOperationTest.java | 121 +++++++++++++++++ 5 files changed, 261 insertions(+), 3 deletions(-) diff --git a/sdk/src/main/java/software/amazon/lambda/durable/operation/SerializableDurableOperation.java b/sdk/src/main/java/software/amazon/lambda/durable/operation/SerializableDurableOperation.java index cebe55662..5c3552bcf 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/operation/SerializableDurableOperation.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/operation/SerializableDurableOperation.java @@ -164,12 +164,24 @@ protected ErrorObject serializeException(Throwable throwable, Integer attempt) { * that data using the parent's unrelated entity identity. */ protected ErrorObject rebindForwardedException(DurableOperationException exception) { + return rebindForwardedException(exception, null); + } + + /** + * Re-serializes an exception forwarded from another durable operation under this operation's attempt context. + * + * @param exception the forwarded durable operation exception + * @param attempt the receiving operation's attempt, or {@code null} when attempts do not apply + * @return error data owned by this operation when the original exception can be reconstructed; otherwise the + * forwarded error data + */ + protected ErrorObject rebindForwardedException(DurableOperationException exception, Integer attempt) { var error = exception.getErrorObject(); if (error == null || exception.getOperation() == null) { return error; } var original = exception.deserializedError(); - return original != null ? serializeException(original) : error; + return original != null ? serializeException(original, attempt) : error; } private boolean shouldDeserializeAfterSerialization() { diff --git a/sdk/src/main/java/software/amazon/lambda/durable/operation/StepOperation.java b/sdk/src/main/java/software/amazon/lambda/durable/operation/StepOperation.java index 09a86ff2e..52cfdb98d 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/operation/StepOperation.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/operation/StepOperation.java @@ -169,7 +169,7 @@ private void handleStepFailure(Throwable exception, int attempt) { final ErrorObject errorObject; if (exception instanceof DurableOperationException durableOperationException) { - errorObject = durableOperationException.getErrorObject(); + errorObject = rebindForwardedException(durableOperationException, attempt); } else { errorObject = serializeException(exception, attempt); } diff --git a/sdk/src/main/java/software/amazon/lambda/durable/operation/WaitForConditionOperation.java b/sdk/src/main/java/software/amazon/lambda/durable/operation/WaitForConditionOperation.java index da9d23df3..b3a51c7c7 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/operation/WaitForConditionOperation.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/operation/WaitForConditionOperation.java @@ -183,7 +183,7 @@ private void handleCheckFailure(Throwable exception, int attempt) { } final var errorObject = (exception instanceof DurableOperationException durableOpEx) - ? durableOpEx.getErrorObject() + ? rebindForwardedException(durableOpEx, attempt) : serializeException(exception, attempt); // Checkpoint FAIL diff --git a/sdk/src/test/java/software/amazon/lambda/durable/operation/StepOperationTest.java b/sdk/src/test/java/software/amazon/lambda/durable/operation/StepOperationTest.java index 669a38714..0e4922a22 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/operation/StepOperationTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/operation/StepOperationTest.java @@ -5,19 +5,27 @@ import static org.junit.jupiter.api.Assertions.*; import static org.mockito.Mockito.*; +import java.nio.file.Path; import java.util.List; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.Executors; import java.util.concurrent.atomic.AtomicReference; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import software.amazon.awssdk.services.lambda.model.CallbackDetails; import software.amazon.awssdk.services.lambda.model.ErrorObject; import software.amazon.awssdk.services.lambda.model.Operation; +import software.amazon.awssdk.services.lambda.model.OperationAction; import software.amazon.awssdk.services.lambda.model.OperationStatus; +import software.amazon.awssdk.services.lambda.model.OperationType; +import software.amazon.awssdk.services.lambda.model.OperationUpdate; import software.amazon.awssdk.services.lambda.model.StepDetails; import software.amazon.lambda.durable.DurableConfig; import software.amazon.lambda.durable.TypeToken; import software.amazon.lambda.durable.config.StepConfig; import software.amazon.lambda.durable.context.DurableContextImpl; +import software.amazon.lambda.durable.exception.CallbackFailedException; import software.amazon.lambda.durable.exception.StepFailedException; import software.amazon.lambda.durable.exception.StepInterruptedException; import software.amazon.lambda.durable.execution.ExecutionManager; @@ -25,12 +33,19 @@ import software.amazon.lambda.durable.execution.ThreadType; import software.amazon.lambda.durable.model.OperationIdentifier; import software.amazon.lambda.durable.model.OperationSubType; +import software.amazon.lambda.durable.retry.RetryStrategies; import software.amazon.lambda.durable.serde.JacksonSerDes; +import software.amazon.lambda.durable.serde.SerDes; import software.amazon.lambda.durable.serde.SerDesContext; +import software.amazon.lambda.durable.serde.SerDesPayloadKind; +import software.amazon.lambda.durable.serde.SerDesRunner; import software.amazon.lambda.durable.serde.SerDesStage; +import software.amazon.lambda.durable.serde.filesystem.FileSystemSerDesStage; class StepOperationTest { + private static final String DURABLE_EXECUTION_ARN = + "arn:aws:lambda:us-east-1:123456789012:function:test:1/durable-execution/execution/invocation"; private static final String OPERATION_ID = "1"; private static final String OPERATION_NAME = "test-step"; private static final String RESULT = "result"; @@ -39,6 +54,9 @@ class StepOperationTest { private ExecutionManager executionManager; private DurableContextImpl durableContext; + @TempDir + Path basePath; + @BeforeEach void setUp() { executionManager = mock(ExecutionManager.class); @@ -136,6 +154,84 @@ public String deserialize(String data, SerDesContext context) { assertNull(observedContext.get().originalValue()); } + @Test + void forwardedFilesystemExceptionIsReboundForFirstExecutionAndReplay() { + when(executionManager.getDurableExecutionArn()).thenReturn(DURABLE_EXECUTION_ARN); + var serDes = + new JacksonSerDes().then(FileSystemSerDesStage.builder(basePath).build()); + var original = new IllegalArgumentException("callback failed"); + var forwarded = forwardedCallbackFailure(serDes, original); + var failedUpdate = new AtomicReference(); + doAnswer(invocation -> { + var update = invocation.getArgument(0); + if (update.action() == OperationAction.FAIL) { + failedUpdate.set(update); + } + return CompletableFuture.completedFuture(null); + }) + .when(executionManager) + .sendOperationUpdate(any()); + when(executionManager.getOperationAndUpdateReplayState(OPERATION_ID)).thenReturn(null); + + var operation = new StepOperation<>( + OPERATION_IDENTIFIER, + ctx -> { + throw forwarded; + }, + TypeToken.get(String.class), + StepConfig.builder() + .retryStrategy(RetryStrategies.Presets.NO_RETRY) + .serDes(serDes) + .build(), + durableContext); + + operation.execute(); + + verify(executionManager, timeout(5_000)) + .sendOperationUpdate(argThat(update -> update.action() == OperationAction.FAIL)); + var checkpointedError = failedUpdate.get().error(); + var stepContext = SerDesContext.forOperation( + DURABLE_EXECUTION_ARN, + OPERATION_ID, + OPERATION_NAME, + null, + OperationType.STEP, + OperationSubType.STEP, + SerDesPayloadKind.EXCEPTION, + 1); + var rebound = new SerDesRunner(null) + .deserialize( + serDes, + checkpointedError.errorData(), + TypeToken.get(IllegalArgumentException.class), + stepContext); + assertEquals("callback failed", rebound.getMessage()); + + var replayedOperation = Operation.builder() + .id(OPERATION_ID) + .name(OPERATION_NAME) + .type(OperationType.STEP) + .subType(OperationSubType.STEP.getValue()) + .status(OperationStatus.FAILED) + .stepDetails(StepDetails.builder() + .attempt(1) + .error(checkpointedError) + .build()) + .build(); + when(executionManager.getOperationAndUpdateReplayState(OPERATION_ID)).thenReturn(replayedOperation); + var replay = new StepOperation<>( + OPERATION_IDENTIFIER, + ctx -> RESULT, + TypeToken.get(String.class), + StepConfig.builder().serDes(serDes).build(), + durableContext); + + replay.execute(); + + var thrown = assertThrows(IllegalArgumentException.class, replay::get); + assertEquals("callback failed", thrown.getMessage()); + } + @Test void getThrowsOriginalExceptionWhenClassIsAvailable() { var serDes = new JacksonSerDes(); @@ -286,4 +382,33 @@ public CustomTestException(String message) { super(message); } } + + private CallbackFailedException forwardedCallbackFailure(SerDes serDes, RuntimeException original) { + var sourceContext = SerDesContext.forOperation( + DURABLE_EXECUTION_ARN, + "callback-1", + "callback", + null, + OperationType.CALLBACK, + OperationSubType.CALLBACK, + SerDesPayloadKind.EXCEPTION, + null); + var error = ErrorObject.builder() + .errorType(original.getClass().getName()) + .errorMessage(original.getMessage()) + .errorData(new SerDesRunner(null).serialize(serDes, original, sourceContext)) + .build(); + var sourceOperation = Operation.builder() + .id("callback-1") + .name("callback") + .type(OperationType.CALLBACK) + .subType(OperationSubType.CALLBACK.getValue()) + .status(OperationStatus.FAILED) + .callbackDetails(CallbackDetails.builder() + .callbackId("callback-id") + .error(error) + .build()) + .build(); + return new CallbackFailedException(sourceOperation, original); + } } diff --git a/sdk/src/test/java/software/amazon/lambda/durable/operation/WaitForConditionOperationTest.java b/sdk/src/test/java/software/amazon/lambda/durable/operation/WaitForConditionOperationTest.java index 69502a3c3..e2e02e0d3 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/operation/WaitForConditionOperationTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/operation/WaitForConditionOperationTest.java @@ -5,21 +5,28 @@ import static org.junit.jupiter.api.Assertions.*; import static org.mockito.Mockito.*; +import java.nio.file.Path; import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.concurrent.Executors; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import software.amazon.awssdk.services.lambda.model.CallbackDetails; import software.amazon.awssdk.services.lambda.model.ErrorObject; import software.amazon.awssdk.services.lambda.model.Operation; +import software.amazon.awssdk.services.lambda.model.OperationAction; import software.amazon.awssdk.services.lambda.model.OperationStatus; import software.amazon.awssdk.services.lambda.model.OperationType; +import software.amazon.awssdk.services.lambda.model.OperationUpdate; import software.amazon.awssdk.services.lambda.model.StepDetails; import software.amazon.lambda.durable.DurableConfig; import software.amazon.lambda.durable.TypeToken; import software.amazon.lambda.durable.config.WaitForConditionConfig; import software.amazon.lambda.durable.context.DurableContextImpl; +import software.amazon.lambda.durable.exception.CallbackFailedException; import software.amazon.lambda.durable.exception.IllegalDurableOperationException; import software.amazon.lambda.durable.exception.NonDeterministicExecutionException; import software.amazon.lambda.durable.exception.SerDesException; @@ -31,9 +38,16 @@ import software.amazon.lambda.durable.model.OperationSubType; import software.amazon.lambda.durable.model.WaitForConditionResult; import software.amazon.lambda.durable.serde.JacksonSerDes; +import software.amazon.lambda.durable.serde.SerDes; +import software.amazon.lambda.durable.serde.SerDesContext; +import software.amazon.lambda.durable.serde.SerDesPayloadKind; +import software.amazon.lambda.durable.serde.SerDesRunner; +import software.amazon.lambda.durable.serde.filesystem.FileSystemSerDesStage; class WaitForConditionOperationTest { + private static final String DURABLE_EXECUTION_ARN = + "arn:aws:lambda:us-east-1:123456789012:function:test:1/durable-execution/execution/invocation"; private static final String OPERATION_ID = "1"; private static final String OPERATION_NAME = "test-wait-for-condition"; private static final JacksonSerDes SERDES = new JacksonSerDes(); @@ -41,6 +55,9 @@ class WaitForConditionOperationTest { private ExecutionManager executionManager; private DurableContextImpl durableContext; + @TempDir + Path basePath; + @BeforeEach void setUp() { executionManager = mock(ExecutionManager.class); @@ -155,6 +172,81 @@ void replayFailedFallsBackToStepFailedException() { assertThrows(WaitForConditionFailedException.class, operation::get); } + @Test + void forwardedFilesystemExceptionIsReboundForFirstExecutionAndReplay() { + when(executionManager.getDurableExecutionArn()).thenReturn(DURABLE_EXECUTION_ARN); + var serDes = + new JacksonSerDes().then(FileSystemSerDesStage.builder(basePath).build()); + var original = new IllegalArgumentException("callback failed"); + var forwarded = forwardedCallbackFailure(serDes, original); + var failedUpdate = new AtomicReference(); + doAnswer(invocation -> { + var update = invocation.getArgument(0); + if (update.action() == OperationAction.FAIL) { + failedUpdate.set(update); + } + return CompletableFuture.completedFuture(null); + }) + .when(executionManager) + .sendOperationUpdate(any()); + when(executionManager.getOperationAndUpdateReplayState(OPERATION_ID)).thenReturn(null); + var config = WaitForConditionConfig.builder() + .initialState(0) + .serDes(serDes) + .build(); + var operation = createOperation( + (state, ctx) -> { + throw forwarded; + }, + config); + + operation.execute(); + + verify(executionManager, timeout(5_000)) + .sendOperationUpdate(argThat(update -> update.action() == OperationAction.FAIL)); + var checkpointedError = failedUpdate.get().error(); + var waitForConditionContext = SerDesContext.forOperation( + DURABLE_EXECUTION_ARN, + OPERATION_ID, + OPERATION_NAME, + null, + OperationType.STEP, + OperationSubType.WAIT_FOR_CONDITION, + SerDesPayloadKind.EXCEPTION, + 1); + var rebound = new SerDesRunner(null) + .deserialize( + serDes, + checkpointedError.errorData(), + TypeToken.get(IllegalArgumentException.class), + waitForConditionContext); + assertEquals("callback failed", rebound.getMessage()); + + var replayedOperation = Operation.builder() + .id(OPERATION_ID) + .name(OPERATION_NAME) + .type(OperationType.STEP) + .subType(OperationSubType.WAIT_FOR_CONDITION.getValue()) + .status(OperationStatus.FAILED) + .stepDetails(StepDetails.builder() + .attempt(1) + .error(checkpointedError) + .build()) + .build(); + when(executionManager.getOperationAndUpdateReplayState(OPERATION_ID)).thenReturn(replayedOperation); + var replay = createOperation( + (state, ctx) -> WaitForConditionResult.stopPolling(state), + WaitForConditionConfig.builder() + .initialState(0) + .serDes(serDes) + .build()); + + replay.execute(); + + var thrown = assertThrows(IllegalArgumentException.class, replay::get); + assertEquals("callback failed", thrown.getMessage()); + } + // ===== Replay STARTED ===== @Test @@ -389,4 +481,33 @@ void replayStartedWithCorruptCheckpointDataThrowsSerDesException() { assertThrows(SerDesException.class, operation::execute); } + + private CallbackFailedException forwardedCallbackFailure(SerDes serDes, RuntimeException original) { + var sourceContext = SerDesContext.forOperation( + DURABLE_EXECUTION_ARN, + "callback-1", + "callback", + null, + OperationType.CALLBACK, + OperationSubType.CALLBACK, + SerDesPayloadKind.EXCEPTION, + null); + var error = ErrorObject.builder() + .errorType(original.getClass().getName()) + .errorMessage(original.getMessage()) + .errorData(new SerDesRunner(null).serialize(serDes, original, sourceContext)) + .build(); + var sourceOperation = Operation.builder() + .id("callback-1") + .name("callback") + .type(OperationType.CALLBACK) + .subType(OperationSubType.CALLBACK.getValue()) + .status(OperationStatus.FAILED) + .callbackDetails(CallbackDetails.builder() + .callbackId("callback-id") + .error(error) + .build()) + .build(); + return new CallbackFailedException(sourceOperation, original); + } } From 0b38e868f9ab7797c80274554aa064ed028a7628 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Wed, 26 Aug 2026 05:15:34 +0000 Subject: [PATCH 56/56] Preserve scalar arrays in SerDes previews --- .../serde/filesystem/SerDesPreview.java | 21 ++++++++++++++++--- .../serde/filesystem/SerDesPreviewTest.java | 20 ++++++++++++++++++ 2 files changed, 38 insertions(+), 3 deletions(-) diff --git a/sdk/src/main/java/software/amazon/lambda/durable/serde/filesystem/SerDesPreview.java b/sdk/src/main/java/software/amazon/lambda/durable/serde/filesystem/SerDesPreview.java index 138c4e0bf..2479a20e1 100644 --- a/sdk/src/main/java/software/amazon/lambda/durable/serde/filesystem/SerDesPreview.java +++ b/sdk/src/main/java/software/amazon/lambda/durable/serde/filesystem/SerDesPreview.java @@ -27,9 +27,10 @@ private SerDesPreview() {} /** * Builds a preview from an object using include, exclude, mask, path-matching, and byte-budget rules. * - *

Object fields are traversed in their Jackson serialization order. Arrays are flattened into their containing - * path, matching the Python and TypeScript preview behavior. Fields whose names contain dots are skipped because - * they cannot be distinguished from dot-separated paths. + *

Object fields are traversed in their Jackson serialization order. Object arrays are flattened into their + * containing path, while scalar arrays are preserved as field values, matching the Python and TypeScript preview + * behavior. Fields whose names contain dots are skipped because they cannot be distinguished from dot-separated + * paths. * * @return a nested preview map, or {@code null} when no fields are visible */ @@ -118,6 +119,8 @@ private static void collect(JsonNode node, String pathPrefix, PreviewConfig conf } if (masked) { pairs.add(new PreviewEntry(path, config.maskString())); + } else if (isScalarArray(field.getValue())) { + pairs.add(new PreviewEntry(path, MAPPER.convertValue(field.getValue(), Object.class))); } else if (field.getValue().isContainerNode()) { collect(field.getValue(), path, config, pairs); } else { @@ -126,6 +129,18 @@ private static void collect(JsonNode node, String pathPrefix, PreviewConfig conf } } + private static boolean isScalarArray(JsonNode node) { + if (!node.isArray()) { + return false; + } + for (var item : node) { + if (item.isContainerNode()) { + return false; + } + } + return true; + } + private static boolean isMatched(String path, List fields) { for (var field : fields) { if (field.match() == FieldMatchMode.PATH) { diff --git a/sdk/src/test/java/software/amazon/lambda/durable/serde/filesystem/SerDesPreviewTest.java b/sdk/src/test/java/software/amazon/lambda/durable/serde/filesystem/SerDesPreviewTest.java index a633594a4..2d1c53f2d 100644 --- a/sdk/src/test/java/software/amazon/lambda/durable/serde/filesystem/SerDesPreviewTest.java +++ b/sdk/src/test/java/software/amazon/lambda/durable/serde/filesystem/SerDesPreviewTest.java @@ -97,6 +97,26 @@ void arraysMergeFieldsAtTheirContainingPath() { assertEquals(Map.of("id", "first", "email", "second@example.com"), nested(preview, "items")); } + @Test + void includeAllPreservesScalarArrayFields() { + var preview = SerDesPreview.buildPreviewFromJson( + "{\"tags\":[\"a\",\"b\"]}", + PreviewConfig.builder(PreviewMode.INCLUDE_ALL).build()); + + assertEquals(Map.of("tags", List.of("a", "b")), preview); + } + + @Test + void excludeAllPreservesExplicitlyIncludedScalarArrayFields() { + var preview = SerDesPreview.buildPreview( + Map.of("tags", List.of("a", "b"), "hidden", List.of("c")), + PreviewConfig.builder(PreviewMode.EXCLUDE_ALL) + .include(PreviewField.path("tags")) + .build()); + + assertEquals(Map.of("tags", List.of("a", "b")), preview); + } + @Test void customMaskStringAndByteBudgetAreApplied() { var value = new LinkedHashMap();