diff --git a/examples/rmap/basics/main.go b/examples/rmap/basics/main.go index b7e3c6e..719bd5d 100644 --- a/examples/rmap/basics/main.go +++ b/examples/rmap/basics/main.go @@ -14,7 +14,7 @@ func main() { ctx := context.Background() // Create Redis client - rdb := redis.NewClient(&redis.Options{Addr: "localhost:6379", Password: os.Getenv("REDIS_PASSWORD")}) + rdb := redis.NewClient(&redis.Options{Addr: os.Getenv("REDIS_ADDR"), Password: os.Getenv("REDIS_PASSWORD")}) // Make sure Redis is up and running and we can connect to it if err := rdb.Ping(ctx).Err(); err != nil { diff --git a/examples/rmap/multi-nodes/main.go b/examples/rmap/multi-nodes/main.go index dca0599..7e57513 100644 --- a/examples/rmap/multi-nodes/main.go +++ b/examples/rmap/multi-nodes/main.go @@ -18,7 +18,7 @@ func main() { ctx := context.Background() // Create Redis client - rdb := redis.NewClient(&redis.Options{Addr: "localhost:6379", Password: os.Getenv("REDIS_PASSWORD")}) + rdb := redis.NewClient(&redis.Options{Addr: os.Getenv("REDIS_ADDR"), Password: os.Getenv("REDIS_PASSWORD")}) // Make sure Redis is up and running and we can connect to it if err := rdb.Ping(ctx).Err(); err != nil { diff --git a/examples/streaming/README.md b/examples/streaming/README.md index 6905002..5b2127c 100644 --- a/examples/streaming/README.md +++ b/examples/streaming/README.md @@ -10,30 +10,34 @@ To run the examples, follow these steps: 2. Open a terminal or command prompt. + The examples read `REDIS_ADDR` and `REDIS_PASSWORD`; from the repository + root, `source .env` selects the default local Redis configuration. + 3. Clone the `goadesign/pulse` repository by running the following command: - ``` + ```bash git clone https://github.com/goadesign/pulse.git ``` 4. Change into the example directory (e.g. `examples/streaming/single-reader`): - ``` + ```bash cd pulse/examples/streaming/single-reader ``` - ``` -5. Install the required dependencies by running the following command: - ``` - go get github.com/redis/go-redis/v9 goa.design/pulse/rmap +5. Download the repository's pinned dependencies: + ```bash + go mod download ``` 6. Build the Go program by executing the following command: - ``` + ```bash go build ``` 7. Run the program using the following command: - ``` + ```bash ./single-reader ``` This will execute the program and demonstrate the basic operations on streaming. +The [`exact-publication`](exact-publication/main.go) example shows +deadline-owned `AddOnce` retries and read-only snapshots. diff --git a/examples/streaming/exact-publication/main.go b/examples/streaming/exact-publication/main.go new file mode 100644 index 0000000..41c1c57 --- /dev/null +++ b/examples/streaming/exact-publication/main.go @@ -0,0 +1,70 @@ +package main + +import ( + "context" + "fmt" + "os" + "time" + + "github.com/redis/go-redis/v9" + + "goa.design/pulse/streaming" + "goa.design/pulse/streaming/options" +) + +func main() { + ctx := context.Background() + rdb := redis.NewClient(&redis.Options{ + Addr: os.Getenv("REDIS_ADDR"), + Password: os.Getenv("REDIS_PASSWORD"), + }) + if err := rdb.Ping(ctx).Err(); err != nil { + panic(err) + } + + deadline := time.Now().Add(time.Minute).Truncate(time.Millisecond) + stream, err := streaming.NewStream( + "exact-publication", + rdb, + options.WithStreamDeadline(deadline), + options.WithStreamMaxLen(1_000), + ) + if err != nil { + panic(err) + } + defer func() { + if err := stream.Destroy(ctx); err != nil { + panic(err) + } + }() + + first, err := stream.AddOnce( + ctx, + "facility-42:alarm-7", + "alarm-opened", + []byte("high discharge pressure"), + options.WithTopic("alarms"), + ) + if err != nil { + panic(err) + } + retry, err := stream.AddOnce( + ctx, + "facility-42:alarm-7", + "alarm-opened", + []byte("high discharge pressure"), + options.WithTopic("alarms"), + ) + if err != nil { + panic(err) + } + fmt.Printf("first=%s retry=%s\n", first, retry) + + events, err := stream.Snapshot(ctx) + if err != nil { + panic(err) + } + for _, event := range events { + fmt.Printf("%s %s: %s\n", event.ID(), event.EventName(), event.Payload()) + } +} diff --git a/examples/streaming/multi-readers/main.go b/examples/streaming/multi-readers/main.go index b8baefd..48cec99 100644 --- a/examples/streaming/multi-readers/main.go +++ b/examples/streaming/multi-readers/main.go @@ -13,7 +13,7 @@ import ( func main() { // Create Redis client - rdb := redis.NewClient(&redis.Options{Addr: "localhost:6379", Password: os.Getenv("REDIS_PASSWORD")}) + rdb := redis.NewClient(&redis.Options{Addr: os.Getenv("REDIS_ADDR"), Password: os.Getenv("REDIS_PASSWORD")}) ctx := context.Background() // Make sure Redis is up and running and we can connect to it diff --git a/examples/streaming/multi-sinks/main.go b/examples/streaming/multi-sinks/main.go index 3a8ae08..7206c09 100644 --- a/examples/streaming/multi-sinks/main.go +++ b/examples/streaming/multi-sinks/main.go @@ -14,7 +14,7 @@ import ( // NOTE: the example below does not handle errors for brevity. func main() { // Create Redis client - rdb := redis.NewClient(&redis.Options{Addr: "localhost:6379", Password: os.Getenv("REDIS_PASSWORD")}) + rdb := redis.NewClient(&redis.Options{Addr: os.Getenv("REDIS_ADDR"), Password: os.Getenv("REDIS_PASSWORD")}) ctx := context.Background() // Make sure Redis is up and running and we can connect to it @@ -29,7 +29,11 @@ func main() { } // Don't forget to destroy the stream when done - defer stream.Destroy(ctx) + defer func() { + if err := stream.Destroy(ctx); err != nil { + panic(err) + } + }() // Write 2 events to the stream id1, err := stream.Add(ctx, "event 1", []byte("payload 1")) @@ -54,7 +58,11 @@ func main() { } // Don't forget to close the sink when done - defer sink1.Close(ctx) + defer func() { + if err := sink1.Close(ctx); err != nil { + panic(err) + } + }() // Read and acknowlege event ev := <-sink1.Subscribe() @@ -70,12 +78,16 @@ func main() { if err != nil { panic(err) } - defer sink2.Close(ctx) + defer func() { + if err := sink2.Close(ctx); err != nil { + panic(err) + } + }() // Read second event ev = <-sink2.Subscribe() fmt.Printf("sink 2, event: %s, payload: %s\n", ev.EventName, ev.Payload) - if sink2.Ack(ctx, ev); err != nil { + if err := sink2.Ack(ctx, ev); err != nil { panic(err) } } diff --git a/examples/streaming/multi-streams/main.go b/examples/streaming/multi-streams/main.go index 482a22e..231bd17 100644 --- a/examples/streaming/multi-streams/main.go +++ b/examples/streaming/multi-streams/main.go @@ -14,7 +14,7 @@ import ( // Note: the example below does not handle errors for brevity. func main() { // Create Redis client - rdb := redis.NewClient(&redis.Options{Addr: "localhost:6379", Password: os.Getenv("REDIS_PASSWORD")}) + rdb := redis.NewClient(&redis.Options{Addr: os.Getenv("REDIS_ADDR"), Password: os.Getenv("REDIS_PASSWORD")}) ctx := context.Background() // Make sure Redis is up and running and we can connect to it @@ -29,7 +29,11 @@ func main() { } // Don't forget to destroy the stream when done - defer stream1.Destroy(ctx) + defer func() { + if err := stream1.Destroy(ctx); err != nil { + panic(err) + } + }() // Create sink sink, err := stream1.NewSink(ctx, "multistreams-sink", @@ -40,7 +44,11 @@ func main() { } // Don't forget to close the sink when done - defer sink.Close(ctx) + defer func() { + if err := sink.Close(ctx); err != nil { + panic(err) + } + }() // Subscribe to events c := sink.Subscribe() @@ -57,7 +65,11 @@ func main() { if err != nil { panic(err) } - defer stream2.Destroy(ctx) + defer func() { + if err := stream2.Destroy(ctx); err != nil { + panic(err) + } + }() // Add stream to sink err = sink.AddStream(ctx, stream2) diff --git a/examples/streaming/pub-sub/main.go b/examples/streaming/pub-sub/main.go index b595128..a729814 100644 --- a/examples/streaming/pub-sub/main.go +++ b/examples/streaming/pub-sub/main.go @@ -14,7 +14,7 @@ import ( // NOTE: the example below does not handle errors for brevity. func main() { // Create Redis client - rdb := redis.NewClient(&redis.Options{Addr: "localhost:6379", Password: os.Getenv("REDIS_PASSWORD")}) + rdb := redis.NewClient(&redis.Options{Addr: os.Getenv("REDIS_ADDR"), Password: os.Getenv("REDIS_PASSWORD")}) ctx := context.Background() if err := rdb.Ping(ctx).Err(); err != nil { panic(err) @@ -27,7 +27,11 @@ func main() { } // Don't forget to destroy the stream when done - defer stream.Destroy(ctx) + defer func() { + if err := stream.Destroy(ctx); err != nil { + panic(err) + } + }() // Add a new event to topic "my-topic" id1, err := stream.Add(ctx, @@ -56,7 +60,11 @@ func main() { } // Don't forget to close the sink when done - defer sink.Close(ctx) + defer func() { + if err := sink.Close(ctx); err != nil { + panic(err) + } + }() // Read both events c := sink.Subscribe() diff --git a/examples/streaming/single-reader/main.go b/examples/streaming/single-reader/main.go index bb51235..f5cab2b 100644 --- a/examples/streaming/single-reader/main.go +++ b/examples/streaming/single-reader/main.go @@ -13,7 +13,7 @@ import ( func main() { // Create Redis client - rdb := redis.NewClient(&redis.Options{Addr: "localhost:6379", Password: os.Getenv("REDIS_PASSWORD")}) + rdb := redis.NewClient(&redis.Options{Addr: os.Getenv("REDIS_ADDR"), Password: os.Getenv("REDIS_PASSWORD")}) ctx := context.Background() if err := rdb.Ping(ctx).Err(); err != nil { panic(err) @@ -26,7 +26,11 @@ func main() { } // Don't forget to destroy the stream when done - defer stream.Destroy(ctx) + defer func() { + if err := stream.Destroy(ctx); err != nil { + panic(err) + } + }() // Add a new event id, err := stream.Add(ctx, "event", []byte("payload")) diff --git a/examples/streaming/single-sink/main.go b/examples/streaming/single-sink/main.go index b11a470..e2d5ffd 100644 --- a/examples/streaming/single-sink/main.go +++ b/examples/streaming/single-sink/main.go @@ -13,7 +13,7 @@ import ( func main() { // Create Redis client - rdb := redis.NewClient(&redis.Options{Addr: "localhost:6379", Password: os.Getenv("REDIS_PASSWORD")}) + rdb := redis.NewClient(&redis.Options{Addr: os.Getenv("REDIS_ADDR"), Password: os.Getenv("REDIS_PASSWORD")}) ctx := context.Background() if err := rdb.Ping(ctx).Err(); err != nil { panic(err) @@ -26,7 +26,11 @@ func main() { } // Don't forget to destroy the stream when done - defer stream.Destroy(ctx) + defer func() { + if err := stream.Destroy(ctx); err != nil { + panic(err) + } + }() // Add a new event id, err := stream.Add(ctx, "event", []byte("payload")) @@ -45,7 +49,11 @@ func main() { } // Don't forget to close the sink when done - defer sink.Close(ctx) + defer func() { + if err := sink.Close(ctx); err != nil { + panic(err) + } + }() // Consume event ev := <-sink.Subscribe() diff --git a/pool/node_test.go b/pool/node_test.go index 9f4c964..16ba44d 100644 --- a/pool/node_test.go +++ b/pool/node_test.go @@ -7,6 +7,7 @@ import ( "strconv" "strings" "sync" + "sync/atomic" "testing" "time" @@ -992,14 +993,15 @@ func TestStaleNodeStreamCleanup(t *testing.T) { rdb = ptesting.NewRedisClient(t) node1 = newFastCleanupTestNode(t, ctx, rdb, testName) node2 = newFastCleanupTestNode(t, ctx, rdb, testName) - numJobs = 0 + numJobs atomic.Int64 ) defer ptesting.CleanupRedis(t, rdb, false, testName) - // Configure nodes to send jobs to specific workers + // Configure nodes to send jobs to specific workers. The hasher is shared + // by both nodes and called from concurrent routing and rebalance + // goroutines, so its state must be synchronized. node1.h = &ptesting.Hasher{IndexFunc: func(key string, numBuckets int64) int64 { - numJobs++ - if numJobs > 2 { + if numJobs.Add(1) > 2 { return 0 // to avoid panics on cleanup where jobs get requeued } if key == "job1" { diff --git a/pulse/goroutine_test.go b/pulse/goroutine_test.go index aa06460..e92beb0 100644 --- a/pulse/goroutine_test.go +++ b/pulse/goroutine_test.go @@ -12,6 +12,15 @@ import ( "goa.design/clue/log" ) +type ( + // synchronizedBuilder lets panic-recovery tests observe asynchronous logger + // output without racing the logger's Write call. + synchronizedBuilder struct { + lock sync.Mutex + buf strings.Builder + } +) + func TestGo(t *testing.T) { t.Run("executes function without panic", func(t *testing.T) { var wg sync.WaitGroup @@ -32,7 +41,7 @@ func TestGo(t *testing.T) { var wg sync.WaitGroup wg.Add(1) - var buf strings.Builder + var buf synchronizedBuilder ctx = log.Context(ctx, log.WithOutput(&buf)) logger := ClueLogger(ctx) @@ -56,7 +65,7 @@ func TestGo(t *testing.T) { var wg sync.WaitGroup wg.Add(1) - var buf strings.Builder + var buf synchronizedBuilder ctx = log.Context(ctx, log.WithOutput(&buf)) logger := ClueLogger(ctx) @@ -73,3 +82,17 @@ func TestGo(t *testing.T) { }, 100*time.Millisecond, 10*time.Millisecond, "Log should contain panic message and stack trace") }) } + +// Write serializes logger output writes. +func (b *synchronizedBuilder) Write(p []byte) (int, error) { + b.lock.Lock() + defer b.lock.Unlock() + return b.buf.Write(p) +} + +// String returns a stable snapshot of logger output. +func (b *synchronizedBuilder) String() string { + b.lock.Lock() + defer b.lock.Unlock() + return b.buf.String() +} diff --git a/rmap/map_test.go b/rmap/map_test.go index b113363..2720351 100644 --- a/rmap/map_test.go +++ b/rmap/map_test.go @@ -6,40 +6,25 @@ import ( "bytes" "context" "fmt" - "os" "strings" "sync" "testing" "time" - "github.com/redis/go-redis/v9" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "goa.design/clue/log" "goa.design/pulse/pulse" + ptesting "goa.design/pulse/testing" ) var ( - redisPwd = "redispassword" - redisAddr = "localhost:6379" - wf = time.Second - tck = time.Millisecond + wf = time.Second + tck = time.Millisecond ) -func init() { - if p := os.Getenv("REDIS_PASSWORD"); p != "" { - redisPwd = p - } - if a := os.Getenv("REDIS_ADDR"); a != "" { - redisAddr = a - } -} - func TestMapLocal(t *testing.T) { - rdb := redis.NewClient(&redis.Options{ - Addr: redisAddr, - Password: redisPwd, - }) + rdb := ptesting.NewRedisClient(t) var buf Buffer ctx := context.Background() ctx = log.Context(ctx, log.WithOutput(&buf)) @@ -144,10 +129,7 @@ func TestMapLocal(t *testing.T) { } func TestMapTTLAbsolute(t *testing.T) { - rdb := redis.NewClient(&redis.Options{ - Addr: redisAddr, - Password: redisPwd, - }) + rdb := ptesting.NewRedisClient(t) ctx := context.Background() m, err := Join(ctx, "ttl-absolute", rdb, WithTTL(2*time.Second)) @@ -178,10 +160,7 @@ func TestMapTTLAbsolute(t *testing.T) { } func TestMapTTLSliding(t *testing.T) { - rdb := redis.NewClient(&redis.Options{ - Addr: redisAddr, - Password: redisPwd, - }) + rdb := ptesting.NewRedisClient(t) ctx := context.Background() m, err := Join(ctx, "ttl-sliding", rdb, WithSlidingTTL(2*time.Second)) @@ -212,10 +191,7 @@ func TestMapTTLSliding(t *testing.T) { } func TestSetAndWait(t *testing.T) { - rdb := redis.NewClient(&redis.Options{ - Addr: redisAddr, - Password: redisPwd, - }) + rdb := ptesting.NewRedisClient(t) var buf Buffer ctx := context.Background() ctx = log.Context(ctx, log.WithOutput(&buf)) @@ -273,10 +249,7 @@ func TestSetAndWait(t *testing.T) { } func TestReadAfterClose(t *testing.T) { - rdb := redis.NewClient(&redis.Options{ - Addr: redisAddr, - Password: redisPwd, - }) + rdb := ptesting.NewRedisClient(t) var buf Buffer ctx := context.Background() ctx = log.Context(ctx, log.WithOutput(&buf)) @@ -339,7 +312,7 @@ func TestReadAfterClose(t *testing.T) { } func TestWriteEmptyString(t *testing.T) { - rdb := redis.NewClient(&redis.Options{Addr: redisAddr, Password: redisPwd}) + rdb := ptesting.NewRedisClient(t) ctx := context.Background() m, err := Join(ctx, "test", rdb) require.NoError(t, err) @@ -359,7 +332,7 @@ func TestWriteEmptyString(t *testing.T) { } func TestSetEx(t *testing.T) { - rdb := redis.NewClient(&redis.Options{Addr: redisAddr, Password: redisPwd}) + rdb := ptesting.NewRedisClient(t) ctx := context.Background() m, err := Join(ctx, "test", rdb) require.NoError(t, err) @@ -394,7 +367,7 @@ func TestSetEx(t *testing.T) { } func TestDeleteEx(t *testing.T) { - rdb := redis.NewClient(&redis.Options{Addr: redisAddr, Password: redisPwd}) + rdb := ptesting.NewRedisClient(t) ctx := context.Background() m, err := Join(ctx, "test", rdb) require.NoError(t, err) @@ -419,7 +392,7 @@ func TestDeleteEx(t *testing.T) { } func TestTestAndSetEx(t *testing.T) { - rdb := redis.NewClient(&redis.Options{Addr: redisAddr, Password: redisPwd}) + rdb := ptesting.NewRedisClient(t) ctx := context.Background() m, err := Join(ctx, "test", rdb) require.NoError(t, err) @@ -453,7 +426,7 @@ func TestTestAndSetEx(t *testing.T) { } func TestTestAndDeleteEx(t *testing.T) { - rdb := redis.NewClient(&redis.Options{Addr: redisAddr, Password: redisPwd}) + rdb := ptesting.NewRedisClient(t) ctx := context.Background() m, err := Join(ctx, "test", rdb) require.NoError(t, err) @@ -487,7 +460,7 @@ func TestTestAndDeleteEx(t *testing.T) { } func TestAppendUniqueValues(t *testing.T) { - rdb := redis.NewClient(&redis.Options{Addr: redisAddr, Password: redisPwd}) + rdb := ptesting.NewRedisClient(t) ctx := context.Background() m, err := Join(ctx, "test", rdb) require.NoError(t, err) @@ -531,7 +504,7 @@ func TestAppendUniqueValues(t *testing.T) { } func TestTestAndDelete(t *testing.T) { - rdb := redis.NewClient(&redis.Options{Addr: redisAddr, Password: redisPwd}) + rdb := ptesting.NewRedisClient(t) ctx := context.Background() m, err := Join(ctx, "test", rdb) require.NoError(t, err) @@ -569,7 +542,7 @@ func TestTestAndDelete(t *testing.T) { } func TestTestAndSet(t *testing.T) { - rdb := redis.NewClient(&redis.Options{Addr: redisAddr, Password: redisPwd}) + rdb := ptesting.NewRedisClient(t) ctx := context.Background() m, err := Join(ctx, "test", rdb) require.NoError(t, err) @@ -594,7 +567,7 @@ func TestTestAndSet(t *testing.T) { } func TestTestAndReset(t *testing.T) { - rdb := redis.NewClient(&redis.Options{Addr: redisAddr, Password: redisPwd}) + rdb := ptesting.NewRedisClient(t) ctx := context.Background() m, err := Join(ctx, "test", rdb) require.NoError(t, err) @@ -650,7 +623,7 @@ func TestTestAndReset(t *testing.T) { } func TestArrays(t *testing.T) { - rdb := redis.NewClient(&redis.Options{Addr: redisAddr, Password: redisPwd}) + rdb := ptesting.NewRedisClient(t) ctx := context.Background() m, err := Join(ctx, "test", rdb) require.NoError(t, err) @@ -686,7 +659,7 @@ func TestArrays(t *testing.T) { } func TestIncrement(t *testing.T) { - rdb := redis.NewClient(&redis.Options{Addr: redisAddr, Password: redisPwd}) + rdb := ptesting.NewRedisClient(t) ctx := context.Background() m, err := Join(ctx, "test", rdb) require.NoError(t, err) @@ -712,7 +685,7 @@ func TestIncrement(t *testing.T) { } func TestLogs(t *testing.T) { - rdb := redis.NewClient(&redis.Options{Addr: redisAddr, Password: redisPwd}) + rdb := ptesting.NewRedisClient(t) var buf Buffer ctx := context.Background() ctx = log.Context(ctx, log.WithOutput(&buf), log.WithDebug(), log.WithFormat(log.FormatText)) @@ -764,7 +737,7 @@ func TestJoinErrors(t *testing.T) { } func TestSetErrors(t *testing.T) { - rdb := redis.NewClient(&redis.Options{Addr: redisAddr, Password: redisPwd}) + rdb := ptesting.NewRedisClient(t) ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -782,7 +755,7 @@ func TestSetErrors(t *testing.T) { } func TestAppendValuesErrors(t *testing.T) { - rdb := redis.NewClient(&redis.Options{Addr: redisAddr, Password: redisPwd}) + rdb := ptesting.NewRedisClient(t) ctx := context.Background() m, err := Join(ctx, "test", rdb) @@ -798,7 +771,7 @@ func TestAppendValuesErrors(t *testing.T) { } func TestRemoveValuesErrors(t *testing.T) { - rdb := redis.NewClient(&redis.Options{Addr: redisAddr, Password: redisPwd}) + rdb := ptesting.NewRedisClient(t) ctx := context.Background() m, err := Join(ctx, "test", rdb) @@ -816,7 +789,7 @@ func TestRemoveValuesErrors(t *testing.T) { } func TestReconnect(t *testing.T) { - rdb := redis.NewClient(&redis.Options{Addr: redisAddr, Password: redisPwd}) + rdb := ptesting.NewRedisClient(t) ctx := context.Background() var buf Buffer ctx = log.Context(ctx, log.WithOutput(&buf)) @@ -852,7 +825,7 @@ func TestReconnect(t *testing.T) { } func TestDestroyAllowsReuse(t *testing.T) { - rdb := redis.NewClient(&redis.Options{Addr: redisAddr, Password: redisPwd}) + rdb := ptesting.NewRedisClient(t) ctx := context.Background() m, err := Join(ctx, "destroy-reuse", rdb) @@ -894,7 +867,7 @@ func TestApplyMessageLockedRejectsDestroyWithoutRevision(t *testing.T) { } func TestReconnectPreservesResetEvent(t *testing.T) { - rdb := redis.NewClient(&redis.Options{Addr: redisAddr, Password: redisPwd}) + rdb := ptesting.NewRedisClient(t) ctx := context.Background() observer, err := Join(ctx, "reconnect-reset-event", rdb) diff --git a/streaming/README.md b/streaming/README.md index 4079fe4..5cfd174 100644 --- a/streaming/README.md +++ b/streaming/README.md @@ -4,7 +4,7 @@ Pulse leverages Redis streams to provide scalable and reliable event streams that can be used to implement distributed architectures. Pulse provides a simple API to create and consume streams, for example: -[![Single Reader](../snippets/single-reader.png)](../examples/streaming/single-reader/main.go#L21-L51) +[![Single Reader](../snippets/single-reader.png)](../examples/streaming/single-reader/main.go#L22-L55) The code above creates a stream and adds a new event to it. The event is then consumed by a reader. The reader is closed after the event is consumed. @@ -29,9 +29,21 @@ flowchart LR Multiple readers can be created for the same stream across many nodes. Readers are independent and each instance receives a copy of the same events. Readers can specify a start position for the stream cursor. The default start position -is the last event in the stream. - -[![Multi Reader](../snippets/multi-reader.png)](../examples/streaming/multi-readers/main.go#L45-L72) +is the last event in the stream. `NewReader` is non-creating: the stream must +already have an active lifecycle established by a writer or an explicit +`Stream.Open`, otherwise it returns `ErrStreamNotFound`. + +Reader and sink Redis block durations are always finite and positive. The +default is five seconds; `WithReaderBlockDuration` and +`WithSinkBlockDuration` may shorten or lengthen that bound but constructors +reject zero and negative durations. This guarantees `Close` can finish after at +most the configured blocking read even when a Redis client does not interrupt +the command on context cancellation. The examples use 100 milliseconds. +Constructor contexts bound setup only. Reader and Sink background loops use +their own lifecycle contexts and stop only when `Close` or a terminal lifecycle +error cancels them. + +[![Multi Reader](../snippets/multi-reader.png)](../examples/streaming/multi-readers/main.go#L44-L78) ```mermaid %%{init: {'themeVariables': { 'background': '#282828', 'edgeLabelBackground': '#7A7A7A'}}}%% @@ -55,9 +67,47 @@ flowchart LR linkStyle 4 stroke:#DDDDDD,color:#DDDDDD,stroke-width:3px; ``` +## Exact publication and snapshots + +`Stream.AddOnce` publishes an event exactly once for a non-empty idempotency key +of at most 256 bytes. The length-delimited name, topic, and payload may total at +most 1 MiB. The first call records those exact canonical bytes with the event +ID; no digest is used. An exact retry returns the same ID, while reusing the key +for different bytes returns `ErrIdempotencyConflict`. The event and its +generation-scoped dedupe record +share one Redis-absolute deadline, checked with Redis `TIME`, so retrying an +ambiguous client result is safe even after max-length trimming removes the +event itself. The returned ID remains the publication result even when that +event is no longer retained. + +`AddOnce` takes no deadline argument: the active stream generation's immutable +deadline is the sole expiry authority. A handle may adopt that existing +deadline by omitting retention options; an explicitly configured handle must +match it exactly. +Ordinary `Add` calls use that same absolute expiry and never extend it. At or +after expiry, Add, AddOnce, and +Snapshot return `ErrDeadlineElapsed`; Sink.Close treats expiry as terminal and +still closes its local subscriptions. Reuse of the logical name requires +explicit `Destroy` followed by construction of a new generation. The lifecycle +record intentionally survives expiry until Destroy so stale handles remain +fenced. + +`Stream.Snapshot` performs one generation-fenced Lua `XRANGE COUNT MaxLen+1` +and returns currently retained immutable `SnapshotEvent` values in Redis ID +order. The generation must have a bounded MaxLen; an unbounded stream returns +`ErrSnapshotUnbounded` before Redis runs `XRANGE`. The count bounds Redis script +work, reply size, and client memory. More than MaxLen physical entries violates +the immutable retention contract and returns `ErrSnapshotBoundExceeded` without +materializing the full stream. Snapshot creates no lifecycle, stream, reader, +consumer group, recovery cursor, or acknowledgement state. An uninitialized +name returns `ErrStreamNotFound`. If any externally written entry is malformed, +Snapshot returns a precise error and no partial result. + +[Exact publication example](../examples/streaming/exact-publication/main.go) + ## Stream retention (TTL) -By default, streams have no retention beyond `MaxLen` trimming and must be +By default, streams have no retention beyond max-length trimming and must be deleted explicitly. Pulse can also set a TTL on the Redis key backing a stream: - `options.WithStreamTTL(ttl)` sets an **absolute TTL** (set once when the key @@ -65,8 +115,83 @@ deleted explicitly. Pulse can also set a TTL on the Redis key backing a stream: - `options.WithStreamSlidingTTL(ttl)` sets a **sliding TTL** (refreshed on every published event). +TTL values and Reader/Sink timing values use Redis millisecond precision and +must be at least one millisecond. TTL options cannot be combined with +`options.WithStreamDeadline`. +`options.WithUnboundedStream` disables trimming and requires callers to delete +settled events explicitly; it cannot be combined with `WithStreamMaxLen`. + +The generation lifecycle owns one immutable retention configuration: maximum +length (or explicit unbounded mode), retention mode, TTL duration and sliding +flag, or absolute deadline. A handle with explicit retention options must match +exactly. A handle with no retention options adopts the active generation; +when its first write creates a name, it establishes the default of maximum +length 1000 with no TTL. Reader, Sink, Snapshot, and Destroy handles can +therefore adopt a writer configuration such as maximum length 50,000 with +sliding TTL without restating it. At the quiescent upgrade boundary, the first +upgraded writer that explicitly adopts bounded retention for a legacy flat +stream atomically trims the physical stream to MaxLen before publishing the +lifecycle configuration. Subsequent bounded writes preserve that hard limit. +`Stream.MaxLen` remains exported for v1 source compatibility and reflects the +constructor value, then the adopted generation value after `Open`. Mutating the +field is unsupported: all operations use the immutable construction/adoption +snapshot, so field mutation cannot weaken retention fencing. + The TTL is applied when the Redis stream key is first created, which happens on the first publish (`XADD`) or when creating a sink (`XGROUP CREATE ... MKSTREAM`). +Lifecycle and sink-recovery metadata do not expire with event data. A sink can +therefore recover its acknowledged cursor after the physical stream expires, +without using `$` and skipping events published before recovery. Explicit +`Stream.Destroy` removes recovery and exact-publication metadata; the small +lifecycle record remains as the monotonic incarnation source. + +## Stream incarnations + +`NewStream` is a local constructor and performs no Redis I/O. `Open` or the +first mutating caller-context operation establishes or loads the one active +Redis-owned generation for a logical stream name. Non-creating reads load only +an existing generation and return `ErrStreamNotFound` when none exists. Every +`Stream`, `Reader`, `Sink`, event acker, and added stream then retains that +immutable generation. Publishing, deleting events, consumer-group recovery, +consumer registration, keep-alive refresh, acknowledgement, and destruction +verify it atomically in Redis, so a destroyed generation's metadata can never +be recreated by a concurrent sink. + +`Stream.Destroy` invalidates and deletes exactly its generation, including its +events, consumer groups, recovery cursors, exact-publication records, sink +configuration, keepalives, leases, and membership. An unbound handle for an +absent name returns `ErrStreamNotFound` without creating a lifecycle. Reads that +begin after invalidation return +`ErrStreamDestroyed` and cannot cross into a later generation. An event fetched +before `Destroy` may already be executing in application code; its +`StreamGeneration` token identifies that old incarnation for handlers that +must fence side effects. Physical-key isolation guarantees that no +later-generation event can enter the old reader, sink, or pending-entry list. +Repeating `Destroy` for that generation is idempotent. Generation-qualified +metadata keys are deleted because they are never reused; the one small +lifecycle record remains as the bounded monotonic source for the logical name. + +Generation one records the compatible `pulse:stream:` physical key, so +Pulse adopts streams and consumer groups created before generation fencing +without moving queued or pending events. After explicit destruction, the next +generation receives a distinct physical key. Each handle stores the +lifecycle-selected key, and reads verify that lifecycle both before and after +Redis returns, so an old reader or sink cannot consume a recreated stream. + +Sink stale recovery uses a Redis-time lease with an owner token and monotonic +fencing token. Each `XAUTOCLAIM` and each stale-consumer inspection/deletion +verifies that exact unexpired capability in the same Lua operation as the PEL, +group, keepalive, and membership mutation. A paused predecessor therefore +cannot mutate recovery state after another replica takes over. + +Retention-config adoption is a quiescent upgrade: stop every reader, sink, and +publisher; deploy the new version everywhere; open one handle per logical name +with the intended options and verify its lifecycle using +`redis-cli HGETALL "pulse:stream::lifecycle"`: the active generation, +physical key, and `retention_config` must match that handle. Then resume +traffic. Rollback also requires stopping every user. Do not run an older binary +after the new config is captured; restore the pre-upgrade Redis snapshot or +Destroy and recreate with the rollback version. There is no mixed-version mode. ## Event Sinks @@ -77,16 +202,78 @@ partitions. Event sinks make it possible for multiple nodes to share the same stream cursor. If a stream contains 3 events and 3 nodes are consuming the stream using the same sink (i.e. a sink with the same name), then each node will receive a unique -event from the sequence. Nodes using a different sink (or a reader) will receive -copies of the same events. +event from the sequence; Redis chooses the replica for each event and does not +promise round-robin distribution. Nodes using a different sink name (or a +reader) receive an independent copy. Events read from a sink must be acknowledged by the client. Pulse automatically requeues events added to a sink that have been read by a node but not acknowledged. +Sinks also restore Redis consumer groups deleted outside Pulse. One +generation-scoped recovery hash stores the durable cursor for each named sink. A +cursor advances only across acknowledged events, so pending events and events +added during recovery remain eligible for delivery. `RemoveStream` and +`Sink.Close` detach only that sink instance's membership; they do not delete +the shared consumer group or cursor. Recovery applies only to streams still +attached to the sink instance. `Sink.Close` returns distributed detach failures +without marking the sink closed; callers can retry it with a fresh context. +Removing a stream preserves its shared group and cursor. Pulse deletes an +individual Redis consumer only when that consumer's own PEL is empty, not when +the whole group's PEL is empty. Consumer PEL inspection, optional Redis +consumer deletion, membership removal, and keepalive removal are one +generation-fenced Lua operation. A consumer with pending entries loses local +membership but remains in Redis until stale recovery claims its entries; failed +setup and removal therefore leave no orphaned membership metadata. Stale empty +consumers are also removed by periodic cleanup. + +Each attached `(stream generation, sink name)` owns its own keepalive map, +shared configuration, and fenced stale-recovery lease. Cross-primary sinks +that attach the same secondary stream therefore coordinate on that secondary +without coupling their other streams. Replicas must use identical topic or +topic-pattern filters, acknowledgement mode, acknowledgement grace period, and +initial cursor for each attachment; `NewSink` or `AddStream` fails before group +membership on a mismatch. Keepalives are written once per acknowledgement +grace period, and stale-message checks run every 500 milliseconds by default. + +Sink and Reader constructors reject non-positive block durations and batch +sizes, negative channel buffers, and non-positive sink acknowledgement grace +periods. Defaults are a five-second block, 1000 events per read, a 1000-event +channel buffer, and a 20-second sink acknowledgement grace period. + +`WithSinkNoAck` preserves at-most-once delivery to subscribers by atomically +acknowledging each event and advancing the same recovery cursor before exposing +the event. It does not use Redis's `NOACK` read mode. Both `Sink.Ack` and direct +calls through `Event.Acker` use this recovery-aware acknowledgement operation. + +Deleting the physical Redis stream key itself is different: its stored event +payloads are gone and cannot be recovered. Pulse recreates the empty stream and +group at the durable cursor so all subsequently published events are delivered, +and logs the data-loss condition. `AddStream` and `RemoveStream` return +`ErrSinkClosed` after sink shutdown begins. Removing the final stream from a +Reader or Sink returns `ErrLastStream`, preserving a valid Redis read set. Explicit +`Stream.Destroy` is the only operation that deletes consumer groups and +recovery state; active sink membership neither blocks nor authorizes exact +generation destruction. + +Once Reader or Sink shutdown starts, new subscriptions return an already +closed channel. Reader stream changes return `ErrReaderClosed`, and Sink stream +changes return `ErrSinkClosed`. A sink also atomically acknowledges filtered +events before advancing past them, because no subscriber can acknowledge an +event it never receives. Reader filters remain observational and do not mutate +stream state. + +## Redis topology + +Streaming follows Pulse's standalone Redis contract. Physical event data, +lifecycle metadata, recovery, and replicated-map keys are intentionally +separate readable keys, and atomic Lua operations may access several of them. +`AddOnce` and `Snapshot` follow the same contract. Redis Cluster cross-slot +execution is not supported. + Creating a sink is as simple as: -[![Single Sink](../snippets/single-sink.png)](../examples/streaming/single-sink/main.go#L37-L56) +[![Single Sink](../snippets/single-sink.png)](../examples/streaming/single-sink/main.go#L42-L65) Note a couple of differences with the reader example above: @@ -118,10 +305,11 @@ flowchart LR linkStyle 3 stroke:#DDDDDD,color:#DDDDDD,stroke-width:3px; ``` -As with readers, multiple sinks can be created for the same stream. Copies of -the same event are distributed among all sinks. +As with readers, multiple sink groups can be created for the same stream. +Different sink names each receive a copy; replicas sharing one sink name divide +that group's events. -[![Multi Sink](../snippets/multi-sink.png)](../examples/streaming/multi-sinks/main.go#L58-L79) +[![Multi Sink](../snippets/multi-sink.png)](../examples/streaming/multi-sinks/main.go#L53-L89) ```mermaid %%{init: {'themeVariables': { 'background': '#282828', 'edgeLabelBackground': '#7A7A7A'}}}%% @@ -184,7 +372,7 @@ work or losing unacknowledged events: Readers and sinks can also read concurrently from multiple streams: -[![Multi Stream](../snippets/multi-stream.png)](../examples/streaming/multi-streams/main.go#L61-62) +[![Multi Stream](../snippets/multi-stream.png)](../examples/streaming/multi-streams/main.go#L63-L83) ```mermaid %%{init: {'themeVariables': { 'background': '#282828', 'edgeLabelBackground': '#7A7A7A'}}}%% @@ -214,16 +402,18 @@ flowchart LR ``` `AddStream` can be called at any time to add new streams to a reader or a sink. -Streams can also be removed using `RemoveStream`. +Streams can also be removed using `RemoveStream`. Topic and topic-pattern +filters are mutually exclusive, as are the cursor-start options; constructors +and `AddStream` reject conflicting choices instead of applying precedence. -[![Remove Stream](../snippets/remove-stream.png)](../examples/streaming/multi-streams/main.go#L87-L91) +[![Remove Stream](../snippets/remove-stream.png)](../examples/streaming/multi-streams/main.go#L100-L104) ## Pub/Sub Streams supports a flexible pub/sub mechanism where events can be attached to topics and readers or sinks can define simple or custom matching logic. -[![Pub/Sub](../snippets/pub-sub.png)](../examples/streaming/pub-sub/main.go#L31-L35) +[![Pub/Sub](../snippets/pub-sub.png)](../examples/streaming/pub-sub/main.go#L36-L40) ```mermaid %%{init: {'themeVariables': { 'background': '#282828', 'edgeLabelBackground': '#7A7A7A'}}}%% @@ -259,11 +449,13 @@ flowchart RL Topics can be matched using their name as in the example above or using complex patterns. For example: -[![Pub/Sub](../snippets/pub-sub-pattern.png)](../examples/streaming/pub-sub/main.go#L76-L79) +[![Pub/Sub](../snippets/pub-sub-pattern.png)](../examples/streaming/pub-sub/main.go#L85-L88) -> Note: Event filtering is done client-side in the sink or reader and does not -> affect the underlying stream. This means that events are still stored in the -> stream and can be consumed by other sinks. +> Note: Event filtering is evaluated client-side and never removes an event +> from the stream. Reader filtering is observational. A sink filter +> acknowledges non-matching events in that sink's consumer group because no +> subscriber can acknowledge an event it never receives; other sink names and +> readers remain independent. ## Examples diff --git a/streaming/exact_publication.go b/streaming/exact_publication.go new file mode 100644 index 0000000..e837cb0 --- /dev/null +++ b/streaming/exact_publication.go @@ -0,0 +1,523 @@ +// Exact publication provides generation-scoped idempotent writes and +// side-effect-free snapshots. Redis Lua owns both linearization points so +// retries, Destroy, and concurrent clients observe one canonical result. +package streaming + +import ( + "bytes" + "context" + "encoding/binary" + "fmt" + "strconv" + + redis "github.com/redis/go-redis/v9" + + "goa.design/pulse/streaming/options" +) + +type ( + // SnapshotEvent is an immutable event returned by Stream.Snapshot. + SnapshotEvent struct { + id string + streamName string + generation string + name string + topic string + payload []byte + } +) + +const ( + maxIdempotencyKeyBytes = 256 + maxAddOnceBodyBytes = 1 << 20 +) + +var ( + // addOnceScript verifies the stream generation and absolute deadline, + // resolves the generation-scoped idempotency record, and publishes exactly + // once. Redis TIME is authoritative for deadline admission. + addOnceScript = redis.NewScript(` +local state = redis.call("HGET", KEYS[1], "state") +local generation = redis.call("HGET", KEYS[1], "generation") +local physical = redis.call("HGET", KEYS[1], ARGV[5]) +local deadline = redis.call("HGET", KEYS[1], ARGV[6]) +local ttl_owned = redis.call("HGET", KEYS[1], ARGV[7]) +local retention = redis.call("HGET", KEYS[1], ARGV[15]) +local recreate = false +local adopting_legacy = not generation + +if ARGV[2] ~= "" then + if state ~= ARGV[1] or generation ~= ARGV[2] or physical ~= ARGV[3] then + return redis.error_reply("STREAMDESTROYED") + end +else + if not generation then + if ARGV[8] == "" then + return redis.error_reply("STREAMDEADLINEREQUIRED") + end + generation = "1" + physical = ARGV[4] + recreate = true + elseif state ~= ARGV[1] then + generation = tostring(tonumber(generation) + 1) + physical = ARGV[4] .. ":generation:" .. generation + deadline = false + ttl_owned = false + retention = false + recreate = true + elseif not physical then + physical = ARGV[4] + end +end + +if retention and ARGV[18] == "1" and retention ~= ARGV[16] then + return redis.error_reply("STREAMCONFIGMISMATCH") +end +if deadline then + if ARGV[8] ~= "" and deadline ~= ARGV[8] then + return redis.error_reply("STREAMDEADLINECONFLICT") + end +elseif ttl_owned == "1" then + return redis.error_reply("STREAMDEADLINECONFLICT") +else + if ARGV[8] == "" then + return redis.error_reply("STREAMDEADLINEREQUIRED") + end + deadline = ARGV[8] +end +local now = redis.call("TIME") +local now_ms = tonumber(now[1]) * 1000 + math.floor(tonumber(now[2]) / 1000) +if now_ms >= tonumber(deadline) then + return redis.error_reply("DEADLINEELAPSED") +end + +local dedupe = ARGV[4] .. ":generation:" .. generation .. ":idempotency" +local recovery = physical .. ":sink-recovery:" .. generation +local resources_key = ARGV[4] .. ":generation:" .. generation .. ":resources" +local physical_type = redis.call("TYPE", physical)["ok"] +local dedupe_type = redis.call("TYPE", dedupe)["ok"] +local recovery_type = redis.call("TYPE", recovery)["ok"] +local resources_type = redis.call("TYPE", resources_key)["ok"] +if (physical_type ~= "none" and physical_type ~= "stream") +or (dedupe_type ~= "none" and dedupe_type ~= "hash") +or (recovery_type ~= "none" and recovery_type ~= "hash") +or (resources_type ~= "none" and resources_type ~= "set") then + return redis.error_reply("STREAMRESOURCEINVALID") +end + +local existing = redis.call("HGET", dedupe, ARGV[10]) +if existing then + local separator = string.find(existing, "\0", 1, true) + if not separator then + return redis.error_reply("IDEMPOTENCYINVALID") + end + local event_id = string.sub(existing, 1, separator - 1) + local existing_identity = string.sub(existing, separator + 1) + if existing_identity ~= ARGV[17] then + return redis.error_reply("IDEMPOTENCYCONFLICT") + end + redis.call("PEXPIREAT", physical, deadline) + redis.call("PEXPIREAT", dedupe, deadline) + redis.call("PEXPIREAT", recovery, deadline) + local existing_resources = redis.call("SMEMBERS", resources_key) + for _, resource in ipairs(existing_resources) do + redis.call("PEXPIREAT", resource, deadline) + end + redis.call("PEXPIREAT", resources_key, deadline) + return {generation, physical, deadline, retention, 0, event_id} +end + +if recreate then + if adopting_legacy and ARGV[18] == "1" and tonumber(ARGV[9]) > 0 then + redis.call("XTRIM", physical, "MAXLEN", "=", ARGV[9]) + end + redis.call("HSET", KEYS[1], + "generation", generation, + "state", ARGV[1], + ARGV[5], physical, + ARGV[6], deadline, + ARGV[15], ARGV[16]) + redis.call("HDEL", KEYS[1], ARGV[7]) +elseif redis.call("HGET", KEYS[1], ARGV[5]) == false then + redis.call("HSET", KEYS[1], ARGV[5], physical) + redis.call("HSET", KEYS[1], ARGV[6], deadline) +elseif redis.call("HGET", KEYS[1], ARGV[6]) == false then + redis.call("HSET", KEYS[1], ARGV[6], deadline) +end +if not retention then + redis.call("HSET", KEYS[1], ARGV[15], ARGV[16]) +end + +local event_id +if ARGV[9] == "0" and ARGV[13] == "1" then + event_id = redis.call("XADD", physical, "*", "n", ARGV[11], "p", ARGV[12], "t", ARGV[14]) +elseif ARGV[9] == "0" then + event_id = redis.call("XADD", physical, "*", "n", ARGV[11], "p", ARGV[12]) +elseif ARGV[13] == "1" then + event_id = redis.call("XADD", physical, "MAXLEN", "=", ARGV[9], "*", + "n", ARGV[11], "p", ARGV[12], "t", ARGV[14]) +else + event_id = redis.call("XADD", physical, "MAXLEN", "=", ARGV[9], "*", + "n", ARGV[11], "p", ARGV[12]) +end +redis.call("HSET", dedupe, ARGV[10], event_id .. "\0" .. ARGV[17]) +redis.call("HSET", recovery, "=deadline", deadline) +redis.call("SADD", resources_key, dedupe, recovery) + +redis.call("PEXPIREAT", physical, deadline) +redis.call("PEXPIREAT", dedupe, deadline) +redis.call("PEXPIREAT", recovery, deadline) +local resources = redis.call("SMEMBERS", resources_key) +for _, resource in ipairs(resources) do + redis.call("PEXPIREAT", resource, deadline) +end +redis.call("PEXPIREAT", resources_key, deadline) +return {generation, physical, deadline, retention or ARGV[16], 1, event_id} +`) + + // snapshotScript binds an unbound handle using the same zero-migration + // lifecycle contract, verifies a bound handle, and returns one XRANGE + // result at the operation's single Redis linearization point. + snapshotScript = redis.NewScript(` +local state = redis.call("HGET", KEYS[1], "state") +local generation = redis.call("HGET", KEYS[1], "generation") +if not generation then + return {0} +end +local physical = redis.call("HGET", KEYS[1], ARGV[4]) +local deadline = redis.call("HGET", KEYS[1], ARGV[5]) +if state ~= ARGV[1] or (ARGV[2] ~= "" and generation ~= ARGV[2]) +or (ARGV[3] ~= "" and physical ~= ARGV[3]) then + return redis.error_reply("STREAMDESTROYED") +end +local retention = redis.call("HGET", KEYS[1], ARGV[6]) +if not retention then + return redis.error_reply("STREAMCONFIGMISSING") +end +if ARGV[8] == "1" and retention ~= ARGV[7] then + return redis.error_reply("STREAMCONFIGMISMATCH") +end +if string.find(retention, "|max=0|", 1, true) then + return redis.error_reply("SNAPSHOTUNBOUNDED") +end +local max_len = tonumber(string.match(retention, "|max=(%d+)|")) +if not max_len or max_len <= 0 then + return redis.error_reply("STREAMCONFIGMISSING") +end + +if deadline then + local now = redis.call("TIME") + local now_ms = tonumber(now[1]) * 1000 + math.floor(tonumber(now[2]) / 1000) + if now_ms >= tonumber(deadline) then + return redis.error_reply("DEADLINEELAPSED") + end +end +local events = redis.call("XRANGE", physical, "-", "+", "COUNT", max_len + 1) +if #events > max_len then + return redis.error_reply("SNAPSHOTBOUNDVIOLATION") +end +return {1, generation, physical, deadline or "", retention, events} +`) +) + +// AddOnce publishes one event for idempotencyKey in this stream generation. +// The first call stores the event ID and exact length-delimited event identity +// until the generation deadline. Exact retries return that ID; content changes +// return ErrIdempotencyConflict. The active generation must be deadline-owned; +// a handle with explicit retention options must match that immutable deadline. +func (s *Stream) AddOnce( + ctx context.Context, + idempotencyKey string, + name string, + payload []byte, + opts ...options.AddEvent, +) (string, error) { + if len(idempotencyKey) == 0 { + return "", fmt.Errorf("pulse streaming: idempotency key must not be empty") + } + if len(idempotencyKey) > maxIdempotencyKeyBytes { + return "", fmt.Errorf( + "pulse streaming: idempotency key exceeds %d bytes", + maxIdempotencyKeyBytes, + ) + } + if name == "" { + return "", fmt.Errorf("pulse streaming: event name must not be empty") + } + o := options.ParseAddEventOptions(opts...) + identity, err := canonicalEventIdentity(name, o.Topic, payload) + if err != nil { + return "", err + } + if o.OnlyIfStreamExists { + return "", fmt.Errorf( + "pulse streaming: AddOnce does not support WithOnlyIfStreamExists", + ) + } + s.generationLock.Lock() + defer s.generationLock.Unlock() + bound := s.generation != "" + result, err := addOnceScript.Run( + ctx, + s.rdb, + []string{s.lifecycleKey}, + streamStateActive, + s.generation, + s.key, + streamKey(s.Name), + streamPhysicalKey, + streamDeadlineKey, + streamTTLOwnedKey, + s.requestedDeadline(), + strconv.Itoa(s.maxLen), + idempotencyKey, + name, + payload, + boolString(o.Topic != ""), + o.Topic, + streamConfigKey, + s.retention, + identity, + boolString(s.retentionExplicit), + ).Slice() + if err != nil { + if redis.HasErrorPrefix(err, "IDEMPOTENCYCONFLICT") { + return "", fmt.Errorf( + "%w: key %q", + ErrIdempotencyConflict, + idempotencyKey, + ) + } + return "", s.lifecycleError(err) + } + if len(result) != 6 { + return "", fmt.Errorf("add once script returned %d values", len(result)) + } + generation, physical, canonicalDeadline, retention, err := parseLifecycleIdentity(result[:4]) + if err != nil { + return "", fmt.Errorf("pulse streaming: AddOnce identity: %w", err) + } + parsedDeadline, err := parseDeadline(canonicalDeadline) + if err != nil { + return "", fmt.Errorf("pulse streaming: AddOnce identity: %w", err) + } + eventID, err := parseAddOnceResult(result[4:]) + if err != nil { + return "", err + } + // A bound handle is an immutable capability for one exact generation: the + // script verified the identity, so only the unbound→bound transition may + // write binding state. Post-bind writes would race with the unlocked reads + // every verified operation performs after its own lock passage. + if !bound { + if err := s.applyRetentionConfig(retention); err != nil { + return "", fmt.Errorf("pulse streaming: AddOnce identity: %w", err) + } + s.generation = generation + s.key = physical + s.deadline = parsedDeadline + } + return eventID, nil +} + +// Snapshot returns every event currently retained by this exact bounded +// stream generation in Redis ID order. The immutable MaxLen contract bounds +// Redis script work and result memory. Redis reads at most MaxLen+1 entries and +// returns ErrSnapshotBoundExceeded if physical data violates that invariant; +// unbounded streams return ErrSnapshotUnbounded before XRANGE. Snapshot creates no reader, consumer +// group, cursor, acknowledgement state, or lifecycle. An absent lifecycle +// returns ErrStreamNotFound. A malformed Redis entry fails the whole snapshot. +func (s *Stream) Snapshot(ctx context.Context) ([]SnapshotEvent, error) { + s.generationLock.Lock() + defer s.generationLock.Unlock() + bound := s.generation != "" + raw, err := snapshotScript.Run( + ctx, + s.rdb, + []string{s.lifecycleKey}, + streamStateActive, + s.generation, + s.key, + streamPhysicalKey, + streamDeadlineKey, + streamConfigKey, + s.retention, + boolString(s.retentionExplicit), + ).Slice() + if err != nil { + return nil, s.lifecycleError(err) + } + if len(raw) == 1 { + return nil, ErrStreamNotFound + } + if len(raw) != 6 { + return nil, fmt.Errorf("snapshot script returned %d values", len(raw)) + } + generation, physical, deadline, retention, err := parseLifecycleIdentity(raw[1:5]) + if err != nil { + return nil, fmt.Errorf("pulse streaming: snapshot identity: %w", err) + } + canonicalDeadline, err := parseDeadline(deadline) + if err != nil { + return nil, fmt.Errorf("pulse streaming: snapshot identity: %w", err) + } + rangeResult, ok := raw[5].([]any) + if !ok { + return nil, fmt.Errorf("pulse streaming: snapshot range has invalid type %T", raw[5]) + } + // Only the unbound→bound transition may write binding state; see AddOnce. + if !bound { + if err := s.applyRetentionConfig(retention); err != nil { + return nil, fmt.Errorf("pulse streaming: snapshot identity: %w", err) + } + s.generation = generation + s.key = physical + s.deadline = canonicalDeadline + } + messages, err := decodeSnapshotRange(rangeResult) + if err != nil { + return nil, err + } + events := make([]SnapshotEvent, len(messages)) + for i, message := range messages { + name, topic, payload, err := decodeRedisEvent(message) + if err != nil { + return nil, fmt.Errorf("pulse streaming: snapshot: %w", err) + } + events[i] = SnapshotEvent{ + id: message.ID, + streamName: s.Name, + generation: s.generation, + name: name, + topic: topic, + payload: append([]byte(nil), payload...), + } + } + return events, nil +} + +// ID returns the immutable Redis event ID. +func (e SnapshotEvent) ID() string { + return e.id +} + +// StreamName returns the logical stream name. +func (e SnapshotEvent) StreamName() string { + return e.streamName +} + +// StreamGeneration returns the immutable stream incarnation. +func (e SnapshotEvent) StreamGeneration() string { + return e.generation +} + +// EventName returns the producer-defined event name. +func (e SnapshotEvent) EventName() string { + return e.name +} + +// Topic returns the producer-defined topic, or empty when absent. +func (e SnapshotEvent) Topic() string { + return e.topic +} + +// Payload returns an independent copy of the event payload. +func (e SnapshotEvent) Payload() []byte { + return append([]byte(nil), e.payload...) +} + +// parseAddOnceResult validates the Lua result at the Redis boundary. +func parseAddOnceResult(result []any) (string, error) { + if len(result) != 2 { + return "", fmt.Errorf("add once script returned %d values", len(result)) + } + status, ok := result[0].(int64) + if !ok || (status != 0 && status != 1) { + return "", fmt.Errorf("add once script returned invalid status %T(%v)", result[0], result[0]) + } + eventID, ok := result[1].(string) + if !ok || eventID == "" { + return "", fmt.Errorf("add once script returned invalid event ID %T", result[1]) + } + return eventID, nil +} + +// canonicalEventIdentity length-prefixes exact event bytes so no field +// concatenation can alias another identity. +func canonicalEventIdentity(name, topic string, payload []byte) ([]byte, error) { + total := len(name) + len(topic) + len(payload) + if total > maxAddOnceBodyBytes { + return nil, fmt.Errorf( + "pulse streaming: AddOnce event identity exceeds %d bytes", + maxAddOnceBodyBytes, + ) + } + var identity bytes.Buffer + for _, field := range [][]byte{[]byte(name), []byte(topic), payload} { + if err := binary.Write(&identity, binary.BigEndian, uint64(len(field))); err != nil { + return nil, fmt.Errorf("pulse streaming: encode AddOnce event identity: %w", err) + } + if _, err := identity.Write(field); err != nil { + return nil, fmt.Errorf("pulse streaming: encode AddOnce event identity: %w", err) + } + } + return identity.Bytes(), nil +} + +// decodeSnapshotRange validates Redis's nested XRANGE response without +// returning a partial message set. +func decodeSnapshotRange(raw []any) ([]redis.XMessage, error) { + messages := make([]redis.XMessage, len(raw)) + for i, value := range raw { + entry, ok := value.([]any) + if !ok || len(entry) != 2 { + return nil, fmt.Errorf( + "pulse streaming: snapshot entry %d must contain ID and fields, got %T", + i, + value, + ) + } + id, ok := entry[0].(string) + if !ok || id == "" { + return nil, fmt.Errorf("pulse streaming: snapshot entry %d has invalid ID %T", i, entry[0]) + } + fields, ok := entry[1].([]any) + if !ok || len(fields)%2 != 0 { + return nil, fmt.Errorf("pulse streaming: snapshot event %s has invalid fields %T", id, entry[1]) + } + values := make(map[string]any, len(fields)/2) + for field := 0; field < len(fields); field += 2 { + key, ok := fields[field].(string) + if !ok { + return nil, fmt.Errorf( + "pulse streaming: snapshot event %s has invalid field name %T", + id, + fields[field], + ) + } + fieldValue, ok := fields[field+1].(string) + if !ok { + return nil, fmt.Errorf( + "pulse streaming: snapshot event %s field %q has invalid value %T", + id, + key, + fields[field+1], + ) + } + values[key] = fieldValue + } + messages[i] = redis.XMessage{ID: id, Values: values} + } + return messages, nil +} + +// idempotencyKeyMap stores generation-scoped publication IDs and content +// digests independently from MAXLEN-trimmed event data. +func idempotencyKeyMap(stream *Stream) string { + return fmt.Sprintf( + "%s%s:generation:%s:idempotency", + streamKeyPrefix, + stream.Name, + stream.generation, + ) +} diff --git a/streaming/exact_publication_test.go b/streaming/exact_publication_test.go new file mode 100644 index 0000000..6ef87ea --- /dev/null +++ b/streaming/exact_publication_test.go @@ -0,0 +1,639 @@ +package streaming + +import ( + "context" + "errors" + "net" + "strconv" + "sync" + "sync/atomic" + "testing" + "time" + + redis "github.com/redis/go-redis/v9" + "github.com/stretchr/testify/require" + + "goa.design/pulse/streaming/options" + ptesting "goa.design/pulse/testing" +) + +type ( + // ambiguousAddOnceHook returns one client error after Redis committed the + // selected AddOnce script, reproducing an ambiguous network outcome. + ambiguousAddOnceHook struct { + fail atomic.Bool + err error + } +) + +func TestAddOnceConcurrentClientsPublishExactlyOnce(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + other := redis.NewClient(rdb.Options()) + defer func() { require.NoError(t, other.Close()) }() + ctx := ptesting.NewTestContext(t) + deadline := time.Now().Add(10 * time.Second).Truncate(time.Millisecond) + const clients = 12 + results := make(chan string, clients) + errs := make(chan error, clients) + var wait sync.WaitGroup + for index := range clients { + wait.Add(1) + go func(client *redis.Client) { + defer wait.Done() + stream, err := NewStream(t.Name(), client, options.WithStreamDeadline(deadline)) + if err != nil { + errs <- err + return + } + result, err := stream.AddOnce( + ctx, + "command", + "created", + []byte("payload"), + options.WithTopic("alarms"), + ) + if err != nil { + errs <- err + return + } + results <- result + }([]*redis.Client{rdb, other}[index%2]) + } + wait.Wait() + close(results) + close(errs) + for err := range errs { + require.NoError(t, err) + } + var ( + eventID string + ) + for result := range results { + if eventID == "" { + eventID = result + } + require.Equal(t, eventID, result) + } + require.EqualValues(t, 1, rdb.XLen(ctx, streamKey(t.Name())).Val()) +} + +func TestAddOnceConflictAndAmbiguousCommitRetry(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + hook := &ambiguousAddOnceHook{err: errors.New("ambiguous client result")} + rdb.AddHook(hook) + ctx := ptesting.NewTestContext(t) + deadline := time.Now().Add(10 * time.Second).Truncate(time.Millisecond) + stream, err := NewStream(t.Name(), rdb, options.WithStreamDeadline(deadline)) + require.NoError(t, err) + + hook.fail.Store(true) + _, err = stream.AddOnce(ctx, "command", "created", []byte("payload")) + require.ErrorIs(t, err, hook.err) + retry, err := stream.AddOnce(ctx, "command", "created", []byte("payload")) + require.NoError(t, err) + require.NotEmpty(t, retry) + require.EqualValues(t, 1, rdb.XLen(ctx, stream.key).Val()) + + _, err = stream.AddOnce(ctx, "command", "created", []byte("different")) + require.ErrorIs(t, err, ErrIdempotencyConflict) + require.EqualValues(t, 1, rdb.XLen(ctx, stream.key).Val()) + require.NoError(t, stream.Destroy(ctx)) +} + +func TestAddOnceMetadataSurvivesMaxLenAndScriptFlush(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + deadline := time.Now().Add(10 * time.Second).Truncate(time.Millisecond) + stream, err := NewStream( + t.Name(), + rdb, + options.WithStreamMaxLen(1), + options.WithStreamDeadline(deadline), + ) + require.NoError(t, err) + first, err := stream.AddOnce(ctx, "first", "first", []byte("payload")) + require.NoError(t, err) + for i := 0; i < 200; i++ { + _, err = stream.Add(ctx, "trim", []byte{byte(i)}) + require.NoError(t, err) + } + require.Empty(t, rdb.XRangeN(ctx, stream.key, first, first, 1).Val()) + + require.NoError(t, rdb.ScriptFlush(ctx).Err()) + retry, err := stream.AddOnce(ctx, "first", "first", []byte("payload")) + require.NoError(t, err) + require.Equal(t, first, retry) + _, err = stream.AddOnce(ctx, "second", "second", []byte("payload")) + require.NoError(t, err) + _, err = stream.Snapshot(ctx) + require.NoError(t, err) + require.NoError(t, stream.Destroy(ctx)) +} + +func TestAddOnceAdoptsPreGenerationFlatStreamInPlace(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + flatKey := streamKey(t.Name()) + legacyID, err := rdb.XAdd(ctx, &redis.XAddArgs{ + Stream: flatKey, + Values: map[string]any{nameKey: "legacy", payloadKey: "payload"}, + }).Result() + require.NoError(t, err) + deadline := time.Now().Add(10 * time.Second).Truncate(time.Millisecond) + stream, err := NewStream(t.Name(), rdb, options.WithStreamDeadline(deadline)) + require.NoError(t, err) + published, err := stream.AddOnce(ctx, "new", "current", []byte("payload")) + require.NoError(t, err) + require.Equal(t, flatKey, stream.key) + require.Equal(t, "1", stream.Generation()) + events, err := stream.Snapshot(ctx) + require.NoError(t, err) + require.Equal(t, []string{legacyID, published}, []string{events[0].ID(), events[1].ID()}) + require.NoError(t, stream.Destroy(ctx)) +} + +func TestExplicitBoundedLegacyAdoptionTrimsBeforePublishingLifecycle(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + const ( + legacyEvents = 5_000 + maxLen = 100 + ) + flatKey := streamKey(t.Name()) + pipe := rdb.Pipeline() + for index := 0; index < legacyEvents; index++ { + pipe.XAdd(ctx, &redis.XAddArgs{ + Stream: flatKey, + Values: map[string]any{ + nameKey: "legacy", + payloadKey: strconv.Itoa(index), + }, + }) + } + _, err := pipe.Exec(ctx) + require.NoError(t, err) + + stream, err := NewStream(t.Name(), rdb, options.WithStreamMaxLen(maxLen)) + require.NoError(t, err) + require.NoError(t, stream.Open(ctx)) + require.EqualValues(t, maxLen, rdb.XLen(ctx, flatKey).Val()) + events, err := stream.Snapshot(ctx) + require.NoError(t, err) + require.Len(t, events, maxLen) + require.Equal(t, strconv.Itoa(legacyEvents-maxLen), string(events[0].Payload())) + require.NoError(t, stream.Destroy(ctx)) +} + +func TestSnapshotRejectsPhysicalBoundViolation(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + const maxLen = 3 + stream, err := NewStream(t.Name(), rdb, options.WithStreamMaxLen(maxLen)) + require.NoError(t, err) + require.NoError(t, stream.Open(ctx)) + for index := 0; index <= maxLen; index++ { + require.NoError(t, rdb.XAdd(ctx, &redis.XAddArgs{ + Stream: stream.key, + Values: map[string]any{nameKey: "event", payloadKey: strconv.Itoa(index)}, + }).Err()) + } + _, err = stream.Snapshot(ctx) + require.ErrorIs(t, err, ErrSnapshotBoundExceeded) + require.NoError(t, stream.Destroy(ctx)) +} + +func TestDeadlineIsAbsoluteAndNeverExtended(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + deadline := time.Now().Add(1500 * time.Millisecond).Truncate(time.Millisecond) + stream, err := NewStream(t.Name(), rdb, options.WithStreamDeadline(deadline)) + require.NoError(t, err) + _, err = stream.Add(ctx, "first", []byte("payload")) + require.NoError(t, err) + firstExpiry, err := rdb.PExpireTime(ctx, stream.key).Result() + require.NoError(t, err) + expectedExpiry := time.Duration(deadline.UnixMilli()) * time.Millisecond + require.Equal(t, expectedExpiry, firstExpiry) + + time.Sleep(100 * time.Millisecond) + reopened, err := NewStream(t.Name(), rdb, options.WithStreamDeadline(deadline)) + require.NoError(t, err) + _, err = reopened.Add(ctx, "second", []byte("payload")) + require.NoError(t, err) + secondExpiry, err := rdb.PExpireTime(ctx, stream.key).Result() + require.NoError(t, err) + require.Equal(t, firstExpiry, secondExpiry) + + _, err = stream.AddOnce(ctx, "command", "third", []byte("payload")) + require.NoError(t, err) + require.Equal(t, expectedExpiry, rdb.PExpireTime(ctx, idempotencyKeyMap(stream)).Val()) + require.Equal(t, expectedExpiry, rdb.PExpireTime(ctx, stream.key).Val()) + require.Equal(t, expectedExpiry, rdb.PExpireTime(ctx, recoveryCursorKey(stream)).Val()) + require.NoError(t, stream.Destroy(ctx)) +} + +func TestDeadlineValidationReopenAndElapsedOperations(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + deadline := time.Now().Add(150 * time.Millisecond).Truncate(time.Millisecond) + stream, err := NewStream(t.Name(), rdb, options.WithStreamDeadline(deadline)) + require.NoError(t, err) + _, err = stream.Add(ctx, "event", []byte("payload")) + require.NoError(t, err) + same, err := NewStream(t.Name(), rdb, options.WithStreamDeadline(deadline)) + require.NoError(t, err) + require.NoError(t, same.Open(ctx)) + conflicting, err := NewStream( + t.Name(), + rdb, + options.WithStreamDeadline(deadline.Add(time.Second)), + ) + require.NoError(t, err) + require.ErrorIs(t, conflicting.Open(ctx), ErrStreamConfigMismatch) + + require.Eventually(t, func() bool { + _, addErr := stream.Add(ctx, "late", []byte("payload")) + return errors.Is(addErr, ErrDeadlineElapsed) + }, time.Second, 10*time.Millisecond) + _, err = stream.AddOnce(ctx, "late", "late", []byte("payload")) + require.ErrorIs(t, err, ErrDeadlineElapsed) + events, err := stream.Snapshot(ctx) + require.ErrorIs(t, err, ErrDeadlineElapsed) + require.Nil(t, events) + require.NoError(t, stream.Destroy(ctx)) +} + +func TestAddOnceDestroyRecreateIsolatesGeneration(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + firstDeadline := time.Now().Add(10 * time.Second).Truncate(time.Millisecond) + first, err := NewStream(t.Name(), rdb, options.WithStreamDeadline(firstDeadline)) + require.NoError(t, err) + old, err := first.AddOnce(ctx, "command", "old", []byte("payload")) + require.NoError(t, err) + oldDedupe := idempotencyKeyMap(first) + require.NoError(t, first.Destroy(ctx)) + require.EqualValues(t, 0, rdb.Exists(ctx, oldDedupe).Val()) + + secondDeadline := firstDeadline.Add(time.Second) + second, err := NewStream(t.Name(), rdb, options.WithStreamDeadline(secondDeadline)) + require.NoError(t, err) + fresh, err := second.AddOnce(ctx, "command", "new", []byte("different")) + require.NoError(t, err) + require.NotEqual(t, first.Generation(), second.Generation()) + require.NotEqual(t, first.key, second.key) + require.ErrorIs(t, func() error { + _, snapshotErr := first.Snapshot(ctx) + return snapshotErr + }(), ErrStreamDestroyed) + require.NotEqual(t, old+"@"+first.key, fresh+"@"+second.key) + require.NoError(t, second.Destroy(ctx)) +} + +func TestAddOnceDestroyRaceCannotCrossGeneration(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + deadline := time.Now().Add(10 * time.Second).Truncate(time.Millisecond) + old, err := NewStream(t.Name(), rdb, options.WithStreamDeadline(deadline)) + require.NoError(t, err) + require.NoError(t, old.Open(ctx)) + start := make(chan struct{}) + result := make(chan error, 1) + go func() { + <-start + _, addErr := old.AddOnce(ctx, "shared", "old", []byte("payload")) + result <- addErr + }() + close(start) + destroyErr := old.Destroy(ctx) + require.NoError(t, destroyErr) + addErr := <-result + if addErr != nil { + require.ErrorIs(t, addErr, ErrStreamDestroyed) + } + + freshDeadline := deadline.Add(time.Second) + fresh, err := NewStream(t.Name(), rdb, options.WithStreamDeadline(freshDeadline)) + require.NoError(t, err) + freshResult, err := fresh.AddOnce( + ctx, + "shared", + "fresh", + []byte("new payload"), + ) + require.NoError(t, err) + require.NotEmpty(t, freshResult) + events, err := fresh.Snapshot(ctx) + require.NoError(t, err) + require.Len(t, events, 1) + require.Equal(t, "fresh", events[0].EventName()) + require.NoError(t, fresh.Destroy(ctx)) +} + +func TestSnapshotOrderedSideEffectFreeAndMalformedAtomic(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + stream, err := NewStream(t.Name(), rdb) + require.NoError(t, err) + firstID, err := stream.Add(ctx, "first", []byte("one")) + require.NoError(t, err) + secondID, err := stream.Add(ctx, "second", []byte("two"), options.WithTopic("topic")) + require.NoError(t, err) + events, err := stream.Snapshot(ctx) + require.NoError(t, err) + require.Len(t, events, 2) + require.Equal(t, []string{firstID, secondID}, []string{events[0].ID(), events[1].ID()}) + require.Equal(t, []string{"first", "second"}, []string{events[0].EventName(), events[1].EventName()}) + require.Equal(t, "topic", events[1].Topic()) + require.EqualValues(t, 0, rdb.Exists(ctx, recoveryCursorKey(stream)).Val()) + groups, err := rdb.XInfoGroups(ctx, stream.key).Result() + require.NoError(t, err) + require.Empty(t, groups) + + _, err = rdb.XAdd(ctx, &redis.XAddArgs{ + Stream: stream.key, + Values: map[string]any{payloadKey: "missing name"}, + }).Result() + require.NoError(t, err) + events, err = stream.Snapshot(ctx) + require.ErrorContains(t, err, "malformed event") + require.Nil(t, events) + require.NoError(t, stream.Destroy(ctx)) +} + +func TestExactPublicationOptionValidation(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + deadline := time.Now().Add(time.Second) + _, err := NewStream( + t.Name(), + rdb, + options.WithStreamTTL(time.Second), + options.WithStreamDeadline(deadline), + ) + require.ErrorContains(t, err, "mutually exclusive") + _, err = NewStream(t.Name(), rdb, options.WithStreamDeadline(time.Time{})) + require.ErrorContains(t, err, "must not be zero") + + stream, err := NewStream(t.Name(), rdb, options.WithStreamDeadline(deadline)) + require.NoError(t, err) + _, err = stream.AddOnce(ctx, "", "event", nil) + require.ErrorContains(t, err, "must not be empty") + _, err = stream.AddOnce(ctx, string(make([]byte, maxIdempotencyKeyBytes+1)), "event", nil) + require.ErrorContains(t, err, "exceeds") + _, err = stream.AddOnce(ctx, "key", "", nil) + require.ErrorContains(t, err, "event name must not be empty") + _, err = stream.AddOnce( + ctx, + "key", + "event", + nil, + options.WithOnlyIfStreamExists(), + ) + require.ErrorContains(t, err, "does not support") + past := time.Now().Add(-time.Second) + pastStream, err := NewStream(t.Name()+"-past", rdb, options.WithStreamDeadline(past)) + require.NoError(t, err) + _, err = pastStream.AddOnce(ctx, "past", "event", nil) + require.ErrorIs(t, err, ErrDeadlineElapsed) + valid, err := stream.AddOnce(ctx, "valid", "event", nil) + require.NoError(t, err) + require.NotEmpty(t, valid) + require.NoError(t, stream.Destroy(ctx)) + + ttlStream, err := NewStream(t.Name()+"-ttl", rdb, options.WithStreamTTL(time.Second)) + require.NoError(t, err) + _, err = ttlStream.Add(ctx, "event", nil) + require.NoError(t, err) + deadlineHandle, err := NewStream( + t.Name()+"-ttl", + rdb, + options.WithStreamDeadline(deadline), + ) + require.NoError(t, err) + require.ErrorIs(t, deadlineHandle.Open(ctx), ErrStreamConfigMismatch) + require.NoError(t, ttlStream.Destroy(ctx)) +} + +func TestSnapshotDoesNotInitializeAbsentStream(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + stream, err := NewStream(t.Name(), rdb) + require.NoError(t, err) + + events, err := stream.Snapshot(ctx) + require.ErrorIs(t, err, ErrStreamNotFound) + require.Nil(t, events) + require.EqualValues(t, 0, rdb.Exists(ctx, stream.lifecycleKey, streamKey(t.Name())).Val()) + require.Empty(t, stream.Generation()) +} + +func TestSnapshotRejectsUnboundedRetentionBeforeRange(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + stream, err := NewStream(t.Name(), rdb, options.WithUnboundedStream()) + require.NoError(t, err) + _, err = stream.Add(ctx, "event", []byte("payload")) + require.NoError(t, err) + + events, err := stream.Snapshot(ctx) + require.ErrorIs(t, err, ErrSnapshotUnbounded) + require.Nil(t, events) + require.NoError(t, stream.Destroy(ctx)) +} + +func TestRejectedLegacyRetentionAdoptionDoesNotMutateLifecycle(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + name := t.Name() + lifecycle := streamLifecycleKey(name) + deadline := time.Now().Add(time.Hour).UnixMilli() + require.NoError(t, rdb.HSet( + ctx, + lifecycle, + "generation", "1", + "state", streamStateActive, + streamPhysicalKey, streamKey(name), + streamDeadlineKey, deadline, + ).Err()) + before, err := rdb.HGetAll(ctx, lifecycle).Result() + require.NoError(t, err) + opener, err := NewStream(name, rdb, options.WithStreamTTL(time.Minute)) + require.NoError(t, err) + + require.ErrorIs(t, opener.Open(ctx), ErrStreamConfigMismatch) + after, err := rdb.HGetAll(ctx, lifecycle).Result() + require.NoError(t, err) + require.Equal(t, before, after) +} + +func TestRetentionConfigurationIsImmutable(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + stream, err := NewStream(t.Name(), rdb, options.WithStreamMaxLen(12)) + require.NoError(t, err) + _, err = stream.Add(ctx, "event", nil) + require.NoError(t, err) + + differentLimit, err := NewStream(t.Name(), rdb, options.WithStreamMaxLen(13)) + require.NoError(t, err) + require.ErrorIs(t, differentLimit.Open(ctx), ErrStreamConfigMismatch) + differentMode, err := NewStream(t.Name(), rdb, options.WithStreamTTL(time.Second)) + require.NoError(t, err) + require.ErrorIs(t, differentMode.Open(ctx), ErrStreamConfigMismatch) + _, err = NewStream( + t.Name()+"-invalid", + rdb, + options.WithStreamMaxLen(1), + options.WithUnboundedStream(), + ) + require.ErrorContains(t, err, "mutually exclusive") + require.NoError(t, stream.Destroy(ctx)) +} + +func TestDefaultHandlesAdoptWriterRetention(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + writer, err := NewStream( + t.Name(), + rdb, + options.WithStreamMaxLen(50_000), + options.WithStreamSlidingTTL(time.Minute), + ) + require.NoError(t, err) + _, err = writer.Add(ctx, "event", nil) + require.NoError(t, err) + + readerHandle, err := NewStream(t.Name(), rdb) + require.NoError(t, err) + reader, err := readerHandle.NewReader(ctx) + require.NoError(t, err) + reader.Close() + require.Equal(t, 50_000, readerHandle.maxLen) + require.Equal(t, 1000, readerHandle.MaxLen, "exported MaxLen stays the construction value") + require.Equal(t, time.Minute, readerHandle.ttl) + require.True(t, readerHandle.ttlSliding) + + destroyHandle, err := NewStream(t.Name(), rdb) + require.NoError(t, err) + require.NoError(t, destroyHandle.Destroy(ctx)) +} + +func TestNonCreatingOperationsDoNotInitializeLifecycle(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + stream, err := NewStream(t.Name(), rdb) + require.NoError(t, err) + + _, err = stream.Add(ctx, "event", nil, options.WithOnlyIfStreamExists()) + require.ErrorIs(t, err, ErrStreamNotFound) + reader, err := stream.NewReader(ctx) + require.Nil(t, reader) + require.ErrorIs(t, err, ErrStreamNotFound) + require.ErrorIs(t, stream.Destroy(ctx), ErrStreamNotFound) + require.EqualValues(t, 0, rdb.Exists(ctx, stream.lifecycleKey, streamKey(t.Name())).Val()) +} + +func TestSubMillisecondRetentionDurationsAreRejected(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + for _, option := range []options.Stream{ + options.WithStreamTTL(0), + options.WithStreamSlidingTTL(0), + options.WithStreamTTL(500 * time.Microsecond), + options.WithStreamSlidingTTL(500 * time.Microsecond), + } { + _, err := NewStream(t.Name(), rdb, option) + require.ErrorContains(t, err, "at least 1ms") + } +} + +func TestAddOnceExactIdentityAndBodyLimit(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + deadline := time.Now().Add(time.Minute).Truncate(time.Millisecond) + stream, err := NewStream(t.Name(), rdb, options.WithStreamDeadline(deadline)) + require.NoError(t, err) + first, err := stream.AddOnce(ctx, "key", "a:1", []byte("b")) + require.NoError(t, err) + retry, err := stream.AddOnce(ctx, "key", "a:1", []byte("b")) + require.NoError(t, err) + require.Equal(t, first, retry) + _, err = stream.AddOnce(ctx, "key", "a", []byte("1:b")) + require.ErrorIs(t, err, ErrIdempotencyConflict) + _, err = stream.AddOnce(ctx, "large", "event", make([]byte, maxAddOnceBodyBytes+1)) + require.ErrorContains(t, err, "exceeds") + require.NoError(t, stream.Destroy(ctx)) +} + +func TestSinkCloseTreatsDeadlineElapsedAsTerminal(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + deadline := time.Now().Add(2 * time.Second).Truncate(time.Millisecond) + stream, err := NewStream(t.Name(), rdb, options.WithStreamDeadline(deadline)) + require.NoError(t, err) + sink, err := stream.NewSink( + ctx, + "sink", + options.WithSinkBlockDuration(25*time.Millisecond), + ) + require.NoError(t, err) + events := sink.Subscribe() + require.Eventually(t, func() bool { + return errors.Is(stream.verifyGeneration(ctx), ErrDeadlineElapsed) + }, 5*time.Second, 10*time.Millisecond) + require.NoError(t, sink.Close(ctx)) + _, open := <-events + require.False(t, open) + require.True(t, sink.IsClosed()) + require.NoError(t, stream.Destroy(ctx)) +} + +// DialHook preserves normal Redis dialing. +func (h *ambiguousAddOnceHook) DialHook(next redis.DialHook) redis.DialHook { + return func(ctx context.Context, network, addr string) (net.Conn, error) { + return next(ctx, network, addr) + } +} + +// ProcessHook injects one error only after Redis committed AddOnce. +func (h *ambiguousAddOnceHook) ProcessHook(next redis.ProcessHook) redis.ProcessHook { + return func(ctx context.Context, cmd redis.Cmder) error { + err := next(ctx, cmd) + if err != nil || cmd.Name() != "evalsha" { + return err + } + args := cmd.Args() + if len(args) > 1 && args[1] == addOnceScript.Hash() && + h.fail.CompareAndSwap(true, false) { + return h.err + } + return err + } +} + +// ProcessPipelineHook preserves Redis pipelines. +func (h *ambiguousAddOnceHook) ProcessPipelineHook(next redis.ProcessPipelineHook) redis.ProcessPipelineHook { + return func(ctx context.Context, cmds []redis.Cmder) error { + return next(ctx, cmds) + } +} diff --git a/streaming/options/add_stream.go b/streaming/options/add_stream.go index cda5a1f..366c086 100644 --- a/streaming/options/add_stream.go +++ b/streaming/options/add_stream.go @@ -9,8 +9,14 @@ type ( // AddStream is an option for adding a stream to a sink. AddStream func(*AddStreamOptions) + // AddStreamOptions keeps its v1 field first; new fields are only ever + // appended so existing keyed construction remains source-compatible. AddStreamOptions struct { + // LastEventID is the ID after which delivery starts for this stream. LastEventID string + // startOptions counts applied start-position options to reject + // conflicting combinations. + startOptions int } ) @@ -21,6 +27,7 @@ type ( func WithAddStreamStartAtNewest() AddStream { return func(o *AddStreamOptions) { o.LastEventID = "$" + o.startOptions++ } } @@ -31,6 +38,7 @@ func WithAddStreamStartAtNewest() AddStream { func WithAddStreamStartAtOldest() AddStream { return func(o *AddStreamOptions) { o.LastEventID = "0" + o.startOptions++ } } @@ -41,6 +49,7 @@ func WithAddStreamStartAtOldest() AddStream { func WithAddStreamStartAfter(id string) AddStream { return func(o *AddStreamOptions) { o.LastEventID = id + o.startOptions++ } } @@ -51,6 +60,7 @@ func WithAddStreamStartAfter(id string) AddStream { func WithAddStreamStartAt(startAt time.Time) AddStream { return func(o *AddStreamOptions) { o.LastEventID = fmt.Sprintf("%d-0", startAt.UnixMilli()) + o.startOptions++ } } @@ -63,6 +73,13 @@ func ParseAddStreamOptions(opts ...AddStream) AddStreamOptions { return options } +// HasConflictingStartOptions reports whether more than one cursor-start option +// was supplied. AddStream rejects this instead of silently accepting the last +// option. +func (o AddStreamOptions) HasConflictingStartOptions() bool { + return o.startOptions > 1 +} + // defaultAddStreamOptions returns the default options. func defaultAddStreamOptions() AddStreamOptions { return AddStreamOptions{} diff --git a/streaming/options/compat_test.go b/streaming/options/compat_test.go index 45ac587..35388fd 100644 --- a/streaming/options/compat_test.go +++ b/streaming/options/compat_test.go @@ -1,28 +1,53 @@ -// Package options_test locks the v1 source compatibility of the exported -// option structs: external code constructing them with unkeyed (positional) -// literals must keep compiling. Adding, removing, reordering, or unexporting -// a field breaks the literals below at compile time. +// Package options_test locks the source compatibility of the exported option +// structs: every v1 field keeps its name, type, and position (v1 fields form a +// stable prefix, in v1 order), and new fields are only ever appended. Keyed +// construction and field access from v1 code must keep compiling; unkeyed +// literals were only guaranteed within the v1 patch line and are not part of +// the contract for feature releases. package options_test import ( + "reflect" "testing" "time" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "goa.design/pulse/pulse" "goa.design/pulse/streaming/options" ) -// TestOptionStructsKeepV1PositionalShape compiles v1 unkeyed literals of +// TestOptionStructsKeepV1Fields compiles keyed literals of every v1 field of // every exported option struct. The assertions only exist to consume the // literals; the test is about compilation. -func TestOptionStructsKeepV1PositionalShape(t *testing.T) { - reader := options.ReaderOptions{time.Second, 100, "topic", "pattern", 10, "0"} - sink := options.SinkOptions{time.Second, 100, "topic", "pattern", 10, "0", true, time.Minute} - addStream := options.AddStreamOptions{"42-0"} - stream := options.StreamOptions{1000, pulse.NoopLogger(), time.Minute, true} - addEvent := options.AddEventOptions{"topic", true} +func TestOptionStructsKeepV1Fields(t *testing.T) { + reader := options.ReaderOptions{ + BlockDuration: time.Second, + MaxPolled: 100, + Topic: "topic", + TopicPattern: "pattern", + BufferSize: 10, + LastEventID: "0", + } + sink := options.SinkOptions{ + BlockDuration: time.Second, + MaxPolled: 100, + Topic: "topic", + TopicPattern: "pattern", + BufferSize: 10, + LastEventID: "0", + NoAck: true, + AckGracePeriod: time.Minute, + } + addStream := options.AddStreamOptions{LastEventID: "42-0"} + stream := options.StreamOptions{ + MaxLen: 1000, + Logger: pulse.NoopLogger(), + TTL: time.Minute, + TTLSliding: true, + } + addEvent := options.AddEventOptions{Topic: "topic", OnlyIfStreamExists: true} assert.Equal(t, time.Second, reader.BlockDuration) assert.Equal(t, time.Minute, sink.AckGracePeriod) @@ -30,3 +55,29 @@ func TestOptionStructsKeepV1PositionalShape(t *testing.T) { assert.Equal(t, 1000, stream.MaxLen) assert.True(t, addEvent.OnlyIfStreamExists) } + +// TestV1FieldsFormStableOrderedPrefix pins the v1 fields to the leading +// positions of every exported option struct, in v1 order. Keyed literals +// cannot detect reordering, so this is the check that enforces the stable +// prefix the package documents. +func TestV1FieldsFormStableOrderedPrefix(t *testing.T) { + assertFieldPrefix(t, options.ReaderOptions{}, + "BlockDuration", "MaxPolled", "Topic", "TopicPattern", "BufferSize", "LastEventID") + assertFieldPrefix(t, options.SinkOptions{}, + "BlockDuration", "MaxPolled", "Topic", "TopicPattern", "BufferSize", "LastEventID", + "NoAck", "AckGracePeriod") + assertFieldPrefix(t, options.AddStreamOptions{}, "LastEventID") + assertFieldPrefix(t, options.StreamOptions{}, "MaxLen", "Logger", "TTL", "TTLSliding") + assertFieldPrefix(t, options.AddEventOptions{}, "Topic", "OnlyIfStreamExists") +} + +// assertFieldPrefix asserts that the struct's leading fields carry exactly +// the given names in order. +func assertFieldPrefix(t *testing.T, v any, names ...string) { + t.Helper() + typ := reflect.TypeOf(v) + require.GreaterOrEqual(t, typ.NumField(), len(names), typ.Name()) + for i, name := range names { + assert.Equal(t, name, typ.Field(i).Name, "%s field %d", typ.Name(), i) + } +} diff --git a/streaming/options/options_test.go b/streaming/options/options_test.go index 1e4f7fc..4b35734 100644 --- a/streaming/options/options_test.go +++ b/streaming/options/options_test.go @@ -11,6 +11,7 @@ import ( ) func TestStreamOptions(t *testing.T) { + deadline := time.Date(2030, 1, 1, 0, 0, 0, 0, time.UTC) cases := []struct { name string opts []Stream @@ -29,9 +30,19 @@ func TestStreamOptions(t *testing.T) { name: "maxlen", opts: []Stream{WithStreamMaxLen(10)}, want: StreamOptions{ - MaxLen: 10, - Logger: pulse.NoopLogger(), - TTL: 0, + MaxLen: 10, + MaxLenSet: true, + Logger: pulse.NoopLogger(), + TTL: 0, + }, + }, + { + name: "unbounded", + opts: []Stream{WithUnboundedStream()}, + want: StreamOptions{ + MaxLen: 1000, + Unbounded: true, + Logger: pulse.NoopLogger(), }, }, { @@ -50,6 +61,7 @@ func TestStreamOptions(t *testing.T) { MaxLen: 1000, Logger: pulse.NoopLogger(), TTL: 10 * time.Second, + TTLSet: true, TTLSliding: false, }, }, @@ -60,9 +72,20 @@ func TestStreamOptions(t *testing.T) { MaxLen: 1000, Logger: pulse.NoopLogger(), TTL: 10 * time.Second, + TTLSet: true, TTLSliding: true, }, }, + { + name: "deadline", + opts: []Stream{WithStreamDeadline(deadline)}, + want: StreamOptions{ + MaxLen: 1000, + Logger: pulse.NoopLogger(), + Deadline: deadline, + DeadlineSet: true, + }, + }, } for _, c := range cases { @@ -152,6 +175,7 @@ func TestReaderOptions(t *testing.T) { MaxPolled: 1000, BufferSize: 1000, LastEventID: "foo", + startOptions: 1, }, }, } @@ -251,6 +275,7 @@ func TestSinkOptions(t *testing.T) { BufferSize: 1000, LastEventID: "foo", AckGracePeriod: 20 * time.Second, + startOptions: 1, }, }, { @@ -262,6 +287,7 @@ func TestSinkOptions(t *testing.T) { BufferSize: 1000, LastEventID: fmt.Sprintf("%d-0", date.UnixMilli()), AckGracePeriod: 20 * time.Second, + startOptions: 1, }, }, { @@ -318,14 +344,16 @@ func TestAddStreamOptions(t *testing.T) { name: "last event ID", opts: []AddStream{WithAddStreamStartAfter("foo")}, want: AddStreamOptions{ - LastEventID: "foo", + LastEventID: "foo", + startOptions: 1, }, }, { name: "start at", opts: []AddStream{WithAddStreamStartAt(date)}, want: AddStreamOptions{ - LastEventID: fmt.Sprintf("%d-0", date.UnixMilli()), + LastEventID: fmt.Sprintf("%d-0", date.UnixMilli()), + startOptions: 1, }, }, } diff --git a/streaming/options/reader.go b/streaming/options/reader.go index 4aca96f..77ca282 100644 --- a/streaming/options/reader.go +++ b/streaming/options/reader.go @@ -9,19 +9,32 @@ type ( // Reader is a sink creation option. Reader func(*ReaderOptions) + // ReaderOptions keeps its v1 fields first, in v1 order; new fields are + // only ever appended so existing keyed construction and field access + // remain source-compatible across feature releases. ReaderOptions struct { + // BlockDuration is the XREAD block duration. BlockDuration time.Duration - MaxPolled int64 - Topic string - TopicPattern string - BufferSize int - LastEventID string + // MaxPolled is the maximum number of events read per XREAD call. + MaxPolled int64 + // Topic delivers only events published with this exact topic. + Topic string + // TopicPattern delivers only events whose topic matches this regex. + TopicPattern string + // BufferSize is the capacity of each subscription channel. + BufferSize int + // LastEventID is the ID after which delivery starts. + LastEventID string + // startOptions counts applied start-position options to reject + // conflicting combinations. + startOptions int } ) // WithReaderBlockDuration sets the maximum amount of time the reader waits for -// MaxPolled events. The default block duration is 5 seconds. If the block -// duration is set to 0 then the reader blocks indefinitely. +// MaxPolled events. The default block duration is 5 seconds. NewReader rejects +// durations below one millisecond because Redis block timing is millisecond +// precision and every read must have a finite shutdown bound. func WithReaderBlockDuration(d time.Duration) Reader { return func(o *ReaderOptions) { o.BlockDuration = d @@ -44,7 +57,7 @@ func WithReaderTopic(topic string) Reader { } // WithReaderTopicPattern sets the reader topic pattern. -// pattern must be a valid regular expression or NewReader panics. +// NewReader returns an error when pattern is not a valid regular expression. func WithReaderTopicPattern(pattern string) Reader { return func(o *ReaderOptions) { o.TopicPattern = pattern @@ -67,6 +80,7 @@ func WithReaderBufferSize(size int) Reader { func WithReaderStartAtNewest() Reader { return func(o *ReaderOptions) { o.LastEventID = "$" + o.startOptions++ } } @@ -76,6 +90,7 @@ func WithReaderStartAtNewest() Reader { func WithReaderStartAtOldest() Reader { return func(o *ReaderOptions) { o.LastEventID = "0" + o.startOptions++ } } @@ -84,6 +99,7 @@ func WithReaderStartAtOldest() Reader { func WithReaderStartAfter(id string) Reader { return func(o *ReaderOptions) { o.LastEventID = id + o.startOptions++ } } @@ -92,6 +108,7 @@ func WithReaderStartAfter(id string) Reader { func WithReaderStartAt(startAt time.Time) Reader { return func(o *ReaderOptions) { o.LastEventID = fmt.Sprintf("%d-0", startAt.UnixMilli()) + o.startOptions++ } } @@ -105,6 +122,13 @@ func ParseReaderOptions(opts ...Reader) ReaderOptions { return o } +// HasConflictingStartOptions reports whether more than one cursor-start option +// was supplied. Constructors reject this instead of silently accepting the +// last option. +func (o ReaderOptions) HasConflictingStartOptions() bool { + return o.startOptions > 1 +} + // defaultReaderOptions returns the default options. func defaultReaderOptions() ReaderOptions { return ReaderOptions{ diff --git a/streaming/options/shape_pin_test.go b/streaming/options/shape_pin_test.go new file mode 100644 index 0000000..041dbfb --- /dev/null +++ b/streaming/options/shape_pin_test.go @@ -0,0 +1,31 @@ +// Positional pins of the full current shape of every exported option struct. +// Any field insertion, reorder, or retype breaks these literals at compile +// time, forcing a deliberate decision; appending a field only updates the +// literal it extends. This complements compat_test.go, which pins the +// external keyed-construction contract for v1 fields. +package options + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + + "goa.design/pulse/pulse" +) + +// TestOptionStructsPositionalShape compiles unkeyed literals of the current +// full shapes. The assertions only consume the literals. +func TestOptionStructsPositionalShape(t *testing.T) { + reader := ReaderOptions{time.Second, 100, "topic", "pattern", 10, "0", 0} + sink := SinkOptions{time.Second, 100, "topic", "pattern", 10, "0", true, time.Minute, 0} + addStream := AddStreamOptions{"42-0", 0} + stream := StreamOptions{1000, pulse.NoopLogger(), time.Minute, true, true, false, true, time.Time{}, false} + addEvent := AddEventOptions{"topic", true} + + assert.Equal(t, time.Second, reader.BlockDuration) + assert.Equal(t, time.Minute, sink.AckGracePeriod) + assert.Equal(t, "42-0", addStream.LastEventID) + assert.Equal(t, 1000, stream.MaxLen) + assert.True(t, addEvent.OnlyIfStreamExists) +} diff --git a/streaming/options/sink.go b/streaming/options/sink.go index f3cb2b4..38b6e36 100644 --- a/streaming/options/sink.go +++ b/streaming/options/sink.go @@ -9,21 +9,37 @@ type ( // Sink is a sink creation option. Sink func(*SinkOptions) + // SinkOptions keeps its v1 fields first, in v1 order; new fields are + // only ever appended so existing keyed construction and field access + // remain source-compatible across feature releases. SinkOptions struct { - BlockDuration time.Duration - MaxPolled int64 - Topic string - TopicPattern string - BufferSize int - LastEventID string - NoAck bool + // BlockDuration is the XREADGROUP block duration. + BlockDuration time.Duration + // MaxPolled is the maximum number of events read per XREADGROUP call. + MaxPolled int64 + // Topic delivers only events published with this exact topic. + Topic string + // TopicPattern delivers only events whose topic matches this regex. + TopicPattern string + // BufferSize is the capacity of each subscription channel. + BufferSize int + // LastEventID is the ID after which delivery starts. + LastEventID string + // NoAck atomically acknowledges each event before delivery. + NoAck bool + // AckGracePeriod bounds how long an unacknowledged event stays owned + // by one consumer before stale recovery may reclaim it. AckGracePeriod time.Duration + // startOptions counts applied start-position options to reject + // conflicting combinations. + startOptions int } ) // WithSinkBlockDuration sets the maximum amount of time the sink waits for -// MaxPolled events. The default block duration is 5 seconds. If the block -// duration is set to 0 then the sink blocks indefinitely. +// MaxPolled events. The default block duration is 5 seconds. NewSink rejects +// durations below one millisecond because Redis block timing is millisecond +// precision and every read must have a finite shutdown bound. func WithSinkBlockDuration(d time.Duration) Sink { return func(o *SinkOptions) { o.BlockDuration = d @@ -46,7 +62,7 @@ func WithSinkTopic(topic string) Sink { } // WithSinkTopicPattern sets the sink topic pattern. -// pattern must be a valid regular expression or NewSink panics. +// NewSink returns an error when pattern is not a valid regular expression. func WithSinkTopicPattern(pattern string) Sink { return func(o *SinkOptions) { o.TopicPattern = pattern @@ -68,6 +84,7 @@ func WithSinkBufferSize(size int) Sink { func WithSinkStartAtNewest() Sink { return func(o *SinkOptions) { o.LastEventID = "$" + o.startOptions++ } } @@ -77,6 +94,7 @@ func WithSinkStartAtNewest() Sink { func WithSinkStartAtOldest() Sink { return func(o *SinkOptions) { o.LastEventID = "0" + o.startOptions++ } } @@ -86,6 +104,7 @@ func WithSinkStartAtOldest() Sink { func WithSinkStartAfter(id string) Sink { return func(o *SinkOptions) { o.LastEventID = id + o.startOptions++ } } @@ -95,18 +114,21 @@ func WithSinkStartAfter(id string) Sink { func WithSinkStartAt(startAt time.Time) Sink { return func(o *SinkOptions) { o.LastEventID = fmt.Sprintf("%d-0", startAt.UnixMilli()) + o.startOptions++ } } -// WithSinkNoAck removes the need to acknowledge events read from the sink. +// WithSinkNoAck atomically acknowledges each event before delivering it to +// subscribers, preserving at-most-once delivery without requiring Sink.Ack. func WithSinkNoAck() Sink { return func(o *SinkOptions) { o.NoAck = true } } -// WithSinkAckGracePeriod sets the grace period for acknowledging events. The -// default grace period is 20 seconds. +// WithSinkAckGracePeriod sets the grace period for acknowledging events. The +// default grace period is 20 seconds; NewSink rejects values below one +// millisecond. // Note: all sinks with identical names must have the same ack grace period. func WithSinkAckGracePeriod(d time.Duration) Sink { return func(o *SinkOptions) { @@ -123,6 +145,13 @@ func ParseSinkOptions(opts ...Sink) SinkOptions { return o } +// HasConflictingStartOptions reports whether more than one cursor-start option +// was supplied. Constructors reject this instead of silently accepting the +// last option. +func (o SinkOptions) HasConflictingStartOptions() bool { + return o.startOptions > 1 +} + // defaultSinkOptions returns the default options. func defaultSinkOptions() SinkOptions { return SinkOptions{ diff --git a/streaming/options/stream.go b/streaming/options/stream.go index 581468b..8c148a3 100644 --- a/streaming/options/stream.go +++ b/streaming/options/stream.go @@ -10,23 +10,47 @@ type ( // Stream is a stream creation option. Stream func(*StreamOptions) + // StreamOptions keeps its v1 fields first, in v1 order; new fields are + // only ever appended so existing keyed construction and field access + // remain source-compatible across feature releases. StreamOptions struct { MaxLen int Logger pulse.Logger - // TTL configures a retention window for the Redis key backing the stream. // When zero, no TTL is applied. TTL time.Duration // TTLSliding controls whether the TTL is refreshed on every published event. // When false, the TTL is applied once (absolute TTL) and never extended. TTLSliding bool + + // MaxLenSet distinguishes an omitted MaxLen from an explicit request. + MaxLenSet bool + // Unbounded disables MAXLEN trimming entirely. + Unbounded bool + // TTLSet distinguishes an omitted TTL from an explicitly invalid zero. + TTLSet bool + // Deadline is the immutable absolute expiry claimed by this stream + // generation when DeadlineSet is true. + Deadline time.Time + // DeadlineSet distinguishes an omitted deadline from the zero time. + DeadlineSet bool } ) -// WithStreamMaxLen sets the maximum number of events stored by the stream. +// WithStreamMaxLen sets the positive maximum number of events stored by the +// stream. NewStream rejects zero and negative values. func WithStreamMaxLen(len int) Stream { return func(o *StreamOptions) { o.MaxLen = len + o.MaxLenSet = true + } +} + +// WithUnboundedStream disables MAXLEN trimming. Callers must remove settled +// events explicitly. It cannot be combined with WithStreamMaxLen. +func WithUnboundedStream() Stream { + return func(o *StreamOptions) { + o.Unbounded = true } } @@ -37,24 +61,39 @@ func WithStreamLogger(logger pulse.Logger) Stream { } } -// WithStreamTTL sets an absolute TTL on the Redis key backing the stream. -// The TTL is set once (when the key is created) and never extended. +// WithStreamTTL sets an absolute TTL on the Redis key backing the stream. The +// TTL is set once and never extended; NewStream rejects values below Redis's +// one-millisecond precision. func WithStreamTTL(ttl time.Duration) Stream { return func(o *StreamOptions) { o.TTL = ttl + o.TTLSet = true o.TTLSliding = false } } // WithStreamSlidingTTL sets a sliding TTL on the Redis key backing the stream. -// The TTL is refreshed on every published event. +// The TTL is refreshed on every published event; NewStream rejects values +// below Redis's one-millisecond precision. func WithStreamSlidingTTL(ttl time.Duration) Stream { return func(o *StreamOptions) { o.TTL = ttl + o.TTLSet = true o.TTLSliding = true } } +// WithStreamDeadline makes the stream generation expire at deadline. The +// deadline is persisted when the handle first opens; handles for the same +// generation must not request a different deadline. TTL options cannot be +// combined with this option. +func WithStreamDeadline(deadline time.Time) Stream { + return func(o *StreamOptions) { + o.Deadline = deadline + o.DeadlineSet = true + } +} + // ParseStreamOptions parses the given options and returns the corresponding // StreamOptions. func ParseStreamOptions(opts ...Stream) StreamOptions { diff --git a/streaming/reader.go b/streaming/reader.go index eaef0b9..cc9eaa7 100644 --- a/streaming/reader.go +++ b/streaming/reader.go @@ -2,21 +2,37 @@ package streaming import ( "context" + "errors" "fmt" "math/rand" "regexp" "strconv" "strings" "sync" + "sync/atomic" "time" redis "github.com/redis/go-redis/v9" + "goa.design/clue/log" "goa.design/pulse/pulse" "goa.design/pulse/streaming/options" ) type ( + // readRetry bounds the command rate while a long-lived reader or sink waits + // for Redis to recover. Successful commands reset the exponential delay. + readRetry struct { + failures int + jitter func(int64) int64 + } + + // readerSnapshot is the exact immutable stream set used by one XREAD. + readerSnapshot struct { + streams map[string]*Stream + args []string + } + // Reader represents a stream reader. Reader struct { // closed is true if Close completed. @@ -48,13 +64,14 @@ type ( closeOnce sync.Once // donechan is the reader donechan channel. donechan chan struct{} - // streamschan notifies the reader when streams are added or - // removed. - streamschan chan struct{} + // ctx bounds blocking Redis reads for the reader lifetime. + ctx context.Context + // cancel interrupts a blocking XREAD when Close begins. + cancel context.CancelFunc // wait is the reader cleanup wait group. wait sync.WaitGroup - // closing is true if Close was called. - closing bool + // closing is true from the instant Close begins. + closing atomic.Bool // eventFilter is the event filter if any. eventFilter eventFilterFunc // logger is the logger used by the reader. @@ -74,6 +91,10 @@ type ( ID string // StreamName is the name of the stream the event belongs to. StreamName string + // StreamGeneration is the immutable stream incarnation that produced the + // event. Events fetched before Destroy may still be in process, so + // handlers that fence side effects can compare this token explicitly. + StreamGeneration string // SinkName is the name of the sink the event belongs to. SinkName string // EventName is the producer-defined event name. @@ -82,30 +103,33 @@ type ( Topic string // Payload is the event payload. Payload []byte - // Acker is the redis client used to acknowledge events. + // Acker acknowledges events according to their sink recovery contract. Acker Acker // streamKey is the Redis key of the stream. streamKey string } - - // readRetry computes jittered exponential backoff delays for reader and - // sink Redis failures. Jitter spans half to full of the current backoff - // so replicas that fail together do not retry in lockstep. - readRetry struct { - backoff time.Duration - } ) const ( - // minReadRetryBackoff is the first retry backoff after a Redis failure. - minReadRetryBackoff = 50 * time.Millisecond - // maxReadRetryBackoff caps the exponential retry backoff. - maxReadRetryBackoff = 5 * time.Second + readRetryInitialDelay = 100 * time.Millisecond + readRetryMaxDelay = 5 * time.Second +) + +var ( + // ErrReaderClosed is returned when stream ownership is changed after reader + // shutdown begins. + ErrReaderClosed = errors.New("pulse streaming: reader is closed") + // ErrLastStream is returned when removing a stream would leave a reader or + // sink without a valid Redis read set. + ErrLastStream = errors.New("pulse streaming: cannot remove final stream") ) // newReader creates a new reader. -func newReader(stream *Stream, opts ...options.Reader) (*Reader, error) { +func newReader(ctx context.Context, stream *Stream, opts ...options.Reader) (*Reader, error) { o := options.ParseReaderOptions(opts...) + if err := validateReaderOptions(o); err != nil { + return nil, err + } var eventFilter eventFilterFunc if o.Topic != "" { eventFilter = func(e *Event) bool { return e.Topic == o.Topic } @@ -117,6 +141,9 @@ func newReader(stream *Stream, opts ...options.Reader) (*Reader, error) { eventFilter = func(e *Event) bool { return topicPatternRegexp.MatchString(e.Topic) } } + logCtx := context.Background() + logCtx = log.WithContext(logCtx, ctx) + runCtx, cancel := context.WithCancel(logCtx) reader := &Reader{ startID: o.LastEventID, streams: []*Stream{stream}, @@ -126,7 +153,8 @@ func newReader(stream *Stream, opts ...options.Reader) (*Reader, error) { maxPolled: o.MaxPolled, bufferSize: o.BufferSize, donechan: make(chan struct{}), - streamschan: make(chan struct{}), + ctx: runCtx, + cancel: cancel, eventFilter: eventFilter, logger: stream.rootLogger.WithPrefix("reader", stream.Name), rdb: stream.rdb, @@ -135,12 +163,17 @@ func newReader(stream *Stream, opts ...options.Reader) (*Reader, error) { return reader, nil } -// Subscribe returns a channel that receives events from the stream. -// The channel is closed when the reader is closed. +// Subscribe returns a channel that receives events from the stream. The +// channel is closed when the reader closes; calls made after shutdown starts +// return an already closed channel. func (r *Reader) Subscribe() <-chan *Event { c := make(chan *Event, r.bufferSize) r.lock.Lock() defer r.lock.Unlock() + if r.closing.Load() { + close(c) + return c + } r.chans = append(r.chans, c) r.start() return c @@ -159,43 +192,75 @@ func (r *Reader) Unsubscribe(c <-chan *Event) { } } -// AddStream adds the stream to the sink. By default the stream cursor starts at -// the same timestamp as the sink main stream cursor. This can be overridden -// with opts. AddStream does nothing if the stream is already part of the sink. +// AddStream adds the stream to the reader. By default the stream cursor starts +// at the same timestamp as the main stream cursor. This can be overridden with +// opts. AddStream does nothing if the stream is already part of the reader and +// returns ErrReaderClosed once shutdown starts. func (r *Reader) AddStream(ctx context.Context, stream *Stream, opts ...options.AddStream) error { r.lock.Lock() defer r.lock.Unlock() - for _, name := range r.streamKeys { - if name == stream.Name { + if r.closing.Load() { + return ErrReaderClosed + } + o := options.ParseAddStreamOptions(opts...) + if err := validateAddStreamOptions(o); err != nil { + return err + } + if err := stream.verifyExistingGeneration(ctx); err != nil { + return err + } + for _, owned := range r.streams { + if owned.Name != stream.Name { + continue + } + if owned.generation == stream.generation { return nil } + return owned.verifyGeneration(ctx) } startID := r.startID - o := options.ParseAddStreamOptions(opts...) if o.LastEventID != "" { startID = o.LastEventID } r.streams = append(r.streams, stream) r.streamKeys = append(r.streamKeys, stream.key) r.streamCursors = append(r.streamCursors, startID) - r.notifyStreamChange() r.logger.Info("added", "stream", stream.Name) return nil } -// RemoveStream removes the stream from the sink, it is idempotent. +// RemoveStream removes the stream from the reader. It is idempotent and returns +// ErrReaderClosed once shutdown starts. Removing the final stream returns +// ErrLastStream. func (r *Reader) RemoveStream(ctx context.Context, stream *Stream) error { r.lock.Lock() defer r.lock.Unlock() + if r.closing.Load() { + return ErrReaderClosed + } + index := -1 for i, st := range r.streams { - if st == stream { - r.streams = append(r.streams[:i], r.streams[i+1:]...) - r.streamKeys = append(r.streamKeys[:i], r.streamKeys[i+1:]...) - r.streamCursors = append(r.streamCursors[:i], r.streamCursors[i+1:]...) + if st == stream || + (stream.generation != "" && + st.Name == stream.Name && + st.generation == stream.generation) { + index = i break } } - r.notifyStreamChange() + if index == -1 { + return nil + } + if len(r.streams) == 1 { + return ErrLastStream + } + attached := r.streams[index] + if err := attached.verifyExistingGeneration(ctx); err != nil { + return err + } + r.streams = append(r.streams[:index], r.streams[index+1:]...) + r.streamKeys = append(r.streamKeys[:index], r.streamKeys[index+1:]...) + r.streamCursors = append(r.streamCursors[:index], r.streamCursors[index+1:]...) r.logger.Info("removed", "stream", stream.Name) return nil } @@ -203,17 +268,21 @@ func (r *Reader) RemoveStream(ctx context.Context, stream *Stream) error { // Close stops event polling and closes the reader channel. It is safe to call // Close multiple times; concurrent callers block until the first Close // completes. Close returns only once the read goroutine has stopped and its -// resources are released, which may take up to one block duration. +// resources are released. The configured finite block duration bounds shutdown +// even when the Redis client does not interrupt a blocking read on cancellation. func (r *Reader) Close() { r.closeOnce.Do(func() { + r.closing.Store(true) + r.cancel() // Close donechan first, without holding the lock, so the signal // reaches the read loop even when it is parked on a fan-out send // to a stalled subscriber (which holds the lock). Otherwise Close // would deadlock acquiring the lock the read loop never releases. close(r.donechan) + // Synchronize with a Subscribe already inside the admission lock so its + // wait-group Add completes before Wait begins. Future subscriptions see + // closing and cannot start the read loop. r.lock.Lock() - r.closing = true - close(r.streamschan) r.lock.Unlock() r.wait.Wait() r.lock.Lock() @@ -244,24 +313,27 @@ func (r *Reader) start() { var xreadFn = (*Reader).xread // read reads events from the streams and sends them to the reader channel. -// Transient Redis failures are retried with jittered exponential backoff so -// reader replicas that fail together do not hammer Redis in lockstep. func (r *Reader) read() { - ctx := context.Background() defer r.cleanup() var retry readRetry for { - streamsEvents, err := xreadFn(r, ctx) + snapshot, err := r.readSnapshot(r.ctx) + if err != nil { + if fatal := fatalReadError(err); fatal != nil { + pulse.Go(r.logger, r.Close) + return + } + if !retry.wait(r.donechan, err, r.logger) { + return + } + continue + } + streamsEvents, err := xreadFn(r, r.ctx, snapshot.args) if r.isClosing() { return } if err != nil { - if err == redis.Nil { - // No event at this time, just loop. - retry.reset() - continue - } - if isFatalReaderError(err) { + if err := fatalReadError(err); err != nil { r.logger.Error(fmt.Errorf("fatal error while reading events: %w, stopping", err)) // Close waits on this goroutine via wait.Wait, so calling it // synchronously here would deadlock and leak the reader and its @@ -270,6 +342,10 @@ func (r *Reader) read() { pulse.Go(r.logger, r.Close) return } + if err == redis.Nil { + retry.reset() + continue + } if !retry.wait(r.donechan, err, r.logger) { return } @@ -277,11 +353,35 @@ func (r *Reader) read() { } retry.reset() - r.lock.Lock() for _, events := range streamsEvents { - streamName := events.Stream[len(streamKeyPrefix):] - if err := streamEvents(ctx, streamName, events.Stream, "", events.Messages, r.rdb, false, r.eventFilter, r.chans, r.donechan, r.logger); err != nil { - r.logger.Error(fmt.Errorf("failed to stream events: %w", err)) + stream := snapshot.streams[events.Stream] + if stream == nil { + continue + } + if verifyErr := stream.verifyGeneration(r.ctx); verifyErr != nil { + err = verifyErr + break + } + r.lock.Lock() + if !r.ownsStream(stream) { + r.lock.Unlock() + continue + } + if err := streamEvents( + r.ctx, + stream, + "", + events.Messages, + false, + r.eventFilter, + r.chans, + r.donechan, + r.rdb, + r.logger, + ); err != nil { + r.logger.Error(fmt.Errorf("failed to stream reader events: %w", err)) + r.lock.Unlock() + continue } for i := range r.streamKeys { if r.streamKeys[i] == events.Stream { @@ -289,19 +389,36 @@ func (r *Reader) read() { break } } + r.lock.Unlock() + } + if fatal := fatalReadError(err); fatal != nil { + pulse.Go(r.logger, r.Close) + return } - r.lock.Unlock() } } -func (r *Reader) xread(ctx context.Context) ([]redis.XStream, error) { - // copy so no two goroutines can share the memory +// readSnapshot verifies and captures the exact stream capabilities used by one +// Redis read. +func (r *Reader) readSnapshot(ctx context.Context) (readerSnapshot, error) { r.lock.Lock() - readStreams := make([]string, len(r.streamKeys)) - copy(readStreams, r.streamKeys) - readStreams = append(readStreams, r.streamCursors...) - r.lock.Unlock() + defer r.lock.Unlock() + snapshot := readerSnapshot{ + streams: make(map[string]*Stream, len(r.streams)), + args: make([]string, 0, len(r.streamKeys)+len(r.streamCursors)), + } + for _, stream := range r.streams { + if err := stream.verifyGeneration(ctx); err != nil { + return readerSnapshot{}, err + } + snapshot.streams[stream.key] = stream + } + snapshot.args = append(snapshot.args, r.streamKeys...) + snapshot.args = append(snapshot.args, r.streamCursors...) + return snapshot, nil +} +func (r *Reader) xread(ctx context.Context, readStreams []string) ([]redis.XStream, error) { r.logger.Debug("reading", "streams", readStreams, "max", r.maxPolled, "block", r.blockDuration) return r.rdb.XRead(ctx, &redis.XReadArgs{ Streams: readStreams, @@ -310,12 +427,15 @@ func (r *Reader) xread(ctx context.Context) ([]redis.XStream, error) { }).Result() } -// notifyStreamChange notifies the reader that the streams have changed. -func (r *Reader) notifyStreamChange() { - select { - case r.streamschan <- struct{}{}: - default: +// ownsStream reports whether the current reader still owns the exact snapshot +// capability after a concurrent RemoveStream. +func (r *Reader) ownsStream(candidate *Stream) bool { + for _, stream := range r.streams { + if stream == candidate { + return true + } } + return false } // cleanup removes the consumer from the consumer groups and removes the reader @@ -333,9 +453,7 @@ func (r *Reader) cleanup() { // isClosing returns true if the reader is stopping. func (r *Reader) isClosing() bool { - r.lock.Lock() - defer r.lock.Unlock() - return r.closing + return r.closing.Load() } // CreatedAt returns the event creation time (millisecond precision). @@ -347,91 +465,66 @@ func (e *Event) CreatedAt() time.Time { return time.Unix(seconds, nanos).UTC() } -// reset clears the backoff so the next failure starts from the minimum delay. -func (r *readRetry) reset() { - r.backoff = 0 -} - -// wait logs err and sleeps for the next jittered backoff delay. It returns -// false when done closes during the wait, signaling the caller to stop. -func (r *readRetry) wait(done <-chan struct{}, err error, logger pulse.Logger) bool { - d := r.next() - logger.Error(fmt.Errorf("failed to read events: %w, retrying in %v", err, d)) - select { - case <-done: - return false - case <-time.After(d): - return true - } -} - -// next returns a delay uniformly distributed in [backoff/2, backoff] and -// doubles the backoff up to maxReadRetryBackoff. -func (r *readRetry) next() time.Duration { - if r.backoff == 0 { - r.backoff = minReadRetryBackoff - } - d := r.backoff/2 + time.Duration(rand.Int63n(int64(r.backoff/2)+1)) - r.backoff = min(2*r.backoff, maxReadRetryBackoff) - return d -} - -// streamEvents filters and streams the Redis messages as events to the -// subscriber channels. The caller is responsible for locking chans. For sinks -// (sinkName non-empty) acker settles events that are never delivered to -// subscribers so the recovery cursor keeps advancing: filtered events are -// acknowledged individually, and when autoAck is set the whole batch is -// settled upfront because delivery is already at-most-once. +// streamEvents filters and streams Redis messages. Sink events receive a +// recovery-aware Acker; auto-ack sinks acknowledge each message before it can +// be exposed to filters or subscribers. func streamEvents( ctx context.Context, - streamName string, - streamKey string, + stream *Stream, sinkName string, msgs []redis.XMessage, - acker Acker, autoAck bool, eventFilter eventFilterFunc, chans []chan *Event, done <-chan struct{}, + rdb *redis.Client, logger pulse.Logger, ) error { if len(msgs) == 0 { return nil } - if autoAck && sinkName != "" { - ids := make([]string, len(msgs)) - for i, msg := range msgs { - ids[i] = msg.ID - } - if err := acker.XAck(ctx, streamKey, sinkName, ids...).Err(); err != nil { - return fmt.Errorf("failed to advance recovery cursor for auto-acked events: %w", err) - } - } for _, event := range msgs { - var topic string - if t, ok := event.Values[topicKey]; ok { - topic = t.(string) + name, topic, payload, err := decodeRedisEvent(event) + if err != nil { + if sinkName != "" { + acker := &recoveryAcker{stream: stream} + if ackErr := acker.XAck(ctx, stream.key, sinkName, event.ID).Err(); ackErr != nil { + return errors.Join(err, fmt.Errorf("acknowledge malformed sink event %s: %w", event.ID, ackErr)) + } + } + logger.Error(err, "stream", stream.Name, "id", event.ID) + continue + } + var acker Acker = rdb + if sinkName != "" { + acker = &recoveryAcker{stream: stream} } ev := &Event{ - ID: event.ID, - StreamName: streamName, - SinkName: sinkName, - EventName: event.Values[nameKey].(string), - Topic: topic, - Payload: []byte(event.Values[payloadKey].(string)), - streamKey: streamKey, - Acker: acker, + ID: event.ID, + StreamName: stream.Name, + StreamGeneration: stream.generation, + SinkName: sinkName, + EventName: name, + Topic: topic, + Payload: payload, + streamKey: stream.key, + Acker: acker, + } + if autoAck { + if err := ev.Acker.XAck(ctx, stream.key, sinkName, event.ID).Err(); err != nil { + return err + } } if eventFilter != nil && !eventFilter(ev) { if sinkName != "" && !autoAck { - if err := acker.XAck(ctx, streamKey, sinkName, ev.ID).Err(); err != nil { - return fmt.Errorf("failed to acknowledge filtered event %s: %w", ev.ID, err) + if err := ev.Acker.XAck(ctx, stream.key, sinkName, event.ID).Err(); err != nil { + return fmt.Errorf("failed to acknowledge filtered sink event %s: %w", event.ID, err) } } - logger.Debug("event filtered", "event", ev.EventName, "id", ev.ID, "stream", streamName) + logger.Debug("event filtered", "event", ev.EventName, "id", ev.ID, "stream", stream.Name) continue } - logger.Debug("event", "stream", streamName, "event", ev.EventName, "id", ev.ID, "channels", len(chans)) + logger.Debug("event", "stream", stream.Name, "event", ev.EventName, "id", ev.ID, "channels", len(chans)) for _, c := range chans { select { case c <- ev: @@ -449,8 +542,91 @@ func streamEvents( return nil } -// isFatalReaderError reports whether the read loop must stop instead of -// retrying, which happens when the underlying stream key was destroyed. -func isFatalReaderError(err error) bool { - return strings.Contains(err.Error(), "stream key no longer exists") +// decodeRedisEvent validates the externally writable Redis stream boundary. +func decodeRedisEvent(event redis.XMessage) (string, string, []byte, error) { + name, ok := event.Values[nameKey].(string) + if !ok || name == "" { + return "", "", nil, fmt.Errorf( + "pulse streaming: malformed event %s: required field %q must be a non-empty string", + event.ID, + nameKey, + ) + } + payload, ok := event.Values[payloadKey].(string) + if !ok { + return "", "", nil, fmt.Errorf( + "pulse streaming: malformed event %s: required field %q must be a string", + event.ID, + payloadKey, + ) + } + var topic string + if value, exists := event.Values[topicKey]; exists { + var valid bool + topic, valid = value.(string) + if !valid { + return "", "", nil, fmt.Errorf( + "pulse streaming: malformed event %s: optional field %q must be a string", + event.ID, + topicKey, + ) + } + } + return name, topic, []byte(payload), nil +} + +// fatalReadError returns errors that permanently invalidate a reader. All +// other errors are transient and are retried through readRetry. +func fatalReadError(err error) error { + if err == nil { + return nil + } + if errors.Is(err, ErrStreamDestroyed) || errors.Is(err, ErrDeadlineElapsed) { + return err + } + return nil +} + +// reset restores the retry delay after Redis processes a read or repairs a +// missing consumer group. +func (r *readRetry) reset() { + r.failures = 0 +} + +// nextDelay returns a half-to-full-jitter exponential delay. The non-zero +// lower bound prevents a hot loop while randomization avoids fleet-wide retry +// synchronization. +func (r *readRetry) nextDelay() time.Duration { + limit := readRetryInitialDelay + for range r.failures { + limit = min(limit*2, readRetryMaxDelay) + if limit == readRetryMaxDelay { + break + } + } + if limit < readRetryMaxDelay { + r.failures++ + } + jitter := r.jitter + if jitter == nil { + jitter = rand.Int63n + } + floor := limit / 2 + return floor + time.Duration(jitter(int64(limit-floor)+1)) +} + +// wait applies the next bounded jittered delay. It returns false when shutdown +// interrupts the wait so Close does not wait for a retry timer. +func (r *readRetry) wait(done <-chan struct{}, err error, logger pulse.Logger) bool { + delay := r.nextDelay() + logger.Error(fmt.Errorf("failed to read events: %w", err), "retry_in", delay) + + timer := time.NewTimer(delay) + defer timer.Stop() + select { + case <-timer.C: + return true + case <-done: + return false + } } diff --git a/streaming/reader_test.go b/streaming/reader_test.go index 788f0b8..f2e0ae2 100644 --- a/streaming/reader_test.go +++ b/streaming/reader_test.go @@ -2,9 +2,9 @@ package streaming import ( "context" - "fmt" "strconv" "strings" + "sync" "testing" "time" @@ -17,6 +17,130 @@ import ( ptesting "goa.design/pulse/testing" ) +func TestReaderRejectsInvalidOptions(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + cases := []struct { + name string + opt options.Reader + }{ + {name: "max polled", opt: options.WithReaderMaxPolled(0)}, + {name: "buffer", opt: options.WithReaderBufferSize(-1)}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + stream, err := NewStream(t.Name(), rdb) + require.NoError(t, err) + _, err = stream.NewReader(ctx, tc.opt) + require.Error(t, err) + }) + } + stream, err := NewStream(t.Name()+"-conflicts", rdb) + require.NoError(t, err) + _, err = stream.NewReader( + ctx, + options.WithReaderTopic("alarms"), + options.WithReaderTopicPattern("alarm.*"), + ) + require.ErrorContains(t, err, "mutually exclusive") + _, err = stream.NewReader( + ctx, + options.WithReaderStartAtNewest(), + options.WithReaderStartAtOldest(), + ) + require.ErrorContains(t, err, "reader cursor-start options are mutually exclusive") + require.ErrorIs(t, stream.Destroy(ctx), ErrStreamNotFound) +} + +func TestReaderDropsMalformedRedisEventWithoutPanicking(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + stream, err := NewStream(t.Name(), rdb) + require.NoError(t, err) + require.NoError(t, stream.Open(ctx)) + reader, err := stream.NewReader( + ctx, + options.WithReaderStartAtOldest(), + options.WithReaderBlockDuration(testBlockDuration), + ) + require.NoError(t, err) + events := reader.Subscribe() + require.NoError(t, rdb.XAdd(ctx, &redis.XAddArgs{ + Stream: stream.key, + Values: map[string]any{payloadKey: "missing name"}, + }).Err()) + _, err = stream.Add(ctx, "valid", []byte("payload")) + require.NoError(t, err) + + select { + case event := <-events: + require.Equal(t, "valid", event.EventName) + require.Equal(t, stream.Generation(), event.StreamGeneration) + case <-time.After(time.Second): + t.Fatal("timed out waiting for valid event after malformed entry") + } + reader.Close() + require.NoError(t, stream.Destroy(ctx)) +} + +func TestReaderClosingFencesSubscriptionsAndStreamChanges(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + stream, err := NewStream(t.Name()+"-main", rdb) + require.NoError(t, err) + added, err := NewStream(t.Name()+"-added", rdb) + require.NoError(t, err) + require.NoError(t, stream.Open(ctx)) + require.NoError(t, added.Open(ctx)) + reader, err := stream.NewReader(ctx, options.WithReaderBlockDuration(testBlockDuration)) + require.NoError(t, err) + reader.Subscribe() + + start := make(chan struct{}) + subscriptions := make(chan (<-chan *Event), 32) + results := make(chan error, 64) + var wait sync.WaitGroup + for range 32 { + wait.Add(2) + go func() { + defer wait.Done() + <-start + subscriptions <- reader.Subscribe() + }() + go func() { + defer wait.Done() + <-start + results <- reader.AddStream(ctx, added) + results <- reader.RemoveStream(ctx, added) + }() + } + close(start) + reader.Close() + wait.Wait() + close(subscriptions) + close(results) + + for result := range results { + if result != nil { + require.ErrorIs(t, result, ErrReaderClosed) + } + } + for subscription := range subscriptions { + _, ok := <-subscription + require.False(t, ok) + } + closed := reader.Subscribe() + _, ok := <-closed + require.False(t, ok) + require.ErrorIs(t, reader.AddStream(ctx, added), ErrReaderClosed) + require.ErrorIs(t, reader.RemoveStream(ctx, added), ErrReaderClosed) + require.NoError(t, stream.Destroy(ctx)) + require.NoError(t, added.Destroy(ctx)) +} + func TestNewReader(t *testing.T) { testName := strings.Replace(t.Name(), "/", "_", -1) rdb := ptesting.NewRedisClient(t) @@ -24,10 +148,12 @@ func TestNewReader(t *testing.T) { ctx := ptesting.NewTestContext(t) s, err := NewStream(testName, rdb, options.WithStreamLogger(pulse.ClueLogger(ctx))) assert.NoError(t, err) + require.NoError(t, s.Open(ctx)) + defer func() { require.NoError(t, s.Destroy(ctx)) }() reader, err := s.NewReader(ctx, options.WithReaderBlockDuration(testBlockDuration)) assert.NoError(t, err) if assert.NotNil(t, reader) { - defer cleanupReader(t, ctx, s, reader) + defer cleanupReader(t, reader) } _, err = s.NewReader(ctx, options.WithReaderTopicPattern("(")) @@ -41,9 +167,11 @@ func TestReaderReadOnce(t *testing.T) { ctx := ptesting.NewTestContext(t) s, err := NewStream(testName, rdb, options.WithStreamLogger(pulse.ClueLogger(ctx))) assert.NoError(t, err) + require.NoError(t, s.Open(ctx)) + defer func() { require.NoError(t, s.Destroy(ctx)) }() reader, err := s.NewReader(ctx, options.WithReaderStartAtOldest(), options.WithReaderBlockDuration(testBlockDuration)) require.NoError(t, err) - defer cleanupReader(t, ctx, s, reader) + defer cleanupReader(t, reader) c := reader.Subscribe() _, err = s.Add(ctx, "event", []byte("payload")) @@ -60,11 +188,13 @@ func TestReaderReadSinceLastEvent(t *testing.T) { ctx := ptesting.NewTestContext(t) s, err := NewStream(testName, rdb, options.WithStreamLogger(pulse.ClueLogger(ctx))) assert.NoError(t, err) + require.NoError(t, s.Open(ctx)) + defer func() { require.NoError(t, s.Destroy(ctx)) }() // Add and read 2 events consecutively reader, err := s.NewReader(ctx, options.WithReaderStartAtOldest(), options.WithReaderBlockDuration(testBlockDuration)) require.NoError(t, err) - defer cleanupReader(t, ctx, s, reader) + defer cleanupReader(t, reader) c := reader.Subscribe() _, err = s.Add(ctx, "event", []byte("payload")) require.NoError(t, err) @@ -81,7 +211,7 @@ func TestReaderReadSinceLastEvent(t *testing.T) { // Create new reader with last event ID set to first event and read last event reader2, err := s.NewReader(ctx, options.WithReaderStartAfter(eventID), options.WithReaderBlockDuration(testBlockDuration)) require.NoError(t, err) - defer cleanupReader(t, ctx, s, reader2) + defer cleanupReader(t, reader2) c2 := reader2.Subscribe() read = readOneReaderEvent(t, c2) assert.Equal(t, "event", read.EventName) @@ -90,7 +220,7 @@ func TestReaderReadSinceLastEvent(t *testing.T) { // Create new reader with last event ID set to 0 and read the 2 events reader3, err := s.NewReader(ctx, options.WithReaderStartAfter("0"), options.WithReaderBlockDuration(testBlockDuration)) require.NoError(t, err) - defer cleanupReader(t, ctx, s, reader3) + defer cleanupReader(t, reader3) c3 := reader3.Subscribe() read = readOneReaderEvent(t, c3) assert.Equal(t, "event", read.EventName) @@ -107,6 +237,7 @@ func TestCleanupReader(t *testing.T) { ctx := ptesting.NewTestContext(t) s, err := NewStream(testName, rdb, options.WithStreamLogger(pulse.ClueLogger(ctx))) assert.NoError(t, err) + require.NoError(t, s.Open(ctx)) reader, err := s.NewReader(ctx, options.WithReaderStartAtOldest(), options.WithReaderBlockDuration(testBlockDuration)) require.NoError(t, err) @@ -133,14 +264,15 @@ func TestReaderCloseOnFatalReadError(t *testing.T) { ctx := ptesting.NewTestContext(t) s, err := NewStream(testName, rdb, options.WithStreamLogger(pulse.ClueLogger(ctx))) require.NoError(t, err) - // Simulate a fatal read error (e.g. the underlying stream key being - // destroyed) before the read goroutine starts. The read loop reacts to a - // fatal error by closing the reader; because Close waits on the read - // goroutine, it must run asynchronously or it would deadlock and leak the - // reader and its Redis connection. - defer func(orig func(*Reader, context.Context) ([]redis.XStream, error)) { xreadFn = orig }(xreadFn) - xreadFn = func(*Reader, context.Context) ([]redis.XStream, error) { - return nil, fmt.Errorf("stream key no longer exists") + require.NoError(t, s.Open(ctx)) + defer func() { require.NoError(t, s.Destroy(ctx)) }() + // Simulate exact-generation destruction before the read goroutine starts. + // The read loop reacts by closing the reader; because Close waits on the + // read goroutine, it must run asynchronously or it would deadlock and leak + // the reader and its Redis connection. + defer func(orig func(*Reader, context.Context, []string) ([]redis.XStream, error)) { xreadFn = orig }(xreadFn) + xreadFn = func(*Reader, context.Context, []string) ([]redis.XStream, error) { + return nil, ErrStreamDestroyed } reader, err := s.NewReader(ctx, options.WithReaderBlockDuration(testBlockDuration)) @@ -151,6 +283,50 @@ func TestReaderCloseOnFatalReadError(t *testing.T) { "reader did not close after a fatal read error (Close likely deadlocked on its own read goroutine)") } +func TestReaderLifetimeDoesNotUseConstructorContext(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + setupCtx, cancel := context.WithCancel(ptesting.NewTestContext(t)) + stream, err := NewStream(t.Name(), rdb) + require.NoError(t, err) + require.NoError(t, stream.Open(setupCtx)) + reader, err := stream.NewReader( + setupCtx, + options.WithReaderStartAtOldest(), + options.WithReaderBlockDuration(testBlockDuration), + ) + require.NoError(t, err) + events := reader.Subscribe() + cancel() + + eventID, err := stream.Add(context.Background(), "event", []byte("payload")) + require.NoError(t, err) + require.Equal(t, eventID, receiveSinkEvent(t, events).ID) + reader.Close() + require.NoError(t, stream.Destroy(context.Background())) +} + +func TestReaderRejectsRemovingFinalStream(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + stream, err := NewStream(t.Name(), rdb) + require.NoError(t, err) + require.NoError(t, stream.Open(ctx)) + reader, err := stream.NewReader(ctx, options.WithReaderBlockDuration(testBlockDuration)) + require.NoError(t, err) + + unattached, err := NewStream(t.Name()+"-unattached", rdb) + require.NoError(t, err) + cancelled, cancel := context.WithCancel(ctx) + cancel() + require.NoError(t, reader.RemoveStream(cancelled, unattached)) + require.Empty(t, unattached.Generation()) + require.ErrorIs(t, reader.RemoveStream(ctx, stream), ErrLastStream) + reader.Close() + require.NoError(t, stream.Destroy(ctx)) +} + func TestAddReaderStream(t *testing.T) { testName := strings.Replace(t.Name(), "/", "_", -1) rdb := ptesting.NewRedisClient(t) @@ -158,14 +334,17 @@ func TestAddReaderStream(t *testing.T) { ctx := ptesting.NewTestContext(t) s, err := NewStream("testAddStream", rdb, options.WithStreamLogger(pulse.ClueLogger(ctx))) assert.NoError(t, err) + require.NoError(t, s.Open(ctx)) + defer func() { assert.NoError(t, s.Destroy(ctx)) }() reader, err := s.NewReader(ctx, options.WithReaderStartAtOldest(), options.WithReaderBlockDuration(testBlockDuration)) require.NoError(t, err) s2, err := NewStream("testAddStream2", rdb, options.WithStreamLogger(pulse.ClueLogger(ctx))) assert.NoError(t, err) + require.NoError(t, s2.Open(ctx)) assert.NoError(t, reader.AddStream(ctx, s2)) assert.NoError(t, reader.AddStream(ctx, s2)) // Make sure it's idempotent defer func() { assert.NoError(t, s2.Destroy(ctx)) }() - defer cleanupReader(t, ctx, s, reader) + defer cleanupReader(t, reader) // Add events to both streams c := reader.Subscribe() @@ -190,13 +369,16 @@ func TestRemoveReaderStream(t *testing.T) { ctx := ptesting.NewTestContext(t) s, err := NewStream("testRemoveStream", rdb, options.WithStreamLogger(pulse.ClueLogger(ctx))) assert.NoError(t, err) + require.NoError(t, s.Open(ctx)) + defer func() { assert.NoError(t, s.Destroy(ctx)) }() s2, err := NewStream("testRemoveStream2", rdb, options.WithStreamLogger(pulse.ClueLogger(ctx))) assert.NoError(t, err) + require.NoError(t, s2.Open(ctx)) reader, err := s.NewReader(ctx, options.WithReaderStartAtOldest(), options.WithReaderBlockDuration(testBlockDuration)) require.NoError(t, err) assert.NoError(t, reader.AddStream(ctx, s2)) defer func() { assert.NoError(t, s2.Destroy(ctx)) }() - defer cleanupReader(t, ctx, s, reader) + defer cleanupReader(t, reader) // Read events from both streams c := reader.Subscribe() @@ -227,6 +409,7 @@ func TestReaderCloseWithStalledSubscriber(t *testing.T) { ctx := ptesting.NewTestContext(t) s, err := NewStream(testName, rdb, options.WithStreamLogger(pulse.ClueLogger(ctx))) require.NoError(t, err) + require.NoError(t, s.Open(ctx)) // Tiny buffer so the read loop's fan-out send blocks after a couple of // events when the subscriber never drains its channel. @@ -267,6 +450,20 @@ func TestReaderCloseWithStalledSubscriber(t *testing.T) { require.NoError(t, s.Destroy(ctx)) } +func TestReaderRejectsSubMillisecondBlockDuration(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + stream, err := NewStream(t.Name(), rdb, options.WithStreamLogger(pulse.ClueLogger(ctx))) + require.NoError(t, err) + for _, duration := range []time.Duration{0, -time.Second, 500 * time.Microsecond} { + reader, err := stream.NewReader(ctx, options.WithReaderBlockDuration(duration)) + require.Nil(t, reader) + require.EqualError(t, err, "reader block duration must be at least 1ms") + } + require.ErrorIs(t, stream.Destroy(ctx), ErrStreamNotFound) +} + func TestEventCreatedAt(t *testing.T) { rdb := ptesting.NewRedisClient(t) defer ptesting.CleanupRedis(t, rdb, false, "") diff --git a/streaming/sink.go b/streaming/sink.go index aa59f2f..f65584b 100644 --- a/streaming/sink.go +++ b/streaming/sink.go @@ -1,10 +1,3 @@ -// This file implements Sink, the consumer-group side of a stream. A sink read -// loop XREADGROUPs events for every stream added to the sink, fans them out -// to subscribers, and settles them through the recovery acker defined in -// sink_recovery.go so the durable recovery cursor tracks exactly what was -// acknowledged. Background goroutines refresh the sink keep-alive and, under -// the fenced lease defined in sink_lease.go, claim idle messages and delete -// stale consumers left behind by dead sink instances. package streaming import ( @@ -13,7 +6,6 @@ import ( "fmt" "regexp" "sort" - "strconv" "sync" "sync/atomic" "time" @@ -27,7 +19,31 @@ import ( "goa.design/pulse/streaming/options" ) +var ( + // checkIdlePeriod is the period at which idle messages are checked. + checkIdlePeriod = 500 * time.Millisecond +) + type ( + // sinkStream is the complete coordination state owned for one attached + // stream incarnation. + sinkStream struct { + stream *Stream + startID string + consumers *rmap.Map + keepAlives *rmap.Map + leaseKey string + leaseOwner string + } + + // sinkSnapshot is the exact stream and consumer set used by one + // XREADGROUP command. + sinkSnapshot struct { + streams map[string]*sinkStream + args []string + consumer string + } + // Sink represents a stream sink. Sink struct { // Name is the sink name. @@ -36,17 +52,14 @@ type ( closed bool // consumer is the sink consumer name. consumer string - // leaseOwner identifies this sink instance in the fenced lease used - // for idle message claiming and stale consumer cleanup. - leaseOwner string // startID is the sink start event ID. startID string // noAck is true if there is no need to acknowledge events. noAck bool // lock is the sink mutex. lock sync.Mutex - // streams are the streams the sink consumes events from, indexed by - // stream Redis key. + // streams is the per-incarnation coordination state indexed by physical + // stream key. streams map[string]*sinkStream // blockDuration is the XREADGROUP timeout. blockDuration time.Duration @@ -59,64 +72,54 @@ type ( chans []chan *Event // donechan is the sink done channel. donechan chan struct{} + // ctx is canceled when Close begins and bounds all sink-owned Redis I/O. + ctx context.Context + // cancel interrupts sink-owned Redis I/O. + cancel context.CancelFunc // wait is the sink cleanup wait group. wait sync.WaitGroup - // closeOnce is used to ensure the sink is closed only once. - closeOnce sync.Once - // closing is set when Close starts so loops stop scheduling work. + // stopOnce cancels background work exactly once; distributed cleanup is + // intentionally retried until it succeeds. + stopOnce sync.Once + // closing is true if Close was called. closing atomic.Bool - // ctx is canceled by Close to abort all sink-owned Redis I/O, - // including blocked XREADGROUP calls and recovery in progress. - ctx context.Context - // cancel cancels ctx. - cancel context.CancelFunc // eventFilter is the event filter if any. eventFilter eventFilterFunc - // consumersKeepAliveMap records consumer keep-alives for this - // sink (i.e. for all in-process instances of the sink). - consumersKeepAliveMap *rmap.Map + // filterKind and filterValue preserve the exact shared configuration + // applied to every later AddStream attachment. + filterKind string + filterValue string // ackGracePeriod is the grace period after which an event is // considered unacknowledged. ackGracePeriod time.Duration - // lastKeepAlive is the last keep-alive timestamp for this consumer - // in Redis-time nanoseconds. + // lastKeepAlive is the last keep-alive timestamp for this consumer. lastKeepAlive int64 - // acker settles events and advances the durable recovery cursor. - acker *recoveryAcker // logger is the logger used by the sink. logger pulse.Logger // rdb is the redis connection. rdb *redis.Client } - // sinkStream is the sink-side state for one consumed stream: the stream - // handle, the start ID used when (re)creating the consumer group, and the - // replicated membership map listing the consumers of each sink. - sinkStream struct { - // stream is the consumed stream. - stream *Stream - // startID is the group start position for brand new groups. - startID string - // consumers is the stream membership map (sink name to consumer - // names), joined for reads and change notifications; all writes go - // through the fenced scripts in sink_recovery.go. - consumers *rmap.Map - } - // eventFilterFunc is the function used to filter events. eventFilterFunc func(*Event) bool ) -// checkIdlePeriod is the period at which idle messages are checked. -var checkIdlePeriod = 500 * time.Millisecond +var ( + // ErrSinkClosed is returned when stream ownership is changed after sink + // shutdown begins. + ErrSinkClosed = errors.New("pulse streaming: sink is closed") +) -// newSink creates a new sink. -// Sinks use one Redis consumer per stream they are consuming from. -// Pulse maintains a pool of consumers per stream and reuses them when possible. -// This is because deleting a consumer causes Redis to drop its pending messages -// which is not the semantics Pulse wants to enforce. +// newSink creates a sink whose consumer identity spans every attached stream. +// Each stream generation owns independent membership, keepalive, recovery, and +// PEL state for that identity. Detachment deletes the Redis consumer only when +// that consumer's own PEL is empty, because deletion would otherwise discard +// pending ownership. func newSink(ctx context.Context, name string, stream *Stream, opts ...options.Sink) (*Sink, error) { o := options.ParseSinkOptions(opts...) + if err := validateSinkOptions(o); err != nil { + return nil, err + } var eventMatcher eventFilterFunc if o.Topic != "" { eventMatcher = func(e *Event) bool { return e.Topic == o.Topic } @@ -127,55 +130,51 @@ func newSink(ctx context.Context, name string, stream *Stream, opts ...options.S } eventMatcher = func(e *Event) bool { return topicPatternRegexp.MatchString(e.Topic) } } - + if err := stream.ensureGeneration(ctx); err != nil { + return nil, err + } logger := stream.rootLogger.WithPrefix("sink", name) - km, err := rmap.Join(ctx, sinkKeepAliveMapName(name), stream.rdb, rmap.WithLogger(logger)) + config := sinkConfigurationForOptions(o, o.LastEventID) + state, err := attachSinkStream(ctx, stream, name, config, logger) if err != nil { - return nil, fmt.Errorf("failed to join replicated map for sink keep-alives %s: %w", name, err) + return nil, err } - // runCtx outlives the caller context and is canceled by Close so all - // sink-owned Redis I/O, including recovery in progress, stops promptly. - runCtx, cancel := context.WithCancel(log.WithContext(context.Background(), ctx)) - + // Preserve the caller's logger while giving all background Redis commands + // a context Close can cancel. + logCtx := context.Background() + logCtx = log.WithContext(logCtx, ctx) + runCtx, cancel := context.WithCancel(logCtx) sink := &Sink{ - Name: name, - leaseOwner: ulid.Make().String(), - startID: o.LastEventID, - noAck: o.NoAck, - streams: make(map[string]*sinkStream, 1), - blockDuration: o.BlockDuration, - maxPolled: o.MaxPolled, - bufferSize: o.BufferSize, - donechan: make(chan struct{}), - ctx: runCtx, - cancel: cancel, - eventFilter: eventMatcher, - consumersKeepAliveMap: km, - ackGracePeriod: o.AckGracePeriod, - acker: &recoveryAcker{rdb: stream.rdb}, - logger: logger, - rdb: stream.rdb, - } - - state, err := sink.attachStream(ctx, stream, o.LastEventID) - if err != nil { - cancel() - km.Close() - return nil, err + Name: name, + startID: o.LastEventID, + noAck: o.NoAck, + streams: map[string]*sinkStream{stream.key: state}, + blockDuration: o.BlockDuration, + maxPolled: o.MaxPolled, + bufferSize: o.BufferSize, + donechan: make(chan struct{}), + ctx: runCtx, + cancel: cancel, + eventFilter: eventMatcher, + filterKind: config.filterKind, + filterValue: config.filterValue, + ackGracePeriod: o.AckGracePeriod, + logger: logger, + rdb: stream.rdb, + } + + // Clean up existing stale consumers under this stream's fenced lease before + // creating our own. + if err := sink.deleteStreamStaleConsumersWithLease(ctx, state); err != nil { + sink.logger.Error(fmt.Errorf("failed to cleanup stale consumers: %w", err)) } - sink.streams[stream.key] = state consumer, err := sink.newConsumer(ctx) if err != nil { - // Compensate the group and cursor created by attachStream (the group - // survives only when other sink instances are members). - if cerr := removeSinkStream(ctx, stream, name, ""); cerr != nil { - err = errors.Join(err, cerr) - } cancel() - state.consumers.Close() - km.Close() + closeSetupMembership(ctx, stream, state.consumers, err) + state.keepAlives.Close() return nil, fmt.Errorf("failed to create consumer: %w", err) } sink.consumer = consumer @@ -191,11 +190,16 @@ func newSink(ctx context.Context, name string, stream *Stream, opts ...options.S return sink, nil } -// Subscribe returns a channel that receives events from the sink. +// Subscribe returns a channel that receives events from the sink. Calls made +// after shutdown starts return an already closed channel. func (s *Sink) Subscribe() <-chan *Event { c := make(chan *Event, s.bufferSize) s.lock.Lock() defer s.lock.Unlock() + if s.closing.Load() { + close(c) + return c + } s.chans = append(s.chans, c) return c } @@ -213,10 +217,9 @@ func (s *Sink) Unsubscribe(c <-chan *Event) { } } -// Ack acknowledges the event and advances the sink recovery cursor. +// Ack acknowledges the event. func (s *Sink) Ack(ctx context.Context, e *Event) error { - err := e.Acker.XAck(ctx, e.streamKey, e.SinkName, e.ID).Err() - if err != nil { + if err := e.Acker.XAck(ctx, e.streamKey, e.SinkName, e.ID).Err(); err != nil { s.logger.Error(err, "ack", e.ID, "stream", e.StreamName) return err } @@ -226,34 +229,48 @@ func (s *Sink) Ack(ctx context.Context, e *Event) error { // AddStream adds the stream to the sink. By default the stream cursor starts at // the same timestamp as the sink main stream cursor. This can be overridden -// with opts. AddStream does nothing if the stream is already part of the sink. -// It returns ErrSinkClosed after Close. +// with opts. AddStream does nothing if the stream is already part of the sink +// and returns ErrSinkClosed once shutdown starts. func (s *Sink) AddStream(ctx context.Context, stream *Stream, opts ...options.AddStream) error { s.lock.Lock() defer s.lock.Unlock() if s.closing.Load() { return ErrSinkClosed } - if _, ok := s.streams[stream.key]; ok { - return nil + addOptions := options.ParseAddStreamOptions(opts...) + if err := validateAddStreamOptions(addOptions); err != nil { + return err + } + if err := stream.verifyGeneration(ctx); err != nil { + return err + } + for _, state := range s.streams { + if state.stream.Name != stream.Name { + continue + } + if state.stream.generation == stream.generation { + return nil + } + return state.stream.verifyGeneration(ctx) } startID := s.startID - o := options.ParseAddStreamOptions(opts...) - if o.LastEventID != "" { - startID = o.LastEventID + if addOptions.LastEventID != "" { + startID = addOptions.LastEventID } - state, err := s.attachStream(ctx, stream, startID) + config := sinkConfiguration{ + filterKind: s.filterKind, + filterValue: s.filterValue, + startID: startID, + noAck: s.noAck, + ackGrace: s.ackGracePeriod, + } + state, err := attachSinkStream(ctx, stream, s.Name, config, s.logger) if err != nil { return err } - if err := registerSinkConsumer(ctx, stream, s.Name, s.consumer); err != nil { - // Compensate the group and cursor created by attachStream so a failed - // AddStream leaves no dangling ownership state (the group survives - // only when other sink instances are members). - if cerr := removeSinkStream(ctx, stream, s.Name, s.consumer); cerr != nil { - err = errors.Join(err, cerr) - } - state.consumers.Close() + if err := registerSinkConsumer(ctx, state, s.Name, s.consumer, s.lastKeepAlive); err != nil { + closeSetupMembership(ctx, stream, state.consumers, err) + state.keepAlives.Close() return err } s.streams[stream.key] = state @@ -261,57 +278,103 @@ func (s *Sink) AddStream(ctx context.Context, stream *Stream, opts ...options.Ad return nil } -// RemoveStream removes the stream from the sink, it is idempotent. The -// distributed effects (membership removal and, for the last member, consumer -// group, recovery cursor, and lease deletion) execute in one atomic script so -// there is no partial state to compensate. It returns ErrSinkClosed after -// Close. +// RemoveStream removes the stream from the sink. It is idempotent and returns +// ErrSinkClosed once shutdown starts. Removing the final stream returns +// ErrLastStream. func (s *Sink) RemoveStream(ctx context.Context, stream *Stream) error { s.lock.Lock() defer s.lock.Unlock() if s.closing.Load() { return ErrSinkClosed } - state, ok := s.streams[stream.key] - if !ok { + var state *sinkStream + for _, candidate := range s.streams { + if candidate.stream == stream || + (stream.generation != "" && + candidate.stream.Name == stream.Name && + candidate.stream.generation == stream.generation) { + state = candidate + break + } + } + if state == nil { return nil } - if err := removeSinkStream(ctx, state.stream, s.Name, s.consumer); err != nil { + if len(s.streams) == 1 { + return ErrLastStream + } + if err := state.stream.verifyGeneration(ctx); err != nil { return err } - delete(s.streams, stream.key) - state.consumers.Close() + if _, err := detachSinkConsumer(ctx, state, s.Name, s.consumer); err != nil { + return fmt.Errorf("failed to detach consumer %s from stream %s: %w", s.consumer, stream.Name, err) + } + delete(s.streams, state.stream.key) + state.close() s.logger.Info("removed", "stream", stream.Name) return nil } -// Close stops event polling, cancels all sink-owned Redis I/O (including any -// recovery in progress), waits for the sink goroutines to stop, and closes -// the sink channels. It is safe to call Close multiple times; concurrent -// callers block until the first Close completes. -func (s *Sink) Close(ctx context.Context) { - s.closeOnce.Do(func() { - // Signal shutdown without holding the lock so the read loop stops - // even when it is parked on a fan-out send to a stalled subscriber - // (which holds the lock) or blocked in a Redis call. +// Close stops event polling and detaches every distributed membership. Failed +// Redis cleanup is returned and may be retried with another context; the sink +// is closed only after all membership and keep-alive side effects complete. +func (s *Sink) Close(ctx context.Context) error { + s.stopOnce.Do(func() { s.closing.Store(true) s.cancel() + // Close donechan first, without holding the lock, so the signal + // reaches the read loop even when it is parked on a fan-out send + // to a stalled subscriber (which holds the lock). Otherwise Close + // would deadlock acquiring the lock the read loop never releases. close(s.donechan) - s.wait.Wait() - s.lock.Lock() - defer s.lock.Unlock() - for _, c := range s.chans { - close(c) + }) + s.wait.Wait() + + s.lock.Lock() + defer s.lock.Unlock() + if s.closed { + return nil + } + var cleanupErr error + for streamKey, state := range s.streams { + stream := state.stream + err := stream.verifyGeneration(ctx) + if errors.Is(err, ErrStreamDestroyed) || errors.Is(err, ErrDeadlineElapsed) { + state.close() + delete(s.streams, streamKey) + continue } - // Note: we do not delete the consumer from the keep-alive and consumer maps - // so that another instance may claim any pending messages. - s.consumersKeepAliveMap.Close() - for _, state := range s.streams { - state.consumers.Close() + if err != nil { + cleanupErr = errors.Join(cleanupErr, fmt.Errorf( + "failed to verify stream %s before detaching consumer %s: %w", + streamKey, + s.consumer, + err, + )) + continue } - s.closed = true - s.logger.Info("closed") - }) + if _, err := detachSinkConsumer(ctx, state, s.Name, s.consumer); err != nil { + cleanupErr = errors.Join(cleanupErr, fmt.Errorf( + "failed to detach consumer %s from stream %s: %w", + s.consumer, + streamKey, + err, + )) + continue + } + state.close() + delete(s.streams, streamKey) + } + if cleanupErr != nil { + return cleanupErr + } + for _, c := range s.chans { + close(c) + } + s.chans = nil + s.closed = true + s.logger.Info("closed") + return nil } // IsClosed returns true if the sink was closed. @@ -321,211 +384,279 @@ func (s *Sink) IsClosed() bool { return s.closed } -// attachStream ensures the consumer group and recovery cursor exist for the -// stream (restoring the stream TTL even on BUSYGROUP) and joins the stream -// membership map. Callers own registering the sink consumer. -func (s *Sink) attachStream(ctx context.Context, stream *Stream, startID string) (*sinkStream, error) { - if _, _, err := ensureConsumerGroup(ctx, stream, s.Name, startID, true); err != nil { - return nil, err +// deleteStreamStaleConsumersWithLease deletes stale consumers for one attached +// stream only while this replica owns its fenced recovery lease. +func (s *Sink) deleteStreamStaleConsumersWithLease(ctx context.Context, state *sinkStream) error { + duration := 2 * checkIdlePeriod + lease, acquired, err := acquireSinkRecoveryLease( + ctx, + state.stream, + state.leaseKey, + state.leaseOwner, + duration, + ) + if err != nil || !acquired { + return err } - cm, err := rmap.Join(ctx, consumersMapName(stream), stream.rdb, consumersMapOptions(stream, s.logger)...) + return s.deleteStreamStaleConsumers(ctx, state, lease, duration) +} + +// deleteStreamStaleConsumers deletes stale consumers for a specific stream in +// one lease-fenced Redis operation. +// s.lock must be held once the sink is running. +func (s *Sink) deleteStreamStaleConsumers( + ctx context.Context, + state *sinkStream, + lease sinkRecoveryLease, + leaseDuration time.Duration, +) error { + removed, malformed, err := cleanupStaleConsumers( + ctx, + state.stream, + state.leaseKey, + lease, + leaseDuration, + s.Name, + s.ackGracePeriod, + ) + if err != nil { + return err + } + for _, consumer := range removed { + s.logger.Info("cleaned up stale consumer", "consumer", consumer) + } + for _, consumer := range malformed { + s.logger.Error( + fmt.Errorf("invalid keep-alive timestamp"), + "stream", state.stream.Name, + "consumer", consumer, + ) + } + return nil +} + +// newConsumer creates one replacement consumer across every owned stream. No +// stream observes local ownership unless all Redis consumers, memberships, and +// the shared keep-alive are established; failures roll back the full prefix. +func (s *Sink) newConsumer(ctx context.Context) (string, error) { + consumer := ulid.Make().String() + now, err := s.rdb.Time(ctx).Result() if err != nil { - return nil, fmt.Errorf("failed to join replicated map for stream %s: %w", stream.Name, err) + return "", fmt.Errorf("failed to read Redis time for new consumer %s: %w", consumer, err) } - return &sinkStream{stream: stream, startID: startID, consumers: cm}, nil + keepAlive := now.UnixNano() + registered := make([]*sinkStream, 0, len(s.streams)) + for _, state := range s.streams { + registered = append(registered, state) + if err := registerSinkConsumer(ctx, state, s.Name, consumer, keepAlive); err != nil { + return "", errors.Join(err, s.rollbackConsumer(ctx, consumer, registered)) + } + } + s.lastKeepAlive = keepAlive + return consumer, nil } -// read reads events from the streams and sends them to the sink channels. -// NOGROUP errors trigger lossless consumer group recovery; transient Redis -// failures are retried with jittered exponential backoff. +// rollbackConsumer removes a replacement consumer from every stream that was +// registered before consumer creation failed. +func (s *Sink) rollbackConsumer(ctx context.Context, consumer string, states []*sinkStream) error { + var rollbackErr error + for i := len(states) - 1; i >= 0; i-- { + state := states[i] + stream := state.stream + if _, err := detachSinkConsumer(ctx, state, s.Name, consumer); err != nil { + rollbackErr = errors.Join(rollbackErr, fmt.Errorf( + "failed to roll back consumer %s from stream %s: %w", + consumer, + stream.Name, + err, + )) + } + } + return rollbackErr +} + +// read reads events from the streams and sends them to the sink channel. func (s *Sink) read() { defer s.logger.Debug("read: exiting") defer s.wait.Done() var retry readRetry for { if err := s.ensureConsumer(s.ctx); err != nil { - if s.closing.Load() { - return - } if !retry.wait(s.donechan, err, s.logger) { return } continue } - args, consumer := s.readArgs() - if len(args) == 0 { - // No streams to read from; wait for AddStream or Close. - select { - case <-s.donechan: + snapshot, err := s.readSnapshot(s.ctx) + if err != nil { + if fatal := fatalReadError(err); fatal != nil { + pulse.Go(s.logger, func() { + if closeErr := s.Close(context.WithoutCancel(s.ctx)); closeErr != nil { + s.logger.Error(fmt.Errorf("failed to close terminal sink: %w", closeErr)) + } + }) return - case <-time.After(s.blockDuration): - continue } + if !retry.wait(s.donechan, err, s.logger) { + return + } + continue } - s.logger.Debug("reading", "streams", args, "max", s.maxPolled, "block", s.blockDuration) + + s.logger.Debug("reading", "streams", snapshot.args, "max", s.maxPolled, "block", s.blockDuration) streams, err := s.rdb.XReadGroup(s.ctx, &redis.XReadGroupArgs{ Group: s.Name, - Consumer: consumer, - Streams: args, + Consumer: snapshot.consumer, + Streams: snapshot.args, Count: s.maxPolled, Block: s.blockDuration, - NoAck: s.noAck, }).Result() + if s.closing.Load() { // Honor the Close contract and do not forward any more events. // Any events in the PEL will be claimed by another consumer. return } if err == nil { - err = s.dispatch(streams) - } - if err == nil || err == redis.Nil { - retry.reset() - continue + for _, events := range streams { + owned, ok := snapshot.streams[events.Stream] + if !ok { + continue + } + stream := owned.stream + if verifyErr := stream.verifyGeneration(s.ctx); verifyErr != nil { + err = verifyErr + break + } + s.lock.Lock() + current := s.ownsStream(stream) + if !current { + s.lock.Unlock() + if ackErr := acknowledgeMessages(s.ctx, stream, s.Name, events.Messages); ackErr != nil { + err = ackErr + break + } + continue + } + err = streamEvents( + s.ctx, + stream, + s.Name, + events.Messages, + s.noAck, + s.eventFilter, + s.chans, + s.donechan, + s.rdb, + s.logger, + ) + s.lock.Unlock() + if err != nil { + break + } + } } - if redis.HasErrorPrefix(err, "NOGROUP") { - if err := s.recoverConsumerGroups(s.ctx); err == nil { + if err != nil { + if redis.HasErrorPrefix(err, "NOGROUP") { + s.lock.Lock() + err = s.recoverConsumerGroups(s.ctx) + s.lock.Unlock() + } + fatal := fatalReadError(err) + if fatal != nil { + s.logger.Error(fmt.Errorf("fatal error while reading events: %w, stopping", fatal)) + pulse.Go(s.logger, func() { + if err := s.Close(context.WithoutCancel(s.ctx)); err != nil { + s.logger.Error(fmt.Errorf("failed to close terminal sink: %w", err)) + } + }) + return + } + if err == nil || err == redis.Nil { retry.reset() continue - } else if s.closing.Load() { - return - } else if !retry.wait(s.donechan, err, s.logger) { + } + if !retry.wait(s.donechan, err, s.logger) { return } continue } - if !retry.wait(s.donechan, err, s.logger) { - return - } + retry.reset() } } -// dispatch fans out one XREADGROUP reply to the subscribers, settling events -// through the recovery acker. Batches for streams removed from the sink -// concurrently with the read are left pending, never acknowledged: if other -// sink instances remain in the group, the fenced idle-claim redelivers the -// entries to one of them (stale-consumer cleanup skips consumers with pending -// events), and if this was the last member the group was already destroyed so -// the pending entries are gone with it. -func (s *Sink) dispatch(streams []redis.XStream) error { - for _, events := range streams { - s.lock.Lock() - state, owned := s.streams[events.Stream] - if !owned { - s.lock.Unlock() - continue - } - err := streamEvents(s.ctx, state.stream.Name, state.stream.key, s.Name, events.Messages, s.acker, s.noAck, s.eventFilter, s.chans, s.donechan, s.logger) - s.lock.Unlock() - if err != nil { - return err - } - } - return nil -} - -// recoverConsumerGroups recreates missing consumer groups at the durable -// recovery cursor after Redis loses group state (e.g. XGROUP DESTROY). A -// stream that was destroyed with Stream.Destroy is dropped from the sink -// instead of being resurrected. -func (s *Sink) recoverConsumerGroups(ctx context.Context) error { +// readSnapshot verifies and captures the exact stream capabilities and +// consumer used by one grouped read. +func (s *Sink) readSnapshot(ctx context.Context) (sinkSnapshot, error) { s.lock.Lock() defer s.lock.Unlock() - for key, state := range s.streams { - created, cursor, err := ensureConsumerGroup(ctx, state.stream, s.Name, state.startID, false) - if err != nil { - if errors.Is(err, ErrStreamDestroyed) { - s.logger.Info("stream destroyed, dropping from sink", "stream", state.stream.Name) - delete(s.streams, key) - state.consumers.Close() - continue - } - return err - } - if created { - s.logger.Info("recovered consumer group", "stream", state.stream.Name, "cursor", cursor) - } + snapshot := sinkSnapshot{ + streams: make(map[string]*sinkStream, len(s.streams)), + args: make([]string, len(s.streams)*2), + consumer: s.consumer, } - return nil -} - -// readArgs snapshots the XREADGROUP stream arguments and the current consumer -// under the sink lock. -func (s *Sink) readArgs() ([]string, string) { - s.lock.Lock() - defer s.lock.Unlock() keys := make([]string, 0, len(s.streams)) for key := range s.streams { keys = append(keys, key) } sort.Strings(keys) - args := make([]string, 2*len(keys)) for i, key := range keys { - args[i] = key - args[len(keys)+i] = ">" + state := s.streams[key] + if err := state.stream.verifyGeneration(ctx); err != nil { + return sinkSnapshot{}, err + } + snapshot.streams[key] = state + snapshot.args[i] = key + snapshot.args[len(keys)+i] = ">" } - return args, s.consumer + return snapshot, nil } -// ensureConsumer rotates the sink consumer when its keep-alive went stale, -// e.g. after this instance was partitioned long enough for its consumer to be -// cleaned up by a replica. Staleness is evaluated against Redis time so -// client clocks do not skew the decision. -func (s *Sink) ensureConsumer(ctx context.Context) error { - s.lock.Lock() - defer s.lock.Unlock() - now, err := s.rdb.Time(ctx).Result() - if err != nil { - return fmt.Errorf("failed to read Redis time: %w", err) +// ownsStream reports whether the sink still owns the exact capability used by +// an in-flight read. +func (s *Sink) ownsStream(candidate *Stream) bool { + for _, state := range s.streams { + if state.stream == candidate { + return true + } } - if now.Sub(time.Unix(0, s.lastKeepAlive)) <= 2*s.ackGracePeriod { + return false +} + +// acknowledgeMessages atomically advances recovery for messages returned by a +// read whose stream was concurrently removed. +func acknowledgeMessages(ctx context.Context, stream *Stream, group string, messages []redis.XMessage) error { + if len(messages) == 0 { return nil } - s.logger.Debug("consumer stale, creating new one") - consumer, err := s.newConsumer(ctx) - if err != nil { - return fmt.Errorf("failed to create new consumer: %w", err) + ids := make([]string, len(messages)) + for i, message := range messages { + ids[i] = message.ID } - s.consumer = consumer - return nil + return (&recoveryAcker{stream: stream}).XAck(ctx, stream.key, group, ids...).Err() } -// newConsumer creates a new consumer, registers it with every sink stream, -// and records its keep-alive. Registration is failure-atomic: when any -// registration fails the consumer is detached from the streams registered so -// far so ownership state never diverges across streams. s.lock must be held. -func (s *Sink) newConsumer(ctx context.Context) (string, error) { - consumer := ulid.Make().String() - registered := make([]*sinkStream, 0, len(s.streams)) - rollback := func(cause error) error { - var errs []error - for _, state := range registered { - if err := detachSinkConsumer(ctx, state.stream, s.Name, consumer); err != nil { - errs = append(errs, err) - } - } - return errors.Join(append([]error{cause}, errs...)...) - } - for _, state := range s.streams { - if err := registerSinkConsumer(ctx, state.stream, s.Name, consumer); err != nil { - return "", rollback(err) - } - registered = append(registered, state) - } +// ensureConsumer ensures that the consumer is still alive. +func (s *Sink) ensureConsumer(ctx context.Context) error { + s.lock.Lock() + defer s.lock.Unlock() now, err := s.rdb.Time(ctx).Result() if err != nil { - return "", rollback(fmt.Errorf("failed to read Redis time for new consumer %s: %w", consumer, err)) + return fmt.Errorf("failed to read Redis time while checking consumer: %w", err) } - keepAlive := now.UnixNano() - if _, err := s.consumersKeepAliveMap.Set(ctx, consumer, strconv.FormatInt(keepAlive, 10)); err != nil { - return "", rollback(fmt.Errorf("failed to set sink keep-alive for new consumer %s: %w", consumer, err)) + if now.Sub(time.Unix(0, s.lastKeepAlive)) > 2*s.ackGracePeriod { + s.logger.Debug("consumer stale, creating new one") + consumer, err := s.newConsumer(ctx) + if err != nil { + s.logger.Error(fmt.Errorf("failed to create new consumer: %w", err)) + return err + } + s.consumer = consumer } - s.lastKeepAlive = keepAlive - return consumer, nil + return nil } -// periodicKeepAlive updates this consumer keep-alive every ack grace period -// using Redis time so replicas evaluating staleness agree on the clock. +// periodicKeepAlive updates this consumer keep-alive every half ack grace period. func (s *Sink) periodicKeepAlive() { defer s.wait.Done() defer s.logger.Debug("periodicKeepAlive: exiting") @@ -538,17 +669,34 @@ func (s *Sink) periodicKeepAlive() { s.lock.Lock() now, err := s.rdb.Time(s.ctx).Result() if err != nil { - s.logger.Error(fmt.Errorf("failed to read Redis time for keep-alive: %w", err)) + s.logger.Error(fmt.Errorf("failed to read Redis time for sink keep-alive: %w", err)) s.lock.Unlock() continue } keepAlive := now.UnixNano() - if _, err := s.consumersKeepAliveMap.Set(s.ctx, s.consumer, strconv.FormatInt(keepAlive, 10)); err != nil { - s.logger.Error(fmt.Errorf("failed to update sink keep-alive: %w", err)) + var updateErr error + for _, state := range s.streams { + err := setSinkKeepAlive(s.ctx, state, s.Name, s.consumer, keepAlive) + switch { + case err == nil: + case errors.Is(err, ErrStreamDestroyed), errors.Is(err, ErrDeadlineElapsed): + // The generation ended; the read loop owns detaching the + // stream. Refreshing nothing is correct here. + s.logger.Debug("keep-alive skipped", "stream", state.stream.Name, "reason", err) + default: + updateErr = errors.Join(updateErr, fmt.Errorf( + "stream %s: %w", + state.stream.Name, + err, + )) + } + } + if updateErr != nil { + s.logger.Error(fmt.Errorf("failed to update sink keep-alive: %w", updateErr)) s.lock.Unlock() continue } - s.lastKeepAlive = keepAlive + s.lastKeepAlive = now.UnixNano() s.lock.Unlock() case <-s.donechan: @@ -557,23 +705,54 @@ func (s *Sink) periodicKeepAlive() { } } -// periodicIdleMessageCheck claims idle messages and deletes stale consumers -// under the per-stream fenced lease. An idle message is one that has not been -// acked for more than the ack grace period. Lease renewal and each guarded -// mutation execute in one atomic script so a stale owner cannot mutate the -// PEL after another instance takes over. +// periodicIdleMessageCheck claims any idle message every check stale period. +// An idle message is one that has not been acked for more than the ack grace period. +// Once all idle messages are claimed, any stale consumer is deleted. func (s *Sink) periodicIdleMessageCheck() { defer s.wait.Done() defer s.logger.Debug("periodicIdleMessageCheck: exiting") ticker := time.NewTicker(checkIdlePeriod) defer ticker.Stop() + leaseDuration := 2 * checkIdlePeriod for { select { case <-ticker.C: s.lock.Lock() for _, state := range s.streams { - s.checkIdleMessages(s.ctx, state) + lease, acquired, err := acquireSinkRecoveryLease( + s.ctx, + state.stream, + state.leaseKey, + state.leaseOwner, + leaseDuration, + ) + if err != nil { + s.logger.Error(fmt.Errorf( + "failed to acquire stale-recovery lease for stream %s: %w", + state.stream.Name, + err, + )) + continue + } + if !acquired { + continue + } + if err := s.claimIdleMessages(s.ctx, state, lease, leaseDuration); err != nil { + s.logger.Error(fmt.Errorf( + "failed to claim idle messages for stream %s: %w", + state.stream.Name, + err, + )) + continue + } + if err := s.deleteStreamStaleConsumers(s.ctx, state, lease, leaseDuration); err != nil { + s.logger.Error(fmt.Errorf( + "failed to delete stale consumers for stream %s: %w", + state.stream.Name, + err, + )) + } } s.lock.Unlock() @@ -583,82 +762,160 @@ func (s *Sink) periodicIdleMessageCheck() { } } -// checkIdleMessages acquires the stream lease and, when held, claims idle -// messages for this consumer and deletes stale consumers. Lease loss and -// stream destruction abort silently: another instance owns the work or the -// stream is gone. s.lock must be held. -func (s *Sink) checkIdleMessages(ctx context.Context, state *sinkStream) { - // Note: the builtin max is shadowed by the package-level test helper - // variable of the same name, hence the explicit floor. - leaseMs := 2 * checkIdlePeriod.Milliseconds() - if leaseMs < 20 { - leaseMs = 20 - } - acquired, fence, err := acquireSinkLease(ctx, state.stream, s.Name, s.leaseOwner, leaseMs) +// claimIdleMessages claims idle messages from one stream through an atomic +// exact-lease fence around every XAUTOCLAIM. +// s.lock must be held. +func (s *Sink) claimIdleMessages( + ctx context.Context, + state *sinkStream, + lease sinkRecoveryLease, + leaseDuration time.Duration, +) error { + start, err := s.claim(ctx, state, lease, leaseDuration, "0-0") if err != nil { - if !errors.Is(err, ErrStreamDestroyed) && ctx.Err() == nil { - s.logger.Error(fmt.Errorf("failed to acquire idle message check lease: %w", err)) - } - return - } - if !acquired { - // Another sink instance owns the lease. - return + return err } - start := "0-0" - for { - msgs, next, err := fencedAutoClaim(ctx, state.stream, s.Name, s.leaseOwner, fence, leaseMs, s.consumer, s.ackGracePeriod.Milliseconds(), start, s.maxPolled) + for start != "0-0" { + start, err = s.claim(ctx, state, lease, leaseDuration, start) if err != nil { - if !isLeaseLostErr(err) && !isStreamDestroyedErr(err) && ctx.Err() == nil { - s.logger.Error(fmt.Errorf("failed to claim idle messages for stream %s: %w", state.stream.Name, err)) - } - return + return err } - if len(msgs) > 0 { - s.logger.Info("claimed", "stream", state.stream.Name, "messages", len(msgs)) - if err := streamEvents(ctx, state.stream.Name, state.stream.key, s.Name, msgs, s.acker, s.noAck, s.eventFilter, s.chans, s.donechan, s.logger); err != nil { - s.logger.Error(fmt.Errorf("failed to stream claimed events: %w", err)) - return - } + } + return nil +} + +// recoverConsumerGroups recreates groups removed outside Pulse for the streams +// the sink still owns. The caller holds s.lock, which serializes recovery with +// AddStream and RemoveStream so an in-flight stale read cannot resurrect a +// removed stream. BUSYGROUP means another sink instance already repaired the +// shared group and is therefore success. +func (s *Sink) recoverConsumerGroups(ctx context.Context) error { + for _, state := range s.streams { + stream := state.stream + deleted, err := ensureConsumerGroup(ctx, stream, s.Name, state.startID) + if err != nil { + return err } - if next == "0-0" { - break + if deleted { + s.logger.Error( + fmt.Errorf("stream data was deleted before consumer group recovery"), + "stream", stream.Name, + "group", s.Name, + "data_loss", true, + ) } - start = next + s.logger.Info("recovered consumer group", "stream", stream.Name, "start", state.startID) } - staleNs := (2 * s.ackGracePeriod).Nanoseconds() - removed, err := fencedCleanupStaleConsumers(ctx, state.stream, s.Name, s.leaseOwner, fence, leaseMs, staleNs, s.consumer) + return nil +} + +// Helper function to claim messages from a stream used by claimIdleMessages. +func (s *Sink) claim( + ctx context.Context, + state *sinkStream, + lease sinkRecoveryLease, + leaseDuration time.Duration, + start string, +) (string, error) { + next, messages, err := fencedAutoClaim( + ctx, + state.stream, + state.leaseKey, + lease, + leaseDuration, + s.Name, + s.consumer, + s.ackGracePeriod, + start, + s.maxPolled, + ) if err != nil { - if !isLeaseLostErr(err) && !isStreamDestroyedErr(err) && ctx.Err() == nil { - s.logger.Error(fmt.Errorf("failed to delete stale consumers for stream %s: %w", state.stream.Name, err)) - } - return + return start, err + } + if len(messages) > 0 { + s.logger.Info("claimed", "stream", state.stream.Name, "messages", len(messages)) + err = streamEvents( + ctx, + state.stream, + s.Name, + messages, + s.noAck, + s.eventFilter, + s.chans, + s.donechan, + s.rdb, + s.logger, + ) + } + return next, err +} + +// attachSinkStream validates the shared contract before joining any group +// membership, then builds the complete per-stream coordination state. +func attachSinkStream( + ctx context.Context, + stream *Stream, + name string, + config sinkConfiguration, + logger pulse.Logger, +) (*sinkStream, error) { + if err := ensureSinkConfiguration(ctx, stream, name, config); err != nil { + return nil, err } - if len(removed) > 0 { - s.logger.Info("deleted stale consumers", "stream", state.stream.Name, "consumers", removed) + consumers, err := rmap.Join( + ctx, + consumersMapName(stream), + stream.rdb, + rmap.WithLogger(logger), + ) + if err != nil { + return nil, fmt.Errorf("failed to join replicated map for sink %s: %w", name, err) } + keepAlives, err := rmap.Join( + ctx, + sinkKeepAliveMapName(stream, name), + stream.rdb, + rmap.WithLogger(logger), + ) + if err != nil { + consumers.Close() + return nil, fmt.Errorf("failed to join replicated map for sink keep-alives %s: %w", name, err) + } + if _, err := ensureConsumerGroup(ctx, stream, name, config.startID); err != nil { + closeSetupMembership(ctx, stream, consumers, err) + keepAlives.Close() + return nil, err + } + return &sinkStream{ + stream: stream, + startID: config.startID, + consumers: consumers, + keepAlives: keepAlives, + leaseKey: staleLockName(stream, name), + leaseOwner: ulid.Make().String(), + }, nil +} + +// close releases both local rmap replicas owned by one attachment. +func (s *sinkStream) close() { + s.consumers.Close() + s.keepAlives.Close() } // consumersMapName is the name of the replicated map that backs a sink. func consumersMapName(stream *Stream) string { - return fmt.Sprintf("stream:%s:sinks", stream.Name) + return fmt.Sprintf("stream:%s:generation:%s:sinks", stream.Name, stream.generation) } -func consumersMapOptions(stream *Stream, logger pulse.Logger) []rmap.MapOption { - opts := []rmap.MapOption{ - rmap.WithLogger(logger), - } - if stream.ttl > 0 { - if stream.ttlSliding { - opts = append(opts, rmap.WithSlidingTTL(stream.ttl)) - } else { - opts = append(opts, rmap.WithTTL(stream.ttl)) +// closeSetupMembership destroys an orphan generation-qualified map when the +// stream was destroyed during setup; otherwise it only releases this replica. +func closeSetupMembership(ctx context.Context, stream *Stream, membership *rmap.Map, setupErr error) { + if errors.Is(setupErr, ErrStreamDestroyed) { + membership.Close() + if err := stream.rdb.Del(ctx, consumersMapContentKey(stream)).Err(); err != nil { + stream.logger.Error(fmt.Errorf("failed to delete orphan sink membership: %w", err)) } + return } - return opts -} - -// sinkKeepAliveMapName is the name of the replicated map that backs a sink keep-alives. -func sinkKeepAliveMapName(sink string) string { - return fmt.Sprintf("sink:%s:keepalive", sink) + membership.Close() } diff --git a/streaming/sink_config.go b/streaming/sink_config.go new file mode 100644 index 0000000..3af0022 --- /dev/null +++ b/streaming/sink_config.go @@ -0,0 +1,666 @@ +// Package streaming persists the behavior and coordination state shared by +// same-name sink replicas. Every attached stream incarnation owns an +// independent configuration, keepalive map, and fenced stale-recovery lease. +package streaming + +import ( + "context" + "errors" + "fmt" + "strconv" + "strings" + "time" + + redis "github.com/redis/go-redis/v9" + + "goa.design/pulse/streaming/options" +) + +type ( + // sinkConfiguration is the delivery contract shared by every replica + // attached to one stream incarnation. + sinkConfiguration struct { + filterKind string + filterValue string + startID string + noAck bool + ackGrace time.Duration + } + + // sinkRecoveryLease is the Redis-issued fencing capability for one stale + // recovery pass. + sinkRecoveryLease struct { + owner string + fence int64 + } +) + +const ( + sinkLeaseUnavailable = int64(0) + sinkLeaseAcquired = int64(1) +) + +var ( + // ensureSinkConfigurationScript validates the exact stream incarnation and + // atomically establishes one immutable configuration for all replicas. + ensureSinkConfigurationScript = redis.NewScript(` +if redis.call("HGET", KEYS[1], "state") ~= ARGV[1] +or redis.call("HGET", KEYS[1], "generation") ~= ARGV[2] +or redis.call("HGET", KEYS[1], "physical_key") ~= ARGV[3] then + return redis.error_reply("STREAMDESTROYED") +end +local deadline = redis.call("HGET", KEYS[1], ARGV[9]) +if deadline then + local clock = redis.call("TIME") + local now = (tonumber(clock[1]) * 1000) + math.floor(tonumber(clock[2]) / 1000) + if now >= tonumber(deadline) then + return redis.error_reply("DEADLINEELAPSED") + end +end + +local existing = redis.call("HMGET", KEYS[2], + "filter_kind", "filter_value", "start_id", "no_ack", "ack_grace_ms") +if existing[1] then + if existing[1] ~= ARGV[4] + or existing[2] ~= ARGV[5] + or existing[3] ~= ARGV[6] + or existing[4] ~= ARGV[7] + or existing[5] ~= ARGV[8] then + return redis.error_reply("SINKCONFIGMISMATCH") + end +else + redis.call("HSET", KEYS[2], + "filter_kind", ARGV[4], + "filter_value", ARGV[5], + "start_id", ARGV[6], + "no_ack", ARGV[7], + "ack_grace_ms", ARGV[8]) +end +redis.call("SADD", KEYS[3], KEYS[2], KEYS[4], KEYS[5]) +if deadline then + redis.call("PEXPIREAT", KEYS[2], deadline) + redis.call("PEXPIREAT", KEYS[3], deadline) + redis.call("PEXPIREAT", KEYS[4], deadline) + redis.call("PEXPIREAT", KEYS[5], deadline) +end +return 1 +`) + + // acquireSinkRecoveryLeaseScript acquires or renews one owner-token lease + // using Redis TIME and returns the monotonically increasing fencing token. + acquireSinkRecoveryLeaseScript = redis.NewScript(` +if redis.call("HGET", KEYS[1], "state") ~= ARGV[1] +or redis.call("HGET", KEYS[1], "generation") ~= ARGV[2] +or redis.call("HGET", KEYS[1], "physical_key") ~= ARGV[3] then + return redis.error_reply("STREAMDESTROYED") +end +local clock = redis.call("TIME") +local now = (tonumber(clock[1]) * 1000) + math.floor(tonumber(clock[2]) / 1000) +local deadline = redis.call("HGET", KEYS[1], ARGV[6]) +if deadline and now >= tonumber(deadline) then + return redis.error_reply("DEADLINEELAPSED") +end +local owner = redis.call("HGET", KEYS[2], "owner") +local lease_until = tonumber(redis.call("HGET", KEYS[2], "lease_until") or "0") +local fence = tonumber(redis.call("HGET", KEYS[2], "fence") or "0") +if lease_until > now and owner ~= ARGV[4] then + return {0, fence} +end +if owner ~= ARGV[4] or lease_until <= now then + fence = fence + 1 +end +redis.call("HSET", KEYS[2], + "owner", ARGV[4], + "fence", tostring(fence), + "lease_until", tostring(now + tonumber(ARGV[5]))) +if deadline then + redis.call("PEXPIREAT", KEYS[2], deadline) +end +return {1, fence} +`) + + // renewSinkRecoveryLeaseScript supports explicit lease maintenance tests + // and administration. Recovery mutations do not rely on separate renewal. + renewSinkRecoveryLeaseScript = redis.NewScript(` +if redis.call("HGET", KEYS[1], "state") ~= ARGV[1] +or redis.call("HGET", KEYS[1], "generation") ~= ARGV[2] +or redis.call("HGET", KEYS[1], "physical_key") ~= ARGV[3] then + return redis.error_reply("STREAMDESTROYED") +end +local clock = redis.call("TIME") +local now = (tonumber(clock[1]) * 1000) + math.floor(tonumber(clock[2]) / 1000) +local deadline = redis.call("HGET", KEYS[1], ARGV[7]) +if deadline and now >= tonumber(deadline) then + return redis.error_reply("DEADLINEELAPSED") +end +if redis.call("HGET", KEYS[2], "owner") ~= ARGV[4] +or redis.call("HGET", KEYS[2], "fence") ~= ARGV[5] +or tonumber(redis.call("HGET", KEYS[2], "lease_until") or "0") <= now then + return redis.error_reply("SINKLEASELOST") +end +redis.call("HSET", KEYS[2], "lease_until", tostring(now + tonumber(ARGV[6]))) +if deadline then + redis.call("PEXPIREAT", KEYS[2], deadline) +end +return 1 +`) + + // fencedAutoClaimScript verifies the exact live fencing capability and + // performs XAUTOCLAIM in the same Redis operation. + fencedAutoClaimScript = redis.NewScript(` +if redis.call("HGET", KEYS[1], "state") ~= ARGV[1] +or redis.call("HGET", KEYS[1], "generation") ~= ARGV[2] +or redis.call("HGET", KEYS[1], "physical_key") ~= ARGV[3] then + return redis.error_reply("STREAMDESTROYED") +end +local clock = redis.call("TIME") +local now = (tonumber(clock[1]) * 1000) + math.floor(tonumber(clock[2]) / 1000) +local deadline = redis.call("HGET", KEYS[1], ARGV[7]) +if deadline and now >= tonumber(deadline) then + return redis.error_reply("DEADLINEELAPSED") +end +if redis.call("HGET", KEYS[2], "owner") ~= ARGV[4] +or redis.call("HGET", KEYS[2], "fence") ~= ARGV[5] +or tonumber(redis.call("HGET", KEYS[2], "lease_until") or "0") <= now then + return redis.error_reply("SINKLEASELOST") +end +redis.call("HSET", KEYS[2], "lease_until", tostring(now + tonumber(ARGV[6]))) +if deadline then + redis.call("PEXPIREAT", KEYS[2], deadline) +end +return redis.call( + "XAUTOCLAIM", + KEYS[3], + ARGV[8], + ARGV[9], + ARGV[10], + ARGV[11], + "COUNT", + ARGV[12] +) +`) + + // fencedStaleConsumerCleanupScript performs stale-consumer inspection and + // every resulting group and replicated-map mutation under one exact lease + // check. A predecessor whose lease expired cannot mutate the PEL or maps. + fencedStaleConsumerCleanupScript = redis.NewScript(` +if redis.call("HGET", KEYS[1], "state") ~= ARGV[1] +or redis.call("HGET", KEYS[1], "generation") ~= ARGV[2] +or redis.call("HGET", KEYS[1], "physical_key") ~= ARGV[3] then + return redis.error_reply("STREAMDESTROYED") +end +local clock = redis.call("TIME") +local now = (tonumber(clock[1]) * 1000) + math.floor(tonumber(clock[2]) / 1000) +local deadline = redis.call("HGET", KEYS[1], ARGV[7]) +if deadline and now >= tonumber(deadline) then + return redis.error_reply("DEADLINEELAPSED") +end +if redis.call("HGET", KEYS[2], "owner") ~= ARGV[4] +or redis.call("HGET", KEYS[2], "fence") ~= ARGV[5] +or tonumber(redis.call("HGET", KEYS[2], "lease_until") or "0") <= now then + return redis.error_reply("SINKLEASELOST") +end +redis.call("HSET", KEYS[2], "lease_until", tostring(now + tonumber(ARGV[6]))) +if deadline then + redis.call("PEXPIREAT", KEYS[2], deadline) +end + +local function publish_delete(content, channel, key) + if redis.call("HDEL", content, key) == 0 then + return + end + local rev = tostring(redis.call("HINCRBY", content, "=rev", 1)) + redis.call("HSET", content, "=kind", "del") + local message = struct.pack("ic0ic0", string.len(key), key, string.len(rev), rev) + redis.call("PUBLISH", channel, "del:" .. message) +end + +local function remove_value(content, channel, key, removed_value) + local encoded = redis.call("HGET", content, key) + if not encoded then + return + end + local values = {} + local decoded_ok, decoded = pcall(cjson.decode, encoded) + if decoded_ok and type(decoded) == "table" then + values = decoded + else + for value in string.gmatch(encoded, "[^,]+") do + table.insert(values, value) + end + end + local remaining = {} + local removed = false + for _, value in ipairs(values) do + if value == removed_value then + removed = true + else + table.insert(remaining, value) + end + end + if not removed then + return + end + if #remaining == 0 then + publish_delete(content, channel, key) + return + end + local replacement = cjson.encode(remaining) + redis.call("HSET", content, key, replacement) + local rev = tostring(redis.call("HINCRBY", content, "=rev", 1)) + redis.call("HSET", content, "=kind", "set") + local message = struct.pack( + "ic0ic0ic0", + string.len(key), key, + string.len(replacement), replacement, + string.len(rev), rev + ) + redis.call("PUBLISH", channel, "set:" .. message) +end + +local removed = {} +local malformed = {} +local consumers = redis.call("XINFO", "CONSUMERS", KEYS[3], ARGV[8]) +for _, consumer in ipairs(consumers) do + local name + local pending + for index = 1, #consumer, 2 do + if consumer[index] == "name" then + name = consumer[index + 1] + elseif consumer[index] == "pending" then + pending = tonumber(consumer[index + 1]) + end + end + if name then + local heartbeat = redis.call("HGET", KEYS[4], name) + local stale = heartbeat == false + if heartbeat then + local heartbeat_ns = tonumber(heartbeat) + if not heartbeat_ns then + table.insert(malformed, name) + else + stale = now - math.floor(heartbeat_ns / 1000000) > tonumber(ARGV[9]) + end + end + if stale then + if pending == 0 then + redis.call("XGROUP", "DELCONSUMER", KEYS[3], ARGV[8], name) + end + publish_delete(KEYS[4], KEYS[5], name) + remove_value(KEYS[6], KEYS[7], ARGV[8], name) + table.insert(removed, name) + end + end +end +return {removed, malformed} +`) +) + +// ensureSinkConfiguration establishes or verifies one attached stream's shared +// sink behavior and registers its generation-owned metadata for Destroy. +func ensureSinkConfiguration( + ctx context.Context, + stream *Stream, + name string, + config sinkConfiguration, +) error { + err := ensureSinkConfigurationScript.Run( + ctx, + stream.rdb, + []string{ + stream.lifecycleKey, + sinkConfigurationKey(stream, name), + streamResourceRegistryKey(stream), + rmapContentKey(sinkKeepAliveMapName(stream, name)), + staleLockName(stream, name), + }, + streamStateActive, + stream.generation, + stream.key, + config.filterKind, + config.filterValue, + config.startID, + boolString(config.noAck), + strconv.FormatInt(config.ackGrace.Milliseconds(), 10), + streamDeadlineKey, + ).Err() + if err == nil { + return nil + } + lifecycleErr := stream.lifecycleError(err) + if errors.Is(lifecycleErr, ErrStreamDestroyed) || + errors.Is(lifecycleErr, ErrDeadlineElapsed) { + return lifecycleErr + } + if strings.Contains(err.Error(), "SINKCONFIGMISMATCH") { + return fmt.Errorf( + "sink %q configuration differs from existing replicas for stream %q generation %s", + name, + stream.Name, + stream.generation, + ) + } + return fmt.Errorf("establish sink %q configuration: %w", name, err) +} + +// sinkConfigurationForOptions returns the canonical persisted contract for a +// sink attachment. +func sinkConfigurationForOptions(o options.SinkOptions, startID string) sinkConfiguration { + filterKind, filterValue := sinkFilterConfiguration(o) + return sinkConfiguration{ + filterKind: filterKind, + filterValue: filterValue, + startID: startID, + noAck: o.NoAck, + ackGrace: o.AckGracePeriod, + } +} + +// sinkFilterConfiguration returns the canonical persisted filter contract. +func sinkFilterConfiguration(o options.SinkOptions) (string, string) { + switch { + case o.Topic != "": + return "topic", o.Topic + case o.TopicPattern != "": + return "pattern", o.TopicPattern + default: + return "all", "" + } +} + +// sinkNamespace is the logical identity shared only by replicas of one sink +// on one stream incarnation. +func sinkNamespace(stream *Stream, sink string) string { + return fmt.Sprintf("stream:%s:generation:%s:sink:%s", stream.Name, stream.generation, sink) +} + +// sinkConfigurationKey stores immutable same-name replica behavior. +func sinkConfigurationKey(stream *Stream, sink string) string { + return "pulse:" + sinkNamespace(stream, sink) + ":config" +} + +// sinkKeepAliveMapName identifies the generation-scoped consumer heartbeat map. +func sinkKeepAliveMapName(stream *Stream, sink string) string { + return sinkNamespace(stream, sink) + ":keepalive" +} + +// staleLockName identifies the generation-scoped stale-recovery lease. +func staleLockName(stream *Stream, sink string) string { + return "pulse:" + sinkNamespace(stream, sink) + ":stalelease" +} + +// acquireSinkRecoveryLease obtains the current fencing token for one attached +// stream. A false result means another replica still owns recovery. +func acquireSinkRecoveryLease( + ctx context.Context, + stream *Stream, + key, owner string, + duration time.Duration, +) (sinkRecoveryLease, bool, error) { + raw, err := acquireSinkRecoveryLeaseScript.Run( + ctx, + stream.rdb, + []string{stream.lifecycleKey, key}, + streamStateActive, + stream.generation, + stream.key, + owner, + strconv.FormatInt(duration.Milliseconds(), 10), + streamDeadlineKey, + ).Slice() + if err != nil { + lifecycleErr := stream.lifecycleError(err) + if errors.Is(lifecycleErr, ErrStreamDestroyed) || + errors.Is(lifecycleErr, ErrDeadlineElapsed) { + return sinkRecoveryLease{}, false, lifecycleErr + } + return sinkRecoveryLease{}, false, fmt.Errorf("acquire sink recovery lease: %w", err) + } + if len(raw) != 2 { + return sinkRecoveryLease{}, false, fmt.Errorf("invalid sink recovery lease result length %d", len(raw)) + } + status, ok := raw[0].(int64) + if !ok { + return sinkRecoveryLease{}, false, fmt.Errorf("invalid sink recovery lease status %T", raw[0]) + } + fence, ok := raw[1].(int64) + if !ok { + return sinkRecoveryLease{}, false, fmt.Errorf("invalid sink recovery lease fence %T", raw[1]) + } + switch status { + case sinkLeaseUnavailable: + return sinkRecoveryLease{}, false, nil + case sinkLeaseAcquired: + return sinkRecoveryLease{owner: owner, fence: fence}, true, nil + default: + return sinkRecoveryLease{}, false, fmt.Errorf("invalid sink recovery lease status %d", status) + } +} + +// renewSinkRecoveryLease extends one exact current capability without +// performing a recovery mutation. +func renewSinkRecoveryLease( + ctx context.Context, + stream *Stream, + key string, + lease sinkRecoveryLease, + duration time.Duration, +) error { + err := renewSinkRecoveryLeaseScript.Run( + ctx, + stream.rdb, + []string{stream.lifecycleKey, key}, + streamStateActive, + stream.generation, + stream.key, + lease.owner, + strconv.FormatInt(lease.fence, 10), + strconv.FormatInt(duration.Milliseconds(), 10), + streamDeadlineKey, + ).Err() + lifecycleErr := stream.lifecycleError(err) + if errors.Is(lifecycleErr, ErrStreamDestroyed) || + errors.Is(lifecycleErr, ErrDeadlineElapsed) { + return lifecycleErr + } + if err != nil { + return fmt.Errorf("renew sink recovery lease: %w", err) + } + return nil +} + +// fencedAutoClaim claims one idle batch only while lease is the exact +// unexpired Redis-owned fencing capability. +func fencedAutoClaim( + ctx context.Context, + stream *Stream, + key string, + lease sinkRecoveryLease, + duration time.Duration, + group, consumer string, + minIdle time.Duration, + start string, + count int64, +) (string, []redis.XMessage, error) { + raw, err := fencedAutoClaimScript.Run( + ctx, + stream.rdb, + []string{stream.lifecycleKey, key, stream.key}, + streamStateActive, + stream.generation, + stream.key, + lease.owner, + strconv.FormatInt(lease.fence, 10), + strconv.FormatInt(duration.Milliseconds(), 10), + streamDeadlineKey, + group, + consumer, + strconv.FormatInt(minIdle.Milliseconds(), 10), + start, + strconv.FormatInt(count, 10), + ).Slice() + lifecycleErr := stream.lifecycleError(err) + if errors.Is(lifecycleErr, ErrStreamDestroyed) || + errors.Is(lifecycleErr, ErrDeadlineElapsed) { + return start, nil, lifecycleErr + } + if err != nil { + return start, nil, fmt.Errorf("fenced sink recovery claim: %w", err) + } + if len(raw) < 2 || len(raw) > 3 { + return start, nil, fmt.Errorf("fenced sink recovery claim returned %d values", len(raw)) + } + next, ok := raw[0].(string) + if !ok { + return start, nil, fmt.Errorf("fenced sink recovery claim returned invalid cursor %T", raw[0]) + } + entries, ok := raw[1].([]any) + if !ok { + return start, nil, fmt.Errorf("fenced sink recovery claim returned invalid messages %T", raw[1]) + } + messages, err := decodeSnapshotRange(entries) + if err != nil { + return start, nil, fmt.Errorf("decode fenced sink recovery claim: %w", err) + } + return next, messages, nil +} + +// cleanupStaleConsumers removes stale group and replicated-map state only +// while lease remains the exact unexpired fencing capability. +func cleanupStaleConsumers( + ctx context.Context, + stream *Stream, + key string, + lease sinkRecoveryLease, + duration time.Duration, + group string, + grace time.Duration, +) ([]string, []string, error) { + raw, err := fencedStaleConsumerCleanupScript.Run( + ctx, + stream.rdb, + []string{ + stream.lifecycleKey, + key, + stream.key, + rmapContentKey(sinkKeepAliveMapName(stream, group)), + rmapChannelKey(sinkKeepAliveMapName(stream, group)), + consumersMapContentKey(stream), + consumersMapChannelKey(stream), + }, + streamStateActive, + stream.generation, + stream.key, + lease.owner, + strconv.FormatInt(lease.fence, 10), + strconv.FormatInt(duration.Milliseconds(), 10), + streamDeadlineKey, + group, + strconv.FormatInt((2*grace).Milliseconds(), 10), + ).Slice() + lifecycleErr := stream.lifecycleError(err) + if errors.Is(lifecycleErr, ErrStreamDestroyed) || + errors.Is(lifecycleErr, ErrDeadlineElapsed) { + return nil, nil, lifecycleErr + } + if err != nil { + return nil, nil, fmt.Errorf("fenced stale-consumer cleanup: %w", err) + } + if len(raw) != 2 { + return nil, nil, fmt.Errorf("fenced stale-consumer cleanup returned %d values", len(raw)) + } + removed, err := stringSlice(raw[0]) + if err != nil { + return nil, nil, fmt.Errorf("decode removed stale consumers: %w", err) + } + malformed, err := stringSlice(raw[1]) + if err != nil { + return nil, nil, fmt.Errorf("decode malformed stale consumers: %w", err) + } + return removed, malformed, nil +} + +// stringSlice validates one Redis Lua array of strings. +func stringSlice(value any) ([]string, error) { + raw, ok := value.([]any) + if !ok { + return nil, fmt.Errorf("expected array, got %T", value) + } + result := make([]string, len(raw)) + for i, item := range raw { + result[i], ok = item.(string) + if !ok { + return nil, fmt.Errorf("item %d has type %T", i, item) + } + } + return result, nil +} + +// streamResourceRegistryKey owns generation-scoped metadata deleted by +// explicit Stream.Destroy. +func streamResourceRegistryKey(stream *Stream) string { + return fmt.Sprintf( + "pulse:stream:%s:generation:%s:resources", + stream.Name, + stream.generation, + ) +} + +// validateSinkOptions rejects values that would cause Redis retry churn, +// invalid channel allocation, or immediate stale-consumer recovery. +func validateSinkOptions(o options.SinkOptions) error { + switch { + case o.Topic != "" && o.TopicPattern != "": + return fmt.Errorf("sink topic and topic pattern are mutually exclusive") + case o.HasConflictingStartOptions(): + return fmt.Errorf("sink cursor-start options are mutually exclusive") + case o.BlockDuration < time.Millisecond: + return fmt.Errorf("sink block duration must be at least 1ms") + case o.MaxPolled <= 0: + return fmt.Errorf("sink maximum polled events must be greater than zero") + case o.BufferSize < 0: + return fmt.Errorf("sink buffer size must be greater than or equal to zero") + case o.AckGracePeriod < time.Millisecond: + return fmt.Errorf("sink acknowledgement grace period must be at least 1ms") + default: + return nil + } +} + +// validateReaderOptions rejects ambiguous selection and cursor contracts plus +// values that would cause Redis retry churn or invalid channel allocation. +func validateReaderOptions(o options.ReaderOptions) error { + switch { + case o.Topic != "" && o.TopicPattern != "": + return fmt.Errorf("reader topic and topic pattern are mutually exclusive") + case o.HasConflictingStartOptions(): + return fmt.Errorf("reader cursor-start options are mutually exclusive") + case o.BlockDuration < time.Millisecond: + return fmt.Errorf("reader block duration must be at least 1ms") + case o.MaxPolled <= 0: + return fmt.Errorf("reader maximum polled events must be greater than zero") + case o.BufferSize < 0: + return fmt.Errorf("reader buffer size must be greater than or equal to zero") + default: + return nil + } +} + +// validateAddStreamOptions rejects ambiguous per-stream cursor contracts. +func validateAddStreamOptions(o options.AddStreamOptions) error { + if o.HasConflictingStartOptions() { + return fmt.Errorf("added stream cursor-start options are mutually exclusive") + } + return nil +} + +// rmapContentKey returns the Redis hash used by the named replicated map. +func rmapContentKey(name string) string { + return fmt.Sprintf("map:%s:content", name) +} + +// rmapChannelKey is the Redis channel carrying rmap update notifications for +// the named replicated map. +func rmapChannelKey(name string) string { + return fmt.Sprintf("map:%s:updates", name) +} diff --git a/streaming/sink_consumer.go b/streaming/sink_consumer.go new file mode 100644 index 0000000..4634339 --- /dev/null +++ b/streaming/sink_consumer.go @@ -0,0 +1,315 @@ +// Sink consumer attachment and detachment are single Redis operations fenced +// on the exact stream generation, so a destroyed generation's membership and +// keep-alive projections can never be recreated after Stream.Destroy deleted +// them. Per-consumer pending state decides whether Redis metadata can be +// deleted, while membership and keep-alive projections always move coherently. +package streaming + +import ( + "context" + "fmt" + "strconv" + + redis "github.com/redis/go-redis/v9" +) + +// registerSinkConsumerScript atomically verifies the exact stream generation, +// creates the Redis consumer, appends it to the sink membership map, and +// writes its initial keep-alive, so no consumer state exists on a destroyed +// generation and membership can never diverge from the consumer group. +// +// KEYS: [1]=lifecycle [2]=stream [3]=membership content [4]=membership channel +// +// [5]=keepalive content [6]=keepalive channel +// +// ARGV: [1]=active state [2]=generation [3]=group [4]=consumer [5]=keep-alive +var registerSinkConsumerScript = redis.NewScript(` +local lifecycle_type = redis.call("TYPE", KEYS[1]).ok +if lifecycle_type ~= "hash" then + return redis.error_reply("SINKLIFECYCLEINVALID " .. lifecycle_type) +end +if redis.call("HGET", KEYS[1], "state") ~= ARGV[1] +or redis.call("HGET", KEYS[1], "generation") ~= ARGV[2] +or redis.call("HGET", KEYS[1], "physical_key") ~= KEYS[2] then + return redis.error_reply("STREAMDESTROYED") +end +local membership_type = redis.call("TYPE", KEYS[3]).ok +if membership_type ~= "hash" and membership_type ~= "none" then + return redis.error_reply("SINKMEMBERSHIPINVALID " .. membership_type) +end +local keepalive_type = redis.call("TYPE", KEYS[5]).ok +if keepalive_type ~= "hash" and keepalive_type ~= "none" then + return redis.error_reply("SINKKEEPALIVEINVALID " .. keepalive_type) +end +local created = redis.pcall("XGROUP", "CREATECONSUMER", KEYS[2], ARGV[3], ARGV[4]) +if type(created) == "table" and created.err then + return redis.error_reply(created.err) +end + +local function publish_set(content, channel, field, value) + local rev = tostring(redis.call("HINCRBY", content, "=rev", 1)) + redis.call("HSET", content, "=kind", "set") + local message = struct.pack( + "ic0ic0ic0", + string.len(field), field, + string.len(value), value, + string.len(rev), rev + ) + redis.call("PUBLISH", channel, "set:" .. message) +end + +local encoded = redis.call("HGET", KEYS[3], ARGV[3]) +local consumers = {} +if encoded then + local ok, decoded = pcall(cjson.decode, encoded) + if not ok or type(decoded) ~= "table" then + return redis.error_reply("SINKMEMBERSHIPINVALID") + end + consumers = decoded +end +local member = false +for _, consumer in ipairs(consumers) do + if consumer == ARGV[4] then + member = true + break + end +end +if not member then + table.insert(consumers, ARGV[4]) + local value = cjson.encode(consumers) + redis.call("HSET", KEYS[3], ARGV[3], value) + publish_set(KEYS[3], KEYS[4], ARGV[3], value) +end + +redis.call("HSET", KEYS[5], ARGV[4], ARGV[5]) +publish_set(KEYS[5], KEYS[6], ARGV[4], ARGV[5]) +return 1 +`) + +// fencedKeepAliveSetScript refreshes one consumer keep-alive only while the +// exact stream generation is active, so the periodic keep-alive loop cannot +// recreate the keep-alive map after Stream.Destroy deleted it. +// +// KEYS: [1]=lifecycle [2]=stream [3]=keepalive content [4]=keepalive channel +// ARGV: [1]=active state [2]=generation [3]=consumer [4]=keep-alive +var fencedKeepAliveSetScript = redis.NewScript(` +if redis.call("HGET", KEYS[1], "state") ~= ARGV[1] +or redis.call("HGET", KEYS[1], "generation") ~= ARGV[2] +or redis.call("HGET", KEYS[1], "physical_key") ~= KEYS[2] then + return redis.error_reply("STREAMDESTROYED") +end +local keepalive_type = redis.call("TYPE", KEYS[3]).ok +if keepalive_type ~= "hash" and keepalive_type ~= "none" then + return redis.error_reply("SINKKEEPALIVEINVALID " .. keepalive_type) +end +redis.call("HSET", KEYS[3], ARGV[3], ARGV[4]) +local rev = tostring(redis.call("HINCRBY", KEYS[3], "=rev", 1)) +redis.call("HSET", KEYS[3], "=kind", "set") +local message = struct.pack( + "ic0ic0ic0", + string.len(ARGV[3]), ARGV[3], + string.len(ARGV[4]), ARGV[4], + string.len(rev), rev +) +redis.call("PUBLISH", KEYS[4], "set:" .. message) +return 1 +`) + +var detachSinkConsumerScript = redis.NewScript(` +local lifecycle_type = redis.call("TYPE", KEYS[1]).ok +if lifecycle_type ~= "hash" then + return redis.error_reply("SINKLIFECYCLEINVALID " .. lifecycle_type) +end +if redis.call("HGET", KEYS[1], "state") ~= ARGV[1] +or redis.call("HGET", KEYS[1], "generation") ~= ARGV[2] +or redis.call("HGET", KEYS[1], "physical_key") ~= KEYS[2] then + return redis.error_reply("STREAMDESTROYED") +end +local stream_type = redis.call("TYPE", KEYS[2]).ok +if stream_type ~= "stream" and stream_type ~= "none" then + return redis.error_reply("SINKSTREAMINVALID " .. stream_type) +end +local membership_type = redis.call("TYPE", KEYS[3]).ok +if membership_type ~= "hash" and membership_type ~= "none" then + return redis.error_reply("SINKMEMBERSHIPINVALID " .. membership_type) +end +local keepalive_type = redis.call("TYPE", KEYS[5]).ok +if keepalive_type ~= "hash" and keepalive_type ~= "none" then + return redis.error_reply("SINKKEEPALIVEINVALID " .. keepalive_type) +end + +local pending = {} +local pending_result = redis.pcall( + "XPENDING", KEYS[2], ARGV[3], "-", "+", 1, ARGV[4] +) +if type(pending_result) == "table" and pending_result.err then + if not string.find(pending_result.err, "NOGROUP", 1, true) then + return redis.error_reply(pending_result.err) + end +elseif type(pending_result) == "table" then + pending = pending_result +end +if #pending == 0 then + local groups = redis.pcall("XGROUP", "DELCONSUMER", KEYS[2], ARGV[3], ARGV[4]) + if type(groups) == "table" and groups.err + and not string.find(groups.err, "NOGROUP", 1, true) then + return redis.error_reply(groups.err) + end +end + +local function publish_delete(content, channel, field) + local rev = tostring(redis.call("HINCRBY", content, "=rev", 1)) + redis.call("HSET", content, "=kind", "del") + local message = struct.pack( + "ic0ic0", + string.len(field), field, + string.len(rev), rev + ) + redis.call("PUBLISH", channel, "del:" .. message) +end + +local function publish_set(content, channel, field, value) + local rev = tostring(redis.call("HINCRBY", content, "=rev", 1)) + redis.call("HSET", content, "=kind", "set") + local message = struct.pack( + "ic0ic0ic0", + string.len(field), field, + string.len(value), value, + string.len(rev), rev + ) + redis.call("PUBLISH", channel, "set:" .. message) +end + +local encoded = redis.call("HGET", KEYS[3], ARGV[3]) +if encoded then + local ok, consumers = pcall(cjson.decode, encoded) + if not ok or type(consumers) ~= "table" then + return redis.error_reply("SINKMEMBERSHIPINVALID") + end + local remaining = {} + for _, consumer in ipairs(consumers) do + if consumer ~= ARGV[4] then + table.insert(remaining, consumer) + end + end + if #remaining == 0 then + redis.call("HDEL", KEYS[3], ARGV[3]) + publish_delete(KEYS[3], KEYS[4], ARGV[3]) + elseif #remaining ~= #consumers then + local value = cjson.encode(remaining) + redis.call("HSET", KEYS[3], ARGV[3], value) + publish_set(KEYS[3], KEYS[4], ARGV[3], value) + end +end + +if redis.call("HDEL", KEYS[5], ARGV[4]) == 1 then + publish_delete(KEYS[5], KEYS[6], ARGV[4]) +end +return #pending +`) + +// detachSinkConsumer removes one consumer's distributed metadata. Redis +// deletes the consumer itself only when its own PEL is empty; pending entries +// remain claimable by stale-consumer recovery. +func detachSinkConsumer( + ctx context.Context, + state *sinkStream, + sinkName, consumer string, +) (bool, error) { + pending, err := detachSinkConsumerScript.Run( + ctx, + state.stream.rdb, + []string{ + state.stream.lifecycleKey, + state.stream.key, + consumersMapContentKey(state.stream), + consumersMapChannelKey(state.stream), + rmapContentKey(sinkKeepAliveMapName(state.stream, sinkName)), + rmapChannelKey(sinkKeepAliveMapName(state.stream, sinkName)), + }, + streamStateActive, + state.stream.generation, + sinkName, + consumer, + ).Int64() + if err != nil { + return false, streamLifecycleBoundaryError(err) + } + if pending < 0 || pending > 1 { + return false, fmt.Errorf("detach sink consumer returned invalid pending count %d", pending) + } + return pending == 0, nil +} + +// registerSinkConsumer establishes one consumer's complete distributed +// metadata in a single generation-fenced operation. On error nothing was +// written, so callers need no Redis rollback for this consumer. +func registerSinkConsumer( + ctx context.Context, + state *sinkStream, + sinkName, consumer string, + keepAlive int64, +) error { + err := registerSinkConsumerScript.Run( + ctx, + state.stream.rdb, + []string{ + state.stream.lifecycleKey, + state.stream.key, + consumersMapContentKey(state.stream), + consumersMapChannelKey(state.stream), + rmapContentKey(sinkKeepAliveMapName(state.stream, sinkName)), + rmapChannelKey(sinkKeepAliveMapName(state.stream, sinkName)), + }, + streamStateActive, + state.stream.generation, + sinkName, + consumer, + strconv.FormatInt(keepAlive, 10), + ).Err() + if err != nil { + return streamLifecycleBoundaryError(fmt.Errorf( + "cannot register consumer %s for sink %s on stream %s: %w", + consumer, + sinkName, + state.stream.Name, + err, + )) + } + return nil +} + +// setSinkKeepAlive refreshes one consumer keep-alive behind the generation +// fence. A destroyed generation returns ErrStreamDestroyed without recreating +// any keep-alive state. +func setSinkKeepAlive( + ctx context.Context, + state *sinkStream, + sinkName, consumer string, + keepAlive int64, +) error { + err := fencedKeepAliveSetScript.Run( + ctx, + state.stream.rdb, + []string{ + state.stream.lifecycleKey, + state.stream.key, + rmapContentKey(sinkKeepAliveMapName(state.stream, sinkName)), + rmapChannelKey(sinkKeepAliveMapName(state.stream, sinkName)), + }, + streamStateActive, + state.stream.generation, + consumer, + strconv.FormatInt(keepAlive, 10), + ).Err() + if err != nil { + return streamLifecycleBoundaryError(fmt.Errorf( + "cannot refresh keep-alive for consumer %s of sink %s on stream %s: %w", + consumer, + sinkName, + state.stream.Name, + err, + )) + } + return nil +} diff --git a/streaming/sink_lease.go b/streaming/sink_lease.go deleted file mode 100644 index 475a9e4..0000000 --- a/streaming/sink_lease.go +++ /dev/null @@ -1,263 +0,0 @@ -// This file implements the fenced sink lease used for idle-message claiming -// and stale-consumer cleanup. -// -// Each sink (consumer group) holds at most one lease per stream, stored in a -// hash at "pulse:streammeta::sink::lease" with fields: -// -// - owner: the sink instance currently allowed to claim and clean up. -// - fence: a counter incremented on every ownership change. -// - expiry: Redis-time (milliseconds) after which the lease is up for grabs. -// -// All timing uses the Redis TIME command evaluated inside the scripts so -// client clock skew cannot grant two instances the lease at once. Lease -// verification, renewal, and the guarded mutation (XAUTOCLAIM or consumer -// deletion) execute in a single script: once another instance takes over the -// lease the fence counter changes and every in-flight script from the stale -// owner fails before touching the PEL. -package streaming - -import ( - "context" - "fmt" - "strings" - - redis "github.com/redis/go-redis/v9" -) - -// leaseLostErrorPrefix is the Redis error prefix returned by fenced scripts -// when the caller no longer holds the lease. -const leaseLostErrorPrefix = "LEASELOST" - -// leaseCheckLua verifies and renews the caller's lease. Scripts using it must -// bind KEYS[2] to the lease key, ARGV[1] to the owner, ARGV[2] to the fence -// value returned by acquisition, and ARGV[3] to the lease duration in -// milliseconds. It defines now_ms for subsequent statements. -const leaseCheckLua = ` -local t = redis.call("TIME") -local now_ms = t[1] * 1000 + math.floor(t[2] / 1000) -local owner = redis.call("HGET", KEYS[2], "owner") -local fence = redis.call("HGET", KEYS[2], "fence") -local expiry = tonumber(redis.call("HGET", KEYS[2], "expiry") or "0") -if owner ~= ARGV[1] or fence ~= ARGV[2] or expiry < now_ms then - return redis.error_reply("LEASELOST sink lease lost") -end -redis.call("HSET", KEYS[2], "expiry", now_ms + tonumber(ARGV[3])) -redis.call("PEXPIRE", KEYS[2], tonumber(ARGV[3]) * 2) -` - -// acquireLeaseScript acquires or renews the sink lease using Redis time. A -// new owner (or a takeover of an expired lease) increments the fence counter -// so scripts still carrying the previous fence value can no longer mutate -// anything. Acquisition is fenced by the stream lifecycle: destroyed streams -// never get a new lease key. -// -// KEYS: [1]=lifecycle [2]=lease -// ARGV: [1]=owner [2]=leaseMs -// Returns {acquired(0|1), fence}. -var acquireLeaseScript = redis.NewScript(` -local state = redis.call("HGET", KEYS[1], "state") -if state ~= "active" then - return redis.error_reply("STREAMDESTROYED stream was destroyed") -end -local t = redis.call("TIME") -local now_ms = t[1] * 1000 + math.floor(t[2] / 1000) -local owner = redis.call("HGET", KEYS[2], "owner") -local expiry = tonumber(redis.call("HGET", KEYS[2], "expiry") or "0") -if owner and owner ~= ARGV[1] and expiry >= now_ms then - return {0, 0} -end -local bump = 1 -if owner == ARGV[1] and expiry >= now_ms then - bump = 0 -end -local fence = redis.call("HINCRBY", KEYS[2], "fence", bump) -redis.call("HSET", KEYS[2], "owner", ARGV[1], "expiry", now_ms + tonumber(ARGV[2])) -redis.call("PEXPIRE", KEYS[2], tonumber(ARGV[2]) * 2) -return {1, fence} -`) - -// fencedAutoClaimScript renews the lease and claims idle messages in one -// atomic operation so a stale owner cannot move PEL entries after takeover. -// -// KEYS: [1]=lifecycle [2]=lease [3]=stream -// ARGV: [1]=owner [2]=fence [3]=leaseMs [4]=group [5]=consumer [6]=minIdleMs -// [7]=start [8]=count -// Returns the XAUTOCLAIM reply, or {"0-0", {}} when the group is gone. -var fencedAutoClaimScript = redis.NewScript(` -local state = redis.call("HGET", KEYS[1], "state") -if state ~= "active" then - return redis.error_reply("STREAMDESTROYED stream was destroyed") -end -` + leaseCheckLua + ` -local res = redis.pcall("XAUTOCLAIM", KEYS[3], ARGV[4], ARGV[5], ARGV[6], ARGV[7], "COUNT", ARGV[8]) -if type(res) == "table" and res["err"] then - if string.find(res["err"], "NOGROUP", 1, true) then - return {"0-0", {}} - end - return res -end -return res -`) - -// fencedCleanupScript renews the lease and deletes stale consumers in one -// atomic operation. A consumer is stale when it has no pending events and its -// keep-alive is missing or older than the staleness threshold; its membership -// map entry and keep-alive record are removed with rmap protocol -// notifications. The caller's live consumer is never deleted. -// -// KEYS: [1]=lifecycle [2]=lease [3]=stream [4]=membership content -// [5]=membership channel [6]=keep-alive content [7]=keep-alive channel -// ARGV: [1]=owner [2]=fence [3]=leaseMs [4]=group [5]=staleNs [6]=live consumer -// Returns the deleted consumer names. -var fencedCleanupScript = redis.NewScript(` -local state = redis.call("HGET", KEYS[1], "state") -if state ~= "active" then - return redis.error_reply("STREAMDESTROYED stream was destroyed") -end -` + leaseCheckLua + membershipRemoveLua + ` -local consumers = redis.pcall("XINFO", "CONSUMERS", KEYS[3], ARGV[4]) -if type(consumers) == "table" and consumers["err"] then - if string.find(consumers["err"], "NOGROUP", 1, true) then - return {} - end - return consumers -end -local now_ns = (t[1] * 1000000 + t[2]) * 1000 -local removed = {} -for _, info in ipairs(consumers) do - local name, pending - for i = 1, #info, 2 do - if info[i] == "name" then name = info[i+1] end - if info[i] == "pending" then pending = info[i+1] end - end - if name ~= ARGV[6] and pending == 0 then - local ka = redis.call("HGET", KEYS[6], name) - if not ka or now_ns - tonumber(ka) > tonumber(ARGV[5]) then - redis.call("XGROUP", "DELCONSUMER", KEYS[3], ARGV[4], name) - if ka then - redis.call("HDEL", KEYS[6], name) - local rev = tostring(redis.call("HINCRBY", KEYS[6], "=rev", 1)) - redis.call("HSET", KEYS[6], "=kind", "del") - local msg = struct.pack("ic0ic0", string.len(name), name, string.len(rev), rev) - redis.call("PUBLISH", KEYS[7], "del:" .. msg) - end - membership_remove(KEYS[4], KEYS[5], ARGV[4], name) - table.insert(removed, name) - end - end -end -return removed -`) - -// acquireSinkLease attempts to acquire or renew the sink lease for the stream -// using Redis time. It returns whether the lease is held and the fence value -// to pass to subsequent fenced operations. -func acquireSinkLease(ctx context.Context, stream *Stream, sinkName, owner string, leaseMs int64) (bool, int64, error) { - keys := []string{lifecycleKey(stream.key), leaseKey(stream.key, sinkName)} - res, err := acquireLeaseScript.Run(ctx, stream.rdb, keys, owner, leaseMs).Slice() - if err != nil { - if isStreamDestroyedErr(err) { - return false, 0, fmt.Errorf("cannot acquire lease for stream %s: %w", stream.Name, ErrStreamDestroyed) - } - return false, 0, fmt.Errorf("failed to acquire lease for stream %s: %w", stream.Name, err) - } - return res[0].(int64) == 1, res[1].(int64), nil -} - -// fencedAutoClaim renews the lease and claims up to count messages idle for -// at least minIdleMs, starting at start, assigning them to consumer. It -// returns the claimed messages and the next XAUTOCLAIM start cursor ("0-0" -// when the scan is complete). -func fencedAutoClaim(ctx context.Context, stream *Stream, sinkName, owner string, fence, leaseMs int64, consumer string, minIdleMs int64, start string, count int64) ([]redis.XMessage, string, error) { - keys := []string{lifecycleKey(stream.key), leaseKey(stream.key, sinkName), stream.key} - res, err := fencedAutoClaimScript.Run(ctx, stream.rdb, keys, - owner, fence, leaseMs, sinkName, consumer, minIdleMs, start, count).Slice() - if err != nil { - return nil, "", err - } - next := res[0].(string) - entries, ok := res[1].([]any) - if !ok { - return nil, "", fmt.Errorf("unexpected XAUTOCLAIM entries type %T", res[1]) - } - msgs, err := decodeClaimedMessages(entries) - if err != nil { - return nil, "", err - } - return msgs, next, nil -} - -// fencedCleanupStaleConsumers renews the lease and deletes consumers of the -// sink group that have no pending events and whose keep-alive is missing or -// older than staleNs nanoseconds of Redis time. liveConsumer is the caller's -// current consumer and is never deleted. It returns the deleted consumer -// names. -func fencedCleanupStaleConsumers(ctx context.Context, stream *Stream, sinkName, owner string, fence, leaseMs int64, staleNs int64, liveConsumer string) ([]string, error) { - keys := []string{ - lifecycleKey(stream.key), - leaseKey(stream.key, sinkName), - stream.key, - membershipContentKey(stream.Name), - membershipChannelKey(stream.Name), - keepAliveContentKey(sinkName), - keepAliveChannelKey(sinkName), - } - res, err := fencedCleanupScript.Run(ctx, stream.rdb, keys, - owner, fence, leaseMs, sinkName, staleNs, liveConsumer).StringSlice() - if err != nil { - return nil, err - } - return res, nil -} - -// decodeClaimedMessages converts the raw Lua XAUTOCLAIM entries reply into -// XMessage values. Nil entries (tombstones of trimmed events reported by -// older Redis servers) are skipped. -func decodeClaimedMessages(entries []any) ([]redis.XMessage, error) { - msgs := make([]redis.XMessage, 0, len(entries)) - for _, entry := range entries { - if entry == nil { - continue - } - pair, ok := entry.([]any) - if !ok || len(pair) != 2 { - return nil, fmt.Errorf("unexpected XAUTOCLAIM entry %v", entry) - } - id, ok := pair[0].(string) - if !ok { - return nil, fmt.Errorf("unexpected XAUTOCLAIM entry ID %v", pair[0]) - } - fields, ok := pair[1].([]any) - if !ok || len(fields)%2 != 0 { - return nil, fmt.Errorf("unexpected XAUTOCLAIM entry fields %v", pair[1]) - } - values := make(map[string]any, len(fields)/2) - for i := 0; i < len(fields); i += 2 { - key, ok := fields[i].(string) - if !ok { - return nil, fmt.Errorf("unexpected XAUTOCLAIM field name %v", fields[i]) - } - values[key] = fields[i+1] - } - msgs = append(msgs, redis.XMessage{ID: id, Values: values}) - } - return msgs, nil -} - -// isLeaseLostErr reports whether err is the fenced-script rejection of an -// operation whose lease was taken over by another sink instance. -func isLeaseLostErr(err error) bool { - return err != nil && strings.HasPrefix(err.Error(), leaseLostErrorPrefix) -} - -// keepAliveContentKey returns the rmap content key of the sink keep-alive map -// (see sinkKeepAliveMapName). -func keepAliveContentKey(sinkName string) string { - return fmt.Sprintf("map:sink:%s:keepalive:content", sinkName) -} - -// keepAliveChannelKey returns the rmap pubsub channel of the sink keep-alive -// map. -func keepAliveChannelKey(sinkName string) string { - return fmt.Sprintf("map:sink:%s:keepalive:updates", sinkName) -} diff --git a/streaming/sink_recovery.go b/streaming/sink_recovery.go index 7d87670..03d1b3d 100644 --- a/streaming/sink_recovery.go +++ b/streaming/sink_recovery.go @@ -1,484 +1,305 @@ -// This file implements lossless consumer-group recovery and the stream -// lifecycle fence shared by sinks and Stream.Destroy. -// -// Every piece of durable sink metadata attached to a stream lives under the -// "pulse:streammeta:" prefix: -// -// - :lifecycle hash with a "state" field ("active" or "destroyed"). -// - :cursors hash mapping sink name to the recovery cursor, the -// highest event ID known to be fully acknowledged for that sink. -// - :sink::lease fenced lease used by sink_lease.go. -// -// The lifecycle hash is the destroy fence: every Lua script in this file and -// in sink_lease.go that mutates stream-scoped metadata reads the lifecycle -// state first, inside the same atomic script invocation as the write. Once -// Stream.Destroy marks a stream destroyed no concurrent sink loop can -// recreate its consumer group, membership map, cursor, or lease; only a -// deliberate NewSink/AddStream call (establish mode) reactivates the name. -// -// The recovery cursor is what makes NOGROUP recovery lossless: when a -// consumer group disappears (XGROUP DESTROY, Redis state loss) the group is -// recreated at the stored cursor rather than "$" so unacknowledged events are -// redelivered. The cursor is advanced by recomputing it from the exact PEL -// state each time events are acknowledged. -// -// Scripts publish replicated-map updates using the exact rmap wire protocol -// (see rmap/scripts.go) so live rmap clients observe fenced membership -// mutations exactly as if they had been made through the rmap API. +// Package streaming persists each sink cursor in generation-qualified metadata +// beside its lifecycle-selected physical stream key. Recovery and +// acknowledgement verify the Redis lifecycle record in the same Lua operation, +// so stale handles cannot recreate groups or mutate a later incarnation. package streaming import ( "context" - "errors" "fmt" - "strings" + "strconv" redis "github.com/redis/go-redis/v9" ) type ( - // recoveryAcker acknowledges sink events and atomically advances the - // durable recovery cursor from the exact PEL state. It is the Acker - // carried by every event delivered through a sink, so both Sink.Ack and - // direct Event.Acker.XAck calls keep the recovery cursor current. + // recoveryAcker implements Event.Acker for one immutable stream generation. recoveryAcker struct { - rdb *redis.Client + stream *Stream } ) var ( - // ErrStreamDestroyed is returned when a sink operation targets a stream - // that was destroyed with Stream.Destroy. The sink drops the stream from - // its set instead of resurrecting its metadata. - ErrStreamDestroyed = errors.New("stream is destroyed") - - // ErrSinkClosed is returned by AddStream and RemoveStream after Close. - ErrSinkClosed = errors.New("sink is closed") -) - -// streamDestroyedErrorPrefix is the Redis error prefix used by fenced scripts -// to reject writes against a destroyed stream. -const streamDestroyedErrorPrefix = "STREAMDESTROYED" - -// lifecycleFenceLua guards a script against destroyed streams and activates -// absent lifecycles. Scripts using it must bind KEYS[1] to the lifecycle key -// and define an `establish` local ("1" reactivates a destroyed stream). -const lifecycleFenceLua = ` -local state = redis.call("HGET", KEYS[1], "state") -if state == "destroyed" then - if establish == "1" then - redis.call("HSET", KEYS[1], "state", "active") - else - return redis.error_reply("STREAMDESTROYED stream was destroyed") - end -elseif not state then - redis.call("HSET", KEYS[1], "state", "active") -end -` - -// recoveryCursorLua defines recovery_cursor(stream, group) which computes the -// highest event ID X such that every entry at or before X is acknowledged: -// the entry preceding the oldest pending entry when the PEL is not empty, the -// group last-delivered-id otherwise. Returns false when the group is gone. -const recoveryCursorLua = ` -local function recovery_cursor(stream, group) - local pending = redis.pcall("XPENDING", stream, group) - if pending["err"] then - return false - end - if pending[1] > 0 then - local prev = redis.call("XREVRANGE", stream, "(" .. pending[2], "-", "COUNT", 1) - if #prev == 0 then - return "0-0" - end - return prev[1][1] - end - local groups = redis.call("XINFO", "GROUPS", stream) - for _, info in ipairs(groups) do - local name, last - for i = 1, #info, 2 do - if info[i] == "name" then name = info[i+1] end - if info[i] == "last-delivered-id" then last = info[i+1] end - end - if name == group then - return last - end - end - return false + // ensureConsumerGroupScript creates or repairs a group, initializes its + // durable cursor, and aligns recovery retention atomically after verifying + // the exact stream generation. + ensureConsumerGroupScript = redis.NewScript(` +if redis.call("HGET", KEYS[1], "state") ~= ARGV[1] +or redis.call("HGET", KEYS[1], "generation") ~= ARGV[2] +or redis.call("HGET", KEYS[1], ARGV[7]) ~= KEYS[2] then + return redis.error_reply("STREAMDESTROYED") end -` - -// membershipRemoveLua defines membership_remove(content, channel, field, -// value) which removes value from the JSON-array field of an rmap content -// hash and publishes the matching rmap protocol notification. -const membershipRemoveLua = ` -local function membership_remove(content, channel, field, value) - local v = redis.call("HGET", content, field) - if not v then - return - end - local values = {} - local ok, decoded = pcall(cjson.decode, v) - if ok and type(decoded) == "table" then - values = decoded - else - for s in string.gmatch(v, "[^,]+") do - table.insert(values, s) - end - end - local remaining = {} - local removed = false - for _, item in ipairs(values) do - if item == value then - removed = true - else - table.insert(remaining, item) - end - end - if not removed then - return - end - if #remaining == 0 then - redis.call("HDEL", content, field) - local rev = tostring(redis.call("HINCRBY", content, "=rev", 1)) - redis.call("HSET", content, "=kind", "del") - local msg = struct.pack("ic0ic0", string.len(field), field, string.len(rev), rev) - redis.call("PUBLISH", channel, "del:" .. msg) - return - end - local encoded = cjson.encode(remaining) - redis.call("HSET", content, field, encoded) - local rev = tostring(redis.call("HINCRBY", content, "=rev", 1)) - redis.call("HSET", content, "=kind", "set") - local msg = struct.pack("ic0ic0ic0", string.len(field), field, string.len(encoded), encoded, string.len(rev), rev) - redis.call("PUBLISH", channel, "set:" .. msg) +local deadline = redis.call("HGET", KEYS[1], ARGV[8]) +if deadline then + local now = redis.call("TIME") + local now_ms = tonumber(now[1]) * 1000 + math.floor(tonumber(now[2]) / 1000) + if now_ms >= tonumber(deadline) then + return redis.error_reply("DEADLINEELAPSED") + end end -` -// ensureConsumerGroupScript creates or repairs the consumer group for a sink -// behind the lifecycle fence and restores the stream TTL even when the group -// already exists (BUSYGROUP). The group is created at the durable recovery -// cursor when one exists, at the caller start ID otherwise, and the cursor is -// recomputed and stored from live group state so it is always defined. -// -// KEYS: [1]=lifecycle [2]=stream [3]=cursors -// ARGV: [1]=group [2]=startID [3]=establish [4]=ttlMs [5]=ttlSliding -// Returns {created(0|1), cursor}. -var ensureConsumerGroupScript = redis.NewScript(` -local establish = ARGV[3] -` + lifecycleFenceLua + recoveryCursorLua + ` -local start = redis.call("HGET", KEYS[3], ARGV[1]) +local absent = redis.call("EXISTS", KEYS[2]) == 0 +local start = redis.call("HGET", KEYS[3], ARGV[3]) if not start then - start = ARGV[2] + start = ARGV[4] end -local created = 1 -local res = redis.pcall("XGROUP", "CREATE", KEYS[2], ARGV[1], start, "MKSTREAM") -if type(res) == "table" and res["err"] then - if not string.find(res["err"], "BUSYGROUP", 1, true) then - return res - end - created = 0 +local created = redis.pcall("XGROUP", "CREATE", KEYS[2], ARGV[3], start, "MKSTREAM") +if type(created) == "table" and created.err +and not string.find(created.err, "BUSYGROUP", 1, true) then + return redis.error_reply(created.err) end -local cursor = recovery_cursor(KEYS[2], ARGV[1]) -redis.call("HSET", KEYS[3], ARGV[1], cursor) -if tonumber(ARGV[4]) > 0 then - if ARGV[5] == "1" then - redis.call("PEXPIRE", KEYS[2], ARGV[4]) - else - redis.call("PEXPIRE", KEYS[2], ARGV[4], "NX") - end + +if redis.call("HEXISTS", KEYS[3], ARGV[3]) == 0 then + local pending = redis.call("XPENDING", KEYS[2], ARGV[3]) + local cursor + if pending[1] == 0 then + local groups = redis.call("XINFO", "GROUPS", KEYS[2]) + for _, group in ipairs(groups) do + local name + local delivered + for i = 1, #group, 2 do + if group[i] == "name" then + name = group[i + 1] + elseif group[i] == "last-delivered-id" then + delivered = group[i + 1] + end + end + if name == ARGV[3] then + cursor = delivered + break + end + end + if not cursor then + return redis.error_reply("NOGROUP consumer group no longer exists") + end + else + local previous = redis.call("XREVRANGE", KEYS[2], "(" .. pending[2], "-", "COUNT", 1) + if #previous == 0 then + cursor = "0-0" + else + cursor = previous[1][1] + end + end + redis.call("HSET", KEYS[3], ARGV[3], cursor) end -return {created, cursor} -`) -// ackEventsScript acknowledges events and atomically advances the recovery -// cursor from the exact PEL state. Acking events of a destroyed stream or of -// a deleted group is a no-op returning 0, matching XACK semantics on missing -// keys. -// -// KEYS: [1]=lifecycle [2]=stream [3]=cursors -// ARGV: [1]=group [2..]=event IDs -var ackEventsScript = redis.NewScript(` -local state = redis.call("HGET", KEYS[1], "state") -if state ~= "active" then - return 0 +local ttl = tonumber(ARGV[5]) +if deadline then + redis.call("PEXPIREAT", KEYS[2], deadline) + redis.call("PEXPIREAT", KEYS[3], deadline) +elseif ttl > 0 and (ARGV[6] == "1" or redis.call("PTTL", KEYS[2]) == -1) then + redis.call("PEXPIRE", KEYS[2], ttl) end -` + recoveryCursorLua + ` -local acked = redis.call("XACK", KEYS[2], ARGV[1], unpack(ARGV, 2)) -local cursor = recovery_cursor(KEYS[2], ARGV[1]) -if cursor then - redis.call("HSET", KEYS[3], ARGV[1], cursor) +if absent then + return 1 end -return acked +return 0 `) -// registerConsumerScript creates a Redis consumer in the sink group and -// appends it to the stream membership map behind the lifecycle fence, in one -// atomic operation so membership and consumer-group state cannot diverge. -// -// KEYS: [1]=lifecycle [2]=stream [3]=membership content [4]=membership channel -// ARGV: [1]=group [2]=consumer -var registerConsumerScript = redis.NewScript(` -local state = redis.call("HGET", KEYS[1], "state") -if state ~= "active" then - return redis.error_reply("STREAMDESTROYED stream was destroyed") + // recoveryCursorScript acknowledges one or more IDs and advances the + // canonical cursor only across the prefix with no pending entry. + recoveryCursorScript = redis.NewScript(` +if redis.call("HGET", KEYS[1], "state") ~= ARGV[1] +or redis.call("HGET", KEYS[1], "generation") ~= ARGV[2] +or redis.call("HGET", KEYS[1], ARGV[4]) ~= KEYS[2] then + return redis.error_reply("STREAMDESTROYED") end -local res = redis.pcall("XGROUP", "CREATECONSUMER", KEYS[2], ARGV[1], ARGV[2]) -if type(res) == "table" and res["err"] then - return res +local deadline = redis.call("HGET", KEYS[1], ARGV[5]) +if deadline then + local now = redis.call("TIME") + local now_ms = tonumber(now[1]) * 1000 + math.floor(tonumber(now[2]) / 1000) + if now_ms >= tonumber(deadline) then + return redis.error_reply("DEADLINEELAPSED") + end end -local field = ARGV[1] -local v = redis.call("HGET", KEYS[3], field) -local values = {} -if v then - local ok, decoded = pcall(cjson.decode, v) - if ok and type(decoded) == "table" then - values = decoded - else - for s in string.gmatch(v, "[^,]+") do - table.insert(values, s) - end - end -end -for _, item in ipairs(values) do - if item == ARGV[2] then - return 1 - end + +local result = redis.call("XACK", KEYS[2], ARGV[3], unpack(ARGV, 6)) +if result == 0 then + return 0 end -table.insert(values, ARGV[2]) -local encoded = cjson.encode(values) -redis.call("HSET", KEYS[3], field, encoded) -local rev = tostring(redis.call("HINCRBY", KEYS[3], "=rev", 1)) -redis.call("HSET", KEYS[3], "=kind", "set") -local msg = struct.pack("ic0ic0ic0", string.len(field), field, string.len(encoded), encoded, string.len(rev), rev) -redis.call("PUBLISH", KEYS[4], "set:" .. msg) -return 1 -`) -// detachConsumerScript rolls back a consumer registration: it removes the -// consumer from the membership map and deletes the Redis consumer when its -// PEL is empty. Used to compensate partial consumer rotation. A destroyed -// stream has no metadata left to detach so the script is then a no-op. -// -// KEYS: [1]=lifecycle [2]=stream [3]=membership content [4]=membership channel -// ARGV: [1]=group [2]=consumer -var detachConsumerScript = redis.NewScript(` -local state = redis.call("HGET", KEYS[1], "state") -if state ~= "active" then - return 0 +local pending = redis.call("XPENDING", KEYS[2], ARGV[3]) +local cursor +if pending[1] == 0 then + local groups = redis.call("XINFO", "GROUPS", KEYS[2]) + for _, group in ipairs(groups) do + local name + local delivered + for i = 1, #group, 2 do + if group[i] == "name" then + name = group[i + 1] + elseif group[i] == "last-delivered-id" then + delivered = group[i + 1] + end + end + if name == ARGV[3] then + cursor = delivered + break + end + end + if not cursor then + return redis.error_reply("NOGROUP consumer group no longer exists") + end +else + local previous = redis.call("XREVRANGE", KEYS[2], "(" .. pending[2], "-", "COUNT", 1) + if #previous == 0 then + cursor = "0-0" + else + cursor = previous[1][1] + end end -` + membershipRemoveLua + ` -local pending = redis.pcall("XPENDING", KEYS[2], ARGV[1], "-", "+", 1, ARGV[2]) -if not pending["err"] and #pending == 0 then - redis.call("XGROUP", "DELCONSUMER", KEYS[2], ARGV[1], ARGV[2]) +redis.call("HSET", KEYS[3], ARGV[3], cursor) +if deadline then + redis.call("PEXPIREAT", KEYS[3], deadline) end -membership_remove(KEYS[3], KEYS[4], ARGV[1], ARGV[2]) -return 1 +return result `) -// removeSinkStreamScript atomically removes a sink consumer from the stream -// membership map and, when it was the last member, destroys the consumer -// group, its recovery cursor, and its lease. Being a single script there is -// no partial failure between membership and group state to compensate. -// -// KEYS: [1]=lifecycle [2]=stream [3]=cursors [4]=membership content -// [5]=membership channel [6]=lease -// ARGV: [1]=group [2]=consumer -var removeSinkStreamScript = redis.NewScript(` + // destroyStreamScript invalidates and deletes exactly one generation. The + // lifecycle record remains as the monotonic source used by the next + // explicit NewStream call. The generation-qualified membership map is never + // reused, so its destroy notification needs no persistent rmap tombstone. + destroyStreamScript = redis.NewScript(` +if redis.call("HGET", KEYS[1], "generation") ~= ARGV[2] then + return redis.error_reply("STREAMDESTROYED") +end local state = redis.call("HGET", KEYS[1], "state") -if state ~= "active" then - return 1 +if state == ARGV[3] then + return 0 end -` + membershipRemoveLua + ` -membership_remove(KEYS[4], KEYS[5], ARGV[1], ARGV[2]) -if not redis.call("HGET", KEYS[4], ARGV[1]) then - redis.call("XGROUP", "DESTROY", KEYS[2], ARGV[1]) - redis.call("HDEL", KEYS[3], ARGV[1]) - redis.call("DEL", KEYS[6]) +if state ~= ARGV[1] then + return redis.error_reply("STREAMDESTROYED") +end +if redis.call("HGET", KEYS[1], ARGV[4]) ~= KEYS[2] then + return redis.error_reply("STREAMDESTROYED") end -return 1 -`) -// destroyStreamScript marks the stream destroyed and deletes every key owned -// by the stream in one atomic operation: events, recovery cursors, sink -// leases (enumerated from the membership map fields), and the membership map -// itself via the rmap destroy protocol. The lifecycle hash is kept as the -// destroyed tombstone that fences concurrent sink metadata writes. -// -// KEYS: [1]=lifecycle [2]=stream [3]=cursors [4]=membership content -// [5]=membership channel -// ARGV: [1]=lease key prefix (lease key is prefix .. sink .. ":lease") -var destroyStreamScript = redis.NewScript(` -redis.call("HSET", KEYS[1], "state", "destroyed") -redis.call("DEL", KEYS[2], KEYS[3]) -if redis.call("EXISTS", KEYS[4]) == 1 then - for _, field in ipairs(redis.call("HKEYS", KEYS[4])) do - if field ~= "=rev" and field ~= "=kind" then - redis.call("DEL", ARGV[1] .. field .. ":lease") - end - end - local rev = redis.call("HINCRBY", KEYS[4], "=rev", 1) - redis.call("DEL", KEYS[4]) - redis.call("HSET", KEYS[4], "=rev", rev, "=kind", "destroy") - redis.call("PUBLISH", KEYS[5], "destroy:" .. tostring(rev)) +redis.call("HSET", KEYS[1], "state", ARGV[3]) +local rev = redis.call("HINCRBY", KEYS[4], "=rev", 1) +local resources = redis.call("SMEMBERS", KEYS[6]) +if #resources > 0 then + redis.call("DEL", unpack(resources)) end +redis.call("DEL", KEYS[2], KEYS[3], KEYS[4]) +redis.call("DEL", KEYS[6]) +redis.call("PUBLISH", KEYS[5], "destroy:" .. tostring(rev)) return 1 `) +) -// ensureConsumerGroup creates or repairs the consumer group for sinkName on -// stream at the durable recovery cursor (startID for brand new groups) and -// restores the stream TTL. establish reactivates a destroyed stream and is -// reserved for deliberate NewSink/AddStream calls; recovery paths pass false -// so a destroyed stream fails with ErrStreamDestroyed instead of being -// resurrected. Returns whether the group was created and the stored cursor. -func ensureConsumerGroup(ctx context.Context, stream *Stream, sinkName, startID string, establish bool) (bool, string, error) { - keys := []string{lifecycleKey(stream.key), stream.key, cursorsKey(stream.key)} - args := []any{sinkName, startID, boolArg(establish), stream.ttl.Milliseconds(), boolArg(stream.ttlSliding)} - res, err := ensureConsumerGroupScript.Run(ctx, stream.rdb, keys, args...).Slice() - if err != nil { - if isStreamDestroyedErr(err) { - return false, "", fmt.Errorf("cannot create consumer group %s for stream %s: %w", sinkName, stream.Name, ErrStreamDestroyed) - } - return false, "", fmt.Errorf("failed to create consumer group %s for stream %s: %w", sinkName, stream.Name, err) - } - return res[0].(int64) == 1, res[1].(string), nil -} - -// registerSinkConsumer atomically creates the Redis consumer for sinkName in -// the stream consumer group and records it in the stream membership map. -func registerSinkConsumer(ctx context.Context, stream *Stream, sinkName, consumer string) error { - keys := []string{ - lifecycleKey(stream.key), - stream.key, - membershipContentKey(stream.Name), - membershipChannelKey(stream.Name), - } - if err := registerConsumerScript.Run(ctx, stream.rdb, keys, sinkName, consumer).Err(); err != nil { - if isStreamDestroyedErr(err) { - return fmt.Errorf("cannot register consumer %s for stream %s: %w", consumer, stream.Name, ErrStreamDestroyed) - } - return fmt.Errorf("failed to register consumer %s for stream %s: %w", consumer, stream.Name, err) - } - return nil -} - -// detachSinkConsumer compensates a partial consumer rotation by removing the -// consumer from the stream membership map and deleting the Redis consumer -// when it has no pending events. -func detachSinkConsumer(ctx context.Context, stream *Stream, sinkName, consumer string) error { - keys := []string{ - lifecycleKey(stream.key), - stream.key, - membershipContentKey(stream.Name), - membershipChannelKey(stream.Name), - } - if err := detachConsumerScript.Run(ctx, stream.rdb, keys, sinkName, consumer).Err(); err != nil { - return fmt.Errorf("failed to detach consumer %s from stream %s: %w", consumer, stream.Name, err) - } - return nil -} - -// removeSinkStream removes the sink consumer from the stream membership map -// and destroys the consumer group, recovery cursor, and lease when the -// consumer was the last member, all in one atomic operation. -func removeSinkStream(ctx context.Context, stream *Stream, sinkName, consumer string) error { - keys := []string{ - lifecycleKey(stream.key), - stream.key, - cursorsKey(stream.key), - membershipContentKey(stream.Name), - membershipChannelKey(stream.Name), - leaseKey(stream.key, sinkName), - } - if err := removeSinkStreamScript.Run(ctx, stream.rdb, keys, sinkName, consumer).Err(); err != nil { - return fmt.Errorf("failed to remove sink %s from stream %s: %w", sinkName, stream.Name, err) +// ensureConsumerGroup creates or repairs a group for this exact generation. +// The boolean result reports whether its physical stream data was absent. +func ensureConsumerGroup(ctx context.Context, stream *Stream, group, configuredStart string) (bool, error) { + if err := stream.ensureGeneration(ctx); err != nil { + return false, err } - return nil -} - -// destroyStream atomically marks the stream destroyed and deletes its events -// and sink metadata. It is idempotent and safe to call on streams that were -// never created. -func destroyStream(ctx context.Context, stream *Stream) error { - keys := []string{ - lifecycleKey(stream.key), - stream.key, - cursorsKey(stream.key), - membershipContentKey(stream.Name), - membershipChannelKey(stream.Name), + absent, err := ensureConsumerGroupScript.Run( + ctx, + stream.rdb, + []string{stream.lifecycleKey, stream.key, recoveryCursorKey(stream)}, + streamStateActive, + stream.generation, + group, + configuredStart, + strconv.FormatInt(stream.ttl.Milliseconds(), 10), + boolString(stream.ttlSliding), + streamPhysicalKey, + streamDeadlineKey, + ).Int64() + if err != nil { + return false, fmt.Errorf( + "failed to ensure Redis consumer group %q for stream %q generation %s: %w", + group, + stream.Name, + stream.generation, + stream.lifecycleError(err), + ) } - return destroyStreamScript.Run(ctx, stream.rdb, keys, leaseKeyPrefix(stream.key)).Err() + return absent == 1, nil } -// XAck acknowledges the events and advances the sink recovery cursor from the -// exact PEL state in one atomic operation. It satisfies the Acker interface -// so acknowledging a sink event through Event.Acker keeps recovery lossless. -func (a *recoveryAcker) XAck(ctx context.Context, streamKey, sinkName string, ids ...string) *redis.IntCmd { +// XAck atomically acknowledges IDs and advances this generation's shared +// recovery cursor. The streamKey argument remains part of the public Acker +// contract; generated events bind the implementation to their own stream. +func (a *recoveryAcker) XAck(ctx context.Context, _ string, group string, ids ...string) *redis.IntCmd { cmd := redis.NewIntCmd(ctx) - keys := []string{lifecycleKey(streamKey), streamKey, cursorsKey(streamKey)} - args := make([]any, 0, len(ids)+1) - args = append(args, sinkName) + args := make([]any, 0, len(ids)+5) + args = append( + args, + streamStateActive, + a.stream.generation, + group, + streamPhysicalKey, + streamDeadlineKey, + ) for _, id := range ids { args = append(args, id) } - acked, err := ackEventsScript.Run(ctx, a.rdb, keys, args...).Int64() + result, err := recoveryCursorScript.Run( + ctx, + a.stream.rdb, + []string{a.stream.lifecycleKey, a.stream.key, recoveryCursorKey(a.stream)}, + args..., + ).Int64() if err != nil { - cmd.SetErr(err) + cmd.SetErr(fmt.Errorf( + "failed to acknowledge events %q and advance consumer group %q on stream %q generation %s: %w", + ids, + group, + a.stream.Name, + a.stream.generation, + a.stream.lifecycleError(err), + )) return cmd } - cmd.SetVal(acked) + cmd.SetVal(result) return cmd } -// streamMetaPrefix returns the metadata key prefix for the stream with the -// given event key. Metadata lives under a prefix distinct from the event key -// so stream names cannot collide with metadata key suffixes. -func streamMetaPrefix(streamKey string) string { - return "pulse:streammeta:" + streamKey[len(streamKeyPrefix):] -} - -// lifecycleKey returns the lifecycle fence key for the stream event key. -func lifecycleKey(streamKey string) string { - return streamMetaPrefix(streamKey) + ":lifecycle" -} - -// cursorsKey returns the recovery cursor hash key for the stream event key. -func cursorsKey(streamKey string) string { - return streamMetaPrefix(streamKey) + ":cursors" -} - -// leaseKeyPrefix returns the prefix of per-sink lease keys for the stream. -func leaseKeyPrefix(streamKey string) string { - return streamMetaPrefix(streamKey) + ":sink:" -} - -// leaseKey returns the fenced lease key for the sink on the stream. -func leaseKey(streamKey, sinkName string) string { - return leaseKeyPrefix(streamKey) + sinkName + ":lease" -} - -// membershipContentKey returns the rmap content key of the stream sink -// membership map (see consumersMapName). -func membershipContentKey(streamName string) string { - return fmt.Sprintf("map:stream:%s:sinks:content", streamName) +// destroyStream atomically invalidates and deletes this exact generation. +func destroyStream(ctx context.Context, stream *Stream) error { + if err := stream.loadExistingGeneration(ctx); err != nil { + return err + } + err := destroyStreamScript.Run( + ctx, + stream.rdb, + []string{ + stream.lifecycleKey, + stream.key, + recoveryCursorKey(stream), + consumersMapContentKey(stream), + consumersMapChannelKey(stream), + streamResourceRegistryKey(stream), + }, + streamStateActive, + stream.generation, + streamStateDestroyed, + streamPhysicalKey, + ).Err() + if err != nil { + return fmt.Errorf( + "failed to destroy stream %q generation %s: %w", + stream.Name, + stream.generation, + stream.lifecycleError(err), + ) + } + return nil } -// membershipChannelKey returns the rmap pubsub channel of the stream sink -// membership map. -func membershipChannelKey(streamName string) string { - return fmt.Sprintf("map:stream:%s:sinks:updates", streamName) +// recoveryCursorKey identifies all named-sink cursors for one generation while +// the physical event key remains stable. +func recoveryCursorKey(stream *Stream) string { + return fmt.Sprintf("%s:sink-recovery:%s", stream.key, stream.generation) } -// isStreamDestroyedErr reports whether err is the fenced-script rejection of -// a write against a destroyed stream. -func isStreamDestroyedErr(err error) bool { - return err != nil && strings.HasPrefix(err.Error(), streamDestroyedErrorPrefix) +// consumersMapContentKey identifies this generation's membership hash. +func consumersMapContentKey(stream *Stream) string { + return fmt.Sprintf("map:%s:content", consumersMapName(stream)) } -// boolArg encodes a boolean as the "0"/"1" convention used by the scripts. -func boolArg(b bool) string { - if b { - return "1" - } - return "0" +// consumersMapChannelKey identifies this generation's membership updates. +func consumersMapChannelKey(stream *Stream) string { + return rmapChannelKey(consumersMapName(stream)) } diff --git a/streaming/sink_recovery_test.go b/streaming/sink_recovery_test.go index d12dcaf..0ac71bc 100644 --- a/streaming/sink_recovery_test.go +++ b/streaming/sink_recovery_test.go @@ -1,17 +1,11 @@ -// Tests for lossless consumer group recovery, the destroy lifecycle fence, -// fenced leases, close semantics, and failure-atomic stream ownership -// changes. All tests run against a live Redis instance. +// Package streaming tests consumer-group recovery against real Redis. The +// cases focus on delivery safety, shared replica state, lifecycle boundaries, +// and compensation when a multi-step ownership change fails. package streaming import ( "context" - "encoding/json" "errors" - "fmt" - "net" - "strings" - "sync" - "sync/atomic" "testing" "time" @@ -20,706 +14,730 @@ import ( "github.com/stretchr/testify/require" "goa.design/pulse/pulse" + "goa.design/pulse/rmap" "goa.design/pulse/streaming/options" ptesting "goa.design/pulse/testing" ) -// TestSinkRecoversGroupAfterExternalDestroy is the core lossless recovery -// scenario: publish events, ack some, force XGROUP DESTROY, and verify the -// group is recreated at the recovery cursor so every unacked event is -// redelivered exactly once and no acked event is redelivered. -func TestSinkRecoversGroupAfterExternalDestroy(t *testing.T) { - testName := strings.Replace(t.Name(), "/", "_", -1) +type ( + // recordingAcker proves Sink.Ack preserves Event's public acknowledgement + // boundary instead of bypassing it with the sink's Redis client. + recordingAcker struct { + streamKey string + group string + ids []string + } +) + +func TestSinkAdoptsFlatStreamWithQueuedAndPendingEvents(t *testing.T) { rdb := ptesting.NewRedisClient(t) - defer ptesting.CleanupRedis(t, rdb, true, testName) + defer ptesting.CleanupRedis(t, rdb, false, "") ctx := ptesting.NewTestContext(t) - s, err := NewStream(testName, rdb, options.WithStreamLogger(pulse.ClueLogger(ctx))) - require.NoError(t, err) - sink, err := s.NewSink(ctx, "sink", + key := streamKey(t.Name()) + + firstID, err := rdb.XAdd(ctx, &redis.XAddArgs{ + Stream: key, + Values: map[string]any{nameKey: "first", payloadKey: "one"}, + }).Result() + require.NoError(t, err) + secondID, err := rdb.XAdd(ctx, &redis.XAddArgs{ + Stream: key, + Values: map[string]any{nameKey: "second", payloadKey: "two"}, + }).Result() + require.NoError(t, err) + require.NoError(t, rdb.XGroupCreate(ctx, key, "sink", "0").Err()) + legacy, err := rdb.XReadGroup(ctx, &redis.XReadGroupArgs{ + Group: "sink", + Consumer: "legacy", + Streams: []string{key, ">"}, + Count: 1, + }).Result() + require.NoError(t, err) + require.Equal(t, firstID, legacy[0].Messages[0].ID) + + stream, err := NewStream(t.Name(), rdb) + require.NoError(t, err) + require.Empty(t, stream.Generation()) + sink, err := stream.NewSink( + ctx, + "sink", options.WithSinkStartAtOldest(), - options.WithSinkBlockDuration(testBlockDuration)) + options.WithSinkAckGracePeriod(50*time.Millisecond), + options.WithSinkBlockDuration(testBlockDuration), + ) require.NoError(t, err) - defer cleanupSink(t, ctx, s, sink) + events := sink.Subscribe() - c := sink.Subscribe() - ids := make([]string, 5) - for i := range ids { - ids[i], err = s.Add(ctx, fmt.Sprintf("event%d", i), []byte("payload")) - require.NoError(t, err) - } - events := make([]*Event, 5) - for i := range events { - events[i] = receiveEvent(t, c) - } - require.NoError(t, sink.Ack(ctx, events[0])) - require.NoError(t, sink.Ack(ctx, events[1])) - // Acks advance the recovery cursor synchronously. - assert.Equal(t, ids[1], recoveryCursor(t, ctx, rdb, s, "sink")) - - // Simulate Redis consumer group state loss. - require.NoError(t, rdb.XGroupDestroy(ctx, s.key, "sink").Err()) - futureID, err := s.Add(ctx, "future", []byte("payload")) - require.NoError(t, err) - - // Every unacked event and the new event must be redelivered exactly once. - var redelivered []string - for range 4 { - ev := receiveEvent(t, c) - redelivered = append(redelivered, ev.ID) - require.NoError(t, sink.Ack(ctx, ev)) - } - assert.Equal(t, []string{ids[2], ids[3], ids[4], futureID}, redelivered) - select { - case ev := <-c: - t.Errorf("unexpected redelivery of event %s", ev.ID) - case <-time.After(4 * testBlockDuration): + received := map[string]*Event{} + for len(received) < 2 { + select { + case event := <-events: + received[event.ID] = event + case <-time.After(max): + t.Fatalf("timed out waiting for upgraded events; received %v", received) + } } - groups, err := rdb.XInfoGroups(ctx, s.key).Result() - require.NoError(t, err) - require.Len(t, groups, 1) - assert.Equal(t, "sink", groups[0].Name) + require.Contains(t, received, firstID) + require.Contains(t, received, secondID) + pending, err := rdb.XPending(ctx, key, "sink").Result() + require.NoError(t, err) + require.EqualValues(t, 2, pending.Count) + require.Equal(t, "1", stream.Generation()) + require.Equal(t, key, stream.key) + require.Equal(t, key, rdb.HGet(ctx, stream.lifecycleKey, streamPhysicalKey).Val()) + + require.NoError(t, sink.Ack(ctx, received[firstID])) + require.NoError(t, sink.Ack(ctx, received[secondID])) + require.NoError(t, sink.Close(ctx)) + require.NoError(t, stream.Destroy(ctx)) } -// TestSinkRecoveryCursorIsSharedAcrossReplicas verifies that sink replicas -// share one durable recovery cursor per stream and group so whichever replica -// recovers first resumes from the same acknowledged position. -func TestSinkRecoveryCursorIsSharedAcrossReplicas(t *testing.T) { - testName := strings.Replace(t.Name(), "/", "_", -1) +func TestSinkAcknowledgesFilteredEvents(t *testing.T) { rdb := ptesting.NewRedisClient(t) - defer ptesting.CleanupRedis(t, rdb, true, testName) + defer ptesting.CleanupRedis(t, rdb, false, "") ctx := ptesting.NewTestContext(t) - s, err := NewStream(testName, rdb, options.WithStreamLogger(pulse.ClueLogger(ctx))) + stream, err := NewStream(t.Name(), rdb) require.NoError(t, err) - sink1, err := s.NewSink(ctx, "sink", + sink, err := stream.NewSink( + ctx, + "sink", options.WithSinkStartAtOldest(), - options.WithSinkBlockDuration(testBlockDuration)) - require.NoError(t, err) - defer cleanupSink(t, ctx, s, sink1) - sink2, err := s.NewSink(ctx, "sink", - options.WithSinkStartAtOldest(), - options.WithSinkBlockDuration(testBlockDuration)) + options.WithSinkTopic("wanted"), + options.WithSinkBlockDuration(testBlockDuration), + ) require.NoError(t, err) - defer sink2.Close(ctx) + events := sink.Subscribe() - c1, c2 := sink1.Subscribe(), sink2.Subscribe() - ids := make([]string, 2) - for i := range ids { - ids[i], err = s.Add(ctx, fmt.Sprintf("event%d", i), []byte("payload")) - require.NoError(t, err) - } - for range ids { - select { - case ev := <-c1: - require.NoError(t, sink1.Ack(ctx, ev)) - case ev := <-c2: - require.NoError(t, sink2.Ack(ctx, ev)) - case <-time.After(max): - t.Fatal("timeout waiting for event") - } - } - assert.Equal(t, ids[1], recoveryCursor(t, ctx, rdb, s, "sink")) + filteredID, err := stream.Add(ctx, "filtered", []byte("filtered"), options.WithTopic("other")) + require.NoError(t, err) + wantedID, err := stream.Add(ctx, "wanted", []byte("wanted"), options.WithTopic("wanted")) + require.NoError(t, err) + wanted := receiveSinkEvent(t, events) + require.Equal(t, wantedID, wanted.ID) - require.NoError(t, rdb.XGroupDestroy(ctx, s.key, "sink").Err()) - futureID, err := s.Add(ctx, "future", []byte("payload")) + require.Eventually(t, func() bool { + pending, err := rdb.XPending(ctx, stream.key, sink.Name).Result() + return err == nil && pending.Count == 1 && pending.Lower == wantedID + }, max, delay) + cursor, err := rdb.HGet(ctx, recoveryCursorKey(stream), sink.Name).Result() require.NoError(t, err) + require.Equal(t, filteredID, cursor) - // Whichever replica recovers, only the new event is delivered. - select { - case ev := <-c1: - assert.Equal(t, futureID, ev.ID) - require.NoError(t, sink1.Ack(ctx, ev)) - case ev := <-c2: - assert.Equal(t, futureID, ev.ID) - require.NoError(t, sink2.Ack(ctx, ev)) - case <-time.After(max): - t.Fatal("timeout waiting for post-recovery event") - } - select { - case ev := <-c1: - t.Errorf("unexpected redelivery of event %s", ev.ID) - case ev := <-c2: - t.Errorf("unexpected redelivery of event %s", ev.ID) - case <-time.After(4 * testBlockDuration): - } + require.NoError(t, sink.Ack(ctx, wanted)) + require.NoError(t, sink.Close(ctx)) + require.NoError(t, stream.Destroy(ctx)) } -// TestEnsureGroupRestoresTTLOnBusyGroup verifies that ensuring an existing -// consumer group (BUSYGROUP path) restores the stream TTL and that recovery -// metadata itself never carries a TTL. -func TestEnsureGroupRestoresTTLOnBusyGroup(t *testing.T) { - testName := strings.Replace(t.Name(), "/", "_", -1) +func TestSinkRecoveryPreservesPendingAndGapEvents(t *testing.T) { rdb := ptesting.NewRedisClient(t) - defer ptesting.CleanupRedis(t, rdb, true, testName) + defer ptesting.CleanupRedis(t, rdb, false, "") + hook := &redisCommandHook{} + rdb.AddHook(hook) ctx := ptesting.NewTestContext(t) - s, err := NewStream(testName, rdb, - options.WithStreamLogger(pulse.ClueLogger(ctx)), - options.WithStreamTTL(time.Hour)) + stream, err := NewStream(t.Name(), rdb, options.WithStreamLogger(pulse.ClueLogger(ctx))) require.NoError(t, err) - sink, err := s.NewSink(ctx, "sink", + sink, err := stream.NewSink( + ctx, + "sink", options.WithSinkStartAtOldest(), - options.WithSinkBlockDuration(testBlockDuration)) + options.WithSinkBlockDuration(testBlockDuration), + ) require.NoError(t, err) - defer cleanupSink(t, ctx, s, sink) + defer cleanupSink(t, ctx, stream, sink) + events := sink.Subscribe() - _, err = s.Add(ctx, "event", []byte("payload")) - require.NoError(t, err) - require.NoError(t, rdb.Persist(ctx, s.key).Err()) - require.Equal(t, time.Duration(-1), rdb.PTTL(ctx, s.key).Val()) + for _, name := range []string{"first", "pending", "out-of-order"} { + _, err := stream.Add(ctx, name, []byte(name)) + require.NoError(t, err) + } + first := receiveSinkEvent(t, events) + pending := receiveSinkEvent(t, events) + outOfOrder := receiveSinkEvent(t, events) + require.NoError(t, sink.Ack(ctx, first)) + require.NoError(t, sink.Ack(ctx, outOfOrder)) - created, _, err := ensureConsumerGroup(ctx, s, "sink", "$", false) + cursor, err := rdb.HGet(ctx, recoveryCursorKey(stream), sink.Name).Result() require.NoError(t, err) - assert.False(t, created, "group must already exist") - assert.Positive(t, rdb.PTTL(ctx, s.key).Val(), "stream TTL must be restored on BUSYGROUP") - assert.Equal(t, time.Duration(-1), rdb.PTTL(ctx, cursorsKey(s.key)).Val(), - "recovery cursors must outlive the event TTL") + require.Equal(t, first.ID, cursor) + + readsBeforeDestroy := hook.xreadGroups.Load() + require.Eventually(t, func() bool { return hook.xreadGroups.Load() > readsBeforeDestroy }, max, delay) + sink.lock.Lock() + destroyed, destroyErr := rdb.XGroupDestroy(ctx, stream.key, sink.Name).Result() + _, gapErr := stream.Add(ctx, "gap", []byte("gap")) + sink.lock.Unlock() + require.NoError(t, destroyErr) + require.EqualValues(t, 1, destroyed) + require.NoError(t, gapErr) + + recovered := []*Event{ + receiveSinkEvent(t, events), + receiveSinkEvent(t, events), + receiveSinkEvent(t, events), + } + assert.Equal(t, []string{"pending", "out-of-order", "gap"}, eventNames(recovered)) + assert.Equal(t, pending.ID, recovered[0].ID) + for _, event := range recovered { + require.NoError(t, sink.Ack(ctx, event)) + } } -// TestSinkRecoveryAfterEventStreamExpires verifies that when the Redis key -// backing a TTL stream expires (deleting the consumer group with it), the -// sink recreates the group from the durable recovery cursor and delivers -// events published afterwards. -func TestSinkRecoveryAfterEventStreamExpires(t *testing.T) { - testName := strings.Replace(t.Name(), "/", "_", -1) +func TestSinkRecoveryCursorIsSharedAcrossReplicas(t *testing.T) { rdb := ptesting.NewRedisClient(t) - defer ptesting.CleanupRedis(t, rdb, true, testName) + defer ptesting.CleanupRedis(t, rdb, false, "") ctx := ptesting.NewTestContext(t) - s, err := NewStream(testName, rdb, - options.WithStreamLogger(pulse.ClueLogger(ctx)), - options.WithStreamTTL(500*time.Millisecond)) + stream, err := NewStream(t.Name(), rdb, options.WithStreamLogger(pulse.ClueLogger(ctx))) require.NoError(t, err) - sink, err := s.NewSink(ctx, "sink", - options.WithSinkStartAtOldest(), - options.WithSinkBlockDuration(testBlockDuration)) + sink1, err := stream.NewSink(ctx, "sink", options.WithSinkStartAtOldest(), options.WithSinkBlockDuration(testBlockDuration)) + require.NoError(t, err) + sink2, err := stream.NewSink(ctx, "sink", options.WithSinkStartAtOldest(), options.WithSinkBlockDuration(testBlockDuration)) require.NoError(t, err) - defer cleanupSink(t, ctx, s, sink) + defer cleanupSink(t, ctx, stream, sink2) + events1 := sink1.Subscribe() + events2 := sink2.Subscribe() - c := sink.Subscribe() - _, err = s.Add(ctx, "event", []byte("payload")) + eventID, err := stream.Add(ctx, "shared", []byte("shared")) require.NoError(t, err) - require.NoError(t, sink.Ack(ctx, receiveEvent(t, c))) + select { + case event := <-events1: + require.NoError(t, sink1.Ack(ctx, event)) + case event := <-events2: + require.NoError(t, sink2.Ack(ctx, event)) + case <-time.After(max): + t.Fatal("timed out waiting for shared event") + } + cursor, err := rdb.HGet(ctx, recoveryCursorKey(stream), sink1.Name).Result() + require.NoError(t, err) + require.Equal(t, eventID, cursor) - // Wait for the stream key (and with it the consumer group) to expire. + require.NoError(t, sink1.Close(ctx)) + require.True(t, sink1.IsClosed()) + destroyed, err := rdb.XGroupDestroy(ctx, stream.key, sink2.Name).Result() + require.NoError(t, err) + require.EqualValues(t, 1, destroyed) require.Eventually(t, func() bool { - return rdb.Exists(ctx, s.key).Val() == 0 - }, 2*time.Second, delay) + return consumerGroupExists(ctx, rdb, stream.key, sink2.Name) + }, max, delay) - futureID, err := s.Add(ctx, "future", []byte("payload")) + _, err = stream.Add(ctx, "future", []byte("future")) require.NoError(t, err) - ev := receiveEvent(t, c) - assert.Equal(t, futureID, ev.ID) - require.NoError(t, sink.Ack(ctx, ev)) + future := receiveSinkEvent(t, events2) + require.Equal(t, "future", future.EventName) + require.NoError(t, sink2.Ack(ctx, future)) } -// TestSinkCloseCancelsBlockingRedisRead verifies that Close cancels -// sink-owned Redis I/O so a blocked XREADGROUP cannot stall shutdown. func TestSinkCloseCancelsBlockingRedisRead(t *testing.T) { - testName := strings.Replace(t.Name(), "/", "_", -1) rdb := ptesting.NewRedisClient(t) - defer ptesting.CleanupRedis(t, rdb, true, testName) + defer ptesting.CleanupRedis(t, rdb, false, "") + hook := &redisCommandHook{ + readStarted: make(chan struct{}), + } + hook.blockRead.Store(true) + rdb.AddHook(hook) ctx := ptesting.NewTestContext(t) - s, err := NewStream(testName, rdb, options.WithStreamLogger(pulse.ClueLogger(ctx))) + stream, err := NewStream(t.Name(), rdb, options.WithStreamLogger(pulse.ClueLogger(ctx))) require.NoError(t, err) - hook := newRedisCommandHook() - rdb.AddHook(hook) - sink, err := s.NewSink(ctx, "sink", - options.WithSinkBlockDuration(time.Hour)) // would block Close without cancellation + sink, err := stream.NewSink(ctx, "sink", options.WithSinkBlockDuration(testBlockDuration)) require.NoError(t, err) - hook.blockRead.Store(true) select { case <-hook.readStarted: case <-time.After(max): - t.Fatal("read loop never blocked on XREADGROUP") + t.Fatal("sink never entered blocking Redis read") } - - done := make(chan struct{}) + closed := make(chan struct{}) go func() { - sink.Close(ctx) - close(done) + require.NoError(t, sink.Close(ctx)) + close(closed) }() select { - case <-done: - case <-time.After(2 * time.Second): - t.Fatal("Close did not cancel the blocked Redis read") + case <-closed: + case <-time.After(max): + t.Fatal("Close did not cancel blocking Redis I/O") } - assert.True(t, sink.IsClosed()) - hook.blockRead.Store(false) - require.NoError(t, s.Destroy(ctx)) + require.True(t, sink.IsClosed()) + require.NoError(t, stream.Destroy(ctx)) } -// TestSinkRejectsStreamMutationAfterClose verifies AddStream and RemoveStream -// fail with ErrSinkClosed once Close was called. -func TestSinkRejectsStreamMutationAfterClose(t *testing.T) { - testName := strings.Replace(t.Name(), "/", "_", -1) +func TestSinkCloseRetriesDistributedDetach(t *testing.T) { rdb := ptesting.NewRedisClient(t) - defer ptesting.CleanupRedis(t, rdb, true, testName) + defer ptesting.CleanupRedis(t, rdb, false, "") + failure := errors.New("injected close detach failure") + hook := &redisCommandHook{failure: failure} + rdb.AddHook(hook) ctx := ptesting.NewTestContext(t) - s, err := NewStream(testName, rdb, options.WithStreamLogger(pulse.ClueLogger(ctx))) + stream, err := NewStream(t.Name(), rdb, options.WithStreamLogger(pulse.ClueLogger(ctx))) require.NoError(t, err) - sink, err := s.NewSink(ctx, "sink", options.WithSinkBlockDuration(testBlockDuration)) + sink, err := stream.NewSink(ctx, "sink", options.WithSinkBlockDuration(testBlockDuration)) require.NoError(t, err) - sink.Close(ctx) - require.True(t, sink.IsClosed()) - s2, err := NewStream(testName+"2", rdb, options.WithStreamLogger(pulse.ClueLogger(ctx))) - require.NoError(t, err) - assert.ErrorIs(t, sink.AddStream(ctx, s2), ErrSinkClosed) - assert.ErrorIs(t, sink.RemoveStream(ctx, s), ErrSinkClosed) - require.NoError(t, s.Destroy(ctx)) + hook.failMembershipKey = consumersMapContentKey(stream) + hook.failRemove.Store(true) + err = sink.Close(ctx) + require.ErrorIs(t, err, failure) + require.False(t, sink.IsClosed()) + require.Contains(t, sink.streams, stream.key) + hook.failRemove.Store(false) + + require.NoError(t, sink.Close(ctx)) + require.True(t, sink.IsClosed()) + require.Empty(t, sink.streams) + require.NoError(t, stream.Destroy(ctx)) } -// TestSinkAddStreamRollsBackPartialFailure verifies that a failed AddStream -// compensates the consumer group and cursor it created so no dangling -// ownership state survives, and that a subsequent AddStream succeeds. -func TestSinkAddStreamRollsBackPartialFailure(t *testing.T) { - testName := strings.Replace(t.Name(), "/", "_", -1) +func TestSinkRejectsStreamMutationAfterClose(t *testing.T) { rdb := ptesting.NewRedisClient(t) - defer ptesting.CleanupRedis(t, rdb, true, testName) + defer ptesting.CleanupRedis(t, rdb, false, "") ctx := ptesting.NewTestContext(t) - s, err := NewStream(testName, rdb, options.WithStreamLogger(pulse.ClueLogger(ctx))) + stream, err := NewStream(t.Name(), rdb, options.WithStreamLogger(pulse.ClueLogger(ctx))) require.NoError(t, err) - s2, err := NewStream(testName+"2", rdb, options.WithStreamLogger(pulse.ClueLogger(ctx))) + added, err := NewStream(t.Name()+"-added", rdb, options.WithStreamLogger(pulse.ClueLogger(ctx))) require.NoError(t, err) - hook := newRedisCommandHook() - rdb.AddHook(hook) - sink, err := s.NewSink(ctx, "sink", - options.WithSinkStartAtOldest(), - options.WithSinkBlockDuration(testBlockDuration)) + defer func() { require.ErrorIs(t, added.Destroy(ctx), ErrStreamNotFound) }() + sink, err := stream.NewSink(ctx, "sink", options.WithSinkBlockDuration(testBlockDuration)) require.NoError(t, err) - defer cleanupSink(t, ctx, s, sink) - defer func() { assert.NoError(t, s2.Destroy(ctx)) }() - - hook.failScript(registerConsumerScript.Hash(), s2.key) - require.Error(t, sink.AddStream(ctx, s2)) - hook.clearFailure() - sink.lock.Lock() - _, owned := sink.streams[s2.key] - sink.lock.Unlock() - assert.False(t, owned, "failed AddStream must not leave the stream owned") - assert.Zero(t, rdb.Exists(ctx, cursorsKey(s2.key)).Val(), "compensation must delete the cursor") - groups, err := rdb.XInfoGroups(ctx, s2.key).Result() - require.NoError(t, err) - assert.Empty(t, groups, "compensation must delete the consumer group") - - // AddStream succeeds once the failure clears. - require.NoError(t, sink.AddStream(ctx, s2)) - c := sink.Subscribe() - _, err = s2.Add(ctx, "event", []byte("payload")) - require.NoError(t, err) - require.NoError(t, sink.Ack(ctx, receiveEvent(t, c))) + require.NoError(t, sink.Close(ctx)) + require.ErrorIs(t, sink.AddStream(ctx, added), ErrSinkClosed) + require.ErrorIs(t, sink.RemoveStream(ctx, stream), ErrSinkClosed) + require.NoError(t, stream.Destroy(ctx)) } -// TestSinkConsumerRotationRegistersEveryStreamOrRollsBack verifies that -// replacement consumer creation is failure-atomic across all sink streams: -// when registration fails for one stream the registrations already made are -// detached so ownership state never diverges. func TestSinkConsumerRotationRegistersEveryStreamOrRollsBack(t *testing.T) { - testName := strings.Replace(t.Name(), "/", "_", -1) rdb := ptesting.NewRedisClient(t) - defer ptesting.CleanupRedis(t, rdb, true, testName) + defer ptesting.CleanupRedis(t, rdb, false, "") + failure := errors.New("injected replacement consumer failure") + hook := &redisCommandHook{failure: failure} + rdb.AddHook(hook) ctx := ptesting.NewTestContext(t) - s, err := NewStream(testName, rdb, options.WithStreamLogger(pulse.ClueLogger(ctx))) + mainStream, err := NewStream(t.Name(), rdb, options.WithStreamLogger(pulse.ClueLogger(ctx))) require.NoError(t, err) - s2, err := NewStream(testName+"2", rdb, options.WithStreamLogger(pulse.ClueLogger(ctx))) + addedStream, err := NewStream(t.Name()+"-added", rdb, options.WithStreamLogger(pulse.ClueLogger(ctx))) require.NoError(t, err) - hook := newRedisCommandHook() - rdb.AddHook(hook) - sink, err := s.NewSink(ctx, "sink", - options.WithSinkStartAtOldest(), - options.WithSinkBlockDuration(testBlockDuration)) + defer func() { require.NoError(t, addedStream.Destroy(ctx)) }() + sink, err := mainStream.NewSink(ctx, "sink", options.WithSinkBlockDuration(testBlockDuration)) require.NoError(t, err) - require.NoError(t, sink.AddStream(ctx, s2)) - defer cleanupSink(t, ctx, s, sink) - defer func() { assert.NoError(t, s2.Destroy(ctx)) }() + defer cleanupSink(t, ctx, mainStream, sink) + require.NoError(t, sink.AddStream(ctx, addedStream)) original := sink.consumer - hook.failScript(registerConsumerScript.Hash(), s2.key) sink.lock.Lock() + hook.failCreateConsumerKey = addedStream.key + hook.failCreateConsumer.Store(true) _, err = sink.newConsumer(ctx) + hook.failCreateConsumer.Store(false) sink.lock.Unlock() - require.Error(t, err) - hook.clearFailure() - - for _, stream := range []*Stream{s, s2} { - assert.Equal(t, []string{original}, memberConsumers(t, ctx, rdb, stream, "sink"), - "membership of %s must be unchanged after rollback", stream.Name) - assert.Equal(t, []string{original}, groupConsumers(t, ctx, rdb, stream, "sink"), - "consumer group of %s must be unchanged after rollback", stream.Name) + require.ErrorIs(t, err, failure) + require.Equal(t, original, sink.consumer) + for _, stream := range []*Stream{mainStream, addedStream} { + require.Eventually(t, func() bool { + members, ok := sink.streams[stream.key].consumers.GetValues(sink.Name) + return ok && assert.ObjectsAreEqual([]string{original}, members) + }, max, delay) + consumers, err := rdb.XInfoConsumers(ctx, stream.key, sink.Name).Result() + require.NoError(t, err) + require.Len(t, consumers, 1) + require.Equal(t, original, consumers[0].Name) } - // Rotation succeeds once the failure clears and registers both streams. sink.lock.Lock() replacement, err := sink.newConsumer(ctx) + if err == nil { + sink.consumer = replacement + } sink.lock.Unlock() require.NoError(t, err) - for _, stream := range []*Stream{s, s2} { - assert.ElementsMatch(t, []string{original, replacement}, memberConsumers(t, ctx, rdb, stream, "sink")) + require.NotEqual(t, original, replacement) + for _, stream := range []*Stream{mainStream, addedStream} { + members, ok := sink.streams[stream.key].consumers.GetValues(sink.Name) + require.True(t, ok) + require.Contains(t, members, replacement) + consumers, err := rdb.XInfoConsumers(ctx, stream.key, sink.Name).Result() + require.NoError(t, err) + assert.Contains(t, consumerNames(consumers), replacement) } } -// TestDestroyFencesSinkMetadataWrites is the P1-A regression test: once -// Stream.Destroy runs, a live sink (setup paths, replacement-consumer -// creation, keepalive-driven loops, lease work) must not resurrect any -// stream-scoped metadata. -func TestDestroyFencesSinkMetadataWrites(t *testing.T) { - testName := strings.Replace(t.Name(), "/", "_", -1) - var origCheckIdlePeriod time.Duration - origCheckIdlePeriod, checkIdlePeriod = checkIdlePeriod, testCheckIdlePeriod - defer func() { checkIdlePeriod = origCheckIdlePeriod }() +func TestSinkStreamMutationRollback(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + failure := errors.New("injected ownership failure") + hook := &redisCommandHook{failure: failure} + rdb.AddHook(hook) + ctx := ptesting.NewTestContext(t) + mainStream, err := NewStream(t.Name(), rdb, options.WithStreamLogger(pulse.ClueLogger(ctx))) + require.NoError(t, err) + addedStream, err := NewStream(t.Name()+"-added", rdb, options.WithStreamLogger(pulse.ClueLogger(ctx))) + require.NoError(t, err) + defer func() { require.NoError(t, addedStream.Destroy(ctx)) }() + sink, err := mainStream.NewSink(ctx, "sink", options.WithSinkBlockDuration(testBlockDuration)) + require.NoError(t, err) + defer cleanupSink(t, ctx, mainStream, sink) + + hook.failCreateConsumer.Store(true) + hook.failCreateConsumerKey = addedStream.key + err = sink.AddStream(ctx, addedStream) + hook.failCreateConsumer.Store(false) + require.ErrorIs(t, err, failure) + assert.False(t, sinkOwnsStream(sink, addedStream)) + assert.NotContains(t, sink.streams, addedStream.key) + assert.True(t, consumerGroupExists(ctx, rdb, addedStream.key, sink.Name)) + assert.True(t, rdb.HExists(ctx, recoveryCursorKey(addedStream), sink.Name).Val()) + + consumerMap, err := rmap.Join(ctx, consumersMapName(addedStream), rdb) + require.NoError(t, err) + assert.NotContains(t, consumerMap.Map(), sink.Name) + consumerMap.Close() + + require.NoError(t, sink.AddStream(ctx, addedStream)) + hook.failMembershipKey = consumersMapContentKey(addedStream) + hook.failRemove.Store(true) + err = sink.RemoveStream(ctx, addedStream) + hook.failRemove.Store(false) + require.ErrorIs(t, err, failure) + assert.True(t, sinkOwnsStream(sink, addedStream)) + assert.Contains(t, sink.streams, addedStream.key) + assert.True(t, consumerGroupExists(ctx, rdb, addedStream.key, sink.Name)) + + require.NoError(t, sink.RemoveStream(ctx, addedStream)) + assert.False(t, sinkOwnsStream(sink, addedStream)) + assert.True(t, consumerGroupExists(ctx, rdb, addedStream.key, sink.Name)) + assert.True(t, rdb.HExists(ctx, recoveryCursorKey(addedStream), sink.Name).Val()) +} +func TestSinkRecoveryMetadataOutlivesEventTTL(t *testing.T) { rdb := ptesting.NewRedisClient(t) - defer ptesting.CleanupRedis(t, rdb, true, testName) + defer ptesting.CleanupRedis(t, rdb, false, "") ctx := ptesting.NewTestContext(t) - s, err := NewStream(testName, rdb, options.WithStreamLogger(pulse.ClueLogger(ctx))) + stream, err := NewStream( + t.Name(), + rdb, + options.WithStreamTTL(5*time.Second), + options.WithStreamLogger(pulse.ClueLogger(ctx)), + ) require.NoError(t, err) - sink, err := s.NewSink(ctx, "sink", - options.WithSinkStartAtOldest(), - options.WithSinkBlockDuration(testBlockDuration), - options.WithSinkAckGracePeriod(testAckDuration)) - require.NoError(t, err) - defer sink.Close(ctx) - - c := sink.Subscribe() - _, err = s.Add(ctx, "event", []byte("payload")) - require.NoError(t, err) - require.NoError(t, sink.Ack(ctx, receiveEvent(t, c))) - require.Equal(t, int64(1), rdb.Exists(ctx, cursorsKey(s.key)).Val()) - - // Destroy the stream while the sink keepalive and idle-check loops are - // due to run. - require.NoError(t, s.Destroy(ctx)) - - // Every fenced metadata write must fail with ErrStreamDestroyed. - assert.ErrorIs(t, registerSinkConsumer(ctx, s, "sink", "ghost"), ErrStreamDestroyed) - _, _, err = ensureConsumerGroup(ctx, s, "sink", "$", false) - assert.ErrorIs(t, err, ErrStreamDestroyed) - _, _, err = acquireSinkLease(ctx, s, "sink", "ghost-owner", 1000) - assert.ErrorIs(t, err, ErrStreamDestroyed) - - // The sink read loop observes NOGROUP, fails recovery with - // ErrStreamDestroyed, and drops the stream instead of resurrecting it. - assert.Eventually(t, func() bool { - sink.lock.Lock() - defer sink.lock.Unlock() - return len(sink.streams) == 0 - }, max, delay, "sink must drop the destroyed stream") - - // Let the periodic keepalive and idle-check loops tick several times, - // then verify no stream-scoped metadata was recreated. - time.Sleep(5 * testCheckIdlePeriod) - assert.Zero(t, rdb.Exists(ctx, s.key).Val(), "event stream must stay deleted") - assert.Zero(t, rdb.Exists(ctx, cursorsKey(s.key)).Val(), "recovery cursor must stay deleted") - assert.Zero(t, rdb.Exists(ctx, leaseKey(s.key, "sink")).Val(), "lease must stay deleted") - content, err := rdb.HGetAll(ctx, membershipContentKey(s.Name)).Result() - require.NoError(t, err) - assert.Equal(t, "destroy", content["=kind"], "membership map must remain a destroy tombstone") - assert.NotContains(t, content, "sink", "membership must not be resurrected") - assert.Equal(t, "destroyed", rdb.HGet(ctx, lifecycleKey(s.key), "state").Val()) + sink, err := stream.NewSink(ctx, "sink", options.WithSinkBlockDuration(testBlockDuration)) + require.NoError(t, err) + defer cleanupSink(t, ctx, stream, sink) + + require.NoError(t, rdb.Persist(ctx, stream.key).Err()) + require.NoError(t, rdb.Persist(ctx, recoveryCursorKey(stream)).Err()) + require.Equal(t, time.Duration(-1), rdb.PTTL(ctx, stream.key).Val()) + require.Equal(t, time.Duration(-1), rdb.PTTL(ctx, recoveryCursorKey(stream)).Val()) + sink.lock.Lock() + err = sink.recoverConsumerGroups(ctx) + sink.lock.Unlock() + require.NoError(t, err) + assert.Greater(t, rdb.PTTL(ctx, stream.key).Val(), time.Duration(0)) + assert.Equal(t, time.Duration(-1), rdb.PTTL(ctx, recoveryCursorKey(stream)).Val()) } -// TestDispatchLeavesRemovedStreamEventsPending verifies that a batch read for -// a stream removed from this sink concurrently with the read is left pending -// for the surviving group members instead of being acknowledged undelivered, -// which would permanently drop the events for the whole group. -func TestDispatchLeavesRemovedStreamEventsPending(t *testing.T) { - testName := strings.Replace(t.Name(), "/", "_", -1) +func TestSinkAcknowledgesBatchFromConcurrentlyRemovedSnapshot(t *testing.T) { rdb := ptesting.NewRedisClient(t) - defer ptesting.CleanupRedis(t, rdb, true, testName) + defer ptesting.CleanupRedis(t, rdb, false, "") + hook := &redisCommandHook{} + rdb.AddHook(hook) ctx := ptesting.NewTestContext(t) - s, err := NewStream(testName, rdb, options.WithStreamLogger(pulse.ClueLogger(ctx))) + main, err := NewStream(t.Name()+"-main", rdb) + require.NoError(t, err) + removed, err := NewStream(t.Name()+"-removed", rdb) + require.NoError(t, err) + sink, err := main.NewSink(ctx, "sink", options.WithSinkBlockDuration(time.Second)) require.NoError(t, err) - removed, err := s.NewSink(ctx, "sink", + require.NoError(t, sink.AddStream(ctx, removed, options.WithAddStreamStartAtOldest())) + events := sink.Subscribe() + reads := hook.xreadGroups.Load() + require.Eventually(t, func() bool { + return hook.xreadGroups.Load() > reads + }, max, delay) + + require.NoError(t, sink.RemoveStream(ctx, removed)) + eventID, err := removed.Add(ctx, "removed", []byte("payload")) + require.NoError(t, err) + require.Eventually(t, func() bool { + cursor := rdb.HGet(ctx, recoveryCursorKey(removed), sink.Name).Val() + pending, pendingErr := rdb.XPending(ctx, removed.key, sink.Name).Result() + return pendingErr == nil && pending.Count == 0 && cursor == eventID + }, max, delay) + select { + case event := <-events: + require.NotEqual(t, eventID, event.ID) + case <-time.After(2 * testBlockDuration): + } + + require.NoError(t, sink.Close(ctx)) + require.NoError(t, main.Destroy(ctx)) + require.NoError(t, removed.Destroy(ctx)) +} + +func TestSinkRecoveryAfterEventStreamExpiresDeliversNewEvents(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + stream, err := NewStream(t.Name(), rdb, options.WithStreamTTL(2*time.Second)) + require.NoError(t, err) + sink, err := stream.NewSink( + ctx, + "sink", options.WithSinkStartAtOldest(), - options.WithSinkBlockDuration(testBlockDuration)) + options.WithSinkBlockDuration(testBlockDuration), + ) + require.NoError(t, err) + events := sink.Subscribe() + _, err = stream.Add(ctx, "old", []byte("payload")) require.NoError(t, err) - defer removed.Close(ctx) - s2, err := NewStream(testName, ptesting.NewRedisClient(t), options.WithStreamLogger(pulse.ClueLogger(ctx))) + require.NoError(t, sink.Ack(ctx, receiveSinkEvent(t, events))) + require.NoError(t, sink.Close(ctx)) + require.NoError(t, rdb.PExpire(ctx, stream.key, 10*time.Millisecond).Err()) + require.Eventually(t, func() bool { + return rdb.Exists(ctx, stream.key).Val() == 0 + }, max, delay) + require.Equal(t, time.Duration(-1), rdb.PTTL(ctx, recoveryCursorKey(stream)).Val()) + + freshID, err := stream.Add(ctx, "fresh", []byte("payload")) require.NoError(t, err) - survivor, err := s2.NewSink(ctx, "sink", + recovered, err := stream.NewSink( + ctx, + "sink", options.WithSinkStartAtOldest(), - options.WithSinkBlockDuration(testBlockDuration)) - require.NoError(t, err) - defer cleanupSink(t, ctx, s2, survivor) - - // Drop the stream from one sink; the group survives through the other - // member. Wait out reads issued before the removal so the event below is - // deterministically delivered to the survivor's consumer PEL, unacked. - c := survivor.Subscribe() - require.NoError(t, removed.RemoveStream(ctx, s)) - time.Sleep(3 * testBlockDuration) - id, err := s.Add(ctx, "event", []byte("payload")) - require.NoError(t, err) - ev := receiveEvent(t, c) - require.Equal(t, id, ev.ID) - - // Replay the racing batch against the sink that no longer owns the - // stream: dispatch must not settle the group's pending entry. - require.NoError(t, removed.dispatch([]redis.XStream{{ - Stream: s.key, - Messages: []redis.XMessage{{ID: id, Values: map[string]any{nameKey: "event", payloadKey: "payload"}}}, - }})) - pending, err := rdb.XPending(ctx, s.key, "sink").Result() - require.NoError(t, err) - assert.Equal(t, int64(1), pending.Count, "unowned batch must stay pending for surviving members") - require.NoError(t, survivor.Ack(ctx, ev)) + options.WithSinkBlockDuration(testBlockDuration), + ) + require.NoError(t, err) + fresh := receiveSinkEvent(t, recovered.Subscribe()) + require.Equal(t, freshID, fresh.ID) + require.NoError(t, recovered.Ack(ctx, fresh)) + require.NoError(t, recovered.Close(ctx)) + require.NoError(t, stream.Destroy(ctx)) } -// TestSinkAcknowledgesFilteredEvents verifies that events dropped by the sink -// topic filter are acknowledged so they cannot hold back the recovery cursor. -func TestSinkAcknowledgesFilteredEvents(t *testing.T) { - testName := strings.Replace(t.Name(), "/", "_", -1) +func TestSinkNoAckUsesPendingEntryListBeforeDelivery(t *testing.T) { rdb := ptesting.NewRedisClient(t) - defer ptesting.CleanupRedis(t, rdb, true, testName) + defer ptesting.CleanupRedis(t, rdb, false, "") + hook := &redisCommandHook{} + rdb.AddHook(hook) ctx := ptesting.NewTestContext(t) - s, err := NewStream(testName, rdb, options.WithStreamLogger(pulse.ClueLogger(ctx))) + stream, err := NewStream(t.Name(), rdb, options.WithStreamLogger(pulse.ClueLogger(ctx))) require.NoError(t, err) - sink, err := s.NewSink(ctx, "sink", + sink, err := stream.NewSink( + ctx, + "sink", options.WithSinkStartAtOldest(), + options.WithSinkNoAck(), options.WithSinkBlockDuration(testBlockDuration), - options.WithSinkTopic("keep")) + ) require.NoError(t, err) - defer cleanupSink(t, ctx, s, sink) + defer cleanupSink(t, ctx, stream, sink) + events := sink.Subscribe() - c := sink.Subscribe() - _, err = s.Add(ctx, "dropped", []byte("payload"), options.WithTopic("drop")) + eventID, err := stream.Add(ctx, "at-most-once", []byte("payload")) require.NoError(t, err) - keepID, err := s.Add(ctx, "kept", []byte("payload"), options.WithTopic("keep")) + event := receiveSinkEvent(t, events) + require.Equal(t, eventID, event.ID) + require.False(t, hook.usedNoAck.Load(), "sink sent Redis NOACK") + pending, err := rdb.XPending(ctx, stream.key, sink.Name).Result() require.NoError(t, err) + require.Zero(t, pending.Count) + cursor, err := rdb.HGet(ctx, recoveryCursorKey(stream), sink.Name).Result() + require.NoError(t, err) + require.Equal(t, eventID, cursor) - ev := receiveEvent(t, c) - assert.Equal(t, keepID, ev.ID) - require.NoError(t, sink.Ack(ctx, ev)) - assert.Equal(t, keepID, recoveryCursor(t, ctx, rdb, s, "sink")) - pending, err := rdb.XPending(ctx, s.key, "sink").Result() + require.NoError(t, rdb.XGroupDestroy(ctx, stream.key, sink.Name).Err()) + require.Eventually(t, func() bool { + return consumerGroupExists(ctx, rdb, stream.key, sink.Name) + }, max, delay) + _, err = stream.Add(ctx, "future", []byte("future")) require.NoError(t, err) - assert.Zero(t, pending.Count, "filtered events must be settled") + require.Equal(t, "future", receiveSinkEvent(t, events).EventName) } -// TestSinkNoAckAdvancesRecoveryCursor verifies that NoAck sinks advance the -// recovery cursor on delivery so recovery never replays delivered events. -func TestSinkNoAckAdvancesRecoveryCursor(t *testing.T) { - testName := strings.Replace(t.Name(), "/", "_", -1) +func TestEventAckerAdvancesRecoveryCursor(t *testing.T) { rdb := ptesting.NewRedisClient(t) - defer ptesting.CleanupRedis(t, rdb, true, testName) + defer ptesting.CleanupRedis(t, rdb, false, "") ctx := ptesting.NewTestContext(t) - s, err := NewStream(testName, rdb, options.WithStreamLogger(pulse.ClueLogger(ctx))) + stream, err := NewStream(t.Name(), rdb, options.WithStreamLogger(pulse.ClueLogger(ctx))) require.NoError(t, err) - sink, err := s.NewSink(ctx, "sink", + sink, err := stream.NewSink( + ctx, + "sink", options.WithSinkStartAtOldest(), options.WithSinkBlockDuration(testBlockDuration), - options.WithSinkNoAck()) + ) require.NoError(t, err) - defer cleanupSink(t, ctx, s, sink) + defer cleanupSink(t, ctx, stream, sink) + events := sink.Subscribe() - c := sink.Subscribe() - id, err := s.Add(ctx, "event", []byte("payload")) + eventID, err := stream.Add(ctx, "direct-ack", []byte("payload")) require.NoError(t, err) - ev := receiveEvent(t, c) - assert.Equal(t, id, ev.ID) - assert.Equal(t, id, recoveryCursor(t, ctx, rdb, s, "sink")) - - require.NoError(t, rdb.XGroupDestroy(ctx, s.key, "sink").Err()) - futureID, err := s.Add(ctx, "future", []byte("payload")) + event := receiveSinkEvent(t, events) + acked, err := event.Acker.XAck(ctx, stream.key, sink.Name, event.ID).Result() require.NoError(t, err) - ev = receiveEvent(t, c) - assert.Equal(t, futureID, ev.ID, "recovery must resume after the delivered event") - select { - case ev := <-c: - t.Errorf("unexpected redelivery of event %s", ev.ID) - case <-time.After(4 * testBlockDuration): - } + require.EqualValues(t, 1, acked) + cursor, err := rdb.HGet(ctx, recoveryCursorKey(stream), sink.Name).Result() + require.NoError(t, err) + require.Equal(t, eventID, cursor) } -// TestEventAckerAdvancesRecoveryCursor verifies the cursor arithmetic of the -// recovery acker: the cursor is always the entry preceding the oldest pending -// event, and the group last-delivered-id once the PEL drains, even when -// events are acknowledged out of order. -func TestEventAckerAdvancesRecoveryCursor(t *testing.T) { - testName := strings.Replace(t.Name(), "/", "_", -1) +func TestEventAckerAcknowledgesMultipleIDs(t *testing.T) { rdb := ptesting.NewRedisClient(t) - defer ptesting.CleanupRedis(t, rdb, true, testName) + defer ptesting.CleanupRedis(t, rdb, false, "") ctx := ptesting.NewTestContext(t) - s, err := NewStream(testName, rdb, options.WithStreamLogger(pulse.ClueLogger(ctx))) + stream, err := NewStream(t.Name(), rdb, options.WithStreamLogger(pulse.ClueLogger(ctx))) require.NoError(t, err) - sink, err := s.NewSink(ctx, "sink", + sink, err := stream.NewSink( + ctx, + "sink", options.WithSinkStartAtOldest(), - options.WithSinkBlockDuration(testBlockDuration)) + options.WithSinkBlockDuration(testBlockDuration), + ) require.NoError(t, err) - defer cleanupSink(t, ctx, s, sink) - - c := sink.Subscribe() - ids := make([]string, 3) - for i := range ids { - ids[i], err = s.Add(ctx, fmt.Sprintf("event%d", i), []byte("payload")) + defer cleanupSink(t, ctx, stream, sink) + events := sink.Subscribe() + for _, name := range []string{"first", "second"} { + _, err := stream.Add(ctx, name, []byte(name)) require.NoError(t, err) } - events := make(map[string]*Event, 3) - for range ids { - ev := receiveEvent(t, c) - events[ev.ID] = ev - } + first := receiveSinkEvent(t, events) + second := receiveSinkEvent(t, events) - // Ack out of order: the middle event first. - require.NoError(t, sink.Ack(ctx, events[ids[1]])) - assert.Equal(t, "0-0", recoveryCursor(t, ctx, rdb, s, "sink"), - "oldest pending event is the first entry so nothing is durably settled") - require.NoError(t, sink.Ack(ctx, events[ids[0]])) - assert.Equal(t, ids[1], recoveryCursor(t, ctx, rdb, s, "sink"), - "cursor must jump past the contiguous acknowledged prefix") - require.NoError(t, sink.Ack(ctx, events[ids[2]])) - assert.Equal(t, ids[2], recoveryCursor(t, ctx, rdb, s, "sink"), - "cursor must reach last-delivered-id once the PEL drains") + acked, err := first.Acker.XAck(ctx, stream.key, sink.Name, first.ID, second.ID).Result() + require.NoError(t, err) + require.EqualValues(t, 2, acked) + cursor, err := rdb.HGet(ctx, recoveryCursorKey(stream), sink.Name).Result() + require.NoError(t, err) + require.Equal(t, second.ID, cursor) } -// TestReadRetryJitterBounds verifies the retry backoff is jittered between -// half and full of the current backoff and doubles up to the cap. -func TestReadRetryJitterBounds(t *testing.T) { - var r readRetry - expected := minReadRetryBackoff - for range 10 { - d := r.next() - assert.GreaterOrEqual(t, d, expected/2) - assert.LessOrEqual(t, d, expected) - expected = 2 * expected - if expected > maxReadRetryBackoff { - expected = maxReadRetryBackoff - } +func TestSinkAckDelegatesToEventAcker(t *testing.T) { + ctx := context.Background() + acker := &recordingAcker{} + sink := &Sink{logger: pulse.NoopLogger()} + event := &Event{ + ID: "1-0", + StreamName: "stream", + SinkName: "sink", + Acker: acker, + streamKey: "pulse:stream:stream", } - r.reset() - d := r.next() - assert.GreaterOrEqual(t, d, minReadRetryBackoff/2) - assert.LessOrEqual(t, d, minReadRetryBackoff) -} -// receiveEvent reads one event from the channel without acknowledging it or -// fails the test after the standard timeout. -func receiveEvent(t *testing.T, c <-chan *Event) *Event { - t.Helper() - select { - case ev := <-c: - require.NotNil(t, ev) - return ev - case <-time.After(max): - t.Fatal("timeout waiting for event") - return nil - } + require.NoError(t, sink.Ack(ctx, event)) + require.Equal(t, event.streamKey, acker.streamKey) + require.Equal(t, event.SinkName, acker.group) + require.Equal(t, []string{event.ID}, acker.ids) } -// recoveryCursor returns the durable recovery cursor stored for the sink on -// the stream. -func recoveryCursor(t *testing.T, ctx context.Context, rdb *redis.Client, s *Stream, sink string) string { - t.Helper() - cursor, err := rdb.HGet(ctx, cursorsKey(s.key), sink).Result() +func TestRecoveryCursorsShareOneStreamHash(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + stream, err := NewStream(t.Name(), rdb, options.WithStreamLogger(pulse.ClueLogger(ctx))) require.NoError(t, err) - return cursor -} - -// memberConsumers returns the consumer names recorded for the sink in the -// stream membership map. -func memberConsumers(t *testing.T, ctx context.Context, rdb *redis.Client, s *Stream, sink string) []string { - t.Helper() - raw, err := rdb.HGet(ctx, membershipContentKey(s.Name), sink).Result() + sink1, err := stream.NewSink(ctx, "first", options.WithSinkBlockDuration(testBlockDuration)) require.NoError(t, err) - var names []string - require.NoError(t, json.Unmarshal([]byte(raw), &names)) - return names -} + sink2, err := stream.NewSink(ctx, "second", options.WithSinkBlockDuration(testBlockDuration)) + require.NoError(t, err) + defer cleanupSink(t, ctx, stream, sink1) + defer func() { require.NoError(t, sink2.Close(ctx)) }() -// groupConsumers returns the Redis consumer names of the sink group on the -// stream. -func groupConsumers(t *testing.T, ctx context.Context, rdb *redis.Client, s *Stream, sink string) []string { - t.Helper() - consumers, err := rdb.XInfoConsumers(ctx, s.key, sink).Result() + cursors, err := rdb.HGetAll(ctx, recoveryCursorKey(stream)).Result() require.NoError(t, err) - names := make([]string, len(consumers)) - for i, c := range consumers { - names[i] = c.Name - } - return names + require.Len(t, cursors, 2) + require.Contains(t, cursors, sink1.Name) + require.Contains(t, cursors, sink2.Name) + legacyKeys, err := rdb.Keys(ctx, stream.key+":sink:*:recovery").Result() + require.NoError(t, err) + require.Empty(t, legacyKeys) } -type ( - // redisCommandHook injects command-specific failures and blocking reads - // into the sink Redis client to prove cancellation, bounded retries, and - // failure-atomic ownership changes. - redisCommandHook struct { - // blockRead blocks XREADGROUP calls until their context is canceled. - blockRead atomic.Bool - // readStarted is closed the first time a read blocks. - readStarted chan struct{} - // started guards readStarted. - started sync.Once - // mu guards the failure configuration below. - mu sync.Mutex - // failHash is the script hash whose EVALSHA calls fail. - failHash string - // failKey restricts injected failures to invocations naming this key. - failKey string +func TestReadRetryJitterBounds(t *testing.T) { + low := readRetry{jitter: func(int64) int64 { return 0 }} + high := readRetry{jitter: func(n int64) int64 { return n - 1 }} + limit := readRetryInitialDelay + for range 10 { + assert.Equal(t, limit/2, low.nextDelay()) + assert.Equal(t, limit, high.nextDelay()) + limit = min(limit*2, readRetryMaxDelay) } -) - -// errInjected is the transport failure injected by redisCommandHook. -var errInjected = errors.New("injected redis failure") +} -// newRedisCommandHook returns a hook with no active failure. -func newRedisCommandHook() *redisCommandHook { - return &redisCommandHook{readStarted: make(chan struct{})} +// XAck records the Event acknowledgement invoked by Sink.Ack. +func (a *recordingAcker) XAck(ctx context.Context, streamKey, group string, ids ...string) *redis.IntCmd { + a.streamKey = streamKey + a.group = group + a.ids = append([]string(nil), ids...) + cmd := redis.NewIntCmd(ctx) + cmd.SetVal(int64(len(ids))) + return cmd } -// failScript makes EVALSHA calls of the script with the given hash fail when -// their arguments include key. -func (h *redisCommandHook) failScript(hash, key string) { - h.mu.Lock() - defer h.mu.Unlock() - h.failHash, h.failKey = hash, key +// receiveSinkEvent reads one event without acknowledging it. +func receiveSinkEvent(t *testing.T, events <-chan *Event) *Event { + t.Helper() + select { + case event := <-events: + require.NotNil(t, event) + return event + case <-time.After(max): + t.Fatal("timed out waiting for sink event") + return nil + } } -// clearFailure removes the active script failure. -func (h *redisCommandHook) clearFailure() { - h.mu.Lock() - defer h.mu.Unlock() - h.failHash, h.failKey = "", "" +// eventNames projects event names in delivery order. +func eventNames(events []*Event) []string { + names := make([]string, len(events)) + for i, event := range events { + names[i] = event.EventName + } + return names } -// DialHook preserves the client's normal Redis connection behavior. -func (h *redisCommandHook) DialHook(next redis.DialHook) redis.DialHook { - return func(ctx context.Context, network, addr string) (net.Conn, error) { - return next(ctx, network, addr) +// consumerNames projects Redis consumer info for membership assertions. +func consumerNames(consumers []redis.XInfoConsumer) []string { + names := make([]string, len(consumers)) + for i, consumer := range consumers { + names[i] = consumer.Name } + return names } -// ProcessHook blocks reads and injects script failures per the hook -// configuration. -func (h *redisCommandHook) ProcessHook(next redis.ProcessHook) redis.ProcessHook { - return func(ctx context.Context, cmd redis.Cmder) error { - switch cmd.Name() { - case "xreadgroup": - if h.blockRead.Load() { - h.started.Do(func() { close(h.readStarted) }) - <-ctx.Done() - return ctx.Err() - } - case "evalsha": - h.mu.Lock() - hash, key := h.failHash, h.failKey - h.mu.Unlock() - if hash != "" && len(cmd.Args()) > 1 && cmd.Args()[1] == hash { - for _, arg := range cmd.Args() { - if s, ok := arg.(string); ok && s == key { - return errInjected - } - } - } +// sinkOwnsStream reports local ownership under the sink lock. +func sinkOwnsStream(sink *Sink, stream *Stream) bool { + sink.lock.Lock() + defer sink.lock.Unlock() + for _, owned := range sink.streams { + if owned.stream == stream { + return true } - return next(ctx, cmd) } + return false } -// ProcessPipelineHook preserves pipeline behavior; the sink does not issue -// pipelines on the paths exercised by these tests. -func (h *redisCommandHook) ProcessPipelineHook(next redis.ProcessPipelineHook) redis.ProcessPipelineHook { - return func(ctx context.Context, cmds []redis.Cmder) error { - return next(ctx, cmds) - } +func TestDestroyedGenerationMetadataCannotBeRecreated(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + stream, err := NewStream(t.Name(), rdb, options.WithStreamLogger(pulse.ClueLogger(ctx))) + require.NoError(t, err) + sink, err := stream.NewSink(ctx, "sink", options.WithSinkBlockDuration(testBlockDuration)) + require.NoError(t, err) + defer func() { require.NoError(t, sink.Close(ctx)) }() + state := sink.streams[stream.key] + require.NotNil(t, state) + keepAliveKey := rmapContentKey(sinkKeepAliveMapName(stream, "sink")) + membershipKey := consumersMapContentKey(stream) + require.EqualValues(t, 1, rdb.Exists(ctx, keepAliveKey).Val()) + require.EqualValues(t, 1, rdb.Exists(ctx, membershipKey).Val()) + + require.NoError(t, stream.Destroy(ctx)) + require.EqualValues(t, 0, rdb.Exists(ctx, keepAliveKey).Val()) + require.EqualValues(t, 0, rdb.Exists(ctx, membershipKey).Val()) + + // Neither a periodic keep-alive tick nor a consumer registration may + // resurrect metadata for the destroyed generation. + err = setSinkKeepAlive(ctx, state, "sink", sink.consumer, time.Now().UnixNano()) + require.ErrorIs(t, err, ErrStreamDestroyed) + err = registerSinkConsumer(ctx, state, "sink", "ghost-consumer", time.Now().UnixNano()) + require.ErrorIs(t, err, ErrStreamDestroyed) + require.EqualValues(t, 0, rdb.Exists(ctx, keepAliveKey).Val()) + require.EqualValues(t, 0, rdb.Exists(ctx, membershipKey).Val()) } diff --git a/streaming/sink_test.go b/streaming/sink_test.go index 67e16e8..bfa8f3b 100644 --- a/streaming/sink_test.go +++ b/streaming/sink_test.go @@ -1,24 +1,596 @@ package streaming import ( + "context" + "errors" + "net" "strings" + "sync" + "sync/atomic" "testing" "time" + redis "github.com/redis/go-redis/v9" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "goa.design/pulse/pulse" + "goa.design/pulse/rmap" "goa.design/pulse/streaming/options" ptesting "goa.design/pulse/testing" ) +// redisCommandHook observes real Redis commands and can fail selected commands +// at the client boundary without replacing Redis in recovery tests. +type redisCommandHook struct { + xreadGroups atomic.Int64 + failRead atomic.Bool + failGroup atomic.Bool + failCreateConsumer atomic.Bool + failRemove atomic.Bool + usedNoAck atomic.Bool + blockRead atomic.Bool + // failMembershipKey selects the replicated membership map whose mutation + // should fail while failRemove is set. + failMembershipKey string + // failGroupKey selects the stream lifecycle key whose group Lua operation + // should fail while failGroup is set. + failGroupKey string + // failCreateConsumerKey narrows consumer creation failure to one stream. + failCreateConsumerKey string + readStarted chan struct{} + started sync.Once + failure error +} + var ( testCheckIdlePeriod = 50 * time.Millisecond testBlockDuration = 50 * time.Millisecond testAckDuration = 50 * time.Millisecond ) +func TestSinkClosingFencesSubscriptions(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + stream, err := NewStream(t.Name(), rdb) + require.NoError(t, err) + sink, err := stream.NewSink( + ctx, + "sink", + options.WithSinkStartAtOldest(), + options.WithSinkBlockDuration(testBlockDuration), + ) + require.NoError(t, err) + + start := make(chan struct{}) + subscriptions := make(chan (<-chan *Event), 64) + var wait sync.WaitGroup + for range 64 { + wait.Add(1) + go func() { + defer wait.Done() + <-start + subscriptions <- sink.Subscribe() + }() + } + close(start) + require.NoError(t, sink.Close(ctx)) + wait.Wait() + close(subscriptions) + for subscription := range subscriptions { + _, ok := <-subscription + require.False(t, ok) + } + closed := sink.Subscribe() + _, ok := <-closed + require.False(t, ok) + require.NoError(t, stream.Destroy(ctx)) +} + +func TestSinkRejectsRemovingFinalStream(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + stream, err := NewStream(t.Name(), rdb) + require.NoError(t, err) + sink, err := stream.NewSink(ctx, "sink", options.WithSinkBlockDuration(testBlockDuration)) + require.NoError(t, err) + + unattached, err := NewStream(t.Name()+"-unattached", rdb) + require.NoError(t, err) + cancelled, cancel := context.WithCancel(ctx) + cancel() + require.NoError(t, sink.RemoveStream(cancelled, unattached)) + require.Empty(t, unattached.Generation()) + require.ErrorIs(t, sink.RemoveStream(ctx, stream), ErrLastStream) + require.NoError(t, sink.Close(ctx)) + require.NoError(t, stream.Destroy(ctx)) +} + +func TestSinkReplicaConfigurationAndLeaseScope(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + first, err := NewStream(t.Name()+"-first", rdb) + require.NoError(t, err) + second, err := NewStream(t.Name()+"-second", rdb) + require.NoError(t, err) + opts := []options.Sink{ + options.WithSinkTopic("alarms"), + options.WithSinkNoAck(), + options.WithSinkAckGracePeriod(testAckDuration), + options.WithSinkBlockDuration(testBlockDuration), + } + firstReplica, err := first.NewSink(ctx, "shared", opts...) + require.NoError(t, err) + secondReplica, err := first.NewSink(ctx, "shared", opts...) + require.NoError(t, err) + unrelated, err := second.NewSink(ctx, "shared", opts...) + require.NoError(t, err) + + firstState := firstReplica.streams[first.key] + secondState := secondReplica.streams[first.key] + unrelatedState := unrelated.streams[second.key] + require.Equal(t, firstState.leaseKey, secondState.leaseKey) + require.NotEqual(t, firstState.leaseKey, unrelatedState.leaseKey) + require.Equal( + t, + firstState.keepAlives.Name, + secondState.keepAlives.Name, + ) + require.NotEqual( + t, + firstState.keepAlives.Name, + unrelatedState.keepAlives.Name, + ) + + _, err = first.NewSink( + ctx, + "shared", + options.WithSinkTopic("readings"), + options.WithSinkNoAck(), + options.WithSinkAckGracePeriod(testAckDuration), + options.WithSinkBlockDuration(testBlockDuration), + ) + require.ErrorContains(t, err, "configuration differs") + + require.NoError(t, firstReplica.Close(ctx)) + require.NoError(t, secondReplica.Close(ctx)) + require.NoError(t, unrelated.Close(ctx)) + firstConfigKey := sinkConfigurationKey(first, "shared") + firstKeepAliveKey := rmapContentKey(sinkKeepAliveMapName(first, "shared")) + firstRegistryKey := streamResourceRegistryKey(first) + require.NoError(t, first.Destroy(ctx)) + require.NoError(t, second.Destroy(ctx)) + require.EqualValues( + t, + 0, + rdb.Exists(ctx, firstConfigKey, firstKeepAliveKey, firstRegistryKey).Val(), + ) +} + +func TestCrossPrimarySinkReplicasCoordinatePerAttachedStream(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + first, err := NewStream(t.Name()+"-first", rdb) + require.NoError(t, err) + second, err := NewStream(t.Name()+"-second", rdb) + require.NoError(t, err) + third, err := NewStream(t.Name()+"-third", rdb) + require.NoError(t, err) + compatible := []options.Sink{ + options.WithSinkTopic("alarms"), + options.WithSinkStartAtOldest(), + options.WithSinkAckGracePeriod(testAckDuration), + options.WithSinkBlockDuration(testBlockDuration), + } + firstPrimary, err := first.NewSink(ctx, "shared", compatible...) + require.NoError(t, err) + secondPrimary, err := second.NewSink(ctx, "shared", compatible...) + require.NoError(t, err) + require.NoError(t, firstPrimary.AddStream(ctx, second, options.WithAddStreamStartAtOldest())) + require.NoError(t, secondPrimary.AddStream(ctx, first, options.WithAddStreamStartAtOldest())) + + require.Equal( + t, + firstPrimary.streams[first.key].keepAlives.Name, + secondPrimary.streams[first.key].keepAlives.Name, + ) + require.Equal( + t, + firstPrimary.streams[second.key].leaseKey, + secondPrimary.streams[second.key].leaseKey, + ) + require.NotEqual( + t, + firstPrimary.streams[second.key].leaseOwner, + secondPrimary.streams[second.key].leaseOwner, + ) + + incompatible, err := third.NewSink( + ctx, + "shared", + options.WithSinkTopic("readings"), + options.WithSinkStartAtOldest(), + options.WithSinkAckGracePeriod(testAckDuration), + options.WithSinkBlockDuration(testBlockDuration), + ) + require.NoError(t, err) + before, err := rdb.XInfoConsumers(ctx, second.key, "shared").Result() + require.NoError(t, err) + err = incompatible.AddStream(ctx, second, options.WithAddStreamStartAtOldest()) + require.ErrorContains(t, err, "configuration differs") + require.NotContains(t, incompatible.streams, second.key) + after, err := rdb.XInfoConsumers(ctx, second.key, "shared").Result() + require.NoError(t, err) + require.Len(t, after, len(before)) + + require.NoError(t, firstPrimary.Close(ctx)) + require.NoError(t, secondPrimary.Close(ctx)) + require.NoError(t, incompatible.Close(ctx)) + require.NoError(t, first.Destroy(ctx)) + require.NoError(t, second.Destroy(ctx)) + require.NoError(t, third.Destroy(ctx)) +} + +func TestSinkRecoveryLeaseFencesExpiredOwner(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + stream, err := NewStream(t.Name(), rdb) + require.NoError(t, err) + sink, err := stream.NewSink( + ctx, + "sink", + options.WithSinkAckGracePeriod(testAckDuration), + options.WithSinkBlockDuration(testBlockDuration), + ) + require.NoError(t, err) + state := sink.streams[stream.key] + duration := time.Second + require.NoError(t, rdb.HSet(ctx, state.leaseKey, "lease_until", "0").Err()) + first, acquired, err := acquireSinkRecoveryLease( + ctx, + stream, + state.leaseKey, + "first-owner", + duration, + ) + require.NoError(t, err) + require.True(t, acquired) + require.NoError(t, rdb.HSet(ctx, state.leaseKey, "lease_until", "0").Err()) + second, acquired, err := acquireSinkRecoveryLease( + ctx, + stream, + state.leaseKey, + "second-owner", + duration, + ) + require.NoError(t, err) + require.True(t, acquired) + require.Greater(t, second.fence, first.fence) + require.ErrorContains( + t, + renewSinkRecoveryLease(ctx, stream, state.leaseKey, first, duration), + "SINKLEASELOST", + ) + require.NoError(t, renewSinkRecoveryLease(ctx, stream, state.leaseKey, second, duration)) + + require.NoError(t, sink.Close(ctx)) + require.NoError(t, stream.Destroy(ctx)) +} + +func TestSinkRecoveryLeaseFencesAutoClaimAfterTakeover(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + stream, err := NewStream(t.Name(), rdb) + require.NoError(t, err) + _, err = stream.Add(ctx, "event", []byte("payload")) + require.NoError(t, err) + require.NoError(t, rdb.XGroupCreate(ctx, stream.key, "sink", "0").Err()) + claimed, err := rdb.XReadGroup(ctx, &redis.XReadGroupArgs{ + Group: "sink", + Consumer: "predecessor", + Streams: []string{stream.key, ">"}, + Count: 1, + }).Result() + require.NoError(t, err) + require.Len(t, claimed, 1) + sink, err := stream.NewSink( + ctx, + "sink", + options.WithSinkStartAtOldest(), + options.WithSinkAckGracePeriod(time.Hour), + options.WithSinkBlockDuration(testBlockDuration), + ) + require.NoError(t, err) + state := sink.streams[stream.key] + + require.NoError(t, rdb.HSet(ctx, state.leaseKey, "lease_until", "0").Err()) + first, acquired, err := acquireSinkRecoveryLease( + ctx, + stream, + state.leaseKey, + "first-owner", + 20*time.Millisecond, + ) + require.NoError(t, err) + require.True(t, acquired) + time.Sleep(30 * time.Millisecond) + second, acquired, err := acquireSinkRecoveryLease( + ctx, + stream, + state.leaseKey, + "second-owner", + time.Second, + ) + require.NoError(t, err) + require.True(t, acquired) + + require.NoError(t, rdb.ScriptFlush(ctx).Err()) + _, _, err = fencedAutoClaim( + ctx, + stream, + state.leaseKey, + first, + time.Second, + sink.Name, + sink.consumer, + 0, + "0-0", + 1, + ) + require.ErrorContains(t, err, "SINKLEASELOST") + pending, err := rdb.XPendingExt(ctx, &redis.XPendingExtArgs{ + Stream: stream.key, + Group: sink.Name, + Start: "-", + End: "+", + Count: 1, + }).Result() + require.NoError(t, err) + require.Equal(t, "predecessor", pending[0].Consumer) + + _, messages, err := fencedAutoClaim( + ctx, + stream, + state.leaseKey, + second, + time.Second, + sink.Name, + sink.consumer, + 0, + "0-0", + 1, + ) + require.NoError(t, err) + require.Len(t, messages, 1) + pending, err = rdb.XPendingExt(ctx, &redis.XPendingExtArgs{ + Stream: stream.key, + Group: sink.Name, + Start: "-", + End: "+", + Count: 1, + }).Result() + require.NoError(t, err) + require.Equal(t, sink.consumer, pending[0].Consumer) + + require.NoError(t, rdb.XGroupCreateConsumer(ctx, stream.key, sink.Name, "stale").Err()) + _, err = state.keepAlives.Set(ctx, "stale", "0") + require.NoError(t, err) + require.NoError(t, rdb.ScriptFlush(ctx).Err()) + _, _, err = cleanupStaleConsumers( + ctx, + stream, + state.leaseKey, + first, + time.Second, + sink.Name, + time.Millisecond, + ) + require.ErrorContains(t, err, "SINKLEASELOST") + consumers, err := rdb.XInfoConsumers(ctx, stream.key, sink.Name).Result() + require.NoError(t, err) + require.Contains(t, consumerNames(consumers), "stale") + + removed, malformed, err := cleanupStaleConsumers( + ctx, + stream, + state.leaseKey, + second, + time.Second, + sink.Name, + time.Millisecond, + ) + require.NoError(t, err) + require.Contains(t, removed, "stale") + require.Empty(t, malformed) + consumers, err = rdb.XInfoConsumers(ctx, stream.key, sink.Name).Result() + require.NoError(t, err) + require.NotContains(t, consumerNames(consumers), "stale") + + require.NoError(t, sink.Close(ctx)) + require.NoError(t, stream.Destroy(ctx)) +} +func TestStaleConsumerSweepSkipsMalformedKeepAliveAndContinues(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + stream, err := NewStream(t.Name(), rdb) + require.NoError(t, err) + sink, err := stream.NewSink( + ctx, + "sink", + options.WithSinkAckGracePeriod(testAckDuration), + options.WithSinkBlockDuration(testBlockDuration), + ) + require.NoError(t, err) + state := sink.streams[stream.key] + require.NoError(t, rdb.XGroupCreateConsumer(ctx, stream.key, sink.Name, "malformed").Err()) + require.NoError(t, rdb.XGroupCreateConsumer(ctx, stream.key, sink.Name, "stale").Err()) + _, err = state.keepAlives.Set(ctx, "malformed", "not-a-timestamp") + require.NoError(t, err) + _, err = state.keepAlives.Set(ctx, "stale", "0") + require.NoError(t, err) + lease, acquired, err := acquireSinkRecoveryLease( + ctx, + stream, + state.leaseKey, + state.leaseOwner, + time.Second, + ) + require.NoError(t, err) + require.True(t, acquired) + require.NoError(t, sink.deleteStreamStaleConsumers(ctx, state, lease, time.Second)) + consumers, err := rdb.XInfoConsumers(ctx, stream.key, sink.Name).Result() + require.NoError(t, err) + names := make([]string, len(consumers)) + for i, consumer := range consumers { + names[i] = consumer.Name + } + require.Contains(t, names, "malformed") + require.NotContains(t, names, "stale") + + require.NoError(t, sink.Close(ctx)) + require.NoError(t, stream.Destroy(ctx)) +} + +func TestStreamingScriptsRecoverAfterScriptFlush(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + stream, err := NewStream(t.Name(), rdb) + require.NoError(t, err) + sink, err := stream.NewSink( + ctx, + "sink", + options.WithSinkStartAtOldest(), + options.WithSinkAckGracePeriod(testAckDuration), + options.WithSinkBlockDuration(testBlockDuration), + ) + require.NoError(t, err) + events := sink.Subscribe() + + require.NoError(t, rdb.ScriptFlush(ctx).Err()) + require.NoError(t, rdb.XGroupDestroy(ctx, stream.key, sink.Name).Err()) + _, err = stream.Add(ctx, "after-flush", []byte("payload")) + require.NoError(t, err) + event := receiveSinkEvent(t, events) + require.Equal(t, "after-flush", event.EventName) + require.NoError(t, sink.Ack(ctx, event)) + + require.NoError(t, sink.Close(ctx)) + require.NoError(t, stream.Destroy(ctx)) +} + +func TestSinkRejectsInvalidOptions(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + cases := []struct { + name string + opt options.Sink + }{ + {name: "max polled", opt: options.WithSinkMaxPolled(0)}, + {name: "buffer", opt: options.WithSinkBufferSize(-1)}, + {name: "ack grace", opt: options.WithSinkAckGracePeriod(0)}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + stream, err := NewStream(t.Name(), rdb) + require.NoError(t, err) + _, err = stream.NewSink(ctx, "sink", tc.opt) + require.Error(t, err) + }) + } + stream, err := NewStream(t.Name()+"-conflicts", rdb) + require.NoError(t, err) + _, err = stream.NewSink( + ctx, + "sink", + options.WithSinkTopic("alarms"), + options.WithSinkTopicPattern("alarm.*"), + ) + require.ErrorContains(t, err, "mutually exclusive") + _, err = stream.NewSink( + ctx, + "sink", + options.WithSinkStartAtNewest(), + options.WithSinkStartAtOldest(), + ) + require.ErrorContains(t, err, "sink cursor-start options are mutually exclusive") + + sink, err := stream.NewSink(ctx, "sink") + require.NoError(t, err) + added, err := NewStream(t.Name()+"-added", rdb) + require.NoError(t, err) + err = sink.AddStream( + ctx, + added, + options.WithAddStreamStartAtNewest(), + options.WithAddStreamStartAtOldest(), + ) + require.ErrorContains(t, err, "added stream cursor-start options are mutually exclusive") + require.NoError(t, sink.Close(ctx)) + require.NoError(t, stream.Destroy(ctx)) + require.ErrorIs(t, added.Destroy(ctx), ErrStreamNotFound) +} + +func TestSinkAcknowledgesMalformedRedisEvent(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + stream, err := NewStream(t.Name(), rdb) + require.NoError(t, err) + require.NoError(t, stream.Open(ctx)) + require.NoError(t, rdb.XAdd(ctx, &redis.XAddArgs{ + Stream: stream.key, + Values: map[string]any{payloadKey: "missing name"}, + }).Err()) + sink, err := stream.NewSink( + ctx, + "sink", + options.WithSinkStartAtOldest(), + options.WithSinkAckGracePeriod(testAckDuration), + options.WithSinkBlockDuration(testBlockDuration), + ) + require.NoError(t, err) + events := sink.Subscribe() + _, err = stream.Add(ctx, "valid", []byte("payload")) + require.NoError(t, err) + event := receiveSinkEvent(t, events) + require.Equal(t, "valid", event.EventName) + require.NoError(t, sink.Ack(ctx, event)) + require.Eventually(t, func() bool { + pending, pendingErr := rdb.XPending(ctx, stream.key, sink.Name).Result() + return pendingErr == nil && pending.Count == 0 + }, time.Second, delay) + + require.NoError(t, sink.Close(ctx)) + require.NoError(t, stream.Destroy(ctx)) +} + +func TestDestroyedStreamSetupRemovesOrphanMembership(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + stream, err := NewStream(t.Name(), rdb) + require.NoError(t, err) + require.NoError(t, stream.Open(ctx)) + membership, err := rmap.Join(ctx, consumersMapName(stream), rdb) + require.NoError(t, err) + membershipKey := consumersMapContentKey(stream) + require.NoError(t, stream.Destroy(ctx)) + + _, setupErr := ensureConsumerGroup(ctx, stream, "sink", "0") + require.ErrorIs(t, setupErr, ErrStreamDestroyed) + closeSetupMembership(ctx, stream, membership, setupErr) + require.EqualValues(t, 0, rdb.Exists(ctx, membershipKey).Val()) +} + func TestSinkCloseWithStalledSubscriber(t *testing.T) { testName := strings.Replace(t.Name(), "/", "_", -1) rdb := ptesting.NewRedisClient(t) @@ -51,13 +623,13 @@ func TestSinkCloseWithStalledSubscriber(t *testing.T) { require.Eventually(t, func() bool { return len(c) == cap(c) }, max, delay) // Close must return even though the subscriber stalled the read loop. - done := make(chan struct{}) + done := make(chan error, 1) go func() { - sink.Close(ctx) - close(done) + done <- sink.Close(ctx) }() select { - case <-done: + case err := <-done: + require.NoError(t, err) case <-time.After(2 * time.Second): t.Fatal("sink.Close() hung with a stalled subscriber") } @@ -83,6 +655,27 @@ func TestNewSink(t *testing.T) { assert.EqualError(t, err, "topic pattern must be a valid regex: error parsing regexp: missing closing ): `(`") } +func TestSinkRejectsSubMillisecondDurations(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + stream, err := NewStream(t.Name(), rdb, options.WithStreamLogger(pulse.ClueLogger(ctx))) + require.NoError(t, err) + for _, duration := range []time.Duration{0, -time.Second, 500 * time.Microsecond} { + sink, err := stream.NewSink(ctx, "sink", options.WithSinkBlockDuration(duration)) + require.Nil(t, sink) + require.EqualError(t, err, "sink block duration must be at least 1ms") + } + sink, err := stream.NewSink( + ctx, + "sink", + options.WithSinkAckGracePeriod(500*time.Microsecond), + ) + require.Nil(t, sink) + require.EqualError(t, err, "sink acknowledgement grace period must be at least 1ms") + require.ErrorIs(t, stream.Destroy(ctx), ErrStreamNotFound) +} + func TestReadOnce(t *testing.T) { testName := strings.Replace(t.Name(), "/", "_", -1) rdb := ptesting.NewRedisClient(t) @@ -111,6 +704,7 @@ func TestReadSinceLastEvent(t *testing.T) { ctx := ptesting.NewTestContext(t) s, err := NewStream(testName, rdb, options.WithStreamLogger(pulse.ClueLogger(ctx))) assert.NoError(t, err) + defer func() { require.NoError(t, s.Destroy(ctx)) }() // Add and read 2 events consecutively sink, err := s.NewSink(ctx, "sink", @@ -118,7 +712,7 @@ func TestReadSinceLastEvent(t *testing.T) { options.WithSinkBlockDuration(testBlockDuration), ) require.NoError(t, err) - defer cleanupSink(t, ctx, s, sink) + defer func() { require.NoError(t, sink.Close(ctx)) }() c := sink.Subscribe() _, err = s.Add(ctx, "event", []byte("payload")) assert.NoError(t, err) @@ -137,7 +731,7 @@ func TestReadSinceLastEvent(t *testing.T) { options.WithSinkStartAfter(eventID), options.WithSinkBlockDuration(testBlockDuration)) require.NoError(t, err) - defer cleanupSink(t, ctx, s, sink2) + defer func() { require.NoError(t, sink2.Close(ctx)) }() c2 := sink2.Subscribe() read = readOneEvent(t, ctx, c2, sink2) assert.Equal(t, "event", read.EventName) @@ -148,7 +742,7 @@ func TestReadSinceLastEvent(t *testing.T) { options.WithSinkStartAfter("0"), options.WithSinkBlockDuration(testBlockDuration)) require.NoError(t, err) - defer cleanupSink(t, ctx, s, sink3) + defer func() { require.NoError(t, sink3.Close(ctx)) }() c3 := sink3.Subscribe() read = readOneEvent(t, ctx, c3, sink3) assert.Equal(t, "event", read.EventName) @@ -179,7 +773,7 @@ func TestCleanup(t *testing.T) { assert.Equal(t, []byte("payload"), read.Payload) // Stop sink, destroy stream and check Redis keys are gone - sink.Close(ctx) + require.NoError(t, sink.Close(ctx)) assert.Eventually(t, func() bool { return sink.IsClosed() }, max, delay) assert.Equal(t, rdb.Exists(ctx, s.key).Val(), int64(1)) assert.NoError(t, s.Destroy(ctx)) @@ -257,14 +851,14 @@ func TestRemoveStream(t *testing.T) { // Remove one stream and read again err = sink.RemoveStream(ctx, s2) assert.NoError(t, err) - eventID, err := s.Add(ctx, "event3", []byte("payload3")) + _, err = s.Add(ctx, "event3", []byte("payload3")) assert.NoError(t, err) read = readOneEvent(t, ctx, c, sink) assert.Equal(t, "event3", read.EventName) assert.Equal(t, []byte("payload3"), read.Payload) // Add back and remove other stream - err = sink.AddStream(ctx, s2, options.WithAddStreamStartAfter(eventID)) + err = sink.AddStream(ctx, s2) assert.NoError(t, err) err = sink.RemoveStream(ctx, s) assert.NoError(t, err) @@ -275,6 +869,273 @@ func TestRemoveStream(t *testing.T) { assert.Equal(t, []byte("payload4"), read.Payload) } +func TestRemoveStreamDeletesOnlyEmptyConsumerMetadata(t *testing.T) { + testName := strings.ReplaceAll(t.Name(), "/", "_") + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + primary, err := NewStream(testName+"-primary", rdb) + require.NoError(t, err) + secondary, err := NewStream(testName+"-secondary", rdb) + require.NoError(t, err) + sink, err := primary.NewSink( + ctx, + "sink", + options.WithSinkStartAtOldest(), + options.WithSinkBlockDuration(testBlockDuration), + ) + require.NoError(t, err) + require.NoError(t, sink.AddStream(ctx, secondary)) + defer func() { + assert.NoError(t, sink.Close(ctx)) + assert.NoError(t, primary.Destroy(ctx)) + assert.NoError(t, secondary.Destroy(ctx)) + }() + + require.NoError(t, rdb.XGroupCreateConsumer( + ctx, + secondary.key, + sink.Name, + "pending-consumer", + ).Err()) + eventID, err := secondary.Add(ctx, "event", nil) + require.NoError(t, err) + require.Eventually(t, func() bool { + pending, pendingErr := rdb.XPending(ctx, secondary.key, sink.Name).Result() + return pendingErr == nil && pending.Count == 1 + }, max, delay) + _, err = rdb.XClaim(ctx, &redis.XClaimArgs{ + Stream: secondary.key, + Group: sink.Name, + Consumer: "pending-consumer", + MinIdle: 0, + Messages: []string{eventID}, + }).Result() + require.NoError(t, err) + require.Eventually(t, func() bool { + pending, pendingErr := rdb.XPendingExt(ctx, &redis.XPendingExtArgs{ + Stream: secondary.key, + Group: sink.Name, + Consumer: sink.consumer, + Start: "-", + End: "+", + Count: 1, + }).Result() + return pendingErr == nil && len(pending) == 0 + }, max, delay) + + require.NoError(t, sink.RemoveStream(ctx, secondary)) + consumers, err := rdb.XInfoConsumers(ctx, secondary.key, sink.Name).Result() + require.NoError(t, err) + names := make([]string, 0, len(consumers)) + for _, consumer := range consumers { + names = append(names, consumer.Name) + } + assert.NotContains(t, names, sink.consumer) + assert.Contains(t, names, "pending-consumer") +} + +func TestSinkRecoversExternallyDeletedStream(t *testing.T) { + testName := strings.ReplaceAll(t.Name(), "/", "_") + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + hook := &redisCommandHook{} + rdb.AddHook(hook) + ctx := ptesting.NewTestContext(t) + + stream, err := NewStream(testName, rdb, options.WithStreamLogger(pulse.ClueLogger(ctx))) + require.NoError(t, err) + sink, err := stream.NewSink(ctx, "sink", options.WithSinkBlockDuration(testBlockDuration)) + require.NoError(t, err) + defer cleanupSink(t, ctx, stream, sink) + events := sink.Subscribe() + + require.Eventually(t, func() bool { return hook.xreadGroups.Load() > 0 }, max, delay) + readsBeforeDelete := hook.xreadGroups.Load() + require.NoError(t, rdb.Del(ctx, stream.key).Err()) + + require.Eventually(t, func() bool { return consumerGroupExists(ctx, rdb, stream.key, sink.Name) }, max, delay) + time.Sleep(4 * testBlockDuration) + assert.LessOrEqual(t, hook.xreadGroups.Load()-readsBeforeDelete, int64(10), + "NOGROUP recovery issued XREADGROUP in a hot loop") + + _, err = stream.Add(ctx, "future", []byte("payload")) + require.NoError(t, err) + read := readOneEvent(t, ctx, events, sink) + assert.Equal(t, "future", read.EventName) + assert.Equal(t, []byte("payload"), read.Payload) +} + +func TestSinkRecoversDestroyedGroupAcrossStreams(t *testing.T) { + testName := strings.ReplaceAll(t.Name(), "/", "_") + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + hook := &redisCommandHook{} + rdb.AddHook(hook) + ctx := ptesting.NewTestContext(t) + + mainStream, err := NewStream(testName, rdb, options.WithStreamLogger(pulse.ClueLogger(ctx))) + require.NoError(t, err) + addedStream, err := NewStream(testName+"-added", rdb, options.WithStreamLogger(pulse.ClueLogger(ctx))) + require.NoError(t, err) + defer func() { assert.NoError(t, addedStream.Destroy(ctx)) }() + + sink, err := mainStream.NewSink(ctx, "sink", options.WithSinkBlockDuration(testBlockDuration)) + require.NoError(t, err) + defer cleanupSink(t, ctx, mainStream, sink) + require.NoError(t, sink.AddStream(ctx, addedStream, options.WithAddStreamStartAtOldest())) + events := sink.Subscribe() + + readsBeforeAdd := hook.xreadGroups.Load() + require.Eventually(t, func() bool { return hook.xreadGroups.Load() > readsBeforeAdd }, max, delay) + + // Hold the ownership lock after the multi-stream read has started. Redis + // reports NOGROUP for that read when the added stream's group is destroyed, + // but recovery cannot run until the event below exists. This proves the + // added stream's own start position is retained for recovery. + sink.lock.Lock() + destroyed, destroyErr := rdb.XGroupDestroy(ctx, addedStream.key, sink.Name).Result() + _, addErr := addedStream.Add(ctx, "recovered", []byte("added payload")) + sink.lock.Unlock() + require.NoError(t, destroyErr) + require.EqualValues(t, 1, destroyed) + require.NoError(t, addErr) + + read := readOneEvent(t, ctx, events, sink) + assert.Equal(t, addedStream.Name, read.StreamName) + assert.Equal(t, "recovered", read.EventName) + require.True(t, consumerGroupExists(ctx, rdb, mainStream.key, sink.Name)) + require.True(t, consumerGroupExists(ctx, rdb, addedStream.key, sink.Name)) + + _, err = mainStream.Add(ctx, "main", []byte("main payload")) + require.NoError(t, err) + read = readOneEvent(t, ctx, events, sink) + assert.Equal(t, mainStream.Name, read.StreamName) + assert.Equal(t, "main", read.EventName) +} + +func TestSinkRecoveryDoesNotRecreateRemovedStream(t *testing.T) { + testName := strings.ReplaceAll(t.Name(), "/", "_") + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + hook := &redisCommandHook{} + rdb.AddHook(hook) + ctx := ptesting.NewTestContext(t) + + mainStream, err := NewStream(testName, rdb, options.WithStreamLogger(pulse.ClueLogger(ctx))) + require.NoError(t, err) + removedStream, err := NewStream(testName+"-removed", rdb, options.WithStreamLogger(pulse.ClueLogger(ctx))) + require.NoError(t, err) + defer func() { assert.NoError(t, removedStream.Destroy(ctx)) }() + + sink, err := mainStream.NewSink(ctx, "sink", options.WithSinkBlockDuration(testBlockDuration)) + require.NoError(t, err) + defer cleanupSink(t, ctx, mainStream, sink) + require.NoError(t, sink.AddStream(ctx, removedStream)) + events := sink.Subscribe() + + readsBeforeAdd := hook.xreadGroups.Load() + require.Eventually(t, func() bool { return hook.xreadGroups.Load() > readsBeforeAdd }, max, delay) + require.NoError(t, sink.RemoveStream(ctx, removedStream)) + destroyed, err := rdb.XGroupDestroy(ctx, removedStream.key, sink.Name).Result() + require.NoError(t, err) + require.EqualValues(t, 1, destroyed) + + assert.Never(t, func() bool { + return consumerGroupExists(ctx, rdb, removedStream.key, sink.Name) + }, 4*testBlockDuration, delay, "recovery recreated a group for a removed stream") + + _, err = mainStream.Add(ctx, "remaining", []byte("payload")) + require.NoError(t, err) + read := readOneEvent(t, ctx, events, sink) + assert.Equal(t, mainStream.Name, read.StreamName) + assert.Equal(t, "remaining", read.EventName) +} + +func TestSinkBoundsRetriesDuringRedisFailure(t *testing.T) { + testName := strings.ReplaceAll(t.Name(), "/", "_") + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + failure := errors.New("test Redis outage") + hook := &redisCommandHook{failure: failure} + rdb.AddHook(hook) + ctx := ptesting.NewTestContext(t) + + stream, err := NewStream(testName, rdb, options.WithStreamLogger(pulse.ClueLogger(ctx))) + require.NoError(t, err) + sink, err := stream.NewSink(ctx, "sink", options.WithSinkBlockDuration(20*time.Millisecond)) + require.NoError(t, err) + defer cleanupSink(t, ctx, stream, sink) + events := sink.Subscribe() + + hook.failGroup.Store(true) + hook.failGroupKey = stream.lifecycleKey + sink.lock.Lock() + err = sink.recoverConsumerGroups(ctx) + sink.lock.Unlock() + require.ErrorIs(t, err, failure) + assert.EqualError(t, err, + `failed to ensure Redis consumer group "sink" for stream "`+testName+`" generation 1: test Redis outage`) + hook.failGroup.Store(false) + + require.Eventually(t, func() bool { return hook.xreadGroups.Load() > 0 }, max, delay) + readsBeforeFailure := hook.xreadGroups.Load() + hook.failRead.Store(true) + time.Sleep(350 * time.Millisecond) + hook.failRead.Store(false) + assert.LessOrEqual(t, hook.xreadGroups.Load()-readsBeforeFailure, int64(4), + "Redis outage retries exceeded the bounded backoff rate") + + _, err = stream.Add(ctx, "after-outage", []byte("payload")) + require.NoError(t, err) + read := readOneEvent(t, ctx, events, sink) + assert.Equal(t, "after-outage", read.EventName) +} + +func TestSinkCloseDuringConsumerGroupRecovery(t *testing.T) { + testName := strings.ReplaceAll(t.Name(), "/", "_") + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + hook := &redisCommandHook{} + rdb.AddHook(hook) + ctx := ptesting.NewTestContext(t) + + stream, err := NewStream(testName, rdb, options.WithStreamLogger(pulse.ClueLogger(ctx))) + require.NoError(t, err) + sink, err := stream.NewSink(ctx, "sink", options.WithSinkBlockDuration(10*time.Millisecond)) + require.NoError(t, err) + + require.Eventually(t, func() bool { return hook.xreadGroups.Load() > 0 }, max, delay) + disruptionDone := make(chan error, 1) + go func() { + for range 50 { + if err := rdb.XGroupDestroy(ctx, stream.key, sink.Name).Err(); err != nil { + disruptionDone <- err + return + } + } + disruptionDone <- nil + }() + + closeDone := make(chan error, 1) + go func() { + closeDone <- sink.Close(ctx) + }() + select { + case err := <-closeDone: + require.NoError(t, err) + case <-time.After(2 * time.Second): + t.Fatal("sink.Close hung during consumer-group recovery") + } + require.NoError(t, <-disruptionDone) + require.True(t, sink.IsClosed()) + + readsAfterClose := hook.xreadGroups.Load() + time.Sleep(4 * testBlockDuration) + assert.Equal(t, readsAfterClose, hook.xreadGroups.Load(), "closed sink continued reading") + require.NoError(t, stream.Destroy(ctx)) +} + func TestMultipleConsumers(t *testing.T) { testName := strings.Replace(t.Name(), "/", "_", -1) rdb := ptesting.NewRedisClient(t) @@ -296,7 +1157,7 @@ func TestMultipleConsumers(t *testing.T) { options.WithSinkAckGracePeriod(testAckDuration)) require.NoError(t, err) defer func() { - sink2.Close(ctx) + require.NoError(t, sink2.Close(ctx)) assert.Eventually(t, func() bool { return sink2.IsClosed() }, max, delay) }() @@ -355,7 +1216,7 @@ func TestClaimStaleMessages(t *testing.T) { var read *Event select { case read = <-c: - case <-time.After(testAckDuration): + case <-time.After(max): t.Fatal("timeout waiting for initial event") } assert.Equal(t, "event", read.EventName) @@ -365,7 +1226,7 @@ func TestClaimStaleMessages(t *testing.T) { select { case read = <-c: assert.NoError(t, sink.Ack(ctx, read)) - case <-time.After(testAckDuration * 2): + case <-time.After(max): t.Fatal("timeout waiting for claimed event") } assert.Equal(t, "event", read.EventName) @@ -406,7 +1267,7 @@ func TestNonAckMessageDeliveredToAnotherConsumer(t *testing.T) { options.WithSinkBlockDuration(testBlockDuration), options.WithSinkAckGracePeriod(testAckDuration)) require.NoError(t, err) - defer sink2.Close(ctx) + defer func() { require.NoError(t, sink2.Close(ctx)) }() // Subscribe to both sinks c1 := sink1.Subscribe() @@ -431,14 +1292,14 @@ func TestNonAckMessageDeliveredToAnotherConsumer(t *testing.T) { receiverSink = sink2 otherSink = sink1 otherChan = c1 - case <-time.After(testAckDuration): + case <-time.After(max): t.Fatal("Timeout waiting for event on first sink") } assert.Equal(t, "test_event", read1.EventName) assert.Equal(t, []byte("test_payload"), read1.Payload) // Close the receiver sink - receiverSink.Close(ctx) + require.NoError(t, receiverSink.Close(ctx)) assert.Eventually(t, func() bool { return receiverSink.IsClosed() }, max, delay) // The message should now be redelivered to the other sink @@ -447,7 +1308,7 @@ func TestNonAckMessageDeliveredToAnotherConsumer(t *testing.T) { case read2 = <-otherChan: logger.Info("Read from other sink") assert.NoError(t, otherSink.Ack(ctx, read2)) - case <-time.After(testAckDuration * 4): + case <-time.After(testAckDuration * 20): t.Fatal("Timeout waiting for event on other sink") } assert.Equal(t, "test_event", read2.EventName) @@ -487,7 +1348,7 @@ func TestStaleConsumerDeletionAndMessageClaiming(t *testing.T) { var read *Event select { case read = <-c1: - case <-time.After(testAckDuration): + case <-time.After(max): t.Fatal("Timeout waiting for event on first sink") } assert.Equal(t, "test_event", read.EventName) @@ -512,7 +1373,7 @@ func TestStaleConsumerDeletionAndMessageClaiming(t *testing.T) { }, max, delay, "Expected two consumers") // Close the sink to stop keep-alive refresh - sink1.Close(ctx) + require.NoError(t, sink1.Close(ctx)) assert.Eventually(t, func() bool { return sink1.IsClosed() }, max, delay) // Verify that the stale consumer is deleted @@ -533,9 +1394,91 @@ func TestStaleConsumerDeletionAndMessageClaiming(t *testing.T) { select { case claimedRead = <-c2: assert.NoError(t, sink2.Ack(ctx, claimedRead)) - case <-time.After(testAckDuration * 2): + case <-time.After(max): t.Fatal("Timeout waiting for claimed event") } assert.Equal(t, "test_event", claimedRead.EventName) assert.Equal(t, []byte("test_payload"), claimedRead.Payload) } + +// consumerGroupExists reports whether Redis currently holds group for stream. +func consumerGroupExists(ctx context.Context, rdb *redis.Client, stream, group string) bool { + groups, err := rdb.XInfoGroups(ctx, stream).Result() + if err != nil { + return false + } + for _, candidate := range groups { + if candidate.Name == group { + return true + } + } + return false +} + +// DialHook preserves the client's normal Redis connection behavior. +func (h *redisCommandHook) DialHook(next redis.DialHook) redis.DialHook { + return func(ctx context.Context, network, addr string) (net.Conn, error) { + return next(ctx, network, addr) + } +} + +// ProcessHook counts group reads and injects command-specific transport +// failures used to prove bounded retries and exact error propagation. +func (h *redisCommandHook) ProcessHook(next redis.ProcessHook) redis.ProcessHook { + return func(ctx context.Context, cmd redis.Cmder) error { + switch cmd.Name() { + case "xreadgroup": + h.xreadGroups.Add(1) + for _, arg := range cmd.Args() { + if value, ok := arg.(string); ok && strings.EqualFold(value, "noack") { + h.usedNoAck.Store(true) + } + } + if h.blockRead.Load() { + h.started.Do(func() { + close(h.readStarted) + }) + <-ctx.Done() + return ctx.Err() + } + if h.failRead.Load() { + return h.failure + } + case "xgroup": + if h.failGroup.Load() { + return h.failure + } + case "evalsha": + if h.failCreateConsumer.Load() && len(cmd.Args()) > 1 && cmd.Args()[1] == registerSinkConsumerScript.Hash() { + for _, arg := range cmd.Args() { + if key, ok := arg.(string); ok && key == h.failCreateConsumerKey { + return h.failure + } + } + } + if h.failGroup.Load() { + for _, arg := range cmd.Args() { + if key, ok := arg.(string); ok && key == h.failGroupKey { + return h.failure + } + } + } + if h.failRemove.Load() { + for _, arg := range cmd.Args() { + if key, ok := arg.(string); ok && key == h.failMembershipKey { + return h.failure + } + } + } + } + return next(ctx, cmd) + } +} + +// ProcessPipelineHook preserves pipeline behavior; recovery uses direct Redis +// commands so no injected failure is expected through this path. +func (h *redisCommandHook) ProcessPipelineHook(next redis.ProcessPipelineHook) redis.ProcessPipelineHook { + return func(ctx context.Context, cmds []redis.Cmder) error { + return next(ctx, cmds) + } +} diff --git a/streaming/stream_lifecycle.go b/streaming/stream_lifecycle.go new file mode 100644 index 0000000..43fc6fc --- /dev/null +++ b/streaming/stream_lifecycle.go @@ -0,0 +1,730 @@ +// Package streaming adopts pre-generation event data in place, then assigns a +// distinct physical Redis key to every explicitly recreated incarnation. +// Stream handles bind lazily to the lifecycle-selected generation and key. +package streaming + +import ( + "context" + "fmt" + "strconv" + "strings" + "time" + + redis "github.com/redis/go-redis/v9" +) + +const ( + streamStateActive = "active" + streamStateDestroyed = "destroyed" + streamDestroyedError = "STREAMDESTROYED" + streamPhysicalKey = "physical_key" + streamDeadlineKey = "deadline_ms" + streamTTLOwnedKey = "ttl_owned" + streamDeadlineError = "DEADLINEELAPSED" + streamConflictError = "STREAMDEADLINECONFLICT" + streamConfigKey = "retention_config" + streamConfigError = "STREAMCONFIGMISMATCH" + streamNotFoundError = "STREAMNOTFOUND" + streamDeadlineRequiredError = "STREAMDEADLINEREQUIRED" + streamConfigMissingError = "STREAMCONFIGMISSING" + streamSnapshotUnboundedError = "SNAPSHOTUNBOUNDED" + streamSnapshotBoundError = "SNAPSHOTBOUNDVIOLATION" + streamFormatVersion = "2" +) + +var ( + // establishStreamScript adopts an existing flat stream for generation one. + // Explicit recreation receives a distinct physical key. + establishStreamScript = redis.NewScript(` +local state = redis.call("HGET", KEYS[1], "state") +local generation = redis.call("HGET", KEYS[1], "generation") +local physical = redis.call("HGET", KEYS[1], ARGV[2]) +local deadline = redis.call("HGET", KEYS[1], ARGV[6]) +local ttl_owned = redis.call("HGET", KEYS[1], ARGV[7]) +local retention = redis.call("HGET", KEYS[1], ARGV[8]) +local physical_missing = not physical +if ARGV[4] ~= "" and ARGV[5] == "1" then + return redis.error_reply("STREAMDEADLINECONFLICT") +end +if ARGV[4] ~= "" then + local clock = redis.call("TIME") + local now = tonumber(clock[1]) * 1000 + math.floor(tonumber(clock[2]) / 1000) + if now >= tonumber(ARGV[4]) then + return redis.error_reply("DEADLINEELAPSED") + end +end +if not generation and ARGV[10] == "1" and tonumber(ARGV[11]) > 0 then + redis.call("XTRIM", ARGV[3], "MAXLEN", "=", ARGV[11]) +end +if not generation then + generation = "1" + physical = ARGV[3] + redis.call("HSET", KEYS[1], "generation", generation, "state", ARGV[1], ARGV[2], physical, ARGV[8], ARGV[9]) + if ARGV[4] ~= "" then + deadline = ARGV[4] + redis.call("HSET", KEYS[1], ARGV[6], deadline) + elseif ARGV[5] == "1" then + ttl_owned = "1" + redis.call("HSET", KEYS[1], ARGV[7], ttl_owned) + end +elseif state ~= ARGV[1] then + generation = tostring(redis.call("HINCRBY", KEYS[1], "generation", 1)) + physical = ARGV[3] .. ":generation:" .. generation + redis.call("HSET", KEYS[1], "state", ARGV[1], ARGV[2], physical, ARGV[8], ARGV[9]) + redis.call("HDEL", KEYS[1], ARGV[6], ARGV[7]) + deadline = false + ttl_owned = false + if ARGV[4] ~= "" then + deadline = ARGV[4] + redis.call("HSET", KEYS[1], ARGV[6], deadline) + elseif ARGV[5] == "1" then + ttl_owned = "1" + redis.call("HSET", KEYS[1], ARGV[7], ttl_owned) + end +elseif not physical then + physical = ARGV[3] +end +if state == ARGV[1] then + if retention and ARGV[10] == "1" and retention ~= ARGV[9] then + return redis.error_reply("STREAMCONFIGMISMATCH") + end + if deadline then + if (ARGV[4] ~= "" and ARGV[4] ~= deadline) or ARGV[5] == "1" then + return redis.error_reply("STREAMDEADLINECONFLICT") + end + elseif ttl_owned == "1" then + if ARGV[4] ~= "" then + return redis.error_reply("STREAMDEADLINECONFLICT") + end + end + if physical_missing then + redis.call("HSET", KEYS[1], ARGV[2], physical) + end + if not retention then + if ARGV[10] == "1" and tonumber(ARGV[11]) > 0 and physical == ARGV[3] then + redis.call("XTRIM", physical, "MAXLEN", "=", ARGV[11]) + end + redis.call("HSET", KEYS[1], ARGV[8], ARGV[9]) + retention = ARGV[9] + end + if not deadline and ttl_owned ~= "1" then + if ARGV[4] ~= "" then + deadline = ARGV[4] + redis.call("HSET", KEYS[1], ARGV[6], deadline) + elseif ARGV[5] == "1" then + redis.call("HSET", KEYS[1], ARGV[7], "1") + end + end +end +return {generation, physical, deadline or "", retention or ARGV[9]} +`) + + // loadCurrentGenerationScript binds an unbound non-creating operation to + // the current generation without advancing a destroyed lifecycle. + loadCurrentGenerationScript = redis.NewScript(` +local state = redis.call("HGET", KEYS[1], "state") +local generation = redis.call("HGET", KEYS[1], "generation") +local physical = redis.call("HGET", KEYS[1], ARGV[2]) +local deadline = redis.call("HGET", KEYS[1], ARGV[6]) +local ttl_owned = redis.call("HGET", KEYS[1], ARGV[7]) +local retention = redis.call("HGET", KEYS[1], ARGV[8]) +if not generation then + return redis.error_reply("STREAMNOTFOUND") +end +if state ~= ARGV[1] then + return redis.error_reply("STREAMDESTROYED") +end +if not physical then + physical = ARGV[3] +end +if retention then + if ARGV[10] == "1" and retention ~= ARGV[9] then + return redis.error_reply("STREAMCONFIGMISMATCH") + end +end +if deadline then + if (ARGV[4] ~= "" and ARGV[4] ~= deadline) or ARGV[5] == "1" then + return redis.error_reply("STREAMDEADLINECONFLICT") + end +elseif ttl_owned == "1" then + if ARGV[4] ~= "" then + return redis.error_reply("STREAMDEADLINECONFLICT") + end +elseif ARGV[4] ~= "" or ARGV[5] == "1" then + return redis.error_reply("STREAMDEADLINECONFLICT") +end +if not retention then + retention = ARGV[9] + redis.call("HSET", KEYS[1], ARGV[8], retention) +end +return {generation, physical, deadline or "", retention} +`) + + // verifyStreamScript is used by constructors whose subsequent Redis + // operations cannot recreate stream state. + verifyStreamScript = redis.NewScript(` +if redis.call("HGET", KEYS[1], "state") ~= ARGV[1] +or redis.call("HGET", KEYS[1], "generation") ~= ARGV[2] +or redis.call("HGET", KEYS[1], ARGV[3]) ~= ARGV[4] then + return redis.error_reply("STREAMDESTROYED") +end +if redis.call("HGET", KEYS[1], ARGV[6]) ~= ARGV[7] then + return redis.error_reply("STREAMCONFIGMISMATCH") +end +local deadline = redis.call("HGET", KEYS[1], ARGV[5]) +if deadline then + local now = redis.call("TIME") + local now_ms = tonumber(now[1]) * 1000 + math.floor(tonumber(now[2]) / 1000) + if now_ms >= tonumber(deadline) then + return redis.error_reply("DEADLINEELAPSED") + end +end +return 1 +`) + + // addStreamEventScript verifies the generation, appends one event, and + // applies stream/recovery retention in one Redis operation. + addStreamEventScript = redis.NewScript(` +if redis.call("HGET", KEYS[1], "state") ~= ARGV[1] +or redis.call("HGET", KEYS[1], "generation") ~= ARGV[2] +or redis.call("HGET", KEYS[1], ARGV[11]) ~= KEYS[2] then + return redis.error_reply("STREAMDESTROYED") +end +if redis.call("HGET", KEYS[1], ARGV[13]) ~= ARGV[14] then + return redis.error_reply("STREAMCONFIGMISMATCH") +end +local deadline = redis.call("HGET", KEYS[1], ARGV[12]) +if deadline then + local now = redis.call("TIME") + local now_ms = tonumber(now[1]) * 1000 + math.floor(tonumber(now[2]) / 1000) + if now_ms >= tonumber(deadline) then + return redis.error_reply("DEADLINEELAPSED") + end + if tonumber(ARGV[9]) > 0 then + return redis.error_reply("STREAMDEADLINECONFLICT") + end +end +if ARGV[6] == "1" and redis.call("EXISTS", KEYS[2]) == 0 then + return {0} +end + +local id +if ARGV[3] == "0" and ARGV[7] == "1" then + if ARGV[6] == "1" then + id = redis.call("XADD", KEYS[2], "NOMKSTREAM", "*", "n", ARGV[4], "p", ARGV[5], "t", ARGV[8]) + else + id = redis.call("XADD", KEYS[2], "*", "n", ARGV[4], "p", ARGV[5], "t", ARGV[8]) + end +elseif ARGV[3] == "0" then + if ARGV[6] == "1" then + id = redis.call("XADD", KEYS[2], "NOMKSTREAM", "*", "n", ARGV[4], "p", ARGV[5]) + else + id = redis.call("XADD", KEYS[2], "*", "n", ARGV[4], "p", ARGV[5]) + end +elseif ARGV[7] == "1" then + if ARGV[6] == "1" then + id = redis.call("XADD", KEYS[2], "NOMKSTREAM", "MAXLEN", "=", ARGV[3], "*", + "n", ARGV[4], "p", ARGV[5], "t", ARGV[8]) + else + id = redis.call("XADD", KEYS[2], "MAXLEN", "=", ARGV[3], "*", + "n", ARGV[4], "p", ARGV[5], "t", ARGV[8]) + end +else + if ARGV[6] == "1" then + id = redis.call("XADD", KEYS[2], "NOMKSTREAM", "MAXLEN", "=", ARGV[3], "*", + "n", ARGV[4], "p", ARGV[5]) + else + id = redis.call("XADD", KEYS[2], "MAXLEN", "=", ARGV[3], "*", + "n", ARGV[4], "p", ARGV[5]) + end +end + +local ttl = tonumber(ARGV[9]) +if deadline then + redis.call("HSET", KEYS[3], "=deadline", deadline) + redis.call("SADD", KEYS[4], KEYS[3]) + redis.call("PEXPIREAT", KEYS[2], deadline) + redis.call("PEXPIREAT", KEYS[3], deadline) + redis.call("PEXPIREAT", KEYS[4], deadline) +elseif ttl > 0 then + if ARGV[10] == "1" then + redis.call("PEXPIRE", KEYS[2], ttl) + elseif redis.call("PTTL", KEYS[2]) == -1 then + redis.call("PEXPIRE", KEYS[2], ttl) + end +end +return {1, id} +`) + + // removeStreamEventsScript prevents stale Stream values from deleting + // events in a newer incarnation. + removeStreamEventsScript = redis.NewScript(` +if redis.call("HGET", KEYS[1], "state") ~= ARGV[1] +or redis.call("HGET", KEYS[1], "generation") ~= ARGV[2] +or redis.call("HGET", KEYS[1], ARGV[3]) ~= KEYS[2] then + return redis.error_reply("STREAMDESTROYED") +end +return redis.call("XDEL", KEYS[2], unpack(ARGV, 4)) +`) +) + +// establishStreamGeneration linearizes the first caller-context operation +// against Destroy and claims or adopts the generation retention contract. +func establishStreamGeneration( + ctx context.Context, + rdb *redis.Client, + name, requestedDeadline, retention string, + maxLen int, + ttlOwned, retentionExplicit bool, +) (string, string, string, string, error) { + result, err := establishStreamScript.Run( + ctx, + rdb, + []string{streamLifecycleKey(name)}, + streamStateActive, + streamPhysicalKey, + streamKey(name), + requestedDeadline, + boolString(ttlOwned), + streamDeadlineKey, + streamTTLOwnedKey, + streamConfigKey, + retention, + boolString(retentionExplicit), + strconv.Itoa(maxLen), + ).Slice() + if err != nil { + return "", "", "", "", streamLifecycleBoundaryError( + fmt.Errorf("pulse stream: establish generation for %q: %w", name, err), + ) + } + generation, physical, deadline, canonicalRetention, err := parseLifecycleIdentity(result) + if err != nil { + return "", "", "", "", fmt.Errorf("pulse stream: establish generation for %q: %w", name, err) + } + return generation, physical, deadline, canonicalRetention, nil +} + +// ensureGeneration binds s to the active Redis generation exactly once. +// Failed boundary I/O leaves the handle unbound so a later caller may retry +// with its own context. +func (s *Stream) ensureGeneration(ctx context.Context) error { + s.generationLock.Lock() + defer s.generationLock.Unlock() + if s.generation != "" { + return nil + } + generation, physical, deadline, retention, err := establishStreamGeneration( + ctx, + s.rdb, + s.Name, + s.requestedDeadline(), + requestedRetention(s), + s.maxLen, + s.ttl > 0, + s.retentionExplicit, + ) + if err != nil { + return err + } + s.generation = generation + s.key = physical + if err := s.applyRetentionConfig(retention); err != nil { + s.generation = "" + s.key = streamKey(s.Name) + return fmt.Errorf("pulse stream: establish generation for %q: %w", s.Name, err) + } + s.deadline, err = parseDeadline(deadline) + if err != nil { + s.generation = "" + s.key = streamKey(s.Name) + return fmt.Errorf("pulse stream: establish generation for %q: %w", s.Name, err) + } + return nil +} + +// loadExistingGeneration binds an unbound non-creating operation to the +// existing lifecycle generation without allocating a destroyed successor. +func (s *Stream) loadExistingGeneration(ctx context.Context) error { + s.generationLock.Lock() + defer s.generationLock.Unlock() + if s.generation != "" { + return nil + } + result, err := loadCurrentGenerationScript.Run( + ctx, + s.rdb, + []string{s.lifecycleKey}, + streamStateActive, + streamPhysicalKey, + streamKey(s.Name), + s.requestedDeadline(), + boolString(s.ttl > 0), + streamDeadlineKey, + streamTTLOwnedKey, + streamConfigKey, + requestedRetention(s), + boolString(s.retentionExplicit), + ).Slice() + if err != nil { + return streamLifecycleBoundaryError( + fmt.Errorf("pulse stream: load current generation for %q: %w", s.Name, err), + ) + } + generation, physical, deadline, retention, err := parseLifecycleIdentity(result) + if err != nil { + return fmt.Errorf("pulse stream: load current generation for %q: %w", s.Name, err) + } + s.generation = generation + s.key = physical + if err := s.applyRetentionConfig(retention); err != nil { + s.generation = "" + s.key = streamKey(s.Name) + return fmt.Errorf("pulse stream: load current generation for %q: %w", s.Name, err) + } + s.deadline, err = parseDeadline(deadline) + if err != nil { + s.generation = "" + s.key = streamKey(s.Name) + return fmt.Errorf("pulse stream: load current generation for %q: %w", s.Name, err) + } + return nil +} + +// verifyGeneration establishes an unbound handle and rejects a stale bound +// capability. +func (s *Stream) verifyGeneration(ctx context.Context) error { + if err := s.ensureGeneration(ctx); err != nil { + return err + } + err := verifyStreamScript.Run( + ctx, + s.rdb, + []string{s.lifecycleKey}, + streamStateActive, + s.generation, + streamPhysicalKey, + s.key, + streamDeadlineKey, + streamConfigKey, + s.retention, + ).Err() + return s.lifecycleError(err) +} + +// verifyExistingGeneration loads without creating, then verifies the exact +// active generation and its adopted retention contract. +func (s *Stream) verifyExistingGeneration(ctx context.Context) error { + if err := s.loadExistingGeneration(ctx); err != nil { + return err + } + return s.verifyGeneration(ctx) +} + +// addEvent atomically verifies this generation and appends one event. +func (s *Stream) addEvent( + ctx context.Context, + name string, + payload []byte, + onlyIfExists bool, + topic string, +) (string, error) { + var err error + if onlyIfExists { + err = s.loadExistingGeneration(ctx) + } else { + err = s.ensureGeneration(ctx) + } + if err != nil { + return "", err + } + topicPresent := topic != "" + result, err := addStreamEventScript.Run( + ctx, + s.rdb, + []string{ + s.lifecycleKey, + s.key, + recoveryCursorKey(s), + streamResourceRegistryKey(s), + }, + streamStateActive, + s.generation, + strconv.Itoa(s.maxLen), + name, + payload, + boolString(onlyIfExists), + boolString(topicPresent), + topic, + strconv.FormatInt(s.ttl.Milliseconds(), 10), + boolString(s.ttlSliding), + streamPhysicalKey, + streamDeadlineKey, + streamConfigKey, + s.retention, + ).Slice() + if err != nil { + return "", s.lifecycleError(err) + } + if len(result) == 0 { + return "", fmt.Errorf("add stream event script returned no status") + } + status, ok := result[0].(int64) + if !ok { + return "", fmt.Errorf("add stream event script returned invalid status %T", result[0]) + } + if status == 0 { + return "", nil + } + if len(result) != 2 { + return "", fmt.Errorf("add stream event script returned %d values for successful add", len(result)) + } + id, ok := result[1].(string) + if !ok { + return "", fmt.Errorf("add stream event script returned invalid event ID %T", result[1]) + } + return id, nil +} + +// removeEvents atomically verifies this generation and deletes event IDs. +func (s *Stream) removeEvents(ctx context.Context, ids []string) error { + if err := s.ensureGeneration(ctx); err != nil { + return err + } + args := make([]any, 0, len(ids)+3) + args = append(args, streamStateActive, s.generation, streamPhysicalKey) + for _, id := range ids { + args = append(args, id) + } + err := removeStreamEventsScript.Run( + ctx, + s.rdb, + []string{s.lifecycleKey, s.key}, + args..., + ).Err() + return s.lifecycleError(err) +} + +// lifecycleError maps Redis's generation mismatch to the public sentinel. +func (s *Stream) lifecycleError(err error) error { + if err == nil { + return nil + } + if redis.HasErrorPrefix(err, streamDestroyedError) { + return fmt.Errorf( + "%w: stream %q generation %s is no longer active", + ErrStreamDestroyed, + s.Name, + s.generation, + ) + } + if redis.HasErrorPrefix(err, streamDeadlineError) { + return fmt.Errorf("%w: stream %q", ErrDeadlineElapsed, s.Name) + } + if redis.HasErrorPrefix(err, streamConflictError) { + return fmt.Errorf( + "%w: stream %q deadline conflicts with active generation", + ErrStreamConfigMismatch, + s.Name, + ) + } + if redis.HasErrorPrefix(err, streamConfigError) { + return fmt.Errorf("%w: stream %q", ErrStreamConfigMismatch, s.Name) + } + if redis.HasErrorPrefix(err, streamDeadlineRequiredError) { + return fmt.Errorf("%w: stream %q requires deadline retention", ErrStreamConfigMismatch, s.Name) + } + if redis.HasErrorPrefix(err, streamConfigMissingError) { + return fmt.Errorf("%w: stream %q has no adopted retention configuration", ErrStreamConfigMismatch, s.Name) + } + if redis.HasErrorPrefix(err, streamSnapshotUnboundedError) { + return fmt.Errorf("%w: stream %q", ErrSnapshotUnbounded, s.Name) + } + if redis.HasErrorPrefix(err, streamSnapshotBoundError) { + return fmt.Errorf("%w: stream %q", ErrSnapshotBoundExceeded, s.Name) + } + if redis.HasErrorPrefix(err, streamNotFoundError) { + return fmt.Errorf("%w: stream %q", ErrStreamNotFound, s.Name) + } + return err +} + +// streamLifecycleKey identifies the canonical generation record for a logical +// stream name. +func streamLifecycleKey(name string) string { + return fmt.Sprintf("%s%s:lifecycle", streamKeyPrefix, name) +} + +// streamKey identifies the flat physical key adopted by generation one. +func streamKey(name string) string { + return streamKeyPrefix + name +} + +// parseStreamIdentity decodes the generation and physical key selected by the +// lifecycle scripts. +func parseStreamIdentity(result []any) (string, string, string, error) { + if len(result) != 3 { + return "", "", "", fmt.Errorf("stream lifecycle returned %d values", len(result)) + } + generation, ok := result[0].(string) + if !ok { + return "", "", "", fmt.Errorf("stream lifecycle returned generation %T", result[0]) + } + physical, ok := result[1].(string) + if !ok { + return "", "", "", fmt.Errorf("stream lifecycle returned physical key %T", result[1]) + } + deadline, ok := result[2].(string) + if !ok { + return "", "", "", fmt.Errorf("stream lifecycle returned deadline %T", result[2]) + } + return generation, physical, deadline, nil +} + +// parseLifecycleIdentity also returns the canonical retention configuration +// selected by the lifecycle owner. +func parseLifecycleIdentity(result []any) (string, string, string, string, error) { + if len(result) != 4 { + return "", "", "", "", fmt.Errorf("stream lifecycle returned %d values", len(result)) + } + generation, physical, deadline, err := parseStreamIdentity(result[:3]) + if err != nil { + return "", "", "", "", err + } + retention, ok := result[3].(string) + if !ok || retention == "" { + return "", "", "", "", fmt.Errorf("stream lifecycle returned retention %T", result[3]) + } + return generation, physical, deadline, retention, nil +} + +// requestedDeadline returns the construction-time absolute deadline in Redis +// millisecond form, or empty when this handle adopts the generation contract. +func (s *Stream) requestedDeadline() string { + if !s.deadlineConfigured { + return "" + } + return strconv.FormatInt(s.deadline.UnixMilli(), 10) +} + +// parseDeadline decodes the canonical lifecycle deadline already validated by +// the owning Redis script. +func parseDeadline(value string) (time.Time, error) { + if value == "" { + return time.Time{}, nil + } + milliseconds, err := strconv.ParseInt(value, 10, 64) + if err != nil { + return time.Time{}, fmt.Errorf("invalid Redis-owned stream deadline %q: %w", value, err) + } + return time.UnixMilli(milliseconds), nil +} + +// streamLifecycleBoundaryError maps lifecycle configuration errors returned +// before a Stream handle has bound its generation. +func streamLifecycleBoundaryError(err error) error { + if redis.HasErrorPrefix(err, streamDestroyedError) { + return ErrStreamDestroyed + } + if redis.HasErrorPrefix(err, streamDeadlineError) { + return ErrDeadlineElapsed + } + if redis.HasErrorPrefix(err, streamConflictError) { + return fmt.Errorf( + "%w: stream deadline conflicts with active generation", + ErrStreamConfigMismatch, + ) + } + if redis.HasErrorPrefix(err, streamConfigError) { + return ErrStreamConfigMismatch + } + if redis.HasErrorPrefix(err, streamNotFoundError) { + return ErrStreamNotFound + } + return err +} + +// retentionConfig serializes the typed immutable retention contract compared +// by every generation-opening Redis operation. +func (s *Stream) retentionConfig() string { + mode := "none" + value := int64(0) + sliding := false + switch { + case s.deadlineConfigured: + mode = "deadline" + value = s.deadline.UnixMilli() + case s.ttl > 0: + mode = "ttl" + value = s.ttl.Milliseconds() + sliding = s.ttlSliding + } + return fmt.Sprintf( + "v=%s|max=%d|mode=%s|value=%d|sliding=%t", + streamFormatVersion, + s.maxLen, + mode, + value, + sliding, + ) +} + +// requestedRetention returns the explicit configuration or the documented +// default used only when a writer creates the first generation. +func requestedRetention(s *Stream) string { + return s.retention +} + +// applyRetentionConfig adopts the Redis-owned immutable configuration into an +// unconfigured local handle. +func (s *Stream) applyRetentionConfig(config string) error { + parts := strings.Split(config, "|") + if len(parts) != 5 || parts[0] != "v="+streamFormatVersion { + return fmt.Errorf("invalid stream retention configuration %q", config) + } + maxLen, err := strconv.Atoi(strings.TrimPrefix(parts[1], "max=")) + if err != nil || maxLen < 0 { + return fmt.Errorf("invalid stream retention maximum in %q", config) + } + mode := strings.TrimPrefix(parts[2], "mode=") + value, err := strconv.ParseInt(strings.TrimPrefix(parts[3], "value="), 10, 64) + if err != nil { + return fmt.Errorf("invalid stream retention value in %q", config) + } + sliding, err := strconv.ParseBool(strings.TrimPrefix(parts[4], "sliding=")) + if err != nil { + return fmt.Errorf("invalid stream retention sliding flag in %q", config) + } + switch mode { + case "none": + if value != 0 || sliding { + return fmt.Errorf("invalid none retention configuration %q", config) + } + s.ttl = 0 + s.ttlSliding = false + case "ttl": + if value <= 0 { + return fmt.Errorf("invalid ttl retention configuration %q", config) + } + s.ttl = time.Duration(value) * time.Millisecond + s.ttlSliding = sliding + case "deadline": + if value <= 0 || sliding { + return fmt.Errorf("invalid deadline retention configuration %q", config) + } + s.ttl = 0 + default: + return fmt.Errorf("invalid stream retention mode %q", mode) + } + s.maxLen = maxLen + s.retention = config + return nil +} + +// boolString encodes booleans for Lua arguments. +func boolString(value bool) string { + if value { + return "1" + } + return "0" +} diff --git a/streaming/streams.go b/streaming/streams.go index 87ff236..125d178 100644 --- a/streaming/streams.go +++ b/streaming/streams.go @@ -2,8 +2,10 @@ package streaming import ( "context" + "errors" "fmt" "regexp" + "sync" "time" redis "github.com/redis/go-redis/v9" @@ -20,23 +22,72 @@ type ( Stream struct { // Name of the stream. Name string - // MaxLen is the maximum number of events in the stream. + // MaxLen is the maximum retained event count requested at + // construction. It is immutable after NewStream and remains exported + // for v1 source compatibility; the generation's canonical bound is + // tracked privately so concurrent readers of this field never race + // with binding. MaxLen int + // maxLen is the canonical maximum retained event count adopted from + // the bound generation. Zero means the stream is explicitly + // unbounded. + maxLen int // ttl configures an expiry for the Redis key backing the stream. ttl time.Duration // ttlSliding controls whether ttl is refreshed on every Add call. ttlSliding bool + // deadline is the Redis-canonical absolute expiry for this generation. + deadline time.Time + // deadlineConfigured records whether construction explicitly requested + // deadline, so lifecycle binding can reject a conflicting value. + deadlineConfigured bool + // retention is the canonical immutable generation configuration. + retention string + // retentionExplicit distinguishes writer configuration from an + // unconfigured handle that adopts the active generation. + retentionExplicit bool // logger is the logger used by the stream. logger pulse.Logger // rootLogger is the prefix-free logger used to create sink loggers. rootLogger pulse.Logger - // key is the redis key used for the stream. + // key is the immutable lifecycle-selected Redis event-stream key. key string + // lifecycleKey is the Redis-owned logical stream identity. + lifecycleKey string + // generation is the immutable incarnation established by the first + // caller-context operation. + generation string + // generationLock serializes lazy generation establishment. + generationLock sync.Mutex // rdb is the redis connection. rdb *redis.Client } ) +var ( + // ErrStreamDestroyed is returned when an operation uses a Stream whose + // immutable generation is no longer active. + ErrStreamDestroyed = errors.New("pulse streaming: stream generation is destroyed") + // ErrIdempotencyConflict is returned when an AddOnce key already identifies + // different event content in the same stream generation. + ErrIdempotencyConflict = errors.New("pulse streaming: idempotency key conflicts with existing event") + // ErrDeadlineElapsed is returned when a deadline-owned stream operation is + // attempted at or after its Redis-authoritative absolute deadline. + ErrDeadlineElapsed = errors.New("pulse streaming: stream deadline elapsed") + // ErrStreamNotFound is returned when Snapshot observes no initialized + // lifecycle. Snapshot never creates one. + ErrStreamNotFound = errors.New("pulse streaming: stream is not initialized") + // ErrStreamConfigMismatch is returned when an explicitly configured handle + // differs from the active generation's immutable retention contract. + ErrStreamConfigMismatch = errors.New("pulse streaming: stream retention configuration mismatch") + // ErrSnapshotUnbounded is returned before XRANGE when Snapshot is called on + // a generation whose immutable retention contract has no MaxLen bound. + ErrSnapshotUnbounded = errors.New("pulse streaming: snapshot requires bounded stream retention") + // ErrSnapshotBoundExceeded reports physical data that exceeds the active + // generation's immutable MaxLen contract. + ErrSnapshotBoundExceeded = errors.New("pulse streaming: snapshot retention bound violated") +) + const ( // streamKeyPrefix is the prefix used for stream keys. streamKeyPrefix = "pulse:stream:" @@ -48,35 +99,78 @@ const ( topicKey = "t" ) -// NewStream returns the stream with the given name. All stream instances -// with the same name share the same events. +// NewStream validates options and returns a local stream handle without +// performing Redis I/O. The first caller-context operation establishes or +// loads the active generation. WithStreamDeadline claims an immutable absolute +// generation deadline and cannot be combined with TTL options. func NewStream(name string, rdb *redis.Client, opts ...options.Stream) (*Stream, error) { if !isValidRedisKeyName(name) { return nil, fmt.Errorf("pulse stream: not a valid name %q", name) } o := options.ParseStreamOptions(opts...) - if o.TTL < 0 { - return nil, fmt.Errorf("pulse stream: ttl must be >= 0") + if o.Unbounded && o.MaxLenSet { + return nil, fmt.Errorf("pulse stream: maximum length and unbounded options are mutually exclusive") + } + if o.Unbounded { + o.MaxLen = 0 + } else if o.MaxLen <= 0 { + return nil, fmt.Errorf("pulse stream: maximum length must be greater than zero") + } + if o.TTLSet && o.TTL < time.Millisecond { + return nil, fmt.Errorf("pulse stream: ttl must be at least 1ms") + } + if o.DeadlineSet && o.Deadline.IsZero() { + return nil, fmt.Errorf("pulse stream: deadline must not be zero") + } + if o.DeadlineSet && o.TTL != 0 { + return nil, fmt.Errorf("pulse stream: deadline and ttl options are mutually exclusive") } - var logger pulse.Logger + var ( + logger pulse.Logger + rootLogger pulse.Logger + ) if o.Logger != nil { + rootLogger = o.Logger logger = o.Logger.WithPrefix("stream", name) } else { - logger = pulse.NoopLogger() + rootLogger = pulse.NoopLogger() + logger = rootLogger } s := &Stream{ - Name: name, - MaxLen: o.MaxLen, - ttl: o.TTL, - ttlSliding: o.TTLSliding, - logger: logger, - rootLogger: o.Logger, - key: streamKeyPrefix + name, - rdb: rdb, + Name: name, + MaxLen: o.MaxLen, + maxLen: o.MaxLen, + ttl: o.TTL, + ttlSliding: o.TTLSliding, + deadline: o.Deadline, + deadlineConfigured: o.DeadlineSet, + retentionExplicit: o.MaxLenSet || o.Unbounded || o.TTLSet || o.DeadlineSet, + logger: logger, + rootLogger: rootLogger, + key: streamKey(name), + lifecycleKey: streamLifecycleKey(name), + rdb: rdb, } + s.retention = s.retentionConfig() return s, nil } +// Generation returns the immutable Redis-owned incarnation established for s, +// or the empty string before the first caller-context operation. +func (s *Stream) Generation() string { + s.generationLock.Lock() + defer s.generationLock.Unlock() + return s.generation +} + +// Open establishes or adopts the active Redis incarnation and verifies that a +// previously bound handle still names the active generation. Higher-level +// runtimes use Open as their lifecycle fence before mutating generation-owned +// resources outside the event stream. +func (s *Stream) Open(ctx context.Context) error { + return s.verifyGeneration(ctx) +} + // NewReader creates a new stream reader. All reader instances get all the // events in the stream. Events are read starting: // - from the last event by default @@ -86,20 +180,28 @@ func NewStream(name string, rdb *redis.Client, opts ...options.Stream) (*Stream, // event is still in the stream, oldest event otherwise // - from the event added on or after the timestamp provided via // WithReaderStartAt if still in the stream, oldest event otherwise +// +// NewReader does not initialize an absent lifecycle; it returns +// ErrStreamNotFound until a writer or Open establishes the stream. func (s *Stream) NewReader(ctx context.Context, opts ...options.Reader) (*Reader, error) { - reader, err := newReader(s, opts...) + reader, err := newReader(ctx, s, opts...) if err != nil { s.logger.Error(fmt.Errorf("failed to create reader: %w", err)) return nil, err } + if err := s.verifyExistingGeneration(ctx); err != nil { + reader.cancel() + return nil, err + } + reader.streamKeys[0] = s.key s.logger.Info("create reader", "start", reader.startID) return reader, nil } // NewSink creates a new stream sink with the given name. All sink instances // with the same name share the same stream cursor. Events read through a sink -// are not removed from the stream until they are acked by the client unless the -// WithNoAck option is used. Events are read starting: +// are not removed from the stream until they are acked by the client unless +// WithSinkNoAck is used. Events are read starting: // - from the last event by default // - from the oldest event stored in the stream if the WithSinkStartAtOldest // option is used @@ -119,58 +221,26 @@ func (s *Stream) NewSink(ctx context.Context, name string, opts ...options.Sink) // Add appends an event to the stream and returns its ID. If the option // WithOnlyIfStreamExists is used and the stream does not exist then no event is // added and the empty string is returned. The stream is created if the option -// is omitted or when NewSink is called. +// is omitted or when NewSink is called. On a deadline-owned generation, Add +// rejects elapsed deadlines and reapplies the same absolute expiry without +// extending it. func (s *Stream) Add(ctx context.Context, name string, payload []byte, opts ...options.AddEvent) (string, error) { o := options.ParseAddEventOptions(opts...) - for _, option := range opts { - option(&o) - } - values := []any{nameKey, name, payloadKey, payload} - if o.Topic != "" { - values = append(values, topicKey, o.Topic) - } - res, err := s.rdb.XAdd(ctx, &redis.XAddArgs{ - Stream: s.key, - Values: values, - MaxLen: int64(s.MaxLen), - Approx: true, - NoMkStream: o.OnlyIfStreamExists, - }).Result() + res, err := s.addEvent(ctx, name, payload, o.OnlyIfStreamExists, o.Topic) if err != nil { - if err == redis.Nil { - // Stream does not exist and OnlyIfStreamExists option was used. - return "", nil - } err = fmt.Errorf("failed to add event: %w", err) s.logger.Error(err, "event", name) return "", err } - if err := s.applyTTL(ctx); err != nil { - err = fmt.Errorf("failed to apply stream TTL: %w", err) - s.logger.Error(err, "event", name) - return "", err - } s.logger.Info("add", "event", name, "id", res) return res, nil } -func (s *Stream) applyTTL(ctx context.Context) error { - if s.ttl <= 0 { - return nil - } - if s.ttlSliding { - return s.rdb.Expire(ctx, s.key, s.ttl).Err() - } - _, err := s.rdb.ExpireNX(ctx, s.key, s.ttl).Result() - return err -} - // Remove removes the events with the given IDs from the stream. // Note: clients should not need to call this method in normal operation, // instead they should use the Ack method to acknowledge events. func (s *Stream) Remove(ctx context.Context, ids ...string) error { - err := s.rdb.XDel(ctx, s.key, ids...).Err() - if err != nil { + if err := s.removeEvents(ctx, ids); err != nil { err = fmt.Errorf("failed to remove events: %w", err) s.logger.Error(err, "events", ids) return err @@ -179,15 +249,14 @@ func (s *Stream) Remove(ctx context.Context, ids ...string) error { return nil } -// Destroy deletes the entire stream: its events and every piece of sink -// metadata (consumer groups, recovery cursors, leases, and the sink -// membership map) in one atomic operation. The stream lifecycle is marked -// destroyed so concurrent sinks cannot resurrect the metadata; only a -// subsequent NewSink or Sink.AddStream call deliberately recreates the -// stream. Destroy is idempotent. +// Destroy invalidates this exact generation and atomically deletes its event +// data, consumer groups, recovery cursors, exact-publication records, and +// membership state. It is idempotent after this handle has bound a generation; +// an unbound absent name returns ErrStreamNotFound without creating lifecycle +// state. Existing Stream, Reader, Sink, Event, and Acker values cannot affect a +// later generation. func (s *Stream) Destroy(ctx context.Context) error { if err := destroyStream(ctx, s); err != nil { - err := fmt.Errorf("failed to destroy stream: %w", err) s.logger.Error(err) return err } diff --git a/streaming/streams_test.go b/streaming/streams_test.go index 46d442b..a47771c 100644 --- a/streaming/streams_test.go +++ b/streaming/streams_test.go @@ -1,11 +1,12 @@ package streaming import ( - "strconv" + "errors" "testing" "time" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "goa.design/pulse/pulse" "goa.design/pulse/streaming/options" @@ -19,10 +20,15 @@ func TestDestroy(t *testing.T) { s, err := NewStream("testDestroy", rdb) assert.NoError(t, err) + require.ErrorIs(t, s.Destroy(ctx), ErrStreamNotFound) + require.NoError(t, s.Open(ctx)) assert.NoError(t, s.Destroy(ctx)) exists, err := rdb.Exists(ctx, s.key).Result() assert.NoError(t, err) assert.Equal(t, int64(0), exists) + generation := s.Generation() + require.NoError(t, s.Destroy(ctx)) + require.Equal(t, generation, s.Generation()) s2, err := NewStream("testDestroy2", rdb) assert.NoError(t, err) @@ -34,11 +40,42 @@ func TestDestroy(t *testing.T) { assert.NoError(t, s2.Destroy(ctx)) } +func TestNewStreamLazilyLoadsActiveRedisGeneration(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + + first, err := NewStream(t.Name(), rdb) + require.NoError(t, err) + second, err := NewStream(t.Name(), rdb) + require.NoError(t, err) + require.Empty(t, first.generation) + require.Empty(t, second.generation) + require.Zero(t, rdb.DBSize(ctx).Val()) + require.Equal(t, first.generation, second.generation) + require.Equal(t, first.key, second.key) + _, err = first.Add(ctx, "first", []byte("payload")) + require.NoError(t, err) + _, err = second.Add(ctx, "second", []byte("payload")) + require.NoError(t, err) + require.NotEmpty(t, first.generation) + require.Equal(t, first.generation, second.generation) + require.NoError(t, first.Destroy(ctx)) + require.NoError(t, second.Destroy(ctx)) +} + func TestOptions(t *testing.T) { - s, err := NewStream("testOptions", nil, options.WithStreamMaxLen(10), options.WithStreamLogger(nil)) + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + s, err := NewStream("testOptions", rdb, options.WithStreamMaxLen(10), options.WithStreamLogger(nil)) assert.NoError(t, err) assert.Equal(t, 10, s.MaxLen) assert.Equal(t, pulse.NoopLogger(), s.logger) + require.NoError(t, s.Open(ctx)) + assert.NoError(t, s.Destroy(ctx)) + _, err = NewStream("invalidMaxLen", rdb, options.WithStreamMaxLen(0)) + assert.EqualError(t, err, "pulse stream: maximum length must be greater than zero") } func TestAdd(t *testing.T) { @@ -122,7 +159,7 @@ func TestStreamTTLSliding(t *testing.T) { assert.NoError(t, s.Destroy(ctx)) } -func TestStreamDestroyUsesRMapDestroyProtocol(t *testing.T) { +func TestStreamDestroyDeletesGenerationMembership(t *testing.T) { rdb := ptesting.NewRedisClient(t) defer ptesting.CleanupRedis(t, rdb, false, "") ctx := ptesting.NewTestContext(t) @@ -132,20 +169,198 @@ func TestStreamDestroyUsesRMapDestroyProtocol(t *testing.T) { sink, err := s.NewSink(ctx, "gateway") assert.NoError(t, err) - sink.Close(ctx) + assert.True(t, rdb.HExists(ctx, recoveryCursorKey(s), sink.Name).Val()) + require.NoError(t, sink.Close(ctx)) - mapKey := "map:stream:testStreamDestroyDeletesSinkMap:sinks:content" + mapKey := consumersMapContentKey(s) exists, err := rdb.Exists(ctx, mapKey).Result() assert.NoError(t, err) assert.EqualValues(t, 1, exists) assert.NoError(t, s.Destroy(ctx)) - content, err := rdb.HGetAll(ctx, mapKey).Result() - assert.NoError(t, err) - assert.Len(t, content, 2) - assert.Equal(t, "destroy", content["=kind"]) - _, err = strconv.ParseUint(content["=rev"], 10, 64) - assert.NoError(t, err) + assert.EqualValues(t, 0, rdb.Exists(ctx, mapKey).Val()) + assert.EqualValues(t, 0, rdb.Exists(ctx, recoveryCursorKey(s)).Val()) +} + +func TestStreamDestroyInvalidatesExactGeneration(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + s, err := NewStream(t.Name(), rdb) + require.NoError(t, err) + sink, err := s.NewSink(ctx, "gateway", options.WithSinkBlockDuration(testBlockDuration)) + require.NoError(t, err) + events := sink.Subscribe() + eventID, err := s.Add(ctx, "before-destroy", []byte("payload")) + require.NoError(t, err) + event := receiveSinkEvent(t, events) + require.Equal(t, eventID, event.ID) + + require.NoError(t, s.Destroy(ctx)) + require.EqualValues(t, 0, rdb.Exists(ctx, s.key).Val()) + require.EqualValues(t, 0, rdb.Exists(ctx, recoveryCursorKey(s)).Val()) + require.EqualValues(t, 0, rdb.Exists(ctx, consumersMapContentKey(s)).Val()) + require.Eventually(t, sink.IsClosed, max, delay) + _, err = s.Add(ctx, "stale", []byte("payload")) + require.ErrorIs(t, err, ErrStreamDestroyed) + require.ErrorIs(t, s.Remove(ctx, eventID), ErrStreamDestroyed) + require.ErrorIs(t, sink.AddStream(ctx, s), ErrSinkClosed) + require.ErrorIs(t, sink.RemoveStream(ctx, s), ErrSinkClosed) + require.ErrorIs(t, sink.Ack(ctx, event), ErrStreamDestroyed) + _, err = s.NewReader(ctx) + require.ErrorIs(t, err, ErrStreamDestroyed) + _, err = s.NewSink(ctx, "new-sink") + require.ErrorIs(t, err, ErrStreamDestroyed) + require.EqualValues(t, 0, rdb.Exists(ctx, consumersMapContentKey(s)).Val()) + + late, err := NewStream(t.Name(), rdb) + require.NoError(t, err) + _, err = late.Add(ctx, "late", []byte("payload"), options.WithOnlyIfStreamExists()) + require.ErrorIs(t, err, ErrStreamDestroyed) + require.Equal(t, s.generation, rdb.HGet(ctx, s.lifecycleKey, "generation").Val()) + require.Equal(t, streamStateDestroyed, rdb.HGet(ctx, s.lifecycleKey, "state").Val()) + + next, err := NewStream(t.Name(), rdb) + require.NoError(t, err) + require.Empty(t, next.generation) + require.Equal(t, s.key, next.key) + _, err = next.Add(ctx, "fresh", []byte("payload")) + require.NoError(t, err) + require.NotEqual(t, s.generation, next.generation) + require.NotEqual(t, s.key, next.key) + require.EqualValues(t, 0, rdb.Exists(ctx, s.key).Val()) + require.EqualValues(t, 1, rdb.Exists(ctx, next.key).Val()) + require.EqualValues(t, 0, rdb.Exists(ctx, recoveryCursorKey(s)).Val()) + require.NoError(t, next.Destroy(ctx)) +} + +func TestReaderStopsWhenGenerationIsDestroyed(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + stream, err := NewStream(t.Name(), rdb) + require.NoError(t, err) + require.NoError(t, stream.Open(ctx)) + reader, err := stream.NewReader(ctx, options.WithReaderBlockDuration(testBlockDuration)) + require.NoError(t, err) + reader.Subscribe() + + require.NoError(t, stream.Destroy(ctx)) + require.Eventually(t, reader.IsClosed, max, delay) +} + +func TestRecreatedStreamIsIsolatedFromStaleReadersAndSinks(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + stream, err := NewStream(t.Name(), rdb) + require.NoError(t, err) + require.NoError(t, stream.Open(ctx)) + reader, err := stream.NewReader(ctx, options.WithReaderStartAtOldest(), options.WithReaderBlockDuration(testBlockDuration)) + require.NoError(t, err) + readerEvents := reader.Subscribe() + sink, err := stream.NewSink( + ctx, + "sink", + options.WithSinkStartAtOldest(), + options.WithSinkBlockDuration(testBlockDuration), + ) + require.NoError(t, err) + sinkEvents := sink.Subscribe() + + require.NoError(t, stream.Destroy(ctx)) + require.Eventually(t, reader.IsClosed, max, delay) + require.Eventually(t, sink.IsClosed, max, delay) + + recreated, err := NewStream(t.Name(), rdb) + require.NoError(t, err) + freshID, err := recreated.Add(ctx, "fresh", []byte("payload")) + require.NoError(t, err) + require.NotEqual(t, stream.key, recreated.key) + select { + case event := <-readerEvents: + require.Nil(t, event) + default: + } + select { + case event := <-sinkEvents: + require.Nil(t, event) + default: + } + + freshReader, err := recreated.NewReader( + ctx, + options.WithReaderStartAtOldest(), + options.WithReaderBlockDuration(testBlockDuration), + ) + require.NoError(t, err) + event := receiveSinkEvent(t, freshReader.Subscribe()) + require.Equal(t, freshID, event.ID) + freshReader.Close() + require.NoError(t, recreated.Destroy(ctx)) +} + +func TestRepeatedStreamGenerationsLeaveBoundedKeys(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + for range 10 { + stream, err := NewStream(t.Name(), rdb) + require.NoError(t, err) + sink, err := stream.NewSink(ctx, "sink", options.WithSinkBlockDuration(testBlockDuration)) + require.NoError(t, err) + require.NoError(t, sink.Close(ctx)) + require.NoError(t, stream.Destroy(ctx)) + } + + generationKeys, err := rdb.Keys(ctx, streamKeyPrefix+t.Name()+":generation:*").Result() + require.NoError(t, err) + require.Empty(t, generationKeys) + streamKeys, err := rdb.Keys(ctx, streamKeyPrefix+t.Name()+"*").Result() + require.NoError(t, err) + require.Equal(t, []string{streamLifecycleKey(t.Name())}, streamKeys) +} + +func TestConcurrentRemoveAndDestroyNeverRecreatesGeneration(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + mainStream, err := NewStream(t.Name()+"-main", rdb) + require.NoError(t, err) + addedStream, err := NewStream(t.Name()+"-added", rdb) + require.NoError(t, err) + sink, err := mainStream.NewSink(ctx, "gateway", options.WithSinkBlockDuration(testBlockDuration)) + require.NoError(t, err) + require.NoError(t, sink.AddStream(ctx, addedStream)) + + start := make(chan struct{}) + removeResult := make(chan error, 1) + destroyResult := make(chan error, 1) + go func() { + <-start + removeResult <- sink.RemoveStream(ctx, addedStream) + }() + go func() { + <-start + destroyResult <- addedStream.Destroy(ctx) + }() + close(start) + + removeErr := <-removeResult + if removeErr != nil { + require.True( + t, + errors.Is(removeErr, ErrStreamDestroyed) || errors.Is(removeErr, ErrSinkClosed), + "unexpected remove result: %v", + removeErr, + ) + } + destroyErr := <-destroyResult + require.NoError(t, destroyErr) + time.Sleep(4 * testBlockDuration) + require.EqualValues(t, 0, rdb.Exists(ctx, addedStream.key, recoveryCursorKey(addedStream)).Val()) + require.NoError(t, sink.Close(ctx)) + require.NoError(t, mainStream.Destroy(ctx)) } func TestRemove(t *testing.T) { diff --git a/streaming/testing.go b/streaming/testing.go index 8bda437..1040277 100644 --- a/streaming/testing.go +++ b/streaming/testing.go @@ -11,12 +11,12 @@ import ( ) var ( - max = time.Second + max = 5 * time.Second delay = 10 * time.Millisecond ) // readOneEvent reads one event from the channel and acks it or fails the test -// if it takes more than a second. +// if it takes more than the shared Redis test bound. func readOneEvent(t *testing.T, ctx context.Context, c <-chan *Event, sink *Sink) *Event { t.Helper() var read *Event @@ -24,7 +24,7 @@ func readOneEvent(t *testing.T, ctx context.Context, c <-chan *Event, sink *Sink w.Add(1) go func() { defer w.Done() - tck := time.NewTicker(time.Second) + tck := time.NewTicker(max) select { case read = <-c: assert.NoError(t, sink.Ack(ctx, read)) @@ -38,7 +38,7 @@ func readOneEvent(t *testing.T, ctx context.Context, c <-chan *Event, sink *Sink } // readOneReaderEvent reads one event from the channel or fails the test if it -// takes more than a second. +// takes more than the shared Redis test bound. func readOneReaderEvent(t *testing.T, c <-chan *Event) *Event { t.Helper() var read *Event @@ -46,7 +46,7 @@ func readOneReaderEvent(t *testing.T, c <-chan *Event) *Event { w.Add(1) go func() { defer w.Done() - tck := time.NewTicker(time.Second) + tck := time.NewTicker(max) select { case read = <-c: return @@ -60,22 +60,25 @@ func readOneReaderEvent(t *testing.T, c <-chan *Event) *Event { return read } -// cleanupSink closes the sink and asserts that it is closed within a second. +// cleanupSink closes the sink and asserts that it is closed within the shared +// Redis test bound. func cleanupSink(t *testing.T, ctx context.Context, s *Stream, sink *Sink) { t.Helper() if sink != nil { - sink.Close(ctx) + require.NoError(t, sink.Close(ctx)) assert.Eventually(t, func() bool { return sink.IsClosed() }, max, delay) } if s != nil { - assert.NoError(t, s.Destroy(ctx)) + require.NoError(t, s.Destroy(ctx)) } } -// cleanupReader closes the reader and asserts that it is closed within a second. -func cleanupReader(t *testing.T, ctx context.Context, s *Stream, reader *Reader) { +// cleanupReader closes the reader and asserts that it is closed within the +// shared Redis test bound. +// Stream destruction remains explicit in each test so duplicate cleanup cannot +// mask ErrStreamDestroyed. +func cleanupReader(t *testing.T, reader *Reader) { t.Helper() reader.Close() assert.Eventually(t, func() bool { return reader.IsClosed() }, max, delay) - assert.NoError(t, s.Destroy(ctx)) } diff --git a/testing/redis.go b/testing/redis.go index 940f4ad..8eef9b4 100644 --- a/testing/redis.go +++ b/testing/redis.go @@ -5,10 +5,13 @@ import ( "fmt" "os" "regexp" + "strconv" "strings" + "sync" "testing" "time" + "github.com/oklog/ulid/v2" redis "github.com/redis/go-redis/v9" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -21,6 +24,43 @@ var ( redisAddr = "localhost:6379" // streamRegexp is a regular expression that matches valid stream keys streamRegexp = regexp.MustCompile(`^pulse:stream:[^:]+:node:.*`) + // renewDatabaseLeaseScript extends only the lease still owned by this test. + renewDatabaseLeaseScript = redis.NewScript(` +if redis.call("GET", KEYS[1]) ~= ARGV[1] then + return 0 +end +redis.call("PEXPIRE", KEYS[1], ARGV[2]) +return 1 +`) + // releaseDatabaseLeaseScript releases only the lease still owned by this + // test process. + releaseDatabaseLeaseScript = redis.NewScript(` +if redis.call("GET", KEYS[1]) ~= ARGV[1] then + return 0 +end +return redis.call("DEL", KEYS[1]) +`) +) + +const ( + databaseLeaseTTL = 2 * time.Minute + databaseLeaseHeartbeat = 30 * time.Second + databaseLeaseWait = 30 * time.Second +) + +type ( + // databaseLease owns one non-coordination Redis database for one test, + // including cross-process heartbeat and release. + databaseLease struct { + coordinator *redis.Client + key string + token string + stop chan struct{} + done chan struct{} + lock sync.Mutex + heartbeatErr error + t *testing.T + } ) func init() { @@ -32,10 +72,18 @@ func init() { } } +// NewRedisClient returns a client backed by a cross-process leased Redis +// database. Package names are irrelevant, and FlushDB is safe because the +// caller holds the database's exclusive lease until test cleanup completes. func NewRedisClient(t *testing.T) *redis.Client { t.Helper() - rdb := redis.NewClient(&redis.Options{Addr: redisAddr, Password: redisPwd}) + lease, db := acquireDatabaseLease(t) + t.Cleanup(func() { + lease.release(t) + }) + rdb := redis.NewClient(&redis.Options{Addr: redisAddr, Password: redisPwd, DB: db}) require.NoError(t, rdb.Ping(context.Background()).Err()) + require.NoError(t, rdb.FlushDB(context.Background()).Err()) return rdb } @@ -47,53 +95,56 @@ func CleanupRedis(t *testing.T, rdb *redis.Client, checkClean bool, testName str t.Helper() ctx := context.Background() if checkClean { - var ( - filtered []string - keysErr error - ) - assert.Eventually(t, func() bool { - var keys []string - keys, keysErr = rdb.Keys(ctx, "*").Result() - if keysErr != nil { - filtered = []string{fmt.Sprintf("keys error: %v", keysErr)} - return false - } - filtered = filtered[:0] - for _, k := range keys { - if strings.HasSuffix(k, ":sinks:content") { - // Sinks content is cleaned up asynchronously, so ignore it - continue - } - if isDestroyTombstone(ctx, rdb, k) { - // Destroy tombstones are intentional rmap protocol state used so - // reconnecting replicas can order the next generation correctly. - continue - } - if isDestroyedStreamLifecycle(ctx, rdb, k) { - // Destroyed stream lifecycles are the intentional fence that - // prevents concurrent sinks from resurrecting stream metadata. - continue - } - if streamRegexp.MatchString(k) { - // Node streams are cleaned up asynchronously, so ignore them - continue - } - if strings.Contains(k, testName) { - filtered = append(filtered, k) - } - } - return len(filtered) == 0 - }, 5*time.Second, time.Millisecond*10, "found keys: %v", filtered) - require.NoError(t, keysErr) + clean := assert.Eventually(t, func() bool { + filtered, err := remainingTestKeys(ctx, rdb, testName) + return err == nil && len(filtered) == 0 + }, 5*time.Second, time.Millisecond*10) + if !clean { + filtered, err := remainingTestKeys(ctx, rdb, testName) + require.NoError(t, err) + t.Errorf("found keys: %v", filtered) + } } assert.NoError(t, rdb.FlushDB(ctx).Err()) + assert.NoError(t, rdb.Close()) +} + +// remainingTestKeys returns keys owned by the named test that are not durable +// protocol state or resources with documented asynchronous cleanup. +func remainingTestKeys(ctx context.Context, rdb *redis.Client, testName string) ([]string, error) { + keys, err := rdb.Keys(ctx, "*").Result() + if err != nil { + return nil, err + } + var filtered []string + for _, key := range keys { + if strings.HasPrefix(key, "pulse:pool:") && strings.HasSuffix(key, ":cleanup-generations") { + continue + } + if strings.HasPrefix(key, "pulse:pool:") && strings.HasSuffix(key, ":resources") { + continue + } + if isDestroyedStreamLifecycle(ctx, rdb, key) { + continue + } + if strings.HasSuffix(key, ":sinks:content") { + continue + } + if isDestroyTombstone(ctx, rdb, key) { + continue + } + if streamRegexp.MatchString(key) { + continue + } + if strings.Contains(key, testName) { + filtered = append(filtered, key) + } + } + return filtered, nil } -// isDestroyedStreamLifecycle reports whether key is the lifecycle fence of a -// destroyed stream. Destroyed lifecycles intentionally outlive Stream.Destroy -// so concurrent sinks cannot recreate the stream metadata. func isDestroyedStreamLifecycle(ctx context.Context, rdb *redis.Client, key string) bool { - if !strings.HasPrefix(key, "pulse:streammeta:") || !strings.HasSuffix(key, ":lifecycle") { + if !strings.HasPrefix(key, "pulse:stream:") || !strings.HasSuffix(key, ":lifecycle") { return false } state, err := rdb.HGet(ctx, key, "state").Result() @@ -117,3 +168,118 @@ func isDestroyTombstone(ctx context.Context, rdb *redis.Client, key string) bool _, ok := content["=rev"] return ok } + +// acquireDatabaseLease reserves one Redis database through DB 0, which is used +// only for coordination. Expired leases make crashed test processes harmless. +func acquireDatabaseLease(t *testing.T) (*databaseLease, int) { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), databaseLeaseWait) + defer cancel() + coordinator := redis.NewClient(&redis.Options{Addr: redisAddr, Password: redisPwd, DB: 0}) + if err := coordinator.Ping(ctx).Err(); err != nil { + _ = coordinator.Close() + require.NoError(t, err) + } + config, err := coordinator.ConfigGet(ctx, "databases").Result() + if err != nil { + _ = coordinator.Close() + require.NoError(t, err) + } + databaseCount, err := strconv.Atoi(config["databases"]) + if err != nil || databaseCount < 2 { + _ = coordinator.Close() + require.NoError(t, fmt.Errorf("invalid Redis database capacity %q", config["databases"])) + } + token := ulid.Make().String() + ticker := time.NewTicker(25 * time.Millisecond) + defer ticker.Stop() + for { + for db := 1; db < databaseCount; db++ { + key := fmt.Sprintf("pulse:test:db-lease:%d", db) + acquired, err := coordinator.SetNX(ctx, key, token, databaseLeaseTTL).Result() + if err != nil { + _ = coordinator.Close() + require.NoError(t, err) + } + if !acquired { + continue + } + lease := &databaseLease{ + coordinator: coordinator, + key: key, + token: token, + stop: make(chan struct{}), + done: make(chan struct{}), + t: t, + } + go lease.heartbeat() + return lease, db + } + select { + case <-ctx.Done(): + _ = coordinator.Close() + require.NoError( + t, + ctx.Err(), + "timed out waiting for one of %d isolated Redis test databases", + databaseCount-1, + ) + case <-ticker.C: + } + } +} + +// heartbeat renews the lease while the test owns its database. +func (l *databaseLease) heartbeat() { + defer close(l.done) + ticker := time.NewTicker(databaseLeaseHeartbeat) + defer ticker.Stop() + for { + select { + case <-ticker.C: + ctx, cancel := context.WithTimeout(context.Background(), databaseLeaseHeartbeat) + renewed, err := renewDatabaseLeaseScript.Run( + ctx, + l.coordinator, + []string{l.key}, + l.token, + databaseLeaseTTL.Milliseconds(), + ).Int64() + cancel() + if err != nil { + heartbeatErr := fmt.Errorf("renew Redis database lease: %w", err) + l.lock.Lock() + l.heartbeatErr = heartbeatErr + l.lock.Unlock() + l.t.Errorf("%v", heartbeatErr) + return + } + if renewed != 1 { + heartbeatErr := fmt.Errorf("Redis database lease %q was lost", l.key) + l.lock.Lock() + l.heartbeatErr = heartbeatErr + l.lock.Unlock() + l.t.Errorf("%v", heartbeatErr) + return + } + case <-l.stop: + return + } + } +} + +// release stops renewal and atomically frees the owned database lease. +func (l *databaseLease) release(t *testing.T) { + t.Helper() + close(l.stop) + <-l.done + l.lock.Lock() + heartbeatErr := l.heartbeatErr + l.lock.Unlock() + assert.NoError(t, heartbeatErr) + ctx, cancel := context.WithTimeout(context.Background(), databaseLeaseHeartbeat) + defer cancel() + err := releaseDatabaseLeaseScript.Run(ctx, l.coordinator, []string{l.key}, l.token).Err() + assert.NoError(t, err) + assert.NoError(t, l.coordinator.Close()) +}