From 92e293ddbd7ba74b6bd2d66c4c35f1ca162b73b8 Mon Sep 17 00:00:00 2001 From: "Raphael (manual office deploy after cloud-state fix)" Date: Fri, 24 Jul 2026 18:27:07 -0700 Subject: [PATCH 1/4] streaming: generation-qualified stream lifecycle and exact publication Streams bind lazily to one Redis-owned generation with a distinct physical key per explicit recreation. Every metadata mutation - publication, event removal, consumer-group recovery, consumer registration, keep-alive refresh, stale recovery, acknowledgement, and destruction - verifies the exact generation atomically in Redis, so destroyed-generation metadata can never be resurrected by a concurrent sink. Adds idempotent AddOnce publication and side-effect-free Snapshot reads, Redis-owned TTL/deadline retention, and a fenced stale-recovery lease. Exported option structs keep their v1 fields as a stable ordered prefix; unkeyed literals are pinned internally and keyed v1 construction is pinned for external compatibility. --- examples/rmap/basics/main.go | 2 +- examples/rmap/multi-nodes/main.go | 2 +- examples/streaming/README.md | 20 +- examples/streaming/exact-publication/main.go | 70 ++ examples/streaming/multi-readers/main.go | 2 +- examples/streaming/multi-sinks/main.go | 22 +- examples/streaming/multi-streams/main.go | 20 +- examples/streaming/pub-sub/main.go | 14 +- examples/streaming/single-reader/main.go | 8 +- examples/streaming/single-sink/main.go | 14 +- pulse/goroutine_test.go | 27 +- rmap/map_test.go | 79 +- streaming/README.md | 230 +++- streaming/exact_publication.go | 523 +++++++++ streaming/exact_publication_test.go | 638 ++++++++++ streaming/options/add_stream.go | 17 + streaming/options/compat_test.go | 45 +- streaming/options/options_test.go | 38 +- streaming/options/reader.go | 40 +- streaming/options/shape_pin_test.go | 31 + streaming/options/sink.go | 55 +- streaming/options/stream.go | 49 +- streaming/reader.go | 440 ++++--- streaming/reader_test.go | 229 +++- streaming/sink.go | 976 ++++++++++------ streaming/sink_config.go | 666 +++++++++++ streaming/sink_consumer.go | 315 +++++ streaming/sink_lease.go | 263 ----- streaming/sink_recovery.go | 661 ++++------- streaming/sink_recovery_test.go | 1100 +++++++++--------- streaming/sink_test.go | 983 +++++++++++++++- streaming/stream_lifecycle.go | 732 ++++++++++++ streaming/streams.go | 193 ++- streaming/streams_test.go | 237 +++- streaming/testing.go | 25 +- testing/redis.go | 252 +++- 36 files changed, 7003 insertions(+), 2015 deletions(-) create mode 100644 examples/streaming/exact-publication/main.go create mode 100644 streaming/exact_publication.go create mode 100644 streaming/exact_publication_test.go create mode 100644 streaming/options/shape_pin_test.go create mode 100644 streaming/sink_config.go create mode 100644 streaming/sink_consumer.go delete mode 100644 streaming/sink_lease.go create mode 100644 streaming/stream_lifecycle.go diff --git a/examples/rmap/basics/main.go b/examples/rmap/basics/main.go index b7e3c6e..719bd5d 100644 --- a/examples/rmap/basics/main.go +++ b/examples/rmap/basics/main.go @@ -14,7 +14,7 @@ func main() { ctx := context.Background() // Create Redis client - rdb := redis.NewClient(&redis.Options{Addr: "localhost:6379", Password: os.Getenv("REDIS_PASSWORD")}) + rdb := redis.NewClient(&redis.Options{Addr: os.Getenv("REDIS_ADDR"), Password: os.Getenv("REDIS_PASSWORD")}) // Make sure Redis is up and running and we can connect to it if err := rdb.Ping(ctx).Err(); err != nil { diff --git a/examples/rmap/multi-nodes/main.go b/examples/rmap/multi-nodes/main.go index dca0599..7e57513 100644 --- a/examples/rmap/multi-nodes/main.go +++ b/examples/rmap/multi-nodes/main.go @@ -18,7 +18,7 @@ func main() { ctx := context.Background() // Create Redis client - rdb := redis.NewClient(&redis.Options{Addr: "localhost:6379", Password: os.Getenv("REDIS_PASSWORD")}) + rdb := redis.NewClient(&redis.Options{Addr: os.Getenv("REDIS_ADDR"), Password: os.Getenv("REDIS_PASSWORD")}) // Make sure Redis is up and running and we can connect to it if err := rdb.Ping(ctx).Err(); err != nil { diff --git a/examples/streaming/README.md b/examples/streaming/README.md index 6905002..5b2127c 100644 --- a/examples/streaming/README.md +++ b/examples/streaming/README.md @@ -10,30 +10,34 @@ To run the examples, follow these steps: 2. Open a terminal or command prompt. + The examples read `REDIS_ADDR` and `REDIS_PASSWORD`; from the repository + root, `source .env` selects the default local Redis configuration. + 3. Clone the `goadesign/pulse` repository by running the following command: - ``` + ```bash git clone https://github.com/goadesign/pulse.git ``` 4. Change into the example directory (e.g. `examples/streaming/single-reader`): - ``` + ```bash cd pulse/examples/streaming/single-reader ``` - ``` -5. Install the required dependencies by running the following command: - ``` - go get github.com/redis/go-redis/v9 goa.design/pulse/rmap +5. Download the repository's pinned dependencies: + ```bash + go mod download ``` 6. Build the Go program by executing the following command: - ``` + ```bash go build ``` 7. Run the program using the following command: - ``` + ```bash ./single-reader ``` This will execute the program and demonstrate the basic operations on streaming. +The [`exact-publication`](exact-publication/main.go) example shows +deadline-owned `AddOnce` retries and read-only snapshots. diff --git a/examples/streaming/exact-publication/main.go b/examples/streaming/exact-publication/main.go new file mode 100644 index 0000000..41c1c57 --- /dev/null +++ b/examples/streaming/exact-publication/main.go @@ -0,0 +1,70 @@ +package main + +import ( + "context" + "fmt" + "os" + "time" + + "github.com/redis/go-redis/v9" + + "goa.design/pulse/streaming" + "goa.design/pulse/streaming/options" +) + +func main() { + ctx := context.Background() + rdb := redis.NewClient(&redis.Options{ + Addr: os.Getenv("REDIS_ADDR"), + Password: os.Getenv("REDIS_PASSWORD"), + }) + if err := rdb.Ping(ctx).Err(); err != nil { + panic(err) + } + + deadline := time.Now().Add(time.Minute).Truncate(time.Millisecond) + stream, err := streaming.NewStream( + "exact-publication", + rdb, + options.WithStreamDeadline(deadline), + options.WithStreamMaxLen(1_000), + ) + if err != nil { + panic(err) + } + defer func() { + if err := stream.Destroy(ctx); err != nil { + panic(err) + } + }() + + first, err := stream.AddOnce( + ctx, + "facility-42:alarm-7", + "alarm-opened", + []byte("high discharge pressure"), + options.WithTopic("alarms"), + ) + if err != nil { + panic(err) + } + retry, err := stream.AddOnce( + ctx, + "facility-42:alarm-7", + "alarm-opened", + []byte("high discharge pressure"), + options.WithTopic("alarms"), + ) + if err != nil { + panic(err) + } + fmt.Printf("first=%s retry=%s\n", first, retry) + + events, err := stream.Snapshot(ctx) + if err != nil { + panic(err) + } + for _, event := range events { + fmt.Printf("%s %s: %s\n", event.ID(), event.EventName(), event.Payload()) + } +} diff --git a/examples/streaming/multi-readers/main.go b/examples/streaming/multi-readers/main.go index b8baefd..48cec99 100644 --- a/examples/streaming/multi-readers/main.go +++ b/examples/streaming/multi-readers/main.go @@ -13,7 +13,7 @@ import ( func main() { // Create Redis client - rdb := redis.NewClient(&redis.Options{Addr: "localhost:6379", Password: os.Getenv("REDIS_PASSWORD")}) + rdb := redis.NewClient(&redis.Options{Addr: os.Getenv("REDIS_ADDR"), Password: os.Getenv("REDIS_PASSWORD")}) ctx := context.Background() // Make sure Redis is up and running and we can connect to it diff --git a/examples/streaming/multi-sinks/main.go b/examples/streaming/multi-sinks/main.go index 3a8ae08..7206c09 100644 --- a/examples/streaming/multi-sinks/main.go +++ b/examples/streaming/multi-sinks/main.go @@ -14,7 +14,7 @@ import ( // NOTE: the example below does not handle errors for brevity. func main() { // Create Redis client - rdb := redis.NewClient(&redis.Options{Addr: "localhost:6379", Password: os.Getenv("REDIS_PASSWORD")}) + rdb := redis.NewClient(&redis.Options{Addr: os.Getenv("REDIS_ADDR"), Password: os.Getenv("REDIS_PASSWORD")}) ctx := context.Background() // Make sure Redis is up and running and we can connect to it @@ -29,7 +29,11 @@ func main() { } // Don't forget to destroy the stream when done - defer stream.Destroy(ctx) + defer func() { + if err := stream.Destroy(ctx); err != nil { + panic(err) + } + }() // Write 2 events to the stream id1, err := stream.Add(ctx, "event 1", []byte("payload 1")) @@ -54,7 +58,11 @@ func main() { } // Don't forget to close the sink when done - defer sink1.Close(ctx) + defer func() { + if err := sink1.Close(ctx); err != nil { + panic(err) + } + }() // Read and acknowlege event ev := <-sink1.Subscribe() @@ -70,12 +78,16 @@ func main() { if err != nil { panic(err) } - defer sink2.Close(ctx) + defer func() { + if err := sink2.Close(ctx); err != nil { + panic(err) + } + }() // Read second event ev = <-sink2.Subscribe() fmt.Printf("sink 2, event: %s, payload: %s\n", ev.EventName, ev.Payload) - if sink2.Ack(ctx, ev); err != nil { + if err := sink2.Ack(ctx, ev); err != nil { panic(err) } } diff --git a/examples/streaming/multi-streams/main.go b/examples/streaming/multi-streams/main.go index 482a22e..231bd17 100644 --- a/examples/streaming/multi-streams/main.go +++ b/examples/streaming/multi-streams/main.go @@ -14,7 +14,7 @@ import ( // Note: the example below does not handle errors for brevity. func main() { // Create Redis client - rdb := redis.NewClient(&redis.Options{Addr: "localhost:6379", Password: os.Getenv("REDIS_PASSWORD")}) + rdb := redis.NewClient(&redis.Options{Addr: os.Getenv("REDIS_ADDR"), Password: os.Getenv("REDIS_PASSWORD")}) ctx := context.Background() // Make sure Redis is up and running and we can connect to it @@ -29,7 +29,11 @@ func main() { } // Don't forget to destroy the stream when done - defer stream1.Destroy(ctx) + defer func() { + if err := stream1.Destroy(ctx); err != nil { + panic(err) + } + }() // Create sink sink, err := stream1.NewSink(ctx, "multistreams-sink", @@ -40,7 +44,11 @@ func main() { } // Don't forget to close the sink when done - defer sink.Close(ctx) + defer func() { + if err := sink.Close(ctx); err != nil { + panic(err) + } + }() // Subscribe to events c := sink.Subscribe() @@ -57,7 +65,11 @@ func main() { if err != nil { panic(err) } - defer stream2.Destroy(ctx) + defer func() { + if err := stream2.Destroy(ctx); err != nil { + panic(err) + } + }() // Add stream to sink err = sink.AddStream(ctx, stream2) diff --git a/examples/streaming/pub-sub/main.go b/examples/streaming/pub-sub/main.go index b595128..a729814 100644 --- a/examples/streaming/pub-sub/main.go +++ b/examples/streaming/pub-sub/main.go @@ -14,7 +14,7 @@ import ( // NOTE: the example below does not handle errors for brevity. func main() { // Create Redis client - rdb := redis.NewClient(&redis.Options{Addr: "localhost:6379", Password: os.Getenv("REDIS_PASSWORD")}) + rdb := redis.NewClient(&redis.Options{Addr: os.Getenv("REDIS_ADDR"), Password: os.Getenv("REDIS_PASSWORD")}) ctx := context.Background() if err := rdb.Ping(ctx).Err(); err != nil { panic(err) @@ -27,7 +27,11 @@ func main() { } // Don't forget to destroy the stream when done - defer stream.Destroy(ctx) + defer func() { + if err := stream.Destroy(ctx); err != nil { + panic(err) + } + }() // Add a new event to topic "my-topic" id1, err := stream.Add(ctx, @@ -56,7 +60,11 @@ func main() { } // Don't forget to close the sink when done - defer sink.Close(ctx) + defer func() { + if err := sink.Close(ctx); err != nil { + panic(err) + } + }() // Read both events c := sink.Subscribe() diff --git a/examples/streaming/single-reader/main.go b/examples/streaming/single-reader/main.go index bb51235..f5cab2b 100644 --- a/examples/streaming/single-reader/main.go +++ b/examples/streaming/single-reader/main.go @@ -13,7 +13,7 @@ import ( func main() { // Create Redis client - rdb := redis.NewClient(&redis.Options{Addr: "localhost:6379", Password: os.Getenv("REDIS_PASSWORD")}) + rdb := redis.NewClient(&redis.Options{Addr: os.Getenv("REDIS_ADDR"), Password: os.Getenv("REDIS_PASSWORD")}) ctx := context.Background() if err := rdb.Ping(ctx).Err(); err != nil { panic(err) @@ -26,7 +26,11 @@ func main() { } // Don't forget to destroy the stream when done - defer stream.Destroy(ctx) + defer func() { + if err := stream.Destroy(ctx); err != nil { + panic(err) + } + }() // Add a new event id, err := stream.Add(ctx, "event", []byte("payload")) diff --git a/examples/streaming/single-sink/main.go b/examples/streaming/single-sink/main.go index b11a470..e2d5ffd 100644 --- a/examples/streaming/single-sink/main.go +++ b/examples/streaming/single-sink/main.go @@ -13,7 +13,7 @@ import ( func main() { // Create Redis client - rdb := redis.NewClient(&redis.Options{Addr: "localhost:6379", Password: os.Getenv("REDIS_PASSWORD")}) + rdb := redis.NewClient(&redis.Options{Addr: os.Getenv("REDIS_ADDR"), Password: os.Getenv("REDIS_PASSWORD")}) ctx := context.Background() if err := rdb.Ping(ctx).Err(); err != nil { panic(err) @@ -26,7 +26,11 @@ func main() { } // Don't forget to destroy the stream when done - defer stream.Destroy(ctx) + defer func() { + if err := stream.Destroy(ctx); err != nil { + panic(err) + } + }() // Add a new event id, err := stream.Add(ctx, "event", []byte("payload")) @@ -45,7 +49,11 @@ func main() { } // Don't forget to close the sink when done - defer sink.Close(ctx) + defer func() { + if err := sink.Close(ctx); err != nil { + panic(err) + } + }() // Consume event ev := <-sink.Subscribe() diff --git a/pulse/goroutine_test.go b/pulse/goroutine_test.go index aa06460..e92beb0 100644 --- a/pulse/goroutine_test.go +++ b/pulse/goroutine_test.go @@ -12,6 +12,15 @@ import ( "goa.design/clue/log" ) +type ( + // synchronizedBuilder lets panic-recovery tests observe asynchronous logger + // output without racing the logger's Write call. + synchronizedBuilder struct { + lock sync.Mutex + buf strings.Builder + } +) + func TestGo(t *testing.T) { t.Run("executes function without panic", func(t *testing.T) { var wg sync.WaitGroup @@ -32,7 +41,7 @@ func TestGo(t *testing.T) { var wg sync.WaitGroup wg.Add(1) - var buf strings.Builder + var buf synchronizedBuilder ctx = log.Context(ctx, log.WithOutput(&buf)) logger := ClueLogger(ctx) @@ -56,7 +65,7 @@ func TestGo(t *testing.T) { var wg sync.WaitGroup wg.Add(1) - var buf strings.Builder + var buf synchronizedBuilder ctx = log.Context(ctx, log.WithOutput(&buf)) logger := ClueLogger(ctx) @@ -73,3 +82,17 @@ func TestGo(t *testing.T) { }, 100*time.Millisecond, 10*time.Millisecond, "Log should contain panic message and stack trace") }) } + +// Write serializes logger output writes. +func (b *synchronizedBuilder) Write(p []byte) (int, error) { + b.lock.Lock() + defer b.lock.Unlock() + return b.buf.Write(p) +} + +// String returns a stable snapshot of logger output. +func (b *synchronizedBuilder) String() string { + b.lock.Lock() + defer b.lock.Unlock() + return b.buf.String() +} diff --git a/rmap/map_test.go b/rmap/map_test.go index b113363..2720351 100644 --- a/rmap/map_test.go +++ b/rmap/map_test.go @@ -6,40 +6,25 @@ import ( "bytes" "context" "fmt" - "os" "strings" "sync" "testing" "time" - "github.com/redis/go-redis/v9" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "goa.design/clue/log" "goa.design/pulse/pulse" + ptesting "goa.design/pulse/testing" ) var ( - redisPwd = "redispassword" - redisAddr = "localhost:6379" - wf = time.Second - tck = time.Millisecond + wf = time.Second + tck = time.Millisecond ) -func init() { - if p := os.Getenv("REDIS_PASSWORD"); p != "" { - redisPwd = p - } - if a := os.Getenv("REDIS_ADDR"); a != "" { - redisAddr = a - } -} - func TestMapLocal(t *testing.T) { - rdb := redis.NewClient(&redis.Options{ - Addr: redisAddr, - Password: redisPwd, - }) + rdb := ptesting.NewRedisClient(t) var buf Buffer ctx := context.Background() ctx = log.Context(ctx, log.WithOutput(&buf)) @@ -144,10 +129,7 @@ func TestMapLocal(t *testing.T) { } func TestMapTTLAbsolute(t *testing.T) { - rdb := redis.NewClient(&redis.Options{ - Addr: redisAddr, - Password: redisPwd, - }) + rdb := ptesting.NewRedisClient(t) ctx := context.Background() m, err := Join(ctx, "ttl-absolute", rdb, WithTTL(2*time.Second)) @@ -178,10 +160,7 @@ func TestMapTTLAbsolute(t *testing.T) { } func TestMapTTLSliding(t *testing.T) { - rdb := redis.NewClient(&redis.Options{ - Addr: redisAddr, - Password: redisPwd, - }) + rdb := ptesting.NewRedisClient(t) ctx := context.Background() m, err := Join(ctx, "ttl-sliding", rdb, WithSlidingTTL(2*time.Second)) @@ -212,10 +191,7 @@ func TestMapTTLSliding(t *testing.T) { } func TestSetAndWait(t *testing.T) { - rdb := redis.NewClient(&redis.Options{ - Addr: redisAddr, - Password: redisPwd, - }) + rdb := ptesting.NewRedisClient(t) var buf Buffer ctx := context.Background() ctx = log.Context(ctx, log.WithOutput(&buf)) @@ -273,10 +249,7 @@ func TestSetAndWait(t *testing.T) { } func TestReadAfterClose(t *testing.T) { - rdb := redis.NewClient(&redis.Options{ - Addr: redisAddr, - Password: redisPwd, - }) + rdb := ptesting.NewRedisClient(t) var buf Buffer ctx := context.Background() ctx = log.Context(ctx, log.WithOutput(&buf)) @@ -339,7 +312,7 @@ func TestReadAfterClose(t *testing.T) { } func TestWriteEmptyString(t *testing.T) { - rdb := redis.NewClient(&redis.Options{Addr: redisAddr, Password: redisPwd}) + rdb := ptesting.NewRedisClient(t) ctx := context.Background() m, err := Join(ctx, "test", rdb) require.NoError(t, err) @@ -359,7 +332,7 @@ func TestWriteEmptyString(t *testing.T) { } func TestSetEx(t *testing.T) { - rdb := redis.NewClient(&redis.Options{Addr: redisAddr, Password: redisPwd}) + rdb := ptesting.NewRedisClient(t) ctx := context.Background() m, err := Join(ctx, "test", rdb) require.NoError(t, err) @@ -394,7 +367,7 @@ func TestSetEx(t *testing.T) { } func TestDeleteEx(t *testing.T) { - rdb := redis.NewClient(&redis.Options{Addr: redisAddr, Password: redisPwd}) + rdb := ptesting.NewRedisClient(t) ctx := context.Background() m, err := Join(ctx, "test", rdb) require.NoError(t, err) @@ -419,7 +392,7 @@ func TestDeleteEx(t *testing.T) { } func TestTestAndSetEx(t *testing.T) { - rdb := redis.NewClient(&redis.Options{Addr: redisAddr, Password: redisPwd}) + rdb := ptesting.NewRedisClient(t) ctx := context.Background() m, err := Join(ctx, "test", rdb) require.NoError(t, err) @@ -453,7 +426,7 @@ func TestTestAndSetEx(t *testing.T) { } func TestTestAndDeleteEx(t *testing.T) { - rdb := redis.NewClient(&redis.Options{Addr: redisAddr, Password: redisPwd}) + rdb := ptesting.NewRedisClient(t) ctx := context.Background() m, err := Join(ctx, "test", rdb) require.NoError(t, err) @@ -487,7 +460,7 @@ func TestTestAndDeleteEx(t *testing.T) { } func TestAppendUniqueValues(t *testing.T) { - rdb := redis.NewClient(&redis.Options{Addr: redisAddr, Password: redisPwd}) + rdb := ptesting.NewRedisClient(t) ctx := context.Background() m, err := Join(ctx, "test", rdb) require.NoError(t, err) @@ -531,7 +504,7 @@ func TestAppendUniqueValues(t *testing.T) { } func TestTestAndDelete(t *testing.T) { - rdb := redis.NewClient(&redis.Options{Addr: redisAddr, Password: redisPwd}) + rdb := ptesting.NewRedisClient(t) ctx := context.Background() m, err := Join(ctx, "test", rdb) require.NoError(t, err) @@ -569,7 +542,7 @@ func TestTestAndDelete(t *testing.T) { } func TestTestAndSet(t *testing.T) { - rdb := redis.NewClient(&redis.Options{Addr: redisAddr, Password: redisPwd}) + rdb := ptesting.NewRedisClient(t) ctx := context.Background() m, err := Join(ctx, "test", rdb) require.NoError(t, err) @@ -594,7 +567,7 @@ func TestTestAndSet(t *testing.T) { } func TestTestAndReset(t *testing.T) { - rdb := redis.NewClient(&redis.Options{Addr: redisAddr, Password: redisPwd}) + rdb := ptesting.NewRedisClient(t) ctx := context.Background() m, err := Join(ctx, "test", rdb) require.NoError(t, err) @@ -650,7 +623,7 @@ func TestTestAndReset(t *testing.T) { } func TestArrays(t *testing.T) { - rdb := redis.NewClient(&redis.Options{Addr: redisAddr, Password: redisPwd}) + rdb := ptesting.NewRedisClient(t) ctx := context.Background() m, err := Join(ctx, "test", rdb) require.NoError(t, err) @@ -686,7 +659,7 @@ func TestArrays(t *testing.T) { } func TestIncrement(t *testing.T) { - rdb := redis.NewClient(&redis.Options{Addr: redisAddr, Password: redisPwd}) + rdb := ptesting.NewRedisClient(t) ctx := context.Background() m, err := Join(ctx, "test", rdb) require.NoError(t, err) @@ -712,7 +685,7 @@ func TestIncrement(t *testing.T) { } func TestLogs(t *testing.T) { - rdb := redis.NewClient(&redis.Options{Addr: redisAddr, Password: redisPwd}) + rdb := ptesting.NewRedisClient(t) var buf Buffer ctx := context.Background() ctx = log.Context(ctx, log.WithOutput(&buf), log.WithDebug(), log.WithFormat(log.FormatText)) @@ -764,7 +737,7 @@ func TestJoinErrors(t *testing.T) { } func TestSetErrors(t *testing.T) { - rdb := redis.NewClient(&redis.Options{Addr: redisAddr, Password: redisPwd}) + rdb := ptesting.NewRedisClient(t) ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -782,7 +755,7 @@ func TestSetErrors(t *testing.T) { } func TestAppendValuesErrors(t *testing.T) { - rdb := redis.NewClient(&redis.Options{Addr: redisAddr, Password: redisPwd}) + rdb := ptesting.NewRedisClient(t) ctx := context.Background() m, err := Join(ctx, "test", rdb) @@ -798,7 +771,7 @@ func TestAppendValuesErrors(t *testing.T) { } func TestRemoveValuesErrors(t *testing.T) { - rdb := redis.NewClient(&redis.Options{Addr: redisAddr, Password: redisPwd}) + rdb := ptesting.NewRedisClient(t) ctx := context.Background() m, err := Join(ctx, "test", rdb) @@ -816,7 +789,7 @@ func TestRemoveValuesErrors(t *testing.T) { } func TestReconnect(t *testing.T) { - rdb := redis.NewClient(&redis.Options{Addr: redisAddr, Password: redisPwd}) + rdb := ptesting.NewRedisClient(t) ctx := context.Background() var buf Buffer ctx = log.Context(ctx, log.WithOutput(&buf)) @@ -852,7 +825,7 @@ func TestReconnect(t *testing.T) { } func TestDestroyAllowsReuse(t *testing.T) { - rdb := redis.NewClient(&redis.Options{Addr: redisAddr, Password: redisPwd}) + rdb := ptesting.NewRedisClient(t) ctx := context.Background() m, err := Join(ctx, "destroy-reuse", rdb) @@ -894,7 +867,7 @@ func TestApplyMessageLockedRejectsDestroyWithoutRevision(t *testing.T) { } func TestReconnectPreservesResetEvent(t *testing.T) { - rdb := redis.NewClient(&redis.Options{Addr: redisAddr, Password: redisPwd}) + rdb := ptesting.NewRedisClient(t) ctx := context.Background() observer, err := Join(ctx, "reconnect-reset-event", rdb) diff --git a/streaming/README.md b/streaming/README.md index 4079fe4..5cfd174 100644 --- a/streaming/README.md +++ b/streaming/README.md @@ -4,7 +4,7 @@ Pulse leverages Redis streams to provide scalable and reliable event streams that can be used to implement distributed architectures. Pulse provides a simple API to create and consume streams, for example: -[![Single Reader](../snippets/single-reader.png)](../examples/streaming/single-reader/main.go#L21-L51) +[![Single Reader](../snippets/single-reader.png)](../examples/streaming/single-reader/main.go#L22-L55) The code above creates a stream and adds a new event to it. The event is then consumed by a reader. The reader is closed after the event is consumed. @@ -29,9 +29,21 @@ flowchart LR Multiple readers can be created for the same stream across many nodes. Readers are independent and each instance receives a copy of the same events. Readers can specify a start position for the stream cursor. The default start position -is the last event in the stream. - -[![Multi Reader](../snippets/multi-reader.png)](../examples/streaming/multi-readers/main.go#L45-L72) +is the last event in the stream. `NewReader` is non-creating: the stream must +already have an active lifecycle established by a writer or an explicit +`Stream.Open`, otherwise it returns `ErrStreamNotFound`. + +Reader and sink Redis block durations are always finite and positive. The +default is five seconds; `WithReaderBlockDuration` and +`WithSinkBlockDuration` may shorten or lengthen that bound but constructors +reject zero and negative durations. This guarantees `Close` can finish after at +most the configured blocking read even when a Redis client does not interrupt +the command on context cancellation. The examples use 100 milliseconds. +Constructor contexts bound setup only. Reader and Sink background loops use +their own lifecycle contexts and stop only when `Close` or a terminal lifecycle +error cancels them. + +[![Multi Reader](../snippets/multi-reader.png)](../examples/streaming/multi-readers/main.go#L44-L78) ```mermaid %%{init: {'themeVariables': { 'background': '#282828', 'edgeLabelBackground': '#7A7A7A'}}}%% @@ -55,9 +67,47 @@ flowchart LR linkStyle 4 stroke:#DDDDDD,color:#DDDDDD,stroke-width:3px; ``` +## Exact publication and snapshots + +`Stream.AddOnce` publishes an event exactly once for a non-empty idempotency key +of at most 256 bytes. The length-delimited name, topic, and payload may total at +most 1 MiB. The first call records those exact canonical bytes with the event +ID; no digest is used. An exact retry returns the same ID, while reusing the key +for different bytes returns `ErrIdempotencyConflict`. The event and its +generation-scoped dedupe record +share one Redis-absolute deadline, checked with Redis `TIME`, so retrying an +ambiguous client result is safe even after max-length trimming removes the +event itself. The returned ID remains the publication result even when that +event is no longer retained. + +`AddOnce` takes no deadline argument: the active stream generation's immutable +deadline is the sole expiry authority. A handle may adopt that existing +deadline by omitting retention options; an explicitly configured handle must +match it exactly. +Ordinary `Add` calls use that same absolute expiry and never extend it. At or +after expiry, Add, AddOnce, and +Snapshot return `ErrDeadlineElapsed`; Sink.Close treats expiry as terminal and +still closes its local subscriptions. Reuse of the logical name requires +explicit `Destroy` followed by construction of a new generation. The lifecycle +record intentionally survives expiry until Destroy so stale handles remain +fenced. + +`Stream.Snapshot` performs one generation-fenced Lua `XRANGE COUNT MaxLen+1` +and returns currently retained immutable `SnapshotEvent` values in Redis ID +order. The generation must have a bounded MaxLen; an unbounded stream returns +`ErrSnapshotUnbounded` before Redis runs `XRANGE`. The count bounds Redis script +work, reply size, and client memory. More than MaxLen physical entries violates +the immutable retention contract and returns `ErrSnapshotBoundExceeded` without +materializing the full stream. Snapshot creates no lifecycle, stream, reader, +consumer group, recovery cursor, or acknowledgement state. An uninitialized +name returns `ErrStreamNotFound`. If any externally written entry is malformed, +Snapshot returns a precise error and no partial result. + +[Exact publication example](../examples/streaming/exact-publication/main.go) + ## Stream retention (TTL) -By default, streams have no retention beyond `MaxLen` trimming and must be +By default, streams have no retention beyond max-length trimming and must be deleted explicitly. Pulse can also set a TTL on the Redis key backing a stream: - `options.WithStreamTTL(ttl)` sets an **absolute TTL** (set once when the key @@ -65,8 +115,83 @@ deleted explicitly. Pulse can also set a TTL on the Redis key backing a stream: - `options.WithStreamSlidingTTL(ttl)` sets a **sliding TTL** (refreshed on every published event). +TTL values and Reader/Sink timing values use Redis millisecond precision and +must be at least one millisecond. TTL options cannot be combined with +`options.WithStreamDeadline`. +`options.WithUnboundedStream` disables trimming and requires callers to delete +settled events explicitly; it cannot be combined with `WithStreamMaxLen`. + +The generation lifecycle owns one immutable retention configuration: maximum +length (or explicit unbounded mode), retention mode, TTL duration and sliding +flag, or absolute deadline. A handle with explicit retention options must match +exactly. A handle with no retention options adopts the active generation; +when its first write creates a name, it establishes the default of maximum +length 1000 with no TTL. Reader, Sink, Snapshot, and Destroy handles can +therefore adopt a writer configuration such as maximum length 50,000 with +sliding TTL without restating it. At the quiescent upgrade boundary, the first +upgraded writer that explicitly adopts bounded retention for a legacy flat +stream atomically trims the physical stream to MaxLen before publishing the +lifecycle configuration. Subsequent bounded writes preserve that hard limit. +`Stream.MaxLen` remains exported for v1 source compatibility and reflects the +constructor value, then the adopted generation value after `Open`. Mutating the +field is unsupported: all operations use the immutable construction/adoption +snapshot, so field mutation cannot weaken retention fencing. + The TTL is applied when the Redis stream key is first created, which happens on the first publish (`XADD`) or when creating a sink (`XGROUP CREATE ... MKSTREAM`). +Lifecycle and sink-recovery metadata do not expire with event data. A sink can +therefore recover its acknowledged cursor after the physical stream expires, +without using `$` and skipping events published before recovery. Explicit +`Stream.Destroy` removes recovery and exact-publication metadata; the small +lifecycle record remains as the monotonic incarnation source. + +## Stream incarnations + +`NewStream` is a local constructor and performs no Redis I/O. `Open` or the +first mutating caller-context operation establishes or loads the one active +Redis-owned generation for a logical stream name. Non-creating reads load only +an existing generation and return `ErrStreamNotFound` when none exists. Every +`Stream`, `Reader`, `Sink`, event acker, and added stream then retains that +immutable generation. Publishing, deleting events, consumer-group recovery, +consumer registration, keep-alive refresh, acknowledgement, and destruction +verify it atomically in Redis, so a destroyed generation's metadata can never +be recreated by a concurrent sink. + +`Stream.Destroy` invalidates and deletes exactly its generation, including its +events, consumer groups, recovery cursors, exact-publication records, sink +configuration, keepalives, leases, and membership. An unbound handle for an +absent name returns `ErrStreamNotFound` without creating a lifecycle. Reads that +begin after invalidation return +`ErrStreamDestroyed` and cannot cross into a later generation. An event fetched +before `Destroy` may already be executing in application code; its +`StreamGeneration` token identifies that old incarnation for handlers that +must fence side effects. Physical-key isolation guarantees that no +later-generation event can enter the old reader, sink, or pending-entry list. +Repeating `Destroy` for that generation is idempotent. Generation-qualified +metadata keys are deleted because they are never reused; the one small +lifecycle record remains as the bounded monotonic source for the logical name. + +Generation one records the compatible `pulse:stream:` physical key, so +Pulse adopts streams and consumer groups created before generation fencing +without moving queued or pending events. After explicit destruction, the next +generation receives a distinct physical key. Each handle stores the +lifecycle-selected key, and reads verify that lifecycle both before and after +Redis returns, so an old reader or sink cannot consume a recreated stream. + +Sink stale recovery uses a Redis-time lease with an owner token and monotonic +fencing token. Each `XAUTOCLAIM` and each stale-consumer inspection/deletion +verifies that exact unexpired capability in the same Lua operation as the PEL, +group, keepalive, and membership mutation. A paused predecessor therefore +cannot mutate recovery state after another replica takes over. + +Retention-config adoption is a quiescent upgrade: stop every reader, sink, and +publisher; deploy the new version everywhere; open one handle per logical name +with the intended options and verify its lifecycle using +`redis-cli HGETALL "pulse:stream::lifecycle"`: the active generation, +physical key, and `retention_config` must match that handle. Then resume +traffic. Rollback also requires stopping every user. Do not run an older binary +after the new config is captured; restore the pre-upgrade Redis snapshot or +Destroy and recreate with the rollback version. There is no mixed-version mode. ## Event Sinks @@ -77,16 +202,78 @@ partitions. Event sinks make it possible for multiple nodes to share the same stream cursor. If a stream contains 3 events and 3 nodes are consuming the stream using the same sink (i.e. a sink with the same name), then each node will receive a unique -event from the sequence. Nodes using a different sink (or a reader) will receive -copies of the same events. +event from the sequence; Redis chooses the replica for each event and does not +promise round-robin distribution. Nodes using a different sink name (or a +reader) receive an independent copy. Events read from a sink must be acknowledged by the client. Pulse automatically requeues events added to a sink that have been read by a node but not acknowledged. +Sinks also restore Redis consumer groups deleted outside Pulse. One +generation-scoped recovery hash stores the durable cursor for each named sink. A +cursor advances only across acknowledged events, so pending events and events +added during recovery remain eligible for delivery. `RemoveStream` and +`Sink.Close` detach only that sink instance's membership; they do not delete +the shared consumer group or cursor. Recovery applies only to streams still +attached to the sink instance. `Sink.Close` returns distributed detach failures +without marking the sink closed; callers can retry it with a fresh context. +Removing a stream preserves its shared group and cursor. Pulse deletes an +individual Redis consumer only when that consumer's own PEL is empty, not when +the whole group's PEL is empty. Consumer PEL inspection, optional Redis +consumer deletion, membership removal, and keepalive removal are one +generation-fenced Lua operation. A consumer with pending entries loses local +membership but remains in Redis until stale recovery claims its entries; failed +setup and removal therefore leave no orphaned membership metadata. Stale empty +consumers are also removed by periodic cleanup. + +Each attached `(stream generation, sink name)` owns its own keepalive map, +shared configuration, and fenced stale-recovery lease. Cross-primary sinks +that attach the same secondary stream therefore coordinate on that secondary +without coupling their other streams. Replicas must use identical topic or +topic-pattern filters, acknowledgement mode, acknowledgement grace period, and +initial cursor for each attachment; `NewSink` or `AddStream` fails before group +membership on a mismatch. Keepalives are written once per acknowledgement +grace period, and stale-message checks run every 500 milliseconds by default. + +Sink and Reader constructors reject non-positive block durations and batch +sizes, negative channel buffers, and non-positive sink acknowledgement grace +periods. Defaults are a five-second block, 1000 events per read, a 1000-event +channel buffer, and a 20-second sink acknowledgement grace period. + +`WithSinkNoAck` preserves at-most-once delivery to subscribers by atomically +acknowledging each event and advancing the same recovery cursor before exposing +the event. It does not use Redis's `NOACK` read mode. Both `Sink.Ack` and direct +calls through `Event.Acker` use this recovery-aware acknowledgement operation. + +Deleting the physical Redis stream key itself is different: its stored event +payloads are gone and cannot be recovered. Pulse recreates the empty stream and +group at the durable cursor so all subsequently published events are delivered, +and logs the data-loss condition. `AddStream` and `RemoveStream` return +`ErrSinkClosed` after sink shutdown begins. Removing the final stream from a +Reader or Sink returns `ErrLastStream`, preserving a valid Redis read set. Explicit +`Stream.Destroy` is the only operation that deletes consumer groups and +recovery state; active sink membership neither blocks nor authorizes exact +generation destruction. + +Once Reader or Sink shutdown starts, new subscriptions return an already +closed channel. Reader stream changes return `ErrReaderClosed`, and Sink stream +changes return `ErrSinkClosed`. A sink also atomically acknowledges filtered +events before advancing past them, because no subscriber can acknowledge an +event it never receives. Reader filters remain observational and do not mutate +stream state. + +## Redis topology + +Streaming follows Pulse's standalone Redis contract. Physical event data, +lifecycle metadata, recovery, and replicated-map keys are intentionally +separate readable keys, and atomic Lua operations may access several of them. +`AddOnce` and `Snapshot` follow the same contract. Redis Cluster cross-slot +execution is not supported. + Creating a sink is as simple as: -[![Single Sink](../snippets/single-sink.png)](../examples/streaming/single-sink/main.go#L37-L56) +[![Single Sink](../snippets/single-sink.png)](../examples/streaming/single-sink/main.go#L42-L65) Note a couple of differences with the reader example above: @@ -118,10 +305,11 @@ flowchart LR linkStyle 3 stroke:#DDDDDD,color:#DDDDDD,stroke-width:3px; ``` -As with readers, multiple sinks can be created for the same stream. Copies of -the same event are distributed among all sinks. +As with readers, multiple sink groups can be created for the same stream. +Different sink names each receive a copy; replicas sharing one sink name divide +that group's events. -[![Multi Sink](../snippets/multi-sink.png)](../examples/streaming/multi-sinks/main.go#L58-L79) +[![Multi Sink](../snippets/multi-sink.png)](../examples/streaming/multi-sinks/main.go#L53-L89) ```mermaid %%{init: {'themeVariables': { 'background': '#282828', 'edgeLabelBackground': '#7A7A7A'}}}%% @@ -184,7 +372,7 @@ work or losing unacknowledged events: Readers and sinks can also read concurrently from multiple streams: -[![Multi Stream](../snippets/multi-stream.png)](../examples/streaming/multi-streams/main.go#L61-62) +[![Multi Stream](../snippets/multi-stream.png)](../examples/streaming/multi-streams/main.go#L63-L83) ```mermaid %%{init: {'themeVariables': { 'background': '#282828', 'edgeLabelBackground': '#7A7A7A'}}}%% @@ -214,16 +402,18 @@ flowchart LR ``` `AddStream` can be called at any time to add new streams to a reader or a sink. -Streams can also be removed using `RemoveStream`. +Streams can also be removed using `RemoveStream`. Topic and topic-pattern +filters are mutually exclusive, as are the cursor-start options; constructors +and `AddStream` reject conflicting choices instead of applying precedence. -[![Remove Stream](../snippets/remove-stream.png)](../examples/streaming/multi-streams/main.go#L87-L91) +[![Remove Stream](../snippets/remove-stream.png)](../examples/streaming/multi-streams/main.go#L100-L104) ## Pub/Sub Streams supports a flexible pub/sub mechanism where events can be attached to topics and readers or sinks can define simple or custom matching logic. -[![Pub/Sub](../snippets/pub-sub.png)](../examples/streaming/pub-sub/main.go#L31-L35) +[![Pub/Sub](../snippets/pub-sub.png)](../examples/streaming/pub-sub/main.go#L36-L40) ```mermaid %%{init: {'themeVariables': { 'background': '#282828', 'edgeLabelBackground': '#7A7A7A'}}}%% @@ -259,11 +449,13 @@ flowchart RL Topics can be matched using their name as in the example above or using complex patterns. For example: -[![Pub/Sub](../snippets/pub-sub-pattern.png)](../examples/streaming/pub-sub/main.go#L76-L79) +[![Pub/Sub](../snippets/pub-sub-pattern.png)](../examples/streaming/pub-sub/main.go#L85-L88) -> Note: Event filtering is done client-side in the sink or reader and does not -> affect the underlying stream. This means that events are still stored in the -> stream and can be consumed by other sinks. +> Note: Event filtering is evaluated client-side and never removes an event +> from the stream. Reader filtering is observational. A sink filter +> acknowledges non-matching events in that sink's consumer group because no +> subscriber can acknowledge an event it never receives; other sink names and +> readers remain independent. ## Examples diff --git a/streaming/exact_publication.go b/streaming/exact_publication.go new file mode 100644 index 0000000..e837cb0 --- /dev/null +++ b/streaming/exact_publication.go @@ -0,0 +1,523 @@ +// Exact publication provides generation-scoped idempotent writes and +// side-effect-free snapshots. Redis Lua owns both linearization points so +// retries, Destroy, and concurrent clients observe one canonical result. +package streaming + +import ( + "bytes" + "context" + "encoding/binary" + "fmt" + "strconv" + + redis "github.com/redis/go-redis/v9" + + "goa.design/pulse/streaming/options" +) + +type ( + // SnapshotEvent is an immutable event returned by Stream.Snapshot. + SnapshotEvent struct { + id string + streamName string + generation string + name string + topic string + payload []byte + } +) + +const ( + maxIdempotencyKeyBytes = 256 + maxAddOnceBodyBytes = 1 << 20 +) + +var ( + // addOnceScript verifies the stream generation and absolute deadline, + // resolves the generation-scoped idempotency record, and publishes exactly + // once. Redis TIME is authoritative for deadline admission. + addOnceScript = redis.NewScript(` +local state = redis.call("HGET", KEYS[1], "state") +local generation = redis.call("HGET", KEYS[1], "generation") +local physical = redis.call("HGET", KEYS[1], ARGV[5]) +local deadline = redis.call("HGET", KEYS[1], ARGV[6]) +local ttl_owned = redis.call("HGET", KEYS[1], ARGV[7]) +local retention = redis.call("HGET", KEYS[1], ARGV[15]) +local recreate = false +local adopting_legacy = not generation + +if ARGV[2] ~= "" then + if state ~= ARGV[1] or generation ~= ARGV[2] or physical ~= ARGV[3] then + return redis.error_reply("STREAMDESTROYED") + end +else + if not generation then + if ARGV[8] == "" then + return redis.error_reply("STREAMDEADLINEREQUIRED") + end + generation = "1" + physical = ARGV[4] + recreate = true + elseif state ~= ARGV[1] then + generation = tostring(tonumber(generation) + 1) + physical = ARGV[4] .. ":generation:" .. generation + deadline = false + ttl_owned = false + retention = false + recreate = true + elseif not physical then + physical = ARGV[4] + end +end + +if retention and ARGV[18] == "1" and retention ~= ARGV[16] then + return redis.error_reply("STREAMCONFIGMISMATCH") +end +if deadline then + if ARGV[8] ~= "" and deadline ~= ARGV[8] then + return redis.error_reply("STREAMDEADLINECONFLICT") + end +elseif ttl_owned == "1" then + return redis.error_reply("STREAMDEADLINECONFLICT") +else + if ARGV[8] == "" then + return redis.error_reply("STREAMDEADLINEREQUIRED") + end + deadline = ARGV[8] +end +local now = redis.call("TIME") +local now_ms = tonumber(now[1]) * 1000 + math.floor(tonumber(now[2]) / 1000) +if now_ms >= tonumber(deadline) then + return redis.error_reply("DEADLINEELAPSED") +end + +local dedupe = ARGV[4] .. ":generation:" .. generation .. ":idempotency" +local recovery = physical .. ":sink-recovery:" .. generation +local resources_key = ARGV[4] .. ":generation:" .. generation .. ":resources" +local physical_type = redis.call("TYPE", physical)["ok"] +local dedupe_type = redis.call("TYPE", dedupe)["ok"] +local recovery_type = redis.call("TYPE", recovery)["ok"] +local resources_type = redis.call("TYPE", resources_key)["ok"] +if (physical_type ~= "none" and physical_type ~= "stream") +or (dedupe_type ~= "none" and dedupe_type ~= "hash") +or (recovery_type ~= "none" and recovery_type ~= "hash") +or (resources_type ~= "none" and resources_type ~= "set") then + return redis.error_reply("STREAMRESOURCEINVALID") +end + +local existing = redis.call("HGET", dedupe, ARGV[10]) +if existing then + local separator = string.find(existing, "\0", 1, true) + if not separator then + return redis.error_reply("IDEMPOTENCYINVALID") + end + local event_id = string.sub(existing, 1, separator - 1) + local existing_identity = string.sub(existing, separator + 1) + if existing_identity ~= ARGV[17] then + return redis.error_reply("IDEMPOTENCYCONFLICT") + end + redis.call("PEXPIREAT", physical, deadline) + redis.call("PEXPIREAT", dedupe, deadline) + redis.call("PEXPIREAT", recovery, deadline) + local existing_resources = redis.call("SMEMBERS", resources_key) + for _, resource in ipairs(existing_resources) do + redis.call("PEXPIREAT", resource, deadline) + end + redis.call("PEXPIREAT", resources_key, deadline) + return {generation, physical, deadline, retention, 0, event_id} +end + +if recreate then + if adopting_legacy and ARGV[18] == "1" and tonumber(ARGV[9]) > 0 then + redis.call("XTRIM", physical, "MAXLEN", "=", ARGV[9]) + end + redis.call("HSET", KEYS[1], + "generation", generation, + "state", ARGV[1], + ARGV[5], physical, + ARGV[6], deadline, + ARGV[15], ARGV[16]) + redis.call("HDEL", KEYS[1], ARGV[7]) +elseif redis.call("HGET", KEYS[1], ARGV[5]) == false then + redis.call("HSET", KEYS[1], ARGV[5], physical) + redis.call("HSET", KEYS[1], ARGV[6], deadline) +elseif redis.call("HGET", KEYS[1], ARGV[6]) == false then + redis.call("HSET", KEYS[1], ARGV[6], deadline) +end +if not retention then + redis.call("HSET", KEYS[1], ARGV[15], ARGV[16]) +end + +local event_id +if ARGV[9] == "0" and ARGV[13] == "1" then + event_id = redis.call("XADD", physical, "*", "n", ARGV[11], "p", ARGV[12], "t", ARGV[14]) +elseif ARGV[9] == "0" then + event_id = redis.call("XADD", physical, "*", "n", ARGV[11], "p", ARGV[12]) +elseif ARGV[13] == "1" then + event_id = redis.call("XADD", physical, "MAXLEN", "=", ARGV[9], "*", + "n", ARGV[11], "p", ARGV[12], "t", ARGV[14]) +else + event_id = redis.call("XADD", physical, "MAXLEN", "=", ARGV[9], "*", + "n", ARGV[11], "p", ARGV[12]) +end +redis.call("HSET", dedupe, ARGV[10], event_id .. "\0" .. ARGV[17]) +redis.call("HSET", recovery, "=deadline", deadline) +redis.call("SADD", resources_key, dedupe, recovery) + +redis.call("PEXPIREAT", physical, deadline) +redis.call("PEXPIREAT", dedupe, deadline) +redis.call("PEXPIREAT", recovery, deadline) +local resources = redis.call("SMEMBERS", resources_key) +for _, resource in ipairs(resources) do + redis.call("PEXPIREAT", resource, deadline) +end +redis.call("PEXPIREAT", resources_key, deadline) +return {generation, physical, deadline, retention or ARGV[16], 1, event_id} +`) + + // snapshotScript binds an unbound handle using the same zero-migration + // lifecycle contract, verifies a bound handle, and returns one XRANGE + // result at the operation's single Redis linearization point. + snapshotScript = redis.NewScript(` +local state = redis.call("HGET", KEYS[1], "state") +local generation = redis.call("HGET", KEYS[1], "generation") +if not generation then + return {0} +end +local physical = redis.call("HGET", KEYS[1], ARGV[4]) +local deadline = redis.call("HGET", KEYS[1], ARGV[5]) +if state ~= ARGV[1] or (ARGV[2] ~= "" and generation ~= ARGV[2]) +or (ARGV[3] ~= "" and physical ~= ARGV[3]) then + return redis.error_reply("STREAMDESTROYED") +end +local retention = redis.call("HGET", KEYS[1], ARGV[6]) +if not retention then + return redis.error_reply("STREAMCONFIGMISSING") +end +if ARGV[8] == "1" and retention ~= ARGV[7] then + return redis.error_reply("STREAMCONFIGMISMATCH") +end +if string.find(retention, "|max=0|", 1, true) then + return redis.error_reply("SNAPSHOTUNBOUNDED") +end +local max_len = tonumber(string.match(retention, "|max=(%d+)|")) +if not max_len or max_len <= 0 then + return redis.error_reply("STREAMCONFIGMISSING") +end + +if deadline then + local now = redis.call("TIME") + local now_ms = tonumber(now[1]) * 1000 + math.floor(tonumber(now[2]) / 1000) + if now_ms >= tonumber(deadline) then + return redis.error_reply("DEADLINEELAPSED") + end +end +local events = redis.call("XRANGE", physical, "-", "+", "COUNT", max_len + 1) +if #events > max_len then + return redis.error_reply("SNAPSHOTBOUNDVIOLATION") +end +return {1, generation, physical, deadline or "", retention, events} +`) +) + +// AddOnce publishes one event for idempotencyKey in this stream generation. +// The first call stores the event ID and exact length-delimited event identity +// until the generation deadline. Exact retries return that ID; content changes +// return ErrIdempotencyConflict. The active generation must be deadline-owned; +// a handle with explicit retention options must match that immutable deadline. +func (s *Stream) AddOnce( + ctx context.Context, + idempotencyKey string, + name string, + payload []byte, + opts ...options.AddEvent, +) (string, error) { + if len(idempotencyKey) == 0 { + return "", fmt.Errorf("pulse streaming: idempotency key must not be empty") + } + if len(idempotencyKey) > maxIdempotencyKeyBytes { + return "", fmt.Errorf( + "pulse streaming: idempotency key exceeds %d bytes", + maxIdempotencyKeyBytes, + ) + } + if name == "" { + return "", fmt.Errorf("pulse streaming: event name must not be empty") + } + o := options.ParseAddEventOptions(opts...) + identity, err := canonicalEventIdentity(name, o.Topic, payload) + if err != nil { + return "", err + } + if o.OnlyIfStreamExists { + return "", fmt.Errorf( + "pulse streaming: AddOnce does not support WithOnlyIfStreamExists", + ) + } + s.generationLock.Lock() + defer s.generationLock.Unlock() + bound := s.generation != "" + result, err := addOnceScript.Run( + ctx, + s.rdb, + []string{s.lifecycleKey}, + streamStateActive, + s.generation, + s.key, + streamKey(s.Name), + streamPhysicalKey, + streamDeadlineKey, + streamTTLOwnedKey, + s.requestedDeadline(), + strconv.Itoa(s.maxLen), + idempotencyKey, + name, + payload, + boolString(o.Topic != ""), + o.Topic, + streamConfigKey, + s.retention, + identity, + boolString(s.retentionExplicit), + ).Slice() + if err != nil { + if redis.HasErrorPrefix(err, "IDEMPOTENCYCONFLICT") { + return "", fmt.Errorf( + "%w: key %q", + ErrIdempotencyConflict, + idempotencyKey, + ) + } + return "", s.lifecycleError(err) + } + if len(result) != 6 { + return "", fmt.Errorf("add once script returned %d values", len(result)) + } + generation, physical, canonicalDeadline, retention, err := parseLifecycleIdentity(result[:4]) + if err != nil { + return "", fmt.Errorf("pulse streaming: AddOnce identity: %w", err) + } + parsedDeadline, err := parseDeadline(canonicalDeadline) + if err != nil { + return "", fmt.Errorf("pulse streaming: AddOnce identity: %w", err) + } + eventID, err := parseAddOnceResult(result[4:]) + if err != nil { + return "", err + } + // A bound handle is an immutable capability for one exact generation: the + // script verified the identity, so only the unbound→bound transition may + // write binding state. Post-bind writes would race with the unlocked reads + // every verified operation performs after its own lock passage. + if !bound { + if err := s.applyRetentionConfig(retention); err != nil { + return "", fmt.Errorf("pulse streaming: AddOnce identity: %w", err) + } + s.generation = generation + s.key = physical + s.deadline = parsedDeadline + } + return eventID, nil +} + +// Snapshot returns every event currently retained by this exact bounded +// stream generation in Redis ID order. The immutable MaxLen contract bounds +// Redis script work and result memory. Redis reads at most MaxLen+1 entries and +// returns ErrSnapshotBoundExceeded if physical data violates that invariant; +// unbounded streams return ErrSnapshotUnbounded before XRANGE. Snapshot creates no reader, consumer +// group, cursor, acknowledgement state, or lifecycle. An absent lifecycle +// returns ErrStreamNotFound. A malformed Redis entry fails the whole snapshot. +func (s *Stream) Snapshot(ctx context.Context) ([]SnapshotEvent, error) { + s.generationLock.Lock() + defer s.generationLock.Unlock() + bound := s.generation != "" + raw, err := snapshotScript.Run( + ctx, + s.rdb, + []string{s.lifecycleKey}, + streamStateActive, + s.generation, + s.key, + streamPhysicalKey, + streamDeadlineKey, + streamConfigKey, + s.retention, + boolString(s.retentionExplicit), + ).Slice() + if err != nil { + return nil, s.lifecycleError(err) + } + if len(raw) == 1 { + return nil, ErrStreamNotFound + } + if len(raw) != 6 { + return nil, fmt.Errorf("snapshot script returned %d values", len(raw)) + } + generation, physical, deadline, retention, err := parseLifecycleIdentity(raw[1:5]) + if err != nil { + return nil, fmt.Errorf("pulse streaming: snapshot identity: %w", err) + } + canonicalDeadline, err := parseDeadline(deadline) + if err != nil { + return nil, fmt.Errorf("pulse streaming: snapshot identity: %w", err) + } + rangeResult, ok := raw[5].([]any) + if !ok { + return nil, fmt.Errorf("pulse streaming: snapshot range has invalid type %T", raw[5]) + } + // Only the unbound→bound transition may write binding state; see AddOnce. + if !bound { + if err := s.applyRetentionConfig(retention); err != nil { + return nil, fmt.Errorf("pulse streaming: snapshot identity: %w", err) + } + s.generation = generation + s.key = physical + s.deadline = canonicalDeadline + } + messages, err := decodeSnapshotRange(rangeResult) + if err != nil { + return nil, err + } + events := make([]SnapshotEvent, len(messages)) + for i, message := range messages { + name, topic, payload, err := decodeRedisEvent(message) + if err != nil { + return nil, fmt.Errorf("pulse streaming: snapshot: %w", err) + } + events[i] = SnapshotEvent{ + id: message.ID, + streamName: s.Name, + generation: s.generation, + name: name, + topic: topic, + payload: append([]byte(nil), payload...), + } + } + return events, nil +} + +// ID returns the immutable Redis event ID. +func (e SnapshotEvent) ID() string { + return e.id +} + +// StreamName returns the logical stream name. +func (e SnapshotEvent) StreamName() string { + return e.streamName +} + +// StreamGeneration returns the immutable stream incarnation. +func (e SnapshotEvent) StreamGeneration() string { + return e.generation +} + +// EventName returns the producer-defined event name. +func (e SnapshotEvent) EventName() string { + return e.name +} + +// Topic returns the producer-defined topic, or empty when absent. +func (e SnapshotEvent) Topic() string { + return e.topic +} + +// Payload returns an independent copy of the event payload. +func (e SnapshotEvent) Payload() []byte { + return append([]byte(nil), e.payload...) +} + +// parseAddOnceResult validates the Lua result at the Redis boundary. +func parseAddOnceResult(result []any) (string, error) { + if len(result) != 2 { + return "", fmt.Errorf("add once script returned %d values", len(result)) + } + status, ok := result[0].(int64) + if !ok || (status != 0 && status != 1) { + return "", fmt.Errorf("add once script returned invalid status %T(%v)", result[0], result[0]) + } + eventID, ok := result[1].(string) + if !ok || eventID == "" { + return "", fmt.Errorf("add once script returned invalid event ID %T", result[1]) + } + return eventID, nil +} + +// canonicalEventIdentity length-prefixes exact event bytes so no field +// concatenation can alias another identity. +func canonicalEventIdentity(name, topic string, payload []byte) ([]byte, error) { + total := len(name) + len(topic) + len(payload) + if total > maxAddOnceBodyBytes { + return nil, fmt.Errorf( + "pulse streaming: AddOnce event identity exceeds %d bytes", + maxAddOnceBodyBytes, + ) + } + var identity bytes.Buffer + for _, field := range [][]byte{[]byte(name), []byte(topic), payload} { + if err := binary.Write(&identity, binary.BigEndian, uint64(len(field))); err != nil { + return nil, fmt.Errorf("pulse streaming: encode AddOnce event identity: %w", err) + } + if _, err := identity.Write(field); err != nil { + return nil, fmt.Errorf("pulse streaming: encode AddOnce event identity: %w", err) + } + } + return identity.Bytes(), nil +} + +// decodeSnapshotRange validates Redis's nested XRANGE response without +// returning a partial message set. +func decodeSnapshotRange(raw []any) ([]redis.XMessage, error) { + messages := make([]redis.XMessage, len(raw)) + for i, value := range raw { + entry, ok := value.([]any) + if !ok || len(entry) != 2 { + return nil, fmt.Errorf( + "pulse streaming: snapshot entry %d must contain ID and fields, got %T", + i, + value, + ) + } + id, ok := entry[0].(string) + if !ok || id == "" { + return nil, fmt.Errorf("pulse streaming: snapshot entry %d has invalid ID %T", i, entry[0]) + } + fields, ok := entry[1].([]any) + if !ok || len(fields)%2 != 0 { + return nil, fmt.Errorf("pulse streaming: snapshot event %s has invalid fields %T", id, entry[1]) + } + values := make(map[string]any, len(fields)/2) + for field := 0; field < len(fields); field += 2 { + key, ok := fields[field].(string) + if !ok { + return nil, fmt.Errorf( + "pulse streaming: snapshot event %s has invalid field name %T", + id, + fields[field], + ) + } + fieldValue, ok := fields[field+1].(string) + if !ok { + return nil, fmt.Errorf( + "pulse streaming: snapshot event %s field %q has invalid value %T", + id, + key, + fields[field+1], + ) + } + values[key] = fieldValue + } + messages[i] = redis.XMessage{ID: id, Values: values} + } + return messages, nil +} + +// idempotencyKeyMap stores generation-scoped publication IDs and content +// digests independently from MAXLEN-trimmed event data. +func idempotencyKeyMap(stream *Stream) string { + return fmt.Sprintf( + "%s%s:generation:%s:idempotency", + streamKeyPrefix, + stream.Name, + stream.generation, + ) +} diff --git a/streaming/exact_publication_test.go b/streaming/exact_publication_test.go new file mode 100644 index 0000000..f4d0ccb --- /dev/null +++ b/streaming/exact_publication_test.go @@ -0,0 +1,638 @@ +package streaming + +import ( + "context" + "errors" + "net" + "strconv" + "sync" + "sync/atomic" + "testing" + "time" + + redis "github.com/redis/go-redis/v9" + "github.com/stretchr/testify/require" + + "goa.design/pulse/streaming/options" + ptesting "goa.design/pulse/testing" +) + +type ( + // ambiguousAddOnceHook returns one client error after Redis committed the + // selected AddOnce script, reproducing an ambiguous network outcome. + ambiguousAddOnceHook struct { + fail atomic.Bool + err error + } +) + +func TestAddOnceConcurrentClientsPublishExactlyOnce(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + other := redis.NewClient(rdb.Options()) + defer func() { require.NoError(t, other.Close()) }() + ctx := ptesting.NewTestContext(t) + deadline := time.Now().Add(10 * time.Second).Truncate(time.Millisecond) + const clients = 12 + results := make(chan string, clients) + errs := make(chan error, clients) + var wait sync.WaitGroup + for index := range clients { + wait.Add(1) + go func(client *redis.Client) { + defer wait.Done() + stream, err := NewStream(t.Name(), client, options.WithStreamDeadline(deadline)) + if err != nil { + errs <- err + return + } + result, err := stream.AddOnce( + ctx, + "command", + "created", + []byte("payload"), + options.WithTopic("alarms"), + ) + if err != nil { + errs <- err + return + } + results <- result + }([]*redis.Client{rdb, other}[index%2]) + } + wait.Wait() + close(results) + close(errs) + for err := range errs { + require.NoError(t, err) + } + var ( + eventID string + ) + for result := range results { + if eventID == "" { + eventID = result + } + require.Equal(t, eventID, result) + } + require.EqualValues(t, 1, rdb.XLen(ctx, streamKey(t.Name())).Val()) +} + +func TestAddOnceConflictAndAmbiguousCommitRetry(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + hook := &ambiguousAddOnceHook{err: errors.New("ambiguous client result")} + rdb.AddHook(hook) + ctx := ptesting.NewTestContext(t) + deadline := time.Now().Add(10 * time.Second).Truncate(time.Millisecond) + stream, err := NewStream(t.Name(), rdb, options.WithStreamDeadline(deadline)) + require.NoError(t, err) + + hook.fail.Store(true) + _, err = stream.AddOnce(ctx, "command", "created", []byte("payload")) + require.ErrorIs(t, err, hook.err) + retry, err := stream.AddOnce(ctx, "command", "created", []byte("payload")) + require.NoError(t, err) + require.NotEmpty(t, retry) + require.EqualValues(t, 1, rdb.XLen(ctx, stream.key).Val()) + + _, err = stream.AddOnce(ctx, "command", "created", []byte("different")) + require.ErrorIs(t, err, ErrIdempotencyConflict) + require.EqualValues(t, 1, rdb.XLen(ctx, stream.key).Val()) + require.NoError(t, stream.Destroy(ctx)) +} + +func TestAddOnceMetadataSurvivesMaxLenAndScriptFlush(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + deadline := time.Now().Add(10 * time.Second).Truncate(time.Millisecond) + stream, err := NewStream( + t.Name(), + rdb, + options.WithStreamMaxLen(1), + options.WithStreamDeadline(deadline), + ) + require.NoError(t, err) + first, err := stream.AddOnce(ctx, "first", "first", []byte("payload")) + require.NoError(t, err) + for i := 0; i < 200; i++ { + _, err = stream.Add(ctx, "trim", []byte{byte(i)}) + require.NoError(t, err) + } + require.Empty(t, rdb.XRangeN(ctx, stream.key, first, first, 1).Val()) + + require.NoError(t, rdb.ScriptFlush(ctx).Err()) + retry, err := stream.AddOnce(ctx, "first", "first", []byte("payload")) + require.NoError(t, err) + require.Equal(t, first, retry) + _, err = stream.AddOnce(ctx, "second", "second", []byte("payload")) + require.NoError(t, err) + _, err = stream.Snapshot(ctx) + require.NoError(t, err) + require.NoError(t, stream.Destroy(ctx)) +} + +func TestAddOnceAdoptsPreGenerationFlatStreamInPlace(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + flatKey := streamKey(t.Name()) + legacyID, err := rdb.XAdd(ctx, &redis.XAddArgs{ + Stream: flatKey, + Values: map[string]any{nameKey: "legacy", payloadKey: "payload"}, + }).Result() + require.NoError(t, err) + deadline := time.Now().Add(10 * time.Second).Truncate(time.Millisecond) + stream, err := NewStream(t.Name(), rdb, options.WithStreamDeadline(deadline)) + require.NoError(t, err) + published, err := stream.AddOnce(ctx, "new", "current", []byte("payload")) + require.NoError(t, err) + require.Equal(t, flatKey, stream.key) + require.Equal(t, "1", stream.Generation()) + events, err := stream.Snapshot(ctx) + require.NoError(t, err) + require.Equal(t, []string{legacyID, published}, []string{events[0].ID(), events[1].ID()}) + require.NoError(t, stream.Destroy(ctx)) +} + +func TestExplicitBoundedLegacyAdoptionTrimsBeforePublishingLifecycle(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + const ( + legacyEvents = 5_000 + maxLen = 100 + ) + flatKey := streamKey(t.Name()) + pipe := rdb.Pipeline() + for index := 0; index < legacyEvents; index++ { + pipe.XAdd(ctx, &redis.XAddArgs{ + Stream: flatKey, + Values: map[string]any{ + nameKey: "legacy", + payloadKey: strconv.Itoa(index), + }, + }) + } + _, err := pipe.Exec(ctx) + require.NoError(t, err) + + stream, err := NewStream(t.Name(), rdb, options.WithStreamMaxLen(maxLen)) + require.NoError(t, err) + require.NoError(t, stream.Open(ctx)) + require.EqualValues(t, maxLen, rdb.XLen(ctx, flatKey).Val()) + events, err := stream.Snapshot(ctx) + require.NoError(t, err) + require.Len(t, events, maxLen) + require.Equal(t, strconv.Itoa(legacyEvents-maxLen), string(events[0].Payload())) + require.NoError(t, stream.Destroy(ctx)) +} + +func TestSnapshotRejectsPhysicalBoundViolation(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + const maxLen = 3 + stream, err := NewStream(t.Name(), rdb, options.WithStreamMaxLen(maxLen)) + require.NoError(t, err) + require.NoError(t, stream.Open(ctx)) + for index := 0; index <= maxLen; index++ { + require.NoError(t, rdb.XAdd(ctx, &redis.XAddArgs{ + Stream: stream.key, + Values: map[string]any{nameKey: "event", payloadKey: strconv.Itoa(index)}, + }).Err()) + } + _, err = stream.Snapshot(ctx) + require.ErrorIs(t, err, ErrSnapshotBoundExceeded) + require.NoError(t, stream.Destroy(ctx)) +} + +func TestDeadlineIsAbsoluteAndNeverExtended(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + deadline := time.Now().Add(1500 * time.Millisecond).Truncate(time.Millisecond) + stream, err := NewStream(t.Name(), rdb, options.WithStreamDeadline(deadline)) + require.NoError(t, err) + _, err = stream.Add(ctx, "first", []byte("payload")) + require.NoError(t, err) + firstExpiry, err := rdb.PExpireTime(ctx, stream.key).Result() + require.NoError(t, err) + expectedExpiry := time.Duration(deadline.UnixMilli()) * time.Millisecond + require.Equal(t, expectedExpiry, firstExpiry) + + time.Sleep(100 * time.Millisecond) + reopened, err := NewStream(t.Name(), rdb, options.WithStreamDeadline(deadline)) + require.NoError(t, err) + _, err = reopened.Add(ctx, "second", []byte("payload")) + require.NoError(t, err) + secondExpiry, err := rdb.PExpireTime(ctx, stream.key).Result() + require.NoError(t, err) + require.Equal(t, firstExpiry, secondExpiry) + + _, err = stream.AddOnce(ctx, "command", "third", []byte("payload")) + require.NoError(t, err) + require.Equal(t, expectedExpiry, rdb.PExpireTime(ctx, idempotencyKeyMap(stream)).Val()) + require.Equal(t, expectedExpiry, rdb.PExpireTime(ctx, stream.key).Val()) + require.Equal(t, expectedExpiry, rdb.PExpireTime(ctx, recoveryCursorKey(stream)).Val()) + require.NoError(t, stream.Destroy(ctx)) +} + +func TestDeadlineValidationReopenAndElapsedOperations(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + deadline := time.Now().Add(150 * time.Millisecond).Truncate(time.Millisecond) + stream, err := NewStream(t.Name(), rdb, options.WithStreamDeadline(deadline)) + require.NoError(t, err) + _, err = stream.Add(ctx, "event", []byte("payload")) + require.NoError(t, err) + same, err := NewStream(t.Name(), rdb, options.WithStreamDeadline(deadline)) + require.NoError(t, err) + require.NoError(t, same.Open(ctx)) + conflicting, err := NewStream( + t.Name(), + rdb, + options.WithStreamDeadline(deadline.Add(time.Second)), + ) + require.NoError(t, err) + require.ErrorIs(t, conflicting.Open(ctx), ErrStreamConfigMismatch) + + require.Eventually(t, func() bool { + _, addErr := stream.Add(ctx, "late", []byte("payload")) + return errors.Is(addErr, ErrDeadlineElapsed) + }, time.Second, 10*time.Millisecond) + _, err = stream.AddOnce(ctx, "late", "late", []byte("payload")) + require.ErrorIs(t, err, ErrDeadlineElapsed) + events, err := stream.Snapshot(ctx) + require.ErrorIs(t, err, ErrDeadlineElapsed) + require.Nil(t, events) + require.NoError(t, stream.Destroy(ctx)) +} + +func TestAddOnceDestroyRecreateIsolatesGeneration(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + firstDeadline := time.Now().Add(10 * time.Second).Truncate(time.Millisecond) + first, err := NewStream(t.Name(), rdb, options.WithStreamDeadline(firstDeadline)) + require.NoError(t, err) + old, err := first.AddOnce(ctx, "command", "old", []byte("payload")) + require.NoError(t, err) + oldDedupe := idempotencyKeyMap(first) + require.NoError(t, first.Destroy(ctx)) + require.EqualValues(t, 0, rdb.Exists(ctx, oldDedupe).Val()) + + secondDeadline := firstDeadline.Add(time.Second) + second, err := NewStream(t.Name(), rdb, options.WithStreamDeadline(secondDeadline)) + require.NoError(t, err) + fresh, err := second.AddOnce(ctx, "command", "new", []byte("different")) + require.NoError(t, err) + require.NotEqual(t, first.Generation(), second.Generation()) + require.NotEqual(t, first.key, second.key) + require.ErrorIs(t, func() error { + _, snapshotErr := first.Snapshot(ctx) + return snapshotErr + }(), ErrStreamDestroyed) + require.NotEqual(t, old+"@"+first.key, fresh+"@"+second.key) + require.NoError(t, second.Destroy(ctx)) +} + +func TestAddOnceDestroyRaceCannotCrossGeneration(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + deadline := time.Now().Add(10 * time.Second).Truncate(time.Millisecond) + old, err := NewStream(t.Name(), rdb, options.WithStreamDeadline(deadline)) + require.NoError(t, err) + require.NoError(t, old.Open(ctx)) + start := make(chan struct{}) + result := make(chan error, 1) + go func() { + <-start + _, addErr := old.AddOnce(ctx, "shared", "old", []byte("payload")) + result <- addErr + }() + close(start) + destroyErr := old.Destroy(ctx) + require.NoError(t, destroyErr) + addErr := <-result + if addErr != nil { + require.ErrorIs(t, addErr, ErrStreamDestroyed) + } + + freshDeadline := deadline.Add(time.Second) + fresh, err := NewStream(t.Name(), rdb, options.WithStreamDeadline(freshDeadline)) + require.NoError(t, err) + freshResult, err := fresh.AddOnce( + ctx, + "shared", + "fresh", + []byte("new payload"), + ) + require.NoError(t, err) + require.NotEmpty(t, freshResult) + events, err := fresh.Snapshot(ctx) + require.NoError(t, err) + require.Len(t, events, 1) + require.Equal(t, "fresh", events[0].EventName()) + require.NoError(t, fresh.Destroy(ctx)) +} + +func TestSnapshotOrderedSideEffectFreeAndMalformedAtomic(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + stream, err := NewStream(t.Name(), rdb) + require.NoError(t, err) + firstID, err := stream.Add(ctx, "first", []byte("one")) + require.NoError(t, err) + secondID, err := stream.Add(ctx, "second", []byte("two"), options.WithTopic("topic")) + require.NoError(t, err) + events, err := stream.Snapshot(ctx) + require.NoError(t, err) + require.Len(t, events, 2) + require.Equal(t, []string{firstID, secondID}, []string{events[0].ID(), events[1].ID()}) + require.Equal(t, []string{"first", "second"}, []string{events[0].EventName(), events[1].EventName()}) + require.Equal(t, "topic", events[1].Topic()) + require.EqualValues(t, 0, rdb.Exists(ctx, recoveryCursorKey(stream)).Val()) + groups, err := rdb.XInfoGroups(ctx, stream.key).Result() + require.NoError(t, err) + require.Empty(t, groups) + + _, err = rdb.XAdd(ctx, &redis.XAddArgs{ + Stream: stream.key, + Values: map[string]any{payloadKey: "missing name"}, + }).Result() + require.NoError(t, err) + events, err = stream.Snapshot(ctx) + require.ErrorContains(t, err, "malformed event") + require.Nil(t, events) + require.NoError(t, stream.Destroy(ctx)) +} + +func TestExactPublicationOptionValidation(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + deadline := time.Now().Add(time.Second) + _, err := NewStream( + t.Name(), + rdb, + options.WithStreamTTL(time.Second), + options.WithStreamDeadline(deadline), + ) + require.ErrorContains(t, err, "mutually exclusive") + _, err = NewStream(t.Name(), rdb, options.WithStreamDeadline(time.Time{})) + require.ErrorContains(t, err, "must not be zero") + + stream, err := NewStream(t.Name(), rdb, options.WithStreamDeadline(deadline)) + require.NoError(t, err) + _, err = stream.AddOnce(ctx, "", "event", nil) + require.ErrorContains(t, err, "must not be empty") + _, err = stream.AddOnce(ctx, string(make([]byte, maxIdempotencyKeyBytes+1)), "event", nil) + require.ErrorContains(t, err, "exceeds") + _, err = stream.AddOnce(ctx, "key", "", nil) + require.ErrorContains(t, err, "event name must not be empty") + _, err = stream.AddOnce( + ctx, + "key", + "event", + nil, + options.WithOnlyIfStreamExists(), + ) + require.ErrorContains(t, err, "does not support") + past := time.Now().Add(-time.Second) + pastStream, err := NewStream(t.Name()+"-past", rdb, options.WithStreamDeadline(past)) + require.NoError(t, err) + _, err = pastStream.AddOnce(ctx, "past", "event", nil) + require.ErrorIs(t, err, ErrDeadlineElapsed) + valid, err := stream.AddOnce(ctx, "valid", "event", nil) + require.NoError(t, err) + require.NotEmpty(t, valid) + require.NoError(t, stream.Destroy(ctx)) + + ttlStream, err := NewStream(t.Name()+"-ttl", rdb, options.WithStreamTTL(time.Second)) + require.NoError(t, err) + _, err = ttlStream.Add(ctx, "event", nil) + require.NoError(t, err) + deadlineHandle, err := NewStream( + t.Name()+"-ttl", + rdb, + options.WithStreamDeadline(deadline), + ) + require.NoError(t, err) + require.ErrorIs(t, deadlineHandle.Open(ctx), ErrStreamConfigMismatch) + require.NoError(t, ttlStream.Destroy(ctx)) +} + +func TestSnapshotDoesNotInitializeAbsentStream(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + stream, err := NewStream(t.Name(), rdb) + require.NoError(t, err) + + events, err := stream.Snapshot(ctx) + require.ErrorIs(t, err, ErrStreamNotFound) + require.Nil(t, events) + require.EqualValues(t, 0, rdb.Exists(ctx, stream.lifecycleKey, streamKey(t.Name())).Val()) + require.Empty(t, stream.Generation()) +} + +func TestSnapshotRejectsUnboundedRetentionBeforeRange(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + stream, err := NewStream(t.Name(), rdb, options.WithUnboundedStream()) + require.NoError(t, err) + _, err = stream.Add(ctx, "event", []byte("payload")) + require.NoError(t, err) + + events, err := stream.Snapshot(ctx) + require.ErrorIs(t, err, ErrSnapshotUnbounded) + require.Nil(t, events) + require.NoError(t, stream.Destroy(ctx)) +} + +func TestRejectedLegacyRetentionAdoptionDoesNotMutateLifecycle(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + name := t.Name() + lifecycle := streamLifecycleKey(name) + deadline := time.Now().Add(time.Hour).UnixMilli() + require.NoError(t, rdb.HSet( + ctx, + lifecycle, + "generation", "1", + "state", streamStateActive, + streamPhysicalKey, streamKey(name), + streamDeadlineKey, deadline, + ).Err()) + before, err := rdb.HGetAll(ctx, lifecycle).Result() + require.NoError(t, err) + opener, err := NewStream(name, rdb, options.WithStreamTTL(time.Minute)) + require.NoError(t, err) + + require.ErrorIs(t, opener.Open(ctx), ErrStreamConfigMismatch) + after, err := rdb.HGetAll(ctx, lifecycle).Result() + require.NoError(t, err) + require.Equal(t, before, after) +} + +func TestRetentionConfigurationIsImmutable(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + stream, err := NewStream(t.Name(), rdb, options.WithStreamMaxLen(12)) + require.NoError(t, err) + _, err = stream.Add(ctx, "event", nil) + require.NoError(t, err) + + differentLimit, err := NewStream(t.Name(), rdb, options.WithStreamMaxLen(13)) + require.NoError(t, err) + require.ErrorIs(t, differentLimit.Open(ctx), ErrStreamConfigMismatch) + differentMode, err := NewStream(t.Name(), rdb, options.WithStreamTTL(time.Second)) + require.NoError(t, err) + require.ErrorIs(t, differentMode.Open(ctx), ErrStreamConfigMismatch) + _, err = NewStream( + t.Name()+"-invalid", + rdb, + options.WithStreamMaxLen(1), + options.WithUnboundedStream(), + ) + require.ErrorContains(t, err, "mutually exclusive") + require.NoError(t, stream.Destroy(ctx)) +} + +func TestDefaultHandlesAdoptWriterRetention(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + writer, err := NewStream( + t.Name(), + rdb, + options.WithStreamMaxLen(50_000), + options.WithStreamSlidingTTL(time.Minute), + ) + require.NoError(t, err) + _, err = writer.Add(ctx, "event", nil) + require.NoError(t, err) + + readerHandle, err := NewStream(t.Name(), rdb) + require.NoError(t, err) + reader, err := readerHandle.NewReader(ctx) + require.NoError(t, err) + reader.Close() + require.Equal(t, 50_000, readerHandle.MaxLen) + require.Equal(t, time.Minute, readerHandle.ttl) + require.True(t, readerHandle.ttlSliding) + + destroyHandle, err := NewStream(t.Name(), rdb) + require.NoError(t, err) + require.NoError(t, destroyHandle.Destroy(ctx)) +} + +func TestNonCreatingOperationsDoNotInitializeLifecycle(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + stream, err := NewStream(t.Name(), rdb) + require.NoError(t, err) + + _, err = stream.Add(ctx, "event", nil, options.WithOnlyIfStreamExists()) + require.ErrorIs(t, err, ErrStreamNotFound) + reader, err := stream.NewReader(ctx) + require.Nil(t, reader) + require.ErrorIs(t, err, ErrStreamNotFound) + require.ErrorIs(t, stream.Destroy(ctx), ErrStreamNotFound) + require.EqualValues(t, 0, rdb.Exists(ctx, stream.lifecycleKey, streamKey(t.Name())).Val()) +} + +func TestSubMillisecondRetentionDurationsAreRejected(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + for _, option := range []options.Stream{ + options.WithStreamTTL(0), + options.WithStreamSlidingTTL(0), + options.WithStreamTTL(500 * time.Microsecond), + options.WithStreamSlidingTTL(500 * time.Microsecond), + } { + _, err := NewStream(t.Name(), rdb, option) + require.ErrorContains(t, err, "at least 1ms") + } +} + +func TestAddOnceExactIdentityAndBodyLimit(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + deadline := time.Now().Add(time.Minute).Truncate(time.Millisecond) + stream, err := NewStream(t.Name(), rdb, options.WithStreamDeadline(deadline)) + require.NoError(t, err) + first, err := stream.AddOnce(ctx, "key", "a:1", []byte("b")) + require.NoError(t, err) + retry, err := stream.AddOnce(ctx, "key", "a:1", []byte("b")) + require.NoError(t, err) + require.Equal(t, first, retry) + _, err = stream.AddOnce(ctx, "key", "a", []byte("1:b")) + require.ErrorIs(t, err, ErrIdempotencyConflict) + _, err = stream.AddOnce(ctx, "large", "event", make([]byte, maxAddOnceBodyBytes+1)) + require.ErrorContains(t, err, "exceeds") + require.NoError(t, stream.Destroy(ctx)) +} + +func TestSinkCloseTreatsDeadlineElapsedAsTerminal(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + deadline := time.Now().Add(2 * time.Second).Truncate(time.Millisecond) + stream, err := NewStream(t.Name(), rdb, options.WithStreamDeadline(deadline)) + require.NoError(t, err) + sink, err := stream.NewSink( + ctx, + "sink", + options.WithSinkBlockDuration(25*time.Millisecond), + ) + require.NoError(t, err) + events := sink.Subscribe() + require.Eventually(t, func() bool { + return errors.Is(stream.verifyGeneration(ctx), ErrDeadlineElapsed) + }, 5*time.Second, 10*time.Millisecond) + require.NoError(t, sink.Close(ctx)) + _, open := <-events + require.False(t, open) + require.True(t, sink.IsClosed()) + require.NoError(t, stream.Destroy(ctx)) +} + +// DialHook preserves normal Redis dialing. +func (h *ambiguousAddOnceHook) DialHook(next redis.DialHook) redis.DialHook { + return func(ctx context.Context, network, addr string) (net.Conn, error) { + return next(ctx, network, addr) + } +} + +// ProcessHook injects one error only after Redis committed AddOnce. +func (h *ambiguousAddOnceHook) ProcessHook(next redis.ProcessHook) redis.ProcessHook { + return func(ctx context.Context, cmd redis.Cmder) error { + err := next(ctx, cmd) + if err != nil || cmd.Name() != "evalsha" { + return err + } + args := cmd.Args() + if len(args) > 1 && args[1] == addOnceScript.Hash() && + h.fail.CompareAndSwap(true, false) { + return h.err + } + return err + } +} + +// ProcessPipelineHook preserves Redis pipelines. +func (h *ambiguousAddOnceHook) ProcessPipelineHook(next redis.ProcessPipelineHook) redis.ProcessPipelineHook { + return func(ctx context.Context, cmds []redis.Cmder) error { + return next(ctx, cmds) + } +} diff --git a/streaming/options/add_stream.go b/streaming/options/add_stream.go index cda5a1f..366c086 100644 --- a/streaming/options/add_stream.go +++ b/streaming/options/add_stream.go @@ -9,8 +9,14 @@ type ( // AddStream is an option for adding a stream to a sink. AddStream func(*AddStreamOptions) + // AddStreamOptions keeps its v1 field first; new fields are only ever + // appended so existing keyed construction remains source-compatible. AddStreamOptions struct { + // LastEventID is the ID after which delivery starts for this stream. LastEventID string + // startOptions counts applied start-position options to reject + // conflicting combinations. + startOptions int } ) @@ -21,6 +27,7 @@ type ( func WithAddStreamStartAtNewest() AddStream { return func(o *AddStreamOptions) { o.LastEventID = "$" + o.startOptions++ } } @@ -31,6 +38,7 @@ func WithAddStreamStartAtNewest() AddStream { func WithAddStreamStartAtOldest() AddStream { return func(o *AddStreamOptions) { o.LastEventID = "0" + o.startOptions++ } } @@ -41,6 +49,7 @@ func WithAddStreamStartAtOldest() AddStream { func WithAddStreamStartAfter(id string) AddStream { return func(o *AddStreamOptions) { o.LastEventID = id + o.startOptions++ } } @@ -51,6 +60,7 @@ func WithAddStreamStartAfter(id string) AddStream { func WithAddStreamStartAt(startAt time.Time) AddStream { return func(o *AddStreamOptions) { o.LastEventID = fmt.Sprintf("%d-0", startAt.UnixMilli()) + o.startOptions++ } } @@ -63,6 +73,13 @@ func ParseAddStreamOptions(opts ...AddStream) AddStreamOptions { return options } +// HasConflictingStartOptions reports whether more than one cursor-start option +// was supplied. AddStream rejects this instead of silently accepting the last +// option. +func (o AddStreamOptions) HasConflictingStartOptions() bool { + return o.startOptions > 1 +} + // defaultAddStreamOptions returns the default options. func defaultAddStreamOptions() AddStreamOptions { return AddStreamOptions{} diff --git a/streaming/options/compat_test.go b/streaming/options/compat_test.go index 45ac587..c621949 100644 --- a/streaming/options/compat_test.go +++ b/streaming/options/compat_test.go @@ -1,7 +1,9 @@ -// Package options_test locks the v1 source compatibility of the exported -// option structs: external code constructing them with unkeyed (positional) -// literals must keep compiling. Adding, removing, reordering, or unexporting -// a field breaks the literals below at compile time. +// Package options_test locks the source compatibility of the exported option +// structs: every v1 field keeps its name, type, and position (v1 fields form a +// stable prefix, in v1 order), and new fields are only ever appended. Keyed +// construction and field access from v1 code must keep compiling; unkeyed +// literals were only guaranteed within the v1 patch line and are not part of +// the contract for feature releases. package options_test import ( @@ -14,15 +16,36 @@ import ( "goa.design/pulse/streaming/options" ) -// TestOptionStructsKeepV1PositionalShape compiles v1 unkeyed literals of +// TestOptionStructsKeepV1Fields compiles keyed literals of every v1 field of // every exported option struct. The assertions only exist to consume the // literals; the test is about compilation. -func TestOptionStructsKeepV1PositionalShape(t *testing.T) { - reader := options.ReaderOptions{time.Second, 100, "topic", "pattern", 10, "0"} - sink := options.SinkOptions{time.Second, 100, "topic", "pattern", 10, "0", true, time.Minute} - addStream := options.AddStreamOptions{"42-0"} - stream := options.StreamOptions{1000, pulse.NoopLogger(), time.Minute, true} - addEvent := options.AddEventOptions{"topic", true} +func TestOptionStructsKeepV1Fields(t *testing.T) { + reader := options.ReaderOptions{ + BlockDuration: time.Second, + MaxPolled: 100, + Topic: "topic", + TopicPattern: "pattern", + BufferSize: 10, + LastEventID: "0", + } + sink := options.SinkOptions{ + BlockDuration: time.Second, + MaxPolled: 100, + Topic: "topic", + TopicPattern: "pattern", + BufferSize: 10, + LastEventID: "0", + NoAck: true, + AckGracePeriod: time.Minute, + } + addStream := options.AddStreamOptions{LastEventID: "42-0"} + stream := options.StreamOptions{ + MaxLen: 1000, + Logger: pulse.NoopLogger(), + TTL: time.Minute, + TTLSliding: true, + } + addEvent := options.AddEventOptions{Topic: "topic", OnlyIfStreamExists: true} assert.Equal(t, time.Second, reader.BlockDuration) assert.Equal(t, time.Minute, sink.AckGracePeriod) diff --git a/streaming/options/options_test.go b/streaming/options/options_test.go index 1e4f7fc..4b35734 100644 --- a/streaming/options/options_test.go +++ b/streaming/options/options_test.go @@ -11,6 +11,7 @@ import ( ) func TestStreamOptions(t *testing.T) { + deadline := time.Date(2030, 1, 1, 0, 0, 0, 0, time.UTC) cases := []struct { name string opts []Stream @@ -29,9 +30,19 @@ func TestStreamOptions(t *testing.T) { name: "maxlen", opts: []Stream{WithStreamMaxLen(10)}, want: StreamOptions{ - MaxLen: 10, - Logger: pulse.NoopLogger(), - TTL: 0, + MaxLen: 10, + MaxLenSet: true, + Logger: pulse.NoopLogger(), + TTL: 0, + }, + }, + { + name: "unbounded", + opts: []Stream{WithUnboundedStream()}, + want: StreamOptions{ + MaxLen: 1000, + Unbounded: true, + Logger: pulse.NoopLogger(), }, }, { @@ -50,6 +61,7 @@ func TestStreamOptions(t *testing.T) { MaxLen: 1000, Logger: pulse.NoopLogger(), TTL: 10 * time.Second, + TTLSet: true, TTLSliding: false, }, }, @@ -60,9 +72,20 @@ func TestStreamOptions(t *testing.T) { MaxLen: 1000, Logger: pulse.NoopLogger(), TTL: 10 * time.Second, + TTLSet: true, TTLSliding: true, }, }, + { + name: "deadline", + opts: []Stream{WithStreamDeadline(deadline)}, + want: StreamOptions{ + MaxLen: 1000, + Logger: pulse.NoopLogger(), + Deadline: deadline, + DeadlineSet: true, + }, + }, } for _, c := range cases { @@ -152,6 +175,7 @@ func TestReaderOptions(t *testing.T) { MaxPolled: 1000, BufferSize: 1000, LastEventID: "foo", + startOptions: 1, }, }, } @@ -251,6 +275,7 @@ func TestSinkOptions(t *testing.T) { BufferSize: 1000, LastEventID: "foo", AckGracePeriod: 20 * time.Second, + startOptions: 1, }, }, { @@ -262,6 +287,7 @@ func TestSinkOptions(t *testing.T) { BufferSize: 1000, LastEventID: fmt.Sprintf("%d-0", date.UnixMilli()), AckGracePeriod: 20 * time.Second, + startOptions: 1, }, }, { @@ -318,14 +344,16 @@ func TestAddStreamOptions(t *testing.T) { name: "last event ID", opts: []AddStream{WithAddStreamStartAfter("foo")}, want: AddStreamOptions{ - LastEventID: "foo", + LastEventID: "foo", + startOptions: 1, }, }, { name: "start at", opts: []AddStream{WithAddStreamStartAt(date)}, want: AddStreamOptions{ - LastEventID: fmt.Sprintf("%d-0", date.UnixMilli()), + LastEventID: fmt.Sprintf("%d-0", date.UnixMilli()), + startOptions: 1, }, }, } diff --git a/streaming/options/reader.go b/streaming/options/reader.go index 4aca96f..77ca282 100644 --- a/streaming/options/reader.go +++ b/streaming/options/reader.go @@ -9,19 +9,32 @@ type ( // Reader is a sink creation option. Reader func(*ReaderOptions) + // ReaderOptions keeps its v1 fields first, in v1 order; new fields are + // only ever appended so existing keyed construction and field access + // remain source-compatible across feature releases. ReaderOptions struct { + // BlockDuration is the XREAD block duration. BlockDuration time.Duration - MaxPolled int64 - Topic string - TopicPattern string - BufferSize int - LastEventID string + // MaxPolled is the maximum number of events read per XREAD call. + MaxPolled int64 + // Topic delivers only events published with this exact topic. + Topic string + // TopicPattern delivers only events whose topic matches this regex. + TopicPattern string + // BufferSize is the capacity of each subscription channel. + BufferSize int + // LastEventID is the ID after which delivery starts. + LastEventID string + // startOptions counts applied start-position options to reject + // conflicting combinations. + startOptions int } ) // WithReaderBlockDuration sets the maximum amount of time the reader waits for -// MaxPolled events. The default block duration is 5 seconds. If the block -// duration is set to 0 then the reader blocks indefinitely. +// MaxPolled events. The default block duration is 5 seconds. NewReader rejects +// durations below one millisecond because Redis block timing is millisecond +// precision and every read must have a finite shutdown bound. func WithReaderBlockDuration(d time.Duration) Reader { return func(o *ReaderOptions) { o.BlockDuration = d @@ -44,7 +57,7 @@ func WithReaderTopic(topic string) Reader { } // WithReaderTopicPattern sets the reader topic pattern. -// pattern must be a valid regular expression or NewReader panics. +// NewReader returns an error when pattern is not a valid regular expression. func WithReaderTopicPattern(pattern string) Reader { return func(o *ReaderOptions) { o.TopicPattern = pattern @@ -67,6 +80,7 @@ func WithReaderBufferSize(size int) Reader { func WithReaderStartAtNewest() Reader { return func(o *ReaderOptions) { o.LastEventID = "$" + o.startOptions++ } } @@ -76,6 +90,7 @@ func WithReaderStartAtNewest() Reader { func WithReaderStartAtOldest() Reader { return func(o *ReaderOptions) { o.LastEventID = "0" + o.startOptions++ } } @@ -84,6 +99,7 @@ func WithReaderStartAtOldest() Reader { func WithReaderStartAfter(id string) Reader { return func(o *ReaderOptions) { o.LastEventID = id + o.startOptions++ } } @@ -92,6 +108,7 @@ func WithReaderStartAfter(id string) Reader { func WithReaderStartAt(startAt time.Time) Reader { return func(o *ReaderOptions) { o.LastEventID = fmt.Sprintf("%d-0", startAt.UnixMilli()) + o.startOptions++ } } @@ -105,6 +122,13 @@ func ParseReaderOptions(opts ...Reader) ReaderOptions { return o } +// HasConflictingStartOptions reports whether more than one cursor-start option +// was supplied. Constructors reject this instead of silently accepting the +// last option. +func (o ReaderOptions) HasConflictingStartOptions() bool { + return o.startOptions > 1 +} + // defaultReaderOptions returns the default options. func defaultReaderOptions() ReaderOptions { return ReaderOptions{ diff --git a/streaming/options/shape_pin_test.go b/streaming/options/shape_pin_test.go new file mode 100644 index 0000000..041dbfb --- /dev/null +++ b/streaming/options/shape_pin_test.go @@ -0,0 +1,31 @@ +// Positional pins of the full current shape of every exported option struct. +// Any field insertion, reorder, or retype breaks these literals at compile +// time, forcing a deliberate decision; appending a field only updates the +// literal it extends. This complements compat_test.go, which pins the +// external keyed-construction contract for v1 fields. +package options + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + + "goa.design/pulse/pulse" +) + +// TestOptionStructsPositionalShape compiles unkeyed literals of the current +// full shapes. The assertions only consume the literals. +func TestOptionStructsPositionalShape(t *testing.T) { + reader := ReaderOptions{time.Second, 100, "topic", "pattern", 10, "0", 0} + sink := SinkOptions{time.Second, 100, "topic", "pattern", 10, "0", true, time.Minute, 0} + addStream := AddStreamOptions{"42-0", 0} + stream := StreamOptions{1000, pulse.NoopLogger(), time.Minute, true, true, false, true, time.Time{}, false} + addEvent := AddEventOptions{"topic", true} + + assert.Equal(t, time.Second, reader.BlockDuration) + assert.Equal(t, time.Minute, sink.AckGracePeriod) + assert.Equal(t, "42-0", addStream.LastEventID) + assert.Equal(t, 1000, stream.MaxLen) + assert.True(t, addEvent.OnlyIfStreamExists) +} diff --git a/streaming/options/sink.go b/streaming/options/sink.go index f3cb2b4..38b6e36 100644 --- a/streaming/options/sink.go +++ b/streaming/options/sink.go @@ -9,21 +9,37 @@ type ( // Sink is a sink creation option. Sink func(*SinkOptions) + // SinkOptions keeps its v1 fields first, in v1 order; new fields are + // only ever appended so existing keyed construction and field access + // remain source-compatible across feature releases. SinkOptions struct { - BlockDuration time.Duration - MaxPolled int64 - Topic string - TopicPattern string - BufferSize int - LastEventID string - NoAck bool + // BlockDuration is the XREADGROUP block duration. + BlockDuration time.Duration + // MaxPolled is the maximum number of events read per XREADGROUP call. + MaxPolled int64 + // Topic delivers only events published with this exact topic. + Topic string + // TopicPattern delivers only events whose topic matches this regex. + TopicPattern string + // BufferSize is the capacity of each subscription channel. + BufferSize int + // LastEventID is the ID after which delivery starts. + LastEventID string + // NoAck atomically acknowledges each event before delivery. + NoAck bool + // AckGracePeriod bounds how long an unacknowledged event stays owned + // by one consumer before stale recovery may reclaim it. AckGracePeriod time.Duration + // startOptions counts applied start-position options to reject + // conflicting combinations. + startOptions int } ) // WithSinkBlockDuration sets the maximum amount of time the sink waits for -// MaxPolled events. The default block duration is 5 seconds. If the block -// duration is set to 0 then the sink blocks indefinitely. +// MaxPolled events. The default block duration is 5 seconds. NewSink rejects +// durations below one millisecond because Redis block timing is millisecond +// precision and every read must have a finite shutdown bound. func WithSinkBlockDuration(d time.Duration) Sink { return func(o *SinkOptions) { o.BlockDuration = d @@ -46,7 +62,7 @@ func WithSinkTopic(topic string) Sink { } // WithSinkTopicPattern sets the sink topic pattern. -// pattern must be a valid regular expression or NewSink panics. +// NewSink returns an error when pattern is not a valid regular expression. func WithSinkTopicPattern(pattern string) Sink { return func(o *SinkOptions) { o.TopicPattern = pattern @@ -68,6 +84,7 @@ func WithSinkBufferSize(size int) Sink { func WithSinkStartAtNewest() Sink { return func(o *SinkOptions) { o.LastEventID = "$" + o.startOptions++ } } @@ -77,6 +94,7 @@ func WithSinkStartAtNewest() Sink { func WithSinkStartAtOldest() Sink { return func(o *SinkOptions) { o.LastEventID = "0" + o.startOptions++ } } @@ -86,6 +104,7 @@ func WithSinkStartAtOldest() Sink { func WithSinkStartAfter(id string) Sink { return func(o *SinkOptions) { o.LastEventID = id + o.startOptions++ } } @@ -95,18 +114,21 @@ func WithSinkStartAfter(id string) Sink { func WithSinkStartAt(startAt time.Time) Sink { return func(o *SinkOptions) { o.LastEventID = fmt.Sprintf("%d-0", startAt.UnixMilli()) + o.startOptions++ } } -// WithSinkNoAck removes the need to acknowledge events read from the sink. +// WithSinkNoAck atomically acknowledges each event before delivering it to +// subscribers, preserving at-most-once delivery without requiring Sink.Ack. func WithSinkNoAck() Sink { return func(o *SinkOptions) { o.NoAck = true } } -// WithSinkAckGracePeriod sets the grace period for acknowledging events. The -// default grace period is 20 seconds. +// WithSinkAckGracePeriod sets the grace period for acknowledging events. The +// default grace period is 20 seconds; NewSink rejects values below one +// millisecond. // Note: all sinks with identical names must have the same ack grace period. func WithSinkAckGracePeriod(d time.Duration) Sink { return func(o *SinkOptions) { @@ -123,6 +145,13 @@ func ParseSinkOptions(opts ...Sink) SinkOptions { return o } +// HasConflictingStartOptions reports whether more than one cursor-start option +// was supplied. Constructors reject this instead of silently accepting the +// last option. +func (o SinkOptions) HasConflictingStartOptions() bool { + return o.startOptions > 1 +} + // defaultSinkOptions returns the default options. func defaultSinkOptions() SinkOptions { return SinkOptions{ diff --git a/streaming/options/stream.go b/streaming/options/stream.go index 581468b..8c148a3 100644 --- a/streaming/options/stream.go +++ b/streaming/options/stream.go @@ -10,23 +10,47 @@ type ( // Stream is a stream creation option. Stream func(*StreamOptions) + // StreamOptions keeps its v1 fields first, in v1 order; new fields are + // only ever appended so existing keyed construction and field access + // remain source-compatible across feature releases. StreamOptions struct { MaxLen int Logger pulse.Logger - // TTL configures a retention window for the Redis key backing the stream. // When zero, no TTL is applied. TTL time.Duration // TTLSliding controls whether the TTL is refreshed on every published event. // When false, the TTL is applied once (absolute TTL) and never extended. TTLSliding bool + + // MaxLenSet distinguishes an omitted MaxLen from an explicit request. + MaxLenSet bool + // Unbounded disables MAXLEN trimming entirely. + Unbounded bool + // TTLSet distinguishes an omitted TTL from an explicitly invalid zero. + TTLSet bool + // Deadline is the immutable absolute expiry claimed by this stream + // generation when DeadlineSet is true. + Deadline time.Time + // DeadlineSet distinguishes an omitted deadline from the zero time. + DeadlineSet bool } ) -// WithStreamMaxLen sets the maximum number of events stored by the stream. +// WithStreamMaxLen sets the positive maximum number of events stored by the +// stream. NewStream rejects zero and negative values. func WithStreamMaxLen(len int) Stream { return func(o *StreamOptions) { o.MaxLen = len + o.MaxLenSet = true + } +} + +// WithUnboundedStream disables MAXLEN trimming. Callers must remove settled +// events explicitly. It cannot be combined with WithStreamMaxLen. +func WithUnboundedStream() Stream { + return func(o *StreamOptions) { + o.Unbounded = true } } @@ -37,24 +61,39 @@ func WithStreamLogger(logger pulse.Logger) Stream { } } -// WithStreamTTL sets an absolute TTL on the Redis key backing the stream. -// The TTL is set once (when the key is created) and never extended. +// WithStreamTTL sets an absolute TTL on the Redis key backing the stream. The +// TTL is set once and never extended; NewStream rejects values below Redis's +// one-millisecond precision. func WithStreamTTL(ttl time.Duration) Stream { return func(o *StreamOptions) { o.TTL = ttl + o.TTLSet = true o.TTLSliding = false } } // WithStreamSlidingTTL sets a sliding TTL on the Redis key backing the stream. -// The TTL is refreshed on every published event. +// The TTL is refreshed on every published event; NewStream rejects values +// below Redis's one-millisecond precision. func WithStreamSlidingTTL(ttl time.Duration) Stream { return func(o *StreamOptions) { o.TTL = ttl + o.TTLSet = true o.TTLSliding = true } } +// WithStreamDeadline makes the stream generation expire at deadline. The +// deadline is persisted when the handle first opens; handles for the same +// generation must not request a different deadline. TTL options cannot be +// combined with this option. +func WithStreamDeadline(deadline time.Time) Stream { + return func(o *StreamOptions) { + o.Deadline = deadline + o.DeadlineSet = true + } +} + // ParseStreamOptions parses the given options and returns the corresponding // StreamOptions. func ParseStreamOptions(opts ...Stream) StreamOptions { diff --git a/streaming/reader.go b/streaming/reader.go index eaef0b9..cc9eaa7 100644 --- a/streaming/reader.go +++ b/streaming/reader.go @@ -2,21 +2,37 @@ package streaming import ( "context" + "errors" "fmt" "math/rand" "regexp" "strconv" "strings" "sync" + "sync/atomic" "time" redis "github.com/redis/go-redis/v9" + "goa.design/clue/log" "goa.design/pulse/pulse" "goa.design/pulse/streaming/options" ) type ( + // readRetry bounds the command rate while a long-lived reader or sink waits + // for Redis to recover. Successful commands reset the exponential delay. + readRetry struct { + failures int + jitter func(int64) int64 + } + + // readerSnapshot is the exact immutable stream set used by one XREAD. + readerSnapshot struct { + streams map[string]*Stream + args []string + } + // Reader represents a stream reader. Reader struct { // closed is true if Close completed. @@ -48,13 +64,14 @@ type ( closeOnce sync.Once // donechan is the reader donechan channel. donechan chan struct{} - // streamschan notifies the reader when streams are added or - // removed. - streamschan chan struct{} + // ctx bounds blocking Redis reads for the reader lifetime. + ctx context.Context + // cancel interrupts a blocking XREAD when Close begins. + cancel context.CancelFunc // wait is the reader cleanup wait group. wait sync.WaitGroup - // closing is true if Close was called. - closing bool + // closing is true from the instant Close begins. + closing atomic.Bool // eventFilter is the event filter if any. eventFilter eventFilterFunc // logger is the logger used by the reader. @@ -74,6 +91,10 @@ type ( ID string // StreamName is the name of the stream the event belongs to. StreamName string + // StreamGeneration is the immutable stream incarnation that produced the + // event. Events fetched before Destroy may still be in process, so + // handlers that fence side effects can compare this token explicitly. + StreamGeneration string // SinkName is the name of the sink the event belongs to. SinkName string // EventName is the producer-defined event name. @@ -82,30 +103,33 @@ type ( Topic string // Payload is the event payload. Payload []byte - // Acker is the redis client used to acknowledge events. + // Acker acknowledges events according to their sink recovery contract. Acker Acker // streamKey is the Redis key of the stream. streamKey string } - - // readRetry computes jittered exponential backoff delays for reader and - // sink Redis failures. Jitter spans half to full of the current backoff - // so replicas that fail together do not retry in lockstep. - readRetry struct { - backoff time.Duration - } ) const ( - // minReadRetryBackoff is the first retry backoff after a Redis failure. - minReadRetryBackoff = 50 * time.Millisecond - // maxReadRetryBackoff caps the exponential retry backoff. - maxReadRetryBackoff = 5 * time.Second + readRetryInitialDelay = 100 * time.Millisecond + readRetryMaxDelay = 5 * time.Second +) + +var ( + // ErrReaderClosed is returned when stream ownership is changed after reader + // shutdown begins. + ErrReaderClosed = errors.New("pulse streaming: reader is closed") + // ErrLastStream is returned when removing a stream would leave a reader or + // sink without a valid Redis read set. + ErrLastStream = errors.New("pulse streaming: cannot remove final stream") ) // newReader creates a new reader. -func newReader(stream *Stream, opts ...options.Reader) (*Reader, error) { +func newReader(ctx context.Context, stream *Stream, opts ...options.Reader) (*Reader, error) { o := options.ParseReaderOptions(opts...) + if err := validateReaderOptions(o); err != nil { + return nil, err + } var eventFilter eventFilterFunc if o.Topic != "" { eventFilter = func(e *Event) bool { return e.Topic == o.Topic } @@ -117,6 +141,9 @@ func newReader(stream *Stream, opts ...options.Reader) (*Reader, error) { eventFilter = func(e *Event) bool { return topicPatternRegexp.MatchString(e.Topic) } } + logCtx := context.Background() + logCtx = log.WithContext(logCtx, ctx) + runCtx, cancel := context.WithCancel(logCtx) reader := &Reader{ startID: o.LastEventID, streams: []*Stream{stream}, @@ -126,7 +153,8 @@ func newReader(stream *Stream, opts ...options.Reader) (*Reader, error) { maxPolled: o.MaxPolled, bufferSize: o.BufferSize, donechan: make(chan struct{}), - streamschan: make(chan struct{}), + ctx: runCtx, + cancel: cancel, eventFilter: eventFilter, logger: stream.rootLogger.WithPrefix("reader", stream.Name), rdb: stream.rdb, @@ -135,12 +163,17 @@ func newReader(stream *Stream, opts ...options.Reader) (*Reader, error) { return reader, nil } -// Subscribe returns a channel that receives events from the stream. -// The channel is closed when the reader is closed. +// Subscribe returns a channel that receives events from the stream. The +// channel is closed when the reader closes; calls made after shutdown starts +// return an already closed channel. func (r *Reader) Subscribe() <-chan *Event { c := make(chan *Event, r.bufferSize) r.lock.Lock() defer r.lock.Unlock() + if r.closing.Load() { + close(c) + return c + } r.chans = append(r.chans, c) r.start() return c @@ -159,43 +192,75 @@ func (r *Reader) Unsubscribe(c <-chan *Event) { } } -// AddStream adds the stream to the sink. By default the stream cursor starts at -// the same timestamp as the sink main stream cursor. This can be overridden -// with opts. AddStream does nothing if the stream is already part of the sink. +// AddStream adds the stream to the reader. By default the stream cursor starts +// at the same timestamp as the main stream cursor. This can be overridden with +// opts. AddStream does nothing if the stream is already part of the reader and +// returns ErrReaderClosed once shutdown starts. func (r *Reader) AddStream(ctx context.Context, stream *Stream, opts ...options.AddStream) error { r.lock.Lock() defer r.lock.Unlock() - for _, name := range r.streamKeys { - if name == stream.Name { + if r.closing.Load() { + return ErrReaderClosed + } + o := options.ParseAddStreamOptions(opts...) + if err := validateAddStreamOptions(o); err != nil { + return err + } + if err := stream.verifyExistingGeneration(ctx); err != nil { + return err + } + for _, owned := range r.streams { + if owned.Name != stream.Name { + continue + } + if owned.generation == stream.generation { return nil } + return owned.verifyGeneration(ctx) } startID := r.startID - o := options.ParseAddStreamOptions(opts...) if o.LastEventID != "" { startID = o.LastEventID } r.streams = append(r.streams, stream) r.streamKeys = append(r.streamKeys, stream.key) r.streamCursors = append(r.streamCursors, startID) - r.notifyStreamChange() r.logger.Info("added", "stream", stream.Name) return nil } -// RemoveStream removes the stream from the sink, it is idempotent. +// RemoveStream removes the stream from the reader. It is idempotent and returns +// ErrReaderClosed once shutdown starts. Removing the final stream returns +// ErrLastStream. func (r *Reader) RemoveStream(ctx context.Context, stream *Stream) error { r.lock.Lock() defer r.lock.Unlock() + if r.closing.Load() { + return ErrReaderClosed + } + index := -1 for i, st := range r.streams { - if st == stream { - r.streams = append(r.streams[:i], r.streams[i+1:]...) - r.streamKeys = append(r.streamKeys[:i], r.streamKeys[i+1:]...) - r.streamCursors = append(r.streamCursors[:i], r.streamCursors[i+1:]...) + if st == stream || + (stream.generation != "" && + st.Name == stream.Name && + st.generation == stream.generation) { + index = i break } } - r.notifyStreamChange() + if index == -1 { + return nil + } + if len(r.streams) == 1 { + return ErrLastStream + } + attached := r.streams[index] + if err := attached.verifyExistingGeneration(ctx); err != nil { + return err + } + r.streams = append(r.streams[:index], r.streams[index+1:]...) + r.streamKeys = append(r.streamKeys[:index], r.streamKeys[index+1:]...) + r.streamCursors = append(r.streamCursors[:index], r.streamCursors[index+1:]...) r.logger.Info("removed", "stream", stream.Name) return nil } @@ -203,17 +268,21 @@ func (r *Reader) RemoveStream(ctx context.Context, stream *Stream) error { // Close stops event polling and closes the reader channel. It is safe to call // Close multiple times; concurrent callers block until the first Close // completes. Close returns only once the read goroutine has stopped and its -// resources are released, which may take up to one block duration. +// resources are released. The configured finite block duration bounds shutdown +// even when the Redis client does not interrupt a blocking read on cancellation. func (r *Reader) Close() { r.closeOnce.Do(func() { + r.closing.Store(true) + r.cancel() // Close donechan first, without holding the lock, so the signal // reaches the read loop even when it is parked on a fan-out send // to a stalled subscriber (which holds the lock). Otherwise Close // would deadlock acquiring the lock the read loop never releases. close(r.donechan) + // Synchronize with a Subscribe already inside the admission lock so its + // wait-group Add completes before Wait begins. Future subscriptions see + // closing and cannot start the read loop. r.lock.Lock() - r.closing = true - close(r.streamschan) r.lock.Unlock() r.wait.Wait() r.lock.Lock() @@ -244,24 +313,27 @@ func (r *Reader) start() { var xreadFn = (*Reader).xread // read reads events from the streams and sends them to the reader channel. -// Transient Redis failures are retried with jittered exponential backoff so -// reader replicas that fail together do not hammer Redis in lockstep. func (r *Reader) read() { - ctx := context.Background() defer r.cleanup() var retry readRetry for { - streamsEvents, err := xreadFn(r, ctx) + snapshot, err := r.readSnapshot(r.ctx) + if err != nil { + if fatal := fatalReadError(err); fatal != nil { + pulse.Go(r.logger, r.Close) + return + } + if !retry.wait(r.donechan, err, r.logger) { + return + } + continue + } + streamsEvents, err := xreadFn(r, r.ctx, snapshot.args) if r.isClosing() { return } if err != nil { - if err == redis.Nil { - // No event at this time, just loop. - retry.reset() - continue - } - if isFatalReaderError(err) { + if err := fatalReadError(err); err != nil { r.logger.Error(fmt.Errorf("fatal error while reading events: %w, stopping", err)) // Close waits on this goroutine via wait.Wait, so calling it // synchronously here would deadlock and leak the reader and its @@ -270,6 +342,10 @@ func (r *Reader) read() { pulse.Go(r.logger, r.Close) return } + if err == redis.Nil { + retry.reset() + continue + } if !retry.wait(r.donechan, err, r.logger) { return } @@ -277,11 +353,35 @@ func (r *Reader) read() { } retry.reset() - r.lock.Lock() for _, events := range streamsEvents { - streamName := events.Stream[len(streamKeyPrefix):] - if err := streamEvents(ctx, streamName, events.Stream, "", events.Messages, r.rdb, false, r.eventFilter, r.chans, r.donechan, r.logger); err != nil { - r.logger.Error(fmt.Errorf("failed to stream events: %w", err)) + stream := snapshot.streams[events.Stream] + if stream == nil { + continue + } + if verifyErr := stream.verifyGeneration(r.ctx); verifyErr != nil { + err = verifyErr + break + } + r.lock.Lock() + if !r.ownsStream(stream) { + r.lock.Unlock() + continue + } + if err := streamEvents( + r.ctx, + stream, + "", + events.Messages, + false, + r.eventFilter, + r.chans, + r.donechan, + r.rdb, + r.logger, + ); err != nil { + r.logger.Error(fmt.Errorf("failed to stream reader events: %w", err)) + r.lock.Unlock() + continue } for i := range r.streamKeys { if r.streamKeys[i] == events.Stream { @@ -289,19 +389,36 @@ func (r *Reader) read() { break } } + r.lock.Unlock() + } + if fatal := fatalReadError(err); fatal != nil { + pulse.Go(r.logger, r.Close) + return } - r.lock.Unlock() } } -func (r *Reader) xread(ctx context.Context) ([]redis.XStream, error) { - // copy so no two goroutines can share the memory +// readSnapshot verifies and captures the exact stream capabilities used by one +// Redis read. +func (r *Reader) readSnapshot(ctx context.Context) (readerSnapshot, error) { r.lock.Lock() - readStreams := make([]string, len(r.streamKeys)) - copy(readStreams, r.streamKeys) - readStreams = append(readStreams, r.streamCursors...) - r.lock.Unlock() + defer r.lock.Unlock() + snapshot := readerSnapshot{ + streams: make(map[string]*Stream, len(r.streams)), + args: make([]string, 0, len(r.streamKeys)+len(r.streamCursors)), + } + for _, stream := range r.streams { + if err := stream.verifyGeneration(ctx); err != nil { + return readerSnapshot{}, err + } + snapshot.streams[stream.key] = stream + } + snapshot.args = append(snapshot.args, r.streamKeys...) + snapshot.args = append(snapshot.args, r.streamCursors...) + return snapshot, nil +} +func (r *Reader) xread(ctx context.Context, readStreams []string) ([]redis.XStream, error) { r.logger.Debug("reading", "streams", readStreams, "max", r.maxPolled, "block", r.blockDuration) return r.rdb.XRead(ctx, &redis.XReadArgs{ Streams: readStreams, @@ -310,12 +427,15 @@ func (r *Reader) xread(ctx context.Context) ([]redis.XStream, error) { }).Result() } -// notifyStreamChange notifies the reader that the streams have changed. -func (r *Reader) notifyStreamChange() { - select { - case r.streamschan <- struct{}{}: - default: +// ownsStream reports whether the current reader still owns the exact snapshot +// capability after a concurrent RemoveStream. +func (r *Reader) ownsStream(candidate *Stream) bool { + for _, stream := range r.streams { + if stream == candidate { + return true + } } + return false } // cleanup removes the consumer from the consumer groups and removes the reader @@ -333,9 +453,7 @@ func (r *Reader) cleanup() { // isClosing returns true if the reader is stopping. func (r *Reader) isClosing() bool { - r.lock.Lock() - defer r.lock.Unlock() - return r.closing + return r.closing.Load() } // CreatedAt returns the event creation time (millisecond precision). @@ -347,91 +465,66 @@ func (e *Event) CreatedAt() time.Time { return time.Unix(seconds, nanos).UTC() } -// reset clears the backoff so the next failure starts from the minimum delay. -func (r *readRetry) reset() { - r.backoff = 0 -} - -// wait logs err and sleeps for the next jittered backoff delay. It returns -// false when done closes during the wait, signaling the caller to stop. -func (r *readRetry) wait(done <-chan struct{}, err error, logger pulse.Logger) bool { - d := r.next() - logger.Error(fmt.Errorf("failed to read events: %w, retrying in %v", err, d)) - select { - case <-done: - return false - case <-time.After(d): - return true - } -} - -// next returns a delay uniformly distributed in [backoff/2, backoff] and -// doubles the backoff up to maxReadRetryBackoff. -func (r *readRetry) next() time.Duration { - if r.backoff == 0 { - r.backoff = minReadRetryBackoff - } - d := r.backoff/2 + time.Duration(rand.Int63n(int64(r.backoff/2)+1)) - r.backoff = min(2*r.backoff, maxReadRetryBackoff) - return d -} - -// streamEvents filters and streams the Redis messages as events to the -// subscriber channels. The caller is responsible for locking chans. For sinks -// (sinkName non-empty) acker settles events that are never delivered to -// subscribers so the recovery cursor keeps advancing: filtered events are -// acknowledged individually, and when autoAck is set the whole batch is -// settled upfront because delivery is already at-most-once. +// streamEvents filters and streams Redis messages. Sink events receive a +// recovery-aware Acker; auto-ack sinks acknowledge each message before it can +// be exposed to filters or subscribers. func streamEvents( ctx context.Context, - streamName string, - streamKey string, + stream *Stream, sinkName string, msgs []redis.XMessage, - acker Acker, autoAck bool, eventFilter eventFilterFunc, chans []chan *Event, done <-chan struct{}, + rdb *redis.Client, logger pulse.Logger, ) error { if len(msgs) == 0 { return nil } - if autoAck && sinkName != "" { - ids := make([]string, len(msgs)) - for i, msg := range msgs { - ids[i] = msg.ID - } - if err := acker.XAck(ctx, streamKey, sinkName, ids...).Err(); err != nil { - return fmt.Errorf("failed to advance recovery cursor for auto-acked events: %w", err) - } - } for _, event := range msgs { - var topic string - if t, ok := event.Values[topicKey]; ok { - topic = t.(string) + name, topic, payload, err := decodeRedisEvent(event) + if err != nil { + if sinkName != "" { + acker := &recoveryAcker{stream: stream} + if ackErr := acker.XAck(ctx, stream.key, sinkName, event.ID).Err(); ackErr != nil { + return errors.Join(err, fmt.Errorf("acknowledge malformed sink event %s: %w", event.ID, ackErr)) + } + } + logger.Error(err, "stream", stream.Name, "id", event.ID) + continue + } + var acker Acker = rdb + if sinkName != "" { + acker = &recoveryAcker{stream: stream} } ev := &Event{ - ID: event.ID, - StreamName: streamName, - SinkName: sinkName, - EventName: event.Values[nameKey].(string), - Topic: topic, - Payload: []byte(event.Values[payloadKey].(string)), - streamKey: streamKey, - Acker: acker, + ID: event.ID, + StreamName: stream.Name, + StreamGeneration: stream.generation, + SinkName: sinkName, + EventName: name, + Topic: topic, + Payload: payload, + streamKey: stream.key, + Acker: acker, + } + if autoAck { + if err := ev.Acker.XAck(ctx, stream.key, sinkName, event.ID).Err(); err != nil { + return err + } } if eventFilter != nil && !eventFilter(ev) { if sinkName != "" && !autoAck { - if err := acker.XAck(ctx, streamKey, sinkName, ev.ID).Err(); err != nil { - return fmt.Errorf("failed to acknowledge filtered event %s: %w", ev.ID, err) + if err := ev.Acker.XAck(ctx, stream.key, sinkName, event.ID).Err(); err != nil { + return fmt.Errorf("failed to acknowledge filtered sink event %s: %w", event.ID, err) } } - logger.Debug("event filtered", "event", ev.EventName, "id", ev.ID, "stream", streamName) + logger.Debug("event filtered", "event", ev.EventName, "id", ev.ID, "stream", stream.Name) continue } - logger.Debug("event", "stream", streamName, "event", ev.EventName, "id", ev.ID, "channels", len(chans)) + logger.Debug("event", "stream", stream.Name, "event", ev.EventName, "id", ev.ID, "channels", len(chans)) for _, c := range chans { select { case c <- ev: @@ -449,8 +542,91 @@ func streamEvents( return nil } -// isFatalReaderError reports whether the read loop must stop instead of -// retrying, which happens when the underlying stream key was destroyed. -func isFatalReaderError(err error) bool { - return strings.Contains(err.Error(), "stream key no longer exists") +// decodeRedisEvent validates the externally writable Redis stream boundary. +func decodeRedisEvent(event redis.XMessage) (string, string, []byte, error) { + name, ok := event.Values[nameKey].(string) + if !ok || name == "" { + return "", "", nil, fmt.Errorf( + "pulse streaming: malformed event %s: required field %q must be a non-empty string", + event.ID, + nameKey, + ) + } + payload, ok := event.Values[payloadKey].(string) + if !ok { + return "", "", nil, fmt.Errorf( + "pulse streaming: malformed event %s: required field %q must be a string", + event.ID, + payloadKey, + ) + } + var topic string + if value, exists := event.Values[topicKey]; exists { + var valid bool + topic, valid = value.(string) + if !valid { + return "", "", nil, fmt.Errorf( + "pulse streaming: malformed event %s: optional field %q must be a string", + event.ID, + topicKey, + ) + } + } + return name, topic, []byte(payload), nil +} + +// fatalReadError returns errors that permanently invalidate a reader. All +// other errors are transient and are retried through readRetry. +func fatalReadError(err error) error { + if err == nil { + return nil + } + if errors.Is(err, ErrStreamDestroyed) || errors.Is(err, ErrDeadlineElapsed) { + return err + } + return nil +} + +// reset restores the retry delay after Redis processes a read or repairs a +// missing consumer group. +func (r *readRetry) reset() { + r.failures = 0 +} + +// nextDelay returns a half-to-full-jitter exponential delay. The non-zero +// lower bound prevents a hot loop while randomization avoids fleet-wide retry +// synchronization. +func (r *readRetry) nextDelay() time.Duration { + limit := readRetryInitialDelay + for range r.failures { + limit = min(limit*2, readRetryMaxDelay) + if limit == readRetryMaxDelay { + break + } + } + if limit < readRetryMaxDelay { + r.failures++ + } + jitter := r.jitter + if jitter == nil { + jitter = rand.Int63n + } + floor := limit / 2 + return floor + time.Duration(jitter(int64(limit-floor)+1)) +} + +// wait applies the next bounded jittered delay. It returns false when shutdown +// interrupts the wait so Close does not wait for a retry timer. +func (r *readRetry) wait(done <-chan struct{}, err error, logger pulse.Logger) bool { + delay := r.nextDelay() + logger.Error(fmt.Errorf("failed to read events: %w", err), "retry_in", delay) + + timer := time.NewTimer(delay) + defer timer.Stop() + select { + case <-timer.C: + return true + case <-done: + return false + } } diff --git a/streaming/reader_test.go b/streaming/reader_test.go index 788f0b8..f2e0ae2 100644 --- a/streaming/reader_test.go +++ b/streaming/reader_test.go @@ -2,9 +2,9 @@ package streaming import ( "context" - "fmt" "strconv" "strings" + "sync" "testing" "time" @@ -17,6 +17,130 @@ import ( ptesting "goa.design/pulse/testing" ) +func TestReaderRejectsInvalidOptions(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + cases := []struct { + name string + opt options.Reader + }{ + {name: "max polled", opt: options.WithReaderMaxPolled(0)}, + {name: "buffer", opt: options.WithReaderBufferSize(-1)}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + stream, err := NewStream(t.Name(), rdb) + require.NoError(t, err) + _, err = stream.NewReader(ctx, tc.opt) + require.Error(t, err) + }) + } + stream, err := NewStream(t.Name()+"-conflicts", rdb) + require.NoError(t, err) + _, err = stream.NewReader( + ctx, + options.WithReaderTopic("alarms"), + options.WithReaderTopicPattern("alarm.*"), + ) + require.ErrorContains(t, err, "mutually exclusive") + _, err = stream.NewReader( + ctx, + options.WithReaderStartAtNewest(), + options.WithReaderStartAtOldest(), + ) + require.ErrorContains(t, err, "reader cursor-start options are mutually exclusive") + require.ErrorIs(t, stream.Destroy(ctx), ErrStreamNotFound) +} + +func TestReaderDropsMalformedRedisEventWithoutPanicking(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + stream, err := NewStream(t.Name(), rdb) + require.NoError(t, err) + require.NoError(t, stream.Open(ctx)) + reader, err := stream.NewReader( + ctx, + options.WithReaderStartAtOldest(), + options.WithReaderBlockDuration(testBlockDuration), + ) + require.NoError(t, err) + events := reader.Subscribe() + require.NoError(t, rdb.XAdd(ctx, &redis.XAddArgs{ + Stream: stream.key, + Values: map[string]any{payloadKey: "missing name"}, + }).Err()) + _, err = stream.Add(ctx, "valid", []byte("payload")) + require.NoError(t, err) + + select { + case event := <-events: + require.Equal(t, "valid", event.EventName) + require.Equal(t, stream.Generation(), event.StreamGeneration) + case <-time.After(time.Second): + t.Fatal("timed out waiting for valid event after malformed entry") + } + reader.Close() + require.NoError(t, stream.Destroy(ctx)) +} + +func TestReaderClosingFencesSubscriptionsAndStreamChanges(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + stream, err := NewStream(t.Name()+"-main", rdb) + require.NoError(t, err) + added, err := NewStream(t.Name()+"-added", rdb) + require.NoError(t, err) + require.NoError(t, stream.Open(ctx)) + require.NoError(t, added.Open(ctx)) + reader, err := stream.NewReader(ctx, options.WithReaderBlockDuration(testBlockDuration)) + require.NoError(t, err) + reader.Subscribe() + + start := make(chan struct{}) + subscriptions := make(chan (<-chan *Event), 32) + results := make(chan error, 64) + var wait sync.WaitGroup + for range 32 { + wait.Add(2) + go func() { + defer wait.Done() + <-start + subscriptions <- reader.Subscribe() + }() + go func() { + defer wait.Done() + <-start + results <- reader.AddStream(ctx, added) + results <- reader.RemoveStream(ctx, added) + }() + } + close(start) + reader.Close() + wait.Wait() + close(subscriptions) + close(results) + + for result := range results { + if result != nil { + require.ErrorIs(t, result, ErrReaderClosed) + } + } + for subscription := range subscriptions { + _, ok := <-subscription + require.False(t, ok) + } + closed := reader.Subscribe() + _, ok := <-closed + require.False(t, ok) + require.ErrorIs(t, reader.AddStream(ctx, added), ErrReaderClosed) + require.ErrorIs(t, reader.RemoveStream(ctx, added), ErrReaderClosed) + require.NoError(t, stream.Destroy(ctx)) + require.NoError(t, added.Destroy(ctx)) +} + func TestNewReader(t *testing.T) { testName := strings.Replace(t.Name(), "/", "_", -1) rdb := ptesting.NewRedisClient(t) @@ -24,10 +148,12 @@ func TestNewReader(t *testing.T) { ctx := ptesting.NewTestContext(t) s, err := NewStream(testName, rdb, options.WithStreamLogger(pulse.ClueLogger(ctx))) assert.NoError(t, err) + require.NoError(t, s.Open(ctx)) + defer func() { require.NoError(t, s.Destroy(ctx)) }() reader, err := s.NewReader(ctx, options.WithReaderBlockDuration(testBlockDuration)) assert.NoError(t, err) if assert.NotNil(t, reader) { - defer cleanupReader(t, ctx, s, reader) + defer cleanupReader(t, reader) } _, err = s.NewReader(ctx, options.WithReaderTopicPattern("(")) @@ -41,9 +167,11 @@ func TestReaderReadOnce(t *testing.T) { ctx := ptesting.NewTestContext(t) s, err := NewStream(testName, rdb, options.WithStreamLogger(pulse.ClueLogger(ctx))) assert.NoError(t, err) + require.NoError(t, s.Open(ctx)) + defer func() { require.NoError(t, s.Destroy(ctx)) }() reader, err := s.NewReader(ctx, options.WithReaderStartAtOldest(), options.WithReaderBlockDuration(testBlockDuration)) require.NoError(t, err) - defer cleanupReader(t, ctx, s, reader) + defer cleanupReader(t, reader) c := reader.Subscribe() _, err = s.Add(ctx, "event", []byte("payload")) @@ -60,11 +188,13 @@ func TestReaderReadSinceLastEvent(t *testing.T) { ctx := ptesting.NewTestContext(t) s, err := NewStream(testName, rdb, options.WithStreamLogger(pulse.ClueLogger(ctx))) assert.NoError(t, err) + require.NoError(t, s.Open(ctx)) + defer func() { require.NoError(t, s.Destroy(ctx)) }() // Add and read 2 events consecutively reader, err := s.NewReader(ctx, options.WithReaderStartAtOldest(), options.WithReaderBlockDuration(testBlockDuration)) require.NoError(t, err) - defer cleanupReader(t, ctx, s, reader) + defer cleanupReader(t, reader) c := reader.Subscribe() _, err = s.Add(ctx, "event", []byte("payload")) require.NoError(t, err) @@ -81,7 +211,7 @@ func TestReaderReadSinceLastEvent(t *testing.T) { // Create new reader with last event ID set to first event and read last event reader2, err := s.NewReader(ctx, options.WithReaderStartAfter(eventID), options.WithReaderBlockDuration(testBlockDuration)) require.NoError(t, err) - defer cleanupReader(t, ctx, s, reader2) + defer cleanupReader(t, reader2) c2 := reader2.Subscribe() read = readOneReaderEvent(t, c2) assert.Equal(t, "event", read.EventName) @@ -90,7 +220,7 @@ func TestReaderReadSinceLastEvent(t *testing.T) { // Create new reader with last event ID set to 0 and read the 2 events reader3, err := s.NewReader(ctx, options.WithReaderStartAfter("0"), options.WithReaderBlockDuration(testBlockDuration)) require.NoError(t, err) - defer cleanupReader(t, ctx, s, reader3) + defer cleanupReader(t, reader3) c3 := reader3.Subscribe() read = readOneReaderEvent(t, c3) assert.Equal(t, "event", read.EventName) @@ -107,6 +237,7 @@ func TestCleanupReader(t *testing.T) { ctx := ptesting.NewTestContext(t) s, err := NewStream(testName, rdb, options.WithStreamLogger(pulse.ClueLogger(ctx))) assert.NoError(t, err) + require.NoError(t, s.Open(ctx)) reader, err := s.NewReader(ctx, options.WithReaderStartAtOldest(), options.WithReaderBlockDuration(testBlockDuration)) require.NoError(t, err) @@ -133,14 +264,15 @@ func TestReaderCloseOnFatalReadError(t *testing.T) { ctx := ptesting.NewTestContext(t) s, err := NewStream(testName, rdb, options.WithStreamLogger(pulse.ClueLogger(ctx))) require.NoError(t, err) - // Simulate a fatal read error (e.g. the underlying stream key being - // destroyed) before the read goroutine starts. The read loop reacts to a - // fatal error by closing the reader; because Close waits on the read - // goroutine, it must run asynchronously or it would deadlock and leak the - // reader and its Redis connection. - defer func(orig func(*Reader, context.Context) ([]redis.XStream, error)) { xreadFn = orig }(xreadFn) - xreadFn = func(*Reader, context.Context) ([]redis.XStream, error) { - return nil, fmt.Errorf("stream key no longer exists") + require.NoError(t, s.Open(ctx)) + defer func() { require.NoError(t, s.Destroy(ctx)) }() + // Simulate exact-generation destruction before the read goroutine starts. + // The read loop reacts by closing the reader; because Close waits on the + // read goroutine, it must run asynchronously or it would deadlock and leak + // the reader and its Redis connection. + defer func(orig func(*Reader, context.Context, []string) ([]redis.XStream, error)) { xreadFn = orig }(xreadFn) + xreadFn = func(*Reader, context.Context, []string) ([]redis.XStream, error) { + return nil, ErrStreamDestroyed } reader, err := s.NewReader(ctx, options.WithReaderBlockDuration(testBlockDuration)) @@ -151,6 +283,50 @@ func TestReaderCloseOnFatalReadError(t *testing.T) { "reader did not close after a fatal read error (Close likely deadlocked on its own read goroutine)") } +func TestReaderLifetimeDoesNotUseConstructorContext(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + setupCtx, cancel := context.WithCancel(ptesting.NewTestContext(t)) + stream, err := NewStream(t.Name(), rdb) + require.NoError(t, err) + require.NoError(t, stream.Open(setupCtx)) + reader, err := stream.NewReader( + setupCtx, + options.WithReaderStartAtOldest(), + options.WithReaderBlockDuration(testBlockDuration), + ) + require.NoError(t, err) + events := reader.Subscribe() + cancel() + + eventID, err := stream.Add(context.Background(), "event", []byte("payload")) + require.NoError(t, err) + require.Equal(t, eventID, receiveSinkEvent(t, events).ID) + reader.Close() + require.NoError(t, stream.Destroy(context.Background())) +} + +func TestReaderRejectsRemovingFinalStream(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + stream, err := NewStream(t.Name(), rdb) + require.NoError(t, err) + require.NoError(t, stream.Open(ctx)) + reader, err := stream.NewReader(ctx, options.WithReaderBlockDuration(testBlockDuration)) + require.NoError(t, err) + + unattached, err := NewStream(t.Name()+"-unattached", rdb) + require.NoError(t, err) + cancelled, cancel := context.WithCancel(ctx) + cancel() + require.NoError(t, reader.RemoveStream(cancelled, unattached)) + require.Empty(t, unattached.Generation()) + require.ErrorIs(t, reader.RemoveStream(ctx, stream), ErrLastStream) + reader.Close() + require.NoError(t, stream.Destroy(ctx)) +} + func TestAddReaderStream(t *testing.T) { testName := strings.Replace(t.Name(), "/", "_", -1) rdb := ptesting.NewRedisClient(t) @@ -158,14 +334,17 @@ func TestAddReaderStream(t *testing.T) { ctx := ptesting.NewTestContext(t) s, err := NewStream("testAddStream", rdb, options.WithStreamLogger(pulse.ClueLogger(ctx))) assert.NoError(t, err) + require.NoError(t, s.Open(ctx)) + defer func() { assert.NoError(t, s.Destroy(ctx)) }() reader, err := s.NewReader(ctx, options.WithReaderStartAtOldest(), options.WithReaderBlockDuration(testBlockDuration)) require.NoError(t, err) s2, err := NewStream("testAddStream2", rdb, options.WithStreamLogger(pulse.ClueLogger(ctx))) assert.NoError(t, err) + require.NoError(t, s2.Open(ctx)) assert.NoError(t, reader.AddStream(ctx, s2)) assert.NoError(t, reader.AddStream(ctx, s2)) // Make sure it's idempotent defer func() { assert.NoError(t, s2.Destroy(ctx)) }() - defer cleanupReader(t, ctx, s, reader) + defer cleanupReader(t, reader) // Add events to both streams c := reader.Subscribe() @@ -190,13 +369,16 @@ func TestRemoveReaderStream(t *testing.T) { ctx := ptesting.NewTestContext(t) s, err := NewStream("testRemoveStream", rdb, options.WithStreamLogger(pulse.ClueLogger(ctx))) assert.NoError(t, err) + require.NoError(t, s.Open(ctx)) + defer func() { assert.NoError(t, s.Destroy(ctx)) }() s2, err := NewStream("testRemoveStream2", rdb, options.WithStreamLogger(pulse.ClueLogger(ctx))) assert.NoError(t, err) + require.NoError(t, s2.Open(ctx)) reader, err := s.NewReader(ctx, options.WithReaderStartAtOldest(), options.WithReaderBlockDuration(testBlockDuration)) require.NoError(t, err) assert.NoError(t, reader.AddStream(ctx, s2)) defer func() { assert.NoError(t, s2.Destroy(ctx)) }() - defer cleanupReader(t, ctx, s, reader) + defer cleanupReader(t, reader) // Read events from both streams c := reader.Subscribe() @@ -227,6 +409,7 @@ func TestReaderCloseWithStalledSubscriber(t *testing.T) { ctx := ptesting.NewTestContext(t) s, err := NewStream(testName, rdb, options.WithStreamLogger(pulse.ClueLogger(ctx))) require.NoError(t, err) + require.NoError(t, s.Open(ctx)) // Tiny buffer so the read loop's fan-out send blocks after a couple of // events when the subscriber never drains its channel. @@ -267,6 +450,20 @@ func TestReaderCloseWithStalledSubscriber(t *testing.T) { require.NoError(t, s.Destroy(ctx)) } +func TestReaderRejectsSubMillisecondBlockDuration(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + stream, err := NewStream(t.Name(), rdb, options.WithStreamLogger(pulse.ClueLogger(ctx))) + require.NoError(t, err) + for _, duration := range []time.Duration{0, -time.Second, 500 * time.Microsecond} { + reader, err := stream.NewReader(ctx, options.WithReaderBlockDuration(duration)) + require.Nil(t, reader) + require.EqualError(t, err, "reader block duration must be at least 1ms") + } + require.ErrorIs(t, stream.Destroy(ctx), ErrStreamNotFound) +} + func TestEventCreatedAt(t *testing.T) { rdb := ptesting.NewRedisClient(t) defer ptesting.CleanupRedis(t, rdb, false, "") diff --git a/streaming/sink.go b/streaming/sink.go index aa59f2f..92718a5 100644 --- a/streaming/sink.go +++ b/streaming/sink.go @@ -1,10 +1,3 @@ -// This file implements Sink, the consumer-group side of a stream. A sink read -// loop XREADGROUPs events for every stream added to the sink, fans them out -// to subscribers, and settles them through the recovery acker defined in -// sink_recovery.go so the durable recovery cursor tracks exactly what was -// acknowledged. Background goroutines refresh the sink keep-alive and, under -// the fenced lease defined in sink_lease.go, claim idle messages and delete -// stale consumers left behind by dead sink instances. package streaming import ( @@ -13,7 +6,6 @@ import ( "fmt" "regexp" "sort" - "strconv" "sync" "sync/atomic" "time" @@ -27,7 +19,31 @@ import ( "goa.design/pulse/streaming/options" ) +var ( + // checkIdlePeriod is the period at which idle messages are checked. + checkIdlePeriod = 500 * time.Millisecond +) + type ( + // sinkStream is the complete coordination state owned for one attached + // stream incarnation. + sinkStream struct { + stream *Stream + startID string + consumers *rmap.Map + keepAlives *rmap.Map + leaseKey string + leaseOwner string + } + + // sinkSnapshot is the exact stream and consumer set used by one + // XREADGROUP command. + sinkSnapshot struct { + streams map[string]*sinkStream + args []string + consumer string + } + // Sink represents a stream sink. Sink struct { // Name is the sink name. @@ -36,17 +52,14 @@ type ( closed bool // consumer is the sink consumer name. consumer string - // leaseOwner identifies this sink instance in the fenced lease used - // for idle message claiming and stale consumer cleanup. - leaseOwner string // startID is the sink start event ID. startID string // noAck is true if there is no need to acknowledge events. noAck bool // lock is the sink mutex. lock sync.Mutex - // streams are the streams the sink consumes events from, indexed by - // stream Redis key. + // streams is the per-incarnation coordination state indexed by physical + // stream key. streams map[string]*sinkStream // blockDuration is the XREADGROUP timeout. blockDuration time.Duration @@ -59,64 +72,54 @@ type ( chans []chan *Event // donechan is the sink done channel. donechan chan struct{} + // ctx is canceled when Close begins and bounds all sink-owned Redis I/O. + ctx context.Context + // cancel interrupts sink-owned Redis I/O. + cancel context.CancelFunc // wait is the sink cleanup wait group. wait sync.WaitGroup - // closeOnce is used to ensure the sink is closed only once. - closeOnce sync.Once - // closing is set when Close starts so loops stop scheduling work. + // stopOnce cancels background work exactly once; distributed cleanup is + // intentionally retried until it succeeds. + stopOnce sync.Once + // closing is true if Close was called. closing atomic.Bool - // ctx is canceled by Close to abort all sink-owned Redis I/O, - // including blocked XREADGROUP calls and recovery in progress. - ctx context.Context - // cancel cancels ctx. - cancel context.CancelFunc // eventFilter is the event filter if any. eventFilter eventFilterFunc - // consumersKeepAliveMap records consumer keep-alives for this - // sink (i.e. for all in-process instances of the sink). - consumersKeepAliveMap *rmap.Map + // filterKind and filterValue preserve the exact shared configuration + // applied to every later AddStream attachment. + filterKind string + filterValue string // ackGracePeriod is the grace period after which an event is // considered unacknowledged. ackGracePeriod time.Duration - // lastKeepAlive is the last keep-alive timestamp for this consumer - // in Redis-time nanoseconds. + // lastKeepAlive is the last keep-alive timestamp for this consumer. lastKeepAlive int64 - // acker settles events and advances the durable recovery cursor. - acker *recoveryAcker // logger is the logger used by the sink. logger pulse.Logger // rdb is the redis connection. rdb *redis.Client } - // sinkStream is the sink-side state for one consumed stream: the stream - // handle, the start ID used when (re)creating the consumer group, and the - // replicated membership map listing the consumers of each sink. - sinkStream struct { - // stream is the consumed stream. - stream *Stream - // startID is the group start position for brand new groups. - startID string - // consumers is the stream membership map (sink name to consumer - // names), joined for reads and change notifications; all writes go - // through the fenced scripts in sink_recovery.go. - consumers *rmap.Map - } - // eventFilterFunc is the function used to filter events. eventFilterFunc func(*Event) bool ) -// checkIdlePeriod is the period at which idle messages are checked. -var checkIdlePeriod = 500 * time.Millisecond +var ( + // ErrSinkClosed is returned when stream ownership is changed after sink + // shutdown begins. + ErrSinkClosed = errors.New("pulse streaming: sink is closed") +) -// newSink creates a new sink. -// Sinks use one Redis consumer per stream they are consuming from. -// Pulse maintains a pool of consumers per stream and reuses them when possible. -// This is because deleting a consumer causes Redis to drop its pending messages -// which is not the semantics Pulse wants to enforce. +// newSink creates a sink whose consumer identity spans every attached stream. +// Each stream generation owns independent membership, keepalive, recovery, and +// PEL state for that identity. Detachment deletes the Redis consumer only when +// that consumer's own PEL is empty, because deletion would otherwise discard +// pending ownership. func newSink(ctx context.Context, name string, stream *Stream, opts ...options.Sink) (*Sink, error) { o := options.ParseSinkOptions(opts...) + if err := validateSinkOptions(o); err != nil { + return nil, err + } var eventMatcher eventFilterFunc if o.Topic != "" { eventMatcher = func(e *Event) bool { return e.Topic == o.Topic } @@ -127,55 +130,51 @@ func newSink(ctx context.Context, name string, stream *Stream, opts ...options.S } eventMatcher = func(e *Event) bool { return topicPatternRegexp.MatchString(e.Topic) } } - + if err := stream.ensureGeneration(ctx); err != nil { + return nil, err + } logger := stream.rootLogger.WithPrefix("sink", name) - km, err := rmap.Join(ctx, sinkKeepAliveMapName(name), stream.rdb, rmap.WithLogger(logger)) + config := sinkConfigurationForOptions(o, o.LastEventID) + state, err := attachSinkStream(ctx, stream, name, config, logger) if err != nil { - return nil, fmt.Errorf("failed to join replicated map for sink keep-alives %s: %w", name, err) + return nil, err } - // runCtx outlives the caller context and is canceled by Close so all - // sink-owned Redis I/O, including recovery in progress, stops promptly. - runCtx, cancel := context.WithCancel(log.WithContext(context.Background(), ctx)) - + // Preserve the caller's logger while giving all background Redis commands + // a context Close can cancel. + logCtx := context.Background() + logCtx = log.WithContext(logCtx, ctx) + runCtx, cancel := context.WithCancel(logCtx) sink := &Sink{ - Name: name, - leaseOwner: ulid.Make().String(), - startID: o.LastEventID, - noAck: o.NoAck, - streams: make(map[string]*sinkStream, 1), - blockDuration: o.BlockDuration, - maxPolled: o.MaxPolled, - bufferSize: o.BufferSize, - donechan: make(chan struct{}), - ctx: runCtx, - cancel: cancel, - eventFilter: eventMatcher, - consumersKeepAliveMap: km, - ackGracePeriod: o.AckGracePeriod, - acker: &recoveryAcker{rdb: stream.rdb}, - logger: logger, - rdb: stream.rdb, - } - - state, err := sink.attachStream(ctx, stream, o.LastEventID) - if err != nil { - cancel() - km.Close() - return nil, err + Name: name, + startID: o.LastEventID, + noAck: o.NoAck, + streams: map[string]*sinkStream{stream.key: state}, + blockDuration: o.BlockDuration, + maxPolled: o.MaxPolled, + bufferSize: o.BufferSize, + donechan: make(chan struct{}), + ctx: runCtx, + cancel: cancel, + eventFilter: eventMatcher, + filterKind: config.filterKind, + filterValue: config.filterValue, + ackGracePeriod: o.AckGracePeriod, + logger: logger, + rdb: stream.rdb, + } + + // Clean up existing stale consumers under this stream's fenced lease before + // creating our own. + if err := sink.deleteStreamStaleConsumersWithLease(ctx, state); err != nil { + sink.logger.Error(fmt.Errorf("failed to cleanup stale consumers: %w", err)) } - sink.streams[stream.key] = state consumer, err := sink.newConsumer(ctx) if err != nil { - // Compensate the group and cursor created by attachStream (the group - // survives only when other sink instances are members). - if cerr := removeSinkStream(ctx, stream, name, ""); cerr != nil { - err = errors.Join(err, cerr) - } cancel() - state.consumers.Close() - km.Close() + closeSetupMembership(ctx, stream, state.consumers, err) + state.keepAlives.Close() return nil, fmt.Errorf("failed to create consumer: %w", err) } sink.consumer = consumer @@ -191,11 +190,16 @@ func newSink(ctx context.Context, name string, stream *Stream, opts ...options.S return sink, nil } -// Subscribe returns a channel that receives events from the sink. +// Subscribe returns a channel that receives events from the sink. Calls made +// after shutdown starts return an already closed channel. func (s *Sink) Subscribe() <-chan *Event { c := make(chan *Event, s.bufferSize) s.lock.Lock() defer s.lock.Unlock() + if s.closing.Load() { + close(c) + return c + } s.chans = append(s.chans, c) return c } @@ -213,10 +217,9 @@ func (s *Sink) Unsubscribe(c <-chan *Event) { } } -// Ack acknowledges the event and advances the sink recovery cursor. +// Ack acknowledges the event. func (s *Sink) Ack(ctx context.Context, e *Event) error { - err := e.Acker.XAck(ctx, e.streamKey, e.SinkName, e.ID).Err() - if err != nil { + if err := e.Acker.XAck(ctx, e.streamKey, e.SinkName, e.ID).Err(); err != nil { s.logger.Error(err, "ack", e.ID, "stream", e.StreamName) return err } @@ -226,34 +229,48 @@ func (s *Sink) Ack(ctx context.Context, e *Event) error { // AddStream adds the stream to the sink. By default the stream cursor starts at // the same timestamp as the sink main stream cursor. This can be overridden -// with opts. AddStream does nothing if the stream is already part of the sink. -// It returns ErrSinkClosed after Close. +// with opts. AddStream does nothing if the stream is already part of the sink +// and returns ErrSinkClosed once shutdown starts. func (s *Sink) AddStream(ctx context.Context, stream *Stream, opts ...options.AddStream) error { s.lock.Lock() defer s.lock.Unlock() if s.closing.Load() { return ErrSinkClosed } - if _, ok := s.streams[stream.key]; ok { - return nil + addOptions := options.ParseAddStreamOptions(opts...) + if err := validateAddStreamOptions(addOptions); err != nil { + return err + } + if err := stream.verifyGeneration(ctx); err != nil { + return err + } + for _, state := range s.streams { + if state.stream.Name != stream.Name { + continue + } + if state.stream.generation == stream.generation { + return nil + } + return state.stream.verifyGeneration(ctx) } startID := s.startID - o := options.ParseAddStreamOptions(opts...) - if o.LastEventID != "" { - startID = o.LastEventID + if addOptions.LastEventID != "" { + startID = addOptions.LastEventID + } + config := sinkConfiguration{ + filterKind: s.filterKind, + filterValue: s.filterValue, + startID: startID, + noAck: s.noAck, + ackGrace: s.ackGracePeriod, } - state, err := s.attachStream(ctx, stream, startID) + state, err := attachSinkStream(ctx, stream, s.Name, config, s.logger) if err != nil { return err } - if err := registerSinkConsumer(ctx, stream, s.Name, s.consumer); err != nil { - // Compensate the group and cursor created by attachStream so a failed - // AddStream leaves no dangling ownership state (the group survives - // only when other sink instances are members). - if cerr := removeSinkStream(ctx, stream, s.Name, s.consumer); cerr != nil { - err = errors.Join(err, cerr) - } - state.consumers.Close() + if err := registerSinkConsumer(ctx, state, s.Name, s.consumer, s.lastKeepAlive); err != nil { + closeSetupMembership(ctx, stream, state.consumers, err) + state.keepAlives.Close() return err } s.streams[stream.key] = state @@ -261,57 +278,103 @@ func (s *Sink) AddStream(ctx context.Context, stream *Stream, opts ...options.Ad return nil } -// RemoveStream removes the stream from the sink, it is idempotent. The -// distributed effects (membership removal and, for the last member, consumer -// group, recovery cursor, and lease deletion) execute in one atomic script so -// there is no partial state to compensate. It returns ErrSinkClosed after -// Close. +// RemoveStream removes the stream from the sink. It is idempotent and returns +// ErrSinkClosed once shutdown starts. Removing the final stream returns +// ErrLastStream. func (s *Sink) RemoveStream(ctx context.Context, stream *Stream) error { s.lock.Lock() defer s.lock.Unlock() if s.closing.Load() { return ErrSinkClosed } - state, ok := s.streams[stream.key] - if !ok { + var state *sinkStream + for _, candidate := range s.streams { + if candidate.stream == stream || + (stream.generation != "" && + candidate.stream.Name == stream.Name && + candidate.stream.generation == stream.generation) { + state = candidate + break + } + } + if state == nil { return nil } - if err := removeSinkStream(ctx, state.stream, s.Name, s.consumer); err != nil { + if len(s.streams) == 1 { + return ErrLastStream + } + if err := state.stream.verifyGeneration(ctx); err != nil { return err } - delete(s.streams, stream.key) - state.consumers.Close() + if _, err := detachSinkConsumer(ctx, state, s.Name, s.consumer); err != nil { + return fmt.Errorf("failed to detach consumer %s from stream %s: %w", s.consumer, stream.Name, err) + } + delete(s.streams, state.stream.key) + state.close() s.logger.Info("removed", "stream", stream.Name) return nil } -// Close stops event polling, cancels all sink-owned Redis I/O (including any -// recovery in progress), waits for the sink goroutines to stop, and closes -// the sink channels. It is safe to call Close multiple times; concurrent -// callers block until the first Close completes. -func (s *Sink) Close(ctx context.Context) { - s.closeOnce.Do(func() { - // Signal shutdown without holding the lock so the read loop stops - // even when it is parked on a fan-out send to a stalled subscriber - // (which holds the lock) or blocked in a Redis call. +// Close stops event polling and detaches every distributed membership. Failed +// Redis cleanup is returned and may be retried with another context; the sink +// is closed only after all membership and keep-alive side effects complete. +func (s *Sink) Close(ctx context.Context) error { + s.stopOnce.Do(func() { s.closing.Store(true) s.cancel() + // Close donechan first, without holding the lock, so the signal + // reaches the read loop even when it is parked on a fan-out send + // to a stalled subscriber (which holds the lock). Otherwise Close + // would deadlock acquiring the lock the read loop never releases. close(s.donechan) - s.wait.Wait() - s.lock.Lock() - defer s.lock.Unlock() - for _, c := range s.chans { - close(c) + }) + s.wait.Wait() + + s.lock.Lock() + defer s.lock.Unlock() + if s.closed { + return nil + } + var cleanupErr error + for streamKey, state := range s.streams { + stream := state.stream + err := stream.verifyGeneration(ctx) + if errors.Is(err, ErrStreamDestroyed) || errors.Is(err, ErrDeadlineElapsed) { + state.close() + delete(s.streams, streamKey) + continue } - // Note: we do not delete the consumer from the keep-alive and consumer maps - // so that another instance may claim any pending messages. - s.consumersKeepAliveMap.Close() - for _, state := range s.streams { - state.consumers.Close() + if err != nil { + cleanupErr = errors.Join(cleanupErr, fmt.Errorf( + "failed to verify stream %s before detaching consumer %s: %w", + streamKey, + s.consumer, + err, + )) + continue } - s.closed = true - s.logger.Info("closed") - }) + if _, err := detachSinkConsumer(ctx, state, s.Name, s.consumer); err != nil { + cleanupErr = errors.Join(cleanupErr, fmt.Errorf( + "failed to detach consumer %s from stream %s: %w", + s.consumer, + streamKey, + err, + )) + continue + } + state.close() + delete(s.streams, streamKey) + } + if cleanupErr != nil { + return cleanupErr + } + for _, c := range s.chans { + close(c) + } + s.chans = nil + s.closed = true + s.logger.Info("closed") + return nil } // IsClosed returns true if the sink was closed. @@ -321,211 +384,296 @@ func (s *Sink) IsClosed() bool { return s.closed } -// attachStream ensures the consumer group and recovery cursor exist for the -// stream (restoring the stream TTL even on BUSYGROUP) and joins the stream -// membership map. Callers own registering the sink consumer. -func (s *Sink) attachStream(ctx context.Context, stream *Stream, startID string) (*sinkStream, error) { - if _, _, err := ensureConsumerGroup(ctx, stream, s.Name, startID, true); err != nil { - return nil, err +// deleteStreamStaleConsumersWithLease deletes stale consumers for one attached +// stream only while this replica owns its fenced recovery lease. +func (s *Sink) deleteStreamStaleConsumersWithLease(ctx context.Context, state *sinkStream) error { + duration := 2 * checkIdlePeriod + lease, acquired, err := acquireSinkRecoveryLease( + ctx, + state.stream, + state.leaseKey, + state.leaseOwner, + duration, + ) + if err != nil || !acquired { + return err + } + return s.deleteStreamStaleConsumers(ctx, state, lease, duration) +} + +// deleteStreamStaleConsumers deletes stale consumers for a specific stream in +// one lease-fenced Redis operation. +// s.lock must be held once the sink is running. +func (s *Sink) deleteStreamStaleConsumers( + ctx context.Context, + state *sinkStream, + lease sinkRecoveryLease, + leaseDuration time.Duration, +) error { + removed, malformed, err := cleanupStaleConsumers( + ctx, + state.stream, + state.leaseKey, + lease, + leaseDuration, + s.Name, + s.ackGracePeriod, + ) + if err != nil { + return err + } + for _, consumer := range removed { + s.logger.Info("cleaned up stale consumer", "consumer", consumer) } - cm, err := rmap.Join(ctx, consumersMapName(stream), stream.rdb, consumersMapOptions(stream, s.logger)...) + for _, consumer := range malformed { + s.logger.Error( + fmt.Errorf("invalid keep-alive timestamp"), + "stream", state.stream.Name, + "consumer", consumer, + ) + } + return nil +} + +// rollbackStreamRegistration detaches membership added by an AddStream that +// failed before local ownership was published. Empty Redis consumer metadata, +// membership, and keepalive state are removed atomically; shared group and +// cursor state remain owned by Stream.Destroy. +func (s *Sink) rollbackStreamRegistration(ctx context.Context, state *sinkStream) error { + _, err := detachSinkConsumer(ctx, state, s.Name, s.consumer) if err != nil { - return nil, fmt.Errorf("failed to join replicated map for stream %s: %w", stream.Name, err) + return fmt.Errorf( + "failed to roll back consumer %s for stream %s: %w", + s.consumer, + state.stream.Name, + err, + ) } - return &sinkStream{stream: stream, startID: startID, consumers: cm}, nil + return nil } -// read reads events from the streams and sends them to the sink channels. -// NOGROUP errors trigger lossless consumer group recovery; transient Redis -// failures are retried with jittered exponential backoff. +// newConsumer creates one replacement consumer across every owned stream. No +// stream observes local ownership unless all Redis consumers, memberships, and +// the shared keep-alive are established; failures roll back the full prefix. +func (s *Sink) newConsumer(ctx context.Context) (string, error) { + consumer := ulid.Make().String() + now, err := s.rdb.Time(ctx).Result() + if err != nil { + return "", fmt.Errorf("failed to read Redis time for new consumer %s: %w", consumer, err) + } + keepAlive := now.UnixNano() + registered := make([]*sinkStream, 0, len(s.streams)) + for _, state := range s.streams { + registered = append(registered, state) + if err := registerSinkConsumer(ctx, state, s.Name, consumer, keepAlive); err != nil { + return "", errors.Join(err, s.rollbackConsumer(ctx, consumer, registered)) + } + } + s.lastKeepAlive = keepAlive + return consumer, nil +} + +// rollbackConsumer removes a replacement consumer from every stream that was +// registered before consumer creation failed. +func (s *Sink) rollbackConsumer(ctx context.Context, consumer string, states []*sinkStream) error { + var rollbackErr error + for i := len(states) - 1; i >= 0; i-- { + state := states[i] + stream := state.stream + if _, err := detachSinkConsumer(ctx, state, s.Name, consumer); err != nil { + rollbackErr = errors.Join(rollbackErr, fmt.Errorf( + "failed to roll back consumer %s from stream %s: %w", + consumer, + stream.Name, + err, + )) + } + } + return rollbackErr +} + +// read reads events from the streams and sends them to the sink channel. func (s *Sink) read() { defer s.logger.Debug("read: exiting") defer s.wait.Done() var retry readRetry for { if err := s.ensureConsumer(s.ctx); err != nil { - if s.closing.Load() { - return - } if !retry.wait(s.donechan, err, s.logger) { return } continue } - args, consumer := s.readArgs() - if len(args) == 0 { - // No streams to read from; wait for AddStream or Close. - select { - case <-s.donechan: + snapshot, err := s.readSnapshot(s.ctx) + if err != nil { + if fatal := fatalReadError(err); fatal != nil { + pulse.Go(s.logger, func() { + if closeErr := s.Close(context.WithoutCancel(s.ctx)); closeErr != nil { + s.logger.Error(fmt.Errorf("failed to close terminal sink: %w", closeErr)) + } + }) + return + } + if !retry.wait(s.donechan, err, s.logger) { return - case <-time.After(s.blockDuration): - continue } + continue } - s.logger.Debug("reading", "streams", args, "max", s.maxPolled, "block", s.blockDuration) + + s.logger.Debug("reading", "streams", snapshot.args, "max", s.maxPolled, "block", s.blockDuration) streams, err := s.rdb.XReadGroup(s.ctx, &redis.XReadGroupArgs{ Group: s.Name, - Consumer: consumer, - Streams: args, + Consumer: snapshot.consumer, + Streams: snapshot.args, Count: s.maxPolled, Block: s.blockDuration, - NoAck: s.noAck, }).Result() + if s.closing.Load() { // Honor the Close contract and do not forward any more events. // Any events in the PEL will be claimed by another consumer. return } if err == nil { - err = s.dispatch(streams) - } - if err == nil || err == redis.Nil { - retry.reset() - continue + for _, events := range streams { + owned, ok := snapshot.streams[events.Stream] + if !ok { + continue + } + stream := owned.stream + if verifyErr := stream.verifyGeneration(s.ctx); verifyErr != nil { + err = verifyErr + break + } + s.lock.Lock() + current := s.ownsStream(stream) + if !current { + s.lock.Unlock() + if ackErr := acknowledgeMessages(s.ctx, stream, s.Name, events.Messages); ackErr != nil { + err = ackErr + break + } + continue + } + err = streamEvents( + s.ctx, + stream, + s.Name, + events.Messages, + s.noAck, + s.eventFilter, + s.chans, + s.donechan, + s.rdb, + s.logger, + ) + s.lock.Unlock() + if err != nil { + break + } + } } - if redis.HasErrorPrefix(err, "NOGROUP") { - if err := s.recoverConsumerGroups(s.ctx); err == nil { + if err != nil { + if redis.HasErrorPrefix(err, "NOGROUP") { + s.lock.Lock() + err = s.recoverConsumerGroups(s.ctx) + s.lock.Unlock() + } + fatal := fatalReadError(err) + if fatal != nil { + s.logger.Error(fmt.Errorf("fatal error while reading events: %w, stopping", fatal)) + pulse.Go(s.logger, func() { + if err := s.Close(context.WithoutCancel(s.ctx)); err != nil { + s.logger.Error(fmt.Errorf("failed to close terminal sink: %w", err)) + } + }) + return + } + if err == nil || err == redis.Nil { retry.reset() continue - } else if s.closing.Load() { - return - } else if !retry.wait(s.donechan, err, s.logger) { + } + if !retry.wait(s.donechan, err, s.logger) { return } continue } - if !retry.wait(s.donechan, err, s.logger) { - return - } - } -} - -// dispatch fans out one XREADGROUP reply to the subscribers, settling events -// through the recovery acker. Batches for streams removed from the sink -// concurrently with the read are left pending, never acknowledged: if other -// sink instances remain in the group, the fenced idle-claim redelivers the -// entries to one of them (stale-consumer cleanup skips consumers with pending -// events), and if this was the last member the group was already destroyed so -// the pending entries are gone with it. -func (s *Sink) dispatch(streams []redis.XStream) error { - for _, events := range streams { - s.lock.Lock() - state, owned := s.streams[events.Stream] - if !owned { - s.lock.Unlock() - continue - } - err := streamEvents(s.ctx, state.stream.Name, state.stream.key, s.Name, events.Messages, s.acker, s.noAck, s.eventFilter, s.chans, s.donechan, s.logger) - s.lock.Unlock() - if err != nil { - return err - } + retry.reset() } - return nil } -// recoverConsumerGroups recreates missing consumer groups at the durable -// recovery cursor after Redis loses group state (e.g. XGROUP DESTROY). A -// stream that was destroyed with Stream.Destroy is dropped from the sink -// instead of being resurrected. -func (s *Sink) recoverConsumerGroups(ctx context.Context) error { +// readSnapshot verifies and captures the exact stream capabilities and +// consumer used by one grouped read. +func (s *Sink) readSnapshot(ctx context.Context) (sinkSnapshot, error) { s.lock.Lock() defer s.lock.Unlock() - for key, state := range s.streams { - created, cursor, err := ensureConsumerGroup(ctx, state.stream, s.Name, state.startID, false) - if err != nil { - if errors.Is(err, ErrStreamDestroyed) { - s.logger.Info("stream destroyed, dropping from sink", "stream", state.stream.Name) - delete(s.streams, key) - state.consumers.Close() - continue - } - return err - } - if created { - s.logger.Info("recovered consumer group", "stream", state.stream.Name, "cursor", cursor) - } + snapshot := sinkSnapshot{ + streams: make(map[string]*sinkStream, len(s.streams)), + args: make([]string, len(s.streams)*2), + consumer: s.consumer, } - return nil -} - -// readArgs snapshots the XREADGROUP stream arguments and the current consumer -// under the sink lock. -func (s *Sink) readArgs() ([]string, string) { - s.lock.Lock() - defer s.lock.Unlock() keys := make([]string, 0, len(s.streams)) for key := range s.streams { keys = append(keys, key) } sort.Strings(keys) - args := make([]string, 2*len(keys)) for i, key := range keys { - args[i] = key - args[len(keys)+i] = ">" + state := s.streams[key] + if err := state.stream.verifyGeneration(ctx); err != nil { + return sinkSnapshot{}, err + } + snapshot.streams[key] = state + snapshot.args[i] = key + snapshot.args[len(keys)+i] = ">" } - return args, s.consumer + return snapshot, nil } -// ensureConsumer rotates the sink consumer when its keep-alive went stale, -// e.g. after this instance was partitioned long enough for its consumer to be -// cleaned up by a replica. Staleness is evaluated against Redis time so -// client clocks do not skew the decision. -func (s *Sink) ensureConsumer(ctx context.Context) error { - s.lock.Lock() - defer s.lock.Unlock() - now, err := s.rdb.Time(ctx).Result() - if err != nil { - return fmt.Errorf("failed to read Redis time: %w", err) +// ownsStream reports whether the sink still owns the exact capability used by +// an in-flight read. +func (s *Sink) ownsStream(candidate *Stream) bool { + for _, state := range s.streams { + if state.stream == candidate { + return true + } } - if now.Sub(time.Unix(0, s.lastKeepAlive)) <= 2*s.ackGracePeriod { + return false +} + +// acknowledgeMessages atomically advances recovery for messages returned by a +// read whose stream was concurrently removed. +func acknowledgeMessages(ctx context.Context, stream *Stream, group string, messages []redis.XMessage) error { + if len(messages) == 0 { return nil } - s.logger.Debug("consumer stale, creating new one") - consumer, err := s.newConsumer(ctx) - if err != nil { - return fmt.Errorf("failed to create new consumer: %w", err) + ids := make([]string, len(messages)) + for i, message := range messages { + ids[i] = message.ID } - s.consumer = consumer - return nil + return (&recoveryAcker{stream: stream}).XAck(ctx, stream.key, group, ids...).Err() } -// newConsumer creates a new consumer, registers it with every sink stream, -// and records its keep-alive. Registration is failure-atomic: when any -// registration fails the consumer is detached from the streams registered so -// far so ownership state never diverges across streams. s.lock must be held. -func (s *Sink) newConsumer(ctx context.Context) (string, error) { - consumer := ulid.Make().String() - registered := make([]*sinkStream, 0, len(s.streams)) - rollback := func(cause error) error { - var errs []error - for _, state := range registered { - if err := detachSinkConsumer(ctx, state.stream, s.Name, consumer); err != nil { - errs = append(errs, err) - } - } - return errors.Join(append([]error{cause}, errs...)...) - } - for _, state := range s.streams { - if err := registerSinkConsumer(ctx, state.stream, s.Name, consumer); err != nil { - return "", rollback(err) - } - registered = append(registered, state) - } +// ensureConsumer ensures that the consumer is still alive. +func (s *Sink) ensureConsumer(ctx context.Context) error { + s.lock.Lock() + defer s.lock.Unlock() now, err := s.rdb.Time(ctx).Result() if err != nil { - return "", rollback(fmt.Errorf("failed to read Redis time for new consumer %s: %w", consumer, err)) + return fmt.Errorf("failed to read Redis time while checking consumer: %w", err) } - keepAlive := now.UnixNano() - if _, err := s.consumersKeepAliveMap.Set(ctx, consumer, strconv.FormatInt(keepAlive, 10)); err != nil { - return "", rollback(fmt.Errorf("failed to set sink keep-alive for new consumer %s: %w", consumer, err)) + if now.Sub(time.Unix(0, s.lastKeepAlive)) > 2*s.ackGracePeriod { + s.logger.Debug("consumer stale, creating new one") + consumer, err := s.newConsumer(ctx) + if err != nil { + s.logger.Error(fmt.Errorf("failed to create new consumer: %w", err)) + return err + } + s.consumer = consumer } - s.lastKeepAlive = keepAlive - return consumer, nil + return nil } -// periodicKeepAlive updates this consumer keep-alive every ack grace period -// using Redis time so replicas evaluating staleness agree on the clock. +// periodicKeepAlive updates this consumer keep-alive every half ack grace period. func (s *Sink) periodicKeepAlive() { defer s.wait.Done() defer s.logger.Debug("periodicKeepAlive: exiting") @@ -538,17 +686,34 @@ func (s *Sink) periodicKeepAlive() { s.lock.Lock() now, err := s.rdb.Time(s.ctx).Result() if err != nil { - s.logger.Error(fmt.Errorf("failed to read Redis time for keep-alive: %w", err)) + s.logger.Error(fmt.Errorf("failed to read Redis time for sink keep-alive: %w", err)) s.lock.Unlock() continue } keepAlive := now.UnixNano() - if _, err := s.consumersKeepAliveMap.Set(s.ctx, s.consumer, strconv.FormatInt(keepAlive, 10)); err != nil { - s.logger.Error(fmt.Errorf("failed to update sink keep-alive: %w", err)) + var updateErr error + for _, state := range s.streams { + err := setSinkKeepAlive(s.ctx, state, s.Name, s.consumer, keepAlive) + switch { + case err == nil: + case errors.Is(err, ErrStreamDestroyed), errors.Is(err, ErrDeadlineElapsed): + // The generation ended; the read loop owns detaching the + // stream. Refreshing nothing is correct here. + s.logger.Debug("keep-alive skipped", "stream", state.stream.Name, "reason", err) + default: + updateErr = errors.Join(updateErr, fmt.Errorf( + "stream %s: %w", + state.stream.Name, + err, + )) + } + } + if updateErr != nil { + s.logger.Error(fmt.Errorf("failed to update sink keep-alive: %w", updateErr)) s.lock.Unlock() continue } - s.lastKeepAlive = keepAlive + s.lastKeepAlive = now.UnixNano() s.lock.Unlock() case <-s.donechan: @@ -557,23 +722,54 @@ func (s *Sink) periodicKeepAlive() { } } -// periodicIdleMessageCheck claims idle messages and deletes stale consumers -// under the per-stream fenced lease. An idle message is one that has not been -// acked for more than the ack grace period. Lease renewal and each guarded -// mutation execute in one atomic script so a stale owner cannot mutate the -// PEL after another instance takes over. +// periodicIdleMessageCheck claims any idle message every check stale period. +// An idle message is one that has not been acked for more than the ack grace period. +// Once all idle messages are claimed, any stale consumer is deleted. func (s *Sink) periodicIdleMessageCheck() { defer s.wait.Done() defer s.logger.Debug("periodicIdleMessageCheck: exiting") ticker := time.NewTicker(checkIdlePeriod) defer ticker.Stop() + leaseDuration := 2 * checkIdlePeriod for { select { case <-ticker.C: s.lock.Lock() for _, state := range s.streams { - s.checkIdleMessages(s.ctx, state) + lease, acquired, err := acquireSinkRecoveryLease( + s.ctx, + state.stream, + state.leaseKey, + state.leaseOwner, + leaseDuration, + ) + if err != nil { + s.logger.Error(fmt.Errorf( + "failed to acquire stale-recovery lease for stream %s: %w", + state.stream.Name, + err, + )) + continue + } + if !acquired { + continue + } + if err := s.claimIdleMessages(s.ctx, state, lease, leaseDuration); err != nil { + s.logger.Error(fmt.Errorf( + "failed to claim idle messages for stream %s: %w", + state.stream.Name, + err, + )) + continue + } + if err := s.deleteStreamStaleConsumers(s.ctx, state, lease, leaseDuration); err != nil { + s.logger.Error(fmt.Errorf( + "failed to delete stale consumers for stream %s: %w", + state.stream.Name, + err, + )) + } } s.lock.Unlock() @@ -583,82 +779,160 @@ func (s *Sink) periodicIdleMessageCheck() { } } -// checkIdleMessages acquires the stream lease and, when held, claims idle -// messages for this consumer and deletes stale consumers. Lease loss and -// stream destruction abort silently: another instance owns the work or the -// stream is gone. s.lock must be held. -func (s *Sink) checkIdleMessages(ctx context.Context, state *sinkStream) { - // Note: the builtin max is shadowed by the package-level test helper - // variable of the same name, hence the explicit floor. - leaseMs := 2 * checkIdlePeriod.Milliseconds() - if leaseMs < 20 { - leaseMs = 20 - } - acquired, fence, err := acquireSinkLease(ctx, state.stream, s.Name, s.leaseOwner, leaseMs) +// claimIdleMessages claims idle messages from one stream through an atomic +// exact-lease fence around every XAUTOCLAIM. +// s.lock must be held. +func (s *Sink) claimIdleMessages( + ctx context.Context, + state *sinkStream, + lease sinkRecoveryLease, + leaseDuration time.Duration, +) error { + start, err := s.claim(ctx, state, lease, leaseDuration, "0-0") if err != nil { - if !errors.Is(err, ErrStreamDestroyed) && ctx.Err() == nil { - s.logger.Error(fmt.Errorf("failed to acquire idle message check lease: %w", err)) - } - return - } - if !acquired { - // Another sink instance owns the lease. - return + return err } - start := "0-0" - for { - msgs, next, err := fencedAutoClaim(ctx, state.stream, s.Name, s.leaseOwner, fence, leaseMs, s.consumer, s.ackGracePeriod.Milliseconds(), start, s.maxPolled) + for start != "0-0" { + start, err = s.claim(ctx, state, lease, leaseDuration, start) if err != nil { - if !isLeaseLostErr(err) && !isStreamDestroyedErr(err) && ctx.Err() == nil { - s.logger.Error(fmt.Errorf("failed to claim idle messages for stream %s: %w", state.stream.Name, err)) - } - return + return err } - if len(msgs) > 0 { - s.logger.Info("claimed", "stream", state.stream.Name, "messages", len(msgs)) - if err := streamEvents(ctx, state.stream.Name, state.stream.key, s.Name, msgs, s.acker, s.noAck, s.eventFilter, s.chans, s.donechan, s.logger); err != nil { - s.logger.Error(fmt.Errorf("failed to stream claimed events: %w", err)) - return - } + } + return nil +} + +// recoverConsumerGroups recreates groups removed outside Pulse for the streams +// the sink still owns. The caller holds s.lock, which serializes recovery with +// AddStream and RemoveStream so an in-flight stale read cannot resurrect a +// removed stream. BUSYGROUP means another sink instance already repaired the +// shared group and is therefore success. +func (s *Sink) recoverConsumerGroups(ctx context.Context) error { + for _, state := range s.streams { + stream := state.stream + deleted, err := ensureConsumerGroup(ctx, stream, s.Name, state.startID) + if err != nil { + return err } - if next == "0-0" { - break + if deleted { + s.logger.Error( + fmt.Errorf("stream data was deleted before consumer group recovery"), + "stream", stream.Name, + "group", s.Name, + "data_loss", true, + ) } - start = next + s.logger.Info("recovered consumer group", "stream", stream.Name, "start", state.startID) } - staleNs := (2 * s.ackGracePeriod).Nanoseconds() - removed, err := fencedCleanupStaleConsumers(ctx, state.stream, s.Name, s.leaseOwner, fence, leaseMs, staleNs, s.consumer) + return nil +} + +// Helper function to claim messages from a stream used by claimIdleMessages. +func (s *Sink) claim( + ctx context.Context, + state *sinkStream, + lease sinkRecoveryLease, + leaseDuration time.Duration, + start string, +) (string, error) { + next, messages, err := fencedAutoClaim( + ctx, + state.stream, + state.leaseKey, + lease, + leaseDuration, + s.Name, + s.consumer, + s.ackGracePeriod, + start, + s.maxPolled, + ) if err != nil { - if !isLeaseLostErr(err) && !isStreamDestroyedErr(err) && ctx.Err() == nil { - s.logger.Error(fmt.Errorf("failed to delete stale consumers for stream %s: %w", state.stream.Name, err)) - } - return + return start, err + } + if len(messages) > 0 { + s.logger.Info("claimed", "stream", state.stream.Name, "messages", len(messages)) + err = streamEvents( + ctx, + state.stream, + s.Name, + messages, + s.noAck, + s.eventFilter, + s.chans, + s.donechan, + s.rdb, + s.logger, + ) + } + return next, err +} + +// attachSinkStream validates the shared contract before joining any group +// membership, then builds the complete per-stream coordination state. +func attachSinkStream( + ctx context.Context, + stream *Stream, + name string, + config sinkConfiguration, + logger pulse.Logger, +) (*sinkStream, error) { + if err := ensureSinkConfiguration(ctx, stream, name, config); err != nil { + return nil, err } - if len(removed) > 0 { - s.logger.Info("deleted stale consumers", "stream", state.stream.Name, "consumers", removed) + consumers, err := rmap.Join( + ctx, + consumersMapName(stream), + stream.rdb, + rmap.WithLogger(logger), + ) + if err != nil { + return nil, fmt.Errorf("failed to join replicated map for sink %s: %w", name, err) } + keepAlives, err := rmap.Join( + ctx, + sinkKeepAliveMapName(stream, name), + stream.rdb, + rmap.WithLogger(logger), + ) + if err != nil { + consumers.Close() + return nil, fmt.Errorf("failed to join replicated map for sink keep-alives %s: %w", name, err) + } + if _, err := ensureConsumerGroup(ctx, stream, name, config.startID); err != nil { + closeSetupMembership(ctx, stream, consumers, err) + keepAlives.Close() + return nil, err + } + return &sinkStream{ + stream: stream, + startID: config.startID, + consumers: consumers, + keepAlives: keepAlives, + leaseKey: staleLockName(stream, name), + leaseOwner: ulid.Make().String(), + }, nil +} + +// close releases both local rmap replicas owned by one attachment. +func (s *sinkStream) close() { + s.consumers.Close() + s.keepAlives.Close() } // consumersMapName is the name of the replicated map that backs a sink. func consumersMapName(stream *Stream) string { - return fmt.Sprintf("stream:%s:sinks", stream.Name) + return fmt.Sprintf("stream:%s:generation:%s:sinks", stream.Name, stream.generation) } -func consumersMapOptions(stream *Stream, logger pulse.Logger) []rmap.MapOption { - opts := []rmap.MapOption{ - rmap.WithLogger(logger), - } - if stream.ttl > 0 { - if stream.ttlSliding { - opts = append(opts, rmap.WithSlidingTTL(stream.ttl)) - } else { - opts = append(opts, rmap.WithTTL(stream.ttl)) +// closeSetupMembership destroys an orphan generation-qualified map when the +// stream was destroyed during setup; otherwise it only releases this replica. +func closeSetupMembership(ctx context.Context, stream *Stream, membership *rmap.Map, setupErr error) { + if errors.Is(setupErr, ErrStreamDestroyed) { + membership.Close() + if err := stream.rdb.Del(ctx, consumersMapContentKey(stream)).Err(); err != nil { + stream.logger.Error(fmt.Errorf("failed to delete orphan sink membership: %w", err)) } + return } - return opts -} - -// sinkKeepAliveMapName is the name of the replicated map that backs a sink keep-alives. -func sinkKeepAliveMapName(sink string) string { - return fmt.Sprintf("sink:%s:keepalive", sink) + membership.Close() } diff --git a/streaming/sink_config.go b/streaming/sink_config.go new file mode 100644 index 0000000..3af0022 --- /dev/null +++ b/streaming/sink_config.go @@ -0,0 +1,666 @@ +// Package streaming persists the behavior and coordination state shared by +// same-name sink replicas. Every attached stream incarnation owns an +// independent configuration, keepalive map, and fenced stale-recovery lease. +package streaming + +import ( + "context" + "errors" + "fmt" + "strconv" + "strings" + "time" + + redis "github.com/redis/go-redis/v9" + + "goa.design/pulse/streaming/options" +) + +type ( + // sinkConfiguration is the delivery contract shared by every replica + // attached to one stream incarnation. + sinkConfiguration struct { + filterKind string + filterValue string + startID string + noAck bool + ackGrace time.Duration + } + + // sinkRecoveryLease is the Redis-issued fencing capability for one stale + // recovery pass. + sinkRecoveryLease struct { + owner string + fence int64 + } +) + +const ( + sinkLeaseUnavailable = int64(0) + sinkLeaseAcquired = int64(1) +) + +var ( + // ensureSinkConfigurationScript validates the exact stream incarnation and + // atomically establishes one immutable configuration for all replicas. + ensureSinkConfigurationScript = redis.NewScript(` +if redis.call("HGET", KEYS[1], "state") ~= ARGV[1] +or redis.call("HGET", KEYS[1], "generation") ~= ARGV[2] +or redis.call("HGET", KEYS[1], "physical_key") ~= ARGV[3] then + return redis.error_reply("STREAMDESTROYED") +end +local deadline = redis.call("HGET", KEYS[1], ARGV[9]) +if deadline then + local clock = redis.call("TIME") + local now = (tonumber(clock[1]) * 1000) + math.floor(tonumber(clock[2]) / 1000) + if now >= tonumber(deadline) then + return redis.error_reply("DEADLINEELAPSED") + end +end + +local existing = redis.call("HMGET", KEYS[2], + "filter_kind", "filter_value", "start_id", "no_ack", "ack_grace_ms") +if existing[1] then + if existing[1] ~= ARGV[4] + or existing[2] ~= ARGV[5] + or existing[3] ~= ARGV[6] + or existing[4] ~= ARGV[7] + or existing[5] ~= ARGV[8] then + return redis.error_reply("SINKCONFIGMISMATCH") + end +else + redis.call("HSET", KEYS[2], + "filter_kind", ARGV[4], + "filter_value", ARGV[5], + "start_id", ARGV[6], + "no_ack", ARGV[7], + "ack_grace_ms", ARGV[8]) +end +redis.call("SADD", KEYS[3], KEYS[2], KEYS[4], KEYS[5]) +if deadline then + redis.call("PEXPIREAT", KEYS[2], deadline) + redis.call("PEXPIREAT", KEYS[3], deadline) + redis.call("PEXPIREAT", KEYS[4], deadline) + redis.call("PEXPIREAT", KEYS[5], deadline) +end +return 1 +`) + + // acquireSinkRecoveryLeaseScript acquires or renews one owner-token lease + // using Redis TIME and returns the monotonically increasing fencing token. + acquireSinkRecoveryLeaseScript = redis.NewScript(` +if redis.call("HGET", KEYS[1], "state") ~= ARGV[1] +or redis.call("HGET", KEYS[1], "generation") ~= ARGV[2] +or redis.call("HGET", KEYS[1], "physical_key") ~= ARGV[3] then + return redis.error_reply("STREAMDESTROYED") +end +local clock = redis.call("TIME") +local now = (tonumber(clock[1]) * 1000) + math.floor(tonumber(clock[2]) / 1000) +local deadline = redis.call("HGET", KEYS[1], ARGV[6]) +if deadline and now >= tonumber(deadline) then + return redis.error_reply("DEADLINEELAPSED") +end +local owner = redis.call("HGET", KEYS[2], "owner") +local lease_until = tonumber(redis.call("HGET", KEYS[2], "lease_until") or "0") +local fence = tonumber(redis.call("HGET", KEYS[2], "fence") or "0") +if lease_until > now and owner ~= ARGV[4] then + return {0, fence} +end +if owner ~= ARGV[4] or lease_until <= now then + fence = fence + 1 +end +redis.call("HSET", KEYS[2], + "owner", ARGV[4], + "fence", tostring(fence), + "lease_until", tostring(now + tonumber(ARGV[5]))) +if deadline then + redis.call("PEXPIREAT", KEYS[2], deadline) +end +return {1, fence} +`) + + // renewSinkRecoveryLeaseScript supports explicit lease maintenance tests + // and administration. Recovery mutations do not rely on separate renewal. + renewSinkRecoveryLeaseScript = redis.NewScript(` +if redis.call("HGET", KEYS[1], "state") ~= ARGV[1] +or redis.call("HGET", KEYS[1], "generation") ~= ARGV[2] +or redis.call("HGET", KEYS[1], "physical_key") ~= ARGV[3] then + return redis.error_reply("STREAMDESTROYED") +end +local clock = redis.call("TIME") +local now = (tonumber(clock[1]) * 1000) + math.floor(tonumber(clock[2]) / 1000) +local deadline = redis.call("HGET", KEYS[1], ARGV[7]) +if deadline and now >= tonumber(deadline) then + return redis.error_reply("DEADLINEELAPSED") +end +if redis.call("HGET", KEYS[2], "owner") ~= ARGV[4] +or redis.call("HGET", KEYS[2], "fence") ~= ARGV[5] +or tonumber(redis.call("HGET", KEYS[2], "lease_until") or "0") <= now then + return redis.error_reply("SINKLEASELOST") +end +redis.call("HSET", KEYS[2], "lease_until", tostring(now + tonumber(ARGV[6]))) +if deadline then + redis.call("PEXPIREAT", KEYS[2], deadline) +end +return 1 +`) + + // fencedAutoClaimScript verifies the exact live fencing capability and + // performs XAUTOCLAIM in the same Redis operation. + fencedAutoClaimScript = redis.NewScript(` +if redis.call("HGET", KEYS[1], "state") ~= ARGV[1] +or redis.call("HGET", KEYS[1], "generation") ~= ARGV[2] +or redis.call("HGET", KEYS[1], "physical_key") ~= ARGV[3] then + return redis.error_reply("STREAMDESTROYED") +end +local clock = redis.call("TIME") +local now = (tonumber(clock[1]) * 1000) + math.floor(tonumber(clock[2]) / 1000) +local deadline = redis.call("HGET", KEYS[1], ARGV[7]) +if deadline and now >= tonumber(deadline) then + return redis.error_reply("DEADLINEELAPSED") +end +if redis.call("HGET", KEYS[2], "owner") ~= ARGV[4] +or redis.call("HGET", KEYS[2], "fence") ~= ARGV[5] +or tonumber(redis.call("HGET", KEYS[2], "lease_until") or "0") <= now then + return redis.error_reply("SINKLEASELOST") +end +redis.call("HSET", KEYS[2], "lease_until", tostring(now + tonumber(ARGV[6]))) +if deadline then + redis.call("PEXPIREAT", KEYS[2], deadline) +end +return redis.call( + "XAUTOCLAIM", + KEYS[3], + ARGV[8], + ARGV[9], + ARGV[10], + ARGV[11], + "COUNT", + ARGV[12] +) +`) + + // fencedStaleConsumerCleanupScript performs stale-consumer inspection and + // every resulting group and replicated-map mutation under one exact lease + // check. A predecessor whose lease expired cannot mutate the PEL or maps. + fencedStaleConsumerCleanupScript = redis.NewScript(` +if redis.call("HGET", KEYS[1], "state") ~= ARGV[1] +or redis.call("HGET", KEYS[1], "generation") ~= ARGV[2] +or redis.call("HGET", KEYS[1], "physical_key") ~= ARGV[3] then + return redis.error_reply("STREAMDESTROYED") +end +local clock = redis.call("TIME") +local now = (tonumber(clock[1]) * 1000) + math.floor(tonumber(clock[2]) / 1000) +local deadline = redis.call("HGET", KEYS[1], ARGV[7]) +if deadline and now >= tonumber(deadline) then + return redis.error_reply("DEADLINEELAPSED") +end +if redis.call("HGET", KEYS[2], "owner") ~= ARGV[4] +or redis.call("HGET", KEYS[2], "fence") ~= ARGV[5] +or tonumber(redis.call("HGET", KEYS[2], "lease_until") or "0") <= now then + return redis.error_reply("SINKLEASELOST") +end +redis.call("HSET", KEYS[2], "lease_until", tostring(now + tonumber(ARGV[6]))) +if deadline then + redis.call("PEXPIREAT", KEYS[2], deadline) +end + +local function publish_delete(content, channel, key) + if redis.call("HDEL", content, key) == 0 then + return + end + local rev = tostring(redis.call("HINCRBY", content, "=rev", 1)) + redis.call("HSET", content, "=kind", "del") + local message = struct.pack("ic0ic0", string.len(key), key, string.len(rev), rev) + redis.call("PUBLISH", channel, "del:" .. message) +end + +local function remove_value(content, channel, key, removed_value) + local encoded = redis.call("HGET", content, key) + if not encoded then + return + end + local values = {} + local decoded_ok, decoded = pcall(cjson.decode, encoded) + if decoded_ok and type(decoded) == "table" then + values = decoded + else + for value in string.gmatch(encoded, "[^,]+") do + table.insert(values, value) + end + end + local remaining = {} + local removed = false + for _, value in ipairs(values) do + if value == removed_value then + removed = true + else + table.insert(remaining, value) + end + end + if not removed then + return + end + if #remaining == 0 then + publish_delete(content, channel, key) + return + end + local replacement = cjson.encode(remaining) + redis.call("HSET", content, key, replacement) + local rev = tostring(redis.call("HINCRBY", content, "=rev", 1)) + redis.call("HSET", content, "=kind", "set") + local message = struct.pack( + "ic0ic0ic0", + string.len(key), key, + string.len(replacement), replacement, + string.len(rev), rev + ) + redis.call("PUBLISH", channel, "set:" .. message) +end + +local removed = {} +local malformed = {} +local consumers = redis.call("XINFO", "CONSUMERS", KEYS[3], ARGV[8]) +for _, consumer in ipairs(consumers) do + local name + local pending + for index = 1, #consumer, 2 do + if consumer[index] == "name" then + name = consumer[index + 1] + elseif consumer[index] == "pending" then + pending = tonumber(consumer[index + 1]) + end + end + if name then + local heartbeat = redis.call("HGET", KEYS[4], name) + local stale = heartbeat == false + if heartbeat then + local heartbeat_ns = tonumber(heartbeat) + if not heartbeat_ns then + table.insert(malformed, name) + else + stale = now - math.floor(heartbeat_ns / 1000000) > tonumber(ARGV[9]) + end + end + if stale then + if pending == 0 then + redis.call("XGROUP", "DELCONSUMER", KEYS[3], ARGV[8], name) + end + publish_delete(KEYS[4], KEYS[5], name) + remove_value(KEYS[6], KEYS[7], ARGV[8], name) + table.insert(removed, name) + end + end +end +return {removed, malformed} +`) +) + +// ensureSinkConfiguration establishes or verifies one attached stream's shared +// sink behavior and registers its generation-owned metadata for Destroy. +func ensureSinkConfiguration( + ctx context.Context, + stream *Stream, + name string, + config sinkConfiguration, +) error { + err := ensureSinkConfigurationScript.Run( + ctx, + stream.rdb, + []string{ + stream.lifecycleKey, + sinkConfigurationKey(stream, name), + streamResourceRegistryKey(stream), + rmapContentKey(sinkKeepAliveMapName(stream, name)), + staleLockName(stream, name), + }, + streamStateActive, + stream.generation, + stream.key, + config.filterKind, + config.filterValue, + config.startID, + boolString(config.noAck), + strconv.FormatInt(config.ackGrace.Milliseconds(), 10), + streamDeadlineKey, + ).Err() + if err == nil { + return nil + } + lifecycleErr := stream.lifecycleError(err) + if errors.Is(lifecycleErr, ErrStreamDestroyed) || + errors.Is(lifecycleErr, ErrDeadlineElapsed) { + return lifecycleErr + } + if strings.Contains(err.Error(), "SINKCONFIGMISMATCH") { + return fmt.Errorf( + "sink %q configuration differs from existing replicas for stream %q generation %s", + name, + stream.Name, + stream.generation, + ) + } + return fmt.Errorf("establish sink %q configuration: %w", name, err) +} + +// sinkConfigurationForOptions returns the canonical persisted contract for a +// sink attachment. +func sinkConfigurationForOptions(o options.SinkOptions, startID string) sinkConfiguration { + filterKind, filterValue := sinkFilterConfiguration(o) + return sinkConfiguration{ + filterKind: filterKind, + filterValue: filterValue, + startID: startID, + noAck: o.NoAck, + ackGrace: o.AckGracePeriod, + } +} + +// sinkFilterConfiguration returns the canonical persisted filter contract. +func sinkFilterConfiguration(o options.SinkOptions) (string, string) { + switch { + case o.Topic != "": + return "topic", o.Topic + case o.TopicPattern != "": + return "pattern", o.TopicPattern + default: + return "all", "" + } +} + +// sinkNamespace is the logical identity shared only by replicas of one sink +// on one stream incarnation. +func sinkNamespace(stream *Stream, sink string) string { + return fmt.Sprintf("stream:%s:generation:%s:sink:%s", stream.Name, stream.generation, sink) +} + +// sinkConfigurationKey stores immutable same-name replica behavior. +func sinkConfigurationKey(stream *Stream, sink string) string { + return "pulse:" + sinkNamespace(stream, sink) + ":config" +} + +// sinkKeepAliveMapName identifies the generation-scoped consumer heartbeat map. +func sinkKeepAliveMapName(stream *Stream, sink string) string { + return sinkNamespace(stream, sink) + ":keepalive" +} + +// staleLockName identifies the generation-scoped stale-recovery lease. +func staleLockName(stream *Stream, sink string) string { + return "pulse:" + sinkNamespace(stream, sink) + ":stalelease" +} + +// acquireSinkRecoveryLease obtains the current fencing token for one attached +// stream. A false result means another replica still owns recovery. +func acquireSinkRecoveryLease( + ctx context.Context, + stream *Stream, + key, owner string, + duration time.Duration, +) (sinkRecoveryLease, bool, error) { + raw, err := acquireSinkRecoveryLeaseScript.Run( + ctx, + stream.rdb, + []string{stream.lifecycleKey, key}, + streamStateActive, + stream.generation, + stream.key, + owner, + strconv.FormatInt(duration.Milliseconds(), 10), + streamDeadlineKey, + ).Slice() + if err != nil { + lifecycleErr := stream.lifecycleError(err) + if errors.Is(lifecycleErr, ErrStreamDestroyed) || + errors.Is(lifecycleErr, ErrDeadlineElapsed) { + return sinkRecoveryLease{}, false, lifecycleErr + } + return sinkRecoveryLease{}, false, fmt.Errorf("acquire sink recovery lease: %w", err) + } + if len(raw) != 2 { + return sinkRecoveryLease{}, false, fmt.Errorf("invalid sink recovery lease result length %d", len(raw)) + } + status, ok := raw[0].(int64) + if !ok { + return sinkRecoveryLease{}, false, fmt.Errorf("invalid sink recovery lease status %T", raw[0]) + } + fence, ok := raw[1].(int64) + if !ok { + return sinkRecoveryLease{}, false, fmt.Errorf("invalid sink recovery lease fence %T", raw[1]) + } + switch status { + case sinkLeaseUnavailable: + return sinkRecoveryLease{}, false, nil + case sinkLeaseAcquired: + return sinkRecoveryLease{owner: owner, fence: fence}, true, nil + default: + return sinkRecoveryLease{}, false, fmt.Errorf("invalid sink recovery lease status %d", status) + } +} + +// renewSinkRecoveryLease extends one exact current capability without +// performing a recovery mutation. +func renewSinkRecoveryLease( + ctx context.Context, + stream *Stream, + key string, + lease sinkRecoveryLease, + duration time.Duration, +) error { + err := renewSinkRecoveryLeaseScript.Run( + ctx, + stream.rdb, + []string{stream.lifecycleKey, key}, + streamStateActive, + stream.generation, + stream.key, + lease.owner, + strconv.FormatInt(lease.fence, 10), + strconv.FormatInt(duration.Milliseconds(), 10), + streamDeadlineKey, + ).Err() + lifecycleErr := stream.lifecycleError(err) + if errors.Is(lifecycleErr, ErrStreamDestroyed) || + errors.Is(lifecycleErr, ErrDeadlineElapsed) { + return lifecycleErr + } + if err != nil { + return fmt.Errorf("renew sink recovery lease: %w", err) + } + return nil +} + +// fencedAutoClaim claims one idle batch only while lease is the exact +// unexpired Redis-owned fencing capability. +func fencedAutoClaim( + ctx context.Context, + stream *Stream, + key string, + lease sinkRecoveryLease, + duration time.Duration, + group, consumer string, + minIdle time.Duration, + start string, + count int64, +) (string, []redis.XMessage, error) { + raw, err := fencedAutoClaimScript.Run( + ctx, + stream.rdb, + []string{stream.lifecycleKey, key, stream.key}, + streamStateActive, + stream.generation, + stream.key, + lease.owner, + strconv.FormatInt(lease.fence, 10), + strconv.FormatInt(duration.Milliseconds(), 10), + streamDeadlineKey, + group, + consumer, + strconv.FormatInt(minIdle.Milliseconds(), 10), + start, + strconv.FormatInt(count, 10), + ).Slice() + lifecycleErr := stream.lifecycleError(err) + if errors.Is(lifecycleErr, ErrStreamDestroyed) || + errors.Is(lifecycleErr, ErrDeadlineElapsed) { + return start, nil, lifecycleErr + } + if err != nil { + return start, nil, fmt.Errorf("fenced sink recovery claim: %w", err) + } + if len(raw) < 2 || len(raw) > 3 { + return start, nil, fmt.Errorf("fenced sink recovery claim returned %d values", len(raw)) + } + next, ok := raw[0].(string) + if !ok { + return start, nil, fmt.Errorf("fenced sink recovery claim returned invalid cursor %T", raw[0]) + } + entries, ok := raw[1].([]any) + if !ok { + return start, nil, fmt.Errorf("fenced sink recovery claim returned invalid messages %T", raw[1]) + } + messages, err := decodeSnapshotRange(entries) + if err != nil { + return start, nil, fmt.Errorf("decode fenced sink recovery claim: %w", err) + } + return next, messages, nil +} + +// cleanupStaleConsumers removes stale group and replicated-map state only +// while lease remains the exact unexpired fencing capability. +func cleanupStaleConsumers( + ctx context.Context, + stream *Stream, + key string, + lease sinkRecoveryLease, + duration time.Duration, + group string, + grace time.Duration, +) ([]string, []string, error) { + raw, err := fencedStaleConsumerCleanupScript.Run( + ctx, + stream.rdb, + []string{ + stream.lifecycleKey, + key, + stream.key, + rmapContentKey(sinkKeepAliveMapName(stream, group)), + rmapChannelKey(sinkKeepAliveMapName(stream, group)), + consumersMapContentKey(stream), + consumersMapChannelKey(stream), + }, + streamStateActive, + stream.generation, + stream.key, + lease.owner, + strconv.FormatInt(lease.fence, 10), + strconv.FormatInt(duration.Milliseconds(), 10), + streamDeadlineKey, + group, + strconv.FormatInt((2*grace).Milliseconds(), 10), + ).Slice() + lifecycleErr := stream.lifecycleError(err) + if errors.Is(lifecycleErr, ErrStreamDestroyed) || + errors.Is(lifecycleErr, ErrDeadlineElapsed) { + return nil, nil, lifecycleErr + } + if err != nil { + return nil, nil, fmt.Errorf("fenced stale-consumer cleanup: %w", err) + } + if len(raw) != 2 { + return nil, nil, fmt.Errorf("fenced stale-consumer cleanup returned %d values", len(raw)) + } + removed, err := stringSlice(raw[0]) + if err != nil { + return nil, nil, fmt.Errorf("decode removed stale consumers: %w", err) + } + malformed, err := stringSlice(raw[1]) + if err != nil { + return nil, nil, fmt.Errorf("decode malformed stale consumers: %w", err) + } + return removed, malformed, nil +} + +// stringSlice validates one Redis Lua array of strings. +func stringSlice(value any) ([]string, error) { + raw, ok := value.([]any) + if !ok { + return nil, fmt.Errorf("expected array, got %T", value) + } + result := make([]string, len(raw)) + for i, item := range raw { + result[i], ok = item.(string) + if !ok { + return nil, fmt.Errorf("item %d has type %T", i, item) + } + } + return result, nil +} + +// streamResourceRegistryKey owns generation-scoped metadata deleted by +// explicit Stream.Destroy. +func streamResourceRegistryKey(stream *Stream) string { + return fmt.Sprintf( + "pulse:stream:%s:generation:%s:resources", + stream.Name, + stream.generation, + ) +} + +// validateSinkOptions rejects values that would cause Redis retry churn, +// invalid channel allocation, or immediate stale-consumer recovery. +func validateSinkOptions(o options.SinkOptions) error { + switch { + case o.Topic != "" && o.TopicPattern != "": + return fmt.Errorf("sink topic and topic pattern are mutually exclusive") + case o.HasConflictingStartOptions(): + return fmt.Errorf("sink cursor-start options are mutually exclusive") + case o.BlockDuration < time.Millisecond: + return fmt.Errorf("sink block duration must be at least 1ms") + case o.MaxPolled <= 0: + return fmt.Errorf("sink maximum polled events must be greater than zero") + case o.BufferSize < 0: + return fmt.Errorf("sink buffer size must be greater than or equal to zero") + case o.AckGracePeriod < time.Millisecond: + return fmt.Errorf("sink acknowledgement grace period must be at least 1ms") + default: + return nil + } +} + +// validateReaderOptions rejects ambiguous selection and cursor contracts plus +// values that would cause Redis retry churn or invalid channel allocation. +func validateReaderOptions(o options.ReaderOptions) error { + switch { + case o.Topic != "" && o.TopicPattern != "": + return fmt.Errorf("reader topic and topic pattern are mutually exclusive") + case o.HasConflictingStartOptions(): + return fmt.Errorf("reader cursor-start options are mutually exclusive") + case o.BlockDuration < time.Millisecond: + return fmt.Errorf("reader block duration must be at least 1ms") + case o.MaxPolled <= 0: + return fmt.Errorf("reader maximum polled events must be greater than zero") + case o.BufferSize < 0: + return fmt.Errorf("reader buffer size must be greater than or equal to zero") + default: + return nil + } +} + +// validateAddStreamOptions rejects ambiguous per-stream cursor contracts. +func validateAddStreamOptions(o options.AddStreamOptions) error { + if o.HasConflictingStartOptions() { + return fmt.Errorf("added stream cursor-start options are mutually exclusive") + } + return nil +} + +// rmapContentKey returns the Redis hash used by the named replicated map. +func rmapContentKey(name string) string { + return fmt.Sprintf("map:%s:content", name) +} + +// rmapChannelKey is the Redis channel carrying rmap update notifications for +// the named replicated map. +func rmapChannelKey(name string) string { + return fmt.Sprintf("map:%s:updates", name) +} diff --git a/streaming/sink_consumer.go b/streaming/sink_consumer.go new file mode 100644 index 0000000..4634339 --- /dev/null +++ b/streaming/sink_consumer.go @@ -0,0 +1,315 @@ +// Sink consumer attachment and detachment are single Redis operations fenced +// on the exact stream generation, so a destroyed generation's membership and +// keep-alive projections can never be recreated after Stream.Destroy deleted +// them. Per-consumer pending state decides whether Redis metadata can be +// deleted, while membership and keep-alive projections always move coherently. +package streaming + +import ( + "context" + "fmt" + "strconv" + + redis "github.com/redis/go-redis/v9" +) + +// registerSinkConsumerScript atomically verifies the exact stream generation, +// creates the Redis consumer, appends it to the sink membership map, and +// writes its initial keep-alive, so no consumer state exists on a destroyed +// generation and membership can never diverge from the consumer group. +// +// KEYS: [1]=lifecycle [2]=stream [3]=membership content [4]=membership channel +// +// [5]=keepalive content [6]=keepalive channel +// +// ARGV: [1]=active state [2]=generation [3]=group [4]=consumer [5]=keep-alive +var registerSinkConsumerScript = redis.NewScript(` +local lifecycle_type = redis.call("TYPE", KEYS[1]).ok +if lifecycle_type ~= "hash" then + return redis.error_reply("SINKLIFECYCLEINVALID " .. lifecycle_type) +end +if redis.call("HGET", KEYS[1], "state") ~= ARGV[1] +or redis.call("HGET", KEYS[1], "generation") ~= ARGV[2] +or redis.call("HGET", KEYS[1], "physical_key") ~= KEYS[2] then + return redis.error_reply("STREAMDESTROYED") +end +local membership_type = redis.call("TYPE", KEYS[3]).ok +if membership_type ~= "hash" and membership_type ~= "none" then + return redis.error_reply("SINKMEMBERSHIPINVALID " .. membership_type) +end +local keepalive_type = redis.call("TYPE", KEYS[5]).ok +if keepalive_type ~= "hash" and keepalive_type ~= "none" then + return redis.error_reply("SINKKEEPALIVEINVALID " .. keepalive_type) +end +local created = redis.pcall("XGROUP", "CREATECONSUMER", KEYS[2], ARGV[3], ARGV[4]) +if type(created) == "table" and created.err then + return redis.error_reply(created.err) +end + +local function publish_set(content, channel, field, value) + local rev = tostring(redis.call("HINCRBY", content, "=rev", 1)) + redis.call("HSET", content, "=kind", "set") + local message = struct.pack( + "ic0ic0ic0", + string.len(field), field, + string.len(value), value, + string.len(rev), rev + ) + redis.call("PUBLISH", channel, "set:" .. message) +end + +local encoded = redis.call("HGET", KEYS[3], ARGV[3]) +local consumers = {} +if encoded then + local ok, decoded = pcall(cjson.decode, encoded) + if not ok or type(decoded) ~= "table" then + return redis.error_reply("SINKMEMBERSHIPINVALID") + end + consumers = decoded +end +local member = false +for _, consumer in ipairs(consumers) do + if consumer == ARGV[4] then + member = true + break + end +end +if not member then + table.insert(consumers, ARGV[4]) + local value = cjson.encode(consumers) + redis.call("HSET", KEYS[3], ARGV[3], value) + publish_set(KEYS[3], KEYS[4], ARGV[3], value) +end + +redis.call("HSET", KEYS[5], ARGV[4], ARGV[5]) +publish_set(KEYS[5], KEYS[6], ARGV[4], ARGV[5]) +return 1 +`) + +// fencedKeepAliveSetScript refreshes one consumer keep-alive only while the +// exact stream generation is active, so the periodic keep-alive loop cannot +// recreate the keep-alive map after Stream.Destroy deleted it. +// +// KEYS: [1]=lifecycle [2]=stream [3]=keepalive content [4]=keepalive channel +// ARGV: [1]=active state [2]=generation [3]=consumer [4]=keep-alive +var fencedKeepAliveSetScript = redis.NewScript(` +if redis.call("HGET", KEYS[1], "state") ~= ARGV[1] +or redis.call("HGET", KEYS[1], "generation") ~= ARGV[2] +or redis.call("HGET", KEYS[1], "physical_key") ~= KEYS[2] then + return redis.error_reply("STREAMDESTROYED") +end +local keepalive_type = redis.call("TYPE", KEYS[3]).ok +if keepalive_type ~= "hash" and keepalive_type ~= "none" then + return redis.error_reply("SINKKEEPALIVEINVALID " .. keepalive_type) +end +redis.call("HSET", KEYS[3], ARGV[3], ARGV[4]) +local rev = tostring(redis.call("HINCRBY", KEYS[3], "=rev", 1)) +redis.call("HSET", KEYS[3], "=kind", "set") +local message = struct.pack( + "ic0ic0ic0", + string.len(ARGV[3]), ARGV[3], + string.len(ARGV[4]), ARGV[4], + string.len(rev), rev +) +redis.call("PUBLISH", KEYS[4], "set:" .. message) +return 1 +`) + +var detachSinkConsumerScript = redis.NewScript(` +local lifecycle_type = redis.call("TYPE", KEYS[1]).ok +if lifecycle_type ~= "hash" then + return redis.error_reply("SINKLIFECYCLEINVALID " .. lifecycle_type) +end +if redis.call("HGET", KEYS[1], "state") ~= ARGV[1] +or redis.call("HGET", KEYS[1], "generation") ~= ARGV[2] +or redis.call("HGET", KEYS[1], "physical_key") ~= KEYS[2] then + return redis.error_reply("STREAMDESTROYED") +end +local stream_type = redis.call("TYPE", KEYS[2]).ok +if stream_type ~= "stream" and stream_type ~= "none" then + return redis.error_reply("SINKSTREAMINVALID " .. stream_type) +end +local membership_type = redis.call("TYPE", KEYS[3]).ok +if membership_type ~= "hash" and membership_type ~= "none" then + return redis.error_reply("SINKMEMBERSHIPINVALID " .. membership_type) +end +local keepalive_type = redis.call("TYPE", KEYS[5]).ok +if keepalive_type ~= "hash" and keepalive_type ~= "none" then + return redis.error_reply("SINKKEEPALIVEINVALID " .. keepalive_type) +end + +local pending = {} +local pending_result = redis.pcall( + "XPENDING", KEYS[2], ARGV[3], "-", "+", 1, ARGV[4] +) +if type(pending_result) == "table" and pending_result.err then + if not string.find(pending_result.err, "NOGROUP", 1, true) then + return redis.error_reply(pending_result.err) + end +elseif type(pending_result) == "table" then + pending = pending_result +end +if #pending == 0 then + local groups = redis.pcall("XGROUP", "DELCONSUMER", KEYS[2], ARGV[3], ARGV[4]) + if type(groups) == "table" and groups.err + and not string.find(groups.err, "NOGROUP", 1, true) then + return redis.error_reply(groups.err) + end +end + +local function publish_delete(content, channel, field) + local rev = tostring(redis.call("HINCRBY", content, "=rev", 1)) + redis.call("HSET", content, "=kind", "del") + local message = struct.pack( + "ic0ic0", + string.len(field), field, + string.len(rev), rev + ) + redis.call("PUBLISH", channel, "del:" .. message) +end + +local function publish_set(content, channel, field, value) + local rev = tostring(redis.call("HINCRBY", content, "=rev", 1)) + redis.call("HSET", content, "=kind", "set") + local message = struct.pack( + "ic0ic0ic0", + string.len(field), field, + string.len(value), value, + string.len(rev), rev + ) + redis.call("PUBLISH", channel, "set:" .. message) +end + +local encoded = redis.call("HGET", KEYS[3], ARGV[3]) +if encoded then + local ok, consumers = pcall(cjson.decode, encoded) + if not ok or type(consumers) ~= "table" then + return redis.error_reply("SINKMEMBERSHIPINVALID") + end + local remaining = {} + for _, consumer in ipairs(consumers) do + if consumer ~= ARGV[4] then + table.insert(remaining, consumer) + end + end + if #remaining == 0 then + redis.call("HDEL", KEYS[3], ARGV[3]) + publish_delete(KEYS[3], KEYS[4], ARGV[3]) + elseif #remaining ~= #consumers then + local value = cjson.encode(remaining) + redis.call("HSET", KEYS[3], ARGV[3], value) + publish_set(KEYS[3], KEYS[4], ARGV[3], value) + end +end + +if redis.call("HDEL", KEYS[5], ARGV[4]) == 1 then + publish_delete(KEYS[5], KEYS[6], ARGV[4]) +end +return #pending +`) + +// detachSinkConsumer removes one consumer's distributed metadata. Redis +// deletes the consumer itself only when its own PEL is empty; pending entries +// remain claimable by stale-consumer recovery. +func detachSinkConsumer( + ctx context.Context, + state *sinkStream, + sinkName, consumer string, +) (bool, error) { + pending, err := detachSinkConsumerScript.Run( + ctx, + state.stream.rdb, + []string{ + state.stream.lifecycleKey, + state.stream.key, + consumersMapContentKey(state.stream), + consumersMapChannelKey(state.stream), + rmapContentKey(sinkKeepAliveMapName(state.stream, sinkName)), + rmapChannelKey(sinkKeepAliveMapName(state.stream, sinkName)), + }, + streamStateActive, + state.stream.generation, + sinkName, + consumer, + ).Int64() + if err != nil { + return false, streamLifecycleBoundaryError(err) + } + if pending < 0 || pending > 1 { + return false, fmt.Errorf("detach sink consumer returned invalid pending count %d", pending) + } + return pending == 0, nil +} + +// registerSinkConsumer establishes one consumer's complete distributed +// metadata in a single generation-fenced operation. On error nothing was +// written, so callers need no Redis rollback for this consumer. +func registerSinkConsumer( + ctx context.Context, + state *sinkStream, + sinkName, consumer string, + keepAlive int64, +) error { + err := registerSinkConsumerScript.Run( + ctx, + state.stream.rdb, + []string{ + state.stream.lifecycleKey, + state.stream.key, + consumersMapContentKey(state.stream), + consumersMapChannelKey(state.stream), + rmapContentKey(sinkKeepAliveMapName(state.stream, sinkName)), + rmapChannelKey(sinkKeepAliveMapName(state.stream, sinkName)), + }, + streamStateActive, + state.stream.generation, + sinkName, + consumer, + strconv.FormatInt(keepAlive, 10), + ).Err() + if err != nil { + return streamLifecycleBoundaryError(fmt.Errorf( + "cannot register consumer %s for sink %s on stream %s: %w", + consumer, + sinkName, + state.stream.Name, + err, + )) + } + return nil +} + +// setSinkKeepAlive refreshes one consumer keep-alive behind the generation +// fence. A destroyed generation returns ErrStreamDestroyed without recreating +// any keep-alive state. +func setSinkKeepAlive( + ctx context.Context, + state *sinkStream, + sinkName, consumer string, + keepAlive int64, +) error { + err := fencedKeepAliveSetScript.Run( + ctx, + state.stream.rdb, + []string{ + state.stream.lifecycleKey, + state.stream.key, + rmapContentKey(sinkKeepAliveMapName(state.stream, sinkName)), + rmapChannelKey(sinkKeepAliveMapName(state.stream, sinkName)), + }, + streamStateActive, + state.stream.generation, + consumer, + strconv.FormatInt(keepAlive, 10), + ).Err() + if err != nil { + return streamLifecycleBoundaryError(fmt.Errorf( + "cannot refresh keep-alive for consumer %s of sink %s on stream %s: %w", + consumer, + sinkName, + state.stream.Name, + err, + )) + } + return nil +} diff --git a/streaming/sink_lease.go b/streaming/sink_lease.go deleted file mode 100644 index 475a9e4..0000000 --- a/streaming/sink_lease.go +++ /dev/null @@ -1,263 +0,0 @@ -// This file implements the fenced sink lease used for idle-message claiming -// and stale-consumer cleanup. -// -// Each sink (consumer group) holds at most one lease per stream, stored in a -// hash at "pulse:streammeta::sink::lease" with fields: -// -// - owner: the sink instance currently allowed to claim and clean up. -// - fence: a counter incremented on every ownership change. -// - expiry: Redis-time (milliseconds) after which the lease is up for grabs. -// -// All timing uses the Redis TIME command evaluated inside the scripts so -// client clock skew cannot grant two instances the lease at once. Lease -// verification, renewal, and the guarded mutation (XAUTOCLAIM or consumer -// deletion) execute in a single script: once another instance takes over the -// lease the fence counter changes and every in-flight script from the stale -// owner fails before touching the PEL. -package streaming - -import ( - "context" - "fmt" - "strings" - - redis "github.com/redis/go-redis/v9" -) - -// leaseLostErrorPrefix is the Redis error prefix returned by fenced scripts -// when the caller no longer holds the lease. -const leaseLostErrorPrefix = "LEASELOST" - -// leaseCheckLua verifies and renews the caller's lease. Scripts using it must -// bind KEYS[2] to the lease key, ARGV[1] to the owner, ARGV[2] to the fence -// value returned by acquisition, and ARGV[3] to the lease duration in -// milliseconds. It defines now_ms for subsequent statements. -const leaseCheckLua = ` -local t = redis.call("TIME") -local now_ms = t[1] * 1000 + math.floor(t[2] / 1000) -local owner = redis.call("HGET", KEYS[2], "owner") -local fence = redis.call("HGET", KEYS[2], "fence") -local expiry = tonumber(redis.call("HGET", KEYS[2], "expiry") or "0") -if owner ~= ARGV[1] or fence ~= ARGV[2] or expiry < now_ms then - return redis.error_reply("LEASELOST sink lease lost") -end -redis.call("HSET", KEYS[2], "expiry", now_ms + tonumber(ARGV[3])) -redis.call("PEXPIRE", KEYS[2], tonumber(ARGV[3]) * 2) -` - -// acquireLeaseScript acquires or renews the sink lease using Redis time. A -// new owner (or a takeover of an expired lease) increments the fence counter -// so scripts still carrying the previous fence value can no longer mutate -// anything. Acquisition is fenced by the stream lifecycle: destroyed streams -// never get a new lease key. -// -// KEYS: [1]=lifecycle [2]=lease -// ARGV: [1]=owner [2]=leaseMs -// Returns {acquired(0|1), fence}. -var acquireLeaseScript = redis.NewScript(` -local state = redis.call("HGET", KEYS[1], "state") -if state ~= "active" then - return redis.error_reply("STREAMDESTROYED stream was destroyed") -end -local t = redis.call("TIME") -local now_ms = t[1] * 1000 + math.floor(t[2] / 1000) -local owner = redis.call("HGET", KEYS[2], "owner") -local expiry = tonumber(redis.call("HGET", KEYS[2], "expiry") or "0") -if owner and owner ~= ARGV[1] and expiry >= now_ms then - return {0, 0} -end -local bump = 1 -if owner == ARGV[1] and expiry >= now_ms then - bump = 0 -end -local fence = redis.call("HINCRBY", KEYS[2], "fence", bump) -redis.call("HSET", KEYS[2], "owner", ARGV[1], "expiry", now_ms + tonumber(ARGV[2])) -redis.call("PEXPIRE", KEYS[2], tonumber(ARGV[2]) * 2) -return {1, fence} -`) - -// fencedAutoClaimScript renews the lease and claims idle messages in one -// atomic operation so a stale owner cannot move PEL entries after takeover. -// -// KEYS: [1]=lifecycle [2]=lease [3]=stream -// ARGV: [1]=owner [2]=fence [3]=leaseMs [4]=group [5]=consumer [6]=minIdleMs -// [7]=start [8]=count -// Returns the XAUTOCLAIM reply, or {"0-0", {}} when the group is gone. -var fencedAutoClaimScript = redis.NewScript(` -local state = redis.call("HGET", KEYS[1], "state") -if state ~= "active" then - return redis.error_reply("STREAMDESTROYED stream was destroyed") -end -` + leaseCheckLua + ` -local res = redis.pcall("XAUTOCLAIM", KEYS[3], ARGV[4], ARGV[5], ARGV[6], ARGV[7], "COUNT", ARGV[8]) -if type(res) == "table" and res["err"] then - if string.find(res["err"], "NOGROUP", 1, true) then - return {"0-0", {}} - end - return res -end -return res -`) - -// fencedCleanupScript renews the lease and deletes stale consumers in one -// atomic operation. A consumer is stale when it has no pending events and its -// keep-alive is missing or older than the staleness threshold; its membership -// map entry and keep-alive record are removed with rmap protocol -// notifications. The caller's live consumer is never deleted. -// -// KEYS: [1]=lifecycle [2]=lease [3]=stream [4]=membership content -// [5]=membership channel [6]=keep-alive content [7]=keep-alive channel -// ARGV: [1]=owner [2]=fence [3]=leaseMs [4]=group [5]=staleNs [6]=live consumer -// Returns the deleted consumer names. -var fencedCleanupScript = redis.NewScript(` -local state = redis.call("HGET", KEYS[1], "state") -if state ~= "active" then - return redis.error_reply("STREAMDESTROYED stream was destroyed") -end -` + leaseCheckLua + membershipRemoveLua + ` -local consumers = redis.pcall("XINFO", "CONSUMERS", KEYS[3], ARGV[4]) -if type(consumers) == "table" and consumers["err"] then - if string.find(consumers["err"], "NOGROUP", 1, true) then - return {} - end - return consumers -end -local now_ns = (t[1] * 1000000 + t[2]) * 1000 -local removed = {} -for _, info in ipairs(consumers) do - local name, pending - for i = 1, #info, 2 do - if info[i] == "name" then name = info[i+1] end - if info[i] == "pending" then pending = info[i+1] end - end - if name ~= ARGV[6] and pending == 0 then - local ka = redis.call("HGET", KEYS[6], name) - if not ka or now_ns - tonumber(ka) > tonumber(ARGV[5]) then - redis.call("XGROUP", "DELCONSUMER", KEYS[3], ARGV[4], name) - if ka then - redis.call("HDEL", KEYS[6], name) - local rev = tostring(redis.call("HINCRBY", KEYS[6], "=rev", 1)) - redis.call("HSET", KEYS[6], "=kind", "del") - local msg = struct.pack("ic0ic0", string.len(name), name, string.len(rev), rev) - redis.call("PUBLISH", KEYS[7], "del:" .. msg) - end - membership_remove(KEYS[4], KEYS[5], ARGV[4], name) - table.insert(removed, name) - end - end -end -return removed -`) - -// acquireSinkLease attempts to acquire or renew the sink lease for the stream -// using Redis time. It returns whether the lease is held and the fence value -// to pass to subsequent fenced operations. -func acquireSinkLease(ctx context.Context, stream *Stream, sinkName, owner string, leaseMs int64) (bool, int64, error) { - keys := []string{lifecycleKey(stream.key), leaseKey(stream.key, sinkName)} - res, err := acquireLeaseScript.Run(ctx, stream.rdb, keys, owner, leaseMs).Slice() - if err != nil { - if isStreamDestroyedErr(err) { - return false, 0, fmt.Errorf("cannot acquire lease for stream %s: %w", stream.Name, ErrStreamDestroyed) - } - return false, 0, fmt.Errorf("failed to acquire lease for stream %s: %w", stream.Name, err) - } - return res[0].(int64) == 1, res[1].(int64), nil -} - -// fencedAutoClaim renews the lease and claims up to count messages idle for -// at least minIdleMs, starting at start, assigning them to consumer. It -// returns the claimed messages and the next XAUTOCLAIM start cursor ("0-0" -// when the scan is complete). -func fencedAutoClaim(ctx context.Context, stream *Stream, sinkName, owner string, fence, leaseMs int64, consumer string, minIdleMs int64, start string, count int64) ([]redis.XMessage, string, error) { - keys := []string{lifecycleKey(stream.key), leaseKey(stream.key, sinkName), stream.key} - res, err := fencedAutoClaimScript.Run(ctx, stream.rdb, keys, - owner, fence, leaseMs, sinkName, consumer, minIdleMs, start, count).Slice() - if err != nil { - return nil, "", err - } - next := res[0].(string) - entries, ok := res[1].([]any) - if !ok { - return nil, "", fmt.Errorf("unexpected XAUTOCLAIM entries type %T", res[1]) - } - msgs, err := decodeClaimedMessages(entries) - if err != nil { - return nil, "", err - } - return msgs, next, nil -} - -// fencedCleanupStaleConsumers renews the lease and deletes consumers of the -// sink group that have no pending events and whose keep-alive is missing or -// older than staleNs nanoseconds of Redis time. liveConsumer is the caller's -// current consumer and is never deleted. It returns the deleted consumer -// names. -func fencedCleanupStaleConsumers(ctx context.Context, stream *Stream, sinkName, owner string, fence, leaseMs int64, staleNs int64, liveConsumer string) ([]string, error) { - keys := []string{ - lifecycleKey(stream.key), - leaseKey(stream.key, sinkName), - stream.key, - membershipContentKey(stream.Name), - membershipChannelKey(stream.Name), - keepAliveContentKey(sinkName), - keepAliveChannelKey(sinkName), - } - res, err := fencedCleanupScript.Run(ctx, stream.rdb, keys, - owner, fence, leaseMs, sinkName, staleNs, liveConsumer).StringSlice() - if err != nil { - return nil, err - } - return res, nil -} - -// decodeClaimedMessages converts the raw Lua XAUTOCLAIM entries reply into -// XMessage values. Nil entries (tombstones of trimmed events reported by -// older Redis servers) are skipped. -func decodeClaimedMessages(entries []any) ([]redis.XMessage, error) { - msgs := make([]redis.XMessage, 0, len(entries)) - for _, entry := range entries { - if entry == nil { - continue - } - pair, ok := entry.([]any) - if !ok || len(pair) != 2 { - return nil, fmt.Errorf("unexpected XAUTOCLAIM entry %v", entry) - } - id, ok := pair[0].(string) - if !ok { - return nil, fmt.Errorf("unexpected XAUTOCLAIM entry ID %v", pair[0]) - } - fields, ok := pair[1].([]any) - if !ok || len(fields)%2 != 0 { - return nil, fmt.Errorf("unexpected XAUTOCLAIM entry fields %v", pair[1]) - } - values := make(map[string]any, len(fields)/2) - for i := 0; i < len(fields); i += 2 { - key, ok := fields[i].(string) - if !ok { - return nil, fmt.Errorf("unexpected XAUTOCLAIM field name %v", fields[i]) - } - values[key] = fields[i+1] - } - msgs = append(msgs, redis.XMessage{ID: id, Values: values}) - } - return msgs, nil -} - -// isLeaseLostErr reports whether err is the fenced-script rejection of an -// operation whose lease was taken over by another sink instance. -func isLeaseLostErr(err error) bool { - return err != nil && strings.HasPrefix(err.Error(), leaseLostErrorPrefix) -} - -// keepAliveContentKey returns the rmap content key of the sink keep-alive map -// (see sinkKeepAliveMapName). -func keepAliveContentKey(sinkName string) string { - return fmt.Sprintf("map:sink:%s:keepalive:content", sinkName) -} - -// keepAliveChannelKey returns the rmap pubsub channel of the sink keep-alive -// map. -func keepAliveChannelKey(sinkName string) string { - return fmt.Sprintf("map:sink:%s:keepalive:updates", sinkName) -} diff --git a/streaming/sink_recovery.go b/streaming/sink_recovery.go index 7d87670..03d1b3d 100644 --- a/streaming/sink_recovery.go +++ b/streaming/sink_recovery.go @@ -1,484 +1,305 @@ -// This file implements lossless consumer-group recovery and the stream -// lifecycle fence shared by sinks and Stream.Destroy. -// -// Every piece of durable sink metadata attached to a stream lives under the -// "pulse:streammeta:" prefix: -// -// - :lifecycle hash with a "state" field ("active" or "destroyed"). -// - :cursors hash mapping sink name to the recovery cursor, the -// highest event ID known to be fully acknowledged for that sink. -// - :sink::lease fenced lease used by sink_lease.go. -// -// The lifecycle hash is the destroy fence: every Lua script in this file and -// in sink_lease.go that mutates stream-scoped metadata reads the lifecycle -// state first, inside the same atomic script invocation as the write. Once -// Stream.Destroy marks a stream destroyed no concurrent sink loop can -// recreate its consumer group, membership map, cursor, or lease; only a -// deliberate NewSink/AddStream call (establish mode) reactivates the name. -// -// The recovery cursor is what makes NOGROUP recovery lossless: when a -// consumer group disappears (XGROUP DESTROY, Redis state loss) the group is -// recreated at the stored cursor rather than "$" so unacknowledged events are -// redelivered. The cursor is advanced by recomputing it from the exact PEL -// state each time events are acknowledged. -// -// Scripts publish replicated-map updates using the exact rmap wire protocol -// (see rmap/scripts.go) so live rmap clients observe fenced membership -// mutations exactly as if they had been made through the rmap API. +// Package streaming persists each sink cursor in generation-qualified metadata +// beside its lifecycle-selected physical stream key. Recovery and +// acknowledgement verify the Redis lifecycle record in the same Lua operation, +// so stale handles cannot recreate groups or mutate a later incarnation. package streaming import ( "context" - "errors" "fmt" - "strings" + "strconv" redis "github.com/redis/go-redis/v9" ) type ( - // recoveryAcker acknowledges sink events and atomically advances the - // durable recovery cursor from the exact PEL state. It is the Acker - // carried by every event delivered through a sink, so both Sink.Ack and - // direct Event.Acker.XAck calls keep the recovery cursor current. + // recoveryAcker implements Event.Acker for one immutable stream generation. recoveryAcker struct { - rdb *redis.Client + stream *Stream } ) var ( - // ErrStreamDestroyed is returned when a sink operation targets a stream - // that was destroyed with Stream.Destroy. The sink drops the stream from - // its set instead of resurrecting its metadata. - ErrStreamDestroyed = errors.New("stream is destroyed") - - // ErrSinkClosed is returned by AddStream and RemoveStream after Close. - ErrSinkClosed = errors.New("sink is closed") -) - -// streamDestroyedErrorPrefix is the Redis error prefix used by fenced scripts -// to reject writes against a destroyed stream. -const streamDestroyedErrorPrefix = "STREAMDESTROYED" - -// lifecycleFenceLua guards a script against destroyed streams and activates -// absent lifecycles. Scripts using it must bind KEYS[1] to the lifecycle key -// and define an `establish` local ("1" reactivates a destroyed stream). -const lifecycleFenceLua = ` -local state = redis.call("HGET", KEYS[1], "state") -if state == "destroyed" then - if establish == "1" then - redis.call("HSET", KEYS[1], "state", "active") - else - return redis.error_reply("STREAMDESTROYED stream was destroyed") - end -elseif not state then - redis.call("HSET", KEYS[1], "state", "active") -end -` - -// recoveryCursorLua defines recovery_cursor(stream, group) which computes the -// highest event ID X such that every entry at or before X is acknowledged: -// the entry preceding the oldest pending entry when the PEL is not empty, the -// group last-delivered-id otherwise. Returns false when the group is gone. -const recoveryCursorLua = ` -local function recovery_cursor(stream, group) - local pending = redis.pcall("XPENDING", stream, group) - if pending["err"] then - return false - end - if pending[1] > 0 then - local prev = redis.call("XREVRANGE", stream, "(" .. pending[2], "-", "COUNT", 1) - if #prev == 0 then - return "0-0" - end - return prev[1][1] - end - local groups = redis.call("XINFO", "GROUPS", stream) - for _, info in ipairs(groups) do - local name, last - for i = 1, #info, 2 do - if info[i] == "name" then name = info[i+1] end - if info[i] == "last-delivered-id" then last = info[i+1] end - end - if name == group then - return last - end - end - return false + // ensureConsumerGroupScript creates or repairs a group, initializes its + // durable cursor, and aligns recovery retention atomically after verifying + // the exact stream generation. + ensureConsumerGroupScript = redis.NewScript(` +if redis.call("HGET", KEYS[1], "state") ~= ARGV[1] +or redis.call("HGET", KEYS[1], "generation") ~= ARGV[2] +or redis.call("HGET", KEYS[1], ARGV[7]) ~= KEYS[2] then + return redis.error_reply("STREAMDESTROYED") end -` - -// membershipRemoveLua defines membership_remove(content, channel, field, -// value) which removes value from the JSON-array field of an rmap content -// hash and publishes the matching rmap protocol notification. -const membershipRemoveLua = ` -local function membership_remove(content, channel, field, value) - local v = redis.call("HGET", content, field) - if not v then - return - end - local values = {} - local ok, decoded = pcall(cjson.decode, v) - if ok and type(decoded) == "table" then - values = decoded - else - for s in string.gmatch(v, "[^,]+") do - table.insert(values, s) - end - end - local remaining = {} - local removed = false - for _, item in ipairs(values) do - if item == value then - removed = true - else - table.insert(remaining, item) - end - end - if not removed then - return - end - if #remaining == 0 then - redis.call("HDEL", content, field) - local rev = tostring(redis.call("HINCRBY", content, "=rev", 1)) - redis.call("HSET", content, "=kind", "del") - local msg = struct.pack("ic0ic0", string.len(field), field, string.len(rev), rev) - redis.call("PUBLISH", channel, "del:" .. msg) - return - end - local encoded = cjson.encode(remaining) - redis.call("HSET", content, field, encoded) - local rev = tostring(redis.call("HINCRBY", content, "=rev", 1)) - redis.call("HSET", content, "=kind", "set") - local msg = struct.pack("ic0ic0ic0", string.len(field), field, string.len(encoded), encoded, string.len(rev), rev) - redis.call("PUBLISH", channel, "set:" .. msg) +local deadline = redis.call("HGET", KEYS[1], ARGV[8]) +if deadline then + local now = redis.call("TIME") + local now_ms = tonumber(now[1]) * 1000 + math.floor(tonumber(now[2]) / 1000) + if now_ms >= tonumber(deadline) then + return redis.error_reply("DEADLINEELAPSED") + end end -` -// ensureConsumerGroupScript creates or repairs the consumer group for a sink -// behind the lifecycle fence and restores the stream TTL even when the group -// already exists (BUSYGROUP). The group is created at the durable recovery -// cursor when one exists, at the caller start ID otherwise, and the cursor is -// recomputed and stored from live group state so it is always defined. -// -// KEYS: [1]=lifecycle [2]=stream [3]=cursors -// ARGV: [1]=group [2]=startID [3]=establish [4]=ttlMs [5]=ttlSliding -// Returns {created(0|1), cursor}. -var ensureConsumerGroupScript = redis.NewScript(` -local establish = ARGV[3] -` + lifecycleFenceLua + recoveryCursorLua + ` -local start = redis.call("HGET", KEYS[3], ARGV[1]) +local absent = redis.call("EXISTS", KEYS[2]) == 0 +local start = redis.call("HGET", KEYS[3], ARGV[3]) if not start then - start = ARGV[2] + start = ARGV[4] end -local created = 1 -local res = redis.pcall("XGROUP", "CREATE", KEYS[2], ARGV[1], start, "MKSTREAM") -if type(res) == "table" and res["err"] then - if not string.find(res["err"], "BUSYGROUP", 1, true) then - return res - end - created = 0 +local created = redis.pcall("XGROUP", "CREATE", KEYS[2], ARGV[3], start, "MKSTREAM") +if type(created) == "table" and created.err +and not string.find(created.err, "BUSYGROUP", 1, true) then + return redis.error_reply(created.err) end -local cursor = recovery_cursor(KEYS[2], ARGV[1]) -redis.call("HSET", KEYS[3], ARGV[1], cursor) -if tonumber(ARGV[4]) > 0 then - if ARGV[5] == "1" then - redis.call("PEXPIRE", KEYS[2], ARGV[4]) - else - redis.call("PEXPIRE", KEYS[2], ARGV[4], "NX") - end + +if redis.call("HEXISTS", KEYS[3], ARGV[3]) == 0 then + local pending = redis.call("XPENDING", KEYS[2], ARGV[3]) + local cursor + if pending[1] == 0 then + local groups = redis.call("XINFO", "GROUPS", KEYS[2]) + for _, group in ipairs(groups) do + local name + local delivered + for i = 1, #group, 2 do + if group[i] == "name" then + name = group[i + 1] + elseif group[i] == "last-delivered-id" then + delivered = group[i + 1] + end + end + if name == ARGV[3] then + cursor = delivered + break + end + end + if not cursor then + return redis.error_reply("NOGROUP consumer group no longer exists") + end + else + local previous = redis.call("XREVRANGE", KEYS[2], "(" .. pending[2], "-", "COUNT", 1) + if #previous == 0 then + cursor = "0-0" + else + cursor = previous[1][1] + end + end + redis.call("HSET", KEYS[3], ARGV[3], cursor) end -return {created, cursor} -`) -// ackEventsScript acknowledges events and atomically advances the recovery -// cursor from the exact PEL state. Acking events of a destroyed stream or of -// a deleted group is a no-op returning 0, matching XACK semantics on missing -// keys. -// -// KEYS: [1]=lifecycle [2]=stream [3]=cursors -// ARGV: [1]=group [2..]=event IDs -var ackEventsScript = redis.NewScript(` -local state = redis.call("HGET", KEYS[1], "state") -if state ~= "active" then - return 0 +local ttl = tonumber(ARGV[5]) +if deadline then + redis.call("PEXPIREAT", KEYS[2], deadline) + redis.call("PEXPIREAT", KEYS[3], deadline) +elseif ttl > 0 and (ARGV[6] == "1" or redis.call("PTTL", KEYS[2]) == -1) then + redis.call("PEXPIRE", KEYS[2], ttl) end -` + recoveryCursorLua + ` -local acked = redis.call("XACK", KEYS[2], ARGV[1], unpack(ARGV, 2)) -local cursor = recovery_cursor(KEYS[2], ARGV[1]) -if cursor then - redis.call("HSET", KEYS[3], ARGV[1], cursor) +if absent then + return 1 end -return acked +return 0 `) -// registerConsumerScript creates a Redis consumer in the sink group and -// appends it to the stream membership map behind the lifecycle fence, in one -// atomic operation so membership and consumer-group state cannot diverge. -// -// KEYS: [1]=lifecycle [2]=stream [3]=membership content [4]=membership channel -// ARGV: [1]=group [2]=consumer -var registerConsumerScript = redis.NewScript(` -local state = redis.call("HGET", KEYS[1], "state") -if state ~= "active" then - return redis.error_reply("STREAMDESTROYED stream was destroyed") + // recoveryCursorScript acknowledges one or more IDs and advances the + // canonical cursor only across the prefix with no pending entry. + recoveryCursorScript = redis.NewScript(` +if redis.call("HGET", KEYS[1], "state") ~= ARGV[1] +or redis.call("HGET", KEYS[1], "generation") ~= ARGV[2] +or redis.call("HGET", KEYS[1], ARGV[4]) ~= KEYS[2] then + return redis.error_reply("STREAMDESTROYED") end -local res = redis.pcall("XGROUP", "CREATECONSUMER", KEYS[2], ARGV[1], ARGV[2]) -if type(res) == "table" and res["err"] then - return res +local deadline = redis.call("HGET", KEYS[1], ARGV[5]) +if deadline then + local now = redis.call("TIME") + local now_ms = tonumber(now[1]) * 1000 + math.floor(tonumber(now[2]) / 1000) + if now_ms >= tonumber(deadline) then + return redis.error_reply("DEADLINEELAPSED") + end end -local field = ARGV[1] -local v = redis.call("HGET", KEYS[3], field) -local values = {} -if v then - local ok, decoded = pcall(cjson.decode, v) - if ok and type(decoded) == "table" then - values = decoded - else - for s in string.gmatch(v, "[^,]+") do - table.insert(values, s) - end - end -end -for _, item in ipairs(values) do - if item == ARGV[2] then - return 1 - end + +local result = redis.call("XACK", KEYS[2], ARGV[3], unpack(ARGV, 6)) +if result == 0 then + return 0 end -table.insert(values, ARGV[2]) -local encoded = cjson.encode(values) -redis.call("HSET", KEYS[3], field, encoded) -local rev = tostring(redis.call("HINCRBY", KEYS[3], "=rev", 1)) -redis.call("HSET", KEYS[3], "=kind", "set") -local msg = struct.pack("ic0ic0ic0", string.len(field), field, string.len(encoded), encoded, string.len(rev), rev) -redis.call("PUBLISH", KEYS[4], "set:" .. msg) -return 1 -`) -// detachConsumerScript rolls back a consumer registration: it removes the -// consumer from the membership map and deletes the Redis consumer when its -// PEL is empty. Used to compensate partial consumer rotation. A destroyed -// stream has no metadata left to detach so the script is then a no-op. -// -// KEYS: [1]=lifecycle [2]=stream [3]=membership content [4]=membership channel -// ARGV: [1]=group [2]=consumer -var detachConsumerScript = redis.NewScript(` -local state = redis.call("HGET", KEYS[1], "state") -if state ~= "active" then - return 0 +local pending = redis.call("XPENDING", KEYS[2], ARGV[3]) +local cursor +if pending[1] == 0 then + local groups = redis.call("XINFO", "GROUPS", KEYS[2]) + for _, group in ipairs(groups) do + local name + local delivered + for i = 1, #group, 2 do + if group[i] == "name" then + name = group[i + 1] + elseif group[i] == "last-delivered-id" then + delivered = group[i + 1] + end + end + if name == ARGV[3] then + cursor = delivered + break + end + end + if not cursor then + return redis.error_reply("NOGROUP consumer group no longer exists") + end +else + local previous = redis.call("XREVRANGE", KEYS[2], "(" .. pending[2], "-", "COUNT", 1) + if #previous == 0 then + cursor = "0-0" + else + cursor = previous[1][1] + end end -` + membershipRemoveLua + ` -local pending = redis.pcall("XPENDING", KEYS[2], ARGV[1], "-", "+", 1, ARGV[2]) -if not pending["err"] and #pending == 0 then - redis.call("XGROUP", "DELCONSUMER", KEYS[2], ARGV[1], ARGV[2]) +redis.call("HSET", KEYS[3], ARGV[3], cursor) +if deadline then + redis.call("PEXPIREAT", KEYS[3], deadline) end -membership_remove(KEYS[3], KEYS[4], ARGV[1], ARGV[2]) -return 1 +return result `) -// removeSinkStreamScript atomically removes a sink consumer from the stream -// membership map and, when it was the last member, destroys the consumer -// group, its recovery cursor, and its lease. Being a single script there is -// no partial failure between membership and group state to compensate. -// -// KEYS: [1]=lifecycle [2]=stream [3]=cursors [4]=membership content -// [5]=membership channel [6]=lease -// ARGV: [1]=group [2]=consumer -var removeSinkStreamScript = redis.NewScript(` + // destroyStreamScript invalidates and deletes exactly one generation. The + // lifecycle record remains as the monotonic source used by the next + // explicit NewStream call. The generation-qualified membership map is never + // reused, so its destroy notification needs no persistent rmap tombstone. + destroyStreamScript = redis.NewScript(` +if redis.call("HGET", KEYS[1], "generation") ~= ARGV[2] then + return redis.error_reply("STREAMDESTROYED") +end local state = redis.call("HGET", KEYS[1], "state") -if state ~= "active" then - return 1 +if state == ARGV[3] then + return 0 end -` + membershipRemoveLua + ` -membership_remove(KEYS[4], KEYS[5], ARGV[1], ARGV[2]) -if not redis.call("HGET", KEYS[4], ARGV[1]) then - redis.call("XGROUP", "DESTROY", KEYS[2], ARGV[1]) - redis.call("HDEL", KEYS[3], ARGV[1]) - redis.call("DEL", KEYS[6]) +if state ~= ARGV[1] then + return redis.error_reply("STREAMDESTROYED") +end +if redis.call("HGET", KEYS[1], ARGV[4]) ~= KEYS[2] then + return redis.error_reply("STREAMDESTROYED") end -return 1 -`) -// destroyStreamScript marks the stream destroyed and deletes every key owned -// by the stream in one atomic operation: events, recovery cursors, sink -// leases (enumerated from the membership map fields), and the membership map -// itself via the rmap destroy protocol. The lifecycle hash is kept as the -// destroyed tombstone that fences concurrent sink metadata writes. -// -// KEYS: [1]=lifecycle [2]=stream [3]=cursors [4]=membership content -// [5]=membership channel -// ARGV: [1]=lease key prefix (lease key is prefix .. sink .. ":lease") -var destroyStreamScript = redis.NewScript(` -redis.call("HSET", KEYS[1], "state", "destroyed") -redis.call("DEL", KEYS[2], KEYS[3]) -if redis.call("EXISTS", KEYS[4]) == 1 then - for _, field in ipairs(redis.call("HKEYS", KEYS[4])) do - if field ~= "=rev" and field ~= "=kind" then - redis.call("DEL", ARGV[1] .. field .. ":lease") - end - end - local rev = redis.call("HINCRBY", KEYS[4], "=rev", 1) - redis.call("DEL", KEYS[4]) - redis.call("HSET", KEYS[4], "=rev", rev, "=kind", "destroy") - redis.call("PUBLISH", KEYS[5], "destroy:" .. tostring(rev)) +redis.call("HSET", KEYS[1], "state", ARGV[3]) +local rev = redis.call("HINCRBY", KEYS[4], "=rev", 1) +local resources = redis.call("SMEMBERS", KEYS[6]) +if #resources > 0 then + redis.call("DEL", unpack(resources)) end +redis.call("DEL", KEYS[2], KEYS[3], KEYS[4]) +redis.call("DEL", KEYS[6]) +redis.call("PUBLISH", KEYS[5], "destroy:" .. tostring(rev)) return 1 `) +) -// ensureConsumerGroup creates or repairs the consumer group for sinkName on -// stream at the durable recovery cursor (startID for brand new groups) and -// restores the stream TTL. establish reactivates a destroyed stream and is -// reserved for deliberate NewSink/AddStream calls; recovery paths pass false -// so a destroyed stream fails with ErrStreamDestroyed instead of being -// resurrected. Returns whether the group was created and the stored cursor. -func ensureConsumerGroup(ctx context.Context, stream *Stream, sinkName, startID string, establish bool) (bool, string, error) { - keys := []string{lifecycleKey(stream.key), stream.key, cursorsKey(stream.key)} - args := []any{sinkName, startID, boolArg(establish), stream.ttl.Milliseconds(), boolArg(stream.ttlSliding)} - res, err := ensureConsumerGroupScript.Run(ctx, stream.rdb, keys, args...).Slice() - if err != nil { - if isStreamDestroyedErr(err) { - return false, "", fmt.Errorf("cannot create consumer group %s for stream %s: %w", sinkName, stream.Name, ErrStreamDestroyed) - } - return false, "", fmt.Errorf("failed to create consumer group %s for stream %s: %w", sinkName, stream.Name, err) - } - return res[0].(int64) == 1, res[1].(string), nil -} - -// registerSinkConsumer atomically creates the Redis consumer for sinkName in -// the stream consumer group and records it in the stream membership map. -func registerSinkConsumer(ctx context.Context, stream *Stream, sinkName, consumer string) error { - keys := []string{ - lifecycleKey(stream.key), - stream.key, - membershipContentKey(stream.Name), - membershipChannelKey(stream.Name), - } - if err := registerConsumerScript.Run(ctx, stream.rdb, keys, sinkName, consumer).Err(); err != nil { - if isStreamDestroyedErr(err) { - return fmt.Errorf("cannot register consumer %s for stream %s: %w", consumer, stream.Name, ErrStreamDestroyed) - } - return fmt.Errorf("failed to register consumer %s for stream %s: %w", consumer, stream.Name, err) - } - return nil -} - -// detachSinkConsumer compensates a partial consumer rotation by removing the -// consumer from the stream membership map and deleting the Redis consumer -// when it has no pending events. -func detachSinkConsumer(ctx context.Context, stream *Stream, sinkName, consumer string) error { - keys := []string{ - lifecycleKey(stream.key), - stream.key, - membershipContentKey(stream.Name), - membershipChannelKey(stream.Name), - } - if err := detachConsumerScript.Run(ctx, stream.rdb, keys, sinkName, consumer).Err(); err != nil { - return fmt.Errorf("failed to detach consumer %s from stream %s: %w", consumer, stream.Name, err) - } - return nil -} - -// removeSinkStream removes the sink consumer from the stream membership map -// and destroys the consumer group, recovery cursor, and lease when the -// consumer was the last member, all in one atomic operation. -func removeSinkStream(ctx context.Context, stream *Stream, sinkName, consumer string) error { - keys := []string{ - lifecycleKey(stream.key), - stream.key, - cursorsKey(stream.key), - membershipContentKey(stream.Name), - membershipChannelKey(stream.Name), - leaseKey(stream.key, sinkName), - } - if err := removeSinkStreamScript.Run(ctx, stream.rdb, keys, sinkName, consumer).Err(); err != nil { - return fmt.Errorf("failed to remove sink %s from stream %s: %w", sinkName, stream.Name, err) +// ensureConsumerGroup creates or repairs a group for this exact generation. +// The boolean result reports whether its physical stream data was absent. +func ensureConsumerGroup(ctx context.Context, stream *Stream, group, configuredStart string) (bool, error) { + if err := stream.ensureGeneration(ctx); err != nil { + return false, err } - return nil -} - -// destroyStream atomically marks the stream destroyed and deletes its events -// and sink metadata. It is idempotent and safe to call on streams that were -// never created. -func destroyStream(ctx context.Context, stream *Stream) error { - keys := []string{ - lifecycleKey(stream.key), - stream.key, - cursorsKey(stream.key), - membershipContentKey(stream.Name), - membershipChannelKey(stream.Name), + absent, err := ensureConsumerGroupScript.Run( + ctx, + stream.rdb, + []string{stream.lifecycleKey, stream.key, recoveryCursorKey(stream)}, + streamStateActive, + stream.generation, + group, + configuredStart, + strconv.FormatInt(stream.ttl.Milliseconds(), 10), + boolString(stream.ttlSliding), + streamPhysicalKey, + streamDeadlineKey, + ).Int64() + if err != nil { + return false, fmt.Errorf( + "failed to ensure Redis consumer group %q for stream %q generation %s: %w", + group, + stream.Name, + stream.generation, + stream.lifecycleError(err), + ) } - return destroyStreamScript.Run(ctx, stream.rdb, keys, leaseKeyPrefix(stream.key)).Err() + return absent == 1, nil } -// XAck acknowledges the events and advances the sink recovery cursor from the -// exact PEL state in one atomic operation. It satisfies the Acker interface -// so acknowledging a sink event through Event.Acker keeps recovery lossless. -func (a *recoveryAcker) XAck(ctx context.Context, streamKey, sinkName string, ids ...string) *redis.IntCmd { +// XAck atomically acknowledges IDs and advances this generation's shared +// recovery cursor. The streamKey argument remains part of the public Acker +// contract; generated events bind the implementation to their own stream. +func (a *recoveryAcker) XAck(ctx context.Context, _ string, group string, ids ...string) *redis.IntCmd { cmd := redis.NewIntCmd(ctx) - keys := []string{lifecycleKey(streamKey), streamKey, cursorsKey(streamKey)} - args := make([]any, 0, len(ids)+1) - args = append(args, sinkName) + args := make([]any, 0, len(ids)+5) + args = append( + args, + streamStateActive, + a.stream.generation, + group, + streamPhysicalKey, + streamDeadlineKey, + ) for _, id := range ids { args = append(args, id) } - acked, err := ackEventsScript.Run(ctx, a.rdb, keys, args...).Int64() + result, err := recoveryCursorScript.Run( + ctx, + a.stream.rdb, + []string{a.stream.lifecycleKey, a.stream.key, recoveryCursorKey(a.stream)}, + args..., + ).Int64() if err != nil { - cmd.SetErr(err) + cmd.SetErr(fmt.Errorf( + "failed to acknowledge events %q and advance consumer group %q on stream %q generation %s: %w", + ids, + group, + a.stream.Name, + a.stream.generation, + a.stream.lifecycleError(err), + )) return cmd } - cmd.SetVal(acked) + cmd.SetVal(result) return cmd } -// streamMetaPrefix returns the metadata key prefix for the stream with the -// given event key. Metadata lives under a prefix distinct from the event key -// so stream names cannot collide with metadata key suffixes. -func streamMetaPrefix(streamKey string) string { - return "pulse:streammeta:" + streamKey[len(streamKeyPrefix):] -} - -// lifecycleKey returns the lifecycle fence key for the stream event key. -func lifecycleKey(streamKey string) string { - return streamMetaPrefix(streamKey) + ":lifecycle" -} - -// cursorsKey returns the recovery cursor hash key for the stream event key. -func cursorsKey(streamKey string) string { - return streamMetaPrefix(streamKey) + ":cursors" -} - -// leaseKeyPrefix returns the prefix of per-sink lease keys for the stream. -func leaseKeyPrefix(streamKey string) string { - return streamMetaPrefix(streamKey) + ":sink:" -} - -// leaseKey returns the fenced lease key for the sink on the stream. -func leaseKey(streamKey, sinkName string) string { - return leaseKeyPrefix(streamKey) + sinkName + ":lease" -} - -// membershipContentKey returns the rmap content key of the stream sink -// membership map (see consumersMapName). -func membershipContentKey(streamName string) string { - return fmt.Sprintf("map:stream:%s:sinks:content", streamName) +// destroyStream atomically invalidates and deletes this exact generation. +func destroyStream(ctx context.Context, stream *Stream) error { + if err := stream.loadExistingGeneration(ctx); err != nil { + return err + } + err := destroyStreamScript.Run( + ctx, + stream.rdb, + []string{ + stream.lifecycleKey, + stream.key, + recoveryCursorKey(stream), + consumersMapContentKey(stream), + consumersMapChannelKey(stream), + streamResourceRegistryKey(stream), + }, + streamStateActive, + stream.generation, + streamStateDestroyed, + streamPhysicalKey, + ).Err() + if err != nil { + return fmt.Errorf( + "failed to destroy stream %q generation %s: %w", + stream.Name, + stream.generation, + stream.lifecycleError(err), + ) + } + return nil } -// membershipChannelKey returns the rmap pubsub channel of the stream sink -// membership map. -func membershipChannelKey(streamName string) string { - return fmt.Sprintf("map:stream:%s:sinks:updates", streamName) +// recoveryCursorKey identifies all named-sink cursors for one generation while +// the physical event key remains stable. +func recoveryCursorKey(stream *Stream) string { + return fmt.Sprintf("%s:sink-recovery:%s", stream.key, stream.generation) } -// isStreamDestroyedErr reports whether err is the fenced-script rejection of -// a write against a destroyed stream. -func isStreamDestroyedErr(err error) bool { - return err != nil && strings.HasPrefix(err.Error(), streamDestroyedErrorPrefix) +// consumersMapContentKey identifies this generation's membership hash. +func consumersMapContentKey(stream *Stream) string { + return fmt.Sprintf("map:%s:content", consumersMapName(stream)) } -// boolArg encodes a boolean as the "0"/"1" convention used by the scripts. -func boolArg(b bool) string { - if b { - return "1" - } - return "0" +// consumersMapChannelKey identifies this generation's membership updates. +func consumersMapChannelKey(stream *Stream) string { + return rmapChannelKey(consumersMapName(stream)) } diff --git a/streaming/sink_recovery_test.go b/streaming/sink_recovery_test.go index d12dcaf..0ac71bc 100644 --- a/streaming/sink_recovery_test.go +++ b/streaming/sink_recovery_test.go @@ -1,17 +1,11 @@ -// Tests for lossless consumer group recovery, the destroy lifecycle fence, -// fenced leases, close semantics, and failure-atomic stream ownership -// changes. All tests run against a live Redis instance. +// Package streaming tests consumer-group recovery against real Redis. The +// cases focus on delivery safety, shared replica state, lifecycle boundaries, +// and compensation when a multi-step ownership change fails. package streaming import ( "context" - "encoding/json" "errors" - "fmt" - "net" - "strings" - "sync" - "sync/atomic" "testing" "time" @@ -20,706 +14,730 @@ import ( "github.com/stretchr/testify/require" "goa.design/pulse/pulse" + "goa.design/pulse/rmap" "goa.design/pulse/streaming/options" ptesting "goa.design/pulse/testing" ) -// TestSinkRecoversGroupAfterExternalDestroy is the core lossless recovery -// scenario: publish events, ack some, force XGROUP DESTROY, and verify the -// group is recreated at the recovery cursor so every unacked event is -// redelivered exactly once and no acked event is redelivered. -func TestSinkRecoversGroupAfterExternalDestroy(t *testing.T) { - testName := strings.Replace(t.Name(), "/", "_", -1) +type ( + // recordingAcker proves Sink.Ack preserves Event's public acknowledgement + // boundary instead of bypassing it with the sink's Redis client. + recordingAcker struct { + streamKey string + group string + ids []string + } +) + +func TestSinkAdoptsFlatStreamWithQueuedAndPendingEvents(t *testing.T) { rdb := ptesting.NewRedisClient(t) - defer ptesting.CleanupRedis(t, rdb, true, testName) + defer ptesting.CleanupRedis(t, rdb, false, "") ctx := ptesting.NewTestContext(t) - s, err := NewStream(testName, rdb, options.WithStreamLogger(pulse.ClueLogger(ctx))) - require.NoError(t, err) - sink, err := s.NewSink(ctx, "sink", + key := streamKey(t.Name()) + + firstID, err := rdb.XAdd(ctx, &redis.XAddArgs{ + Stream: key, + Values: map[string]any{nameKey: "first", payloadKey: "one"}, + }).Result() + require.NoError(t, err) + secondID, err := rdb.XAdd(ctx, &redis.XAddArgs{ + Stream: key, + Values: map[string]any{nameKey: "second", payloadKey: "two"}, + }).Result() + require.NoError(t, err) + require.NoError(t, rdb.XGroupCreate(ctx, key, "sink", "0").Err()) + legacy, err := rdb.XReadGroup(ctx, &redis.XReadGroupArgs{ + Group: "sink", + Consumer: "legacy", + Streams: []string{key, ">"}, + Count: 1, + }).Result() + require.NoError(t, err) + require.Equal(t, firstID, legacy[0].Messages[0].ID) + + stream, err := NewStream(t.Name(), rdb) + require.NoError(t, err) + require.Empty(t, stream.Generation()) + sink, err := stream.NewSink( + ctx, + "sink", options.WithSinkStartAtOldest(), - options.WithSinkBlockDuration(testBlockDuration)) + options.WithSinkAckGracePeriod(50*time.Millisecond), + options.WithSinkBlockDuration(testBlockDuration), + ) require.NoError(t, err) - defer cleanupSink(t, ctx, s, sink) + events := sink.Subscribe() - c := sink.Subscribe() - ids := make([]string, 5) - for i := range ids { - ids[i], err = s.Add(ctx, fmt.Sprintf("event%d", i), []byte("payload")) - require.NoError(t, err) - } - events := make([]*Event, 5) - for i := range events { - events[i] = receiveEvent(t, c) - } - require.NoError(t, sink.Ack(ctx, events[0])) - require.NoError(t, sink.Ack(ctx, events[1])) - // Acks advance the recovery cursor synchronously. - assert.Equal(t, ids[1], recoveryCursor(t, ctx, rdb, s, "sink")) - - // Simulate Redis consumer group state loss. - require.NoError(t, rdb.XGroupDestroy(ctx, s.key, "sink").Err()) - futureID, err := s.Add(ctx, "future", []byte("payload")) - require.NoError(t, err) - - // Every unacked event and the new event must be redelivered exactly once. - var redelivered []string - for range 4 { - ev := receiveEvent(t, c) - redelivered = append(redelivered, ev.ID) - require.NoError(t, sink.Ack(ctx, ev)) - } - assert.Equal(t, []string{ids[2], ids[3], ids[4], futureID}, redelivered) - select { - case ev := <-c: - t.Errorf("unexpected redelivery of event %s", ev.ID) - case <-time.After(4 * testBlockDuration): + received := map[string]*Event{} + for len(received) < 2 { + select { + case event := <-events: + received[event.ID] = event + case <-time.After(max): + t.Fatalf("timed out waiting for upgraded events; received %v", received) + } } - groups, err := rdb.XInfoGroups(ctx, s.key).Result() - require.NoError(t, err) - require.Len(t, groups, 1) - assert.Equal(t, "sink", groups[0].Name) + require.Contains(t, received, firstID) + require.Contains(t, received, secondID) + pending, err := rdb.XPending(ctx, key, "sink").Result() + require.NoError(t, err) + require.EqualValues(t, 2, pending.Count) + require.Equal(t, "1", stream.Generation()) + require.Equal(t, key, stream.key) + require.Equal(t, key, rdb.HGet(ctx, stream.lifecycleKey, streamPhysicalKey).Val()) + + require.NoError(t, sink.Ack(ctx, received[firstID])) + require.NoError(t, sink.Ack(ctx, received[secondID])) + require.NoError(t, sink.Close(ctx)) + require.NoError(t, stream.Destroy(ctx)) } -// TestSinkRecoveryCursorIsSharedAcrossReplicas verifies that sink replicas -// share one durable recovery cursor per stream and group so whichever replica -// recovers first resumes from the same acknowledged position. -func TestSinkRecoveryCursorIsSharedAcrossReplicas(t *testing.T) { - testName := strings.Replace(t.Name(), "/", "_", -1) +func TestSinkAcknowledgesFilteredEvents(t *testing.T) { rdb := ptesting.NewRedisClient(t) - defer ptesting.CleanupRedis(t, rdb, true, testName) + defer ptesting.CleanupRedis(t, rdb, false, "") ctx := ptesting.NewTestContext(t) - s, err := NewStream(testName, rdb, options.WithStreamLogger(pulse.ClueLogger(ctx))) + stream, err := NewStream(t.Name(), rdb) require.NoError(t, err) - sink1, err := s.NewSink(ctx, "sink", + sink, err := stream.NewSink( + ctx, + "sink", options.WithSinkStartAtOldest(), - options.WithSinkBlockDuration(testBlockDuration)) - require.NoError(t, err) - defer cleanupSink(t, ctx, s, sink1) - sink2, err := s.NewSink(ctx, "sink", - options.WithSinkStartAtOldest(), - options.WithSinkBlockDuration(testBlockDuration)) + options.WithSinkTopic("wanted"), + options.WithSinkBlockDuration(testBlockDuration), + ) require.NoError(t, err) - defer sink2.Close(ctx) + events := sink.Subscribe() - c1, c2 := sink1.Subscribe(), sink2.Subscribe() - ids := make([]string, 2) - for i := range ids { - ids[i], err = s.Add(ctx, fmt.Sprintf("event%d", i), []byte("payload")) - require.NoError(t, err) - } - for range ids { - select { - case ev := <-c1: - require.NoError(t, sink1.Ack(ctx, ev)) - case ev := <-c2: - require.NoError(t, sink2.Ack(ctx, ev)) - case <-time.After(max): - t.Fatal("timeout waiting for event") - } - } - assert.Equal(t, ids[1], recoveryCursor(t, ctx, rdb, s, "sink")) + filteredID, err := stream.Add(ctx, "filtered", []byte("filtered"), options.WithTopic("other")) + require.NoError(t, err) + wantedID, err := stream.Add(ctx, "wanted", []byte("wanted"), options.WithTopic("wanted")) + require.NoError(t, err) + wanted := receiveSinkEvent(t, events) + require.Equal(t, wantedID, wanted.ID) - require.NoError(t, rdb.XGroupDestroy(ctx, s.key, "sink").Err()) - futureID, err := s.Add(ctx, "future", []byte("payload")) + require.Eventually(t, func() bool { + pending, err := rdb.XPending(ctx, stream.key, sink.Name).Result() + return err == nil && pending.Count == 1 && pending.Lower == wantedID + }, max, delay) + cursor, err := rdb.HGet(ctx, recoveryCursorKey(stream), sink.Name).Result() require.NoError(t, err) + require.Equal(t, filteredID, cursor) - // Whichever replica recovers, only the new event is delivered. - select { - case ev := <-c1: - assert.Equal(t, futureID, ev.ID) - require.NoError(t, sink1.Ack(ctx, ev)) - case ev := <-c2: - assert.Equal(t, futureID, ev.ID) - require.NoError(t, sink2.Ack(ctx, ev)) - case <-time.After(max): - t.Fatal("timeout waiting for post-recovery event") - } - select { - case ev := <-c1: - t.Errorf("unexpected redelivery of event %s", ev.ID) - case ev := <-c2: - t.Errorf("unexpected redelivery of event %s", ev.ID) - case <-time.After(4 * testBlockDuration): - } + require.NoError(t, sink.Ack(ctx, wanted)) + require.NoError(t, sink.Close(ctx)) + require.NoError(t, stream.Destroy(ctx)) } -// TestEnsureGroupRestoresTTLOnBusyGroup verifies that ensuring an existing -// consumer group (BUSYGROUP path) restores the stream TTL and that recovery -// metadata itself never carries a TTL. -func TestEnsureGroupRestoresTTLOnBusyGroup(t *testing.T) { - testName := strings.Replace(t.Name(), "/", "_", -1) +func TestSinkRecoveryPreservesPendingAndGapEvents(t *testing.T) { rdb := ptesting.NewRedisClient(t) - defer ptesting.CleanupRedis(t, rdb, true, testName) + defer ptesting.CleanupRedis(t, rdb, false, "") + hook := &redisCommandHook{} + rdb.AddHook(hook) ctx := ptesting.NewTestContext(t) - s, err := NewStream(testName, rdb, - options.WithStreamLogger(pulse.ClueLogger(ctx)), - options.WithStreamTTL(time.Hour)) + stream, err := NewStream(t.Name(), rdb, options.WithStreamLogger(pulse.ClueLogger(ctx))) require.NoError(t, err) - sink, err := s.NewSink(ctx, "sink", + sink, err := stream.NewSink( + ctx, + "sink", options.WithSinkStartAtOldest(), - options.WithSinkBlockDuration(testBlockDuration)) + options.WithSinkBlockDuration(testBlockDuration), + ) require.NoError(t, err) - defer cleanupSink(t, ctx, s, sink) + defer cleanupSink(t, ctx, stream, sink) + events := sink.Subscribe() - _, err = s.Add(ctx, "event", []byte("payload")) - require.NoError(t, err) - require.NoError(t, rdb.Persist(ctx, s.key).Err()) - require.Equal(t, time.Duration(-1), rdb.PTTL(ctx, s.key).Val()) + for _, name := range []string{"first", "pending", "out-of-order"} { + _, err := stream.Add(ctx, name, []byte(name)) + require.NoError(t, err) + } + first := receiveSinkEvent(t, events) + pending := receiveSinkEvent(t, events) + outOfOrder := receiveSinkEvent(t, events) + require.NoError(t, sink.Ack(ctx, first)) + require.NoError(t, sink.Ack(ctx, outOfOrder)) - created, _, err := ensureConsumerGroup(ctx, s, "sink", "$", false) + cursor, err := rdb.HGet(ctx, recoveryCursorKey(stream), sink.Name).Result() require.NoError(t, err) - assert.False(t, created, "group must already exist") - assert.Positive(t, rdb.PTTL(ctx, s.key).Val(), "stream TTL must be restored on BUSYGROUP") - assert.Equal(t, time.Duration(-1), rdb.PTTL(ctx, cursorsKey(s.key)).Val(), - "recovery cursors must outlive the event TTL") + require.Equal(t, first.ID, cursor) + + readsBeforeDestroy := hook.xreadGroups.Load() + require.Eventually(t, func() bool { return hook.xreadGroups.Load() > readsBeforeDestroy }, max, delay) + sink.lock.Lock() + destroyed, destroyErr := rdb.XGroupDestroy(ctx, stream.key, sink.Name).Result() + _, gapErr := stream.Add(ctx, "gap", []byte("gap")) + sink.lock.Unlock() + require.NoError(t, destroyErr) + require.EqualValues(t, 1, destroyed) + require.NoError(t, gapErr) + + recovered := []*Event{ + receiveSinkEvent(t, events), + receiveSinkEvent(t, events), + receiveSinkEvent(t, events), + } + assert.Equal(t, []string{"pending", "out-of-order", "gap"}, eventNames(recovered)) + assert.Equal(t, pending.ID, recovered[0].ID) + for _, event := range recovered { + require.NoError(t, sink.Ack(ctx, event)) + } } -// TestSinkRecoveryAfterEventStreamExpires verifies that when the Redis key -// backing a TTL stream expires (deleting the consumer group with it), the -// sink recreates the group from the durable recovery cursor and delivers -// events published afterwards. -func TestSinkRecoveryAfterEventStreamExpires(t *testing.T) { - testName := strings.Replace(t.Name(), "/", "_", -1) +func TestSinkRecoveryCursorIsSharedAcrossReplicas(t *testing.T) { rdb := ptesting.NewRedisClient(t) - defer ptesting.CleanupRedis(t, rdb, true, testName) + defer ptesting.CleanupRedis(t, rdb, false, "") ctx := ptesting.NewTestContext(t) - s, err := NewStream(testName, rdb, - options.WithStreamLogger(pulse.ClueLogger(ctx)), - options.WithStreamTTL(500*time.Millisecond)) + stream, err := NewStream(t.Name(), rdb, options.WithStreamLogger(pulse.ClueLogger(ctx))) require.NoError(t, err) - sink, err := s.NewSink(ctx, "sink", - options.WithSinkStartAtOldest(), - options.WithSinkBlockDuration(testBlockDuration)) + sink1, err := stream.NewSink(ctx, "sink", options.WithSinkStartAtOldest(), options.WithSinkBlockDuration(testBlockDuration)) + require.NoError(t, err) + sink2, err := stream.NewSink(ctx, "sink", options.WithSinkStartAtOldest(), options.WithSinkBlockDuration(testBlockDuration)) require.NoError(t, err) - defer cleanupSink(t, ctx, s, sink) + defer cleanupSink(t, ctx, stream, sink2) + events1 := sink1.Subscribe() + events2 := sink2.Subscribe() - c := sink.Subscribe() - _, err = s.Add(ctx, "event", []byte("payload")) + eventID, err := stream.Add(ctx, "shared", []byte("shared")) require.NoError(t, err) - require.NoError(t, sink.Ack(ctx, receiveEvent(t, c))) + select { + case event := <-events1: + require.NoError(t, sink1.Ack(ctx, event)) + case event := <-events2: + require.NoError(t, sink2.Ack(ctx, event)) + case <-time.After(max): + t.Fatal("timed out waiting for shared event") + } + cursor, err := rdb.HGet(ctx, recoveryCursorKey(stream), sink1.Name).Result() + require.NoError(t, err) + require.Equal(t, eventID, cursor) - // Wait for the stream key (and with it the consumer group) to expire. + require.NoError(t, sink1.Close(ctx)) + require.True(t, sink1.IsClosed()) + destroyed, err := rdb.XGroupDestroy(ctx, stream.key, sink2.Name).Result() + require.NoError(t, err) + require.EqualValues(t, 1, destroyed) require.Eventually(t, func() bool { - return rdb.Exists(ctx, s.key).Val() == 0 - }, 2*time.Second, delay) + return consumerGroupExists(ctx, rdb, stream.key, sink2.Name) + }, max, delay) - futureID, err := s.Add(ctx, "future", []byte("payload")) + _, err = stream.Add(ctx, "future", []byte("future")) require.NoError(t, err) - ev := receiveEvent(t, c) - assert.Equal(t, futureID, ev.ID) - require.NoError(t, sink.Ack(ctx, ev)) + future := receiveSinkEvent(t, events2) + require.Equal(t, "future", future.EventName) + require.NoError(t, sink2.Ack(ctx, future)) } -// TestSinkCloseCancelsBlockingRedisRead verifies that Close cancels -// sink-owned Redis I/O so a blocked XREADGROUP cannot stall shutdown. func TestSinkCloseCancelsBlockingRedisRead(t *testing.T) { - testName := strings.Replace(t.Name(), "/", "_", -1) rdb := ptesting.NewRedisClient(t) - defer ptesting.CleanupRedis(t, rdb, true, testName) + defer ptesting.CleanupRedis(t, rdb, false, "") + hook := &redisCommandHook{ + readStarted: make(chan struct{}), + } + hook.blockRead.Store(true) + rdb.AddHook(hook) ctx := ptesting.NewTestContext(t) - s, err := NewStream(testName, rdb, options.WithStreamLogger(pulse.ClueLogger(ctx))) + stream, err := NewStream(t.Name(), rdb, options.WithStreamLogger(pulse.ClueLogger(ctx))) require.NoError(t, err) - hook := newRedisCommandHook() - rdb.AddHook(hook) - sink, err := s.NewSink(ctx, "sink", - options.WithSinkBlockDuration(time.Hour)) // would block Close without cancellation + sink, err := stream.NewSink(ctx, "sink", options.WithSinkBlockDuration(testBlockDuration)) require.NoError(t, err) - hook.blockRead.Store(true) select { case <-hook.readStarted: case <-time.After(max): - t.Fatal("read loop never blocked on XREADGROUP") + t.Fatal("sink never entered blocking Redis read") } - - done := make(chan struct{}) + closed := make(chan struct{}) go func() { - sink.Close(ctx) - close(done) + require.NoError(t, sink.Close(ctx)) + close(closed) }() select { - case <-done: - case <-time.After(2 * time.Second): - t.Fatal("Close did not cancel the blocked Redis read") + case <-closed: + case <-time.After(max): + t.Fatal("Close did not cancel blocking Redis I/O") } - assert.True(t, sink.IsClosed()) - hook.blockRead.Store(false) - require.NoError(t, s.Destroy(ctx)) + require.True(t, sink.IsClosed()) + require.NoError(t, stream.Destroy(ctx)) } -// TestSinkRejectsStreamMutationAfterClose verifies AddStream and RemoveStream -// fail with ErrSinkClosed once Close was called. -func TestSinkRejectsStreamMutationAfterClose(t *testing.T) { - testName := strings.Replace(t.Name(), "/", "_", -1) +func TestSinkCloseRetriesDistributedDetach(t *testing.T) { rdb := ptesting.NewRedisClient(t) - defer ptesting.CleanupRedis(t, rdb, true, testName) + defer ptesting.CleanupRedis(t, rdb, false, "") + failure := errors.New("injected close detach failure") + hook := &redisCommandHook{failure: failure} + rdb.AddHook(hook) ctx := ptesting.NewTestContext(t) - s, err := NewStream(testName, rdb, options.WithStreamLogger(pulse.ClueLogger(ctx))) + stream, err := NewStream(t.Name(), rdb, options.WithStreamLogger(pulse.ClueLogger(ctx))) require.NoError(t, err) - sink, err := s.NewSink(ctx, "sink", options.WithSinkBlockDuration(testBlockDuration)) + sink, err := stream.NewSink(ctx, "sink", options.WithSinkBlockDuration(testBlockDuration)) require.NoError(t, err) - sink.Close(ctx) - require.True(t, sink.IsClosed()) - s2, err := NewStream(testName+"2", rdb, options.WithStreamLogger(pulse.ClueLogger(ctx))) - require.NoError(t, err) - assert.ErrorIs(t, sink.AddStream(ctx, s2), ErrSinkClosed) - assert.ErrorIs(t, sink.RemoveStream(ctx, s), ErrSinkClosed) - require.NoError(t, s.Destroy(ctx)) + hook.failMembershipKey = consumersMapContentKey(stream) + hook.failRemove.Store(true) + err = sink.Close(ctx) + require.ErrorIs(t, err, failure) + require.False(t, sink.IsClosed()) + require.Contains(t, sink.streams, stream.key) + hook.failRemove.Store(false) + + require.NoError(t, sink.Close(ctx)) + require.True(t, sink.IsClosed()) + require.Empty(t, sink.streams) + require.NoError(t, stream.Destroy(ctx)) } -// TestSinkAddStreamRollsBackPartialFailure verifies that a failed AddStream -// compensates the consumer group and cursor it created so no dangling -// ownership state survives, and that a subsequent AddStream succeeds. -func TestSinkAddStreamRollsBackPartialFailure(t *testing.T) { - testName := strings.Replace(t.Name(), "/", "_", -1) +func TestSinkRejectsStreamMutationAfterClose(t *testing.T) { rdb := ptesting.NewRedisClient(t) - defer ptesting.CleanupRedis(t, rdb, true, testName) + defer ptesting.CleanupRedis(t, rdb, false, "") ctx := ptesting.NewTestContext(t) - s, err := NewStream(testName, rdb, options.WithStreamLogger(pulse.ClueLogger(ctx))) + stream, err := NewStream(t.Name(), rdb, options.WithStreamLogger(pulse.ClueLogger(ctx))) require.NoError(t, err) - s2, err := NewStream(testName+"2", rdb, options.WithStreamLogger(pulse.ClueLogger(ctx))) + added, err := NewStream(t.Name()+"-added", rdb, options.WithStreamLogger(pulse.ClueLogger(ctx))) require.NoError(t, err) - hook := newRedisCommandHook() - rdb.AddHook(hook) - sink, err := s.NewSink(ctx, "sink", - options.WithSinkStartAtOldest(), - options.WithSinkBlockDuration(testBlockDuration)) + defer func() { require.ErrorIs(t, added.Destroy(ctx), ErrStreamNotFound) }() + sink, err := stream.NewSink(ctx, "sink", options.WithSinkBlockDuration(testBlockDuration)) require.NoError(t, err) - defer cleanupSink(t, ctx, s, sink) - defer func() { assert.NoError(t, s2.Destroy(ctx)) }() - - hook.failScript(registerConsumerScript.Hash(), s2.key) - require.Error(t, sink.AddStream(ctx, s2)) - hook.clearFailure() - sink.lock.Lock() - _, owned := sink.streams[s2.key] - sink.lock.Unlock() - assert.False(t, owned, "failed AddStream must not leave the stream owned") - assert.Zero(t, rdb.Exists(ctx, cursorsKey(s2.key)).Val(), "compensation must delete the cursor") - groups, err := rdb.XInfoGroups(ctx, s2.key).Result() - require.NoError(t, err) - assert.Empty(t, groups, "compensation must delete the consumer group") - - // AddStream succeeds once the failure clears. - require.NoError(t, sink.AddStream(ctx, s2)) - c := sink.Subscribe() - _, err = s2.Add(ctx, "event", []byte("payload")) - require.NoError(t, err) - require.NoError(t, sink.Ack(ctx, receiveEvent(t, c))) + require.NoError(t, sink.Close(ctx)) + require.ErrorIs(t, sink.AddStream(ctx, added), ErrSinkClosed) + require.ErrorIs(t, sink.RemoveStream(ctx, stream), ErrSinkClosed) + require.NoError(t, stream.Destroy(ctx)) } -// TestSinkConsumerRotationRegistersEveryStreamOrRollsBack verifies that -// replacement consumer creation is failure-atomic across all sink streams: -// when registration fails for one stream the registrations already made are -// detached so ownership state never diverges. func TestSinkConsumerRotationRegistersEveryStreamOrRollsBack(t *testing.T) { - testName := strings.Replace(t.Name(), "/", "_", -1) rdb := ptesting.NewRedisClient(t) - defer ptesting.CleanupRedis(t, rdb, true, testName) + defer ptesting.CleanupRedis(t, rdb, false, "") + failure := errors.New("injected replacement consumer failure") + hook := &redisCommandHook{failure: failure} + rdb.AddHook(hook) ctx := ptesting.NewTestContext(t) - s, err := NewStream(testName, rdb, options.WithStreamLogger(pulse.ClueLogger(ctx))) + mainStream, err := NewStream(t.Name(), rdb, options.WithStreamLogger(pulse.ClueLogger(ctx))) require.NoError(t, err) - s2, err := NewStream(testName+"2", rdb, options.WithStreamLogger(pulse.ClueLogger(ctx))) + addedStream, err := NewStream(t.Name()+"-added", rdb, options.WithStreamLogger(pulse.ClueLogger(ctx))) require.NoError(t, err) - hook := newRedisCommandHook() - rdb.AddHook(hook) - sink, err := s.NewSink(ctx, "sink", - options.WithSinkStartAtOldest(), - options.WithSinkBlockDuration(testBlockDuration)) + defer func() { require.NoError(t, addedStream.Destroy(ctx)) }() + sink, err := mainStream.NewSink(ctx, "sink", options.WithSinkBlockDuration(testBlockDuration)) require.NoError(t, err) - require.NoError(t, sink.AddStream(ctx, s2)) - defer cleanupSink(t, ctx, s, sink) - defer func() { assert.NoError(t, s2.Destroy(ctx)) }() + defer cleanupSink(t, ctx, mainStream, sink) + require.NoError(t, sink.AddStream(ctx, addedStream)) original := sink.consumer - hook.failScript(registerConsumerScript.Hash(), s2.key) sink.lock.Lock() + hook.failCreateConsumerKey = addedStream.key + hook.failCreateConsumer.Store(true) _, err = sink.newConsumer(ctx) + hook.failCreateConsumer.Store(false) sink.lock.Unlock() - require.Error(t, err) - hook.clearFailure() - - for _, stream := range []*Stream{s, s2} { - assert.Equal(t, []string{original}, memberConsumers(t, ctx, rdb, stream, "sink"), - "membership of %s must be unchanged after rollback", stream.Name) - assert.Equal(t, []string{original}, groupConsumers(t, ctx, rdb, stream, "sink"), - "consumer group of %s must be unchanged after rollback", stream.Name) + require.ErrorIs(t, err, failure) + require.Equal(t, original, sink.consumer) + for _, stream := range []*Stream{mainStream, addedStream} { + require.Eventually(t, func() bool { + members, ok := sink.streams[stream.key].consumers.GetValues(sink.Name) + return ok && assert.ObjectsAreEqual([]string{original}, members) + }, max, delay) + consumers, err := rdb.XInfoConsumers(ctx, stream.key, sink.Name).Result() + require.NoError(t, err) + require.Len(t, consumers, 1) + require.Equal(t, original, consumers[0].Name) } - // Rotation succeeds once the failure clears and registers both streams. sink.lock.Lock() replacement, err := sink.newConsumer(ctx) + if err == nil { + sink.consumer = replacement + } sink.lock.Unlock() require.NoError(t, err) - for _, stream := range []*Stream{s, s2} { - assert.ElementsMatch(t, []string{original, replacement}, memberConsumers(t, ctx, rdb, stream, "sink")) + require.NotEqual(t, original, replacement) + for _, stream := range []*Stream{mainStream, addedStream} { + members, ok := sink.streams[stream.key].consumers.GetValues(sink.Name) + require.True(t, ok) + require.Contains(t, members, replacement) + consumers, err := rdb.XInfoConsumers(ctx, stream.key, sink.Name).Result() + require.NoError(t, err) + assert.Contains(t, consumerNames(consumers), replacement) } } -// TestDestroyFencesSinkMetadataWrites is the P1-A regression test: once -// Stream.Destroy runs, a live sink (setup paths, replacement-consumer -// creation, keepalive-driven loops, lease work) must not resurrect any -// stream-scoped metadata. -func TestDestroyFencesSinkMetadataWrites(t *testing.T) { - testName := strings.Replace(t.Name(), "/", "_", -1) - var origCheckIdlePeriod time.Duration - origCheckIdlePeriod, checkIdlePeriod = checkIdlePeriod, testCheckIdlePeriod - defer func() { checkIdlePeriod = origCheckIdlePeriod }() +func TestSinkStreamMutationRollback(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + failure := errors.New("injected ownership failure") + hook := &redisCommandHook{failure: failure} + rdb.AddHook(hook) + ctx := ptesting.NewTestContext(t) + mainStream, err := NewStream(t.Name(), rdb, options.WithStreamLogger(pulse.ClueLogger(ctx))) + require.NoError(t, err) + addedStream, err := NewStream(t.Name()+"-added", rdb, options.WithStreamLogger(pulse.ClueLogger(ctx))) + require.NoError(t, err) + defer func() { require.NoError(t, addedStream.Destroy(ctx)) }() + sink, err := mainStream.NewSink(ctx, "sink", options.WithSinkBlockDuration(testBlockDuration)) + require.NoError(t, err) + defer cleanupSink(t, ctx, mainStream, sink) + + hook.failCreateConsumer.Store(true) + hook.failCreateConsumerKey = addedStream.key + err = sink.AddStream(ctx, addedStream) + hook.failCreateConsumer.Store(false) + require.ErrorIs(t, err, failure) + assert.False(t, sinkOwnsStream(sink, addedStream)) + assert.NotContains(t, sink.streams, addedStream.key) + assert.True(t, consumerGroupExists(ctx, rdb, addedStream.key, sink.Name)) + assert.True(t, rdb.HExists(ctx, recoveryCursorKey(addedStream), sink.Name).Val()) + + consumerMap, err := rmap.Join(ctx, consumersMapName(addedStream), rdb) + require.NoError(t, err) + assert.NotContains(t, consumerMap.Map(), sink.Name) + consumerMap.Close() + + require.NoError(t, sink.AddStream(ctx, addedStream)) + hook.failMembershipKey = consumersMapContentKey(addedStream) + hook.failRemove.Store(true) + err = sink.RemoveStream(ctx, addedStream) + hook.failRemove.Store(false) + require.ErrorIs(t, err, failure) + assert.True(t, sinkOwnsStream(sink, addedStream)) + assert.Contains(t, sink.streams, addedStream.key) + assert.True(t, consumerGroupExists(ctx, rdb, addedStream.key, sink.Name)) + + require.NoError(t, sink.RemoveStream(ctx, addedStream)) + assert.False(t, sinkOwnsStream(sink, addedStream)) + assert.True(t, consumerGroupExists(ctx, rdb, addedStream.key, sink.Name)) + assert.True(t, rdb.HExists(ctx, recoveryCursorKey(addedStream), sink.Name).Val()) +} +func TestSinkRecoveryMetadataOutlivesEventTTL(t *testing.T) { rdb := ptesting.NewRedisClient(t) - defer ptesting.CleanupRedis(t, rdb, true, testName) + defer ptesting.CleanupRedis(t, rdb, false, "") ctx := ptesting.NewTestContext(t) - s, err := NewStream(testName, rdb, options.WithStreamLogger(pulse.ClueLogger(ctx))) + stream, err := NewStream( + t.Name(), + rdb, + options.WithStreamTTL(5*time.Second), + options.WithStreamLogger(pulse.ClueLogger(ctx)), + ) require.NoError(t, err) - sink, err := s.NewSink(ctx, "sink", - options.WithSinkStartAtOldest(), - options.WithSinkBlockDuration(testBlockDuration), - options.WithSinkAckGracePeriod(testAckDuration)) - require.NoError(t, err) - defer sink.Close(ctx) - - c := sink.Subscribe() - _, err = s.Add(ctx, "event", []byte("payload")) - require.NoError(t, err) - require.NoError(t, sink.Ack(ctx, receiveEvent(t, c))) - require.Equal(t, int64(1), rdb.Exists(ctx, cursorsKey(s.key)).Val()) - - // Destroy the stream while the sink keepalive and idle-check loops are - // due to run. - require.NoError(t, s.Destroy(ctx)) - - // Every fenced metadata write must fail with ErrStreamDestroyed. - assert.ErrorIs(t, registerSinkConsumer(ctx, s, "sink", "ghost"), ErrStreamDestroyed) - _, _, err = ensureConsumerGroup(ctx, s, "sink", "$", false) - assert.ErrorIs(t, err, ErrStreamDestroyed) - _, _, err = acquireSinkLease(ctx, s, "sink", "ghost-owner", 1000) - assert.ErrorIs(t, err, ErrStreamDestroyed) - - // The sink read loop observes NOGROUP, fails recovery with - // ErrStreamDestroyed, and drops the stream instead of resurrecting it. - assert.Eventually(t, func() bool { - sink.lock.Lock() - defer sink.lock.Unlock() - return len(sink.streams) == 0 - }, max, delay, "sink must drop the destroyed stream") - - // Let the periodic keepalive and idle-check loops tick several times, - // then verify no stream-scoped metadata was recreated. - time.Sleep(5 * testCheckIdlePeriod) - assert.Zero(t, rdb.Exists(ctx, s.key).Val(), "event stream must stay deleted") - assert.Zero(t, rdb.Exists(ctx, cursorsKey(s.key)).Val(), "recovery cursor must stay deleted") - assert.Zero(t, rdb.Exists(ctx, leaseKey(s.key, "sink")).Val(), "lease must stay deleted") - content, err := rdb.HGetAll(ctx, membershipContentKey(s.Name)).Result() - require.NoError(t, err) - assert.Equal(t, "destroy", content["=kind"], "membership map must remain a destroy tombstone") - assert.NotContains(t, content, "sink", "membership must not be resurrected") - assert.Equal(t, "destroyed", rdb.HGet(ctx, lifecycleKey(s.key), "state").Val()) + sink, err := stream.NewSink(ctx, "sink", options.WithSinkBlockDuration(testBlockDuration)) + require.NoError(t, err) + defer cleanupSink(t, ctx, stream, sink) + + require.NoError(t, rdb.Persist(ctx, stream.key).Err()) + require.NoError(t, rdb.Persist(ctx, recoveryCursorKey(stream)).Err()) + require.Equal(t, time.Duration(-1), rdb.PTTL(ctx, stream.key).Val()) + require.Equal(t, time.Duration(-1), rdb.PTTL(ctx, recoveryCursorKey(stream)).Val()) + sink.lock.Lock() + err = sink.recoverConsumerGroups(ctx) + sink.lock.Unlock() + require.NoError(t, err) + assert.Greater(t, rdb.PTTL(ctx, stream.key).Val(), time.Duration(0)) + assert.Equal(t, time.Duration(-1), rdb.PTTL(ctx, recoveryCursorKey(stream)).Val()) } -// TestDispatchLeavesRemovedStreamEventsPending verifies that a batch read for -// a stream removed from this sink concurrently with the read is left pending -// for the surviving group members instead of being acknowledged undelivered, -// which would permanently drop the events for the whole group. -func TestDispatchLeavesRemovedStreamEventsPending(t *testing.T) { - testName := strings.Replace(t.Name(), "/", "_", -1) +func TestSinkAcknowledgesBatchFromConcurrentlyRemovedSnapshot(t *testing.T) { rdb := ptesting.NewRedisClient(t) - defer ptesting.CleanupRedis(t, rdb, true, testName) + defer ptesting.CleanupRedis(t, rdb, false, "") + hook := &redisCommandHook{} + rdb.AddHook(hook) ctx := ptesting.NewTestContext(t) - s, err := NewStream(testName, rdb, options.WithStreamLogger(pulse.ClueLogger(ctx))) + main, err := NewStream(t.Name()+"-main", rdb) + require.NoError(t, err) + removed, err := NewStream(t.Name()+"-removed", rdb) + require.NoError(t, err) + sink, err := main.NewSink(ctx, "sink", options.WithSinkBlockDuration(time.Second)) require.NoError(t, err) - removed, err := s.NewSink(ctx, "sink", + require.NoError(t, sink.AddStream(ctx, removed, options.WithAddStreamStartAtOldest())) + events := sink.Subscribe() + reads := hook.xreadGroups.Load() + require.Eventually(t, func() bool { + return hook.xreadGroups.Load() > reads + }, max, delay) + + require.NoError(t, sink.RemoveStream(ctx, removed)) + eventID, err := removed.Add(ctx, "removed", []byte("payload")) + require.NoError(t, err) + require.Eventually(t, func() bool { + cursor := rdb.HGet(ctx, recoveryCursorKey(removed), sink.Name).Val() + pending, pendingErr := rdb.XPending(ctx, removed.key, sink.Name).Result() + return pendingErr == nil && pending.Count == 0 && cursor == eventID + }, max, delay) + select { + case event := <-events: + require.NotEqual(t, eventID, event.ID) + case <-time.After(2 * testBlockDuration): + } + + require.NoError(t, sink.Close(ctx)) + require.NoError(t, main.Destroy(ctx)) + require.NoError(t, removed.Destroy(ctx)) +} + +func TestSinkRecoveryAfterEventStreamExpiresDeliversNewEvents(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + stream, err := NewStream(t.Name(), rdb, options.WithStreamTTL(2*time.Second)) + require.NoError(t, err) + sink, err := stream.NewSink( + ctx, + "sink", options.WithSinkStartAtOldest(), - options.WithSinkBlockDuration(testBlockDuration)) + options.WithSinkBlockDuration(testBlockDuration), + ) + require.NoError(t, err) + events := sink.Subscribe() + _, err = stream.Add(ctx, "old", []byte("payload")) require.NoError(t, err) - defer removed.Close(ctx) - s2, err := NewStream(testName, ptesting.NewRedisClient(t), options.WithStreamLogger(pulse.ClueLogger(ctx))) + require.NoError(t, sink.Ack(ctx, receiveSinkEvent(t, events))) + require.NoError(t, sink.Close(ctx)) + require.NoError(t, rdb.PExpire(ctx, stream.key, 10*time.Millisecond).Err()) + require.Eventually(t, func() bool { + return rdb.Exists(ctx, stream.key).Val() == 0 + }, max, delay) + require.Equal(t, time.Duration(-1), rdb.PTTL(ctx, recoveryCursorKey(stream)).Val()) + + freshID, err := stream.Add(ctx, "fresh", []byte("payload")) require.NoError(t, err) - survivor, err := s2.NewSink(ctx, "sink", + recovered, err := stream.NewSink( + ctx, + "sink", options.WithSinkStartAtOldest(), - options.WithSinkBlockDuration(testBlockDuration)) - require.NoError(t, err) - defer cleanupSink(t, ctx, s2, survivor) - - // Drop the stream from one sink; the group survives through the other - // member. Wait out reads issued before the removal so the event below is - // deterministically delivered to the survivor's consumer PEL, unacked. - c := survivor.Subscribe() - require.NoError(t, removed.RemoveStream(ctx, s)) - time.Sleep(3 * testBlockDuration) - id, err := s.Add(ctx, "event", []byte("payload")) - require.NoError(t, err) - ev := receiveEvent(t, c) - require.Equal(t, id, ev.ID) - - // Replay the racing batch against the sink that no longer owns the - // stream: dispatch must not settle the group's pending entry. - require.NoError(t, removed.dispatch([]redis.XStream{{ - Stream: s.key, - Messages: []redis.XMessage{{ID: id, Values: map[string]any{nameKey: "event", payloadKey: "payload"}}}, - }})) - pending, err := rdb.XPending(ctx, s.key, "sink").Result() - require.NoError(t, err) - assert.Equal(t, int64(1), pending.Count, "unowned batch must stay pending for surviving members") - require.NoError(t, survivor.Ack(ctx, ev)) + options.WithSinkBlockDuration(testBlockDuration), + ) + require.NoError(t, err) + fresh := receiveSinkEvent(t, recovered.Subscribe()) + require.Equal(t, freshID, fresh.ID) + require.NoError(t, recovered.Ack(ctx, fresh)) + require.NoError(t, recovered.Close(ctx)) + require.NoError(t, stream.Destroy(ctx)) } -// TestSinkAcknowledgesFilteredEvents verifies that events dropped by the sink -// topic filter are acknowledged so they cannot hold back the recovery cursor. -func TestSinkAcknowledgesFilteredEvents(t *testing.T) { - testName := strings.Replace(t.Name(), "/", "_", -1) +func TestSinkNoAckUsesPendingEntryListBeforeDelivery(t *testing.T) { rdb := ptesting.NewRedisClient(t) - defer ptesting.CleanupRedis(t, rdb, true, testName) + defer ptesting.CleanupRedis(t, rdb, false, "") + hook := &redisCommandHook{} + rdb.AddHook(hook) ctx := ptesting.NewTestContext(t) - s, err := NewStream(testName, rdb, options.WithStreamLogger(pulse.ClueLogger(ctx))) + stream, err := NewStream(t.Name(), rdb, options.WithStreamLogger(pulse.ClueLogger(ctx))) require.NoError(t, err) - sink, err := s.NewSink(ctx, "sink", + sink, err := stream.NewSink( + ctx, + "sink", options.WithSinkStartAtOldest(), + options.WithSinkNoAck(), options.WithSinkBlockDuration(testBlockDuration), - options.WithSinkTopic("keep")) + ) require.NoError(t, err) - defer cleanupSink(t, ctx, s, sink) + defer cleanupSink(t, ctx, stream, sink) + events := sink.Subscribe() - c := sink.Subscribe() - _, err = s.Add(ctx, "dropped", []byte("payload"), options.WithTopic("drop")) + eventID, err := stream.Add(ctx, "at-most-once", []byte("payload")) require.NoError(t, err) - keepID, err := s.Add(ctx, "kept", []byte("payload"), options.WithTopic("keep")) + event := receiveSinkEvent(t, events) + require.Equal(t, eventID, event.ID) + require.False(t, hook.usedNoAck.Load(), "sink sent Redis NOACK") + pending, err := rdb.XPending(ctx, stream.key, sink.Name).Result() require.NoError(t, err) + require.Zero(t, pending.Count) + cursor, err := rdb.HGet(ctx, recoveryCursorKey(stream), sink.Name).Result() + require.NoError(t, err) + require.Equal(t, eventID, cursor) - ev := receiveEvent(t, c) - assert.Equal(t, keepID, ev.ID) - require.NoError(t, sink.Ack(ctx, ev)) - assert.Equal(t, keepID, recoveryCursor(t, ctx, rdb, s, "sink")) - pending, err := rdb.XPending(ctx, s.key, "sink").Result() + require.NoError(t, rdb.XGroupDestroy(ctx, stream.key, sink.Name).Err()) + require.Eventually(t, func() bool { + return consumerGroupExists(ctx, rdb, stream.key, sink.Name) + }, max, delay) + _, err = stream.Add(ctx, "future", []byte("future")) require.NoError(t, err) - assert.Zero(t, pending.Count, "filtered events must be settled") + require.Equal(t, "future", receiveSinkEvent(t, events).EventName) } -// TestSinkNoAckAdvancesRecoveryCursor verifies that NoAck sinks advance the -// recovery cursor on delivery so recovery never replays delivered events. -func TestSinkNoAckAdvancesRecoveryCursor(t *testing.T) { - testName := strings.Replace(t.Name(), "/", "_", -1) +func TestEventAckerAdvancesRecoveryCursor(t *testing.T) { rdb := ptesting.NewRedisClient(t) - defer ptesting.CleanupRedis(t, rdb, true, testName) + defer ptesting.CleanupRedis(t, rdb, false, "") ctx := ptesting.NewTestContext(t) - s, err := NewStream(testName, rdb, options.WithStreamLogger(pulse.ClueLogger(ctx))) + stream, err := NewStream(t.Name(), rdb, options.WithStreamLogger(pulse.ClueLogger(ctx))) require.NoError(t, err) - sink, err := s.NewSink(ctx, "sink", + sink, err := stream.NewSink( + ctx, + "sink", options.WithSinkStartAtOldest(), options.WithSinkBlockDuration(testBlockDuration), - options.WithSinkNoAck()) + ) require.NoError(t, err) - defer cleanupSink(t, ctx, s, sink) + defer cleanupSink(t, ctx, stream, sink) + events := sink.Subscribe() - c := sink.Subscribe() - id, err := s.Add(ctx, "event", []byte("payload")) + eventID, err := stream.Add(ctx, "direct-ack", []byte("payload")) require.NoError(t, err) - ev := receiveEvent(t, c) - assert.Equal(t, id, ev.ID) - assert.Equal(t, id, recoveryCursor(t, ctx, rdb, s, "sink")) - - require.NoError(t, rdb.XGroupDestroy(ctx, s.key, "sink").Err()) - futureID, err := s.Add(ctx, "future", []byte("payload")) + event := receiveSinkEvent(t, events) + acked, err := event.Acker.XAck(ctx, stream.key, sink.Name, event.ID).Result() require.NoError(t, err) - ev = receiveEvent(t, c) - assert.Equal(t, futureID, ev.ID, "recovery must resume after the delivered event") - select { - case ev := <-c: - t.Errorf("unexpected redelivery of event %s", ev.ID) - case <-time.After(4 * testBlockDuration): - } + require.EqualValues(t, 1, acked) + cursor, err := rdb.HGet(ctx, recoveryCursorKey(stream), sink.Name).Result() + require.NoError(t, err) + require.Equal(t, eventID, cursor) } -// TestEventAckerAdvancesRecoveryCursor verifies the cursor arithmetic of the -// recovery acker: the cursor is always the entry preceding the oldest pending -// event, and the group last-delivered-id once the PEL drains, even when -// events are acknowledged out of order. -func TestEventAckerAdvancesRecoveryCursor(t *testing.T) { - testName := strings.Replace(t.Name(), "/", "_", -1) +func TestEventAckerAcknowledgesMultipleIDs(t *testing.T) { rdb := ptesting.NewRedisClient(t) - defer ptesting.CleanupRedis(t, rdb, true, testName) + defer ptesting.CleanupRedis(t, rdb, false, "") ctx := ptesting.NewTestContext(t) - s, err := NewStream(testName, rdb, options.WithStreamLogger(pulse.ClueLogger(ctx))) + stream, err := NewStream(t.Name(), rdb, options.WithStreamLogger(pulse.ClueLogger(ctx))) require.NoError(t, err) - sink, err := s.NewSink(ctx, "sink", + sink, err := stream.NewSink( + ctx, + "sink", options.WithSinkStartAtOldest(), - options.WithSinkBlockDuration(testBlockDuration)) + options.WithSinkBlockDuration(testBlockDuration), + ) require.NoError(t, err) - defer cleanupSink(t, ctx, s, sink) - - c := sink.Subscribe() - ids := make([]string, 3) - for i := range ids { - ids[i], err = s.Add(ctx, fmt.Sprintf("event%d", i), []byte("payload")) + defer cleanupSink(t, ctx, stream, sink) + events := sink.Subscribe() + for _, name := range []string{"first", "second"} { + _, err := stream.Add(ctx, name, []byte(name)) require.NoError(t, err) } - events := make(map[string]*Event, 3) - for range ids { - ev := receiveEvent(t, c) - events[ev.ID] = ev - } + first := receiveSinkEvent(t, events) + second := receiveSinkEvent(t, events) - // Ack out of order: the middle event first. - require.NoError(t, sink.Ack(ctx, events[ids[1]])) - assert.Equal(t, "0-0", recoveryCursor(t, ctx, rdb, s, "sink"), - "oldest pending event is the first entry so nothing is durably settled") - require.NoError(t, sink.Ack(ctx, events[ids[0]])) - assert.Equal(t, ids[1], recoveryCursor(t, ctx, rdb, s, "sink"), - "cursor must jump past the contiguous acknowledged prefix") - require.NoError(t, sink.Ack(ctx, events[ids[2]])) - assert.Equal(t, ids[2], recoveryCursor(t, ctx, rdb, s, "sink"), - "cursor must reach last-delivered-id once the PEL drains") + acked, err := first.Acker.XAck(ctx, stream.key, sink.Name, first.ID, second.ID).Result() + require.NoError(t, err) + require.EqualValues(t, 2, acked) + cursor, err := rdb.HGet(ctx, recoveryCursorKey(stream), sink.Name).Result() + require.NoError(t, err) + require.Equal(t, second.ID, cursor) } -// TestReadRetryJitterBounds verifies the retry backoff is jittered between -// half and full of the current backoff and doubles up to the cap. -func TestReadRetryJitterBounds(t *testing.T) { - var r readRetry - expected := minReadRetryBackoff - for range 10 { - d := r.next() - assert.GreaterOrEqual(t, d, expected/2) - assert.LessOrEqual(t, d, expected) - expected = 2 * expected - if expected > maxReadRetryBackoff { - expected = maxReadRetryBackoff - } +func TestSinkAckDelegatesToEventAcker(t *testing.T) { + ctx := context.Background() + acker := &recordingAcker{} + sink := &Sink{logger: pulse.NoopLogger()} + event := &Event{ + ID: "1-0", + StreamName: "stream", + SinkName: "sink", + Acker: acker, + streamKey: "pulse:stream:stream", } - r.reset() - d := r.next() - assert.GreaterOrEqual(t, d, minReadRetryBackoff/2) - assert.LessOrEqual(t, d, minReadRetryBackoff) -} -// receiveEvent reads one event from the channel without acknowledging it or -// fails the test after the standard timeout. -func receiveEvent(t *testing.T, c <-chan *Event) *Event { - t.Helper() - select { - case ev := <-c: - require.NotNil(t, ev) - return ev - case <-time.After(max): - t.Fatal("timeout waiting for event") - return nil - } + require.NoError(t, sink.Ack(ctx, event)) + require.Equal(t, event.streamKey, acker.streamKey) + require.Equal(t, event.SinkName, acker.group) + require.Equal(t, []string{event.ID}, acker.ids) } -// recoveryCursor returns the durable recovery cursor stored for the sink on -// the stream. -func recoveryCursor(t *testing.T, ctx context.Context, rdb *redis.Client, s *Stream, sink string) string { - t.Helper() - cursor, err := rdb.HGet(ctx, cursorsKey(s.key), sink).Result() +func TestRecoveryCursorsShareOneStreamHash(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + stream, err := NewStream(t.Name(), rdb, options.WithStreamLogger(pulse.ClueLogger(ctx))) require.NoError(t, err) - return cursor -} - -// memberConsumers returns the consumer names recorded for the sink in the -// stream membership map. -func memberConsumers(t *testing.T, ctx context.Context, rdb *redis.Client, s *Stream, sink string) []string { - t.Helper() - raw, err := rdb.HGet(ctx, membershipContentKey(s.Name), sink).Result() + sink1, err := stream.NewSink(ctx, "first", options.WithSinkBlockDuration(testBlockDuration)) require.NoError(t, err) - var names []string - require.NoError(t, json.Unmarshal([]byte(raw), &names)) - return names -} + sink2, err := stream.NewSink(ctx, "second", options.WithSinkBlockDuration(testBlockDuration)) + require.NoError(t, err) + defer cleanupSink(t, ctx, stream, sink1) + defer func() { require.NoError(t, sink2.Close(ctx)) }() -// groupConsumers returns the Redis consumer names of the sink group on the -// stream. -func groupConsumers(t *testing.T, ctx context.Context, rdb *redis.Client, s *Stream, sink string) []string { - t.Helper() - consumers, err := rdb.XInfoConsumers(ctx, s.key, sink).Result() + cursors, err := rdb.HGetAll(ctx, recoveryCursorKey(stream)).Result() require.NoError(t, err) - names := make([]string, len(consumers)) - for i, c := range consumers { - names[i] = c.Name - } - return names + require.Len(t, cursors, 2) + require.Contains(t, cursors, sink1.Name) + require.Contains(t, cursors, sink2.Name) + legacyKeys, err := rdb.Keys(ctx, stream.key+":sink:*:recovery").Result() + require.NoError(t, err) + require.Empty(t, legacyKeys) } -type ( - // redisCommandHook injects command-specific failures and blocking reads - // into the sink Redis client to prove cancellation, bounded retries, and - // failure-atomic ownership changes. - redisCommandHook struct { - // blockRead blocks XREADGROUP calls until their context is canceled. - blockRead atomic.Bool - // readStarted is closed the first time a read blocks. - readStarted chan struct{} - // started guards readStarted. - started sync.Once - // mu guards the failure configuration below. - mu sync.Mutex - // failHash is the script hash whose EVALSHA calls fail. - failHash string - // failKey restricts injected failures to invocations naming this key. - failKey string +func TestReadRetryJitterBounds(t *testing.T) { + low := readRetry{jitter: func(int64) int64 { return 0 }} + high := readRetry{jitter: func(n int64) int64 { return n - 1 }} + limit := readRetryInitialDelay + for range 10 { + assert.Equal(t, limit/2, low.nextDelay()) + assert.Equal(t, limit, high.nextDelay()) + limit = min(limit*2, readRetryMaxDelay) } -) - -// errInjected is the transport failure injected by redisCommandHook. -var errInjected = errors.New("injected redis failure") +} -// newRedisCommandHook returns a hook with no active failure. -func newRedisCommandHook() *redisCommandHook { - return &redisCommandHook{readStarted: make(chan struct{})} +// XAck records the Event acknowledgement invoked by Sink.Ack. +func (a *recordingAcker) XAck(ctx context.Context, streamKey, group string, ids ...string) *redis.IntCmd { + a.streamKey = streamKey + a.group = group + a.ids = append([]string(nil), ids...) + cmd := redis.NewIntCmd(ctx) + cmd.SetVal(int64(len(ids))) + return cmd } -// failScript makes EVALSHA calls of the script with the given hash fail when -// their arguments include key. -func (h *redisCommandHook) failScript(hash, key string) { - h.mu.Lock() - defer h.mu.Unlock() - h.failHash, h.failKey = hash, key +// receiveSinkEvent reads one event without acknowledging it. +func receiveSinkEvent(t *testing.T, events <-chan *Event) *Event { + t.Helper() + select { + case event := <-events: + require.NotNil(t, event) + return event + case <-time.After(max): + t.Fatal("timed out waiting for sink event") + return nil + } } -// clearFailure removes the active script failure. -func (h *redisCommandHook) clearFailure() { - h.mu.Lock() - defer h.mu.Unlock() - h.failHash, h.failKey = "", "" +// eventNames projects event names in delivery order. +func eventNames(events []*Event) []string { + names := make([]string, len(events)) + for i, event := range events { + names[i] = event.EventName + } + return names } -// DialHook preserves the client's normal Redis connection behavior. -func (h *redisCommandHook) DialHook(next redis.DialHook) redis.DialHook { - return func(ctx context.Context, network, addr string) (net.Conn, error) { - return next(ctx, network, addr) +// consumerNames projects Redis consumer info for membership assertions. +func consumerNames(consumers []redis.XInfoConsumer) []string { + names := make([]string, len(consumers)) + for i, consumer := range consumers { + names[i] = consumer.Name } + return names } -// ProcessHook blocks reads and injects script failures per the hook -// configuration. -func (h *redisCommandHook) ProcessHook(next redis.ProcessHook) redis.ProcessHook { - return func(ctx context.Context, cmd redis.Cmder) error { - switch cmd.Name() { - case "xreadgroup": - if h.blockRead.Load() { - h.started.Do(func() { close(h.readStarted) }) - <-ctx.Done() - return ctx.Err() - } - case "evalsha": - h.mu.Lock() - hash, key := h.failHash, h.failKey - h.mu.Unlock() - if hash != "" && len(cmd.Args()) > 1 && cmd.Args()[1] == hash { - for _, arg := range cmd.Args() { - if s, ok := arg.(string); ok && s == key { - return errInjected - } - } - } +// sinkOwnsStream reports local ownership under the sink lock. +func sinkOwnsStream(sink *Sink, stream *Stream) bool { + sink.lock.Lock() + defer sink.lock.Unlock() + for _, owned := range sink.streams { + if owned.stream == stream { + return true } - return next(ctx, cmd) } + return false } -// ProcessPipelineHook preserves pipeline behavior; the sink does not issue -// pipelines on the paths exercised by these tests. -func (h *redisCommandHook) ProcessPipelineHook(next redis.ProcessPipelineHook) redis.ProcessPipelineHook { - return func(ctx context.Context, cmds []redis.Cmder) error { - return next(ctx, cmds) - } +func TestDestroyedGenerationMetadataCannotBeRecreated(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + stream, err := NewStream(t.Name(), rdb, options.WithStreamLogger(pulse.ClueLogger(ctx))) + require.NoError(t, err) + sink, err := stream.NewSink(ctx, "sink", options.WithSinkBlockDuration(testBlockDuration)) + require.NoError(t, err) + defer func() { require.NoError(t, sink.Close(ctx)) }() + state := sink.streams[stream.key] + require.NotNil(t, state) + keepAliveKey := rmapContentKey(sinkKeepAliveMapName(stream, "sink")) + membershipKey := consumersMapContentKey(stream) + require.EqualValues(t, 1, rdb.Exists(ctx, keepAliveKey).Val()) + require.EqualValues(t, 1, rdb.Exists(ctx, membershipKey).Val()) + + require.NoError(t, stream.Destroy(ctx)) + require.EqualValues(t, 0, rdb.Exists(ctx, keepAliveKey).Val()) + require.EqualValues(t, 0, rdb.Exists(ctx, membershipKey).Val()) + + // Neither a periodic keep-alive tick nor a consumer registration may + // resurrect metadata for the destroyed generation. + err = setSinkKeepAlive(ctx, state, "sink", sink.consumer, time.Now().UnixNano()) + require.ErrorIs(t, err, ErrStreamDestroyed) + err = registerSinkConsumer(ctx, state, "sink", "ghost-consumer", time.Now().UnixNano()) + require.ErrorIs(t, err, ErrStreamDestroyed) + require.EqualValues(t, 0, rdb.Exists(ctx, keepAliveKey).Val()) + require.EqualValues(t, 0, rdb.Exists(ctx, membershipKey).Val()) } diff --git a/streaming/sink_test.go b/streaming/sink_test.go index 67e16e8..bfa8f3b 100644 --- a/streaming/sink_test.go +++ b/streaming/sink_test.go @@ -1,24 +1,596 @@ package streaming import ( + "context" + "errors" + "net" "strings" + "sync" + "sync/atomic" "testing" "time" + redis "github.com/redis/go-redis/v9" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "goa.design/pulse/pulse" + "goa.design/pulse/rmap" "goa.design/pulse/streaming/options" ptesting "goa.design/pulse/testing" ) +// redisCommandHook observes real Redis commands and can fail selected commands +// at the client boundary without replacing Redis in recovery tests. +type redisCommandHook struct { + xreadGroups atomic.Int64 + failRead atomic.Bool + failGroup atomic.Bool + failCreateConsumer atomic.Bool + failRemove atomic.Bool + usedNoAck atomic.Bool + blockRead atomic.Bool + // failMembershipKey selects the replicated membership map whose mutation + // should fail while failRemove is set. + failMembershipKey string + // failGroupKey selects the stream lifecycle key whose group Lua operation + // should fail while failGroup is set. + failGroupKey string + // failCreateConsumerKey narrows consumer creation failure to one stream. + failCreateConsumerKey string + readStarted chan struct{} + started sync.Once + failure error +} + var ( testCheckIdlePeriod = 50 * time.Millisecond testBlockDuration = 50 * time.Millisecond testAckDuration = 50 * time.Millisecond ) +func TestSinkClosingFencesSubscriptions(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + stream, err := NewStream(t.Name(), rdb) + require.NoError(t, err) + sink, err := stream.NewSink( + ctx, + "sink", + options.WithSinkStartAtOldest(), + options.WithSinkBlockDuration(testBlockDuration), + ) + require.NoError(t, err) + + start := make(chan struct{}) + subscriptions := make(chan (<-chan *Event), 64) + var wait sync.WaitGroup + for range 64 { + wait.Add(1) + go func() { + defer wait.Done() + <-start + subscriptions <- sink.Subscribe() + }() + } + close(start) + require.NoError(t, sink.Close(ctx)) + wait.Wait() + close(subscriptions) + for subscription := range subscriptions { + _, ok := <-subscription + require.False(t, ok) + } + closed := sink.Subscribe() + _, ok := <-closed + require.False(t, ok) + require.NoError(t, stream.Destroy(ctx)) +} + +func TestSinkRejectsRemovingFinalStream(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + stream, err := NewStream(t.Name(), rdb) + require.NoError(t, err) + sink, err := stream.NewSink(ctx, "sink", options.WithSinkBlockDuration(testBlockDuration)) + require.NoError(t, err) + + unattached, err := NewStream(t.Name()+"-unattached", rdb) + require.NoError(t, err) + cancelled, cancel := context.WithCancel(ctx) + cancel() + require.NoError(t, sink.RemoveStream(cancelled, unattached)) + require.Empty(t, unattached.Generation()) + require.ErrorIs(t, sink.RemoveStream(ctx, stream), ErrLastStream) + require.NoError(t, sink.Close(ctx)) + require.NoError(t, stream.Destroy(ctx)) +} + +func TestSinkReplicaConfigurationAndLeaseScope(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + first, err := NewStream(t.Name()+"-first", rdb) + require.NoError(t, err) + second, err := NewStream(t.Name()+"-second", rdb) + require.NoError(t, err) + opts := []options.Sink{ + options.WithSinkTopic("alarms"), + options.WithSinkNoAck(), + options.WithSinkAckGracePeriod(testAckDuration), + options.WithSinkBlockDuration(testBlockDuration), + } + firstReplica, err := first.NewSink(ctx, "shared", opts...) + require.NoError(t, err) + secondReplica, err := first.NewSink(ctx, "shared", opts...) + require.NoError(t, err) + unrelated, err := second.NewSink(ctx, "shared", opts...) + require.NoError(t, err) + + firstState := firstReplica.streams[first.key] + secondState := secondReplica.streams[first.key] + unrelatedState := unrelated.streams[second.key] + require.Equal(t, firstState.leaseKey, secondState.leaseKey) + require.NotEqual(t, firstState.leaseKey, unrelatedState.leaseKey) + require.Equal( + t, + firstState.keepAlives.Name, + secondState.keepAlives.Name, + ) + require.NotEqual( + t, + firstState.keepAlives.Name, + unrelatedState.keepAlives.Name, + ) + + _, err = first.NewSink( + ctx, + "shared", + options.WithSinkTopic("readings"), + options.WithSinkNoAck(), + options.WithSinkAckGracePeriod(testAckDuration), + options.WithSinkBlockDuration(testBlockDuration), + ) + require.ErrorContains(t, err, "configuration differs") + + require.NoError(t, firstReplica.Close(ctx)) + require.NoError(t, secondReplica.Close(ctx)) + require.NoError(t, unrelated.Close(ctx)) + firstConfigKey := sinkConfigurationKey(first, "shared") + firstKeepAliveKey := rmapContentKey(sinkKeepAliveMapName(first, "shared")) + firstRegistryKey := streamResourceRegistryKey(first) + require.NoError(t, first.Destroy(ctx)) + require.NoError(t, second.Destroy(ctx)) + require.EqualValues( + t, + 0, + rdb.Exists(ctx, firstConfigKey, firstKeepAliveKey, firstRegistryKey).Val(), + ) +} + +func TestCrossPrimarySinkReplicasCoordinatePerAttachedStream(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + first, err := NewStream(t.Name()+"-first", rdb) + require.NoError(t, err) + second, err := NewStream(t.Name()+"-second", rdb) + require.NoError(t, err) + third, err := NewStream(t.Name()+"-third", rdb) + require.NoError(t, err) + compatible := []options.Sink{ + options.WithSinkTopic("alarms"), + options.WithSinkStartAtOldest(), + options.WithSinkAckGracePeriod(testAckDuration), + options.WithSinkBlockDuration(testBlockDuration), + } + firstPrimary, err := first.NewSink(ctx, "shared", compatible...) + require.NoError(t, err) + secondPrimary, err := second.NewSink(ctx, "shared", compatible...) + require.NoError(t, err) + require.NoError(t, firstPrimary.AddStream(ctx, second, options.WithAddStreamStartAtOldest())) + require.NoError(t, secondPrimary.AddStream(ctx, first, options.WithAddStreamStartAtOldest())) + + require.Equal( + t, + firstPrimary.streams[first.key].keepAlives.Name, + secondPrimary.streams[first.key].keepAlives.Name, + ) + require.Equal( + t, + firstPrimary.streams[second.key].leaseKey, + secondPrimary.streams[second.key].leaseKey, + ) + require.NotEqual( + t, + firstPrimary.streams[second.key].leaseOwner, + secondPrimary.streams[second.key].leaseOwner, + ) + + incompatible, err := third.NewSink( + ctx, + "shared", + options.WithSinkTopic("readings"), + options.WithSinkStartAtOldest(), + options.WithSinkAckGracePeriod(testAckDuration), + options.WithSinkBlockDuration(testBlockDuration), + ) + require.NoError(t, err) + before, err := rdb.XInfoConsumers(ctx, second.key, "shared").Result() + require.NoError(t, err) + err = incompatible.AddStream(ctx, second, options.WithAddStreamStartAtOldest()) + require.ErrorContains(t, err, "configuration differs") + require.NotContains(t, incompatible.streams, second.key) + after, err := rdb.XInfoConsumers(ctx, second.key, "shared").Result() + require.NoError(t, err) + require.Len(t, after, len(before)) + + require.NoError(t, firstPrimary.Close(ctx)) + require.NoError(t, secondPrimary.Close(ctx)) + require.NoError(t, incompatible.Close(ctx)) + require.NoError(t, first.Destroy(ctx)) + require.NoError(t, second.Destroy(ctx)) + require.NoError(t, third.Destroy(ctx)) +} + +func TestSinkRecoveryLeaseFencesExpiredOwner(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + stream, err := NewStream(t.Name(), rdb) + require.NoError(t, err) + sink, err := stream.NewSink( + ctx, + "sink", + options.WithSinkAckGracePeriod(testAckDuration), + options.WithSinkBlockDuration(testBlockDuration), + ) + require.NoError(t, err) + state := sink.streams[stream.key] + duration := time.Second + require.NoError(t, rdb.HSet(ctx, state.leaseKey, "lease_until", "0").Err()) + first, acquired, err := acquireSinkRecoveryLease( + ctx, + stream, + state.leaseKey, + "first-owner", + duration, + ) + require.NoError(t, err) + require.True(t, acquired) + require.NoError(t, rdb.HSet(ctx, state.leaseKey, "lease_until", "0").Err()) + second, acquired, err := acquireSinkRecoveryLease( + ctx, + stream, + state.leaseKey, + "second-owner", + duration, + ) + require.NoError(t, err) + require.True(t, acquired) + require.Greater(t, second.fence, first.fence) + require.ErrorContains( + t, + renewSinkRecoveryLease(ctx, stream, state.leaseKey, first, duration), + "SINKLEASELOST", + ) + require.NoError(t, renewSinkRecoveryLease(ctx, stream, state.leaseKey, second, duration)) + + require.NoError(t, sink.Close(ctx)) + require.NoError(t, stream.Destroy(ctx)) +} + +func TestSinkRecoveryLeaseFencesAutoClaimAfterTakeover(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + stream, err := NewStream(t.Name(), rdb) + require.NoError(t, err) + _, err = stream.Add(ctx, "event", []byte("payload")) + require.NoError(t, err) + require.NoError(t, rdb.XGroupCreate(ctx, stream.key, "sink", "0").Err()) + claimed, err := rdb.XReadGroup(ctx, &redis.XReadGroupArgs{ + Group: "sink", + Consumer: "predecessor", + Streams: []string{stream.key, ">"}, + Count: 1, + }).Result() + require.NoError(t, err) + require.Len(t, claimed, 1) + sink, err := stream.NewSink( + ctx, + "sink", + options.WithSinkStartAtOldest(), + options.WithSinkAckGracePeriod(time.Hour), + options.WithSinkBlockDuration(testBlockDuration), + ) + require.NoError(t, err) + state := sink.streams[stream.key] + + require.NoError(t, rdb.HSet(ctx, state.leaseKey, "lease_until", "0").Err()) + first, acquired, err := acquireSinkRecoveryLease( + ctx, + stream, + state.leaseKey, + "first-owner", + 20*time.Millisecond, + ) + require.NoError(t, err) + require.True(t, acquired) + time.Sleep(30 * time.Millisecond) + second, acquired, err := acquireSinkRecoveryLease( + ctx, + stream, + state.leaseKey, + "second-owner", + time.Second, + ) + require.NoError(t, err) + require.True(t, acquired) + + require.NoError(t, rdb.ScriptFlush(ctx).Err()) + _, _, err = fencedAutoClaim( + ctx, + stream, + state.leaseKey, + first, + time.Second, + sink.Name, + sink.consumer, + 0, + "0-0", + 1, + ) + require.ErrorContains(t, err, "SINKLEASELOST") + pending, err := rdb.XPendingExt(ctx, &redis.XPendingExtArgs{ + Stream: stream.key, + Group: sink.Name, + Start: "-", + End: "+", + Count: 1, + }).Result() + require.NoError(t, err) + require.Equal(t, "predecessor", pending[0].Consumer) + + _, messages, err := fencedAutoClaim( + ctx, + stream, + state.leaseKey, + second, + time.Second, + sink.Name, + sink.consumer, + 0, + "0-0", + 1, + ) + require.NoError(t, err) + require.Len(t, messages, 1) + pending, err = rdb.XPendingExt(ctx, &redis.XPendingExtArgs{ + Stream: stream.key, + Group: sink.Name, + Start: "-", + End: "+", + Count: 1, + }).Result() + require.NoError(t, err) + require.Equal(t, sink.consumer, pending[0].Consumer) + + require.NoError(t, rdb.XGroupCreateConsumer(ctx, stream.key, sink.Name, "stale").Err()) + _, err = state.keepAlives.Set(ctx, "stale", "0") + require.NoError(t, err) + require.NoError(t, rdb.ScriptFlush(ctx).Err()) + _, _, err = cleanupStaleConsumers( + ctx, + stream, + state.leaseKey, + first, + time.Second, + sink.Name, + time.Millisecond, + ) + require.ErrorContains(t, err, "SINKLEASELOST") + consumers, err := rdb.XInfoConsumers(ctx, stream.key, sink.Name).Result() + require.NoError(t, err) + require.Contains(t, consumerNames(consumers), "stale") + + removed, malformed, err := cleanupStaleConsumers( + ctx, + stream, + state.leaseKey, + second, + time.Second, + sink.Name, + time.Millisecond, + ) + require.NoError(t, err) + require.Contains(t, removed, "stale") + require.Empty(t, malformed) + consumers, err = rdb.XInfoConsumers(ctx, stream.key, sink.Name).Result() + require.NoError(t, err) + require.NotContains(t, consumerNames(consumers), "stale") + + require.NoError(t, sink.Close(ctx)) + require.NoError(t, stream.Destroy(ctx)) +} +func TestStaleConsumerSweepSkipsMalformedKeepAliveAndContinues(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + stream, err := NewStream(t.Name(), rdb) + require.NoError(t, err) + sink, err := stream.NewSink( + ctx, + "sink", + options.WithSinkAckGracePeriod(testAckDuration), + options.WithSinkBlockDuration(testBlockDuration), + ) + require.NoError(t, err) + state := sink.streams[stream.key] + require.NoError(t, rdb.XGroupCreateConsumer(ctx, stream.key, sink.Name, "malformed").Err()) + require.NoError(t, rdb.XGroupCreateConsumer(ctx, stream.key, sink.Name, "stale").Err()) + _, err = state.keepAlives.Set(ctx, "malformed", "not-a-timestamp") + require.NoError(t, err) + _, err = state.keepAlives.Set(ctx, "stale", "0") + require.NoError(t, err) + lease, acquired, err := acquireSinkRecoveryLease( + ctx, + stream, + state.leaseKey, + state.leaseOwner, + time.Second, + ) + require.NoError(t, err) + require.True(t, acquired) + require.NoError(t, sink.deleteStreamStaleConsumers(ctx, state, lease, time.Second)) + consumers, err := rdb.XInfoConsumers(ctx, stream.key, sink.Name).Result() + require.NoError(t, err) + names := make([]string, len(consumers)) + for i, consumer := range consumers { + names[i] = consumer.Name + } + require.Contains(t, names, "malformed") + require.NotContains(t, names, "stale") + + require.NoError(t, sink.Close(ctx)) + require.NoError(t, stream.Destroy(ctx)) +} + +func TestStreamingScriptsRecoverAfterScriptFlush(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + stream, err := NewStream(t.Name(), rdb) + require.NoError(t, err) + sink, err := stream.NewSink( + ctx, + "sink", + options.WithSinkStartAtOldest(), + options.WithSinkAckGracePeriod(testAckDuration), + options.WithSinkBlockDuration(testBlockDuration), + ) + require.NoError(t, err) + events := sink.Subscribe() + + require.NoError(t, rdb.ScriptFlush(ctx).Err()) + require.NoError(t, rdb.XGroupDestroy(ctx, stream.key, sink.Name).Err()) + _, err = stream.Add(ctx, "after-flush", []byte("payload")) + require.NoError(t, err) + event := receiveSinkEvent(t, events) + require.Equal(t, "after-flush", event.EventName) + require.NoError(t, sink.Ack(ctx, event)) + + require.NoError(t, sink.Close(ctx)) + require.NoError(t, stream.Destroy(ctx)) +} + +func TestSinkRejectsInvalidOptions(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + cases := []struct { + name string + opt options.Sink + }{ + {name: "max polled", opt: options.WithSinkMaxPolled(0)}, + {name: "buffer", opt: options.WithSinkBufferSize(-1)}, + {name: "ack grace", opt: options.WithSinkAckGracePeriod(0)}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + stream, err := NewStream(t.Name(), rdb) + require.NoError(t, err) + _, err = stream.NewSink(ctx, "sink", tc.opt) + require.Error(t, err) + }) + } + stream, err := NewStream(t.Name()+"-conflicts", rdb) + require.NoError(t, err) + _, err = stream.NewSink( + ctx, + "sink", + options.WithSinkTopic("alarms"), + options.WithSinkTopicPattern("alarm.*"), + ) + require.ErrorContains(t, err, "mutually exclusive") + _, err = stream.NewSink( + ctx, + "sink", + options.WithSinkStartAtNewest(), + options.WithSinkStartAtOldest(), + ) + require.ErrorContains(t, err, "sink cursor-start options are mutually exclusive") + + sink, err := stream.NewSink(ctx, "sink") + require.NoError(t, err) + added, err := NewStream(t.Name()+"-added", rdb) + require.NoError(t, err) + err = sink.AddStream( + ctx, + added, + options.WithAddStreamStartAtNewest(), + options.WithAddStreamStartAtOldest(), + ) + require.ErrorContains(t, err, "added stream cursor-start options are mutually exclusive") + require.NoError(t, sink.Close(ctx)) + require.NoError(t, stream.Destroy(ctx)) + require.ErrorIs(t, added.Destroy(ctx), ErrStreamNotFound) +} + +func TestSinkAcknowledgesMalformedRedisEvent(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + stream, err := NewStream(t.Name(), rdb) + require.NoError(t, err) + require.NoError(t, stream.Open(ctx)) + require.NoError(t, rdb.XAdd(ctx, &redis.XAddArgs{ + Stream: stream.key, + Values: map[string]any{payloadKey: "missing name"}, + }).Err()) + sink, err := stream.NewSink( + ctx, + "sink", + options.WithSinkStartAtOldest(), + options.WithSinkAckGracePeriod(testAckDuration), + options.WithSinkBlockDuration(testBlockDuration), + ) + require.NoError(t, err) + events := sink.Subscribe() + _, err = stream.Add(ctx, "valid", []byte("payload")) + require.NoError(t, err) + event := receiveSinkEvent(t, events) + require.Equal(t, "valid", event.EventName) + require.NoError(t, sink.Ack(ctx, event)) + require.Eventually(t, func() bool { + pending, pendingErr := rdb.XPending(ctx, stream.key, sink.Name).Result() + return pendingErr == nil && pending.Count == 0 + }, time.Second, delay) + + require.NoError(t, sink.Close(ctx)) + require.NoError(t, stream.Destroy(ctx)) +} + +func TestDestroyedStreamSetupRemovesOrphanMembership(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + stream, err := NewStream(t.Name(), rdb) + require.NoError(t, err) + require.NoError(t, stream.Open(ctx)) + membership, err := rmap.Join(ctx, consumersMapName(stream), rdb) + require.NoError(t, err) + membershipKey := consumersMapContentKey(stream) + require.NoError(t, stream.Destroy(ctx)) + + _, setupErr := ensureConsumerGroup(ctx, stream, "sink", "0") + require.ErrorIs(t, setupErr, ErrStreamDestroyed) + closeSetupMembership(ctx, stream, membership, setupErr) + require.EqualValues(t, 0, rdb.Exists(ctx, membershipKey).Val()) +} + func TestSinkCloseWithStalledSubscriber(t *testing.T) { testName := strings.Replace(t.Name(), "/", "_", -1) rdb := ptesting.NewRedisClient(t) @@ -51,13 +623,13 @@ func TestSinkCloseWithStalledSubscriber(t *testing.T) { require.Eventually(t, func() bool { return len(c) == cap(c) }, max, delay) // Close must return even though the subscriber stalled the read loop. - done := make(chan struct{}) + done := make(chan error, 1) go func() { - sink.Close(ctx) - close(done) + done <- sink.Close(ctx) }() select { - case <-done: + case err := <-done: + require.NoError(t, err) case <-time.After(2 * time.Second): t.Fatal("sink.Close() hung with a stalled subscriber") } @@ -83,6 +655,27 @@ func TestNewSink(t *testing.T) { assert.EqualError(t, err, "topic pattern must be a valid regex: error parsing regexp: missing closing ): `(`") } +func TestSinkRejectsSubMillisecondDurations(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + stream, err := NewStream(t.Name(), rdb, options.WithStreamLogger(pulse.ClueLogger(ctx))) + require.NoError(t, err) + for _, duration := range []time.Duration{0, -time.Second, 500 * time.Microsecond} { + sink, err := stream.NewSink(ctx, "sink", options.WithSinkBlockDuration(duration)) + require.Nil(t, sink) + require.EqualError(t, err, "sink block duration must be at least 1ms") + } + sink, err := stream.NewSink( + ctx, + "sink", + options.WithSinkAckGracePeriod(500*time.Microsecond), + ) + require.Nil(t, sink) + require.EqualError(t, err, "sink acknowledgement grace period must be at least 1ms") + require.ErrorIs(t, stream.Destroy(ctx), ErrStreamNotFound) +} + func TestReadOnce(t *testing.T) { testName := strings.Replace(t.Name(), "/", "_", -1) rdb := ptesting.NewRedisClient(t) @@ -111,6 +704,7 @@ func TestReadSinceLastEvent(t *testing.T) { ctx := ptesting.NewTestContext(t) s, err := NewStream(testName, rdb, options.WithStreamLogger(pulse.ClueLogger(ctx))) assert.NoError(t, err) + defer func() { require.NoError(t, s.Destroy(ctx)) }() // Add and read 2 events consecutively sink, err := s.NewSink(ctx, "sink", @@ -118,7 +712,7 @@ func TestReadSinceLastEvent(t *testing.T) { options.WithSinkBlockDuration(testBlockDuration), ) require.NoError(t, err) - defer cleanupSink(t, ctx, s, sink) + defer func() { require.NoError(t, sink.Close(ctx)) }() c := sink.Subscribe() _, err = s.Add(ctx, "event", []byte("payload")) assert.NoError(t, err) @@ -137,7 +731,7 @@ func TestReadSinceLastEvent(t *testing.T) { options.WithSinkStartAfter(eventID), options.WithSinkBlockDuration(testBlockDuration)) require.NoError(t, err) - defer cleanupSink(t, ctx, s, sink2) + defer func() { require.NoError(t, sink2.Close(ctx)) }() c2 := sink2.Subscribe() read = readOneEvent(t, ctx, c2, sink2) assert.Equal(t, "event", read.EventName) @@ -148,7 +742,7 @@ func TestReadSinceLastEvent(t *testing.T) { options.WithSinkStartAfter("0"), options.WithSinkBlockDuration(testBlockDuration)) require.NoError(t, err) - defer cleanupSink(t, ctx, s, sink3) + defer func() { require.NoError(t, sink3.Close(ctx)) }() c3 := sink3.Subscribe() read = readOneEvent(t, ctx, c3, sink3) assert.Equal(t, "event", read.EventName) @@ -179,7 +773,7 @@ func TestCleanup(t *testing.T) { assert.Equal(t, []byte("payload"), read.Payload) // Stop sink, destroy stream and check Redis keys are gone - sink.Close(ctx) + require.NoError(t, sink.Close(ctx)) assert.Eventually(t, func() bool { return sink.IsClosed() }, max, delay) assert.Equal(t, rdb.Exists(ctx, s.key).Val(), int64(1)) assert.NoError(t, s.Destroy(ctx)) @@ -257,14 +851,14 @@ func TestRemoveStream(t *testing.T) { // Remove one stream and read again err = sink.RemoveStream(ctx, s2) assert.NoError(t, err) - eventID, err := s.Add(ctx, "event3", []byte("payload3")) + _, err = s.Add(ctx, "event3", []byte("payload3")) assert.NoError(t, err) read = readOneEvent(t, ctx, c, sink) assert.Equal(t, "event3", read.EventName) assert.Equal(t, []byte("payload3"), read.Payload) // Add back and remove other stream - err = sink.AddStream(ctx, s2, options.WithAddStreamStartAfter(eventID)) + err = sink.AddStream(ctx, s2) assert.NoError(t, err) err = sink.RemoveStream(ctx, s) assert.NoError(t, err) @@ -275,6 +869,273 @@ func TestRemoveStream(t *testing.T) { assert.Equal(t, []byte("payload4"), read.Payload) } +func TestRemoveStreamDeletesOnlyEmptyConsumerMetadata(t *testing.T) { + testName := strings.ReplaceAll(t.Name(), "/", "_") + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + primary, err := NewStream(testName+"-primary", rdb) + require.NoError(t, err) + secondary, err := NewStream(testName+"-secondary", rdb) + require.NoError(t, err) + sink, err := primary.NewSink( + ctx, + "sink", + options.WithSinkStartAtOldest(), + options.WithSinkBlockDuration(testBlockDuration), + ) + require.NoError(t, err) + require.NoError(t, sink.AddStream(ctx, secondary)) + defer func() { + assert.NoError(t, sink.Close(ctx)) + assert.NoError(t, primary.Destroy(ctx)) + assert.NoError(t, secondary.Destroy(ctx)) + }() + + require.NoError(t, rdb.XGroupCreateConsumer( + ctx, + secondary.key, + sink.Name, + "pending-consumer", + ).Err()) + eventID, err := secondary.Add(ctx, "event", nil) + require.NoError(t, err) + require.Eventually(t, func() bool { + pending, pendingErr := rdb.XPending(ctx, secondary.key, sink.Name).Result() + return pendingErr == nil && pending.Count == 1 + }, max, delay) + _, err = rdb.XClaim(ctx, &redis.XClaimArgs{ + Stream: secondary.key, + Group: sink.Name, + Consumer: "pending-consumer", + MinIdle: 0, + Messages: []string{eventID}, + }).Result() + require.NoError(t, err) + require.Eventually(t, func() bool { + pending, pendingErr := rdb.XPendingExt(ctx, &redis.XPendingExtArgs{ + Stream: secondary.key, + Group: sink.Name, + Consumer: sink.consumer, + Start: "-", + End: "+", + Count: 1, + }).Result() + return pendingErr == nil && len(pending) == 0 + }, max, delay) + + require.NoError(t, sink.RemoveStream(ctx, secondary)) + consumers, err := rdb.XInfoConsumers(ctx, secondary.key, sink.Name).Result() + require.NoError(t, err) + names := make([]string, 0, len(consumers)) + for _, consumer := range consumers { + names = append(names, consumer.Name) + } + assert.NotContains(t, names, sink.consumer) + assert.Contains(t, names, "pending-consumer") +} + +func TestSinkRecoversExternallyDeletedStream(t *testing.T) { + testName := strings.ReplaceAll(t.Name(), "/", "_") + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + hook := &redisCommandHook{} + rdb.AddHook(hook) + ctx := ptesting.NewTestContext(t) + + stream, err := NewStream(testName, rdb, options.WithStreamLogger(pulse.ClueLogger(ctx))) + require.NoError(t, err) + sink, err := stream.NewSink(ctx, "sink", options.WithSinkBlockDuration(testBlockDuration)) + require.NoError(t, err) + defer cleanupSink(t, ctx, stream, sink) + events := sink.Subscribe() + + require.Eventually(t, func() bool { return hook.xreadGroups.Load() > 0 }, max, delay) + readsBeforeDelete := hook.xreadGroups.Load() + require.NoError(t, rdb.Del(ctx, stream.key).Err()) + + require.Eventually(t, func() bool { return consumerGroupExists(ctx, rdb, stream.key, sink.Name) }, max, delay) + time.Sleep(4 * testBlockDuration) + assert.LessOrEqual(t, hook.xreadGroups.Load()-readsBeforeDelete, int64(10), + "NOGROUP recovery issued XREADGROUP in a hot loop") + + _, err = stream.Add(ctx, "future", []byte("payload")) + require.NoError(t, err) + read := readOneEvent(t, ctx, events, sink) + assert.Equal(t, "future", read.EventName) + assert.Equal(t, []byte("payload"), read.Payload) +} + +func TestSinkRecoversDestroyedGroupAcrossStreams(t *testing.T) { + testName := strings.ReplaceAll(t.Name(), "/", "_") + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + hook := &redisCommandHook{} + rdb.AddHook(hook) + ctx := ptesting.NewTestContext(t) + + mainStream, err := NewStream(testName, rdb, options.WithStreamLogger(pulse.ClueLogger(ctx))) + require.NoError(t, err) + addedStream, err := NewStream(testName+"-added", rdb, options.WithStreamLogger(pulse.ClueLogger(ctx))) + require.NoError(t, err) + defer func() { assert.NoError(t, addedStream.Destroy(ctx)) }() + + sink, err := mainStream.NewSink(ctx, "sink", options.WithSinkBlockDuration(testBlockDuration)) + require.NoError(t, err) + defer cleanupSink(t, ctx, mainStream, sink) + require.NoError(t, sink.AddStream(ctx, addedStream, options.WithAddStreamStartAtOldest())) + events := sink.Subscribe() + + readsBeforeAdd := hook.xreadGroups.Load() + require.Eventually(t, func() bool { return hook.xreadGroups.Load() > readsBeforeAdd }, max, delay) + + // Hold the ownership lock after the multi-stream read has started. Redis + // reports NOGROUP for that read when the added stream's group is destroyed, + // but recovery cannot run until the event below exists. This proves the + // added stream's own start position is retained for recovery. + sink.lock.Lock() + destroyed, destroyErr := rdb.XGroupDestroy(ctx, addedStream.key, sink.Name).Result() + _, addErr := addedStream.Add(ctx, "recovered", []byte("added payload")) + sink.lock.Unlock() + require.NoError(t, destroyErr) + require.EqualValues(t, 1, destroyed) + require.NoError(t, addErr) + + read := readOneEvent(t, ctx, events, sink) + assert.Equal(t, addedStream.Name, read.StreamName) + assert.Equal(t, "recovered", read.EventName) + require.True(t, consumerGroupExists(ctx, rdb, mainStream.key, sink.Name)) + require.True(t, consumerGroupExists(ctx, rdb, addedStream.key, sink.Name)) + + _, err = mainStream.Add(ctx, "main", []byte("main payload")) + require.NoError(t, err) + read = readOneEvent(t, ctx, events, sink) + assert.Equal(t, mainStream.Name, read.StreamName) + assert.Equal(t, "main", read.EventName) +} + +func TestSinkRecoveryDoesNotRecreateRemovedStream(t *testing.T) { + testName := strings.ReplaceAll(t.Name(), "/", "_") + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + hook := &redisCommandHook{} + rdb.AddHook(hook) + ctx := ptesting.NewTestContext(t) + + mainStream, err := NewStream(testName, rdb, options.WithStreamLogger(pulse.ClueLogger(ctx))) + require.NoError(t, err) + removedStream, err := NewStream(testName+"-removed", rdb, options.WithStreamLogger(pulse.ClueLogger(ctx))) + require.NoError(t, err) + defer func() { assert.NoError(t, removedStream.Destroy(ctx)) }() + + sink, err := mainStream.NewSink(ctx, "sink", options.WithSinkBlockDuration(testBlockDuration)) + require.NoError(t, err) + defer cleanupSink(t, ctx, mainStream, sink) + require.NoError(t, sink.AddStream(ctx, removedStream)) + events := sink.Subscribe() + + readsBeforeAdd := hook.xreadGroups.Load() + require.Eventually(t, func() bool { return hook.xreadGroups.Load() > readsBeforeAdd }, max, delay) + require.NoError(t, sink.RemoveStream(ctx, removedStream)) + destroyed, err := rdb.XGroupDestroy(ctx, removedStream.key, sink.Name).Result() + require.NoError(t, err) + require.EqualValues(t, 1, destroyed) + + assert.Never(t, func() bool { + return consumerGroupExists(ctx, rdb, removedStream.key, sink.Name) + }, 4*testBlockDuration, delay, "recovery recreated a group for a removed stream") + + _, err = mainStream.Add(ctx, "remaining", []byte("payload")) + require.NoError(t, err) + read := readOneEvent(t, ctx, events, sink) + assert.Equal(t, mainStream.Name, read.StreamName) + assert.Equal(t, "remaining", read.EventName) +} + +func TestSinkBoundsRetriesDuringRedisFailure(t *testing.T) { + testName := strings.ReplaceAll(t.Name(), "/", "_") + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + failure := errors.New("test Redis outage") + hook := &redisCommandHook{failure: failure} + rdb.AddHook(hook) + ctx := ptesting.NewTestContext(t) + + stream, err := NewStream(testName, rdb, options.WithStreamLogger(pulse.ClueLogger(ctx))) + require.NoError(t, err) + sink, err := stream.NewSink(ctx, "sink", options.WithSinkBlockDuration(20*time.Millisecond)) + require.NoError(t, err) + defer cleanupSink(t, ctx, stream, sink) + events := sink.Subscribe() + + hook.failGroup.Store(true) + hook.failGroupKey = stream.lifecycleKey + sink.lock.Lock() + err = sink.recoverConsumerGroups(ctx) + sink.lock.Unlock() + require.ErrorIs(t, err, failure) + assert.EqualError(t, err, + `failed to ensure Redis consumer group "sink" for stream "`+testName+`" generation 1: test Redis outage`) + hook.failGroup.Store(false) + + require.Eventually(t, func() bool { return hook.xreadGroups.Load() > 0 }, max, delay) + readsBeforeFailure := hook.xreadGroups.Load() + hook.failRead.Store(true) + time.Sleep(350 * time.Millisecond) + hook.failRead.Store(false) + assert.LessOrEqual(t, hook.xreadGroups.Load()-readsBeforeFailure, int64(4), + "Redis outage retries exceeded the bounded backoff rate") + + _, err = stream.Add(ctx, "after-outage", []byte("payload")) + require.NoError(t, err) + read := readOneEvent(t, ctx, events, sink) + assert.Equal(t, "after-outage", read.EventName) +} + +func TestSinkCloseDuringConsumerGroupRecovery(t *testing.T) { + testName := strings.ReplaceAll(t.Name(), "/", "_") + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + hook := &redisCommandHook{} + rdb.AddHook(hook) + ctx := ptesting.NewTestContext(t) + + stream, err := NewStream(testName, rdb, options.WithStreamLogger(pulse.ClueLogger(ctx))) + require.NoError(t, err) + sink, err := stream.NewSink(ctx, "sink", options.WithSinkBlockDuration(10*time.Millisecond)) + require.NoError(t, err) + + require.Eventually(t, func() bool { return hook.xreadGroups.Load() > 0 }, max, delay) + disruptionDone := make(chan error, 1) + go func() { + for range 50 { + if err := rdb.XGroupDestroy(ctx, stream.key, sink.Name).Err(); err != nil { + disruptionDone <- err + return + } + } + disruptionDone <- nil + }() + + closeDone := make(chan error, 1) + go func() { + closeDone <- sink.Close(ctx) + }() + select { + case err := <-closeDone: + require.NoError(t, err) + case <-time.After(2 * time.Second): + t.Fatal("sink.Close hung during consumer-group recovery") + } + require.NoError(t, <-disruptionDone) + require.True(t, sink.IsClosed()) + + readsAfterClose := hook.xreadGroups.Load() + time.Sleep(4 * testBlockDuration) + assert.Equal(t, readsAfterClose, hook.xreadGroups.Load(), "closed sink continued reading") + require.NoError(t, stream.Destroy(ctx)) +} + func TestMultipleConsumers(t *testing.T) { testName := strings.Replace(t.Name(), "/", "_", -1) rdb := ptesting.NewRedisClient(t) @@ -296,7 +1157,7 @@ func TestMultipleConsumers(t *testing.T) { options.WithSinkAckGracePeriod(testAckDuration)) require.NoError(t, err) defer func() { - sink2.Close(ctx) + require.NoError(t, sink2.Close(ctx)) assert.Eventually(t, func() bool { return sink2.IsClosed() }, max, delay) }() @@ -355,7 +1216,7 @@ func TestClaimStaleMessages(t *testing.T) { var read *Event select { case read = <-c: - case <-time.After(testAckDuration): + case <-time.After(max): t.Fatal("timeout waiting for initial event") } assert.Equal(t, "event", read.EventName) @@ -365,7 +1226,7 @@ func TestClaimStaleMessages(t *testing.T) { select { case read = <-c: assert.NoError(t, sink.Ack(ctx, read)) - case <-time.After(testAckDuration * 2): + case <-time.After(max): t.Fatal("timeout waiting for claimed event") } assert.Equal(t, "event", read.EventName) @@ -406,7 +1267,7 @@ func TestNonAckMessageDeliveredToAnotherConsumer(t *testing.T) { options.WithSinkBlockDuration(testBlockDuration), options.WithSinkAckGracePeriod(testAckDuration)) require.NoError(t, err) - defer sink2.Close(ctx) + defer func() { require.NoError(t, sink2.Close(ctx)) }() // Subscribe to both sinks c1 := sink1.Subscribe() @@ -431,14 +1292,14 @@ func TestNonAckMessageDeliveredToAnotherConsumer(t *testing.T) { receiverSink = sink2 otherSink = sink1 otherChan = c1 - case <-time.After(testAckDuration): + case <-time.After(max): t.Fatal("Timeout waiting for event on first sink") } assert.Equal(t, "test_event", read1.EventName) assert.Equal(t, []byte("test_payload"), read1.Payload) // Close the receiver sink - receiverSink.Close(ctx) + require.NoError(t, receiverSink.Close(ctx)) assert.Eventually(t, func() bool { return receiverSink.IsClosed() }, max, delay) // The message should now be redelivered to the other sink @@ -447,7 +1308,7 @@ func TestNonAckMessageDeliveredToAnotherConsumer(t *testing.T) { case read2 = <-otherChan: logger.Info("Read from other sink") assert.NoError(t, otherSink.Ack(ctx, read2)) - case <-time.After(testAckDuration * 4): + case <-time.After(testAckDuration * 20): t.Fatal("Timeout waiting for event on other sink") } assert.Equal(t, "test_event", read2.EventName) @@ -487,7 +1348,7 @@ func TestStaleConsumerDeletionAndMessageClaiming(t *testing.T) { var read *Event select { case read = <-c1: - case <-time.After(testAckDuration): + case <-time.After(max): t.Fatal("Timeout waiting for event on first sink") } assert.Equal(t, "test_event", read.EventName) @@ -512,7 +1373,7 @@ func TestStaleConsumerDeletionAndMessageClaiming(t *testing.T) { }, max, delay, "Expected two consumers") // Close the sink to stop keep-alive refresh - sink1.Close(ctx) + require.NoError(t, sink1.Close(ctx)) assert.Eventually(t, func() bool { return sink1.IsClosed() }, max, delay) // Verify that the stale consumer is deleted @@ -533,9 +1394,91 @@ func TestStaleConsumerDeletionAndMessageClaiming(t *testing.T) { select { case claimedRead = <-c2: assert.NoError(t, sink2.Ack(ctx, claimedRead)) - case <-time.After(testAckDuration * 2): + case <-time.After(max): t.Fatal("Timeout waiting for claimed event") } assert.Equal(t, "test_event", claimedRead.EventName) assert.Equal(t, []byte("test_payload"), claimedRead.Payload) } + +// consumerGroupExists reports whether Redis currently holds group for stream. +func consumerGroupExists(ctx context.Context, rdb *redis.Client, stream, group string) bool { + groups, err := rdb.XInfoGroups(ctx, stream).Result() + if err != nil { + return false + } + for _, candidate := range groups { + if candidate.Name == group { + return true + } + } + return false +} + +// DialHook preserves the client's normal Redis connection behavior. +func (h *redisCommandHook) DialHook(next redis.DialHook) redis.DialHook { + return func(ctx context.Context, network, addr string) (net.Conn, error) { + return next(ctx, network, addr) + } +} + +// ProcessHook counts group reads and injects command-specific transport +// failures used to prove bounded retries and exact error propagation. +func (h *redisCommandHook) ProcessHook(next redis.ProcessHook) redis.ProcessHook { + return func(ctx context.Context, cmd redis.Cmder) error { + switch cmd.Name() { + case "xreadgroup": + h.xreadGroups.Add(1) + for _, arg := range cmd.Args() { + if value, ok := arg.(string); ok && strings.EqualFold(value, "noack") { + h.usedNoAck.Store(true) + } + } + if h.blockRead.Load() { + h.started.Do(func() { + close(h.readStarted) + }) + <-ctx.Done() + return ctx.Err() + } + if h.failRead.Load() { + return h.failure + } + case "xgroup": + if h.failGroup.Load() { + return h.failure + } + case "evalsha": + if h.failCreateConsumer.Load() && len(cmd.Args()) > 1 && cmd.Args()[1] == registerSinkConsumerScript.Hash() { + for _, arg := range cmd.Args() { + if key, ok := arg.(string); ok && key == h.failCreateConsumerKey { + return h.failure + } + } + } + if h.failGroup.Load() { + for _, arg := range cmd.Args() { + if key, ok := arg.(string); ok && key == h.failGroupKey { + return h.failure + } + } + } + if h.failRemove.Load() { + for _, arg := range cmd.Args() { + if key, ok := arg.(string); ok && key == h.failMembershipKey { + return h.failure + } + } + } + } + return next(ctx, cmd) + } +} + +// ProcessPipelineHook preserves pipeline behavior; recovery uses direct Redis +// commands so no injected failure is expected through this path. +func (h *redisCommandHook) ProcessPipelineHook(next redis.ProcessPipelineHook) redis.ProcessPipelineHook { + return func(ctx context.Context, cmds []redis.Cmder) error { + return next(ctx, cmds) + } +} diff --git a/streaming/stream_lifecycle.go b/streaming/stream_lifecycle.go new file mode 100644 index 0000000..946fc06 --- /dev/null +++ b/streaming/stream_lifecycle.go @@ -0,0 +1,732 @@ +// Package streaming adopts pre-generation event data in place, then assigns a +// distinct physical Redis key to every explicitly recreated incarnation. +// Stream handles bind lazily to the lifecycle-selected generation and key. +package streaming + +import ( + "context" + "fmt" + "strconv" + "strings" + "time" + + redis "github.com/redis/go-redis/v9" +) + +const ( + streamStateActive = "active" + streamStateDestroyed = "destroyed" + streamDestroyedError = "STREAMDESTROYED" + streamPhysicalKey = "physical_key" + streamDeadlineKey = "deadline_ms" + streamTTLOwnedKey = "ttl_owned" + streamDeadlineError = "DEADLINEELAPSED" + streamConflictError = "STREAMDEADLINECONFLICT" + streamConfigKey = "retention_config" + streamConfigError = "STREAMCONFIGMISMATCH" + streamNotFoundError = "STREAMNOTFOUND" + streamDeadlineRequiredError = "STREAMDEADLINEREQUIRED" + streamConfigMissingError = "STREAMCONFIGMISSING" + streamSnapshotUnboundedError = "SNAPSHOTUNBOUNDED" + streamSnapshotBoundError = "SNAPSHOTBOUNDVIOLATION" + streamFormatVersion = "2" +) + +var ( + // establishStreamScript adopts an existing flat stream for generation one. + // Explicit recreation receives a distinct physical key. + establishStreamScript = redis.NewScript(` +local state = redis.call("HGET", KEYS[1], "state") +local generation = redis.call("HGET", KEYS[1], "generation") +local physical = redis.call("HGET", KEYS[1], ARGV[2]) +local deadline = redis.call("HGET", KEYS[1], ARGV[6]) +local ttl_owned = redis.call("HGET", KEYS[1], ARGV[7]) +local retention = redis.call("HGET", KEYS[1], ARGV[8]) +local physical_missing = not physical +if ARGV[4] ~= "" and ARGV[5] == "1" then + return redis.error_reply("STREAMDEADLINECONFLICT") +end +if ARGV[4] ~= "" then + local clock = redis.call("TIME") + local now = tonumber(clock[1]) * 1000 + math.floor(tonumber(clock[2]) / 1000) + if now >= tonumber(ARGV[4]) then + return redis.error_reply("DEADLINEELAPSED") + end +end +if not generation and ARGV[10] == "1" and tonumber(ARGV[11]) > 0 then + redis.call("XTRIM", ARGV[3], "MAXLEN", "=", ARGV[11]) +end +if not generation then + generation = "1" + physical = ARGV[3] + redis.call("HSET", KEYS[1], "generation", generation, "state", ARGV[1], ARGV[2], physical, ARGV[8], ARGV[9]) + if ARGV[4] ~= "" then + deadline = ARGV[4] + redis.call("HSET", KEYS[1], ARGV[6], deadline) + elseif ARGV[5] == "1" then + ttl_owned = "1" + redis.call("HSET", KEYS[1], ARGV[7], ttl_owned) + end +elseif state ~= ARGV[1] then + generation = tostring(redis.call("HINCRBY", KEYS[1], "generation", 1)) + physical = ARGV[3] .. ":generation:" .. generation + redis.call("HSET", KEYS[1], "state", ARGV[1], ARGV[2], physical, ARGV[8], ARGV[9]) + redis.call("HDEL", KEYS[1], ARGV[6], ARGV[7]) + deadline = false + ttl_owned = false + if ARGV[4] ~= "" then + deadline = ARGV[4] + redis.call("HSET", KEYS[1], ARGV[6], deadline) + elseif ARGV[5] == "1" then + ttl_owned = "1" + redis.call("HSET", KEYS[1], ARGV[7], ttl_owned) + end +elseif not physical then + physical = ARGV[3] +end +if state == ARGV[1] then + if retention and ARGV[10] == "1" and retention ~= ARGV[9] then + return redis.error_reply("STREAMCONFIGMISMATCH") + end + if deadline then + if (ARGV[4] ~= "" and ARGV[4] ~= deadline) or ARGV[5] == "1" then + return redis.error_reply("STREAMDEADLINECONFLICT") + end + elseif ttl_owned == "1" then + if ARGV[4] ~= "" then + return redis.error_reply("STREAMDEADLINECONFLICT") + end + end + if physical_missing then + redis.call("HSET", KEYS[1], ARGV[2], physical) + end + if not retention then + if ARGV[10] == "1" and tonumber(ARGV[11]) > 0 and physical == ARGV[3] then + redis.call("XTRIM", physical, "MAXLEN", "=", ARGV[11]) + end + redis.call("HSET", KEYS[1], ARGV[8], ARGV[9]) + retention = ARGV[9] + end + if not deadline and ttl_owned ~= "1" then + if ARGV[4] ~= "" then + deadline = ARGV[4] + redis.call("HSET", KEYS[1], ARGV[6], deadline) + elseif ARGV[5] == "1" then + redis.call("HSET", KEYS[1], ARGV[7], "1") + end + end +end +return {generation, physical, deadline or "", retention or ARGV[9]} +`) + + // loadCurrentGenerationScript binds an unbound non-creating operation to + // the current generation without advancing a destroyed lifecycle. + loadCurrentGenerationScript = redis.NewScript(` +local state = redis.call("HGET", KEYS[1], "state") +local generation = redis.call("HGET", KEYS[1], "generation") +local physical = redis.call("HGET", KEYS[1], ARGV[2]) +local deadline = redis.call("HGET", KEYS[1], ARGV[6]) +local ttl_owned = redis.call("HGET", KEYS[1], ARGV[7]) +local retention = redis.call("HGET", KEYS[1], ARGV[8]) +if not generation then + return redis.error_reply("STREAMNOTFOUND") +end +if state ~= ARGV[1] then + return redis.error_reply("STREAMDESTROYED") +end +if not physical then + physical = ARGV[3] +end +if retention then + if ARGV[10] == "1" and retention ~= ARGV[9] then + return redis.error_reply("STREAMCONFIGMISMATCH") + end +end +if deadline then + if (ARGV[4] ~= "" and ARGV[4] ~= deadline) or ARGV[5] == "1" then + return redis.error_reply("STREAMDEADLINECONFLICT") + end +elseif ttl_owned == "1" then + if ARGV[4] ~= "" then + return redis.error_reply("STREAMDEADLINECONFLICT") + end +elseif ARGV[4] ~= "" or ARGV[5] == "1" then + return redis.error_reply("STREAMDEADLINECONFLICT") +end +if not retention then + retention = ARGV[9] + redis.call("HSET", KEYS[1], ARGV[8], retention) +end +return {generation, physical, deadline or "", retention} +`) + + // verifyStreamScript is used by constructors whose subsequent Redis + // operations cannot recreate stream state. + verifyStreamScript = redis.NewScript(` +if redis.call("HGET", KEYS[1], "state") ~= ARGV[1] +or redis.call("HGET", KEYS[1], "generation") ~= ARGV[2] +or redis.call("HGET", KEYS[1], ARGV[3]) ~= ARGV[4] then + return redis.error_reply("STREAMDESTROYED") +end +if redis.call("HGET", KEYS[1], ARGV[6]) ~= ARGV[7] then + return redis.error_reply("STREAMCONFIGMISMATCH") +end +local deadline = redis.call("HGET", KEYS[1], ARGV[5]) +if deadline then + local now = redis.call("TIME") + local now_ms = tonumber(now[1]) * 1000 + math.floor(tonumber(now[2]) / 1000) + if now_ms >= tonumber(deadline) then + return redis.error_reply("DEADLINEELAPSED") + end +end +return 1 +`) + + // addStreamEventScript verifies the generation, appends one event, and + // applies stream/recovery retention in one Redis operation. + addStreamEventScript = redis.NewScript(` +if redis.call("HGET", KEYS[1], "state") ~= ARGV[1] +or redis.call("HGET", KEYS[1], "generation") ~= ARGV[2] +or redis.call("HGET", KEYS[1], ARGV[11]) ~= KEYS[2] then + return redis.error_reply("STREAMDESTROYED") +end +if redis.call("HGET", KEYS[1], ARGV[13]) ~= ARGV[14] then + return redis.error_reply("STREAMCONFIGMISMATCH") +end +local deadline = redis.call("HGET", KEYS[1], ARGV[12]) +if deadline then + local now = redis.call("TIME") + local now_ms = tonumber(now[1]) * 1000 + math.floor(tonumber(now[2]) / 1000) + if now_ms >= tonumber(deadline) then + return redis.error_reply("DEADLINEELAPSED") + end + if tonumber(ARGV[9]) > 0 then + return redis.error_reply("STREAMDEADLINECONFLICT") + end +end +if ARGV[6] == "1" and redis.call("EXISTS", KEYS[2]) == 0 then + return {0} +end + +local id +if ARGV[3] == "0" and ARGV[7] == "1" then + if ARGV[6] == "1" then + id = redis.call("XADD", KEYS[2], "NOMKSTREAM", "*", "n", ARGV[4], "p", ARGV[5], "t", ARGV[8]) + else + id = redis.call("XADD", KEYS[2], "*", "n", ARGV[4], "p", ARGV[5], "t", ARGV[8]) + end +elseif ARGV[3] == "0" then + if ARGV[6] == "1" then + id = redis.call("XADD", KEYS[2], "NOMKSTREAM", "*", "n", ARGV[4], "p", ARGV[5]) + else + id = redis.call("XADD", KEYS[2], "*", "n", ARGV[4], "p", ARGV[5]) + end +elseif ARGV[7] == "1" then + if ARGV[6] == "1" then + id = redis.call("XADD", KEYS[2], "NOMKSTREAM", "MAXLEN", "=", ARGV[3], "*", + "n", ARGV[4], "p", ARGV[5], "t", ARGV[8]) + else + id = redis.call("XADD", KEYS[2], "MAXLEN", "=", ARGV[3], "*", + "n", ARGV[4], "p", ARGV[5], "t", ARGV[8]) + end +else + if ARGV[6] == "1" then + id = redis.call("XADD", KEYS[2], "NOMKSTREAM", "MAXLEN", "=", ARGV[3], "*", + "n", ARGV[4], "p", ARGV[5]) + else + id = redis.call("XADD", KEYS[2], "MAXLEN", "=", ARGV[3], "*", + "n", ARGV[4], "p", ARGV[5]) + end +end + +local ttl = tonumber(ARGV[9]) +if deadline then + redis.call("HSET", KEYS[3], "=deadline", deadline) + redis.call("SADD", KEYS[4], KEYS[3]) + redis.call("PEXPIREAT", KEYS[2], deadline) + redis.call("PEXPIREAT", KEYS[3], deadline) + redis.call("PEXPIREAT", KEYS[4], deadline) +elseif ttl > 0 then + if ARGV[10] == "1" then + redis.call("PEXPIRE", KEYS[2], ttl) + elseif redis.call("PTTL", KEYS[2]) == -1 then + redis.call("PEXPIRE", KEYS[2], ttl) + end +end +return {1, id} +`) + + // removeStreamEventsScript prevents stale Stream values from deleting + // events in a newer incarnation. + removeStreamEventsScript = redis.NewScript(` +if redis.call("HGET", KEYS[1], "state") ~= ARGV[1] +or redis.call("HGET", KEYS[1], "generation") ~= ARGV[2] +or redis.call("HGET", KEYS[1], ARGV[3]) ~= KEYS[2] then + return redis.error_reply("STREAMDESTROYED") +end +return redis.call("XDEL", KEYS[2], unpack(ARGV, 4)) +`) +) + +// establishStreamGeneration linearizes the first caller-context operation +// against Destroy and claims or adopts the generation retention contract. +func establishStreamGeneration( + ctx context.Context, + rdb *redis.Client, + name, requestedDeadline, retention string, + maxLen int, + ttlOwned, retentionExplicit bool, +) (string, string, string, string, error) { + result, err := establishStreamScript.Run( + ctx, + rdb, + []string{streamLifecycleKey(name)}, + streamStateActive, + streamPhysicalKey, + streamKey(name), + requestedDeadline, + boolString(ttlOwned), + streamDeadlineKey, + streamTTLOwnedKey, + streamConfigKey, + retention, + boolString(retentionExplicit), + strconv.Itoa(maxLen), + ).Slice() + if err != nil { + return "", "", "", "", streamLifecycleBoundaryError( + fmt.Errorf("pulse stream: establish generation for %q: %w", name, err), + ) + } + generation, physical, deadline, canonicalRetention, err := parseLifecycleIdentity(result) + if err != nil { + return "", "", "", "", fmt.Errorf("pulse stream: establish generation for %q: %w", name, err) + } + return generation, physical, deadline, canonicalRetention, nil +} + +// ensureGeneration binds s to the active Redis generation exactly once. +// Failed boundary I/O leaves the handle unbound so a later caller may retry +// with its own context. +func (s *Stream) ensureGeneration(ctx context.Context) error { + s.generationLock.Lock() + defer s.generationLock.Unlock() + if s.generation != "" { + return nil + } + generation, physical, deadline, retention, err := establishStreamGeneration( + ctx, + s.rdb, + s.Name, + s.requestedDeadline(), + requestedRetention(s), + s.maxLen, + s.ttl > 0, + s.retentionExplicit, + ) + if err != nil { + return err + } + s.generation = generation + s.key = physical + if err := s.applyRetentionConfig(retention); err != nil { + s.generation = "" + s.key = streamKey(s.Name) + return fmt.Errorf("pulse stream: establish generation for %q: %w", s.Name, err) + } + s.deadline, err = parseDeadline(deadline) + if err != nil { + s.generation = "" + s.key = streamKey(s.Name) + return fmt.Errorf("pulse stream: establish generation for %q: %w", s.Name, err) + } + return nil +} + +// loadExistingGeneration binds an unbound non-creating operation to the +// existing lifecycle generation without allocating a destroyed successor. +func (s *Stream) loadExistingGeneration(ctx context.Context) error { + s.generationLock.Lock() + defer s.generationLock.Unlock() + if s.generation != "" { + return nil + } + result, err := loadCurrentGenerationScript.Run( + ctx, + s.rdb, + []string{s.lifecycleKey}, + streamStateActive, + streamPhysicalKey, + streamKey(s.Name), + s.requestedDeadline(), + boolString(s.ttl > 0), + streamDeadlineKey, + streamTTLOwnedKey, + streamConfigKey, + requestedRetention(s), + boolString(s.retentionExplicit), + ).Slice() + if err != nil { + return streamLifecycleBoundaryError( + fmt.Errorf("pulse stream: load current generation for %q: %w", s.Name, err), + ) + } + generation, physical, deadline, retention, err := parseLifecycleIdentity(result) + if err != nil { + return fmt.Errorf("pulse stream: load current generation for %q: %w", s.Name, err) + } + s.generation = generation + s.key = physical + if err := s.applyRetentionConfig(retention); err != nil { + s.generation = "" + s.key = streamKey(s.Name) + return fmt.Errorf("pulse stream: load current generation for %q: %w", s.Name, err) + } + s.deadline, err = parseDeadline(deadline) + if err != nil { + s.generation = "" + s.key = streamKey(s.Name) + return fmt.Errorf("pulse stream: load current generation for %q: %w", s.Name, err) + } + return nil +} + +// verifyGeneration establishes an unbound handle and rejects a stale bound +// capability. +func (s *Stream) verifyGeneration(ctx context.Context) error { + if err := s.ensureGeneration(ctx); err != nil { + return err + } + err := verifyStreamScript.Run( + ctx, + s.rdb, + []string{s.lifecycleKey}, + streamStateActive, + s.generation, + streamPhysicalKey, + s.key, + streamDeadlineKey, + streamConfigKey, + s.retention, + ).Err() + return s.lifecycleError(err) +} + +// verifyExistingGeneration loads without creating, then verifies the exact +// active generation and its adopted retention contract. +func (s *Stream) verifyExistingGeneration(ctx context.Context) error { + if err := s.loadExistingGeneration(ctx); err != nil { + return err + } + return s.verifyGeneration(ctx) +} + +// addEvent atomically verifies this generation and appends one event. +func (s *Stream) addEvent( + ctx context.Context, + name string, + payload []byte, + onlyIfExists bool, + topic string, +) (string, error) { + var err error + if onlyIfExists { + err = s.loadExistingGeneration(ctx) + } else { + err = s.ensureGeneration(ctx) + } + if err != nil { + return "", err + } + topicPresent := topic != "" + result, err := addStreamEventScript.Run( + ctx, + s.rdb, + []string{ + s.lifecycleKey, + s.key, + recoveryCursorKey(s), + streamResourceRegistryKey(s), + }, + streamStateActive, + s.generation, + strconv.Itoa(s.maxLen), + name, + payload, + boolString(onlyIfExists), + boolString(topicPresent), + topic, + strconv.FormatInt(s.ttl.Milliseconds(), 10), + boolString(s.ttlSliding), + streamPhysicalKey, + streamDeadlineKey, + streamConfigKey, + s.retention, + ).Slice() + if err != nil { + return "", s.lifecycleError(err) + } + if len(result) == 0 { + return "", fmt.Errorf("add stream event script returned no status") + } + status, ok := result[0].(int64) + if !ok { + return "", fmt.Errorf("add stream event script returned invalid status %T", result[0]) + } + if status == 0 { + return "", nil + } + if len(result) != 2 { + return "", fmt.Errorf("add stream event script returned %d values for successful add", len(result)) + } + id, ok := result[1].(string) + if !ok { + return "", fmt.Errorf("add stream event script returned invalid event ID %T", result[1]) + } + return id, nil +} + +// removeEvents atomically verifies this generation and deletes event IDs. +func (s *Stream) removeEvents(ctx context.Context, ids []string) error { + if err := s.ensureGeneration(ctx); err != nil { + return err + } + args := make([]any, 0, len(ids)+3) + args = append(args, streamStateActive, s.generation, streamPhysicalKey) + for _, id := range ids { + args = append(args, id) + } + err := removeStreamEventsScript.Run( + ctx, + s.rdb, + []string{s.lifecycleKey, s.key}, + args..., + ).Err() + return s.lifecycleError(err) +} + + +// lifecycleError maps Redis's generation mismatch to the public sentinel. +func (s *Stream) lifecycleError(err error) error { + if err == nil { + return nil + } + if redis.HasErrorPrefix(err, streamDestroyedError) { + return fmt.Errorf( + "%w: stream %q generation %s is no longer active", + ErrStreamDestroyed, + s.Name, + s.generation, + ) + } + if redis.HasErrorPrefix(err, streamDeadlineError) { + return fmt.Errorf("%w: stream %q", ErrDeadlineElapsed, s.Name) + } + if redis.HasErrorPrefix(err, streamConflictError) { + return fmt.Errorf( + "%w: stream %q deadline conflicts with active generation", + ErrStreamConfigMismatch, + s.Name, + ) + } + if redis.HasErrorPrefix(err, streamConfigError) { + return fmt.Errorf("%w: stream %q", ErrStreamConfigMismatch, s.Name) + } + if redis.HasErrorPrefix(err, streamDeadlineRequiredError) { + return fmt.Errorf("%w: stream %q requires deadline retention", ErrStreamConfigMismatch, s.Name) + } + if redis.HasErrorPrefix(err, streamConfigMissingError) { + return fmt.Errorf("%w: stream %q has no adopted retention configuration", ErrStreamConfigMismatch, s.Name) + } + if redis.HasErrorPrefix(err, streamSnapshotUnboundedError) { + return fmt.Errorf("%w: stream %q", ErrSnapshotUnbounded, s.Name) + } + if redis.HasErrorPrefix(err, streamSnapshotBoundError) { + return fmt.Errorf("%w: stream %q", ErrSnapshotBoundExceeded, s.Name) + } + if redis.HasErrorPrefix(err, streamNotFoundError) { + return fmt.Errorf("%w: stream %q", ErrStreamNotFound, s.Name) + } + return err +} + +// streamLifecycleKey identifies the canonical generation record for a logical +// stream name. +func streamLifecycleKey(name string) string { + return fmt.Sprintf("%s%s:lifecycle", streamKeyPrefix, name) +} + +// streamKey identifies the flat physical key adopted by generation one. +func streamKey(name string) string { + return streamKeyPrefix + name +} + +// parseStreamIdentity decodes the generation and physical key selected by the +// lifecycle scripts. +func parseStreamIdentity(result []any) (string, string, string, error) { + if len(result) != 3 { + return "", "", "", fmt.Errorf("stream lifecycle returned %d values", len(result)) + } + generation, ok := result[0].(string) + if !ok { + return "", "", "", fmt.Errorf("stream lifecycle returned generation %T", result[0]) + } + physical, ok := result[1].(string) + if !ok { + return "", "", "", fmt.Errorf("stream lifecycle returned physical key %T", result[1]) + } + deadline, ok := result[2].(string) + if !ok { + return "", "", "", fmt.Errorf("stream lifecycle returned deadline %T", result[2]) + } + return generation, physical, deadline, nil +} + +// parseLifecycleIdentity also returns the canonical retention configuration +// selected by the lifecycle owner. +func parseLifecycleIdentity(result []any) (string, string, string, string, error) { + if len(result) != 4 { + return "", "", "", "", fmt.Errorf("stream lifecycle returned %d values", len(result)) + } + generation, physical, deadline, err := parseStreamIdentity(result[:3]) + if err != nil { + return "", "", "", "", err + } + retention, ok := result[3].(string) + if !ok || retention == "" { + return "", "", "", "", fmt.Errorf("stream lifecycle returned retention %T", result[3]) + } + return generation, physical, deadline, retention, nil +} + +// requestedDeadline returns the construction-time absolute deadline in Redis +// millisecond form, or empty when this handle adopts the generation contract. +func (s *Stream) requestedDeadline() string { + if !s.deadlineConfigured { + return "" + } + return strconv.FormatInt(s.deadline.UnixMilli(), 10) +} + +// parseDeadline decodes the canonical lifecycle deadline already validated by +// the owning Redis script. +func parseDeadline(value string) (time.Time, error) { + if value == "" { + return time.Time{}, nil + } + milliseconds, err := strconv.ParseInt(value, 10, 64) + if err != nil { + return time.Time{}, fmt.Errorf("invalid Redis-owned stream deadline %q: %w", value, err) + } + return time.UnixMilli(milliseconds), nil +} + +// streamLifecycleBoundaryError maps lifecycle configuration errors returned +// before a Stream handle has bound its generation. +func streamLifecycleBoundaryError(err error) error { + if redis.HasErrorPrefix(err, streamDestroyedError) { + return ErrStreamDestroyed + } + if redis.HasErrorPrefix(err, streamDeadlineError) { + return ErrDeadlineElapsed + } + if redis.HasErrorPrefix(err, streamConflictError) { + return fmt.Errorf( + "%w: stream deadline conflicts with active generation", + ErrStreamConfigMismatch, + ) + } + if redis.HasErrorPrefix(err, streamConfigError) { + return ErrStreamConfigMismatch + } + if redis.HasErrorPrefix(err, streamNotFoundError) { + return ErrStreamNotFound + } + return err +} + +// retentionConfig serializes the typed immutable retention contract compared +// by every generation-opening Redis operation. +func (s *Stream) retentionConfig() string { + mode := "none" + value := int64(0) + sliding := false + switch { + case s.deadlineConfigured: + mode = "deadline" + value = s.deadline.UnixMilli() + case s.ttl > 0: + mode = "ttl" + value = s.ttl.Milliseconds() + sliding = s.ttlSliding + } + return fmt.Sprintf( + "v=%s|max=%d|mode=%s|value=%d|sliding=%t", + streamFormatVersion, + s.maxLen, + mode, + value, + sliding, + ) +} + +// requestedRetention returns the explicit configuration or the documented +// default used only when a writer creates the first generation. +func requestedRetention(s *Stream) string { + return s.retention +} + +// applyRetentionConfig adopts the Redis-owned immutable configuration into an +// unconfigured local handle. +func (s *Stream) applyRetentionConfig(config string) error { + parts := strings.Split(config, "|") + if len(parts) != 5 || parts[0] != "v="+streamFormatVersion { + return fmt.Errorf("invalid stream retention configuration %q", config) + } + maxLen, err := strconv.Atoi(strings.TrimPrefix(parts[1], "max=")) + if err != nil || maxLen < 0 { + return fmt.Errorf("invalid stream retention maximum in %q", config) + } + mode := strings.TrimPrefix(parts[2], "mode=") + value, err := strconv.ParseInt(strings.TrimPrefix(parts[3], "value="), 10, 64) + if err != nil { + return fmt.Errorf("invalid stream retention value in %q", config) + } + sliding, err := strconv.ParseBool(strings.TrimPrefix(parts[4], "sliding=")) + if err != nil { + return fmt.Errorf("invalid stream retention sliding flag in %q", config) + } + switch mode { + case "none": + if value != 0 || sliding { + return fmt.Errorf("invalid none retention configuration %q", config) + } + s.ttl = 0 + s.ttlSliding = false + case "ttl": + if value <= 0 { + return fmt.Errorf("invalid ttl retention configuration %q", config) + } + s.ttl = time.Duration(value) * time.Millisecond + s.ttlSliding = sliding + case "deadline": + if value <= 0 || sliding { + return fmt.Errorf("invalid deadline retention configuration %q", config) + } + s.ttl = 0 + default: + return fmt.Errorf("invalid stream retention mode %q", mode) + } + s.maxLen = maxLen + s.MaxLen = maxLen + s.retention = config + return nil +} + +// boolString encodes booleans for Lua arguments. +func boolString(value bool) string { + if value { + return "1" + } + return "0" +} diff --git a/streaming/streams.go b/streaming/streams.go index 87ff236..768fc3f 100644 --- a/streaming/streams.go +++ b/streaming/streams.go @@ -2,8 +2,10 @@ package streaming import ( "context" + "errors" "fmt" "regexp" + "sync" "time" redis "github.com/redis/go-redis/v9" @@ -20,23 +22,70 @@ type ( Stream struct { // Name of the stream. Name string - // MaxLen is the maximum number of events in the stream. + // MaxLen is the maximum retained event count requested at construction + // and, after Open, the active generation's canonical bound. It remains + // exported for v1 source compatibility. Mutating it after construction + // is unsupported; operations use the immutable private snapshot. MaxLen int + // maxLen is the immutable maximum retained event count. Zero means the + // stream is explicitly unbounded. + maxLen int // ttl configures an expiry for the Redis key backing the stream. ttl time.Duration // ttlSliding controls whether ttl is refreshed on every Add call. ttlSliding bool + // deadline is the Redis-canonical absolute expiry for this generation. + deadline time.Time + // deadlineConfigured records whether construction explicitly requested + // deadline, so lifecycle binding can reject a conflicting value. + deadlineConfigured bool + // retention is the canonical immutable generation configuration. + retention string + // retentionExplicit distinguishes writer configuration from an + // unconfigured handle that adopts the active generation. + retentionExplicit bool // logger is the logger used by the stream. logger pulse.Logger // rootLogger is the prefix-free logger used to create sink loggers. rootLogger pulse.Logger - // key is the redis key used for the stream. + // key is the immutable lifecycle-selected Redis event-stream key. key string + // lifecycleKey is the Redis-owned logical stream identity. + lifecycleKey string + // generation is the immutable incarnation established by the first + // caller-context operation. + generation string + // generationLock serializes lazy generation establishment. + generationLock sync.Mutex // rdb is the redis connection. rdb *redis.Client } ) +var ( + // ErrStreamDestroyed is returned when an operation uses a Stream whose + // immutable generation is no longer active. + ErrStreamDestroyed = errors.New("pulse streaming: stream generation is destroyed") + // ErrIdempotencyConflict is returned when an AddOnce key already identifies + // different event content in the same stream generation. + ErrIdempotencyConflict = errors.New("pulse streaming: idempotency key conflicts with existing event") + // ErrDeadlineElapsed is returned when a deadline-owned stream operation is + // attempted at or after its Redis-authoritative absolute deadline. + ErrDeadlineElapsed = errors.New("pulse streaming: stream deadline elapsed") + // ErrStreamNotFound is returned when Snapshot observes no initialized + // lifecycle. Snapshot never creates one. + ErrStreamNotFound = errors.New("pulse streaming: stream is not initialized") + // ErrStreamConfigMismatch is returned when an explicitly configured handle + // differs from the active generation's immutable retention contract. + ErrStreamConfigMismatch = errors.New("pulse streaming: stream retention configuration mismatch") + // ErrSnapshotUnbounded is returned before XRANGE when Snapshot is called on + // a generation whose immutable retention contract has no MaxLen bound. + ErrSnapshotUnbounded = errors.New("pulse streaming: snapshot requires bounded stream retention") + // ErrSnapshotBoundExceeded reports physical data that exceeds the active + // generation's immutable MaxLen contract. + ErrSnapshotBoundExceeded = errors.New("pulse streaming: snapshot retention bound violated") +) + const ( // streamKeyPrefix is the prefix used for stream keys. streamKeyPrefix = "pulse:stream:" @@ -48,35 +97,78 @@ const ( topicKey = "t" ) -// NewStream returns the stream with the given name. All stream instances -// with the same name share the same events. +// NewStream validates options and returns a local stream handle without +// performing Redis I/O. The first caller-context operation establishes or +// loads the active generation. WithStreamDeadline claims an immutable absolute +// generation deadline and cannot be combined with TTL options. func NewStream(name string, rdb *redis.Client, opts ...options.Stream) (*Stream, error) { if !isValidRedisKeyName(name) { return nil, fmt.Errorf("pulse stream: not a valid name %q", name) } o := options.ParseStreamOptions(opts...) - if o.TTL < 0 { - return nil, fmt.Errorf("pulse stream: ttl must be >= 0") + if o.Unbounded && o.MaxLenSet { + return nil, fmt.Errorf("pulse stream: maximum length and unbounded options are mutually exclusive") + } + if o.Unbounded { + o.MaxLen = 0 + } else if o.MaxLen <= 0 { + return nil, fmt.Errorf("pulse stream: maximum length must be greater than zero") + } + if o.TTLSet && o.TTL < time.Millisecond { + return nil, fmt.Errorf("pulse stream: ttl must be at least 1ms") + } + if o.DeadlineSet && o.Deadline.IsZero() { + return nil, fmt.Errorf("pulse stream: deadline must not be zero") + } + if o.DeadlineSet && o.TTL != 0 { + return nil, fmt.Errorf("pulse stream: deadline and ttl options are mutually exclusive") } - var logger pulse.Logger + var ( + logger pulse.Logger + rootLogger pulse.Logger + ) if o.Logger != nil { + rootLogger = o.Logger logger = o.Logger.WithPrefix("stream", name) } else { - logger = pulse.NoopLogger() + rootLogger = pulse.NoopLogger() + logger = rootLogger } s := &Stream{ - Name: name, - MaxLen: o.MaxLen, - ttl: o.TTL, - ttlSliding: o.TTLSliding, - logger: logger, - rootLogger: o.Logger, - key: streamKeyPrefix + name, - rdb: rdb, + Name: name, + MaxLen: o.MaxLen, + maxLen: o.MaxLen, + ttl: o.TTL, + ttlSliding: o.TTLSliding, + deadline: o.Deadline, + deadlineConfigured: o.DeadlineSet, + retentionExplicit: o.MaxLenSet || o.Unbounded || o.TTLSet || o.DeadlineSet, + logger: logger, + rootLogger: rootLogger, + key: streamKey(name), + lifecycleKey: streamLifecycleKey(name), + rdb: rdb, } + s.retention = s.retentionConfig() return s, nil } +// Generation returns the immutable Redis-owned incarnation established for s, +// or the empty string before the first caller-context operation. +func (s *Stream) Generation() string { + s.generationLock.Lock() + defer s.generationLock.Unlock() + return s.generation +} + +// Open establishes or adopts the active Redis incarnation and verifies that a +// previously bound handle still names the active generation. Higher-level +// runtimes use Open as their lifecycle fence before mutating generation-owned +// resources outside the event stream. +func (s *Stream) Open(ctx context.Context) error { + return s.verifyGeneration(ctx) +} + // NewReader creates a new stream reader. All reader instances get all the // events in the stream. Events are read starting: // - from the last event by default @@ -86,20 +178,28 @@ func NewStream(name string, rdb *redis.Client, opts ...options.Stream) (*Stream, // event is still in the stream, oldest event otherwise // - from the event added on or after the timestamp provided via // WithReaderStartAt if still in the stream, oldest event otherwise +// +// NewReader does not initialize an absent lifecycle; it returns +// ErrStreamNotFound until a writer or Open establishes the stream. func (s *Stream) NewReader(ctx context.Context, opts ...options.Reader) (*Reader, error) { - reader, err := newReader(s, opts...) + reader, err := newReader(ctx, s, opts...) if err != nil { s.logger.Error(fmt.Errorf("failed to create reader: %w", err)) return nil, err } + if err := s.verifyExistingGeneration(ctx); err != nil { + reader.cancel() + return nil, err + } + reader.streamKeys[0] = s.key s.logger.Info("create reader", "start", reader.startID) return reader, nil } // NewSink creates a new stream sink with the given name. All sink instances // with the same name share the same stream cursor. Events read through a sink -// are not removed from the stream until they are acked by the client unless the -// WithNoAck option is used. Events are read starting: +// are not removed from the stream until they are acked by the client unless +// WithSinkNoAck is used. Events are read starting: // - from the last event by default // - from the oldest event stored in the stream if the WithSinkStartAtOldest // option is used @@ -119,58 +219,26 @@ func (s *Stream) NewSink(ctx context.Context, name string, opts ...options.Sink) // Add appends an event to the stream and returns its ID. If the option // WithOnlyIfStreamExists is used and the stream does not exist then no event is // added and the empty string is returned. The stream is created if the option -// is omitted or when NewSink is called. +// is omitted or when NewSink is called. On a deadline-owned generation, Add +// rejects elapsed deadlines and reapplies the same absolute expiry without +// extending it. func (s *Stream) Add(ctx context.Context, name string, payload []byte, opts ...options.AddEvent) (string, error) { o := options.ParseAddEventOptions(opts...) - for _, option := range opts { - option(&o) - } - values := []any{nameKey, name, payloadKey, payload} - if o.Topic != "" { - values = append(values, topicKey, o.Topic) - } - res, err := s.rdb.XAdd(ctx, &redis.XAddArgs{ - Stream: s.key, - Values: values, - MaxLen: int64(s.MaxLen), - Approx: true, - NoMkStream: o.OnlyIfStreamExists, - }).Result() + res, err := s.addEvent(ctx, name, payload, o.OnlyIfStreamExists, o.Topic) if err != nil { - if err == redis.Nil { - // Stream does not exist and OnlyIfStreamExists option was used. - return "", nil - } err = fmt.Errorf("failed to add event: %w", err) s.logger.Error(err, "event", name) return "", err } - if err := s.applyTTL(ctx); err != nil { - err = fmt.Errorf("failed to apply stream TTL: %w", err) - s.logger.Error(err, "event", name) - return "", err - } s.logger.Info("add", "event", name, "id", res) return res, nil } -func (s *Stream) applyTTL(ctx context.Context) error { - if s.ttl <= 0 { - return nil - } - if s.ttlSliding { - return s.rdb.Expire(ctx, s.key, s.ttl).Err() - } - _, err := s.rdb.ExpireNX(ctx, s.key, s.ttl).Result() - return err -} - // Remove removes the events with the given IDs from the stream. // Note: clients should not need to call this method in normal operation, // instead they should use the Ack method to acknowledge events. func (s *Stream) Remove(ctx context.Context, ids ...string) error { - err := s.rdb.XDel(ctx, s.key, ids...).Err() - if err != nil { + if err := s.removeEvents(ctx, ids); err != nil { err = fmt.Errorf("failed to remove events: %w", err) s.logger.Error(err, "events", ids) return err @@ -179,15 +247,14 @@ func (s *Stream) Remove(ctx context.Context, ids ...string) error { return nil } -// Destroy deletes the entire stream: its events and every piece of sink -// metadata (consumer groups, recovery cursors, leases, and the sink -// membership map) in one atomic operation. The stream lifecycle is marked -// destroyed so concurrent sinks cannot resurrect the metadata; only a -// subsequent NewSink or Sink.AddStream call deliberately recreates the -// stream. Destroy is idempotent. +// Destroy invalidates this exact generation and atomically deletes its event +// data, consumer groups, recovery cursors, exact-publication records, and +// membership state. It is idempotent after this handle has bound a generation; +// an unbound absent name returns ErrStreamNotFound without creating lifecycle +// state. Existing Stream, Reader, Sink, Event, and Acker values cannot affect a +// later generation. func (s *Stream) Destroy(ctx context.Context) error { if err := destroyStream(ctx, s); err != nil { - err := fmt.Errorf("failed to destroy stream: %w", err) s.logger.Error(err) return err } diff --git a/streaming/streams_test.go b/streaming/streams_test.go index 46d442b..a47771c 100644 --- a/streaming/streams_test.go +++ b/streaming/streams_test.go @@ -1,11 +1,12 @@ package streaming import ( - "strconv" + "errors" "testing" "time" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "goa.design/pulse/pulse" "goa.design/pulse/streaming/options" @@ -19,10 +20,15 @@ func TestDestroy(t *testing.T) { s, err := NewStream("testDestroy", rdb) assert.NoError(t, err) + require.ErrorIs(t, s.Destroy(ctx), ErrStreamNotFound) + require.NoError(t, s.Open(ctx)) assert.NoError(t, s.Destroy(ctx)) exists, err := rdb.Exists(ctx, s.key).Result() assert.NoError(t, err) assert.Equal(t, int64(0), exists) + generation := s.Generation() + require.NoError(t, s.Destroy(ctx)) + require.Equal(t, generation, s.Generation()) s2, err := NewStream("testDestroy2", rdb) assert.NoError(t, err) @@ -34,11 +40,42 @@ func TestDestroy(t *testing.T) { assert.NoError(t, s2.Destroy(ctx)) } +func TestNewStreamLazilyLoadsActiveRedisGeneration(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + + first, err := NewStream(t.Name(), rdb) + require.NoError(t, err) + second, err := NewStream(t.Name(), rdb) + require.NoError(t, err) + require.Empty(t, first.generation) + require.Empty(t, second.generation) + require.Zero(t, rdb.DBSize(ctx).Val()) + require.Equal(t, first.generation, second.generation) + require.Equal(t, first.key, second.key) + _, err = first.Add(ctx, "first", []byte("payload")) + require.NoError(t, err) + _, err = second.Add(ctx, "second", []byte("payload")) + require.NoError(t, err) + require.NotEmpty(t, first.generation) + require.Equal(t, first.generation, second.generation) + require.NoError(t, first.Destroy(ctx)) + require.NoError(t, second.Destroy(ctx)) +} + func TestOptions(t *testing.T) { - s, err := NewStream("testOptions", nil, options.WithStreamMaxLen(10), options.WithStreamLogger(nil)) + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + s, err := NewStream("testOptions", rdb, options.WithStreamMaxLen(10), options.WithStreamLogger(nil)) assert.NoError(t, err) assert.Equal(t, 10, s.MaxLen) assert.Equal(t, pulse.NoopLogger(), s.logger) + require.NoError(t, s.Open(ctx)) + assert.NoError(t, s.Destroy(ctx)) + _, err = NewStream("invalidMaxLen", rdb, options.WithStreamMaxLen(0)) + assert.EqualError(t, err, "pulse stream: maximum length must be greater than zero") } func TestAdd(t *testing.T) { @@ -122,7 +159,7 @@ func TestStreamTTLSliding(t *testing.T) { assert.NoError(t, s.Destroy(ctx)) } -func TestStreamDestroyUsesRMapDestroyProtocol(t *testing.T) { +func TestStreamDestroyDeletesGenerationMembership(t *testing.T) { rdb := ptesting.NewRedisClient(t) defer ptesting.CleanupRedis(t, rdb, false, "") ctx := ptesting.NewTestContext(t) @@ -132,20 +169,198 @@ func TestStreamDestroyUsesRMapDestroyProtocol(t *testing.T) { sink, err := s.NewSink(ctx, "gateway") assert.NoError(t, err) - sink.Close(ctx) + assert.True(t, rdb.HExists(ctx, recoveryCursorKey(s), sink.Name).Val()) + require.NoError(t, sink.Close(ctx)) - mapKey := "map:stream:testStreamDestroyDeletesSinkMap:sinks:content" + mapKey := consumersMapContentKey(s) exists, err := rdb.Exists(ctx, mapKey).Result() assert.NoError(t, err) assert.EqualValues(t, 1, exists) assert.NoError(t, s.Destroy(ctx)) - content, err := rdb.HGetAll(ctx, mapKey).Result() - assert.NoError(t, err) - assert.Len(t, content, 2) - assert.Equal(t, "destroy", content["=kind"]) - _, err = strconv.ParseUint(content["=rev"], 10, 64) - assert.NoError(t, err) + assert.EqualValues(t, 0, rdb.Exists(ctx, mapKey).Val()) + assert.EqualValues(t, 0, rdb.Exists(ctx, recoveryCursorKey(s)).Val()) +} + +func TestStreamDestroyInvalidatesExactGeneration(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + s, err := NewStream(t.Name(), rdb) + require.NoError(t, err) + sink, err := s.NewSink(ctx, "gateway", options.WithSinkBlockDuration(testBlockDuration)) + require.NoError(t, err) + events := sink.Subscribe() + eventID, err := s.Add(ctx, "before-destroy", []byte("payload")) + require.NoError(t, err) + event := receiveSinkEvent(t, events) + require.Equal(t, eventID, event.ID) + + require.NoError(t, s.Destroy(ctx)) + require.EqualValues(t, 0, rdb.Exists(ctx, s.key).Val()) + require.EqualValues(t, 0, rdb.Exists(ctx, recoveryCursorKey(s)).Val()) + require.EqualValues(t, 0, rdb.Exists(ctx, consumersMapContentKey(s)).Val()) + require.Eventually(t, sink.IsClosed, max, delay) + _, err = s.Add(ctx, "stale", []byte("payload")) + require.ErrorIs(t, err, ErrStreamDestroyed) + require.ErrorIs(t, s.Remove(ctx, eventID), ErrStreamDestroyed) + require.ErrorIs(t, sink.AddStream(ctx, s), ErrSinkClosed) + require.ErrorIs(t, sink.RemoveStream(ctx, s), ErrSinkClosed) + require.ErrorIs(t, sink.Ack(ctx, event), ErrStreamDestroyed) + _, err = s.NewReader(ctx) + require.ErrorIs(t, err, ErrStreamDestroyed) + _, err = s.NewSink(ctx, "new-sink") + require.ErrorIs(t, err, ErrStreamDestroyed) + require.EqualValues(t, 0, rdb.Exists(ctx, consumersMapContentKey(s)).Val()) + + late, err := NewStream(t.Name(), rdb) + require.NoError(t, err) + _, err = late.Add(ctx, "late", []byte("payload"), options.WithOnlyIfStreamExists()) + require.ErrorIs(t, err, ErrStreamDestroyed) + require.Equal(t, s.generation, rdb.HGet(ctx, s.lifecycleKey, "generation").Val()) + require.Equal(t, streamStateDestroyed, rdb.HGet(ctx, s.lifecycleKey, "state").Val()) + + next, err := NewStream(t.Name(), rdb) + require.NoError(t, err) + require.Empty(t, next.generation) + require.Equal(t, s.key, next.key) + _, err = next.Add(ctx, "fresh", []byte("payload")) + require.NoError(t, err) + require.NotEqual(t, s.generation, next.generation) + require.NotEqual(t, s.key, next.key) + require.EqualValues(t, 0, rdb.Exists(ctx, s.key).Val()) + require.EqualValues(t, 1, rdb.Exists(ctx, next.key).Val()) + require.EqualValues(t, 0, rdb.Exists(ctx, recoveryCursorKey(s)).Val()) + require.NoError(t, next.Destroy(ctx)) +} + +func TestReaderStopsWhenGenerationIsDestroyed(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + stream, err := NewStream(t.Name(), rdb) + require.NoError(t, err) + require.NoError(t, stream.Open(ctx)) + reader, err := stream.NewReader(ctx, options.WithReaderBlockDuration(testBlockDuration)) + require.NoError(t, err) + reader.Subscribe() + + require.NoError(t, stream.Destroy(ctx)) + require.Eventually(t, reader.IsClosed, max, delay) +} + +func TestRecreatedStreamIsIsolatedFromStaleReadersAndSinks(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + stream, err := NewStream(t.Name(), rdb) + require.NoError(t, err) + require.NoError(t, stream.Open(ctx)) + reader, err := stream.NewReader(ctx, options.WithReaderStartAtOldest(), options.WithReaderBlockDuration(testBlockDuration)) + require.NoError(t, err) + readerEvents := reader.Subscribe() + sink, err := stream.NewSink( + ctx, + "sink", + options.WithSinkStartAtOldest(), + options.WithSinkBlockDuration(testBlockDuration), + ) + require.NoError(t, err) + sinkEvents := sink.Subscribe() + + require.NoError(t, stream.Destroy(ctx)) + require.Eventually(t, reader.IsClosed, max, delay) + require.Eventually(t, sink.IsClosed, max, delay) + + recreated, err := NewStream(t.Name(), rdb) + require.NoError(t, err) + freshID, err := recreated.Add(ctx, "fresh", []byte("payload")) + require.NoError(t, err) + require.NotEqual(t, stream.key, recreated.key) + select { + case event := <-readerEvents: + require.Nil(t, event) + default: + } + select { + case event := <-sinkEvents: + require.Nil(t, event) + default: + } + + freshReader, err := recreated.NewReader( + ctx, + options.WithReaderStartAtOldest(), + options.WithReaderBlockDuration(testBlockDuration), + ) + require.NoError(t, err) + event := receiveSinkEvent(t, freshReader.Subscribe()) + require.Equal(t, freshID, event.ID) + freshReader.Close() + require.NoError(t, recreated.Destroy(ctx)) +} + +func TestRepeatedStreamGenerationsLeaveBoundedKeys(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + for range 10 { + stream, err := NewStream(t.Name(), rdb) + require.NoError(t, err) + sink, err := stream.NewSink(ctx, "sink", options.WithSinkBlockDuration(testBlockDuration)) + require.NoError(t, err) + require.NoError(t, sink.Close(ctx)) + require.NoError(t, stream.Destroy(ctx)) + } + + generationKeys, err := rdb.Keys(ctx, streamKeyPrefix+t.Name()+":generation:*").Result() + require.NoError(t, err) + require.Empty(t, generationKeys) + streamKeys, err := rdb.Keys(ctx, streamKeyPrefix+t.Name()+"*").Result() + require.NoError(t, err) + require.Equal(t, []string{streamLifecycleKey(t.Name())}, streamKeys) +} + +func TestConcurrentRemoveAndDestroyNeverRecreatesGeneration(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + mainStream, err := NewStream(t.Name()+"-main", rdb) + require.NoError(t, err) + addedStream, err := NewStream(t.Name()+"-added", rdb) + require.NoError(t, err) + sink, err := mainStream.NewSink(ctx, "gateway", options.WithSinkBlockDuration(testBlockDuration)) + require.NoError(t, err) + require.NoError(t, sink.AddStream(ctx, addedStream)) + + start := make(chan struct{}) + removeResult := make(chan error, 1) + destroyResult := make(chan error, 1) + go func() { + <-start + removeResult <- sink.RemoveStream(ctx, addedStream) + }() + go func() { + <-start + destroyResult <- addedStream.Destroy(ctx) + }() + close(start) + + removeErr := <-removeResult + if removeErr != nil { + require.True( + t, + errors.Is(removeErr, ErrStreamDestroyed) || errors.Is(removeErr, ErrSinkClosed), + "unexpected remove result: %v", + removeErr, + ) + } + destroyErr := <-destroyResult + require.NoError(t, destroyErr) + time.Sleep(4 * testBlockDuration) + require.EqualValues(t, 0, rdb.Exists(ctx, addedStream.key, recoveryCursorKey(addedStream)).Val()) + require.NoError(t, sink.Close(ctx)) + require.NoError(t, mainStream.Destroy(ctx)) } func TestRemove(t *testing.T) { diff --git a/streaming/testing.go b/streaming/testing.go index 8bda437..1040277 100644 --- a/streaming/testing.go +++ b/streaming/testing.go @@ -11,12 +11,12 @@ import ( ) var ( - max = time.Second + max = 5 * time.Second delay = 10 * time.Millisecond ) // readOneEvent reads one event from the channel and acks it or fails the test -// if it takes more than a second. +// if it takes more than the shared Redis test bound. func readOneEvent(t *testing.T, ctx context.Context, c <-chan *Event, sink *Sink) *Event { t.Helper() var read *Event @@ -24,7 +24,7 @@ func readOneEvent(t *testing.T, ctx context.Context, c <-chan *Event, sink *Sink w.Add(1) go func() { defer w.Done() - tck := time.NewTicker(time.Second) + tck := time.NewTicker(max) select { case read = <-c: assert.NoError(t, sink.Ack(ctx, read)) @@ -38,7 +38,7 @@ func readOneEvent(t *testing.T, ctx context.Context, c <-chan *Event, sink *Sink } // readOneReaderEvent reads one event from the channel or fails the test if it -// takes more than a second. +// takes more than the shared Redis test bound. func readOneReaderEvent(t *testing.T, c <-chan *Event) *Event { t.Helper() var read *Event @@ -46,7 +46,7 @@ func readOneReaderEvent(t *testing.T, c <-chan *Event) *Event { w.Add(1) go func() { defer w.Done() - tck := time.NewTicker(time.Second) + tck := time.NewTicker(max) select { case read = <-c: return @@ -60,22 +60,25 @@ func readOneReaderEvent(t *testing.T, c <-chan *Event) *Event { return read } -// cleanupSink closes the sink and asserts that it is closed within a second. +// cleanupSink closes the sink and asserts that it is closed within the shared +// Redis test bound. func cleanupSink(t *testing.T, ctx context.Context, s *Stream, sink *Sink) { t.Helper() if sink != nil { - sink.Close(ctx) + require.NoError(t, sink.Close(ctx)) assert.Eventually(t, func() bool { return sink.IsClosed() }, max, delay) } if s != nil { - assert.NoError(t, s.Destroy(ctx)) + require.NoError(t, s.Destroy(ctx)) } } -// cleanupReader closes the reader and asserts that it is closed within a second. -func cleanupReader(t *testing.T, ctx context.Context, s *Stream, reader *Reader) { +// cleanupReader closes the reader and asserts that it is closed within the +// shared Redis test bound. +// Stream destruction remains explicit in each test so duplicate cleanup cannot +// mask ErrStreamDestroyed. +func cleanupReader(t *testing.T, reader *Reader) { t.Helper() reader.Close() assert.Eventually(t, func() bool { return reader.IsClosed() }, max, delay) - assert.NoError(t, s.Destroy(ctx)) } diff --git a/testing/redis.go b/testing/redis.go index 940f4ad..8eef9b4 100644 --- a/testing/redis.go +++ b/testing/redis.go @@ -5,10 +5,13 @@ import ( "fmt" "os" "regexp" + "strconv" "strings" + "sync" "testing" "time" + "github.com/oklog/ulid/v2" redis "github.com/redis/go-redis/v9" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -21,6 +24,43 @@ var ( redisAddr = "localhost:6379" // streamRegexp is a regular expression that matches valid stream keys streamRegexp = regexp.MustCompile(`^pulse:stream:[^:]+:node:.*`) + // renewDatabaseLeaseScript extends only the lease still owned by this test. + renewDatabaseLeaseScript = redis.NewScript(` +if redis.call("GET", KEYS[1]) ~= ARGV[1] then + return 0 +end +redis.call("PEXPIRE", KEYS[1], ARGV[2]) +return 1 +`) + // releaseDatabaseLeaseScript releases only the lease still owned by this + // test process. + releaseDatabaseLeaseScript = redis.NewScript(` +if redis.call("GET", KEYS[1]) ~= ARGV[1] then + return 0 +end +return redis.call("DEL", KEYS[1]) +`) +) + +const ( + databaseLeaseTTL = 2 * time.Minute + databaseLeaseHeartbeat = 30 * time.Second + databaseLeaseWait = 30 * time.Second +) + +type ( + // databaseLease owns one non-coordination Redis database for one test, + // including cross-process heartbeat and release. + databaseLease struct { + coordinator *redis.Client + key string + token string + stop chan struct{} + done chan struct{} + lock sync.Mutex + heartbeatErr error + t *testing.T + } ) func init() { @@ -32,10 +72,18 @@ func init() { } } +// NewRedisClient returns a client backed by a cross-process leased Redis +// database. Package names are irrelevant, and FlushDB is safe because the +// caller holds the database's exclusive lease until test cleanup completes. func NewRedisClient(t *testing.T) *redis.Client { t.Helper() - rdb := redis.NewClient(&redis.Options{Addr: redisAddr, Password: redisPwd}) + lease, db := acquireDatabaseLease(t) + t.Cleanup(func() { + lease.release(t) + }) + rdb := redis.NewClient(&redis.Options{Addr: redisAddr, Password: redisPwd, DB: db}) require.NoError(t, rdb.Ping(context.Background()).Err()) + require.NoError(t, rdb.FlushDB(context.Background()).Err()) return rdb } @@ -47,53 +95,56 @@ func CleanupRedis(t *testing.T, rdb *redis.Client, checkClean bool, testName str t.Helper() ctx := context.Background() if checkClean { - var ( - filtered []string - keysErr error - ) - assert.Eventually(t, func() bool { - var keys []string - keys, keysErr = rdb.Keys(ctx, "*").Result() - if keysErr != nil { - filtered = []string{fmt.Sprintf("keys error: %v", keysErr)} - return false - } - filtered = filtered[:0] - for _, k := range keys { - if strings.HasSuffix(k, ":sinks:content") { - // Sinks content is cleaned up asynchronously, so ignore it - continue - } - if isDestroyTombstone(ctx, rdb, k) { - // Destroy tombstones are intentional rmap protocol state used so - // reconnecting replicas can order the next generation correctly. - continue - } - if isDestroyedStreamLifecycle(ctx, rdb, k) { - // Destroyed stream lifecycles are the intentional fence that - // prevents concurrent sinks from resurrecting stream metadata. - continue - } - if streamRegexp.MatchString(k) { - // Node streams are cleaned up asynchronously, so ignore them - continue - } - if strings.Contains(k, testName) { - filtered = append(filtered, k) - } - } - return len(filtered) == 0 - }, 5*time.Second, time.Millisecond*10, "found keys: %v", filtered) - require.NoError(t, keysErr) + clean := assert.Eventually(t, func() bool { + filtered, err := remainingTestKeys(ctx, rdb, testName) + return err == nil && len(filtered) == 0 + }, 5*time.Second, time.Millisecond*10) + if !clean { + filtered, err := remainingTestKeys(ctx, rdb, testName) + require.NoError(t, err) + t.Errorf("found keys: %v", filtered) + } } assert.NoError(t, rdb.FlushDB(ctx).Err()) + assert.NoError(t, rdb.Close()) +} + +// remainingTestKeys returns keys owned by the named test that are not durable +// protocol state or resources with documented asynchronous cleanup. +func remainingTestKeys(ctx context.Context, rdb *redis.Client, testName string) ([]string, error) { + keys, err := rdb.Keys(ctx, "*").Result() + if err != nil { + return nil, err + } + var filtered []string + for _, key := range keys { + if strings.HasPrefix(key, "pulse:pool:") && strings.HasSuffix(key, ":cleanup-generations") { + continue + } + if strings.HasPrefix(key, "pulse:pool:") && strings.HasSuffix(key, ":resources") { + continue + } + if isDestroyedStreamLifecycle(ctx, rdb, key) { + continue + } + if strings.HasSuffix(key, ":sinks:content") { + continue + } + if isDestroyTombstone(ctx, rdb, key) { + continue + } + if streamRegexp.MatchString(key) { + continue + } + if strings.Contains(key, testName) { + filtered = append(filtered, key) + } + } + return filtered, nil } -// isDestroyedStreamLifecycle reports whether key is the lifecycle fence of a -// destroyed stream. Destroyed lifecycles intentionally outlive Stream.Destroy -// so concurrent sinks cannot recreate the stream metadata. func isDestroyedStreamLifecycle(ctx context.Context, rdb *redis.Client, key string) bool { - if !strings.HasPrefix(key, "pulse:streammeta:") || !strings.HasSuffix(key, ":lifecycle") { + if !strings.HasPrefix(key, "pulse:stream:") || !strings.HasSuffix(key, ":lifecycle") { return false } state, err := rdb.HGet(ctx, key, "state").Result() @@ -117,3 +168,118 @@ func isDestroyTombstone(ctx context.Context, rdb *redis.Client, key string) bool _, ok := content["=rev"] return ok } + +// acquireDatabaseLease reserves one Redis database through DB 0, which is used +// only for coordination. Expired leases make crashed test processes harmless. +func acquireDatabaseLease(t *testing.T) (*databaseLease, int) { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), databaseLeaseWait) + defer cancel() + coordinator := redis.NewClient(&redis.Options{Addr: redisAddr, Password: redisPwd, DB: 0}) + if err := coordinator.Ping(ctx).Err(); err != nil { + _ = coordinator.Close() + require.NoError(t, err) + } + config, err := coordinator.ConfigGet(ctx, "databases").Result() + if err != nil { + _ = coordinator.Close() + require.NoError(t, err) + } + databaseCount, err := strconv.Atoi(config["databases"]) + if err != nil || databaseCount < 2 { + _ = coordinator.Close() + require.NoError(t, fmt.Errorf("invalid Redis database capacity %q", config["databases"])) + } + token := ulid.Make().String() + ticker := time.NewTicker(25 * time.Millisecond) + defer ticker.Stop() + for { + for db := 1; db < databaseCount; db++ { + key := fmt.Sprintf("pulse:test:db-lease:%d", db) + acquired, err := coordinator.SetNX(ctx, key, token, databaseLeaseTTL).Result() + if err != nil { + _ = coordinator.Close() + require.NoError(t, err) + } + if !acquired { + continue + } + lease := &databaseLease{ + coordinator: coordinator, + key: key, + token: token, + stop: make(chan struct{}), + done: make(chan struct{}), + t: t, + } + go lease.heartbeat() + return lease, db + } + select { + case <-ctx.Done(): + _ = coordinator.Close() + require.NoError( + t, + ctx.Err(), + "timed out waiting for one of %d isolated Redis test databases", + databaseCount-1, + ) + case <-ticker.C: + } + } +} + +// heartbeat renews the lease while the test owns its database. +func (l *databaseLease) heartbeat() { + defer close(l.done) + ticker := time.NewTicker(databaseLeaseHeartbeat) + defer ticker.Stop() + for { + select { + case <-ticker.C: + ctx, cancel := context.WithTimeout(context.Background(), databaseLeaseHeartbeat) + renewed, err := renewDatabaseLeaseScript.Run( + ctx, + l.coordinator, + []string{l.key}, + l.token, + databaseLeaseTTL.Milliseconds(), + ).Int64() + cancel() + if err != nil { + heartbeatErr := fmt.Errorf("renew Redis database lease: %w", err) + l.lock.Lock() + l.heartbeatErr = heartbeatErr + l.lock.Unlock() + l.t.Errorf("%v", heartbeatErr) + return + } + if renewed != 1 { + heartbeatErr := fmt.Errorf("Redis database lease %q was lost", l.key) + l.lock.Lock() + l.heartbeatErr = heartbeatErr + l.lock.Unlock() + l.t.Errorf("%v", heartbeatErr) + return + } + case <-l.stop: + return + } + } +} + +// release stops renewal and atomically frees the owned database lease. +func (l *databaseLease) release(t *testing.T) { + t.Helper() + close(l.stop) + <-l.done + l.lock.Lock() + heartbeatErr := l.heartbeatErr + l.lock.Unlock() + assert.NoError(t, heartbeatErr) + ctx, cancel := context.WithTimeout(context.Background(), databaseLeaseHeartbeat) + defer cancel() + err := releaseDatabaseLeaseScript.Run(ctx, l.coordinator, []string{l.key}, l.token).Err() + assert.NoError(t, err) + assert.NoError(t, l.coordinator.Close()) +} From a7a10fdcc796cae1e51e62cc171cf3b1318fc6d6 Mon Sep 17 00:00:00 2001 From: "Raphael (manual office deploy after cloud-state fix)" Date: Fri, 24 Jul 2026 21:54:36 -0700 Subject: [PATCH 2/4] streaming: address review findings Delete the orphaned rollbackStreamRegistration helper (atomic registration made Redis rollback unnecessary), keep the exported Stream.MaxLen immutable after construction so concurrent readers never race with generation binding (the canonical bound lives in the private snapshot), and pin the v1 option fields to their leading positions with a reflect-based prefix check that keyed literals cannot provide. --- streaming/exact_publication_test.go | 3 ++- streaming/options/compat_test.go | 28 ++++++++++++++++++++++++++++ streaming/sink.go | 17 ----------------- streaming/stream_lifecycle.go | 1 - streaming/streams.go | 14 ++++++++------ 5 files changed, 38 insertions(+), 25 deletions(-) diff --git a/streaming/exact_publication_test.go b/streaming/exact_publication_test.go index f4d0ccb..6ef87ea 100644 --- a/streaming/exact_publication_test.go +++ b/streaming/exact_publication_test.go @@ -526,7 +526,8 @@ func TestDefaultHandlesAdoptWriterRetention(t *testing.T) { reader, err := readerHandle.NewReader(ctx) require.NoError(t, err) reader.Close() - require.Equal(t, 50_000, readerHandle.MaxLen) + require.Equal(t, 50_000, readerHandle.maxLen) + require.Equal(t, 1000, readerHandle.MaxLen, "exported MaxLen stays the construction value") require.Equal(t, time.Minute, readerHandle.ttl) require.True(t, readerHandle.ttlSliding) diff --git a/streaming/options/compat_test.go b/streaming/options/compat_test.go index c621949..35388fd 100644 --- a/streaming/options/compat_test.go +++ b/streaming/options/compat_test.go @@ -7,10 +7,12 @@ package options_test import ( + "reflect" "testing" "time" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "goa.design/pulse/pulse" "goa.design/pulse/streaming/options" @@ -53,3 +55,29 @@ func TestOptionStructsKeepV1Fields(t *testing.T) { assert.Equal(t, 1000, stream.MaxLen) assert.True(t, addEvent.OnlyIfStreamExists) } + +// TestV1FieldsFormStableOrderedPrefix pins the v1 fields to the leading +// positions of every exported option struct, in v1 order. Keyed literals +// cannot detect reordering, so this is the check that enforces the stable +// prefix the package documents. +func TestV1FieldsFormStableOrderedPrefix(t *testing.T) { + assertFieldPrefix(t, options.ReaderOptions{}, + "BlockDuration", "MaxPolled", "Topic", "TopicPattern", "BufferSize", "LastEventID") + assertFieldPrefix(t, options.SinkOptions{}, + "BlockDuration", "MaxPolled", "Topic", "TopicPattern", "BufferSize", "LastEventID", + "NoAck", "AckGracePeriod") + assertFieldPrefix(t, options.AddStreamOptions{}, "LastEventID") + assertFieldPrefix(t, options.StreamOptions{}, "MaxLen", "Logger", "TTL", "TTLSliding") + assertFieldPrefix(t, options.AddEventOptions{}, "Topic", "OnlyIfStreamExists") +} + +// assertFieldPrefix asserts that the struct's leading fields carry exactly +// the given names in order. +func assertFieldPrefix(t *testing.T, v any, names ...string) { + t.Helper() + typ := reflect.TypeOf(v) + require.GreaterOrEqual(t, typ.NumField(), len(names), typ.Name()) + for i, name := range names { + assert.Equal(t, name, typ.Field(i).Name, "%s field %d", typ.Name(), i) + } +} diff --git a/streaming/sink.go b/streaming/sink.go index 92718a5..f65584b 100644 --- a/streaming/sink.go +++ b/streaming/sink.go @@ -435,23 +435,6 @@ func (s *Sink) deleteStreamStaleConsumers( return nil } -// rollbackStreamRegistration detaches membership added by an AddStream that -// failed before local ownership was published. Empty Redis consumer metadata, -// membership, and keepalive state are removed atomically; shared group and -// cursor state remain owned by Stream.Destroy. -func (s *Sink) rollbackStreamRegistration(ctx context.Context, state *sinkStream) error { - _, err := detachSinkConsumer(ctx, state, s.Name, s.consumer) - if err != nil { - return fmt.Errorf( - "failed to roll back consumer %s for stream %s: %w", - s.consumer, - state.stream.Name, - err, - ) - } - return nil -} - // newConsumer creates one replacement consumer across every owned stream. No // stream observes local ownership unless all Redis consumers, memberships, and // the shared keep-alive are established; failures roll back the full prefix. diff --git a/streaming/stream_lifecycle.go b/streaming/stream_lifecycle.go index 946fc06..4ebe2ce 100644 --- a/streaming/stream_lifecycle.go +++ b/streaming/stream_lifecycle.go @@ -718,7 +718,6 @@ func (s *Stream) applyRetentionConfig(config string) error { return fmt.Errorf("invalid stream retention mode %q", mode) } s.maxLen = maxLen - s.MaxLen = maxLen s.retention = config return nil } diff --git a/streaming/streams.go b/streaming/streams.go index 768fc3f..125d178 100644 --- a/streaming/streams.go +++ b/streaming/streams.go @@ -22,13 +22,15 @@ type ( Stream struct { // Name of the stream. Name string - // MaxLen is the maximum retained event count requested at construction - // and, after Open, the active generation's canonical bound. It remains - // exported for v1 source compatibility. Mutating it after construction - // is unsupported; operations use the immutable private snapshot. + // MaxLen is the maximum retained event count requested at + // construction. It is immutable after NewStream and remains exported + // for v1 source compatibility; the generation's canonical bound is + // tracked privately so concurrent readers of this field never race + // with binding. MaxLen int - // maxLen is the immutable maximum retained event count. Zero means the - // stream is explicitly unbounded. + // maxLen is the canonical maximum retained event count adopted from + // the bound generation. Zero means the stream is explicitly + // unbounded. maxLen int // ttl configures an expiry for the Redis key backing the stream. ttl time.Duration From e4c8218fd201063b4dcaabfc17e18182eea65b47 Mon Sep 17 00:00:00 2001 From: "Raphael (manual office deploy after cloud-state fix)" Date: Fri, 24 Jul 2026 22:13:41 -0700 Subject: [PATCH 3/4] pool: synchronize the shared test hasher in TestStaleNodeStreamCleanup The hasher closure is shared by two nodes and invoked from concurrent routing and rebalance goroutines; its job counter must be atomic. Latent on main, exposed by the streaming lifecycle's timing changes under -race. --- pool/node_test.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/pool/node_test.go b/pool/node_test.go index 9f4c964..16ba44d 100644 --- a/pool/node_test.go +++ b/pool/node_test.go @@ -7,6 +7,7 @@ import ( "strconv" "strings" "sync" + "sync/atomic" "testing" "time" @@ -992,14 +993,15 @@ func TestStaleNodeStreamCleanup(t *testing.T) { rdb = ptesting.NewRedisClient(t) node1 = newFastCleanupTestNode(t, ctx, rdb, testName) node2 = newFastCleanupTestNode(t, ctx, rdb, testName) - numJobs = 0 + numJobs atomic.Int64 ) defer ptesting.CleanupRedis(t, rdb, false, testName) - // Configure nodes to send jobs to specific workers + // Configure nodes to send jobs to specific workers. The hasher is shared + // by both nodes and called from concurrent routing and rebalance + // goroutines, so its state must be synchronized. node1.h = &ptesting.Hasher{IndexFunc: func(key string, numBuckets int64) int64 { - numJobs++ - if numJobs > 2 { + if numJobs.Add(1) > 2 { return 0 // to avoid panics on cleanup where jobs get requeued } if key == "job1" { From e8f047c7ad2c96015ccc77afc082f0342d5f8cd5 Mon Sep 17 00:00:00 2001 From: "Raphael (manual office deploy after cloud-state fix)" Date: Fri, 24 Jul 2026 22:14:33 -0700 Subject: [PATCH 4/4] streaming: gofmt --- streaming/stream_lifecycle.go | 1 - 1 file changed, 1 deletion(-) diff --git a/streaming/stream_lifecycle.go b/streaming/stream_lifecycle.go index 4ebe2ce..43fc6fc 100644 --- a/streaming/stream_lifecycle.go +++ b/streaming/stream_lifecycle.go @@ -505,7 +505,6 @@ func (s *Stream) removeEvents(ctx context.Context, ids []string) error { return s.lifecycleError(err) } - // lifecycleError maps Redis's generation mismatch to the public sentinel. func (s *Stream) lifecycleError(err error) error { if err == nil {