From 5ce75246f6a0d25dc5f6ef5bc6ebd96badf49658 Mon Sep 17 00:00:00 2001 From: "Raphael (manual office deploy after cloud-state fix)" Date: Fri, 24 Jul 2026 18:27:27 -0700 Subject: [PATCH 1/3] pool: exact dispatch redesign with stale worker and node fencing Dispatch admission, retry resolution, and the start event become one Redis linearization point with real MaxQueuedJobs admission and durable generation-immutable terminal dispatch records. Cleanup runs exclusively under persisted Redis-time leases; scheduler planning and transitions run under distributed ticker ownership with foreign-job protection; settlement is node-owned with retry/join semantics so definitive outcomes are never abandoned. Every mutation a stale process could attempt is fenced at its own Redis linearization point: worker start claims re-verify worker registration, the cleanup fence, and the admitted payload identity; graceful requeue deactivation refuses under a cleanup fence and never recreates a removed registration; and job dispatch plus every scheduler transition and ownership script re-check the node keep-alive registration and node-cleanup field. Cleanup also removes legacy dispatch-map keys so adopted deployments retain no stale metadata. --- examples/pool/README.md | 6 +- examples/pool/producer/main.go | 2 +- examples/pool/scheduler/main.go | 12 +- examples/pool/worker/main.go | 2 +- pool/README.md | 219 ++- pool/cleanup.go | 432 ++++++ pool/exact_dispatch_test.go | 693 ++++++++++ pool/marshal.go | 249 ++-- pool/marshal_test.go | 106 +- pool/node.go | 2209 ++++++++++++++++++++++++------- pool/node_cleanup.go | 231 ++++ pool/node_cleanup_test.go | 147 ++ pool/node_jobloss_test.go | 38 +- pool/node_options.go | 128 +- pool/node_test.go | 1036 ++++++++++++--- pool/pool_map.go | 248 ++++ pool/resources.go | 379 ++++++ pool/scheduler.go | 465 +++++-- pool/scheduler_test.go | 364 ++++- pool/scheduler_transition.go | 555 ++++++++ pool/scripts.go | 390 +++++- pool/settlement.go | 223 ++++ pool/testing.go | 38 +- pool/ticker.go | 127 +- pool/ticker_test.go | 46 +- pool/worker.go | 546 ++++++-- pool/worker_cleanup.go | 656 +++++++++ pool/worker_cleanup_test.go | 256 ++++ pool/worker_test.go | 174 ++- scripts/run-examples | 8 +- 30 files changed, 8791 insertions(+), 1194 deletions(-) create mode 100644 pool/cleanup.go create mode 100644 pool/exact_dispatch_test.go create mode 100644 pool/node_cleanup.go create mode 100644 pool/node_cleanup_test.go create mode 100644 pool/pool_map.go create mode 100644 pool/resources.go create mode 100644 pool/scheduler_transition.go create mode 100644 pool/settlement.go create mode 100644 pool/worker_cleanup.go create mode 100644 pool/worker_cleanup_test.go diff --git a/examples/pool/README.md b/examples/pool/README.md index ed23c2f..abfa8f9 100644 --- a/examples/pool/README.md +++ b/examples/pool/README.md @@ -5,7 +5,7 @@ This example shows how to use the pool package to create a pool of workers. It h 1. The `worker` process registers a worker with the node and waits for jobs. 2. The `producer` process starts and stops two jobs. It also notifies the worker that owns the second job. -3. The `scheduler` process starts runs a schedule that starts and stops jobs alternately. +3. The `scheduler` process runs a schedule that starts and stops jobs alternately. ## Running the example @@ -24,8 +24,8 @@ $ source .env $ go run examples/pool/worker/main.go ``` -The above start two workers that wait for jobs. Then, in separate terminals, run -the following commands: +The above starts two workers that wait for jobs. Then, in a separate terminal, +run the following command: ```bash $ source .env diff --git a/examples/pool/producer/main.go b/examples/pool/producer/main.go index bfd91dc..733f055 100644 --- a/examples/pool/producer/main.go +++ b/examples/pool/producer/main.go @@ -14,7 +14,7 @@ import ( func main() { // Setup Redis connection rdb := redis.NewClient(&redis.Options{ - Addr: "localhost:6379", + Addr: os.Getenv("REDIS_ADDR"), Password: os.Getenv("REDIS_PASSWORD"), }) diff --git a/examples/pool/scheduler/main.go b/examples/pool/scheduler/main.go index be7fe5d..48f1d30 100644 --- a/examples/pool/scheduler/main.go +++ b/examples/pool/scheduler/main.go @@ -14,7 +14,7 @@ import ( func main() { // Setup Redis connection rdb := redis.NewClient(&redis.Options{ - Addr: "localhost:6379", + Addr: os.Getenv("REDIS_ADDR"), Password: os.Getenv("REDIS_PASSWORD"), }) @@ -65,8 +65,16 @@ func (p *producer) Name() string { return "example" } -// Plan is called by the scheduler to determine the next job to start or stop. +// Plan preserves the v1 producer contract. func (p *producer) Plan() (*pool.JobPlan, error) { + return p.PlanContext(context.Background()) +} + +// PlanContext computes the next jobs and stops promptly when the node closes. +func (p *producer) PlanContext(ctx context.Context) (*pool.JobPlan, error) { + if err := ctx.Err(); err != nil { + return nil, err + } p.iter++ if p.iter > 10 { log.Infof(p.logctx, "done") diff --git a/examples/pool/worker/main.go b/examples/pool/worker/main.go index 9199ddc..a6c733f 100644 --- a/examples/pool/worker/main.go +++ b/examples/pool/worker/main.go @@ -37,7 +37,7 @@ type ( func main() { // Setup Redis connection rdb := redis.NewClient(&redis.Options{ - Addr: "localhost:6379", + Addr: os.Getenv("REDIS_ADDR"), Password: os.Getenv("REDIS_PASSWORD"), }) diff --git a/pool/README.md b/pool/README.md index b9c096a..b03520b 100644 --- a/pool/README.md +++ b/pool/README.md @@ -106,7 +106,7 @@ func (h *JobHandler) HandleNotification(key string, payload []byte) error { The function `AddNode` is used to create a new pool node. It takes as input a name, a Redis client and a set of options. -[![Pool AddNode](../snippets/pool-addnode.png)](../examples/pool/worker/main.go#L43-L47) +[![Pool AddNode](../snippets/pool-addnode.png)](../examples/pool/worker/main.go#L53-L57) The `AddNode` function returns a new pool node and an error. The pool node should be closed when it is no longer needed (see below). @@ -121,28 +121,140 @@ available: a worker can go without sending a health check before it's considered inactive and removed from the pool. If a worker doesn't report its status within this time frame, it will be removed, allowing the pool to reassign its jobs to other - active workers. The default value is 30 seconds. -* `WithWorkerShutdownTTL` - specifies the maximum time to wait for a worker to - shutdown gracefully. This is the duration the pool will wait for a worker to - finish its current job and perform any cleanup operations before forcefully - terminating it. If the worker doesn't shut down within this time, it will be - forcefully stopped. The default value is 2 minutes. -* `WithMaxQueuedJobs` - sets the maximum number of jobs that can be queued - before the pool starts rejecting new jobs. This limit applies to the entire - pool across all nodes. When this limit is reached, any attempt to dispatch - new jobs will result in an error. The default value is 1000 jobs. -* `WithAckGracePeriod` - sets the grace period for job acknowledgment. If a - worker doesn't acknowledge starting a job within this duration, the job - becomes available for other workers to claim. This prevents jobs from being - stuck if a worker fails to start processing them. The default value is 20 - seconds. + active workers. It is immutable for the pool generation and every node, + including client-only nodes, must configure the same value. The same Redis-time + threshold governs stale workers and stale nodes. The default value is 30 seconds. +* `WithRequeueTimeout` - bounds one local worker's concurrent requeue + handoff attempt during graceful removal or `Close`. It does not govern remote + takeover: `WithWorkerTTL` determines stale-worker liveness and Redis owns a + renewable cleanup lease for each stale worker. Pulse cannot forcibly + terminate application handler goroutines. The default is 2 minutes. + `WithWorkerShutdownTTL` remains as a deprecated v1-compatible alias. +* `WithMaxQueuedJobs` - sets the immutable generation-wide count of active + job-key admissions. Admission fails with `ErrPoolCapacity` at the limit. + The default is 1000. +* `WithDispatchTimeout` - sets how long a caller waits for the worker result. + Timeout removes only the local waiter; durable admission remains until a + definitive worker completion. The default is 40 seconds. +* `WithDispatchResultRetention` - sets the immutable generation-wide replay + window for terminal `DispatchJobOnce` results. It must exceed both dispatch + timeout and recovery grace. During this window, any node retrying the same + dispatch ID and exact identity receives the original event ID and terminal + outcome. After expiry that replay guarantee ends and the ID may be admitted + as a new dispatch. The default is five minutes. +* `WithRecoveryGrace` - sets sink stale recovery and orphan convergence grace. + The default is 20 seconds. `WithAckGracePeriod` remains as a deprecated + v1-compatible alias. +* `WithCleanupLease` - sets the immutable durable pool-cleanup owner lease. + Cleanup renews this Redis-time lease before each destructive generation-owned + step; if the owner dies, another process may take over only after the lease is + stale. It does not govern worker or node liveness. The default is 30 seconds. + +`DispatchJobOnce` accepts a caller-owned, globally unique dispatch ID. One Lua +operation stores its exact length-delimited job-key and payload identity, +claims the separate job-key admission index, appends the untrimmed start event, +and records its event ID. An exact retry returns that event ID or its original +terminal outcome; reusing the ID with different bytes returns +`ErrDispatchConflict`. Another active dispatch for the key returns +`ErrJobExists`. Terminal settlement atomically persists the result, clears +admission, acknowledges the consumer event, advances recovery, and deletes the +settled event. Active dispatch records have no expiry. Settlement removes them +from the active cleanup index and applies `DispatchResultRetention` to the +per-dispatch terminal record, bounding replay memory. Callers on every node use +local completion only as a wake-up hint and re-read that record after polling, +notification, cancellation, and timeout edges. Once a handler returns, its node +owns retrying settlement independently of worker intake cancellation. Worker +removal and graceful node closure join those obligations and report an error +while any remain pending. If the process dies first, the original pool event +remains pending with its dispatch ID and active record; sink recovery may +re-execute the handler under that same identity. Because a process can die +after the handler's external side effect but before settlement, handlers must +make those side effects idempotent or fence them with the dispatch identity. +Pool events are never MAXLEN-trimmed while unsettled. + +Schedulers acquire one renewable Redis-time owner for each due transition. +That exact owner remains fenced across planning, +start/apply/stop work, authoritative ownership scans, and canonical next-time +commit; no later tick can be claimed while the transition remains live. Each +job first receives one persisted scheduler dispatch ID and is then admitted +through the same owner fence. A foreign `ErrJobExists` never proves scheduler +ownership. Stop and StopAll publish a stop only when both the transition lease +and exact scheduler dispatch capability remain current, so ordinary jobs and +jobs owned by another schedule are untouched. The v1 `JobProducer.Plan()` +contract remains supported and is inherently non-cancellable. Producers whose +planning may block should also implement `ContextJobProducer.PlanContext(ctx)`; +the scheduler prefers that method, and `Node.Close` cancels and joins it. + +`StopJob` durably publishes a stop request and returns after Redis accepts that +request. It does not mean the current handler has already completed; observe +worker/job state when completion matters. + +### Quiescent upgrades + +This is a persisted pool-format cutover, not a Go API major-version migration; +the v1 producer interface and deprecated option aliases remain source +compatible. + +The generation/resource and dispatch wire formats do not support mixed +versions. Upgrade a pool by: (1) stop every producer, client-only node, routing +node, worker, and scheduler; (2) verify every flat map listed below contains no +user entries, discover and check legacy per-producer scheduler maps, and verify +the flat pool stream has no entries or groups; (3) +deploy the new version everywhere; (4) start one node and verify its resource +manifest has `format_version=7`, generation-qualified scheduler resources, and +the intended `max_queued_jobs`, `worker_ttl_ms`, `cleanup_lease_ms`, and +`dispatch_result_retention_ms`; then (5) +resume the remaining nodes and producers. The first upgraded node refuses to +adopt manifestless legacy resources while any actual flat resource proves a +writer may still be active, returning `ErrQuiescenceRequired`. + +For a pool named `$POOL`, inspect the exact standalone-Redis evidence before +deploying. Every map below must contain no fields except rmap metadata +(`=rev`/`=kind`), and both stream commands must report no retained work or +consumer groups: + +```bash +redis-cli HGETALL "map:${POOL}:node-keepalive:content" +redis-cli HGETALL "map:${POOL}:shutdown:content" +redis-cli HGETALL "map:${POOL}:workers:content" +redis-cli HGETALL "map:${POOL}:worker-keepalive:content" +redis-cli HGETALL "map:${POOL}:worker-cleanup:content" +redis-cli HGETALL "map:${POOL}:jobs:content" +redis-cli HGETALL "map:${POOL}:pending-jobs:content" +redis-cli HGETALL "map:${POOL}:dispatches:content" +redis-cli HGETALL "map:${POOL}:job-payloads:content" +redis-cli HGETALL "map:${POOL}:tickers:content" +redis-cli HGETALL "map:${POOL}:scheduler-jobs:content" +redis-cli --scan --pattern "map:${POOL}:*:content" +redis-cli XLEN "pulse:stream:${POOL}:pool" +redis-cli XINFO GROUPS "pulse:stream:${POOL}:pool" +redis-cli --scan --pattern "pulse:stream:${POOL}:node:*" +``` + +Inspect every key returned by the scheduler-map scan; legacy versions created +dynamic `map:${POOL}::content` hashes. Any non-metadata field requires +the old writer to be stopped before upgrade. Once all listed evidence proves +quiescence, first adoption may delete orphaned bare +`pulse:stream:${POOL}:node:` streams that have no lifecycle record. +This store-owned upgrade cleanup is not part of normal `Stream.Destroy`. + +After the first upgraded node starts, verify the selected generation contract +with `redis-cli HGETALL "pulse:pool:${POOL}:resources"`. + +Rollback requires the same stop-all-users boundary. Do not start an older +binary against a manifest already marked format 7; restore Redis from the +pre-upgrade snapshot or complete cleanup and create a fresh pool generation +before starting the older version. ### Closing A Node The `Close` method closes the pool node and releases all resources associated -with it. It should be called when the node is no longer needed. +with it. It should be called when the node is no longer needed. Closing is an +immediate admission fence: once it begins, the node rejects new workers, job or +message dispatches, stop requests, and notifications while already admitted +operations finish. -[![Pool Close](../snippets/pool-close.png)](../examples/pool/producer/main.go#L31-L36) +[![Pool Close](../snippets/pool-close.png)](../examples/pool/producer/main.go#L66-L70) Note that closing a pool node does not stop remote workers. It only stops the local pool node. Remote workers can be stopped by calling the `Shutdown` method @@ -151,9 +263,39 @@ described below. ### Shutting Down A Pool The `Shutdown` method shuts down the entire pool by stopping all its workers -gracefully. It should be called when the pool is no longer needed. - -[![Pool Shutdown](../snippets/pool-shutdown.png)](../examples/pool/worker/main.go#L62-L64) +gracefully. It should be called when the pool is no longer needed. Shutdown +publishes one Redis-owned obligation even when a local `Close` is concurrent. +After every live node detaches, final cleanup is owned by a persisted +owner-and-lease claim based on Redis time. Another process can reclaim an +expired claim and finish cleanup, so an interrupted shutdown cannot permanently +block reuse of the pool name. Successful cleanup compacts the claim to one +bounded generation completion marker. Every destructive stream or map mutation +atomically verifies the exact cleanup generation, owner, and unexpired lease; +a paused former owner cannot delete resources created after takeover and pool +reuse. + +Stale worker and node takeover use the same rule at a narrower scope. One Lua +operation reads the authoritative heartbeat, compares it with immutable +`WorkerTTL` using Redis time, and installs an exact owner/fence. Heartbeat +scripts reject that fence, and every mutation a resumed stale process could +attempt re-verifies liveness at its own Redis linearization point: a worker +start claim re-checks the worker's registration and cleanup fence, graceful +requeue deactivation refuses under a fence and never recreates a removed +registration, and job dispatch plus every scheduler transition and ownership +script re-check the node's keep-alive registration and node-cleanup field. +Requeue, dispatch-release, stream destruction, and discovery removal verify +the same unexpired owner token. + +The pool stream generation selects every shared map and stream as one immutable +resource manifest. The first deployment seen by this version adopts existing +flat names without migration. After explicit shutdown cleanup, the next +generation uses qualified names. A paused stale node therefore retains access +only to its old keepalive, worker, job, ticker, scheduler, dispatch-record, and +stream resources and cannot mutate a reused pool. Every node-owned map mutation also +checks that exact stream generation inside the same Redis operation, preventing +a paused stale node from recreating an old map key after cleanup. + +[![Pool Shutdown](../snippets/pool-shutdown.png)](../examples/pool/worker/main.go#L90-L92) See the [Data Flows](#data-flows) section below for more details on the shutdown process. @@ -163,7 +305,7 @@ shutdown process. The function `AddWorker` is used to create a new worker. It takes as input a job handler object. -[![Worker AddWorker](../snippets/pool-addworker.png)](../examples/pool/worker/main.go#L55-L57) +[![Worker AddWorker](../snippets/pool-addworker.png)](../examples/pool/worker/main.go#L59-L63) The job handler must implement the `Start` and `Stop` methods used to start and stop durable jobs. The handler may also implement `HandleMessage` to receive @@ -185,8 +327,24 @@ The job key is used to route the job to the proper worker. If the worker starts the job successfully, the worker owns that key until the job stops or moves during rebalancing. The job payload is passed to the worker's `Start` method. -The `DispatchJob` method returns an error if the job could not be dispatched. -This can happen if the pool is full or if the job key is invalid. +Pulse establishes the durable dispatch record in the same operation that +publishes the start event, so even an immediate worker acknowledgement cannot +outrun correlation. Concurrent local retries share one broadcast completion +signal. If a caller cancels or times out while completion is unknown, Pulse +retains job-key admission and removes only that caller's wait. Definitive worker +acknowledgement persists the terminal record and releases admission even if the +caller or dispatching process is gone. Job keys are reusable command identities: +while a start is +pending or the job is running, duplicates return `ErrJobExists`; after +definitive completion and a later stop removes ownership, the same key may be +dispatched again. `DispatchJob` also returns errors for invalid keys, duplicate +active/pending jobs, and worker start failures. + +If a worker's `Start` handler fails, Pulse atomically removes that worker's +ownership and the durable payload before it publishes the failure +acknowledgement or releases singleton admission. A Redis failure during this +cleanup leaves the start event pending for retry; it is never reported as +complete with orphaned ownership or payload. ### Dispatching A Message @@ -195,6 +353,11 @@ worker currently assigned by the pool hash ring. Messages are the right primitiv for short-lived work that needs stable key-based routing but must not create a durable job. +Pool streams are poison-message tolerant: malformed external binary envelopes, +jobs, acknowledgements, and keyed payloads are logged and acknowledged as +terminal malformed input. Routing and worker loops continue processing later +valid events. + Messages do not write job payloads and do not require any worker to own a job with the same key. The receiving worker must implement `HandleMessage`. A message handler can return `ErrRequeue` to leave the message pending for @@ -224,5 +387,13 @@ dispatched or stopped on a recurring basis. The method takes as input a job producer and invokes it at the specified interval. The job producer returns a list of jobs to be started and stopped. +The producer's shared ticker establishes distributed ownership before the +initial `Plan` and every later transition, so same-name producers on different +nodes do not plan concurrently. Applied job ownership is canonical in the +generation-qualified scheduler map. Planning, dispatch, stop, and ownership-map +failures return to the tick owner and are retried on a later owned transition; +they are never treated as successful progress. Scheduler intervals use +millisecond precision and must be at least one millisecond. + `Schedule` makes it possible to maintain a pool of jobs for example in a multi-tenant system. See the [examples](../examples/pool) for more details. diff --git a/pool/cleanup.go b/pool/cleanup.go new file mode 100644 index 0000000..8fa7532 --- /dev/null +++ b/pool/cleanup.go @@ -0,0 +1,432 @@ +// Package pool keeps final pool cleanup as a Redis-leased obligation. The +// cleanup record is bounded to one generation, any process may reclaim an +// expired owner, and the shutdown admission fence remains present until every +// shared stream and map has been removed. +package pool + +import ( + "context" + "fmt" + "strconv" + "strings" + "time" + + redis "github.com/redis/go-redis/v9" +) + +const ( + poolCleanupFinishingState = "finishing" + poolCleanupCompleteState = "complete" + + poolCleanupBusy = int64(0) + poolCleanupClaimed = int64(1) + poolCleanupAlreadyComplete = int64(2) +) + +var ( + // claimPoolCleanupScript atomically verifies the empty node barrier and + // acquires or renews the cleanup lease using Redis's clock. + claimPoolCleanupScript = redis.NewScript(` +for _, key in ipairs(redis.call("HKEYS", KEYS[1])) do + if string.sub(key, 1, 1) ~= "=" then + return 0 + end +end + +local clock = redis.call("TIME") +local now = (tonumber(clock[1]) * 1000) + math.floor(tonumber(clock[2]) / 1000) +local state = redis.call("HGET", KEYS[2], "state") +local generation = redis.call("HGET", KEYS[2], "generation") +local owner = redis.call("HGET", KEYS[2], "owner") +local lease_until = tonumber(redis.call("HGET", KEYS[2], "lease_until") or "0") + +if state == ARGV[4] and generation == ARGV[1] then + return 2 +end +if state == ARGV[3] and owner ~= ARGV[2] and lease_until > now then + return 0 +end + +redis.call("HSET", KEYS[2], + "state", ARGV[3], + "generation", ARGV[1], + "owner", ARGV[2], + "lease_until", tostring(now + tonumber(ARGV[5]))) +return 1 +`) + + // completePoolCleanupScript verifies ownership, then compacts all historical + // claim fields to one bounded completion marker. + completePoolCleanupScript = redis.NewScript(` +local clock = redis.call("TIME") +local now = (tonumber(clock[1]) * 1000) + math.floor(tonumber(clock[2]) / 1000) +if redis.call("HGET", KEYS[1], "state") ~= ARGV[3] +or redis.call("HGET", KEYS[1], "generation") ~= ARGV[1] +or redis.call("HGET", KEYS[1], "owner") ~= ARGV[2] +or tonumber(redis.call("HGET", KEYS[1], "lease_until") or "0") <= now then + return redis.error_reply("POOLCLEANUPLOST") +end +if redis.call("HGET", KEYS[2], "state") ~= ARGV[5] +or redis.call("HGET", KEYS[2], "generation") ~= ARGV[1] then + return redis.error_reply("POOLRESOURCELOST") +end +redis.call("DEL", KEYS[1]) +redis.call("HSET", KEYS[1], "state", ARGV[4], "generation", ARGV[1]) +redis.call("HSET", KEYS[2], "state", ARGV[6]) +return 1 +`) + + // destroyCleanupStreamScript verifies the live cleanup lease and exact + // stream incarnation in the same operation that invalidates and deletes it. + destroyCleanupStreamScript = redis.NewScript(` +local clock = redis.call("TIME") +local now = (tonumber(clock[1]) * 1000) + math.floor(tonumber(clock[2]) / 1000) +if redis.call("HGET", KEYS[1], "state") ~= ARGV[3] +or redis.call("HGET", KEYS[1], "generation") ~= ARGV[1] +or redis.call("HGET", KEYS[1], "owner") ~= ARGV[2] +or tonumber(redis.call("HGET", KEYS[1], "lease_until") or "0") <= now then + return redis.error_reply("POOLCLEANUPLOST") +end +redis.call("HSET", KEYS[1], "lease_until", tostring(now + tonumber(ARGV[4]))) + +if redis.call("HGET", KEYS[2], "generation") ~= ARGV[1] then + return redis.error_reply("STREAMDESTROYED") +end +local state = redis.call("HGET", KEYS[2], "state") +if state == "destroyed" then + return 0 +end +if state ~= "active" then + return redis.error_reply("STREAMDESTROYED") +end +local physical = redis.call("HGET", KEYS[2], "physical_key") +if not physical then + return redis.error_reply("STREAMDESTROYED") +end +redis.call("HSET", KEYS[2], "state", "destroyed") +local rev = redis.call("HINCRBY", KEYS[3], "=rev", 1) +local resources = redis.call("SMEMBERS", KEYS[5]) +if #resources > 0 then + redis.call("DEL", unpack(resources)) +end +redis.call("DEL", physical, physical .. ":sink-recovery:" .. ARGV[1], KEYS[3]) +redis.call("DEL", KEYS[5]) +redis.call("PUBLISH", KEYS[4], "destroy:" .. tostring(rev)) +return 1 +`) + + // destroyCleanupMapScript deletes one pool map only while this exact cleanup + // owner still holds an unexpired lease. + destroyCleanupMapScript = redis.NewScript(` +local clock = redis.call("TIME") +local now = (tonumber(clock[1]) * 1000) + math.floor(tonumber(clock[2]) / 1000) +if redis.call("HGET", KEYS[1], "state") ~= ARGV[3] +or redis.call("HGET", KEYS[1], "generation") ~= ARGV[1] +or redis.call("HGET", KEYS[1], "owner") ~= ARGV[2] +or tonumber(redis.call("HGET", KEYS[1], "lease_until") or "0") <= now then + return redis.error_reply("POOLCLEANUPLOST") +end +redis.call("HSET", KEYS[1], "lease_until", tostring(now + tonumber(ARGV[4]))) +local rev = redis.call("HINCRBY", KEYS[2], "=rev", 1) +redis.call("DEL", KEYS[2]) +redis.call("PUBLISH", KEYS[3], "destroy:" .. tostring(rev)) +return 1 +`) + + // destroyCleanupDispatchesScript removes all unsettled per-dispatch records + // while the exact cleanup owner holds its Redis-time lease. Settled records + // are self-bounded by DispatchResultRetention and need no generation index. + destroyCleanupDispatchesScript = redis.NewScript(` +local clock = redis.call("TIME") +local now = (tonumber(clock[1]) * 1000) + math.floor(tonumber(clock[2]) / 1000) +if redis.call("HGET", KEYS[1], "state") ~= ARGV[3] +or redis.call("HGET", KEYS[1], "generation") ~= ARGV[1] +or redis.call("HGET", KEYS[1], "owner") ~= ARGV[2] +or tonumber(redis.call("HGET", KEYS[1], "lease_until") or "0") <= now then + return redis.error_reply("POOLCLEANUPLOST") +end +redis.call("HSET", KEYS[1], "lease_until", tostring(now + tonumber(ARGV[4]))) +local records = redis.call("SMEMBERS", KEYS[2]) +if #records > 0 then + redis.call("DEL", unpack(records)) +end +redis.call("DEL", KEYS[2]) +return 1 +`) +) + +// resumeExpiredPoolCleanup lets AddNode complete an abandoned finishing claim +// before attempting admission under the same pool name. +func resumeExpiredPoolCleanup( + ctx context.Context, + pool, owner string, + lease time.Duration, + rdb *redis.Client, +) error { + record, err := rdb.HGetAll(ctx, poolCleanupGenerationsKey(pool)).Result() + if err != nil { + return fmt.Errorf("failed to read pool cleanup claim: %w", err) + } + if record["state"] != poolCleanupFinishingState { + return nil + } + generation := record["generation"] + if generation == "" { + return fmt.Errorf("pool %q has a finishing cleanup without a generation", pool) + } + resources, err := loadPoolResources(ctx, rdb, pool, generation) + if err != nil { + return fmt.Errorf("failed to load pool cleanup resources: %w", err) + } + if err := reapExpiredPoolNodes(ctx, rdb, pool, generation, owner, resources.workerTTL); err != nil { + return err + } + status, err := claimPoolCleanup(ctx, rdb, pool, generation, owner, lease) + if err != nil { + return fmt.Errorf("failed to reclaim pool cleanup: %w", err) + } + if status == poolCleanupBusy { + return fmt.Errorf("pool %q is shutting down", pool) + } + if status == poolCleanupAlreadyComplete { + return nil + } + if err := cleanupPoolResources(ctx, rdb, pool, generation, owner, lease); err != nil { + return fmt.Errorf("failed to resume pool cleanup: %w", err) + } + return nil +} + +// claimPoolCleanup acquires or renews the persisted cleanup owner and lease. +func claimPoolCleanup( + ctx context.Context, + rdb *redis.Client, + pool, generation, owner string, + lease time.Duration, +) (int64, error) { + resources, err := loadPoolResources(ctx, rdb, pool, generation) + if err != nil { + return 0, err + } + if lease != resources.cleanupLease { + return 0, fmt.Errorf( + "%w: pool %q cleanup lease is %v, requested %v", + ErrPoolConfigMismatch, + pool, + resources.cleanupLease, + lease, + ) + } + return claimPoolCleanupScript.Run( + ctx, + rdb, + []string{ + rmapContentKey(resources.nodeKeepAlive), + poolCleanupGenerationsKey(pool), + }, + generation, + owner, + poolCleanupFinishingState, + poolCleanupCompleteState, + strconv.FormatInt(lease.Milliseconds(), 10), + ).Int64() +} + +// cleanupPoolResources renews ownership before each idempotent destructive +// step. The shutdown map is destroyed last, immediately before the bounded +// completion marker replaces the finishing claim. +func cleanupPoolResources( + ctx context.Context, + rdb *redis.Client, + pool, generation, owner string, + lease time.Duration, +) error { + resources, err := loadPoolResources(ctx, rdb, pool, generation) + if err != nil { + return err + } + if lease != resources.cleanupLease { + return fmt.Errorf( + "%w: pool %q cleanup lease is %v, requested %v", + ErrPoolConfigMismatch, + pool, + resources.cleanupLease, + lease, + ) + } + if err := destroyCleanupStream(ctx, rdb, pool, generation, owner, lease); err != nil { + return fmt.Errorf("destroy pool stream: %w", err) + } + if err := destroyPoolDispatches(ctx, rdb, pool, resources.dispatches, generation, owner, lease); err != nil { + return err + } + + for _, name := range resources.mapNames() { + if name == resources.nodeShutdown { + continue + } + if err := destroyPoolMap(ctx, rdb, pool, name, generation, owner, lease); err != nil { + return err + } + } + if err := destroyPoolMap(ctx, rdb, pool, resources.nodeShutdown, generation, owner, lease); err != nil { + return err + } + if err := completePoolCleanupScript.Run( + ctx, + rdb, + []string{poolCleanupGenerationsKey(pool), poolResourcesKey(pool)}, + generation, + owner, + poolCleanupFinishingState, + poolCleanupCompleteState, + poolResourceStateActive, + poolResourceStateDestroyed, + ).Err(); err != nil { + return fmt.Errorf("failed to record cleanup completion: %w", err) + } + return nil +} + +// destroyPoolDispatches deletes every active dispatch record and its bounded +// generation index under the current cleanup lease. +func destroyPoolDispatches( + ctx context.Context, + rdb *redis.Client, + pool, resource, generation, owner string, + lease time.Duration, +) error { + err := destroyCleanupDispatchesScript.Run( + ctx, + rdb, + []string{ + poolCleanupGenerationsKey(pool), + dispatchActiveKey(resource), + }, + generation, + owner, + poolCleanupFinishingState, + strconv.FormatInt(lease.Milliseconds(), 10), + ).Err() + if err != nil { + return fmt.Errorf("destroy pool dispatch records: %w", err) + } + return nil +} + +// renewPoolCleanup proves that this process still owns cleanup and extends its +// Redis-time lease. +func renewPoolCleanup( + ctx context.Context, + rdb *redis.Client, + pool, generation, owner string, + lease time.Duration, +) error { + status, err := claimPoolCleanup(ctx, rdb, pool, generation, owner, lease) + if err != nil { + return fmt.Errorf("renew cleanup lease: %w", err) + } + if status == poolCleanupBusy { + return fmt.Errorf("cleanup lease for pool %q is owned by another process", pool) + } + if status == poolCleanupAlreadyComplete { + return fmt.Errorf("cleanup for pool %q is already complete", pool) + } + return nil +} + +// reapExpiredPoolNodes removes stale registrations before an abandoned cleanup +// claim is considered busy or failed. +func reapExpiredPoolNodes( + ctx context.Context, + rdb *redis.Client, + pool, generation, owner string, + ttl time.Duration, +) error { + resources, err := loadPoolResources(ctx, rdb, pool, generation) + if err != nil { + return err + } + if ttl != resources.workerTTL { + return fmt.Errorf( + "%w: pool %q worker TTL is %v, requested %v", + ErrPoolConfigMismatch, + pool, + resources.workerTTL, + ttl, + ) + } + key := rmapContentKey(resources.nodeKeepAlive) + nodes, err := rdb.HKeys(ctx, key).Result() + if err != nil { + return fmt.Errorf("read pool node leases: %w", err) + } + for _, nodeID := range nodes { + if strings.HasPrefix(nodeID, "=") { + continue + } + if _, err := cleanupStalePoolNode( + ctx, + rdb, + resources, + nodeID, + owner, + ); err != nil { + return fmt.Errorf("reap stale node %q: %w", nodeID, err) + } + } + return nil +} + +// destroyPoolMap joins one known pool map solely to publish its canonical +// destroy operation, then releases the local replica. +func destroyPoolMap( + ctx context.Context, + rdb *redis.Client, + pool, name, generation, owner string, + lease time.Duration, +) error { + if err := destroyCleanupMapScript.Run( + ctx, + rdb, + []string{ + poolCleanupGenerationsKey(pool), + rmapContentKey(name), + rmapUpdateChannel(name), + }, + generation, + owner, + poolCleanupFinishingState, + strconv.FormatInt(lease.Milliseconds(), 10), + ).Err(); err != nil { + return fmt.Errorf("destroy pool map %q: %w", name, err) + } + return nil +} + +// destroyCleanupStream invalidates the exact pool stream generation under the +// same Redis-time lease check that deletes its physical data and metadata. +func destroyCleanupStream( + ctx context.Context, + rdb *redis.Client, + pool, generation, owner string, + lease time.Duration, +) error { + name := poolStreamName(pool) + membership := rmapContentKey(fmt.Sprintf("stream:%s:generation:%s:sinks", name, generation)) + return destroyCleanupStreamScript.Run( + ctx, + rdb, + []string{ + poolCleanupGenerationsKey(pool), + fmt.Sprintf("pulse:stream:%s:lifecycle", name), + membership, + rmapUpdateChannel(fmt.Sprintf("stream:%s:generation:%s:sinks", name, generation)), + fmt.Sprintf("pulse:stream:%s:generation:%s:resources", name, generation), + }, + generation, + owner, + poolCleanupFinishingState, + strconv.FormatInt(lease.Milliseconds(), 10), + ).Err() +} diff --git a/pool/exact_dispatch_test.go b/pool/exact_dispatch_test.go new file mode 100644 index 0000000..c7b3fca --- /dev/null +++ b/pool/exact_dispatch_test.go @@ -0,0 +1,693 @@ +// Atomic dispatch tests exercise the Redis-owned admission/publication point, +// immutable pool capacity, and quiescent legacy adoption. +package pool + +import ( + "context" + "errors" + "fmt" + "net" + "sync" + "sync/atomic" + "testing" + "time" + + redis "github.com/redis/go-redis/v9" + "github.com/stretchr/testify/require" + + ptesting "goa.design/pulse/testing" +) + +type ( + // ambiguousDispatchHook returns one client error after the dispatch script + // has committed, reproducing an ambiguous transport response. + ambiguousDispatchHook struct { + fail atomic.Bool + err error + scriptHash string + } + + // cleanupDelayHook pauses after one cleanup-map lease renewal, proving the + // next destructive step uses CleanupLease rather than WorkerTTL. + cleanupDelayHook struct { + delayed atomic.Bool + delay time.Duration + scriptHash string + } + + // settlementFailureHook holds exact terminal settlement unavailable until + // a test releases it. + settlementFailureHook struct { + fail atomic.Bool + attempted chan struct{} + once sync.Once + err error + scriptHash string + } +) + +func TestDispatchJobOnceRetriesAmbiguousResponse(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + hook := &ambiguousDispatchHook{ + err: errors.New("ambiguous dispatch response"), + scriptHash: luaDispatchJob.Hash(), + } + rdb.AddHook(hook) + ctx := ptesting.NewTestContext(t) + node, err := AddNode( + ctx, + t.Name(), + rdb, + WithDispatchTimeout(25*time.Millisecond), + ) + require.NoError(t, err) + require.NoError(t, luaDispatchJob.Load(ctx, rdb).Err()) + + hook.fail.Store(true) + _, err = node.DispatchJobOnce(ctx, "dispatch-1", "job", []byte("payload")) + require.ErrorIs(t, err, hook.err) + eventID, err := node.DispatchJobOnce(ctx, "dispatch-1", "job", []byte("payload")) + require.ErrorContains(t, err, "timed out") + require.NotEmpty(t, eventID) + require.EqualValues(t, 1, rdb.XLen(ctx, generationStreamKey(ctx, rdb, node.poolStream.Name)).Val()) + require.NoError(t, node.completeDispatch(ctx, "job", "dispatch-1")) + require.EqualValues(t, 0, rdb.XLen(ctx, generationStreamKey(ctx, rdb, node.poolStream.Name)).Val()) + retryID, err := node.DispatchJobOnce(ctx, "dispatch-1", "job", []byte("payload")) + require.NoError(t, err) + require.Equal(t, eventID, retryID) + _, err = node.DispatchJobOnce(ctx, "dispatch-1", "job", []byte("different")) + require.ErrorIs(t, err, ErrDispatchConflict) + require.NoError(t, node.Close(ctx)) + require.NoError(t, node.poolStream.Destroy(ctx)) +} + +func TestDispatchSettlementRetryReturnsOriginalOutcome(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + hook := &ambiguousDispatchHook{ + err: errors.New("ambiguous settlement response"), + scriptHash: luaSettleDispatch.Hash(), + } + rdb.AddHook(hook) + ctx := ptesting.NewTestContext(t) + node, err := AddNode(ctx, t.Name(), rdb, WithDispatchTimeout(20*time.Millisecond)) + require.NoError(t, err) + eventID, err := node.DispatchJobOnce(ctx, "dispatch", "job", []byte("payload")) + require.ErrorContains(t, err, "timed out") + require.NoError(t, luaSettleDispatch.Load(ctx, rdb).Err()) + + hook.fail.Store(true) + _, err = node.settleDispatch(ctx, "job", "dispatch", errors.New("worker rejected")) + require.ErrorIs(t, err, hook.err) + record, err := node.settleDispatch(ctx, "job", "dispatch", errors.New("different")) + require.NoError(t, err) + require.Equal(t, eventID, record.eventID) + require.Equal(t, "worker rejected", record.err) + retryID, err := node.DispatchJobOnce(ctx, "dispatch", "job", []byte("payload")) + require.EqualError(t, err, "worker rejected") + require.Equal(t, eventID, retryID) + require.NoError(t, node.Close(ctx)) + require.NoError(t, node.poolStream.Destroy(ctx)) +} + +func TestConcurrentLocalDispatchRetriesShareCompletion(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + node, err := AddNode(ctx, t.Name(), rdb, WithDispatchTimeout(time.Second)) + require.NoError(t, err) + type dispatchResult struct { + id string + err error + } + results := make(chan dispatchResult, 2) + for range 2 { + go func() { + id, dispatchErr := node.DispatchJobOnce(ctx, "dispatch", "job", []byte("payload")) + results <- dispatchResult{id: id, err: dispatchErr} + }() + } + require.Eventually(t, func() bool { + value, ok := node.pendingJobChannels.Load("dispatch") + return ok && value.(*dispatchWaiter).refs.Load() == 2 + }, time.Second, time.Millisecond) + record, err := node.settleDispatch(ctx, "job", "dispatch", nil) + require.NoError(t, err) + for range 2 { + result := <-results + require.NoError(t, result.err) + require.Equal(t, record.eventID, result.id) + } + require.Eventually(t, func() bool { + _, ok := node.pendingJobChannels.Load("dispatch") + return !ok + }, time.Second, time.Millisecond) + require.NoError(t, node.Close(ctx)) + require.NoError(t, node.poolStream.Destroy(ctx)) +} + +func TestCrossNodeDispatchWaiterReadsDurableTerminalState(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + opts := []NodeOption{ + WithDispatchTimeout(time.Second), + WithRecoveryGrace(100 * time.Millisecond), + WithDispatchResultRetention(2 * time.Second), + } + caller, err := AddNode(ctx, t.Name(), rdb, opts...) + require.NoError(t, err) + settler, err := AddNode(ctx, t.Name(), rdb, opts...) + require.NoError(t, err) + + type crossNodeResult struct { + eventID string + err error + } + resultCh := make(chan crossNodeResult, 1) + go func() { + eventID, dispatchErr := caller.DispatchJobOnce( + ctx, + "cross-node-dispatch", + "job", + []byte("payload"), + ) + resultCh <- crossNodeResult{eventID: eventID, err: dispatchErr} + }() + require.Eventually(t, func() bool { + return rdb.HGet( + ctx, + rmapContentKey(caller.resources.jobPending), + "job", + ).Val() == "cross-node-dispatch" + }, time.Second, time.Millisecond) + settled, err := settler.settleDispatch( + ctx, + "job", + "cross-node-dispatch", + errors.New("worker rejected"), + ) + require.NoError(t, err) + + outcome := <-resultCh + require.Equal(t, settled.eventID, outcome.eventID) + require.EqualError(t, outcome.err, "worker rejected") + require.NoError(t, caller.Close(ctx)) + require.NoError(t, settler.Close(ctx)) + require.NoError(t, caller.poolStream.Destroy(ctx)) +} + +func TestDispatchResultRetentionBoundsReplayState(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + const dispatchID = "retained-dispatch" + retention := 80 * time.Millisecond + node, err := AddNode( + ctx, + t.Name(), + rdb, + WithDispatchTimeout(10*time.Millisecond), + WithRecoveryGrace(5*time.Millisecond), + WithDispatchResultRetention(retention), + ) + require.NoError(t, err) + recordKey := dispatchRecordKey(node.resources.dispatches, dispatchID) + + eventID, err := node.publishDispatch( + ctx, + "job", + dispatchID, + marshalJob(&Job{Key: "job", Payload: []byte("payload"), dispatchID: dispatchID}), + ) + require.NoError(t, err) + require.EqualValues(t, -1, rdb.PTTL(ctx, recordKey).Val()) + require.True(t, rdb.SIsMember(ctx, dispatchActiveKey(node.resources.dispatches), recordKey).Val()) + + settled, err := node.settleDispatch(ctx, "job", dispatchID, errors.New("terminal")) + require.NoError(t, err) + require.Equal(t, eventID, settled.eventID) + require.False(t, rdb.SIsMember(ctx, dispatchActiveKey(node.resources.dispatches), recordKey).Val()) + require.Positive(t, rdb.PTTL(ctx, recordKey).Val()) + replayedID, err := node.DispatchJobOnce(ctx, dispatchID, "job", []byte("payload")) + require.EqualError(t, err, "terminal") + require.Equal(t, eventID, replayedID) + + require.Eventually(t, func() bool { + return rdb.Exists(ctx, recordKey).Val() == 0 + }, time.Second, 10*time.Millisecond) + newEventID, err := node.publishDispatch( + ctx, + "job", + dispatchID, + marshalJob(&Job{Key: "job", Payload: []byte("payload"), dispatchID: dispatchID}), + ) + require.NoError(t, err) + require.NotEqual(t, eventID, newEventID, "expired replay identity is a new admission") + _, err = node.settleDispatch(ctx, "job", dispatchID, nil) + require.NoError(t, err) + for i := range 10 { + id := fmt.Sprintf("bounded-%d", i) + key := fmt.Sprintf("job-%d", i) + _, err := node.publishDispatch( + ctx, + key, + id, + marshalJob(&Job{Key: key, dispatchID: id}), + ) + require.NoError(t, err) + _, err = node.settleDispatch(ctx, key, id, nil) + require.NoError(t, err) + } + recordPattern := dispatchRecordKey(node.resources.dispatches, "") + "*" + require.Eventually(t, func() bool { + return len(rdb.Keys(ctx, recordPattern).Val()) == 0 + }, time.Second, 10*time.Millisecond) + require.NoError(t, node.Close(ctx)) + require.NoError(t, node.poolStream.Destroy(ctx)) +} + +func TestMaxQueuedJobsIsAtomicActiveCapacity(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + node, err := AddNode( + ctx, + t.Name(), + rdb, + WithMaxQueuedJobs(2), + WithDispatchTimeout(20*time.Millisecond), + ) + require.NoError(t, err) + + for _, dispatch := range []string{"one", "two"} { + eventID, dispatchErr := node.DispatchJobOnce(ctx, dispatch, dispatch, nil) + require.ErrorContains(t, dispatchErr, "timed out") + require.NotEmpty(t, eventID) + } + _, err = node.DispatchJobOnce(ctx, "three", "three", nil) + require.ErrorIs(t, err, ErrPoolCapacity) + streamKey := generationStreamKey(ctx, rdb, node.poolStream.Name) + require.EqualValues(t, 2, rdb.XLen(ctx, streamKey).Val()) + require.NoError(t, node.completeDispatch(ctx, "one", "one")) + require.NoError(t, node.completeDispatch(ctx, "two", "two")) + require.EqualValues(t, 0, rdb.XLen(ctx, streamKey).Val()) + require.NoError(t, node.Close(ctx)) + require.NoError(t, node.poolStream.Destroy(ctx)) +} + +func TestPoolCapacityConfigurationMustMatch(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + first, err := AddNode(ctx, t.Name(), rdb, WithClientOnly(), WithMaxQueuedJobs(2)) + require.NoError(t, err) + _, err = AddNode(ctx, t.Name(), rdb, WithClientOnly(), WithMaxQueuedJobs(3)) + require.ErrorIs(t, err, ErrPoolConfigMismatch) + require.NoError(t, first.Close(ctx)) + require.NoError(t, first.poolStream.Destroy(ctx)) +} + +func TestPoolLeaseAndDispatchRetentionConfigurationMustMatch(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + first, err := AddNode( + ctx, + t.Name(), + rdb, + WithClientOnly(), + WithCleanupLease(10*time.Second), + WithDispatchResultRetention(2*time.Minute), + ) + require.NoError(t, err) + _, err = AddNode( + ctx, + t.Name(), + rdb, + WithClientOnly(), + WithCleanupLease(11*time.Second), + WithDispatchResultRetention(2*time.Minute), + ) + require.ErrorIs(t, err, ErrPoolConfigMismatch) + _, err = AddNode( + ctx, + t.Name(), + rdb, + WithClientOnly(), + WithCleanupLease(10*time.Second), + WithDispatchResultRetention(3*time.Minute), + ) + require.ErrorIs(t, err, ErrPoolConfigMismatch) + require.NoError(t, first.Close(ctx)) + require.NoError(t, first.poolStream.Destroy(ctx)) +} + +func TestPoolWorkerTTLConfigurationMustMatchWithoutFalseReap(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + first, err := AddNode( + ctx, + t.Name(), + rdb, + WithWorkerTTL(200*time.Millisecond), + WithJobSinkBlockDuration(10*time.Millisecond), + ) + require.NoError(t, err) + worker := newTestWorker(t, ctx, first) + + _, err = AddNode( + ctx, + t.Name(), + rdb, + WithWorkerTTL(5*time.Millisecond), + WithJobSinkBlockDuration(time.Millisecond), + ) + require.ErrorIs(t, err, ErrPoolConfigMismatch) + time.Sleep(20 * time.Millisecond) + first.cleanupInactiveWorkers(ctx) + _, exists := first.workerMap.Get(worker.ID) + require.True(t, exists, "rejected short-TTL node must not reap a healthy worker") + require.Equal( + t, + "200", + rdb.HGet(ctx, poolResourcesKey(t.Name()), "worker_ttl_ms").Val(), + ) + + require.NoError(t, first.Close(ctx)) + require.NoError(t, first.poolStream.Destroy(ctx)) +} + +func TestTerminalSettlementOutlivesWorkerRemoval(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + hook := &settlementFailureHook{ + attempted: make(chan struct{}), + err: errors.New("injected settlement outage"), + scriptHash: luaSettleDispatch.Hash(), + } + hook.fail.Store(true) + rdb.AddHook(hook) + ctx := ptesting.NewTestContext(t) + node := newTestNode(t, ctx, rdb, t.Name()) + started := make(chan struct{}) + handler := newMockHandler() + handler.startFunc = func(*Job) error { + close(started) + return nil + } + worker, err := node.AddWorker(ctx, handler) + require.NoError(t, err) + + result := make(chan error, 1) + go func() { + _, dispatchErr := node.DispatchJobOnce(ctx, "dispatch", "job", []byte("payload")) + result <- dispatchErr + }() + require.Eventually(t, func() bool { + select { + case <-started: + return true + default: + return false + } + }, max, delay) + <-hook.attempted + + removeCtx, cancel := context.WithTimeout(ctx, 30*time.Millisecond) + defer cancel() + err = node.RemoveWorker(removeCtx, worker) + require.ErrorIs(t, err, context.DeadlineExceeded) + record, err := node.readDispatchRecord(ctx, "dispatch", mustDispatchIdentity(t, "job", []byte("payload"))) + require.NoError(t, err) + require.Equal(t, dispatchClaimed, record.status) + require.Equal(t, "dispatch", node.jobPendingMap.Map()["job"]) + + hook.fail.Store(false) + require.NoError(t, node.RemoveWorker(ctx, worker)) + require.NoError(t, <-result) + record, err = node.readDispatchRecord(ctx, "dispatch", mustDispatchIdentity(t, "job", []byte("payload"))) + require.NoError(t, err) + require.Equal(t, dispatchTerminal, record.status) + require.NoError(t, node.Close(ctx)) + require.NoError(t, node.poolStream.Destroy(ctx)) +} + +func TestTerminalSettlementOutlivesNodeClose(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + hook := &settlementFailureHook{ + attempted: make(chan struct{}), + err: errors.New("injected settlement outage"), + scriptHash: luaSettleDispatch.Hash(), + } + hook.fail.Store(true) + rdb.AddHook(hook) + ctx := ptesting.NewTestContext(t) + node := newTestNode(t, ctx, rdb, t.Name()) + started := make(chan struct{}) + handler := newMockHandler() + handler.startFunc = func(*Job) error { + close(started) + return nil + } + _, err := node.AddWorker(ctx, handler) + require.NoError(t, err) + go func() { + _, _ = node.DispatchJobOnce(ctx, "dispatch", "job", nil) + }() + <-started + <-hook.attempted + + closeCtx, cancel := context.WithTimeout(ctx, 30*time.Millisecond) + defer cancel() + err = node.Close(closeCtx) + require.ErrorIs(t, err, context.DeadlineExceeded) + require.False(t, node.IsClosed()) + hook.fail.Store(false) + require.NoError(t, node.Close(ctx)) + require.True(t, node.IsClosed()) + require.NoError(t, node.poolStream.Destroy(ctx)) +} + +func TestCrashedExactDispatchReclaimsOriginalEvent(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + node := newTestNode(t, ctx, rdb, t.Name()) + node.stopOnce.Do(func() { + close(node.stop) + }) + node.wg.Wait() + crashed := newTestWorker(t, ctx, node) + replacement := newTestWorker(t, ctx, node) + job := &Job{ + Key: "job", + Payload: []byte("payload"), + CreatedAt: time.Now(), + NodeID: node.ID, + dispatchID: "dispatch", + } + eventID, err := node.publishDispatch(ctx, job.Key, job.dispatchID, marshalJob(job)) + require.NoError(t, err) + claimed, err := crashed.claimDispatchedStart(ctx, job) + require.NoError(t, err) + require.True(t, claimed) + + require.NoError(t, rdb.HSet( + ctx, + rmapContentKey(node.resources.workerKeepAlive), + crashed.ID, + "0", + ).Err()) + node.cleanupWorker(ctx, crashed.ID) + require.Equal( + t, + "dispatch", + rdb.HGet(ctx, rmapContentKey(node.resources.jobPending), job.Key).Val(), + ) + require.EqualValues(t, 1, rdb.XLen(ctx, generationStreamKey(ctx, rdb, node.poolStream.Name)).Val()) + record, err := node.readDispatchRecord( + ctx, + job.dispatchID, + mustDispatchIdentity(t, job.Key, job.Payload), + ) + require.NoError(t, err) + require.Equal(t, eventID, record.eventID) + require.Equal(t, dispatchClaimed, record.status) + durable := rdb.HGetAll( + ctx, + dispatchRecordKey(node.resources.dispatches, job.dispatchID), + ).Val() + require.Equal(t, "dispatch", durable["id"]) + require.Equal(t, "job", durable["key"]) + require.Equal(t, "pending", durable["state"]) + claimed, err = replacement.claimDispatchedStart(ctx, job) + require.NoError(t, err) + require.True(t, claimed, "replacement must reclaim the original dispatch identity") + _, err = node.settleDispatch(ctx, job.Key, job.dispatchID, nil) + require.NoError(t, err) + require.NoError(t, node.Close(ctx)) + require.NoError(t, node.poolStream.Destroy(ctx)) +} + +func TestCleanupUsesConfiguredLeaseNotWorkerTTL(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + hook := &cleanupDelayHook{ + delay: 20 * time.Millisecond, + scriptHash: destroyCleanupMapScript.Hash(), + } + rdb.AddHook(hook) + ctx := ptesting.NewTestContext(t) + node, err := AddNode( + ctx, + t.Name(), + rdb, + WithWorkerTTL(4*time.Millisecond), + WithJobSinkBlockDuration(time.Millisecond), + WithCleanupLease(200*time.Millisecond), + ) + require.NoError(t, err) + require.NoError(t, node.Shutdown(ctx)) + require.True(t, hook.delayed.Load()) +} + +func TestLegacyPoolAdoptionRequiresQuiescence(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + poolName := t.Name() + resources := flatPoolResources(poolName, "1") + require.NoError(t, rdb.HSet(ctx, rmapContentKey(resources.nodeKeepAlive), "legacy-node", "1").Err()) + + _, err := AddNode(ctx, poolName, rdb, WithClientOnly()) + require.ErrorIs(t, err, ErrQuiescenceRequired) + require.EqualValues(t, 0, rdb.Exists(ctx, poolResourcesKey(poolName)).Val()) + require.NoError(t, rdb.HDel(ctx, rmapContentKey(resources.nodeKeepAlive), "legacy-node").Err()) + dynamicScheduler := rmapContentKey(poolName + ":legacy-producer") + require.NoError(t, rdb.HSet(ctx, dynamicScheduler, "job", "owned").Err()) + _, err = AddNode(ctx, poolName, rdb, WithClientOnly()) + require.ErrorIs(t, err, ErrQuiescenceRequired) + require.NoError(t, rdb.Del(ctx, dynamicScheduler).Err()) + require.NoError(t, rdb.HSet(ctx, rmapContentKey(resources.nodeShutdown), "shutdown", "legacy").Err()) + _, err = AddNode(ctx, poolName, rdb, WithClientOnly()) + require.ErrorIs(t, err, ErrQuiescenceRequired) + require.NoError(t, rdb.Del(ctx, rmapContentKey(resources.nodeShutdown)).Err()) + require.NoError(t, rdb.HSet(ctx, rmapContentKey(resources.dispatches), "dispatch:state", "pending").Err()) + _, err = AddNode(ctx, poolName, rdb, WithClientOnly()) + require.ErrorIs(t, err, ErrQuiescenceRequired) + require.NoError(t, rdb.Del(ctx, rmapContentKey(resources.dispatches)).Err()) + legacyNodeStream := "pulse:stream:" + nodeStreamName(poolName, "LEGACYNODE") + require.NoError(t, rdb.XAdd(ctx, &redis.XAddArgs{ + Stream: legacyNodeStream, + Values: map[string]any{"n": evInit, "p": "legacy"}, + }).Err()) + node, err := AddNode(ctx, poolName, rdb, WithClientOnly()) + require.NoError(t, err) + require.EqualValues(t, 0, rdb.Exists(ctx, legacyNodeStream).Val()) + require.Equal(t, "7", rdb.HGet(ctx, poolResourcesKey(poolName), "format_version").Val()) + require.NoError(t, node.Close(ctx)) + require.NoError(t, node.poolStream.Destroy(ctx)) +} + +// mustDispatchIdentity returns the canonical exact-dispatch identity for tests. +func mustDispatchIdentity(t *testing.T, key string, payload []byte) []byte { + t.Helper() + identity, err := dispatchIdentity(key, payload) + require.NoError(t, err) + return identity +} + +// DialHook preserves normal Redis dialing. +func (h *ambiguousDispatchHook) DialHook(next redis.DialHook) redis.DialHook { + return func(ctx context.Context, network, addr string) (net.Conn, error) { + return next(ctx, network, addr) + } +} + +// ProcessHook injects one error only after Redis committed atomic dispatch. +func (h *ambiguousDispatchHook) ProcessHook(next redis.ProcessHook) redis.ProcessHook { + return func(ctx context.Context, cmd redis.Cmder) error { + err := next(ctx, cmd) + if err != nil { + return err + } + args := cmd.Args() + isDispatch := len(args) > 1 && cmd.Name() == "evalsha" && + args[1] == h.scriptHash + if isDispatch && + h.fail.CompareAndSwap(true, false) { + return h.err + } + return nil + } +} + +// ProcessPipelineHook preserves Redis pipelines. +func (h *ambiguousDispatchHook) ProcessPipelineHook(next redis.ProcessPipelineHook) redis.ProcessPipelineHook { + return func(ctx context.Context, cmds []redis.Cmder) error { + return next(ctx, cmds) + } +} + +// DialHook preserves normal Redis dialing. +func (h *cleanupDelayHook) DialHook(next redis.DialHook) redis.DialHook { + return func(ctx context.Context, network, addr string) (net.Conn, error) { + return next(ctx, network, addr) + } +} + +// ProcessHook delays after the first successful map cleanup script. +func (h *cleanupDelayHook) ProcessHook(next redis.ProcessHook) redis.ProcessHook { + return func(ctx context.Context, cmd redis.Cmder) error { + err := next(ctx, cmd) + if err != nil { + return err + } + args := cmd.Args() + if len(args) > 1 && cmd.Name() == "evalsha" && + args[1] == h.scriptHash && h.delayed.CompareAndSwap(false, true) { + time.Sleep(h.delay) + } + return nil + } +} + +// ProcessPipelineHook preserves Redis pipelines. +func (h *cleanupDelayHook) ProcessPipelineHook(next redis.ProcessPipelineHook) redis.ProcessPipelineHook { + return func(ctx context.Context, cmds []redis.Cmder) error { + return next(ctx, cmds) + } +} + +// DialHook preserves normal Redis dialing. +func (h *settlementFailureHook) DialHook(next redis.DialHook) redis.DialHook { + return func(ctx context.Context, network, addr string) (net.Conn, error) { + return next(ctx, network, addr) + } +} + +// ProcessHook fails settlement before Redis can commit it. +func (h *settlementFailureHook) ProcessHook(next redis.ProcessHook) redis.ProcessHook { + return func(ctx context.Context, cmd redis.Cmder) error { + args := cmd.Args() + if len(args) > 1 && cmd.Name() == "evalsha" && + args[1] == h.scriptHash && h.fail.Load() { + h.once.Do(func() { + close(h.attempted) + }) + return h.err + } + return next(ctx, cmd) + } +} + +// ProcessPipelineHook preserves Redis pipelines. +func (h *settlementFailureHook) ProcessPipelineHook(next redis.ProcessPipelineHook) redis.ProcessPipelineHook { + return func(ctx context.Context, cmds []redis.Cmder) error { + return next(ctx, cmds) + } +} diff --git a/pool/marshal.go b/pool/marshal.go index 714ed39..4516d4c 100644 --- a/pool/marshal.go +++ b/pool/marshal.go @@ -3,6 +3,7 @@ package pool import ( "bytes" "encoding/binary" + "fmt" "time" ) @@ -33,58 +34,57 @@ func marshalJob(job *Job) []byte { if err := binary.Write(&buf, binary.LittleEndian, job.Requeued); err != nil { panic(err) } - return buf.Bytes() -} - -// unmarshalJob unmarshals a job from a byte slice created by marshalJob. -func unmarshalJob(data []byte) *Job { - reader := bytes.NewReader(data) - var keyLength int32 - if err := binary.Read(reader, binary.LittleEndian, &keyLength); err != nil { + if err := binary.Write(&buf, binary.LittleEndian, int32(len(job.dispatchID))); err != nil { panic(err) } - keyBytes := make([]byte, keyLength) - if err := binary.Read(reader, binary.LittleEndian, &keyBytes); err != nil { + if err := binary.Write(&buf, binary.LittleEndian, []byte(job.dispatchID)); err != nil { panic(err) } - var nodeIDLength int32 - if err := binary.Read(reader, binary.LittleEndian, &nodeIDLength); err != nil { - panic(err) + return buf.Bytes() +} + +// unmarshalJob decodes the current complete job payload. Mixed-version stream +// entries are rejected because pool upgrades require a quiescent boundary. +func unmarshalJob(data []byte) (*Job, error) { + reader := bytes.NewReader(data) + key, err := unmarshalString(reader, "job key") + if err != nil { + return nil, err } - nodeIDBytes := make([]byte, nodeIDLength) - if err := binary.Read(reader, binary.LittleEndian, &nodeIDBytes); err != nil { - panic(err) + nodeID, err := unmarshalString(reader, "job node ID") + if err != nil { + return nil, err } - nodeID := string(nodeIDBytes) - var payloadLength int32 - if err := binary.Read(reader, binary.LittleEndian, &payloadLength); err != nil { - panic(err) + payload, err := unmarshalBytes(reader, "job payload") + if err != nil { + return nil, err } - var payload []byte - if payloadLength > 0 { - payload = make([]byte, payloadLength) - if err := binary.Read(reader, binary.LittleEndian, &payload); err != nil { - panic(err) - } + if len(payload) == 0 { + payload = nil } var createdAtTimestamp int64 if err := binary.Read(reader, binary.LittleEndian, &createdAtTimestamp); err != nil { - panic(err) + return nil, fmt.Errorf("decode job created-at: %w", err) } - requeued := false - // v1.6.4 and earlier persisted start-job events without the Requeued flag. - // Those events can remain in Redis streams across a rolling upgrade, so the - // decoder treats the missing trailing field as the old dispatch contract. - if reader.Len() > 0 { - requeued = unmarshalBool(reader) + requeued, err := unmarshalBool(reader, "job requeued") + if err != nil { + return nil, err } - return &Job{ - Key: string(keyBytes), - Payload: payload, - CreatedAt: time.Unix(0, createdAtTimestamp).UTC(), - NodeID: nodeID, - Requeued: requeued, + dispatchID, err := unmarshalString(reader, "job dispatch ID") + if err != nil { + return nil, err + } + if reader.Len() != 0 { + return nil, fmt.Errorf("decode job: %d trailing bytes", reader.Len()) } + return &Job{ + Key: key, + Payload: payload, + CreatedAt: time.Unix(0, createdAtTimestamp).UTC(), + NodeID: nodeID, + Requeued: requeued, + dispatchID: dispatchID, + }, nil } // marshalJobKey marshals a job key into a byte slice. @@ -99,46 +99,59 @@ func marshalJobKey(key string) []byte { return buf.Bytes() } -func unmarshalJobKey(data []byte) string { +// unmarshalJobKey decodes one complete job-key payload. +func unmarshalJobKey(data []byte) (string, error) { reader := bytes.NewReader(data) - var keyLength int32 - if err := binary.Read(reader, binary.LittleEndian, &keyLength); err != nil { - panic(err) + key, err := unmarshalString(reader, "job key") + if err != nil { + return "", err } - keyBytes := make([]byte, keyLength) - if err := binary.Read(reader, binary.LittleEndian, &keyBytes); err != nil { - panic(err) + if reader.Len() != 0 { + return "", fmt.Errorf("decode job key: %d trailing bytes", reader.Len()) } - return string(keyBytes) + return key, nil } -func unmarshalJobKeyAndNodeID(data []byte) (string, string) { - reader := bytes.NewReader(data) - var keyLength int32 - if err := binary.Read(reader, binary.LittleEndian, &keyLength); err != nil { - panic(err) +// unmarshalBool decodes the exact binary bool representation. +func unmarshalBool(reader *bytes.Reader, field string) (bool, error) { + value, err := reader.ReadByte() + if err != nil { + return false, fmt.Errorf("decode %s: %w", field, err) + } + switch value { + case 0: + return false, nil + case 1: + return true, nil + default: + return false, fmt.Errorf("decode %s: invalid boolean value %d", field, value) } - keyBytes := make([]byte, keyLength) - if err := binary.Read(reader, binary.LittleEndian, &keyBytes); err != nil { - panic(err) +} + +// unmarshalString reads one validated length-prefixed string. +func unmarshalString(reader *bytes.Reader, field string) (string, error) { + value, err := unmarshalBytes(reader, field) + return string(value), err +} + +// unmarshalBytes rejects negative, oversized, and truncated length-prefixed +// fields before allocating. +func unmarshalBytes(reader *bytes.Reader, field string) ([]byte, error) { + var length int32 + if err := binary.Read(reader, binary.LittleEndian, &length); err != nil { + return nil, fmt.Errorf("decode %s length: %w", field, err) } - var nodeIDLength int32 - if err := binary.Read(reader, binary.LittleEndian, &nodeIDLength); err != nil { - panic(err) + if length < 0 { + return nil, fmt.Errorf("decode %s: negative length %d", field, length) } - nodeIDBytes := make([]byte, nodeIDLength) - if err := binary.Read(reader, binary.LittleEndian, &nodeIDBytes); err != nil { - panic(err) + if int64(length) > int64(reader.Len()) { + return nil, fmt.Errorf("decode %s: length %d exceeds remaining %d bytes", field, length, reader.Len()) } - return string(keyBytes), string(nodeIDBytes) -} - -func unmarshalBool(reader *bytes.Reader) bool { - var value bool + value := make([]byte, length) if err := binary.Read(reader, binary.LittleEndian, &value); err != nil { - panic(err) + return nil, fmt.Errorf("decode %s: %w", field, err) } - return value + return value, nil } // marshalKeyedPayload marshals the shared wire shape used by events whose @@ -160,27 +173,21 @@ func marshalKeyedPayload(key string, payload []byte) []byte { return buf.Bytes() } -// unmarshalKeyedPayload unmarshals data produced by marshalKeyedPayload. -func unmarshalKeyedPayload(data []byte) (string, []byte) { +// unmarshalKeyedPayload decodes one complete keyed payload. +func unmarshalKeyedPayload(data []byte) (string, []byte, error) { reader := bytes.NewReader(data) - var keyLength int32 - if err := binary.Read(reader, binary.LittleEndian, &keyLength); err != nil { - panic(err) - } - keyBytes := make([]byte, keyLength) - if err := binary.Read(reader, binary.LittleEndian, &keyBytes); err != nil { - panic(err) + key, err := unmarshalString(reader, "keyed payload key") + if err != nil { + return "", nil, err } - // read payload - var payloadLength int32 - if err := binary.Read(reader, binary.LittleEndian, &payloadLength); err != nil { - panic(err) + payload, err := unmarshalBytes(reader, "keyed payload") + if err != nil { + return "", nil, err } - payload := make([]byte, payloadLength) - if err := binary.Read(reader, binary.LittleEndian, &payload); err != nil { - panic(err) + if reader.Len() != 0 { + return "", nil, fmt.Errorf("decode keyed payload: %d trailing bytes", reader.Len()) } - return string(keyBytes), payload + return key, payload, nil } // Envelope used to identify event sender. @@ -201,29 +208,21 @@ func marshalEnvelope(sender string, payload []byte) []byte { return buf.Bytes() } -// unmarshalEnvelope unmarshals an envelope from a byte slice created by marshalEnvelope. -func unmarshalEnvelope(data []byte) (string, []byte) { +// unmarshalEnvelope decodes one complete sender envelope. +func unmarshalEnvelope(data []byte) (string, []byte, error) { reader := bytes.NewReader(data) - var senderLength int32 - if err := binary.Read(reader, binary.LittleEndian, &senderLength); err != nil { - panic(err) + sender, err := unmarshalString(reader, "envelope sender") + if err != nil { + return "", nil, err } - senderBytes := make([]byte, senderLength) - if err := binary.Read(reader, binary.LittleEndian, &senderBytes); err != nil { - panic(err) - } - var payloadLength int32 - if err := binary.Read(reader, binary.LittleEndian, &payloadLength); err != nil { - panic(err) + payload, err := unmarshalBytes(reader, "envelope payload") + if err != nil { + return "", nil, err } - var payload []byte - if payloadLength > 0 { - payload = make([]byte, payloadLength) - if err := binary.Read(reader, binary.LittleEndian, &payload); err != nil { - panic(err) - } + if reader.Len() != 0 { + return "", nil, fmt.Errorf("decode envelope: %d trailing bytes", reader.Len()) } - return string(senderBytes), payload + return sender, payload, nil } // marshalAck marshals an ack into a byte slice. @@ -241,30 +240,38 @@ func marshalAck(ak *ack) []byte { if err := binary.Write(&buf, binary.LittleEndian, []byte(ak.Error)); err != nil { panic(err) } + if err := binary.Write(&buf, binary.LittleEndian, int32(len(ak.JobKey))); err != nil { + panic(err) + } + if err := binary.Write(&buf, binary.LittleEndian, []byte(ak.JobKey)); err != nil { + panic(err) + } return buf.Bytes() } -// unmarshalAck unmarshals an ack from a byte slice created by marshalAck. -func unmarshalAck(data []byte) *ack { +// unmarshalAck decodes the current complete acknowledgement. Mixed wire +// versions are rejected by the mandatory quiescent-upgrade contract. +func unmarshalAck(data []byte) (*ack, error) { reader := bytes.NewReader(data) - var eventIDLength int32 - if err := binary.Read(reader, binary.LittleEndian, &eventIDLength); err != nil { - panic(err) + eventID, err := unmarshalString(reader, "ack event ID") + if err != nil { + return nil, err } - eventIDBytes := make([]byte, eventIDLength) - if err := binary.Read(reader, binary.LittleEndian, &eventIDBytes); err != nil { - panic(err) + errorMessage, err := unmarshalString(reader, "ack error") + if err != nil { + return nil, err } - var errorLength int32 - if err := binary.Read(reader, binary.LittleEndian, &errorLength); err != nil { - panic(err) + jobKey, err := unmarshalString(reader, "ack job key") + if err != nil { + return nil, err } - errorBytes := make([]byte, errorLength) - if err := binary.Read(reader, binary.LittleEndian, &errorBytes); err != nil { - panic(err) + result := &ack{ + EventID: eventID, + Error: errorMessage, + JobKey: jobKey, } - return &ack{ - EventID: string(eventIDBytes), - Error: string(errorBytes), + if reader.Len() != 0 { + return nil, fmt.Errorf("decode acknowledgement: %d trailing bytes", reader.Len()) } + return result, nil } diff --git a/pool/marshal_test.go b/pool/marshal_test.go index ee4e3c4..abe2854 100644 --- a/pool/marshal_test.go +++ b/pool/marshal_test.go @@ -6,6 +6,7 @@ import ( "time" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestMarshalJob(t *testing.T) { @@ -41,7 +42,8 @@ func TestMarshalJob(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { marshaled := marshalJob(&tc.job) - job := unmarshalJob(marshaled) + job, err := unmarshalJob(marshaled) + require.NoError(t, err) // Compare original and unmarshaled Job structs assert.Equal(t, tc.job.Key, job.Key) @@ -54,13 +56,14 @@ func TestMarshalJob(t *testing.T) { assert.True(t, bytes.Equal(marshaled, marshaled2)) // Compare unmarshaled job key - key := unmarshalJobKey(marshaled) + key, err := unmarshalJobKey(marshalJobKey(tc.job.Key)) + require.NoError(t, err) assert.Equal(t, tc.job.Key, key) }) } } -func TestUnmarshalLegacyJob(t *testing.T) { +func TestUnmarshalJobRejectsLegacyFormats(t *testing.T) { job := &Job{ Key: "test-key", Payload: []byte("test-payload"), @@ -69,16 +72,12 @@ func TestUnmarshalLegacyJob(t *testing.T) { Requeued: true, } marshaled := marshalJob(job) - legacy := marshaled[:len(marshaled)-1] + legacy := marshaled[:len(marshaled)-5] - assert.NotPanics(t, func() { - decoded := unmarshalJob(legacy) - assert.Equal(t, job.Key, decoded.Key) - assert.Equal(t, job.Payload, decoded.Payload) - assert.Equal(t, job.CreatedAt, decoded.CreatedAt) - assert.Equal(t, job.NodeID, decoded.NodeID) - assert.False(t, decoded.Requeued) - }) + _, err := unmarshalJob(legacy) + require.Error(t, err) + _, err = unmarshalJob(marshaled[:len(marshaled)-4]) + require.Error(t, err) } func TestMarshalKeyedPayload(t *testing.T) { @@ -86,9 +85,88 @@ func TestMarshalKeyedPayload(t *testing.T) { payload := []byte("test-payload") marshaled := marshalKeyedPayload(key, payload) - gotKey, gotPayload := unmarshalKeyedPayload(marshaled) + gotKey, gotPayload, err := unmarshalKeyedPayload(marshaled) + require.NoError(t, err) assert.Equal(t, key, gotKey) assert.Equal(t, payload, gotPayload) - assert.Equal(t, key, unmarshalJobKey(marshaled)) + decodedKey, err := unmarshalJobKey(marshalJobKey(key)) + require.NoError(t, err) + assert.Equal(t, key, decodedKey) +} + +func TestPoolDecodersRejectMalformedPayloads(t *testing.T) { + job := marshalJob(&Job{Key: "job", CreatedAt: time.Unix(1, 0)}) + jobKey := marshalJobKey("job") + keyed := marshalKeyedPayload("job", []byte("payload")) + envelope := marshalEnvelope("node", []byte("payload")) + ackPayload := marshalAck(&ack{EventID: "event", JobKey: "job"}) + + cases := []struct { + name string + decode func([]byte) error + data []byte + }{ + { + name: "job truncated", + decode: func(data []byte) error { + _, err := unmarshalJob(data) + return err + }, + data: job[:len(job)-1], + }, + { + name: "job trailing", + decode: func(data []byte) error { + _, err := unmarshalJob(data) + return err + }, + data: append(job, 1), + }, + { + name: "job key negative length", + decode: func(data []byte) error { + _, err := unmarshalJobKey(data) + return err + }, + data: []byte{0xff, 0xff, 0xff, 0xff}, + }, + { + name: "job key trailing", + decode: func(data []byte) error { + _, err := unmarshalJobKey(data) + return err + }, + data: append(jobKey, 1), + }, + { + name: "keyed payload truncated", + decode: func(data []byte) error { + _, _, err := unmarshalKeyedPayload(data) + return err + }, + data: keyed[:len(keyed)-1], + }, + { + name: "envelope trailing", + decode: func(data []byte) error { + _, _, err := unmarshalEnvelope(data) + return err + }, + data: append(envelope, 1), + }, + { + name: "ack truncated", + decode: func(data []byte) error { + _, err := unmarshalAck(data) + return err + }, + data: ackPayload[:len(ackPayload)-1], + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + require.Error(t, tc.decode(tc.data)) + }) + } } diff --git a/pool/node.go b/pool/node.go index a3c06be..a31daa1 100644 --- a/pool/node.go +++ b/pool/node.go @@ -1,7 +1,10 @@ package pool import ( + "bytes" "context" + "encoding/binary" + "encoding/hex" "errors" "fmt" "hash" @@ -27,42 +30,77 @@ import ( type ( // Node is a pool of workers. Node struct { - ID string - PoolName string - poolStream *streaming.Stream // pool event stream for dispatching jobs - poolSink *streaming.Sink // pool event sink - nodeStream *streaming.Stream // node event stream for receiving worker events - nodeReader *streaming.Reader // node event reader - nodeKeepAliveMap *rmap.Map // node keep-alive timestamps indexed by ID - nodeShutdownMap *rmap.Map // key is node ID that requested shutdown - workerMap *rmap.Map // worker creation times by ID - workerKeepAliveMap *rmap.Map // worker keep-alive timestamps indexed by ID - workerCleanupMap *rmap.Map // key is stale worker ID that needs cleanup - jobMap *rmap.Map // jobs by worker ID - jobPendingMap *rmap.Map // pending jobs by job key - jobPayloadMap *rmap.Map // job payloads by job key - tickerMap *rmap.Map // ticker next tick time indexed by name - workerTTL time.Duration // Worker considered dead if keep-alive not updated after this duration - workerShutdownTTL time.Duration // Worker considered dead if not shutdown after this duration - ackGracePeriod time.Duration // Wait for return status up to this duration - clientOnly bool - logger pulse.Logger - h hasher - stop chan struct{} // closed when node is stopped - closed chan struct{} // closed when node is closed - wg sync.WaitGroup // allows to wait until all goroutines exit - rdb *redis.Client + ID string + PoolName string + poolStream *streaming.Stream // pool event stream for dispatching jobs + poolSink *streaming.Sink // pool event sink + nodeStream *streaming.Stream // node event stream for receiving worker events + nodeReader *streaming.Reader // node event reader + nodeKeepAliveMap *rmap.Map // node keep-alive timestamps indexed by ID + nodeShutdownMap *rmap.Map // key is node ID that requested shutdown + workerMap *rmap.Map // worker creation times by ID + workerKeepAliveMap *rmap.Map // worker keep-alive timestamps indexed by ID + workerCleanupMap *rmap.Map // key is stale worker ID that needs cleanup + jobMap *rmap.Map // jobs by worker ID + jobPendingMap *rmap.Map // pending jobs by job key + jobPayloadMap *rmap.Map // job payloads by job key + tickerMap *rmap.Map // ticker next tick time indexed by name + schedulerJobMap *rmap.Map // generation-fenced scheduler-owned jobs + workerTTL time.Duration // Worker considered dead if keep-alive not updated after this duration + requeueTimeout time.Duration // Bounds one local worker requeue handoff attempt + dispatchTimeout time.Duration + dispatchResultRetention time.Duration + recoveryGrace time.Duration + cleanupLease time.Duration + maxQueuedJobs int + clientOnly bool + logger pulse.Logger + h hasher + resources poolResources + stop chan struct{} // closed when node is stopped + closed chan struct{} // closed when node is closed + wg sync.WaitGroup // allows to wait until all goroutines exit + scheduleCtx context.Context + scheduleCancel context.CancelFunc + scheduleWG sync.WaitGroup + settlements *dispatchSettlements + rdb *redis.Client localWorkers sync.Map // workers created by this node workerStreams sync.Map // worker streams indexed by ID nodeStreams sync.Map // streams for worker acks indexed by ID - pendingJobChannels sync.Map // channels used to send DispatchJob results, nil if event is requeued + pendingJobChannels sync.Map // dispatch nonce -> *dispatchWaiter pendingEvents sync.Map // pending events indexed by sender and event IDs orphanedPayloads sync.Map // job key -> first time observed orphaned payload (unix nanos) - lock sync.RWMutex - closing bool - shutdown bool + dispatchWaitersLock sync.Mutex + closeLock sync.Mutex + stopOnce sync.Once + shutdownOnce sync.Once + terminalOnce sync.Once + lock sync.RWMutex + closing bool + closedState bool + shutdown bool + cleanupComplete bool + closeAfterCleanupErr error + } + + // dispatchWaiter owns one admitted dispatch until its worker result arrives + // or its persisted guard expires. + dispatchWaiter struct { + done chan struct{} + once sync.Once + refs atomic.Int64 + } + + // dispatchRecord is the Redis-owned publication and terminal outcome for + // one globally unique dispatch ID. + dispatchRecord struct { + status int64 + eventID string + result string + err string } // hasher is the interface implemented by types that can hash keys. @@ -78,6 +116,9 @@ type ( ) const ( + // shutdownErrorPoll bounds how quickly the initiator observes a peer's + // authoritative close failure. + shutdownErrorPoll = 100 * time.Millisecond // evInit is the event used to initialize a node or worker stream. evInit string = "i" // evStartJob is the event used to send new job to workers. @@ -90,9 +131,6 @@ const ( evStopJob string = "s" // evAck is the worker event used to ack a pool event. evAck string = "a" - // evDispatchReturn is the event used to forward the worker start return - // status to the node that dispatched the job. - evDispatchReturn string = "d" ) // pendingEventTTL is the TTL for pending events. @@ -101,9 +139,97 @@ var pendingEventTTL = 2 * time.Minute var ( // ErrJobExists is returned when attempting to dispatch a job with a key that already exists. ErrJobExists = errors.New("job already exists") + // ErrPoolCapacity is returned when active pending dispatches have reached + // the generation's immutable MaxQueuedJobs contract. + ErrPoolCapacity = errors.New("pool dispatch capacity reached") + // ErrDispatchConflict is returned when a dispatch ID is reused with + // different exact job key or payload bytes. + ErrDispatchConflict = errors.New("pool dispatch idempotency conflict") + // ErrPoolGenerationLost is returned when a node mutates an inactive pool. + ErrPoolGenerationLost = errors.New("pool generation lost") + // ErrPoolConfigMismatch is returned when node configuration differs from + // the active generation's immutable configuration. + ErrPoolConfigMismatch = errors.New("pool configuration mismatch") + // ErrQuiescenceRequired is returned when legacy resources prove that old + // pool writers are still active during a hard upgrade. + ErrQuiescenceRequired = errors.New("pool quiescence required") errJobAwaitingOwner = errors.New("job awaiting active owner") errJobNotFound = errors.New("job not found") + + // registerPoolNodeScript linearizes node registration against shutdown and + // final cleanup, verifies the exact pool incarnation, and publishes the + // keep-alive map update in the same operation. + registerPoolNodeScript = redis.NewScript(` +if redis.call("HGET", KEYS[1], "state") ~= ARGV[3] or + redis.call("HGET", KEYS[1], "generation") ~= ARGV[2] then + return redis.error_reply("POOLGENERATIONLOST") +end +if redis.call("HGET", KEYS[3], ARGV[4]) then + return {2} +end +for _, state in ipairs(redis.call("HVALS", KEYS[5])) do + if state == "finishing" then + return {0} + end +end +if redis.call("HEXISTS", KEYS[2], "shutdown") == 1 then + return {0} +end +local now = redis.call("TIME") +local timestamp = now[1] .. string.format("%06d", now[2]) .. "000" +redis.call("HSET", KEYS[3], ARGV[1], timestamp) +local rev = tostring(redis.call("HINCRBY", KEYS[3], "=rev", 1)) +redis.call("HSET", KEYS[3], "=kind", "set") +local msg = struct.pack( + "ic0ic0ic0", + string.len(ARGV[1]), ARGV[1], + string.len(timestamp), timestamp, + string.len(rev), rev +) +redis.call("PUBLISH", KEYS[4], "set:" .. msg) +return {1, timestamp} +`) + + // refreshPoolNodeScript renews an existing node through shutdown but + // rejects a stale-node cleanup fence or removed registration. + refreshPoolNodeScript = redis.NewScript(` +if redis.call("HGET", KEYS[1], "state") ~= ARGV[3] or + redis.call("HGET", KEYS[1], "generation") ~= ARGV[2] then + return redis.error_reply("POOLGENERATIONLOST") +end +if redis.call("HGET", KEYS[2], ARGV[4]) +or not redis.call("HGET", KEYS[2], ARGV[1]) then + return redis.error_reply("NODECLEANUPLOST") +end +local now = redis.call("TIME") +local timestamp = now[1] .. string.format("%06d", now[2]) .. "000" +redis.call("HSET", KEYS[2], ARGV[1], timestamp) +local rev = tostring(redis.call("HINCRBY", KEYS[2], "=rev", 1)) +redis.call("HSET", KEYS[2], "=kind", "set") +local msg = struct.pack( + "ic0ic0ic0", + string.len(ARGV[1]), ARGV[1], + string.len(timestamp), timestamp, + string.len(rev), rev +) +redis.call("PUBLISH", KEYS[3], "set:" .. msg) +return timestamp +`) + + // publishPoolShutdownScript records the distributed shutdown obligation and + // emits the rmap wire notification without depending on a local map handle + // that a concurrent Close may already have closed. + publishPoolShutdownScript = redis.NewScript(` +local key = "shutdown" +local value = ARGV[1] +redis.call("HSET", KEYS[1], key, value) +local rev = tostring(redis.call("HINCRBY", KEYS[1], "=rev", 1)) +redis.call("HSET", KEYS[1], "=kind", "set") +local msg = struct.pack("ic0ic0ic0", string.len(key), key, string.len(value), value, string.len(rev), rev) +redis.call("PUBLISH", KEYS[2], "set:" .. msg) +return rev +`) ) // AddNode adds a new node to the pool with the given name and returns it. The @@ -114,8 +240,11 @@ var ( // The options WithClientOnly can be used to create a node that can only be used // to dispatch jobs. Such a node does not route or process jobs in the // background. -func AddNode(ctx context.Context, poolName string, rdb *redis.Client, opts ...NodeOption) (*Node, error) { +func AddNode(ctx context.Context, poolName string, rdb *redis.Client, opts ...NodeOption) (_ *Node, resultErr error) { o := parseOptions(opts...) + if err := validateNodeOptions(o); err != nil { + return nil, fmt.Errorf("AddNode: %w", err) + } logger := o.logger nodeID := ulid.Make().String() if logger == nil { @@ -127,32 +256,67 @@ func AddNode(ctx context.Context, poolName string, rdb *redis.Client, opts ...No "client_only", o.clientOnly, "max_queued_jobs", o.maxQueuedJobs, "worker_ttl", o.workerTTL, - "worker_shutdown_ttl", o.workerShutdownTTL, - "ack_grace_period", o.ackGracePeriod) + "worker_requeue_timeout", o.requeueTimeout, + "dispatch_timeout", o.dispatchTimeout, + "dispatch_result_retention", o.dispatchResultRetention, + "recovery_grace", o.recoveryGrace, + "cleanup_lease", o.cleanupLease) - nsm, err := rmap.Join(ctx, nodeShutdownMapName(poolName), rdb, rmap.WithLogger(logger)) + poolStream, err := streaming.NewStream(poolStreamName(poolName), rdb, + options.WithUnboundedStream(), + options.WithStreamLogger(logger)) if err != nil { - return nil, fmt.Errorf("AddNode: failed to join shutdown replicated map %q: %w", nodeShutdownMapName(poolName), err) + return nil, fmt.Errorf("AddNode: failed to create pool job stream %q: %w", poolStreamName(poolName), err) + } + if err := resumeExpiredPoolCleanup(ctx, poolName, nodeID, o.cleanupLease, rdb); err != nil { + return nil, fmt.Errorf("AddNode: %w", err) + } + if err := poolStream.Open(ctx); err != nil { + return nil, fmt.Errorf("AddNode: failed to open pool job stream %q: %w", poolStreamName(poolName), err) + } + resources, err := establishPoolResources( + ctx, + rdb, + poolName, + poolStream.Generation(), + o.maxQueuedJobs, + o.workerTTL, + o.cleanupLease, + o.dispatchResultRetention, + ) + if err != nil { + return nil, fmt.Errorf("AddNode: %w", err) + } + nsm, err := rmap.Join(ctx, resources.nodeShutdown, rdb, rmap.WithLogger(logger)) + if err != nil { + return nil, fmt.Errorf("AddNode: failed to join shutdown replicated map %q: %w", resources.nodeShutdown, err) } + shutdownUpdates := nsm.Subscribe() if nsm.Len() > 0 { + nsm.Unsubscribe(shutdownUpdates) + nsm.Close() return nil, fmt.Errorf("AddNode: pool %q is shutting down", poolName) } - nkm, err := rmap.Join(ctx, nodeKeepAliveMapName(poolName), rdb, rmap.WithLogger(logger)) + nkm, err := rmap.Join(ctx, resources.nodeKeepAlive, rdb, rmap.WithLogger(logger)) if err != nil { - return nil, fmt.Errorf("AddNode: failed to join node keep-alive map %q: %w", nodeKeepAliveMapName(poolName), err) - } - if _, err := nkm.Set(ctx, nodeID, strconv.FormatInt(time.Now().UnixNano(), 10)); err != nil { - return nil, fmt.Errorf("AddNode: failed to set initial node keep-alive: %w", err) + nsm.Unsubscribe(shutdownUpdates) + nsm.Close() + return nil, fmt.Errorf("AddNode: failed to join node keep-alive map %q: %w", resources.nodeKeepAlive, err) } - - poolStream, err := streaming.NewStream(poolStreamName(poolName), rdb, - options.WithStreamMaxLen(o.maxQueuedJobs), - options.WithStreamLogger(logger)) + registered, _, err := registerPoolNode(ctx, rdb, resources, nodeID) if err != nil { - return nil, fmt.Errorf("AddNode: failed to create pool job stream %q: %w", poolStreamName(poolName), err) + nkm.Close() + nsm.Unsubscribe(shutdownUpdates) + nsm.Close() + return nil, fmt.Errorf("AddNode: failed to register node: %w", err) + } + if !registered { + nkm.Close() + nsm.Unsubscribe(shutdownUpdates) + nsm.Close() + return nil, fmt.Errorf("AddNode: pool %q is shutting down", poolName) } - var ( wm *rmap.Map jm *rmap.Map @@ -160,66 +324,118 @@ func AddNode(ctx context.Context, poolName string, rdb *redis.Client, opts ...No jpem *rmap.Map wkm *rmap.Map tm *rmap.Map + sjm *rmap.Map wcm *rmap.Map - poolSink *streaming.Sink - nodeStream *streaming.Stream - nodeReader *streaming.Reader - closed chan struct{} + poolSink *streaming.Sink + nodeStream *streaming.Stream + nodeReader *streaming.Reader + nodeStreamActive bool + setupComplete bool ) + // Registration is the first externally visible setup step. Every later + // failure unwinds local resources and that registration in reverse order. + defer func() { + if setupComplete { + return + } + var rollbackErr error + if nodeReader != nil { + nodeReader.Close() + } + if nodeStreamActive { + if err := nodeStream.Destroy(context.WithoutCancel(ctx)); err != nil { + rollbackErr = errors.Join(rollbackErr, fmt.Errorf("destroy node stream: %w", err)) + } + } + if poolSink != nil { + if err := poolSink.Close(context.WithoutCancel(ctx)); err != nil { + rollbackErr = errors.Join(rollbackErr, fmt.Errorf("close pool sink: %w", err)) + } + } + for _, m := range []*rmap.Map{jpem, wcm, sjm, tm, wkm, jpm, jm, wm} { + if m != nil { + m.Close() + } + } + if _, err := nkm.Delete(context.WithoutCancel(ctx), nodeID); err != nil { + rollbackErr = errors.Join(rollbackErr, fmt.Errorf("remove node registration: %w", err)) + } + nkm.Close() + nsm.Unsubscribe(shutdownUpdates) + nsm.Close() + if rollbackErr != nil { + resultErr = errors.Join(resultErr, fmt.Errorf("AddNode: rollback failed: %w", rollbackErr)) + } + }() + registrationCtx, stopRegistrationLease := context.WithCancel(ctx) + registrationLeaseDone := make(chan struct{}) + pulse.Go(logger, func() { + maintainNodeRegistrationLease(registrationCtx, rdb, resources, nodeID, o.workerTTL, logger) + close(registrationLeaseDone) + }) + defer func() { + stopRegistrationLease() + <-registrationLeaseDone + }() if !o.clientOnly { - wm, err = rmap.Join(ctx, workerMapName(poolName), rdb, rmap.WithLogger(logger)) + wm, err = rmap.Join(ctx, resources.workers, rdb, rmap.WithLogger(logger)) if err != nil { - return nil, fmt.Errorf("AddNode: failed to join pool workers replicated map %q: %w", workerMapName(poolName), err) + return nil, fmt.Errorf("AddNode: failed to join pool workers replicated map %q: %w", resources.workers, err) } workerIDs := wm.Keys() logger.Info("joined", "workers", workerIDs) - jm, err = rmap.Join(ctx, jobMapName(poolName), rdb, rmap.WithLogger(logger)) + jm, err = rmap.Join(ctx, resources.jobs, rdb, rmap.WithLogger(logger)) if err != nil { - return nil, fmt.Errorf("AddNode: failed to join pool jobs replicated map %q: %w", jobMapName(poolName), err) + return nil, fmt.Errorf("AddNode: failed to join pool jobs replicated map %q: %w", resources.jobs, err) } - jpm, err = rmap.Join(ctx, jobPayloadMapName(poolName), rdb, rmap.WithLogger(logger)) + jpm, err = rmap.Join(ctx, resources.jobPayloads, rdb, rmap.WithLogger(logger)) if err != nil { - return nil, fmt.Errorf("AddNode: failed to join pool job payloads replicated map %q: %w", jobPayloadMapName(poolName), err) + return nil, fmt.Errorf("AddNode: failed to join pool job payloads replicated map %q: %w", resources.jobPayloads, err) } - wkm, err = rmap.Join(ctx, workerKeepAliveMapName(poolName), rdb, rmap.WithLogger(logger)) + wkm, err = rmap.Join(ctx, resources.workerKeepAlive, rdb, rmap.WithLogger(logger)) if err != nil { - return nil, fmt.Errorf("AddNode: failed to join worker keep-alive replicated map %q: %w", workerKeepAliveMapName(poolName), err) + return nil, fmt.Errorf("AddNode: failed to join worker keep-alive replicated map %q: %w", resources.workerKeepAlive, err) } - tm, err = rmap.Join(ctx, tickerMapName(poolName), rdb, rmap.WithLogger(logger)) + tm, err = rmap.Join(ctx, resources.tickers, rdb, rmap.WithLogger(logger)) if err != nil { - return nil, fmt.Errorf("AddNode: failed to join pool ticker replicated map %q: %w", tickerMapName(poolName), err) + return nil, fmt.Errorf("AddNode: failed to join pool ticker replicated map %q: %w", resources.tickers, err) } - wcm, err = rmap.Join(ctx, workerCleanupMapName(poolName), rdb, rmap.WithLogger(logger)) + sjm, err = rmap.Join(ctx, resources.schedulerJobs, rdb, rmap.WithLogger(logger)) if err != nil { - return nil, fmt.Errorf("AddNode: failed to join pool cleanup replicated map %q: %w", workerCleanupMapName(poolName), err) + return nil, fmt.Errorf("AddNode: failed to join scheduler jobs replicated map %q: %w", resources.schedulerJobs, err) + } + + wcm, err = rmap.Join(ctx, resources.workerCleanup, rdb, rmap.WithLogger(logger)) + if err != nil { + return nil, fmt.Errorf("AddNode: failed to join pool cleanup replicated map %q: %w", resources.workerCleanup, err) } // Initialize and join pending jobs map - jpem, err = rmap.Join(ctx, jobPendingMapName(poolName), rdb, rmap.WithLogger(logger)) + jpem, err = rmap.Join(ctx, resources.jobPending, rdb, rmap.WithLogger(logger)) if err != nil { - return nil, fmt.Errorf("AddNode: failed to join pending jobs replicated map %q: %w", jobPendingMapName(poolName), err) + return nil, fmt.Errorf("AddNode: failed to join pending jobs replicated map %q: %w", resources.jobPending, err) } poolSink, err = poolStream.NewSink(ctx, "events", options.WithSinkBlockDuration(o.jobSinkBlockDuration), - options.WithSinkAckGracePeriod(o.ackGracePeriod)) + options.WithSinkAckGracePeriod(o.recoveryGrace)) if err != nil { return nil, fmt.Errorf("AddNode: failed to create events sink for stream %q: %w", poolStreamName(poolName), err) } - closed = make(chan struct{}) } nodeStream, err = streaming.NewStream(nodeStreamName(poolName, nodeID), rdb, options.WithStreamLogger(logger)) if err != nil { return nil, fmt.Errorf("AddNode: failed to create node event stream %q: %w", nodeStreamName(poolName, nodeID), err) } + nodeStreamActive = true if _, err = nodeStream.Add(ctx, evInit, []byte(nodeID)); err != nil { return nil, fmt.Errorf("AddNode: failed to add init event to node event stream %q: %w", nodeStreamName(poolName, nodeID), err) } @@ -229,62 +445,86 @@ func AddNode(ctx context.Context, poolName string, rdb *redis.Client, opts ...No return nil, fmt.Errorf("AddNode: failed to create node event reader for stream %q: %w", nodeStreamName(poolName, nodeID), err) } + scheduleCtx, scheduleCancel := context.WithCancel(context.Background()) p := &Node{ - ID: nodeID, - PoolName: poolName, - nodeKeepAliveMap: nkm, - nodeShutdownMap: nsm, - workerMap: wm, - workerKeepAliveMap: wkm, - workerCleanupMap: wcm, - jobMap: jm, - jobPayloadMap: jpm, - jobPendingMap: jpem, - tickerMap: tm, - workerStreams: sync.Map{}, - nodeStreams: sync.Map{}, - pendingJobChannels: sync.Map{}, - pendingEvents: sync.Map{}, - poolStream: poolStream, - poolSink: poolSink, - nodeStream: nodeStream, - nodeReader: nodeReader, - clientOnly: o.clientOnly, - workerTTL: o.workerTTL, - workerShutdownTTL: o.workerShutdownTTL, - ackGracePeriod: o.ackGracePeriod, - h: &jumpHash{h: crc64.New(crc64.MakeTable(crc64.ECMA))}, - stop: make(chan struct{}), - closed: closed, - rdb: rdb, - logger: logger, + ID: nodeID, + PoolName: poolName, + nodeKeepAliveMap: nkm, + nodeShutdownMap: nsm, + workerMap: wm, + workerKeepAliveMap: wkm, + workerCleanupMap: wcm, + jobMap: jm, + jobPayloadMap: jpm, + jobPendingMap: jpem, + tickerMap: tm, + schedulerJobMap: sjm, + workerStreams: sync.Map{}, + nodeStreams: sync.Map{}, + pendingJobChannels: sync.Map{}, + pendingEvents: sync.Map{}, + poolStream: poolStream, + poolSink: poolSink, + nodeStream: nodeStream, + nodeReader: nodeReader, + clientOnly: o.clientOnly, + workerTTL: resources.workerTTL, + requeueTimeout: o.requeueTimeout, + dispatchTimeout: o.dispatchTimeout, + dispatchResultRetention: o.dispatchResultRetention, + recoveryGrace: o.recoveryGrace, + cleanupLease: o.cleanupLease, + maxQueuedJobs: o.maxQueuedJobs, + resources: resources, + h: &jumpHash{h: crc64.New(crc64.MakeTable(crc64.ECMA))}, + stop: make(chan struct{}), + closed: make(chan struct{}), + scheduleCtx: scheduleCtx, + scheduleCancel: scheduleCancel, + settlements: newDispatchSettlements(), + rdb: rdb, + logger: logger, } nch := nodeReader.Subscribe() + // Preserve the caller's logging context for background goroutines. + logCtx := context.Background() + logCtx = log.WithContext(logCtx, ctx) + if o.clientOnly { logger.Info("client-only") - p.wg.Add(3) + p.wg.Add(4) pulse.Go(logger, func() { p.handleNodeEvents(nch) }) // to handle job acks + pulse.Go(logger, func() { p.watchShutdown(logCtx, shutdownUpdates) }) pulse.Go(logger, func() { p.processInactiveNodes() }) pulse.Go(logger, func() { p.updateNodeKeepAlive() }) - return p, nil + } else { + p.wg.Add(7) // Increment for all background goroutines + pulse.Go(logger, func() { p.handlePoolEvents(poolSink.Subscribe()) }) + pulse.Go(logger, func() { p.handleNodeEvents(nch) }) + pulse.Go(logger, func() { p.watchWorkers(logCtx) }) + pulse.Go(logger, func() { p.watchShutdown(logCtx, shutdownUpdates) }) + pulse.Go(logger, func() { p.processInactiveNodes() }) + pulse.Go(logger, func() { p.processInactiveWorkers(logCtx) }) + pulse.Go(logger, func() { p.updateNodeKeepAlive() }) } - // create new logger context for goroutines. - logCtx := context.Background() - logCtx = log.WithContext(logCtx, ctx) - - p.wg.Add(8) // Increment for all background goroutines - pulse.Go(logger, func() { p.handlePoolEvents(poolSink.Subscribe()) }) - pulse.Go(logger, func() { p.handleNodeEvents(nch) }) - pulse.Go(logger, func() { p.watchWorkers(logCtx) }) - pulse.Go(logger, func() { p.watchShutdown(logCtx) }) - pulse.Go(logger, func() { p.processInactiveNodes() }) - pulse.Go(logger, func() { p.processInactiveWorkers(logCtx) }) - pulse.Go(logger, func() { p.processInactiveJobs(logCtx) }) - pulse.Go(logger, func() { p.updateNodeKeepAlive() }) + shuttingDown, err := rdb.HExists(ctx, rmapContentKey(resources.nodeShutdown), "shutdown").Result() + if err != nil { + if closeErr := p.close(ctx, true); closeErr != nil { + return nil, errors.Join( + fmt.Errorf("AddNode: failed post-registration shutdown check: %w", err), + fmt.Errorf("AddNode: failed to close node after shutdown check: %w", closeErr), + ) + } + return nil, fmt.Errorf("AddNode: failed post-registration shutdown check: %w", err) + } + if shuttingDown { + p.ownShutdown(logCtx) + } + setupComplete = true return p, nil } @@ -293,12 +533,17 @@ func AddNode(ctx context.Context, poolName string, rdb *redis.Client, opts ...No // NotificationHandler and MessageHandler interfaces to handle job-scoped // notifications and hash-routed messages. func (node *Node) AddWorker(ctx context.Context, handler JobHandler) (*Worker, error) { - if node.IsClosed() { + node.lock.RLock() + defer node.lock.RUnlock() + if node.closing { return nil, fmt.Errorf("AddWorker: pool %q is closed", node.PoolName) } if node.clientOnly { return nil, fmt.Errorf("AddWorker: pool %q is client-only", node.PoolName) } + if err := node.ensureGenerationActive(ctx); err != nil { + return nil, fmt.Errorf("AddWorker: %w", err) + } w, err := newWorker(ctx, node, handler) if err != nil { return nil, err @@ -311,11 +556,23 @@ func (node *Node) AddWorker(ctx context.Context, handler JobHandler) (*Worker, e // RemoveWorker stops the worker, removes it from the pool and requeues all its // jobs. func (node *Node) RemoveWorker(ctx context.Context, w *Worker) error { - w.stop(ctx) + node.lock.RLock() + defer node.lock.RUnlock() + if node.closing { + return fmt.Errorf("RemoveWorker: pool %q is closed", node.PoolName) + } + if err := w.stop(ctx); err != nil { + return fmt.Errorf("RemoveWorker: failed to stop worker %q: %w", w.ID, err) + } + if err := node.settlements.waitWorker(ctx, w.ID); err != nil { + return fmt.Errorf("RemoveWorker: terminal outcomes for worker %q remain unsettled: %w", w.ID, err) + } if err := w.requeueJobs(ctx); err != nil { - node.logger.Error(fmt.Errorf("RemoveWorker: failed to requeue jobs for worker %q: %w", w.ID, err)) + return fmt.Errorf("RemoveWorker: failed to requeue jobs for worker %q: %w", w.ID, err) + } + if err := node.removeWorker(ctx, w.ID); err != nil { + return fmt.Errorf("RemoveWorker: failed to remove worker %q: %w", w.ID, err) } - node.removeWorker(ctx, w.ID) node.localWorkers.Delete(w.ID) node.logger.Info("removed worker", "worker", w.ID) return nil @@ -360,17 +617,51 @@ func (node *Node) PoolWorkers() []*Worker { // // The method blocks until one of the above conditions is met. func (node *Node) DispatchJob(ctx context.Context, key string, payload []byte) error { - job := marshalJob(&Job{Key: key, Payload: payload, CreatedAt: time.Now(), NodeID: node.ID}) - return node.dispatchJob(ctx, key, job) + _, err := node.DispatchJobOnce(ctx, ulid.Make().String(), key, payload) + return err +} + +// DispatchJobOnce atomically publishes one start event for dispatchID and +// waits for its worker result. Retrying the same dispatchID returns the same +// Redis event ID and never starts a duplicate handler. +func (node *Node) DispatchJobOnce( + ctx context.Context, + dispatchID, key string, + payload []byte, +) (string, error) { + node.lock.RLock() + defer node.lock.RUnlock() + if node.closing { + return "", fmt.Errorf("DispatchJob: pool %q is closed", node.PoolName) + } + if err := node.ensureGenerationActive(ctx); err != nil { + return "", fmt.Errorf("DispatchJob: %w", err) + } + if dispatchID == "" { + return "", fmt.Errorf("DispatchJob: dispatch ID cannot be empty") + } + job := &Job{ + Key: key, + Payload: payload, + CreatedAt: time.Now(), + NodeID: node.ID, + dispatchID: dispatchID, + } + return node.dispatchJob(ctx, dispatchID, key, job) } // DispatchMessage sends a keyed message to the worker currently assigned by the // pool hash ring. Messages do not create job ownership and are intended for // fire-and-forget work that should be load-balanced by key. func (node *Node) DispatchMessage(ctx context.Context, key string, payload []byte) error { - if node.IsClosed() { + node.lock.RLock() + defer node.lock.RUnlock() + if node.closing { return fmt.Errorf("DispatchMessage: pool %q is closed", node.PoolName) } + if err := node.ensureGenerationActive(ctx); err != nil { + return fmt.Errorf("DispatchMessage: %w", err) + } if _, err := node.poolStream.Add(ctx, evMessage, marshalKeyedPayload(key, payload)); err != nil { return fmt.Errorf("DispatchMessage: failed to add message to stream %q: %w", node.poolStream.Name, err) } @@ -378,129 +669,385 @@ func (node *Node) DispatchMessage(ctx context.Context, key string, payload []byt return nil } -func (node *Node) dispatchJob(ctx context.Context, key string, job []byte) error { - if node.IsClosed() { - return fmt.Errorf("DispatchJob: pool %q is closed", node.PoolName) - } +func (node *Node) dispatchJob(ctx context.Context, dispatchID, key string, job *Job) (string, error) { + waiter := node.acquireDispatchWaiter(dispatchID) + defer node.releaseDispatchWaiter(dispatchID, waiter) - pendingTS, err := node.claimDispatch(ctx, key) + record, err := node.publishDispatchRecord(ctx, key, dispatchID, job.Payload, marshalJob(job)) if err != nil { - return err + return "", err } - - eventID, err := node.poolStream.Add(ctx, evStartJob, job) - if err != nil { - // Clean up pending entry on failure - node.releaseDispatchPending(key, pendingTS) - return fmt.Errorf("DispatchJob: failed to add job to stream %q: %w", node.poolStream.Name, err) + if record.status == dispatchTerminal { + return record.eventID, dispatchTerminalError(record) } - - cherr := make(chan error, 1) - node.pendingJobChannels.Store(eventID, cherr) - - timer := time.NewTimer(2 * node.ackGracePeriod) - defer timer.Stop() - - select { - case err = <-cherr: - case <-timer.C: - err = fmt.Errorf("DispatchJob: job %q timed out, TTL: %v", key, 2*node.ackGracePeriod) - case <-ctx.Done(): - err = ctx.Err() + identity, err := dispatchIdentity(key, job.Payload) + if err != nil { + return record.eventID, err } - - node.pendingJobChannels.Delete(eventID) - close(cherr) - - // Clean up pending entry - node.releaseDispatchPending(key, pendingTS) - + record, err = node.awaitDispatch(ctx, waiter, dispatchID, identity, record) if err != nil { node.logger.Error(fmt.Errorf("DispatchJob: failed to dispatch job: %w", err), "key", key) - return err + return record.eventID, err } - node.logger.Info("dispatched", "key", key) - return nil + return record.eventID, dispatchTerminalError(record) +} + +// awaitDispatch treats local completion as a wake-up hint and Redis as the +// authoritative terminal state. Polling is bounded by DispatchTimeout and each +// wake, cancellation, or timeout edge performs one final durable read. +func (node *Node) awaitDispatch( + ctx context.Context, + waiter *dispatchWaiter, + dispatchID string, + identity []byte, + record dispatchRecord, +) (dispatchRecord, error) { + poll := min(50*time.Millisecond, node.dispatchTimeout) + ticker := time.NewTicker(poll) + defer ticker.Stop() + timer := time.NewTimer(node.dispatchTimeout) + defer timer.Stop() + for { + var edgeErr error + select { + case <-waiter.done: + case <-ticker.C: + case <-timer.C: + edgeErr = fmt.Errorf( + "DispatchJob: dispatch %q timed out after %v", + dispatchID, + node.dispatchTimeout, + ) + case <-ctx.Done(): + edgeErr = ctx.Err() + } + var ( + current dispatchRecord + err error + ) + if edgeErr != nil { + current, err = node.readDispatchRecordAfterEdge(ctx, dispatchID, identity) + } else { + current, err = node.readDispatchRecord(ctx, dispatchID, identity) + } + if err != nil { + return record, err + } + record = current + if record.status == dispatchTerminal { + return record, nil + } + if edgeErr != nil { + return record, edgeErr + } + } } -// claimDispatch atomically decides whether a job key may be dispatched. Redis is -// the source of truth for both states that matter to singleton admission: a -// durable payload means the job is already running, and a pending guard means a -// worker is currently starting it. -func (node *Node) claimDispatch(ctx context.Context, key string) (string, error) { +// readDispatchRecordAfterEdge gives cancellation one bounded authoritative +// Redis read so a concurrently committed terminal result wins the race. +func (node *Node) readDispatchRecordAfterEdge( + ctx context.Context, + dispatchID string, + identity []byte, +) (dispatchRecord, error) { + readCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 100*time.Millisecond) + defer cancel() + return node.readDispatchRecord(readCtx, dispatchID, identity) +} + +// acquireDispatchWaiter joins all local callers for one dispatch ID to the +// same completion broadcast without racing the final caller's removal. +func (node *Node) acquireDispatchWaiter(dispatchID string) *dispatchWaiter { + node.dispatchWaitersLock.Lock() + defer node.dispatchWaitersLock.Unlock() + candidate := &dispatchWaiter{done: make(chan struct{})} + value, _ := node.pendingJobChannels.LoadOrStore(dispatchID, candidate) + waiter := value.(*dispatchWaiter) + waiter.refs.Add(1) + return waiter +} + +// releaseDispatchWaiter removes the local completion broadcast only after the +// final joined caller has stopped waiting. +func (node *Node) releaseDispatchWaiter(dispatchID string, waiter *dispatchWaiter) { + node.dispatchWaitersLock.Lock() + defer node.dispatchWaitersLock.Unlock() + if waiter.refs.Add(-1) == 0 { + node.pendingJobChannels.CompareAndDelete(dispatchID, waiter) + } +} + +// publishDispatch atomically admits and appends one generation-fenced start +// event and durable exact-identity record. +func (node *Node) publishDispatchRecord( + ctx context.Context, + key, dispatchID string, + jobPayload, eventPayload []byte, +) (dispatchRecord, error) { if key == "" { - return "", fmt.Errorf("DispatchJob: job key cannot be empty") + return dispatchRecord{}, fmt.Errorf("DispatchJob: job key cannot be empty") } if strings.Contains(key, "=") { - return "", fmt.Errorf("DispatchJob: job key %q cannot contain '='", key) + return dispatchRecord{}, fmt.Errorf("DispatchJob: job key %q cannot contain '='", key) } - now := time.Now() - pendingUntil := strconv.FormatInt(now.Add(2*node.ackGracePeriod).UnixNano(), 10) - raw, err := luaClaimDispatch.Run(ctx, node.rdb, []string{ - rmapContentKey(jobPayloadMapName(node.PoolName)), - rmapContentKey(jobPendingMapName(node.PoolName)), - rmapUpdateChannel(jobPendingMapName(node.PoolName)), - }, key, strconv.FormatInt(now.UnixNano(), 10), pendingUntil).Result() + identity, err := dispatchIdentity(key, jobPayload) if err != nil { - return "", fmt.Errorf("DispatchJob: failed to claim job %q: %w", key, err) + return dispatchRecord{}, err + } + raw, err := luaDispatchJob.Run(ctx, node.rdb, []string{ + fmt.Sprintf("pulse:stream:%s:lifecycle", node.poolStream.Name), + rmapContentKey(node.resources.jobPayloads), + rmapContentKey(node.resources.jobPending), + rmapUpdateChannel(node.resources.jobPending), + dispatchRecordKey(node.resources.dispatches, dispatchID), + dispatchActiveKey(node.resources.dispatches), + rmapContentKey(node.resources.nodeKeepAlive), + }, + key, + dispatchID, + "active", + node.resources.generation, + node.maxQueuedJobs, + evStartJob, + eventPayload, + "physical_key", + identity, + node.ID, + nodeCleanupField(node.ID), + ).Result() + if err != nil { + if redis.HasErrorPrefix(err, "DISPATCHIDEMPOTENCYCONFLICT") { + return dispatchRecord{}, fmt.Errorf("%w: dispatch %q", ErrDispatchConflict, dispatchID) + } + return dispatchRecord{}, fmt.Errorf("DispatchJob: failed to claim job %q: %w", key, poolBoundaryError(err)) } - status, value, err := parseDispatchClaim(raw) + record, err := parseDispatchRecord(raw) if err != nil { - return "", fmt.Errorf("DispatchJob: failed to parse claim result for job %q: %w", key, err) + return dispatchRecord{}, fmt.Errorf("DispatchJob: failed to parse claim result for job %q: %w", key, err) } - switch status { - case dispatchClaimed: - return value, nil + switch record.status { + case dispatchClaimed, dispatchTerminal: + return record, nil case dispatchAlreadyPending: node.logger.Info("DispatchJob: job already dispatched", "key", key) - return "", fmt.Errorf("%w: job %q is already dispatched", ErrJobExists, key) + return dispatchRecord{}, fmt.Errorf("%w: job %q is already dispatched", ErrJobExists, key) case dispatchAlreadyRunning: node.logger.Info("DispatchJob: job already exists", "key", key) - return "", fmt.Errorf("%w: job %q", ErrJobExists, key) - case dispatchMalformedPending: - return "", fmt.Errorf("DispatchJob: malformed pending guard for job %q: %q", key, value) + return dispatchRecord{}, fmt.Errorf("%w: job %q", ErrJobExists, key) + case dispatchCapacityReached: + return dispatchRecord{}, fmt.Errorf("%w: maximum %d pending jobs", ErrPoolCapacity, node.maxQueuedJobs) default: - return "", fmt.Errorf("DispatchJob: unexpected claim status %d for job %q", status, key) + return dispatchRecord{}, fmt.Errorf("DispatchJob: unexpected claim status %d for job %q", record.status, key) } } -// releaseDispatchPending clears the pending guard only if this dispatch still -// owns it. Dispatch callers can time out while another node later claims a -// stale pending key, so unconditional deletion would erase a newer guard. -func (node *Node) releaseDispatchPending(key, pendingTS string) { - if _, err := luaReleaseDispatch.Run(context.Background(), node.rdb, []string{ - rmapContentKey(jobPendingMapName(node.PoolName)), - rmapUpdateChannel(jobPendingMapName(node.PoolName)), - }, key, pendingTS).Result(); err != nil { - node.logger.Error(fmt.Errorf("DispatchJob: failed to clean up pending entry for job %q: %w", key, err)) +// publishDispatch admits a marshaled job for focused package tests. +func (node *Node) publishDispatch( + ctx context.Context, + key, dispatchID string, + eventPayload []byte, +) (string, error) { + job, err := unmarshalJob(eventPayload) + if err != nil { + return "", err + } + record, err := node.publishDispatchRecord(ctx, key, dispatchID, job.Payload, eventPayload) + return record.eventID, err +} + +// settleDispatch durably records one terminal result and settles its exact +// stream event. Exact retries return the original immutable outcome. +func (node *Node) settleDispatch(ctx context.Context, key, dispatchID string, resultErr error) (dispatchRecord, error) { + errorText := "" + if resultErr != nil { + errorText = resultErr.Error() + } + raw, err := luaSettleDispatch.Run(ctx, node.rdb, []string{ + fmt.Sprintf("pulse:stream:%s:lifecycle", node.poolStream.Name), + rmapContentKey(node.resources.jobPending), + rmapUpdateChannel(node.resources.jobPending), + dispatchRecordKey(node.resources.dispatches, dispatchID), + dispatchActiveKey(node.resources.dispatches), + }, key, dispatchID, "active", node.resources.generation, "physical_key", + "", errorText, "events", node.dispatchResultRetention.Milliseconds()).Result() + if err != nil { + return dispatchRecord{}, fmt.Errorf("settle dispatch %q: %w", key, poolBoundaryError(err)) + } + record, err := parseTerminalDispatch(raw) + if err != nil { + return dispatchRecord{}, fmt.Errorf("settle dispatch %q: %w", key, err) + } + node.dispatchWaitersLock.Lock() + if value, ok := node.pendingJobChannels.Load(dispatchID); ok { + waiter := value.(*dispatchWaiter) + waiter.once.Do(func() { + close(waiter.done) + }) + } + node.dispatchWaitersLock.Unlock() + return record, nil +} + +// readDispatchRecord returns the exact Redis-owned status for one dispatch. +func (node *Node) readDispatchRecord( + ctx context.Context, + dispatchID string, + identity []byte, +) (dispatchRecord, error) { + raw, err := readDispatchRecordScript.Run( + ctx, + node.rdb, + []string{ + fmt.Sprintf("pulse:stream:%s:lifecycle", node.poolStream.Name), + dispatchRecordKey(node.resources.dispatches, dispatchID), + }, + "active", + node.resources.generation, + identity, + ).Result() + if err != nil { + switch { + case redis.HasErrorPrefix(err, "DISPATCHIDEMPOTENCYCONFLICT"): + return dispatchRecord{}, fmt.Errorf("%w: dispatch %q", ErrDispatchConflict, dispatchID) + case redis.HasErrorPrefix(err, "DISPATCHRECORDNOTFOUND"): + return dispatchRecord{}, fmt.Errorf( + "DispatchJob: durable record for dispatch %q is unavailable", + dispatchID, + ) + default: + return dispatchRecord{}, fmt.Errorf( + "DispatchJob: read durable dispatch %q: %w", + dispatchID, + poolBoundaryError(err), + ) + } } + record, err := parseDispatchRecord(raw) + if err != nil { + return dispatchRecord{}, fmt.Errorf("DispatchJob: parse durable dispatch %q: %w", dispatchID, err) + } + return record, nil } -// parseDispatchClaim decodes the Lua admission result into its status code and -// payload. The script owns the result schema so malformed data is a programming -// error, not a recoverable distributed state. -func parseDispatchClaim(raw any) (int64, string, error) { +// completeDispatch settles a successful dispatch for focused package tests. +func (node *Node) completeDispatch(ctx context.Context, key, dispatchID string) error { + _, err := node.settleDispatch(ctx, key, dispatchID, nil) + return err +} + +// parseDispatchRecord validates the durable admission result boundary. +func parseDispatchRecord(raw any) (dispatchRecord, error) { values, ok := raw.([]any) - if !ok || len(values) != 2 { - return 0, "", fmt.Errorf("invalid claim result %T", raw) + if !ok || len(values) != 4 { + return dispatchRecord{}, fmt.Errorf("invalid dispatch result %T", raw) } status, ok := values[0].(int64) if !ok { - return 0, "", fmt.Errorf("invalid claim status %T", values[0]) + return dispatchRecord{}, fmt.Errorf("invalid dispatch status %T", values[0]) } - value, ok := values[1].(string) - if !ok { - return 0, "", fmt.Errorf("invalid claim value %T", values[1]) + decoded := make([]string, 3) + for i := range decoded { + value, ok := values[i+1].(string) + if !ok { + return dispatchRecord{}, fmt.Errorf("invalid dispatch field %d type %T", i, values[i+1]) + } + decoded[i] = value + } + return dispatchRecord{status: status, eventID: decoded[0], result: decoded[1], err: decoded[2]}, nil +} + +// parseTerminalDispatch validates an atomic settlement result. +func parseTerminalDispatch(raw any) (dispatchRecord, error) { + values, ok := raw.([]any) + if !ok || len(values) != 3 { + return dispatchRecord{}, fmt.Errorf("invalid terminal dispatch result %T", raw) + } + decoded := make([]string, len(values)) + for i, value := range values { + text, ok := value.(string) + if !ok { + return dispatchRecord{}, fmt.Errorf("invalid terminal dispatch field %d type %T", i, value) + } + decoded[i] = text + } + return dispatchRecord{ + status: dispatchTerminal, + eventID: decoded[0], + result: decoded[1], + err: decoded[2], + }, nil +} + +// dispatchTerminalError reconstructs the public terminal error text persisted +// by the worker settlement owner. +func dispatchTerminalError(record dispatchRecord) error { + if record.err == "" { + return nil } - return status, value, nil + return errors.New(record.err) } -// StopJob stops the job with the given key. +// dispatchIdentity length-prefixes exact key and payload bytes. +func dispatchIdentity(key string, payload []byte) ([]byte, error) { + var identity bytes.Buffer + for _, field := range [][]byte{[]byte(key), payload} { + if err := binary.Write(&identity, binary.BigEndian, uint64(len(field))); err != nil { + return nil, fmt.Errorf("encode dispatch identity: %w", err) + } + if _, err := identity.Write(field); err != nil { + return nil, fmt.Errorf("encode dispatch identity: %w", err) + } + } + return identity.Bytes(), nil +} + +// dispatchRecordToken produces a collision-free Redis hash field namespace. +func dispatchRecordToken(dispatchID string) string { + return hex.EncodeToString([]byte(dispatchID)) +} + +// dispatchRecordKey returns one generation-qualified per-dispatch Redis hash. +func dispatchRecordKey(resource, dispatchID string) string { + return fmt.Sprintf( + "pulse:pool:dispatch:%s:%s", + hex.EncodeToString([]byte(resource)), + dispatchRecordToken(dispatchID), + ) +} + +// dispatchActiveKey indexes only unsettled records for exact cleanup. +func dispatchActiveKey(resource string) string { + return fmt.Sprintf("pulse:pool:dispatch:%s:active", hex.EncodeToString([]byte(resource))) +} + +// poolBoundaryError maps Redis-owned pool contracts to typed public errors. +func poolBoundaryError(err error) error { + switch { + case redis.HasErrorPrefix(err, "POOLGENERATIONLOST"): + return fmt.Errorf("%w: %v", ErrPoolGenerationLost, err) + case redis.HasErrorPrefix(err, "NODECLEANUPLOST"): + return fmt.Errorf("%w: %v", ErrPoolGenerationLost, err) + case redis.HasErrorPrefix(err, "POOLCONFIGMISMATCH"): + return fmt.Errorf("%w: %v", ErrPoolConfigMismatch, err) + case redis.HasErrorPrefix(err, "POOLQUIESCENCEREQUIRED"): + return fmt.Errorf("%w: %v", ErrQuiescenceRequired, err) + default: + return err + } +} + +// StopJob durably publishes a stop request for the job with the given key. +// Success means Redis accepted the request, not that the handler has completed. func (node *Node) StopJob(ctx context.Context, key string) error { - if node.IsClosed() { + node.lock.RLock() + defer node.lock.RUnlock() + if node.closing { return fmt.Errorf("StopJob: pool %q is closed", node.PoolName) } + if err := node.ensureGenerationActive(ctx); err != nil { + return fmt.Errorf("StopJob: %w", err) + } if _, err := node.poolStream.Add(ctx, evStopJob, marshalJobKey(key)); err != nil { return fmt.Errorf("StopJob: failed to add stop job to stream %q: %w", node.poolStream.Name, err) } @@ -540,9 +1087,14 @@ func (node *Node) JobPayload(key string) ([]byte, bool) { // NotifyWorker notifies the worker that currently owns the job with the given // key. func (node *Node) NotifyWorker(ctx context.Context, key string, payload []byte) error { - if node.IsClosed() { + node.lock.RLock() + defer node.lock.RUnlock() + if node.closing { return fmt.Errorf("NotifyWorker: pool %q is closed", node.PoolName) } + if err := node.ensureGenerationActive(ctx); err != nil { + return fmt.Errorf("NotifyWorker: %w", err) + } if _, err := node.poolStream.Add(ctx, evNotify, marshalKeyedPayload(key, payload)); err != nil { return fmt.Errorf("NotifyWorker: failed to add notification to stream %q: %w", node.poolStream.Name, err) } @@ -550,6 +1102,47 @@ func (node *Node) NotifyWorker(ctx context.Context, key string, payload []byte) return nil } +// ensureGenerationActive fences every node-owned mutation against the exact +// pool stream incarnation. Cleanup invalidates the stream before deleting its +// maps, so a successful preflight can only race with deletion that follows it; +// once deletion completes, stale nodes fail before recreating any old key. +func (node *Node) ensureGenerationActive(ctx context.Context) error { + err := node.poolStream.Open(ctx) + if err == nil { + _, refreshErr := refreshPoolNode(ctx, node.rdb, node.resources, node.ID) + if refreshErr == nil { + return nil + } + if strings.Contains(refreshErr.Error(), "NODECLEANUPLOST") { + node.stopAfterLifecycleLoss("stale node cleanup fence", false) + } + return fmt.Errorf("%w: %v", ErrPoolGenerationLost, refreshErr) + } + if errors.Is(err, streaming.ErrStreamDestroyed) { + node.stopAfterLifecycleLoss("pool generation ended", true) + } + return fmt.Errorf("%w: %v", ErrPoolGenerationLost, err) +} + +// stopAfterLifecycleLoss immediately fences local admission and asynchronously +// closes the node after a Redis-owned terminal lifecycle transition. +func (node *Node) stopAfterLifecycleLoss(reason string, shutdown bool) { + node.lock.Lock() + closing := node.closing + node.closing = true + node.lock.Unlock() + if closing { + return + } + node.terminalOnce.Do(func() { + pulse.Go(node.logger, func() { + if closeErr := node.closeAfterDistributedLoss(context.Background(), shutdown); closeErr != nil { + node.logger.Error(fmt.Errorf("stop node after %s: %w", reason, closeErr)) + } + }) + }) +} + // Shutdown stops the pool workers gracefully across all nodes. It notifies all // workers and waits until they are completed. Shutdown prevents the pool nodes // from creating new workers and the pool workers from accepting new jobs. After @@ -557,29 +1150,211 @@ func (node *Node) NotifyWorker(ctx context.Context, key string, payload []byte) // discarded. One of Shutdown or Close should be called before the node is // garbage collected unless it is client-only. func (node *Node) Shutdown(ctx context.Context) error { - if node.IsClosed() { - return nil - } if node.clientOnly { return fmt.Errorf("Shutdown: client-only node cannot shutdown worker pool") } - - // Signal all nodes to shutdown. - if _, err := node.nodeShutdownMap.Set(ctx, "shutdown", node.ID); err != nil { - node.logger.Error(fmt.Errorf("Shutdown: failed to set shutdown status in shutdown map: %w", err)) + node.lock.RLock() + cleanupComplete := node.cleanupComplete + node.lock.RUnlock() + if cleanupComplete { + return node.closeAfterCleanup(ctx) + } + cleanupComplete, err := node.poolCleanupComplete(ctx) + if err != nil { + return err + } + if cleanupComplete { + node.lock.Lock() + node.cleanupComplete = true + node.lock.Unlock() + return node.closeAfterCleanup(ctx) + } + // Publish through Redis directly because concurrent Close may already have + // closed the local shutdown-map replica. The obligation must exist before + // this caller joins or resumes the distributed barrier. + if err := node.publishShutdown(ctx); err != nil { + return err + } + if err := node.close(ctx, true); err != nil { + return fmt.Errorf("Shutdown: failed to close local node: %w", err) + } + if err := node.waitForPoolNodes(ctx); err != nil { + return err + } + if err := node.cleanupPool(ctx); err != nil { + return err } - <-node.closed // Wait for this node to be closed - node.cleanupPool(ctx) + node.lock.Lock() + node.cleanupComplete = true + node.shutdown = true + node.lock.Unlock() node.logger.Info("shutdown") return nil } -// Close stops the node workers and closes the Redis connection but does -// not stop workers running in other nodes. It requeues all the jobs run by -// workers of the node. One of Shutdown or Close should be called before the -// node is garbage collected unless it is client-only. +// publishShutdown durably records this node's pool-wide shutdown obligation. +func (node *Node) publishShutdown(ctx context.Context) error { + err := publishPoolShutdownScript.Run( + ctx, + node.rdb, + []string{ + rmapContentKey(node.resources.nodeShutdown), + rmapUpdateChannel(node.resources.nodeShutdown), + }, + node.ID, + ).Err() + if err != nil { + return fmt.Errorf("Shutdown: failed to publish distributed shutdown obligation: %w", err) + } + return nil +} + +// poolCleanupComplete reads this pool-stream generation's durable completion +// marker. Local closure and distributed cleanup completion are separate states. +func (node *Node) poolCleanupComplete(ctx context.Context) (bool, error) { + values, err := node.rdb.HMGet( + ctx, + poolCleanupGenerationsKey(node.PoolName), + "state", + "generation", + ).Result() + if err != nil { + return false, fmt.Errorf("Shutdown: failed to read pool cleanup completion: %w", err) + } + state, _ := values[0].(string) + generation, _ := values[1].(string) + return state == poolCleanupCompleteState && generation == node.poolStream.Generation(), nil +} + +// registerPoolNode reserves and publishes the node's authoritative lease only +// while its exact pool generation is active and shutdown/final cleanup are +// absent. +func registerPoolNode( + ctx context.Context, + rdb *redis.Client, + resources poolResources, + nodeID string, +) (bool, string, error) { + result, err := registerPoolNodeScript.Run( + ctx, + rdb, + []string{ + fmt.Sprintf("pulse:stream:%s:lifecycle", poolStreamName(resources.pool)), + rmapContentKey(resources.nodeShutdown), + rmapContentKey(resources.nodeKeepAlive), + rmapUpdateChannel(resources.nodeKeepAlive), + poolCleanupGenerationsKey(resources.pool), + }, + nodeID, + resources.generation, + "active", + nodeCleanupField(nodeID), + ).Slice() + if err != nil { + return false, "", err + } + if len(result) == 0 { + return false, "", fmt.Errorf("registration script returned no status") + } + status, ok := result[0].(int64) + if !ok { + return false, "", fmt.Errorf("registration script returned invalid status %T", result[0]) + } + if status == 0 { + return false, "", nil + } + if status == 2 { + return false, "", errors.New("NODECLEANUPLOST") + } + if len(result) != 2 { + return false, "", fmt.Errorf("registration script returned %d values", len(result)) + } + timestamp, ok := result[1].(string) + if !ok { + return false, "", fmt.Errorf("registration script returned invalid timestamp %T", result[1]) + } + return true, timestamp, nil +} + +// refreshPoolNode renews an existing node heartbeat without reopening +// admission during shutdown. +func refreshPoolNode( + ctx context.Context, + rdb *redis.Client, + resources poolResources, + nodeID string, +) (string, error) { + timestamp, err := refreshPoolNodeScript.Run( + ctx, + rdb, + []string{ + fmt.Sprintf("pulse:stream:%s:lifecycle", poolStreamName(resources.pool)), + rmapContentKey(resources.nodeKeepAlive), + rmapUpdateChannel(resources.nodeKeepAlive), + }, + nodeID, + resources.generation, + "active", + nodeCleanupField(nodeID), + ).Text() + return timestamp, err +} + +// maintainNodeRegistrationLease keeps a node visible to the shutdown barrier +// while AddNode constructs its streams and maps. The regular node heartbeat is +// running before this temporary lease owner is stopped. +func maintainNodeRegistrationLease( + ctx context.Context, + rdb *redis.Client, + resources poolResources, + nodeID string, + ttl time.Duration, + logger pulse.Logger, +) { + ticker := time.NewTicker(ttl / 2) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + registered, _, err := registerPoolNode(ctx, rdb, resources, nodeID) + if err != nil { + logger.Error(fmt.Errorf("AddNode: failed to refresh registration time: %w", err)) + continue + } + if !registered { + logger.Error(fmt.Errorf("AddNode: pool generation stopped accepting registrations")) + return + } + } + } +} + +// Close immediately rejects new node work, stops local workers, requeues their +// jobs, and detaches the node's Redis-owned resources. It does not close the +// caller-owned Redis client or stop workers in other nodes. A distributed +// detach failure is returned without marking the node closed, so Close may be +// retried with a fresh context. One of Shutdown or Close should be called +// before the node is garbage collected unless it is client-only. func (node *Node) Close(ctx context.Context) error { + node.lock.RLock() + cleanupComplete := node.cleanupComplete + node.lock.RUnlock() + if cleanupComplete { + return node.closeAfterCleanup(ctx) + } + cleanupComplete, err := node.poolCleanupComplete(ctx) + if err != nil { + return err + } + if cleanupComplete { + node.lock.Lock() + node.cleanupComplete = true + node.lock.Unlock() + return node.closeAfterCleanup(ctx) + } return node.close(ctx, false) } @@ -594,7 +1369,7 @@ func (node *Node) IsShutdown() bool { func (node *Node) IsClosed() bool { node.lock.RLock() defer node.lock.RUnlock() - return node.closing + return node.closedState } // close stops the node and its workers, optionally requeuing jobs. If shutdown @@ -603,73 +1378,214 @@ func (node *Node) IsClosed() bool { // waits for background goroutines to complete, cleans up resources and closes // connections. It is idempotent and can be called multiple times safely. func (node *Node) close(ctx context.Context, shutdown bool) error { + node.closeLock.Lock() + defer node.closeLock.Unlock() + node.lock.Lock() - if node.closing { + if node.closedState { node.lock.Unlock() return nil } - node.closing = true + if !node.closing { + node.closing = true + } node.lock.Unlock() - // If we're shutting down then stop all the jobs. + node.scheduleCancel() + node.scheduleWG.Wait() + + var stopJobsErr error if shutdown { - node.stopAllJobs(ctx) + stopJobsErr = node.stopAllJobs(ctx) } - // Stop all workers before waiting for goroutines. - // - // IMPORTANT: do NOT remove workers from the replicated maps here. - // Removing the worker deletes the worker->jobs mapping which is what other - // nodes use to recover/requeue jobs if this node dies mid-close. We only - // remove workers from maps after we've attempted to requeue. - var wg sync.WaitGroup - node.localWorkers.Range(func(key, value any) bool { + var workerStopErr error + var workerStopLock sync.Mutex + var workerStopWait sync.WaitGroup + node.localWorkers.Range(func(_, value any) bool { worker := value.(*Worker) - wg.Add(1) + workerStopWait.Add(1) pulse.Go(node.logger, func() { - defer wg.Done() - worker.stop(ctx) + defer workerStopWait.Done() + if err := worker.stop(ctx); err != nil { + workerStopLock.Lock() + workerStopErr = errors.Join(workerStopErr, err) + workerStopLock.Unlock() + } }) return true }) - wg.Wait() + workerStopWait.Wait() - // Stop all goroutines - close(node.stop) + node.stopOnce.Do(func() { + close(node.stop) + }) node.wg.Wait() - - // Requeue jobs if not shutting down. - // - // This is done after stopping node goroutines so we don't route any new pool - // events to workers that have already been stopped. + var localTeardownErr error + if stopJobsErr != nil { + localTeardownErr = errors.Join( + localTeardownErr, + fmt.Errorf("close: failed to stop jobs: %w", stopJobsErr), + ) + } + if workerStopErr != nil { + localTeardownErr = errors.Join( + localTeardownErr, + fmt.Errorf("close: failed to stop workers: %w", workerStopErr), + ) + } + if err := node.settlements.waitAll(ctx); err != nil { + localTeardownErr = errors.Join( + localTeardownErr, + fmt.Errorf("close: terminal dispatch outcomes remain unsettled: %w", err), + ) + } + if localTeardownErr != nil { + return localTeardownErr + } + + // Requeue and distributed worker cleanup are retried on every Close attempt. + // A worker remains locally discoverable until all of its map records are + // removed, so a failed attempt cannot hide incomplete cleanup. if !shutdown { if err := node.requeueAllJobs(ctx); err != nil { - node.logger.Error(fmt.Errorf("close: failed to requeue jobs: %w", err)) + return fmt.Errorf("close: failed to requeue jobs: %w", err) } } - - // Now that we attempted requeue, remove all local workers from pool maps. + var workerCleanupErr error node.localWorkers.Range(func(key, value any) bool { worker := value.(*Worker) - node.removeWorker(ctx, worker.ID) + if err := node.removeWorker(ctx, worker.ID); err != nil { + workerCleanupErr = errors.Join(workerCleanupErr, err) + return true + } node.localWorkers.Delete(key) return true }) + if workerCleanupErr != nil { + return fmt.Errorf("close: failed to remove local workers: %w", workerCleanupErr) + } - // Cleanup resources - node.cleanupNode(ctx) + // Detach distributed membership before closing the maps needed to retry it. + // Local goroutines were stopped above on the first attempt, so subsequent + // calls repeat only these idempotent distributed side effects. + if node.poolSink != nil { + if err := node.poolSink.Close(ctx); err != nil { + return fmt.Errorf("close: pending distributed cleanup: failed to detach pool sink: %w", err) + } + } + node.pendingEvents.Range(func(key, _ any) bool { + node.pendingEvents.Delete(key) + return true + }) + node.pendingJobChannels.Range(func(key, _ any) bool { + node.pendingJobChannels.Delete(key) + return true + }) + if _, err := node.nodeKeepAliveMap.Delete(ctx, node.ID); err != nil { + return fmt.Errorf("close: pending distributed cleanup: failed to detach node from pool: %w", err) + } - // Signal that the node is closed - close(node.closed) + // Local stream destruction is part of distributed cleanup. Keep the maps + // open until it succeeds so Close can retry truthfully. + if err := node.cleanupNode(ctx); err != nil { + return fmt.Errorf("close: pending distributed cleanup: %w", err) + } + // Publish closure and shutdown ownership atomically after all node-owned + // side effects complete. + node.lock.Lock() + node.closedState = true + if shutdown { + node.shutdown = true + } + node.lock.Unlock() + close(node.closed) node.logger.Info("closed") return nil } +// closeAfterCleanup performs every local teardown obligation after another +// process has already destroyed the distributed pool generation. No Redis +// resource deletion is attempted, but local intake, schedules, workers, +// readers, settlement goroutines, maps, and public closure state are joined. +func (node *Node) closeAfterCleanup(ctx context.Context) error { + return node.closeAfterDistributedLoss(ctx, true) +} + +// closeAfterDistributedLoss performs complete local teardown after Redis has +// fenced this node or destroyed its pool generation. +func (node *Node) closeAfterDistributedLoss(ctx context.Context, shutdown bool) error { + node.closeLock.Lock() + defer node.closeLock.Unlock() + + node.lock.Lock() + if node.closedState { + node.shutdown = node.shutdown || shutdown + err := node.closeAfterCleanupErr + node.lock.Unlock() + return err + } + node.closing = true + node.lock.Unlock() + + node.scheduleCancel() + node.scheduleWG.Wait() + node.localWorkers.Range(func(key, value any) bool { + value.(*Worker).stopLocal() + node.localWorkers.Delete(key) + return true + }) + node.stopOnce.Do(func() { + close(node.stop) + }) + if node.nodeReader != nil { + node.nodeReader.Close() + } + node.wg.Wait() + + var teardownErr error + if err := node.settlements.waitAll(ctx); err != nil { + teardownErr = errors.Join( + teardownErr, + fmt.Errorf("close after distributed cleanup: local terminal settlements: %w", err), + ) + } + if node.poolSink != nil { + if err := node.poolSink.Close(ctx); err != nil { + teardownErr = errors.Join(teardownErr, fmt.Errorf("close local pool sink: %w", err)) + } + } + for _, m := range node.maps() { + if m != nil { + m.Close() + } + } + node.pendingEvents.Range(func(key, _ any) bool { + node.pendingEvents.Delete(key) + return true + }) + node.pendingJobChannels.Range(func(key, _ any) bool { + node.pendingJobChannels.Delete(key) + return true + }) + + node.lock.Lock() + node.closedState = true + node.shutdown = shutdown + node.closeAfterCleanupErr = teardownErr + node.lock.Unlock() + close(node.closed) + node.logger.Info("closed after distributed lifecycle loss") + return teardownErr +} + // stopAllJobs stops all jobs running on the node. -func (node *Node) stopAllJobs(ctx context.Context) { +func (node *Node) stopAllJobs(ctx context.Context) error { var wg sync.WaitGroup var total atomic.Int32 + var stopErr error + var errLock sync.Mutex node.localWorkers.Range(func(key, value any) bool { wg.Add(1) worker := value.(*Worker) @@ -678,6 +1594,9 @@ func (node *Node) stopAllJobs(ctx context.Context) { for _, job := range worker.Jobs() { if err := worker.stopJob(ctx, job.Key); err != nil { node.logger.Error(fmt.Errorf("Close: failed to stop job %q for worker %q: %w", job.Key, worker.ID, err)) + errLock.Lock() + stopErr = errors.Join(stopErr, err) + errLock.Unlock() } total.Add(1) } @@ -686,6 +1605,7 @@ func (node *Node) stopAllJobs(ctx context.Context) { }) wg.Wait() node.logger.Info("stopped all jobs", "total", total.Load()) + return stopErr } // handlePoolEvents reads events from the pool job stream. @@ -694,12 +1614,14 @@ func (node *Node) handlePoolEvents(c <-chan *streaming.Event) { for { select { - case ev := <-c: + case ev, ok := <-c: + if !ok { + return + } if err := node.routeWorkerEvent(ev); err != nil { node.logger.Error(fmt.Errorf("handlePoolEvents: failed to route event: %w", err)) } case <-node.stop: - node.poolSink.Close(context.Background()) return } } @@ -708,17 +1630,36 @@ func (node *Node) handlePoolEvents(c <-chan *streaming.Event) { // routeWorkerEvent routes a dispatched event to the proper worker. func (node *Node) routeWorkerEvent(ev *streaming.Event) error { // Filter out stale events - if time.Since(ev.CreatedAt()) > pendingEventTTL { - node.logger.Debug("routeWorkerEvent: stale event, not routing", "event", ev.EventName, "id", ev.ID, "since", time.Since(ev.CreatedAt()), "TTL", pendingEventTTL) + now, err := node.rdb.Time(context.Background()).Result() + if err != nil { + return fmt.Errorf("routeWorkerEvent: read Redis time: %w", err) + } + age := now.Sub(ev.CreatedAt()) + if age > pendingEventTTL && ev.EventName != evStartJob { + node.logger.Debug("routeWorkerEvent: stale event, not routing", "event", ev.EventName, "id", ev.ID, "since", age, "TTL", pendingEventTTL) + settled, err := node.releaseTerminalDispatch(ev, errors.New("pool event expired before routing")) + if err != nil { + node.logger.Error(err, "event", ev.EventName, "id", ev.ID) + } // Ack the sink event so it does not get redelivered. - if err := node.poolSink.Ack(context.Background(), ev); err != nil { + if !settled { + err = node.settlePoolEvent(context.Background(), ev) + } + if err != nil { node.logger.Error(fmt.Errorf("routeWorkerEvent: failed to ack event: %w", err), "event", ev.EventName, "id", ev.ID) } return nil } // Compute the worker ID that will handle the event key. - key := unmarshalJobKey(ev.Payload) + key, err := poolEventKey(ev) + if err != nil { + node.logger.Error(fmt.Errorf("routeWorkerEvent: malformed event: %w", err), "event", ev.EventName, "id", ev.ID) + if ackErr := node.settlePoolEvent(context.Background(), ev); ackErr != nil { + return fmt.Errorf("routeWorkerEvent: acknowledge malformed event %s: %w", ev.ID, ackErr) + } + return nil + } wid, err := node.workerForEvent(ev.EventName, key) if err != nil { if errors.Is(err, errJobAwaitingOwner) { @@ -726,7 +1667,7 @@ func (node *Node) routeWorkerEvent(ev *streaming.Event) error { return nil } if errors.Is(err, errJobNotFound) { - if ackErr := node.poolSink.Ack(context.Background(), ev); ackErr != nil { + if ackErr := node.settlePoolEvent(context.Background(), ev); ackErr != nil { node.logger.Error(fmt.Errorf("routeWorkerEvent: failed to ack event for missing job: %w", ackErr), "event", ev.EventName, "id", ev.ID) } return nil @@ -743,6 +1684,18 @@ func (node *Node) routeWorkerEvent(ev *streaming.Event) error { if err != nil { return fmt.Errorf("routeWorkerEvent: failed to add event %s to worker stream %q: %w", ev.EventName, workerStreamName(wid), err) } + if eventID == "" { + settled, settleErr := node.releaseTerminalDispatch(ev, errors.New("no worker accepted dispatch")) + if settleErr != nil { + return settleErr + } + if !settled { + if err := node.settlePoolEvent(context.Background(), ev); err != nil { + return fmt.Errorf("routeWorkerEvent: failed to acknowledge unroutable event %s: %w", ev.ID, err) + } + } + return nil + } node.logger.Debug("routed", "event", ev.EventName, "id", ev.ID, "worker", wid, "worker-event-id", eventID) // Record the event in the pending events map for future ack. @@ -751,6 +1704,50 @@ func (node *Node) routeWorkerEvent(ev *streaming.Event) error { return nil } +// releaseTerminalDispatch clears singleton admission when routing has +// definitively discarded a start event before any worker could process it. +func (node *Node) releaseTerminalDispatch(event *streaming.Event, cause error) (bool, error) { + if event.EventName != evStartJob { + return false, nil + } + job, err := unmarshalJob(event.Payload) + if err != nil { + return false, fmt.Errorf("release terminal dispatch: decode start job: %w", err) + } + if job.dispatchID != "" { + _, err := node.settleDispatch(context.Background(), job.Key, job.dispatchID, cause) + return err == nil, err + } + return false, nil +} + +// poolEventKey decodes the exact wire shape selected by the event kind and +// rejects trailing or incompatible data before routing. +func poolEventKey(event *streaming.Event) (string, error) { + switch event.EventName { + case evStartJob: + job, err := unmarshalJob(event.Payload) + if err != nil { + return "", fmt.Errorf("decode start job: %w", err) + } + return job.Key, nil + case evMessage, evNotify: + key, _, err := unmarshalKeyedPayload(event.Payload) + if err != nil { + return "", fmt.Errorf("decode keyed event: %w", err) + } + return key, nil + case evStopJob: + key, err := unmarshalJobKey(event.Payload) + if err != nil { + return "", fmt.Errorf("decode stop job: %w", err) + } + return key, nil + default: + return "", fmt.Errorf("unknown pool event %q", event.EventName) + } +} + // handleNodeEvents reads events from the node event stream and acks the pending // events that correspond to jobs that are now running or done. func (node *Node) handleNodeEvents(c <-chan *streaming.Event) { @@ -758,7 +1755,10 @@ func (node *Node) handleNodeEvents(c <-chan *streaming.Event) { for { select { - case ev := <-c: + case ev, ok := <-c: + if !ok { + return + } node.processNodeEvent(ev) case <-node.stop: node.nodeReader.Close() @@ -777,19 +1777,22 @@ func (node *Node) processNodeEvent(ev *streaming.Event) { // Event sent by worker to ack a dispatched job. node.logger.Debug("handleNodeEvents: received ack", "event", ev.EventName, "id", ev.ID) node.ackWorkerEvent(ev) - case evDispatchReturn: - // Event sent by pool node to node that originally dispatched the job. - node.logger.Debug("handleNodeEvents: received dispatch return", "event", ev.EventName, "id", ev.ID) - node.returnDispatchStatus(ev) } } -// ackWorkerEvent acks the pending event that corresponds to the acked job. If -// the event was a dispatched job then it sends a dispatch return event to the -// node that dispatched the job. +// ackWorkerEvent removes the routing node's local tracking after the worker has +// durably settled the corresponding pool event. func (node *Node) ackWorkerEvent(ev *streaming.Event) { - workerID, payload := unmarshalEnvelope(ev.Payload) - ack := unmarshalAck(payload) + workerID, payload, err := unmarshalEnvelope(ev.Payload) + if err != nil { + node.dropMalformedNodeEvent(ev, fmt.Errorf("decode worker acknowledgement envelope: %w", err)) + return + } + ack, err := unmarshalAck(payload) + if err != nil { + node.dropMalformedNodeEvent(ev, fmt.Errorf("decode worker acknowledgement: %w", err)) + return + } key := pendingEventKey(workerID, ack.EventID) val, ok := node.pendingEvents.Load(key) if !ok { @@ -798,62 +1801,71 @@ func (node *Node) ackWorkerEvent(ev *streaming.Event) { } pending := val.(*streaming.Event) ctx := context.Background() + dispatchSettled := false // If a dispatched job then send a return event to the node that // dispatched the job. if pending.EventName == evStartJob { - _, nodeID := unmarshalJobKeyAndNodeID(pending.Payload) - stream, err := node.getNodeStream(nodeID) + job, err := unmarshalJob(pending.Payload) if err != nil { - node.logger.Error(fmt.Errorf("ackWorkerEvent: failed to create node event stream %q: %w", nodeStreamName(node.PoolName, nodeID), err)) + node.logger.Error(fmt.Errorf( + "ackWorkerEvent: decode pending start event %s: %w", + pending.ID, + err, + )) + if ackErr := node.settlePoolEvent(context.Background(), pending); ackErr != nil { + node.logger.Error(fmt.Errorf("ackWorkerEvent: drop malformed pending event: %w", ackErr)) + return + } + node.pendingEvents.Delete(key) return } - ack.EventID = pending.ID - if _, err := stream.Add(ctx, evDispatchReturn, marshalAck(ack), options.WithOnlyIfStreamExists()); err != nil { - node.logger.Error(fmt.Errorf("ackWorkerEvent: failed to dispatch return to stream %q: %w", nodeStreamName(node.PoolName, nodeID), err)) + if !job.Requeued { + ack.JobKey = job.Key + if job.dispatchID != "" { + var resultErr error + if ack.Error != "" { + resultErr = errors.New(ack.Error) + } + if _, err := node.settleDispatch(ctx, job.Key, job.dispatchID, resultErr); err != nil { + node.logger.Error(err) + return + } + dispatchSettled = true + } } } // Ack the sink event so it does not get redelivered. - if err := node.poolSink.Ack(ctx, pending); err != nil { + if !dispatchSettled { + err = node.settlePoolEvent(ctx, pending) + } + if err != nil { node.logger.Error(fmt.Errorf("ackWorkerEvent: failed to ack event: %w", err), "event", pending.EventName, "id", pending.ID) + return } node.pendingEvents.Delete(key) - - // Garbage collect stale events. - var staleKeys []string - node.pendingEvents.Range(func(key, value any) bool { - ev := value.(*streaming.Event) - if time.Since(ev.CreatedAt()) > pendingEventTTL { - staleKeys = append(staleKeys, key.(string)) - node.logger.Error(fmt.Errorf("ackWorkerEvent: stale event, removing from pending events"), "event", ev.EventName, "id", ev.ID, "since", time.Since(ev.CreatedAt()), "TTL", pendingEventTTL) - } - return true - }) - for _, key := range staleKeys { - node.pendingEvents.Delete(key) - } } -// returnDispatchStatus returns the start job result to the caller. -func (node *Node) returnDispatchStatus(ev *streaming.Event) { - ack := unmarshalAck(ev.Payload) - val, ok := node.pendingJobChannels.Load(ack.EventID) - if !ok { - node.logger.Error(fmt.Errorf("returnDispatchStatus: received dispatch return for unknown event"), "id", ack.EventID) - return +// settlePoolEvent acknowledges and deletes one terminal pool-stream event. +// The unbounded pool stream therefore retains only unsettled work. +func (node *Node) settlePoolEvent(ctx context.Context, event *streaming.Event) error { + if err := node.poolSink.Ack(ctx, event); err != nil { + return err } - node.logger.Debug("dispatch return", "event", ev.EventName, "id", ev.ID, "ack-id", ack.EventID) - if val == nil { - // Event was requeued, just clean up - node.pendingJobChannels.Delete(ack.EventID) - return + if err := node.poolStream.Remove(ctx, event.ID); err != nil { + return fmt.Errorf("delete settled pool event %s: %w", event.ID, err) } - var err error - if ack.Error != "" { - err = errors.New(ack.Error) + return nil +} + +// dropMalformedNodeEvent logs and removes a poison entry so the permanent node +// reader remains live across restarts. +func (node *Node) dropMalformedNodeEvent(event *streaming.Event, decodeErr error) { + node.logger.Error(decodeErr, "event", event.EventName, "id", event.ID) + if err := node.nodeStream.Remove(context.Background(), event.ID); err != nil { + node.logger.Error(fmt.Errorf("drop malformed node event %s: %w", event.ID, err)) } - val.(chan error) <- err } // workerForEvent returns the worker that should receive a pool event. Start and @@ -890,7 +1902,7 @@ func (node *Node) workerForEvent(eventName, key string) (string, error) { // jobPayloadExists reads the durable job record from Redis, which is the source // of truth when the local ownership map has no active owner during handoff. func (node *Node) jobPayloadExists(ctx context.Context, key string) (bool, error) { - exists, err := node.rdb.HExists(ctx, rmapContentKey(jobPayloadMapName(node.PoolName)), key).Result() + exists, err := node.rdb.HExists(ctx, rmapContentKey(node.resources.jobPayloads), key).Result() if err != nil { return false, fmt.Errorf("routeWorkerEvent: failed to check job payload %q: %w", key, err) } @@ -952,6 +1964,10 @@ func (node *Node) handleWorkerMapUpdate(ctx context.Context) { if node.IsClosed() { return } + if err := node.ensureGenerationActive(ctx); err != nil { + node.logger.Error(err) + return + } // First cleanup the local workers that are no longer active. node.localWorkers.Range(func(key, value any) bool { worker := value.(*Worker) @@ -962,7 +1978,10 @@ func (node *Node) handleWorkerMapUpdate(ctx context.Context) { if err := node.deleteWorker(worker.ID); err != nil { node.logger.Error(fmt.Errorf("handleWorkerMapUpdate: failed to delete inactive worker %q: %w", worker.ID, err), "worker", worker.ID) } - worker.stop(ctx) + if err := worker.stop(ctx); err != nil { + node.logger.Error(fmt.Errorf("handleWorkerMapUpdate: failed to stop inactive worker %q: %w", worker.ID, err)) + return true + } node.localWorkers.Delete(key) return true } @@ -981,39 +2000,86 @@ func (node *Node) handleWorkerMapUpdate(ctx context.Context) { }) } -// watchShutdown monitors the pool shutdown map and initiates node shutdown when updated. -func (node *Node) watchShutdown(ctx context.Context) { +// watchShutdown monitors the subscription established before node registration, +// so a shutdown concurrent with AddNode cannot be missed. +func (node *Node) watchShutdown(ctx context.Context, updates <-chan rmap.EventKind) { defer node.wg.Done() + defer node.nodeShutdownMap.Unsubscribe(updates) for { select { case <-node.stop: return - case <-node.nodeShutdownMap.Subscribe(): + case _, ok := <-updates: + if !ok { + return + } + if _, shutdown := node.nodeShutdownMap.Get("shutdown"); !shutdown { + continue + } node.logger.Debug("watchShutdown: shutdown map updated") - // Handle shutdown in a separate goroutine to allow this one to exit - pulse.Go(node.logger, func() { node.handleShutdown(ctx) }) + node.ownShutdown(ctx) } } } -// handleShutdown closes the node. +// ownShutdown starts exactly one peer-shutdown owner. +func (node *Node) ownShutdown(ctx context.Context) { + node.shutdownOnce.Do(func() { + pulse.Go(node.logger, func() { node.handleShutdown(ctx) }) + }) +} + +// handleShutdown retries local cleanup within one node lease and publishes each +// failure in the shutdown map so the initiating node can return it immediately. func (node *Node) handleShutdown(ctx context.Context) { - if node.IsClosed() { - return - } - sm := node.nodeShutdownMap.Map() - var requestingNode string - for _, node := range sm { - // There is only one value in the map - requestingNode = node - } + requestingNode, _ := node.nodeShutdownMap.Get("shutdown") node.logger.Debug("handleShutdown: shutting down", "requested-by", requestingNode) - node.close(ctx, true) + failureKey := shutdownErrorKey(node.ID) + deadline := time.NewTimer(node.workerTTL) + defer deadline.Stop() + for { + err := node.close(ctx, true) + if err == nil { + node.lock.Lock() + node.shutdown = true + node.lock.Unlock() + if err := node.rdb.HDel( + ctx, + rmapContentKey(node.resources.nodeShutdown), + failureKey, + ).Err(); err != nil { + node.logger.Error(fmt.Errorf("handleShutdown: failed to clear shutdown error: %w", err)) + if waitForShutdownRetry(ctx, deadline.C, node.workerTTL) { + continue + } + return + } + node.logger.Info("shutdown", "requested-by", requestingNode) + return + } + if setErr := node.setPoolMap(ctx, node.resources.nodeShutdown, failureKey, err.Error()); setErr != nil { + node.logger.Error(fmt.Errorf("handleShutdown: failed to publish shutdown error: %w", setErr)) + } + node.logger.Error(fmt.Errorf("handleShutdown: failed to close node: %w", err)) + if !waitForShutdownRetry(ctx, deadline.C, node.workerTTL) { + return + } + } +} - node.lock.Lock() - node.shutdown = true - node.lock.Unlock() - node.logger.Info("shutdown", "requested-by", requestingNode) +// waitForShutdownRetry spaces peer cleanup attempts while respecting the +// caller context and the node lease deadline. +func waitForShutdownRetry(ctx context.Context, deadline <-chan time.Time, ttl time.Duration) bool { + retry := time.NewTimer(min(100*time.Millisecond, ttl)) + defer retry.Stop() + select { + case <-retry.C: + return true + case <-deadline: + return false + case <-ctx.Done(): + return false + } } // processInactiveNodes periodically checks for inactive nodes and destroys their streams. @@ -1035,26 +2101,28 @@ func (node *Node) processInactiveNodes() { // cleanupInactiveNodes checks for inactive nodes, destroys their streams and // removes them from the keep-alive map. func (node *Node) cleanupInactiveNodes() { - nodeMap := node.nodeKeepAliveMap.Map() - for nodeID, lastSeen := range nodeMap { - if nodeID == node.ID || node.isWithinTTL(lastSeen, node.workerTTL) { + if err := node.ensureGenerationActive(context.Background()); err != nil { + node.logger.Error(err) + return + } + for nodeID := range node.nodeKeepAliveMap.Map() { + if nodeID == node.ID || strings.HasPrefix(nodeID, "=") { continue } - - node.logger.Info("cleaning up inactive node", "node", nodeID) - - // Clean up node's stream ctx := context.Background() - stream := nodeStreamName(node.PoolName, nodeID) - if s, err := streaming.NewStream(stream, node.rdb, options.WithStreamLogger(node.logger)); err == nil { - if err := s.Destroy(ctx); err != nil { - node.logger.Error(fmt.Errorf("cleanupInactiveNodes: failed to destroy stream: %w", err)) - } + cleaned, err := cleanupStalePoolNode( + ctx, + node.rdb, + node.resources, + nodeID, + node.ID, + ) + if err != nil { + node.logger.Error(fmt.Errorf("cleanupInactiveNodes: failed to clean node: %w", err)) + continue } - - // Remove from keep-alive map - if _, err := node.nodeKeepAliveMap.Delete(ctx, nodeID); err != nil { - node.logger.Error(fmt.Errorf("cleanupInactiveNodes: failed to delete node: %w", err)) + if cleaned { + node.logger.Info("cleaned up inactive node", "node", nodeID) } } } @@ -1087,13 +2155,12 @@ func (node *Node) processInactiveWorkers(ctx context.Context) { // lock acquisition. Jobs are requeued and will be reassigned to active workers // through consistent hashing. func (node *Node) cleanupInactiveWorkers(ctx context.Context) { - active := node.activeWorkers() - activeMap := make(map[string]struct{}) - for _, id := range active { - activeMap[id] = struct{}{} + if err := node.ensureGenerationActive(ctx); err != nil { + node.logger.Error(err) + return } - - // Get all workers that need cleanup (either in jobMap or workerMap) + // Discovery may be eventually replicated, but the cleanup decision is made + // only by acquireWorkerCleanup against the authoritative Redis heartbeat. workersToCheck := make(map[string]struct{}) for _, workerID := range node.jobMap.Keys() { workersToCheck[workerID] = struct{}{} @@ -1102,23 +2169,7 @@ func (node *Node) cleanupInactiveWorkers(ctx context.Context) { workersToCheck[workerID] = struct{}{} } - // Check each worker for workerID := range workersToCheck { - // Skip active workers - if _, ok := activeMap[workerID]; ok { - continue - } - - // Skip workers being cleaned up - if cleanupTS, exists := node.workerCleanupMap.Get(workerID); exists { - if node.isWithinTTL(cleanupTS, node.workerTTL) { - node.logger.Debug("cleanupInactiveWorkers: worker already being cleaned up", "worker", workerID) - continue - } - } - - // Worker needs cleanup - node.logger.Info("cleanupInactiveWorkers: found inactive worker", "worker", workerID) node.cleanupWorker(ctx, workerID) } @@ -1149,8 +2200,8 @@ func (node *Node) requeueOrphanedPayloads(ctx context.Context) { // Use a short grace period: we want recovery to be fast under churn, // but still avoid requeuing during brief map inconsistencies. grace := 2 * node.workerTTL - if grace < node.ackGracePeriod { - grace = node.ackGracePeriod + if grace < node.recoveryGrace { + grace = node.recoveryGrace } now := time.Now() @@ -1159,6 +2210,29 @@ func (node *Node) requeueOrphanedPayloads(ctx context.Context) { node.orphanedPayloads.Delete(key) continue } + dispatchID, err := node.activeDispatchID(ctx, key) + if err != nil { + node.logger.Error(err, "key", key) + continue + } + if dispatchID != "" { + released, err := node.releaseCrashedDispatchStart(ctx, nil, key, dispatchID) + if err != nil { + node.logger.Error(err, "key", key, "dispatch", dispatchID) + continue + } + if released { + node.orphanedPayloads.Delete(key) + node.logger.Info( + "released orphaned exact dispatch for stream recovery", + "key", + key, + "dispatch", + dispatchID, + ) + continue + } + } firstAny, ok := node.orphanedPayloads.Load(key) if !ok { @@ -1189,18 +2263,35 @@ func (node *Node) requeueOrphanedPayloads(ctx context.Context) { // cleanupWorker requeues the jobs assigned to the worker and deletes it from // the pool. func (node *Node) cleanupWorker(ctx context.Context, workerID string) { - // Try to acquire or clear stale cleanup lock - if !node.acquireCleanupLock(ctx, workerID) { + if err := node.ensureGenerationActive(ctx); err != nil { + node.logger.Error(err) + return + } + lease, err := node.acquireWorkerCleanup(ctx, workerID) + if err != nil { + node.logger.Error(fmt.Errorf("cleanupWorker: acquire lease: %w", err), "worker", workerID) + return + } + if lease == nil { return } + complete := false + defer func() { + releaseCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), time.Second) + defer cancel() + if err := node.releaseWorkerCleanup(releaseCtx, lease, complete); err != nil { + node.logger.Error(fmt.Errorf("cleanupWorker: release lease: %w", err), "worker", workerID) + } + }() // Get the worker's jobs keys, ok := node.jobMap.GetValues(workerID) if !ok || len(keys) == 0 { - // Worker has no jobs, just delete it - if err := node.deleteWorker(workerID); err != nil { + if err := node.deleteStaleWorker(ctx, lease); err != nil { node.logger.Error(fmt.Errorf("cleanupWorkerJobs: failed to delete worker: %w", err), "worker", workerID) + return } + complete = true node.logger.Info("cleaned up worker with no jobs", "worker", workerID) return } @@ -1211,16 +2302,41 @@ func (node *Node) cleanupWorker(ctx context.Context, workerID string) { processed int // jobs that were either requeued or cleaned up as stale ) for _, key := range keys { + dispatchID, err := node.activeDispatchID(ctx, key) + if err != nil { + node.logger.Error(err, "job", key, "worker", workerID) + continue + } + if dispatchID != "" { + released, err := node.releaseCrashedDispatchStart(ctx, lease, key, dispatchID) + if err != nil { + node.logger.Error(err, "job", key, "worker", workerID, "dispatch", dispatchID) + continue + } + if released { + node.logger.Info( + "released crashed exact dispatch for stream recovery", + "job", + key, + "worker", + workerID, + "dispatch", + dispatchID, + ) + processed++ + continue + } + } payload, ok := node.JobPayload(key) if !ok { - // The job key can remain in the jobs map even if the payload has already - // been removed (e.g. the job was stopped, or another node already handled - // the requeue). Treat it as a stale entry and remove it so future cleanup - // attempts don't keep looping on it. - if _, _, err := node.jobMap.RemoveValues(ctx, workerID, key); err != nil { + removed, err := node.removeStaleWorkerJob(ctx, lease, key) + if err != nil { node.logger.Error(fmt.Errorf("cleanupWorker: failed to remove stale job from jobs map: %w", err), "job", key, "worker", workerID) continue } + if !removed { + continue + } node.logger.Info("cleanupWorker: removed stale job key with missing payload", "job", key, "worker", workerID) processed++ continue @@ -1229,10 +2345,27 @@ func (node *Node) cleanupWorker(ctx context.Context, workerID string) { // Requeue by adding an event back to the pool stream. // We intentionally do not wait for the job to start (which can time out // under heavy churn) - the pool sink will retry routing until it is acked. - if _, err := node.poolStream.Add(ctx, evStartJob, marshalJob(job)); err != nil { + status, err := node.publishWorkerRequeue(ctx, lease, job) + if err != nil { node.logger.Error(fmt.Errorf("requeueWorkerJobs: failed to requeue job: %w", err), "job", job.Key, "worker", workerID) continue } + if status == 2 { + processed++ + continue + } + if status == 3 { + removed, err := node.removeStaleWorkerJob(ctx, lease, key) + if err != nil { + node.logger.Error(fmt.Errorf("cleanupWorker: failed to remove stale job from jobs map: %w", err), "job", key, "worker", workerID) + continue + } + if !removed { + continue + } + processed++ + continue + } requeued++ processed++ } @@ -1243,93 +2376,35 @@ func (node *Node) cleanupWorker(ctx context.Context, workerID string) { // Delete worker node.logger.Info("cleaned up worker", "worker", workerID, "requeued", requeued) - if err := node.deleteWorker(workerID); err != nil { + if err := node.deleteStaleWorker(ctx, lease); err != nil { node.logger.Error(fmt.Errorf("cleanupWorkerJobs: failed to delete worker: %w", err), "worker", workerID) + return } + complete = true } -// processInactiveJobs periodically checks for and removes stale entries in the pending jobs map. -func (node *Node) processInactiveJobs(ctx context.Context) { - defer node.wg.Done() - ticker := time.NewTicker(node.ackGracePeriod) // Run at ackGracePeriod frequency since pending jobs expire after 2*ackGracePeriod - defer ticker.Stop() - - for { - select { - case <-node.stop: - return - case <-ticker.C: - node.cleanupStalePendingJobs(ctx) - } - } -} - -// cleanupStalePendingJobs checks for and removes stale entries in the pending jobs map. -// An entry is considered stale if its timestamp has expired. -func (node *Node) cleanupStalePendingJobs(ctx context.Context) { - for key, pendingTS := range node.jobPendingMap.Map() { - if _, err := strconv.ParseInt(pendingTS, 10, 64); err != nil { - node.logger.Error(fmt.Errorf("cleanupStalePendingJobs: malformed pending timestamp for job %q: %w", key, err)) - continue - } - if node.isWithinTTL(pendingTS, 0) { - continue - } - prev, err := node.jobPendingMap.TestAndDelete(ctx, key, pendingTS) - if err != nil { - node.logger.Error(fmt.Errorf("cleanupStalePendingJobs: failed to delete stale pending entry: %w", err)) - } - if prev == pendingTS { - node.logger.Info("cleanupStalePendingJobs: removed stale pending entry", "key", key) - } - } -} - -// acquireCleanupLock tries to acquire the cleanup lock for a worker. -// It returns true if the lock was acquired, false if another node holds the lock. -// It will clear any stale or invalid locks it finds. -func (node *Node) acquireCleanupLock(ctx context.Context, workerID string) bool { - // Check for existing lock - if existingTS, exists := node.workerCleanupMap.Get(workerID); exists { - if !node.isWithinTTL(existingTS, node.workerTTL) { - // Invalid or stale lock, delete it - if _, err := node.workerCleanupMap.Delete(ctx, workerID); err != nil { - node.logger.Error(fmt.Errorf("cleanupWorkerJobs: failed to delete stale cleanup timestamp: %w", err), "worker", workerID) - return false - } - node.logger.Info("cleanupWorkerJobs: cleared stale cleanup lock", "worker", workerID, "ts", existingTS, "ttl", node.workerTTL) - } else { - // Lock is still valid - node.logger.Debug("cleanupWorkerJobs: cleanup already in progress", "worker", workerID) - return false - } - } - - // Try to acquire lock - now := strconv.FormatInt(time.Now().UnixNano(), 10) - ok, err := node.workerCleanupMap.SetIfNotExists(ctx, workerID, now) +// isWithinTTL checks if a timestamp is within a TTL. If lastSeen is not a valid +// timestamp, false is returned. lastSeen is a string representation of a unix +// timestamp in nanoseconds. +func (node *Node) isWithinTTL(lastSeen string, ttl time.Duration) bool { + now, err := node.rdb.Time(context.Background()).Result() if err != nil { - node.logger.Error(fmt.Errorf("cleanupWorkerJobs: failed to set cleanup timestamp: %w", err), "worker", workerID) + node.logger.Error(fmt.Errorf("isWithinTTL: failed to read Redis time: %w", err)) return false } - if !ok { - node.logger.Debug("cleanupWorkerJobs: cleanup already in progress", "worker", workerID) - return false - } - - return true + return node.isWithinTTLAt(lastSeen, ttl, now) } -// isWithinTTL checks if a timestamp is within a TTL. If lastSeen is not a valid -// timestamp, false is returned. lastSeen is a string representation of a unix -// timestamp in nanoseconds. -func (node *Node) isWithinTTL(lastSeen string, ttl time.Duration) bool { +// isWithinTTLAt compares one persisted timestamp to an already-read Redis +// clock value so callers evaluating a set do not issue one TIME command per +// member. +func (node *Node) isWithinTTLAt(lastSeen string, ttl time.Duration, now time.Time) bool { lsi, err := strconv.ParseInt(lastSeen, 10, 64) if err != nil { node.logger.Error(fmt.Errorf("isWithinTTL: failed to parse last seen timestamp: %w", err)) return false } - return time.Since(time.Unix(0, lsi)) <= ttl + return now.Sub(time.Unix(0, lsi)) <= ttl } // Keep node alive @@ -1344,9 +2419,18 @@ func (node *Node) updateNodeKeepAlive() { case <-node.stop: return case <-ticker.C: - if _, err := node.nodeKeepAliveMap.Set(ctx, node.ID, - strconv.FormatInt(time.Now().UnixNano(), 10)); err != nil { + if err := node.ensureGenerationActive(ctx); err != nil { + node.logger.Error(err) + return + } + _, err := refreshPoolNode(ctx, node.rdb, node.resources, node.ID) + if err != nil { node.logger.Error(fmt.Errorf("updateNodeKeepAlive: failed to update timestamp: %w", err)) + if strings.Contains(err.Error(), "NODECLEANUPLOST") { + node.stopAfterLifecycleLoss("stale node cleanup fence", false) + return + } + continue } } } @@ -1354,6 +2438,11 @@ func (node *Node) updateNodeKeepAlive() { // activeWorkers returns the IDs of the active workers in the pool. func (node *Node) activeWorkers() []string { + now, err := node.rdb.Time(context.Background()).Result() + if err != nil { + node.logger.Error(fmt.Errorf("activeWorkers: failed to read Redis time: %w", err)) + return nil + } workers := node.workerMap.Map() workerCreatedAtByID := make(map[string]int64) var sortedIDs []string @@ -1362,10 +2451,15 @@ func (node *Node) activeWorkers() []string { continue // worker is in the process of being removed } - // Skip workers that are being cleaned up - if cleanupTS, exists := node.workerCleanupMap.Get(id); exists { - if node.isWithinTTL(cleanupTS, node.workerTTL) { - continue // Skip workers being actively cleaned up + // Skip workers under an exact unexpired Redis-time cleanup lease. + if cleanupLease, exists := node.workerCleanupMap.Get(id); exists { + active, err := workerCleanupLeaseActive(cleanupLease, now) + if err != nil { + node.logger.Error(err, "worker", id) + continue + } + if active { + continue } } cai, err := strconv.ParseInt(createdAt, 10, 64) @@ -1391,7 +2485,7 @@ func (node *Node) activeWorkers() []string { // the workers map deletion. continue } - if !node.isWithinTTL(ls, node.workerTTL) { + if !node.isWithinTTLAt(ls, node.workerTTL, now) { continue } activeIDs = append(activeIDs, id) @@ -1405,38 +2499,39 @@ func (node *Node) deleteWorker(id string) error { ctx := context.Background() node.logger.Debug("deleteWorker: deleting worker", "worker", id) - // Remove from all maps including cleanup map - node.removeWorkerFromMaps(ctx, id) - - // Destroy the worker's stream + // Destroy before removing the records that make failed cleanup discoverable. stream, err := node.getWorkerStream(id) if err != nil { return fmt.Errorf("deleteWorker: failed to retrieve worker stream for %q: %w", id, err) } if err := stream.Destroy(ctx); err != nil { - node.logger.Error(fmt.Errorf("deleteWorker: failed to delete worker stream: %w", err)) + return fmt.Errorf("deleteWorker: failed to delete worker stream: %w", err) } - return nil + return node.removeWorkerFromMaps(ctx, id) } // removeWorker removes a worker that was created by this node. // This is used during graceful shutdown or explicit worker removal. -func (node *Node) removeWorker(ctx context.Context, id string) { - node.removeWorkerFromMaps(ctx, id) +func (node *Node) removeWorker(ctx context.Context, id string) error { + if err := node.removeWorkerFromMaps(ctx, id); err != nil { + return err + } node.workerStreams.Delete(id) + return nil } // removeWorkerFromMaps removes the worker from all tracking maps. // This is the common cleanup needed for both local and remote worker removal. -func (node *Node) removeWorkerFromMaps(ctx context.Context, id string) { - if _, err := node.workerMap.Delete(ctx, id); err != nil { - node.logger.Error(fmt.Errorf("removeWorkerFromMaps: failed to remove worker %s from worker map: %w", id, err)) +func (node *Node) removeWorkerFromMaps(ctx context.Context, id string) error { + var cleanupErr error + if err := node.deletePoolMap(ctx, node.resources.workers, id); err != nil { + cleanupErr = errors.Join(cleanupErr, fmt.Errorf("remove worker %s from worker map: %w", id, err)) } - if _, err := node.workerKeepAliveMap.Delete(ctx, id); err != nil { - node.logger.Error(fmt.Errorf("removeWorkerFromMaps: failed to remove worker %s from keep-alive map: %w", id, err)) + if err := node.deletePoolMap(ctx, node.resources.workerKeepAlive, id); err != nil { + cleanupErr = errors.Join(cleanupErr, fmt.Errorf("remove worker %s from keep-alive map: %w", id, err)) } - if _, err := node.workerCleanupMap.Delete(ctx, id); err != nil { - node.logger.Error(fmt.Errorf("removeWorkerFromMaps: failed to remove cleanup timestamp: %w", err), "worker", id) + if err := node.deletePoolMap(ctx, node.resources.workerCleanup, id); err != nil { + cleanupErr = errors.Join(cleanupErr, fmt.Errorf("remove worker %s cleanup timestamp: %w", id, err)) } // NOTE: Do not delete job payloads here. // @@ -1447,9 +2542,10 @@ func (node *Node) removeWorkerFromMaps(ctx context.Context, id string) { // // Payloads are deleted when jobs stop (see Worker.stopJob) and any remaining // orphaned payloads are eventually collected by cleanupOrphanedJobPayloads. - if _, err := node.jobMap.Delete(ctx, id); err != nil { - node.logger.Error(fmt.Errorf("removeWorkerFromMaps: failed to remove worker %s from jobs map: %w", id, err)) + if err := node.deletePoolMap(ctx, node.resources.jobs, id); err != nil { + cleanupErr = errors.Join(cleanupErr, fmt.Errorf("remove worker %s from jobs map: %w", id, err)) } + return cleanupErr } // getWorkerStream retrieves the stream for a worker. It caches the result in the @@ -1513,30 +2609,154 @@ func (node *Node) requeueAllJobs(ctx context.Context) error { return nil } -// cleanupPool removes the pool resources from Redis. -func (node *Node) cleanupPool(ctx context.Context) { - for _, m := range node.maps() { - if m != nil { - if err := m.Destroy(ctx); err != nil { - node.logger.Error(fmt.Errorf("cleanupPool: failed to destroy map: %w", err)) +// cleanupPool completes the takeover-owned cleanup claimed by +// waitForPoolNodes. The persisted owner and lease make every destructive step +// retryable by another process after interruption. +func (node *Node) cleanupPool(ctx context.Context) error { + complete, err := node.poolCleanupComplete(ctx) + if err != nil { + return err + } + if complete { + return nil + } + err = cleanupPoolResources( + ctx, + node.rdb, + node.PoolName, + node.poolStream.Generation(), + node.ID, + node.cleanupLease, + ) + if err != nil { + return fmt.Errorf("cleanupPool: %w", err) + } + return nil +} + +// waitForPoolNodes reaps crashed-node leases and blocks until every live node +// detaches. Redis TIME is the sole lease clock, so host clock skew cannot hold +// or prematurely pass the destructive cleanup barrier. +func (node *Node) waitForPoolNodes(ctx context.Context) error { + nodes, err := rmap.Join( + ctx, + node.resources.nodeKeepAlive, + node.rdb, + rmap.WithLogger(node.logger), + ) + if err != nil { + return fmt.Errorf("Shutdown: failed to join node shutdown barrier: %w", err) + } + defer nodes.Close() + updates := nodes.Subscribe() + defer nodes.Unsubscribe(updates) + for { + now, err := node.rdb.Time(ctx).Result() + if err != nil { + return fmt.Errorf("Shutdown: failed to read Redis time for node barrier: %w", err) + } + shutdownState, err := node.rdb.HGetAll( + ctx, + rmapContentKey(node.resources.nodeShutdown), + ).Result() + if err != nil { + return fmt.Errorf("Shutdown: failed to read peer shutdown state: %w", err) + } + nextExpiry := node.workerTTL + active := 0 + activeNodes := make(map[string]struct{}) + heartbeats, err := node.rdb.HGetAll( + ctx, + rmapContentKey(node.resources.nodeKeepAlive), + ).Result() + if err != nil { + return fmt.Errorf("Shutdown: failed to read authoritative node heartbeats: %w", err) + } + for nodeID, timestamp := range heartbeats { + if strings.HasPrefix(nodeID, "=") { + continue } + lastSeen, err := strconv.ParseInt(timestamp, 10, 64) + if err != nil { + return fmt.Errorf("Shutdown: invalid node keep-alive for %q: %w", nodeID, err) + } + remaining := node.workerTTL - now.Sub(time.Unix(0, lastSeen)) + if remaining <= 0 { + cleaned, err := cleanupStalePoolNode( + ctx, + node.rdb, + node.resources, + nodeID, + node.ID, + ) + if err != nil { + return fmt.Errorf("Shutdown: failed to reap stale node %q: %w", nodeID, err) + } + if cleaned { + continue + } + remaining = shutdownErrorPoll + } + active++ + activeNodes[nodeID] = struct{}{} + nextExpiry = min(nextExpiry, remaining) + } + for key, message := range shutdownState { + if !strings.HasPrefix(key, "error:") { + continue + } + nodeID := strings.TrimPrefix(key, "error:") + if _, active := activeNodes[nodeID]; !active { + continue + } + return fmt.Errorf("Shutdown: node %q failed to close: %s", nodeID, message) + } + if active == 0 { + status, err := claimPoolCleanup( + ctx, + node.rdb, + node.PoolName, + node.poolStream.Generation(), + node.ID, + node.cleanupLease, + ) + if err != nil { + return fmt.Errorf("Shutdown: failed to claim pool cleanup: %w", err) + } + if status == poolCleanupClaimed || status == poolCleanupAlreadyComplete { + return nil + } + active = 1 + } + nextExpiry = min(nextExpiry, shutdownErrorPoll) + timer := time.NewTimer(nextExpiry) + select { + case <-updates: + if !timer.Stop() { + <-timer.C + } + case <-timer.C: + case <-ctx.Done(): + if !timer.Stop() { + <-timer.C + } + return fmt.Errorf("Shutdown: waiting for pool nodes to close: %w", ctx.Err()) } - } - if err := node.poolStream.Destroy(ctx); err != nil { - node.logger.Error(fmt.Errorf("cleanupPool: failed to destroy pool stream: %w", err)) } } -// cleanupNode closes the node resources. -func (node *Node) cleanupNode(ctx context.Context) { +// cleanupNode destroys the node stream before closing maps. A destruction +// failure leaves map handles open and is returned so Close can be retried. +func (node *Node) cleanupNode(ctx context.Context) error { + if err := node.nodeStream.Destroy(ctx); err != nil { + return fmt.Errorf("failed to destroy node stream: %w", err) + } for _, m := range node.maps() { if m != nil { m.Close() } } - if err := node.nodeStream.Destroy(ctx); err != nil { - node.logger.Error(fmt.Errorf("cleanupNode: failed to destroy node stream: %w", err)) - } + return nil } // maps returns the maps managed by the node. @@ -1551,6 +2771,7 @@ func (node *Node) maps() []*rmap.Map { node.jobPendingMap, node.jobPayloadMap, node.tickerMap, + node.schedulerJobMap, } } @@ -1595,6 +2816,16 @@ func nodeShutdownMapName(pool string) string { return fmt.Sprintf("%s:shutdown", pool) } +// shutdownErrorKey identifies one peer's authoritative close failure. +func shutdownErrorKey(nodeID string) string { + return fmt.Sprintf("error:%s", nodeID) +} + +// poolCleanupGenerationsKey records completed pool-stream generations. +func poolCleanupGenerationsKey(pool string) string { + return fmt.Sprintf("pulse:pool:%s:cleanup-generations", pool) +} + // workerMapName returns the name of the replicated map used to store the // worker creation timestamps. func workerMapName(pool string) string { @@ -1625,6 +2856,11 @@ func jobPendingMapName(poolName string) string { return poolName + ":pending-jobs" } +// dispatchMapName returns the generation-owned durable dispatch record map. +func dispatchMapName(pool string) string { + return fmt.Sprintf("%s:dispatches", pool) +} + // jobPayloadMapName returns the name of the replicated map used to store the // job payloads by job key. func jobPayloadMapName(pool string) string { @@ -1647,6 +2883,11 @@ func tickerMapName(pool string) string { return fmt.Sprintf("%s:tickers", pool) } +// schedulerJobMapName returns the pre-generation scheduler ownership map name. +func schedulerJobMapName(pool string) string { + return fmt.Sprintf("%s:scheduler-jobs", pool) +} + // poolStreamName returns the name of the stream used by pool events. func poolStreamName(pool string) string { return fmt.Sprintf("%s:pool", pool) diff --git a/pool/node_cleanup.go b/pool/node_cleanup.go new file mode 100644 index 0000000..076ca03 --- /dev/null +++ b/pool/node_cleanup.go @@ -0,0 +1,231 @@ +// Stale-node cleanup uses the node keep-alive map as one Redis-owned lease +// record. Heartbeat expiry, cleanup fencing, stream destruction, and discovery +// removal are decided by Redis TIME and exact owner tokens. +package pool + +import ( + "context" + "encoding/hex" + "errors" + "fmt" + "strconv" + + "github.com/oklog/ulid/v2" + redis "github.com/redis/go-redis/v9" + + "goa.design/pulse/streaming" +) + +type ( + // nodeCleanupLease is the exact stale-node cleanup capability. + nodeCleanupLease struct { + nodeID string + owner string + fence string + } +) + +var ( + // acquireNodeCleanupScript atomically verifies heartbeat expiry and installs + // one renewable cleanup fence using Redis TIME. + acquireNodeCleanupScript = redis.NewScript(` +if redis.call("HGET", KEYS[1], "state") ~= ARGV[1] +or redis.call("HGET", KEYS[1], "generation") ~= ARGV[2] then + return redis.error_reply("POOLGENERATIONLOST") +end +local clock = redis.call("TIME") +local now = tonumber(clock[1]) * 1000 + math.floor(tonumber(clock[2]) / 1000) +local now_ns = tonumber(clock[1]) * 1000000000 + tonumber(clock[2]) * 1000 +local heartbeat = redis.call("HGET", KEYS[2], ARGV[3]) +if heartbeat then + local heartbeat_ns = tonumber(heartbeat) + if not heartbeat_ns then + return redis.error_reply("NODEHEARTBEATINVALID") + end + if heartbeat_ns + tonumber(ARGV[6]) >= now_ns then + return {0, "live"} + end +end +local current = redis.call("HGET", KEYS[2], ARGV[7]) +if current then + if string.sub(current, 1, 5) == "dead|" then + return {0, "dead"} + end + local current_owner, current_fence, current_until = + string.match(current, "^([^|]+)|([^|]+)|(%d+)$") + if not current_owner then + return redis.error_reply("NODECLEANUPINVALID") + end + if current_owner ~= ARGV[4] and tonumber(current_until) > now then + return {0, current_fence} + end + if current_owner == ARGV[4] and tonumber(current_until) > now then + redis.call("HSET", KEYS[2], ARGV[7], + current_owner .. "|" .. current_fence .. "|" .. tostring(now + tonumber(ARGV[5]))) + return {1, current_fence} + end +end +local fence = tostring(redis.call("HINCRBY", KEYS[2], ARGV[8], 1)) +redis.call("HSET", KEYS[2], ARGV[7], + ARGV[4] .. "|" .. fence .. "|" .. tostring(now + tonumber(ARGV[5]))) +return {1, fence} +`) + + // destroyStaleNodeScript destroys one exact node stream and removes its + // heartbeat only while the supplied cleanup owner remains current. + destroyStaleNodeScript = redis.NewScript(` +if redis.call("HGET", KEYS[1], "state") ~= ARGV[1] +or redis.call("HGET", KEYS[1], "generation") ~= ARGV[2] then + return redis.error_reply("POOLGENERATIONLOST") +end +local clock = redis.call("TIME") +local now = tonumber(clock[1]) * 1000 + math.floor(tonumber(clock[2]) / 1000) +local current = redis.call("HGET", KEYS[2], ARGV[7]) +local owner, fence, lease_until = string.match(current or "", "^([^|]+)|([^|]+)|(%d+)$") +if owner ~= ARGV[4] or fence ~= ARGV[5] or tonumber(lease_until or "0") <= now then + return redis.error_reply("NODECLEANUPLOST") +end +redis.call("HSET", KEYS[2], ARGV[7], + owner .. "|" .. fence .. "|" .. tostring(now + tonumber(ARGV[6]))) + +if redis.call("HGET", KEYS[4], "generation") == ARGV[9] then + local state = redis.call("HGET", KEYS[4], "state") + if state == "active" then + local physical = redis.call("HGET", KEYS[4], "physical_key") + if not physical then + return redis.error_reply("STREAMDESTROYED") + end + redis.call("HSET", KEYS[4], "state", "destroyed") + local resources = redis.call("SMEMBERS", KEYS[5]) + if #resources > 0 then + redis.call("DEL", unpack(resources)) + end + redis.call("DEL", physical, physical .. ":sink-recovery:" .. ARGV[9], KEYS[5]) + elseif state ~= "destroyed" then + return redis.error_reply("STREAMDESTROYED") + end +end + +redis.call("HDEL", KEYS[2], ARGV[3]) +redis.call("HSET", KEYS[2], ARGV[7], "dead|" .. ARGV[5] .. "|0") +local rev = tostring(redis.call("HINCRBY", KEYS[2], "=rev", 1)) +redis.call("HSET", KEYS[2], "=kind", "del") +local message = struct.pack( + "ic0ic0", + string.len(ARGV[3]), ARGV[3], + string.len(rev), rev +) +redis.call("PUBLISH", KEYS[3], "del:" .. message) +redis.call("HDEL", KEYS[6], "error:" .. ARGV[3]) +return 1 +`) +) + +// acquireNodeCleanup returns an exact cleanup lease only when Redis proves the +// node heartbeat expired by the generation's immutable WorkerTTL. +func acquireNodeCleanup( + ctx context.Context, + rdb *redis.Client, + resources poolResources, + nodeID, owner string, +) (*nodeCleanupLease, error) { + raw, err := acquireNodeCleanupScript.Run( + ctx, + rdb, + []string{ + fmt.Sprintf("pulse:stream:%s:lifecycle", poolStreamName(resources.pool)), + rmapContentKey(resources.nodeKeepAlive), + }, + "active", + resources.generation, + nodeID, + owner, + strconv.FormatInt(resources.cleanupLease.Milliseconds(), 10), + strconv.FormatInt(resources.workerTTL.Nanoseconds(), 10), + nodeCleanupField(nodeID), + nodeCleanupFenceField(nodeID), + ).Slice() + if err != nil { + return nil, poolBoundaryError(err) + } + if len(raw) != 2 { + return nil, fmt.Errorf("node cleanup claim returned %d fields", len(raw)) + } + status, ok := raw[0].(int64) + if !ok || (status != 0 && status != 1) { + return nil, fmt.Errorf("node cleanup claim returned invalid status %T(%v)", raw[0], raw[0]) + } + if status == 0 { + return nil, nil + } + fence, ok := raw[1].(string) + if !ok || fence == "" { + return nil, fmt.Errorf("node cleanup claim returned invalid fence %T", raw[1]) + } + return &nodeCleanupLease{nodeID: nodeID, owner: owner, fence: fence}, nil +} + +// cleanupStalePoolNode atomically fences a stale node before destroying its +// exact stream incarnation and removing discovery. +func cleanupStalePoolNode( + ctx context.Context, + rdb *redis.Client, + resources poolResources, + nodeID, owner string, +) (bool, error) { + lease, err := acquireNodeCleanup(ctx, rdb, resources, nodeID, owner) + if err != nil || lease == nil { + return false, err + } + stream, err := streaming.NewStream( + nodeStreamName(resources.pool, nodeID), + rdb, + ) + if err != nil { + return false, err + } + if err := stream.Open(ctx); err != nil && !errors.Is(err, streaming.ErrStreamNotFound) { + return false, err + } + generation := stream.Generation() + err = destroyStaleNodeScript.Run( + ctx, + rdb, + []string{ + fmt.Sprintf("pulse:stream:%s:lifecycle", poolStreamName(resources.pool)), + rmapContentKey(resources.nodeKeepAlive), + rmapUpdateChannel(resources.nodeKeepAlive), + fmt.Sprintf("pulse:stream:%s:lifecycle", stream.Name), + fmt.Sprintf("pulse:stream:%s:generation:%s:resources", stream.Name, generation), + rmapContentKey(resources.nodeShutdown), + }, + "active", + resources.generation, + nodeID, + lease.owner, + lease.fence, + strconv.FormatInt(resources.cleanupLease.Milliseconds(), 10), + nodeCleanupField(nodeID), + nodeCleanupFenceField(nodeID), + generation, + ).Err() + if err != nil { + return false, poolBoundaryError(err) + } + return true, nil +} + +// newNodeCleanupOwner returns a process-unique cleanup owner token. +func newNodeCleanupOwner(prefix string) string { + return prefix + "-" + ulid.Make().String() +} + +// nodeCleanupField stores the terminal cleanup fence beside the heartbeat. +func nodeCleanupField(nodeID string) string { + return "=node-cleanup:" + hex.EncodeToString([]byte(nodeID)) +} + +// nodeCleanupFenceField stores the monotonic ABA counter for one node ID. +func nodeCleanupFenceField(nodeID string) string { + return "=node-cleanup-fence:" + hex.EncodeToString([]byte(nodeID)) +} diff --git a/pool/node_cleanup_test.go b/pool/node_cleanup_test.go new file mode 100644 index 0000000..7503297 --- /dev/null +++ b/pool/node_cleanup_test.go @@ -0,0 +1,147 @@ +// Node cleanup tests prove Redis heartbeat/fence linearization. +package pool + +import ( + "context" + "encoding/hex" + "fmt" + "strconv" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "goa.design/pulse/pulse" + "goa.design/pulse/streaming" + ptesting "goa.design/pulse/testing" +) + +func TestNodeCleanupAcquisitionFencesResumedNode(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + node := newTestNode(t, ctx, rdb, t.Name()) + staleID := "paused-node" + staleStream, err := streaming.NewStream(nodeStreamName(node.PoolName, staleID), rdb) + require.NoError(t, err) + _, err = staleStream.Add(ctx, evInit, []byte(staleID)) + require.NoError(t, err) + + registered, _, err := registerPoolNode(ctx, rdb, node.resources, staleID) + require.NoError(t, err) + require.True(t, registered) + lease, err := acquireNodeCleanup( + ctx, + rdb, + node.resources, + staleID, + newNodeCleanupOwner(node.ID), + ) + require.NoError(t, err) + require.Nil(t, lease, "a heartbeat committed before acquisition must win") + + redisNow, err := rdb.Time(ctx).Result() + require.NoError(t, err) + require.NoError(t, rdb.HSet( + ctx, + rmapContentKey(node.resources.nodeKeepAlive), + staleID, + strconv.FormatInt(redisNow.Add(-2*node.workerTTL).UnixNano(), 10), + ).Err()) + lease, err = acquireNodeCleanup( + ctx, + rdb, + node.resources, + staleID, + newNodeCleanupOwner(node.ID), + ) + require.NoError(t, err) + require.NotNil(t, lease) + + _, _, err = registerPoolNode(ctx, rdb, node.resources, staleID) + require.ErrorContains(t, err, "NODECLEANUPLOST") + require.NoError(t, rdb.HSet( + ctx, + rmapContentKey(node.resources.nodeKeepAlive), + nodeCleanupField(staleID), + fmt.Sprintf("%s|%s|0", lease.owner, lease.fence), + ).Err()) + cleaned, err := cleanupStalePoolNode( + ctx, + rdb, + node.resources, + staleID, + newNodeCleanupOwner(node.ID), + ) + require.NoError(t, err) + require.True(t, cleaned) + require.False(t, rdb.HExists( + ctx, + rmapContentKey(node.resources.nodeKeepAlive), + staleID, + ).Val()) + _, err = staleStream.Add(ctx, evInit, []byte("resumed")) + require.ErrorIs(t, err, streaming.ErrStreamDestroyed) + require.NoError(t, node.Shutdown(context.Background())) +} + +func TestStaleNodeCannotDispatchOrScheduleAfterCleanup(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + node := newTestNode(t, ctx, rdb, t.Name()) + + // A live registered node admits dispatches. + record, err := node.publishDispatchRecord( + ctx, "job-live", "dispatch-live", []byte("payload"), marshalJob(&Job{ + Key: "job-live", + Payload: []byte("payload"), + CreatedAt: time.Now(), + NodeID: node.ID, + }), + ) + require.NoError(t, err) + require.Equal(t, dispatchClaimed, record.status) + + sched := &scheduler{ + name: t.Name(), + interval: time.Second, + node: node, + transitionPrefix: "=transition:" + hex.EncodeToString([]byte(t.Name())) + ":", + owner: "test-transition-owner", + lease: node.workerTTL, + logger: pulse.NoopLogger(), + } + _, err = sched.claimTransition(ctx) + require.NoError(t, err) + + // An installed node-cleanup fence rejects dispatch and scheduling. + require.NoError(t, rdb.HSet( + ctx, + rmapContentKey(node.resources.nodeKeepAlive), + nodeCleanupField(node.ID), + "owner|1|0", + ).Err()) + _, err = node.publishDispatchRecord( + ctx, "job-fenced", "dispatch-fenced", []byte("payload"), []byte("event"), + ) + require.ErrorContains(t, err, "NODECLEANUPLOST") + _, err = sched.claimTransition(ctx) + require.ErrorContains(t, err, "NODECLEANUPLOST") + + // A node whose heartbeat registration was removed is equally fenced. + require.NoError(t, rdb.HDel(ctx, rmapContentKey(node.resources.nodeKeepAlive), nodeCleanupField(node.ID)).Err()) + heartbeat := rdb.HGet(ctx, rmapContentKey(node.resources.nodeKeepAlive), node.ID).Val() + require.NotEmpty(t, heartbeat) + require.NoError(t, rdb.HDel(ctx, rmapContentKey(node.resources.nodeKeepAlive), node.ID).Err()) + _, err = node.publishDispatchRecord( + ctx, "job-removed", "dispatch-removed", []byte("payload"), []byte("event"), + ) + require.ErrorContains(t, err, "NODECLEANUPLOST") + _, err = sched.claimTransition(ctx) + require.ErrorContains(t, err, "NODECLEANUPLOST") + + // Restore liveness so shutdown completes normally. + require.NoError(t, rdb.HSet(ctx, rmapContentKey(node.resources.nodeKeepAlive), node.ID, heartbeat).Err()) + require.NoError(t, node.Shutdown(context.Background())) +} diff --git a/pool/node_jobloss_test.go b/pool/node_jobloss_test.go index a0eedce..332e96a 100644 --- a/pool/node_jobloss_test.go +++ b/pool/node_jobloss_test.go @@ -39,12 +39,17 @@ func TestJobLossDuringConcurrentWorkerCleanup(t *testing.T) { // asynchronous cleanup. We only require the DB to be flushed at the end. defer ptesting.CleanupRedis(t, rdb, false, testName) - // Use aggressive TTLs to make the race easier to hit on buggy implementations. + // Node closure drives the cleanup race deterministically. Keep leases long + // enough that scheduler delay cannot make a healthy worker look stale before + // the test begins the cascade. opts := []NodeOption{ - WithWorkerTTL(200 * time.Millisecond), - WithWorkerShutdownTTL(200 * time.Millisecond), + WithWorkerTTL(10 * time.Second), + WithRequeueTimeout(10 * time.Second), WithJobSinkBlockDuration(50 * time.Millisecond), - WithAckGracePeriod(300 * time.Millisecond), + // Dispatch acknowledgements are not part of the stale-worker timing + // under test and need enough headroom for concurrent repository runs. + WithDispatchTimeout(16 * time.Second), + WithRecoveryGrace(8 * time.Second), // Keep logs quiet so failures are easy to see (and tool output isn't truncated). WithLogger(pulse.NoopLogger()), } @@ -148,12 +153,16 @@ func TestJobLossDuringConcurrentWorkerCleanup(t *testing.T) { // Phase 2: crash node1. fmt.Printf("\n=== PHASE 2: closing node1 (%s) ===\n", worker1.ID) - go func() { _ = node1.Close(ctx) }() + node1Closed := make(chan error, 1) + go func() { + node1Closed <- node1.Close(ctx) + }() time.Sleep(50 * time.Millisecond) // Monitor requeues to node2 and crash node2 mid-requeue. uniqueOnNode2 := make(map[string]bool) crashed2 := false + node2Closed := make(chan error, 1) requeueTimeout := time.After(2 * time.Second) for { select { @@ -163,7 +172,9 @@ func TestJobLossDuringConcurrentWorkerCleanup(t *testing.T) { if len(uniqueOnNode2) >= 1 && !crashed2 { crashed2 = true fmt.Printf("\n=== PHASE 3: closing node2 during requeue (%s) ===\n", worker2.ID) - go func() { _ = node2.Close(ctx) }() + go func() { + node2Closed <- node2.Close(ctx) + }() time.Sleep(50 * time.Millisecond) } case <-requeueTimeout: @@ -176,7 +187,7 @@ phase4: // expires. The test stresses background cleanup, so the correctness condition // is eventual recovery rather than a fixed short sleep. jobsOnNode3 := make(map[string]bool) - recoveryTimeout := time.After(10 * time.Second) + recoveryTimeout := time.After(30 * time.Second) for len(jobsOnNode3) < numJobs { select { case key := <-jobRequeuedToNode3: @@ -207,12 +218,15 @@ done: } } - // Cleanup (best-effort). - shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + require.NoError(t, <-node1Closed) + if crashed2 { + require.NoError(t, <-node2Closed) + } + + // Cleanup. + shutdownCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() - _ = node3.Shutdown(shutdownCtx) - _ = node1.Shutdown(shutdownCtx) - _ = node2.Shutdown(shutdownCtx) + require.NoError(t, node3.Shutdown(shutdownCtx)) assert.Equal(t, numJobs, len(jobsOnNode3), "BUG: Not all jobs made it to node3 after cascading failures; missing=%d", len(missing)) diff --git a/pool/node_options.go b/pool/node_options.go index 77ba37d..e01ce68 100644 --- a/pool/node_options.go +++ b/pool/node_options.go @@ -1,6 +1,7 @@ package pool import ( + "fmt" "time" "goa.design/pulse/pulse" @@ -11,33 +12,44 @@ type ( NodeOption func(*nodeOptions) nodeOptions struct { - workerTTL time.Duration - workerShutdownTTL time.Duration - maxQueuedJobs int - clientOnly bool - jobSinkBlockDuration time.Duration - ackGracePeriod time.Duration - logger pulse.Logger + workerTTL time.Duration + requeueTimeout time.Duration + maxQueuedJobs int + clientOnly bool + jobSinkBlockDuration time.Duration + dispatchTimeout time.Duration + dispatchResultRetention time.Duration + recoveryGrace time.Duration + cleanupLease time.Duration + logger pulse.Logger } ) -// WithWorkerTTL sets the duration after which the worker is removed from the pool in -// case of network partitioning. The default is 10s. A lower number causes more -// frequent keep-alive updates from all workers. +// WithWorkerTTL sets the immutable pool-generation threshold for stale workers +// and nodes. Every node must configure the same value before joining. The +// default is 30s. A lower number causes more frequent keep-alive updates. func WithWorkerTTL(ttl time.Duration) NodeOption { return func(o *nodeOptions) { o.workerTTL = ttl } } -// WithWorkerShutdownTTL sets the maximum time to wait for workers to -// shutdown. The default is 2 minutes. -func WithWorkerShutdownTTL(ttl time.Duration) NodeOption { +// WithRequeueTimeout sets the timeout for one local worker's concurrent +// requeue handoff attempt during graceful removal or Close. It does not control +// remote stale-worker detection or cleanup ownership; those use WithWorkerTTL +// and a renewable Redis-time lease. The default is 2 minutes. +func WithRequeueTimeout(timeout time.Duration) NodeOption { return func(o *nodeOptions) { - o.workerShutdownTTL = ttl + o.requeueTimeout = timeout } } +// WithWorkerShutdownTTL is the deprecated v1 name for WithRequeueTimeout. +// Deprecated: use WithRequeueTimeout. +func WithWorkerShutdownTTL(ttl time.Duration) NodeOption { + return WithRequeueTimeout(ttl) +} + // WithJobSinkBlockDuration sets the duration to block when reading from the // job stream. The default is 5s. This option is mostly useful for testing. func WithJobSinkBlockDuration(d time.Duration) NodeOption { @@ -63,11 +75,43 @@ func WithClientOnly() NodeOption { } } -// WithAckGracePeriod sets the duration after which a job is made available to -// other workers if it wasn't started. The default is 20s. -func WithAckGracePeriod(ttl time.Duration) NodeOption { +// WithDispatchTimeout sets how long DispatchJob waits for a definitive worker +// response. Timing out removes only the local waiter. +func WithDispatchTimeout(timeout time.Duration) NodeOption { + return func(o *nodeOptions) { + o.dispatchTimeout = timeout + } +} + +// WithDispatchResultRetention sets how long a settled DispatchJobOnce record +// remains replayable. After expiry the dispatch ID may be admitted as new. The +// value is immutable for a pool generation and must exceed both DispatchTimeout +// and RecoveryGrace. The default is five minutes. +func WithDispatchResultRetention(retention time.Duration) NodeOption { + return func(o *nodeOptions) { + o.dispatchResultRetention = retention + } +} + +// WithRecoveryGrace sets the sink idle-recovery and orphan convergence grace. +func WithRecoveryGrace(grace time.Duration) NodeOption { return func(o *nodeOptions) { - o.ackGracePeriod = ttl + o.recoveryGrace = grace + } +} + +// WithAckGracePeriod is the deprecated v1 name for WithRecoveryGrace. +// Deprecated: use WithRecoveryGrace. +func WithAckGracePeriod(grace time.Duration) NodeOption { + return WithRecoveryGrace(grace) +} + +// WithCleanupLease sets the immutable Redis-time lease for takeover of +// interrupted pool-generation cleanup. Worker and node stale reaping use +// WithWorkerTTL instead. +func WithCleanupLease(lease time.Duration) NodeOption { + return func(o *nodeOptions) { + o.cleanupLease = lease } } @@ -88,14 +132,50 @@ func parseOptions(opts ...NodeOption) *nodeOptions { return o } +// validateNodeOptions rejects values that would cause immediate lease expiry, +// Redis retry churn, or invalid stream limits. +func validateNodeOptions(o *nodeOptions) error { + replayWindow := o.dispatchTimeout + if o.recoveryGrace > replayWindow { + replayWindow = o.recoveryGrace + } + switch { + case o.workerTTL < 2*time.Millisecond: + return fmt.Errorf("pool worker TTL must be at least 2ms") + case o.requeueTimeout < time.Millisecond: + return fmt.Errorf("pool worker requeue timeout must be at least 1ms") + case o.jobSinkBlockDuration < time.Millisecond: + return fmt.Errorf("pool job sink block duration must be at least 1ms") + case o.maxQueuedJobs <= 0: + return fmt.Errorf("pool maximum queued jobs must be greater than zero") + case o.dispatchTimeout < time.Millisecond: + return fmt.Errorf("pool dispatch timeout must be at least 1ms") + case o.recoveryGrace < time.Millisecond: + return fmt.Errorf("pool recovery grace must be at least 1ms") + case o.cleanupLease < time.Millisecond: + return fmt.Errorf("pool cleanup lease must be at least 1ms") + case o.dispatchResultRetention < time.Millisecond: + return fmt.Errorf("pool dispatch result retention must be at least 1ms") + case o.dispatchResultRetention <= replayWindow: + return fmt.Errorf( + "pool dispatch result retention must exceed dispatch timeout and recovery grace", + ) + default: + return nil + } +} + // defaultPoolOptions returns the default options. func defaultPoolOptions() *nodeOptions { return &nodeOptions{ - workerTTL: 30 * time.Second, - workerShutdownTTL: 2 * time.Minute, - jobSinkBlockDuration: 5 * time.Second, - maxQueuedJobs: 1000, - ackGracePeriod: 20 * time.Second, - logger: pulse.NoopLogger(), + workerTTL: 30 * time.Second, + requeueTimeout: 2 * time.Minute, + jobSinkBlockDuration: 5 * time.Second, + maxQueuedJobs: 1000, + dispatchTimeout: 40 * time.Second, + dispatchResultRetention: 5 * time.Minute, + recoveryGrace: 20 * time.Second, + cleanupLease: 30 * time.Second, + logger: pulse.NoopLogger(), } } diff --git a/pool/node_test.go b/pool/node_test.go index 16ba44d..b937dad 100644 --- a/pool/node_test.go +++ b/pool/node_test.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "net" "strconv" "strings" "sync" @@ -15,6 +16,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "goa.design/pulse/pulse" "goa.design/pulse/streaming" ptesting "goa.design/pulse/testing" ) @@ -26,6 +28,342 @@ const ( max = time.Second ) +// poolRedisHook injects one rmap destroy failure for cleanup retry tests. +type poolRedisHook struct { + failDestroy atomic.Bool + failCompletion atomic.Bool + failDetach atomic.Bool + failNodeInit atomic.Bool + failStartCleanup atomic.Bool + blockShutdownCheck atomic.Bool + key string + detachKey string + completionSHA string + startCleanupSHA string + shutdownKey string + nodeStreamPrefix string + shutdownCheckStart chan struct{} + releaseShutdownCheck chan struct{} + failure error +} + +// DialHook preserves normal Redis dialing. +func (h *poolRedisHook) DialHook(next redis.DialHook) redis.DialHook { + return func(ctx context.Context, network, addr string) (net.Conn, error) { + return next(ctx, network, addr) + } +} + +// ProcessHook fails the selected map's Lua destroy operation. +func (h *poolRedisHook) ProcessHook(next redis.ProcessHook) redis.ProcessHook { + return func(ctx context.Context, cmd redis.Cmder) error { + if h.blockShutdownCheck.Load() && cmd.Name() == "hexists" { + args := cmd.Args() + if len(args) == 3 && args[1] == h.shutdownKey && args[2] == "shutdown" && + h.blockShutdownCheck.CompareAndSwap(true, false) { + close(h.shutdownCheckStart) + <-h.releaseShutdownCheck + } + } + if h.failCompletion.Load() && cmd.Name() == "evalsha" { + args := cmd.Args() + if len(args) > 1 && args[1] == h.completionSHA { + return h.failure + } + } + if h.failStartCleanup.Load() && cmd.Name() == "evalsha" { + args := cmd.Args() + if len(args) > 1 && args[1] == h.startCleanupSHA { + return h.failure + } + } + if h.failDetach.Load() && cmd.Name() == "evalsha" { + for _, arg := range cmd.Args() { + if value, ok := arg.(string); ok && strings.HasPrefix(value, h.detachKey) { + return h.failure + } + } + } + if h.failNodeInit.Load() && cmd.Name() == "evalsha" { + for _, arg := range cmd.Args() { + value, ok := arg.(string) + if ok && strings.HasPrefix(value, h.nodeStreamPrefix) && + h.failNodeInit.CompareAndSwap(true, false) { + return h.failure + } + } + } + if h.failDestroy.Load() && cmd.Name() == "evalsha" { + for _, arg := range cmd.Args() { + if key, ok := arg.(string); ok && key == h.key { + return h.failure + } + } + } + return next(ctx, cmd) + } +} + +// ProcessPipelineHook preserves normal Redis pipelines. +func (h *poolRedisHook) ProcessPipelineHook(next redis.ProcessPipelineHook) redis.ProcessPipelineHook { + return func(ctx context.Context, cmds []redis.Cmder) error { + return next(ctx, cmds) + } +} + +// generationStreamKey returns the lifecycle-selected physical stream key for a +// logical name, or the empty string before lifecycle binding. +func generationStreamKey(ctx context.Context, rdb *redis.Client, name string) string { + key := rdb.HGet(ctx, "pulse:stream:"+name+":lifecycle", "physical_key").Val() + if key == "" { + return "" + } + if rdb.Type(ctx, key).Val() != "stream" { + return "" + } + return key +} + +func TestAddNodeRollsBackPostRegistrationFailure(t *testing.T) { + ctx := ptesting.NewTestContext(t) + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + poolName := strings.ReplaceAll(t.Name(), "/", "_") + failure := errors.New("injected node stream initialization failure") + hook := &poolRedisHook{ + nodeStreamPrefix: "pulse:stream:" + nodeStreamName(poolName, ""), + failure: failure, + } + hook.failNodeInit.Store(true) + rdb.AddHook(hook) + + _, err := AddNode(ctx, poolName, rdb, WithJobSinkBlockDuration(50*time.Millisecond)) + require.ErrorIs(t, err, failure) + leases, err := rdb.HKeys(ctx, rmapContentKey(nodeKeepAliveMapName(poolName))).Result() + require.NoError(t, err) + for _, key := range leases { + require.True(t, key == "=rev" || key == "=kind", "leaked node registration %q", key) + } + keys, err := rdb.Keys(ctx, "pulse:stream:"+nodeStreamName(poolName, "")+"*").Result() + require.NoError(t, err) + for _, key := range keys { + require.NotEqual(t, "stream", rdb.Type(ctx, key).Val(), "leaked node stream %q", key) + } + + node, err := AddNode(ctx, poolName, rdb, WithJobSinkBlockDuration(50*time.Millisecond)) + require.NoError(t, err) + require.NoError(t, node.Shutdown(ctx)) +} + +func TestAddNodeRejectsInvalidOptions(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + cases := []struct { + name string + opts []NodeOption + }{ + {name: "worker TTL", opts: []NodeOption{WithWorkerTTL(time.Millisecond)}}, + {name: "requeue timeout", opts: []NodeOption{WithRequeueTimeout(500 * time.Microsecond)}}, + {name: "job sink block", opts: []NodeOption{WithJobSinkBlockDuration(500 * time.Microsecond)}}, + {name: "queued jobs", opts: []NodeOption{WithMaxQueuedJobs(0)}}, + {name: "dispatch timeout", opts: []NodeOption{WithDispatchTimeout(500 * time.Microsecond)}}, + {name: "recovery grace", opts: []NodeOption{WithRecoveryGrace(500 * time.Microsecond)}}, + {name: "cleanup lease", opts: []NodeOption{WithCleanupLease(500 * time.Microsecond)}}, + { + name: "dispatch retention precision", + opts: []NodeOption{WithDispatchResultRetention(500 * time.Microsecond)}, + }, + { + name: "dispatch retention window", + opts: []NodeOption{ + WithDispatchTimeout(time.Second), + WithDispatchResultRetention(time.Second), + }, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + _, err := AddNode(ctx, t.Name(), rdb, tc.opts...) + require.Error(t, err) + }) + } +} + +func TestClosingImmediatelyFencesAdmission(t *testing.T) { + ctx := ptesting.NewTestContext(t) + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + node := newTestNode(t, ctx, rdb, strings.ReplaceAll(t.Name(), "/", "_")) + + closed := make(chan error, 1) + go func() { + closed <- node.Close(ctx) + }() + require.Eventually(t, func() bool { + node.lock.RLock() + defer node.lock.RUnlock() + return node.closing + }, max, delay) + + _, err := node.AddWorker(ctx, &mockHandler{}) + require.ErrorContains(t, err, "closed") + require.ErrorContains(t, node.DispatchJob(ctx, "job", nil), "closed") + require.ErrorContains(t, node.DispatchMessage(ctx, "message", nil), "closed") + require.ErrorContains(t, node.StopJob(ctx, "job"), "closed") + require.ErrorContains(t, node.NotifyWorker(ctx, "job", nil), "closed") + require.NoError(t, <-closed) +} + +func TestShutdownPublishesWhileCloseIsInProgress(t *testing.T) { + ctx := ptesting.NewTestContext(t) + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + poolName := strings.ReplaceAll(t.Name(), "/", "_") + first := newTestNode(t, ctx, rdb, poolName) + peer := newTestNode(t, ctx, rdb, poolName) + + closeResult := make(chan error, 1) + go func() { + closeResult <- first.Close(ctx) + }() + require.Eventually(t, func() bool { + first.lock.RLock() + defer first.lock.RUnlock() + return first.closing + }, max, delay) + require.NoError(t, first.Shutdown(ctx)) + require.NoError(t, <-closeResult) + require.True(t, first.IsShutdown()) + require.True(t, peer.IsShutdown()) +} + +func TestAddNodeTakesOverExpiredPoolCleanup(t *testing.T) { + ctx := ptesting.NewTestContext(t) + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + poolName := strings.ReplaceAll(t.Name(), "/", "_") + old := newTestNode(t, ctx, rdb, poolName) + generation := old.poolStream.Generation() + require.NoError(t, old.Close(ctx)) + require.NoError(t, rdb.HSet( + ctx, + poolCleanupGenerationsKey(poolName), + "state", poolCleanupFinishingState, + "generation", generation, + "owner", "crashed", + "lease_until", "0", + ).Err()) + require.NoError(t, rdb.HSet( + ctx, + rmapContentKey(nodeShutdownMapName(poolName)), + "shutdown", "crashed", + ).Err()) + + next := newTestNode(t, ctx, rdb, poolName) + require.NotEqual(t, generation, next.poolStream.Generation()) + record, err := rdb.HGetAll(ctx, poolCleanupGenerationsKey(poolName)).Result() + require.NoError(t, err) + require.Equal(t, map[string]string{ + "state": poolCleanupCompleteState, + "generation": generation, + }, record) + require.NoError(t, next.Shutdown(ctx)) + record, err = rdb.HGetAll(ctx, poolCleanupGenerationsKey(poolName)).Result() + require.NoError(t, err) + require.Len(t, record, 2) +} + +func TestExpiredCleanupOwnerCannotDeleteReusedPool(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + poolName := t.Name() + stream, err := streaming.NewStream(poolStreamName(poolName), rdb) + require.NoError(t, err) + require.NoError(t, stream.Open(ctx)) + generation := stream.Generation() + resources, err := establishPoolResources( + ctx, + rdb, + poolName, + generation, + 1000, + 30*time.Second, + time.Second, + 5*time.Minute, + ) + require.NoError(t, err) + _, err = stream.Add(ctx, evInit, nil) + require.NoError(t, err) + + status, err := claimPoolCleanup(ctx, rdb, poolName, generation, "paused", time.Second) + require.NoError(t, err) + require.Equal(t, poolCleanupClaimed, status) + require.NoError(t, rdb.HSet(ctx, poolCleanupGenerationsKey(poolName), "lease_until", "0").Err()) + status, err = claimPoolCleanup(ctx, rdb, poolName, generation, "takeover", time.Second) + require.NoError(t, err) + require.Equal(t, poolCleanupClaimed, status) + + require.NoError(t, destroyCleanupStream(ctx, rdb, poolName, generation, "takeover", time.Second)) + require.NoError(t, destroyPoolMap( + ctx, + rdb, + poolName, + resources.jobs, + generation, + "takeover", + time.Second, + )) + require.NoError(t, completePoolCleanupScript.Run( + ctx, + rdb, + []string{poolCleanupGenerationsKey(poolName), poolResourcesKey(poolName)}, + generation, + "takeover", + poolCleanupFinishingState, + poolCleanupCompleteState, + poolResourceStateActive, + poolResourceStateDestroyed, + ).Err()) + require.Error(t, renewPoolCleanup(ctx, rdb, poolName, generation, "takeover", time.Second)) + + recreated, err := streaming.NewStream(poolStreamName(poolName), rdb) + require.NoError(t, err) + _, err = recreated.Add(ctx, evInit, nil) + require.NoError(t, err) + require.NotEqual(t, generation, recreated.Generation()) + replacementResources, err := establishPoolResources( + ctx, + rdb, + poolName, + recreated.Generation(), + 1000, + 30*time.Second, + 30*time.Second, + 5*time.Minute, + ) + require.NoError(t, err) + require.NotEqual(t, resources.jobs, replacementResources.jobs) + replacementKey := rmapContentKey(replacementResources.jobs) + require.NoError(t, rdb.HSet(ctx, replacementKey, "replacement", "value").Err()) + for _, pair := range [][2]string{ + {resources.nodeKeepAlive, replacementResources.nodeKeepAlive}, + {resources.workers, replacementResources.workers}, + {resources.jobs, replacementResources.jobs}, + {resources.jobPending, replacementResources.jobPending}, + } { + require.NotEqual(t, pair[0], pair[1]) + require.NoError(t, rdb.HSet(ctx, rmapContentKey(pair[0]), "stale", "old").Err()) + require.False(t, rdb.HExists(ctx, rmapContentKey(pair[1]), "stale").Val()) + } + + err = destroyPoolMap(ctx, rdb, poolName, resources.jobs, generation, "paused", time.Second) + require.ErrorContains(t, err, "POOLCLEANUPLOST") + require.Equal(t, "value", rdb.HGet(ctx, replacementKey, "replacement").Val()) + require.NoError(t, recreated.Destroy(ctx)) +} + func TestWorkers(t *testing.T) { testName := strings.Replace(t.Name(), "/", "_", -1) ctx := ptesting.NewTestContext(t) @@ -399,81 +737,51 @@ func TestDispatchJobRaceCondition(t *testing.T) { t.Run("claim rejects active pending job from redis", func(t *testing.T) { jobKey := "active-redis-pending-job" - pendingHash := rmapContentKey(jobPendingMapName(testName)) - pendingUntil := strconv.FormatInt(time.Now().Add(time.Hour).UnixNano(), 10) - require.NoError(t, rdb.HSet(ctx, pendingHash, jobKey, pendingUntil).Err()) + pendingHash := rmapContentKey(node2.resources.jobPending) + const pendingNonce = "active-dispatch" + storedGuard := pendingNonce + "\x00" + "1-0" + require.NoError(t, rdb.HSet(ctx, pendingHash, jobKey, storedGuard).Err()) _, localExists := node2.jobPendingMap.Get(jobKey) require.False(t, localExists) - pendingTS, err := node2.claimDispatch(ctx, jobKey) - require.Empty(t, pendingTS) + _, err := node2.publishDispatch(ctx, jobKey, "replacement", marshalJob(&Job{Key: jobKey})) require.True(t, errors.Is(err, ErrJobExists), "Expected ErrJobExists, got: %v", err) stored, err := rdb.HGet(ctx, pendingHash, jobKey).Result() require.NoError(t, err) - require.Equal(t, pendingUntil, stored) + require.Equal(t, storedGuard, stored) }) - t.Run("claim replaces stale pending job and publishes it", func(t *testing.T) { - jobKey := "stale-redis-pending-job" - pendingHash := rmapContentKey(jobPendingMapName(testName)) - staleUntil := strconv.FormatInt(time.Now().Add(-time.Hour).UnixNano(), 10) - require.NoError(t, rdb.HSet(ctx, pendingHash, jobKey, staleUntil).Err()) + t.Run("claim never replaces pending dispatch without completion", func(t *testing.T) { + jobKey := "unknown-redis-pending-job" + pendingHash := rmapContentKey(node2.resources.jobPending) + const pendingNonce = "unknown-dispatch" + storedGuard := pendingNonce + "\x00" + "1-0" + require.NoError(t, rdb.HSet(ctx, pendingHash, jobKey, storedGuard).Err()) - pendingTS, err := node2.claimDispatch(ctx, jobKey) - require.NoError(t, err) - require.NotEmpty(t, pendingTS) + _, err := node2.publishDispatch(ctx, jobKey, "replacement", marshalJob(&Job{Key: jobKey})) + require.ErrorIs(t, err, ErrJobExists) stored, err := rdb.HGet(ctx, pendingHash, jobKey).Result() require.NoError(t, err) - require.Equal(t, pendingTS, stored) - require.Eventually(t, func() bool { - local, exists := node2.jobPendingMap.Get(jobKey) - return exists && local == pendingTS - }, max, delay) - node2.releaseDispatchPending(jobKey, pendingTS) + require.Equal(t, storedGuard, stored) + require.Error(t, node2.completeDispatch(ctx, jobKey, pendingNonce)) + require.NoError(t, rdb.HDel(ctx, pendingHash, jobKey).Err()) }) - t.Run("claim rejects malformed pending job", func(t *testing.T) { - jobKey := "malformed-redis-pending-job" - pendingHash := rmapContentKey(jobPendingMapName(testName)) - const malformedPending = "not-a-timestamp" - require.NoError(t, rdb.HSet(ctx, pendingHash, jobKey, malformedPending).Err()) + t.Run("claim treats every persisted value as an owned nonce", func(t *testing.T) { + jobKey := "opaque-redis-pending-job" + pendingHash := rmapContentKey(node2.resources.jobPending) + const opaquePending = "not-a-timestamp" + require.NoError(t, rdb.HSet(ctx, pendingHash, jobKey, opaquePending).Err()) - pendingTS, err := node2.claimDispatch(ctx, jobKey) - require.Empty(t, pendingTS) - require.Error(t, err) - require.False(t, errors.Is(err, ErrJobExists)) + _, err := node2.publishDispatch(ctx, jobKey, "replacement", marshalJob(&Job{Key: jobKey})) + require.ErrorIs(t, err, ErrJobExists) stored, err := rdb.HGet(ctx, pendingHash, jobKey).Result() require.NoError(t, err) - require.Equal(t, malformedPending, stored) - }) - - t.Run("release only removes owned pending guard", func(t *testing.T) { - jobKey := "owned-pending-release-job" - pendingHash := rmapContentKey(jobPendingMapName(testName)) - pendingTS, err := node2.claimDispatch(ctx, jobKey) - require.NoError(t, err) - require.NotEmpty(t, pendingTS) - require.Eventually(t, func() bool { - local, exists := node2.jobPendingMap.Get(jobKey) - return exists && local == pendingTS - }, max, delay) - - node2.releaseDispatchPending(jobKey, "not-"+pendingTS) - stored, err := rdb.HGet(ctx, pendingHash, jobKey).Result() - require.NoError(t, err) - require.Equal(t, pendingTS, stored) - - node2.releaseDispatchPending(jobKey, pendingTS) - pendingExists, err := rdb.HExists(ctx, pendingHash, jobKey).Result() - require.NoError(t, err) - require.False(t, pendingExists) - require.Eventually(t, func() bool { - _, exists := node2.jobPendingMap.Get(jobKey) - return !exists - }, max, delay) + require.Equal(t, opaquePending, stored) + require.NoError(t, rdb.HDel(ctx, pendingHash, jobKey).Err()) }) t.Run("concurrent atomic claims admit one dispatcher", func(t *testing.T) { @@ -483,14 +791,20 @@ func TestDispatchJobRaceCondition(t *testing.T) { var wg sync.WaitGroup for i := 0; i < 20; i++ { wg.Add(1) - go func() { + go func(i int) { defer wg.Done() - pendingTS, err := node2.claimDispatch(ctx, jobKey) + dispatchID := fmt.Sprintf("dispatch-%d", i) + _, err := node2.publishDispatch( + ctx, + jobKey, + dispatchID, + marshalJob(&Job{Key: jobKey, dispatchID: dispatchID}), + ) if err == nil { - pendingCh <- pendingTS + pendingCh <- dispatchID } errCh <- err - }() + }(i) } wg.Wait() close(errCh) @@ -511,14 +825,14 @@ func TestDispatchJobRaceCondition(t *testing.T) { } require.Equal(t, 1, successCount) require.Equal(t, 19, errorCount) - node2.releaseDispatchPending(jobKey, <-pendingCh) + require.NoError(t, node2.completeDispatch(ctx, jobKey, <-pendingCh)) }) t.Run("dispatch checks redis when local payload replica is stale", func(t *testing.T) { jobKey := "stale-local-payload-job" payload := []byte("test payload") - payloadHash := rmapContentKey(jobPayloadMapName(testName)) - pendingHash := rmapContentKey(jobPendingMapName(testName)) + payloadHash := rmapContentKey(node2.resources.jobPayloads) + pendingHash := rmapContentKey(node2.resources.jobPending) // Simulate the production race: Redis already has the live job payload, // but this node's local rmap replica has not applied that update yet. @@ -533,22 +847,20 @@ func TestDispatchJobRaceCondition(t *testing.T) { require.False(t, pendingExists) }) - t.Run("dispatch after pending job times out succeeds", func(t *testing.T) { + t.Run("dispatch never reopens unknown pending job", func(t *testing.T) { jobKey := "timeout-job" payload := []byte("test payload") - // Set a stale pending timestamp - staleTS := time.Now().Add(-time.Hour).UnixNano() - _, err := node1.jobPendingMap.SetAndWait(ctx, jobKey, strconv.FormatInt(staleTS, 10)) - require.NoError(t, err, "Failed to set stale pending timestamp") + const nonce = "unknown-dispatch" + _, err := node1.jobPendingMap.SetAndWait(ctx, jobKey, nonce) + require.NoError(t, err) defer func() { _, err = node1.jobPendingMap.Delete(ctx, jobKey) - assert.NoError(t, err, "Failed to delete pending timestamp") + assert.NoError(t, err) }() - // Dispatch should succeed because pending timestamp is in the past err = node1.DispatchJob(ctx, jobKey, payload) - assert.NoError(t, err, "Dispatch should succeed after pending timeout") + require.ErrorIs(t, err, ErrJobExists) }) t.Run("dispatch cleans up pending entry on success", func(t *testing.T) { @@ -566,38 +878,37 @@ func TestDispatchJobRaceCondition(t *testing.T) { }, max, delay, "Pending entry should be cleaned up after successful dispatch") }) - t.Run("dispatch rejects invalid pending timestamp", func(t *testing.T) { + t.Run("dispatch treats pending guard as opaque nonce", func(t *testing.T) { jobKey := "invalid-timestamp-job" payload := []byte("test payload") - // Set an invalid pending timestamp _, err := node1.jobPendingMap.SetAndWait(ctx, jobKey, "invalid-timestamp") - require.NoError(t, err, "Failed to set invalid pending timestamp") + require.NoError(t, err) err = node1.DispatchJob(ctx, jobKey, payload) - require.Error(t, err) - require.False(t, errors.Is(err, ErrJobExists)) - require.Contains(t, err.Error(), "malformed pending guard") - stored, err := rdb.HGet(ctx, rmapContentKey(jobPendingMapName(testName)), jobKey).Result() + require.ErrorIs(t, err, ErrJobExists) + stored, err := rdb.HGet(ctx, rmapContentKey(node2.resources.jobPending), jobKey).Result() require.NoError(t, err) require.Equal(t, "invalid-timestamp", stored) }) - // Keep this test last, it destroys the stream + // Keep this test last because it temporarily replaces the stream key. t.Run("dispatch cleans up pending entry on failure", func(t *testing.T) { jobKey := "cleanup-job" payload := []byte("test payload") - // Replace the Redis stream with a string so XADD fails. Deleting the - // stream no longer forces this path because the pool sink now - // recovers externally deleted consumer groups. - streamKey := "pulse:stream:" + poolStreamName(node1.PoolName) + // Replace the Redis stream with a string so XADD fails after the pending + // dispatch guard is claimed. Deleting the stream no longer forces this + // path because the pool sink now repairs externally deleted streams. + streamKey := generationStreamKey(ctx, rdb, poolStreamName(node1.PoolName)) + require.NotEmpty(t, streamKey) require.NoError(t, rdb.Del(ctx, streamKey).Err()) - require.NoError(t, rdb.Set(ctx, streamKey, "wrong-type", 0).Err()) + err := rdb.Set(ctx, streamKey, "wrong-type", 0).Err() + require.NoError(t, err) defer func() { require.NoError(t, rdb.Del(ctx, streamKey).Err()) }() // Attempt dispatch (should fail) - err := node1.DispatchJob(ctx, jobKey, payload) + err = node1.DispatchJob(ctx, jobKey, payload) require.Error(t, err, "Expected dispatch to fail") // Verify pending entry was cleaned up @@ -609,6 +920,137 @@ func TestDispatchJobRaceCondition(t *testing.T) { } +func TestDispatchCancellationRetainsAdmissionUntilDefinitiveCompletion(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + node, err := AddNode( + ctx, + t.Name(), + rdb, + WithDispatchTimeout(100*time.Millisecond), + WithRecoveryGrace(50*time.Millisecond), + ) + require.NoError(t, err) + + dispatchCtx, cancel := context.WithTimeout(ctx, 20*time.Millisecond) + defer cancel() + err = node.DispatchJob(dispatchCtx, "job", []byte("payload")) + require.ErrorIs(t, err, context.DeadlineExceeded) + pendingKey := rmapContentKey(jobPendingMapName(t.Name())) + guard, err := rdb.HGet(ctx, pendingKey, "job").Result() + require.NoError(t, err) + require.NotEmpty(t, guard) + dispatchID := guard + require.ErrorIs(t, node.DispatchJob(ctx, "job", []byte("duplicate")), ErrJobExists) + _, waiterExists := node.pendingJobChannels.Load(dispatchID) + require.False(t, waiterExists) + + require.NoError(t, node.completeDispatch(ctx, "job", dispatchID)) + replacement, err := node.publishDispatch( + ctx, + "job", + "replacement", + marshalJob(&Job{Key: "job", dispatchID: "replacement"}), + ) + require.NoError(t, err) + require.NotEmpty(t, replacement) + require.NoError(t, node.completeDispatch(ctx, "job", "replacement")) + require.NoError(t, node.Close(ctx)) + require.NoError(t, node.poolStream.Destroy(ctx)) +} + +func TestPoolRoutingDropsMalformedEventAndContinues(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + node := newTestNode(t, ctx, rdb, t.Name()) + received := make(chan string, 1) + handler := &mockMessageHandler{ + mockHandler: newMockHandler(), + messageFunc: func(key string, payload []byte) error { + received <- key + ":" + string(payload) + return nil + }, + } + _, err := node.AddWorker(ctx, handler) + require.NoError(t, err) + + _, err = node.poolStream.Add(ctx, evMessage, []byte{1, 2, 3}) + require.NoError(t, err) + require.NoError(t, node.DispatchMessage(ctx, "valid", []byte("payload"))) + require.Equal(t, "valid:payload", <-received) + require.Eventually(t, func() bool { + pending, pendingErr := rdb.XPending( + ctx, + generationStreamKey(ctx, rdb, node.poolStream.Name), + node.poolSink.Name, + ).Result() + return pendingErr == nil && pending.Count == 0 + }, max, delay) + + require.NoError(t, node.Shutdown(ctx)) +} + +func TestWorkerAckReleasesDispatchAfterCallerNodeCrash(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + node := newTestNode(t, ctx, rdb, t.Name()) + const ( + jobKey = "job" + dispatchID = "crashed-dispatch" + workerID = "worker" + workerEvent = "worker-event" + ) + eventID, err := node.publishDispatch( + ctx, + jobKey, + dispatchID, + marshalJob(&Job{Key: jobKey, NodeID: "crashed-node", dispatchID: dispatchID}), + ) + require.NoError(t, err) + pending := &streaming.Event{ + ID: eventID, + EventName: evStartJob, + Payload: marshalJob(&Job{ + Key: jobKey, + NodeID: "crashed-node", + dispatchID: dispatchID, + }), + Acker: &mockAcker{ + XAckFunc: func(ctx context.Context, _, _ string, _ ...string) *redis.IntCmd { + return redis.NewIntCmd(ctx, 1) + }, + }, + } + node.pendingEvents.Store(pendingEventKey(workerID, workerEvent), pending) + + node.ackWorkerEvent(&streaming.Event{ + Payload: marshalEnvelope(workerID, marshalAck(&ack{EventID: workerEvent})), + }) + require.Eventually(t, func() bool { + return !rdb.HExists(ctx, rmapContentKey(node.resources.jobPending), jobKey).Val() + }, max, delay) + _, exists := node.pendingEvents.Load(pendingEventKey(workerID, workerEvent)) + require.False(t, exists) + require.NoError(t, node.Shutdown(ctx)) +} + +func TestPoolScriptsRecoverAfterScriptFlush(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + node := newTestNode(t, ctx, rdb, t.Name()) + worker := newTestWorker(t, ctx, node) + requireActiveWorkerRing(t, []*Node{node}, worker.ID) + require.NoError(t, rdb.ScriptFlush(ctx).Err()) + + require.NoError(t, node.DispatchJob(ctx, "job", []byte("payload"))) + require.NoError(t, node.StopJob(ctx, "job")) + require.NoError(t, node.Shutdown(ctx)) +} + func TestNotifyWorker(t *testing.T) { testName := strings.Replace(t.Name(), "/", "_", -1) ctx := ptesting.NewTestContext(t) @@ -653,7 +1095,7 @@ func TestNotifyWorkerNoHandler(t *testing.T) { testName := strings.Replace(t.Name(), "/", "_", -1) ctx, buf := ptesting.NewBufferedLogContext(t) rdb := ptesting.NewRedisClient(t) - node := newTestNode(t, ctx, rdb, testName) + node := newTestNodeWithLogger(t, ctx, rdb, testName, pulse.ClueLogger(ctx)) defer ptesting.CleanupRedis(t, rdb, true, testName) // Create a worker without NotificationHandler implementation @@ -737,7 +1179,7 @@ func TestDispatchMessageRequiresHandler(t *testing.T) { testName := strings.Replace(t.Name(), "/", "_", -1) ctx, buf := ptesting.NewBufferedLogContext(t) rdb := ptesting.NewRedisClient(t) - node := newTestNode(t, ctx, rdb, testName) + node := newTestNodeWithLogger(t, ctx, rdb, testName, pulse.ClueLogger(ctx)) defer ptesting.CleanupRedis(t, rdb, true, testName) worker := newTestWorkerWithoutOptionalHandlers(t, ctx, node) @@ -797,6 +1239,275 @@ func TestClose(t *testing.T) { assert.NoError(t, node.Shutdown(ctx)) } +func TestShutdownReapsStaleNodeWithRedisTime(t *testing.T) { + ctx := ptesting.NewTestContext(t) + testName := strings.ReplaceAll(t.Name(), "/", "_") + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, true, testName) + node := newTestNode(t, ctx, rdb, testName) + now, err := rdb.Time(ctx).Result() + require.NoError(t, err) + _, err = node.nodeKeepAliveMap.Set( + ctx, + "crashed-node", + strconv.FormatInt(now.Add(-2*node.workerTTL).UnixNano(), 10), + ) + require.NoError(t, err) + + require.NoError(t, node.Shutdown(ctx)) + require.True(t, node.cleanupComplete) +} + +func TestShutdownRetriesPartialCleanup(t *testing.T) { + ctx := ptesting.NewTestContext(t) + testName := strings.ReplaceAll(t.Name(), "/", "_") + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, true, testName) + failure := errors.New("injected pool cleanup failure") + hook := &poolRedisHook{ + key: rmapContentKey(workerMapName(testName)), + completionSHA: completePoolCleanupScript.Hash(), + failure: failure, + } + rdb.AddHook(hook) + node := newTestNode(t, ctx, rdb, testName) + hook.failDestroy.Store(true) + + err := node.Shutdown(ctx) + require.ErrorIs(t, err, failure) + require.True(t, node.IsClosed()) + require.False(t, node.cleanupComplete) + state, err := rdb.HGet(ctx, "pulse:stream:"+poolStreamName(testName)+":lifecycle", "state").Result() + require.NoError(t, err) + require.Equal(t, "destroyed", state) + require.NotEqual(t, "destroy", rdb.HGet(ctx, hook.key, "=kind").Val()) + + hook.failDestroy.Store(false) + hook.failCompletion.Store(true) + err = node.Shutdown(ctx) + require.ErrorIs(t, err, failure) + require.Contains(t, err.Error(), "failed to record cleanup completion") + _, err = AddNode(ctx, testName, rdb) + require.EqualError(t, err, `AddNode: pool "`+testName+`" is shutting down`) + + hook.failCompletion.Store(false) + require.NoError(t, node.Shutdown(ctx)) + require.True(t, node.cleanupComplete) + require.EqualValues(t, 0, rdb.Exists(ctx, rmapContentKey(nodeShutdownMapName(testName))).Val()) +} + +func TestCompletedCleanupStillClosesLocalNode(t *testing.T) { + for _, test := range []struct { + name string + close func(context.Context, *Node) error + }{ + {name: "Shutdown", close: func(ctx context.Context, node *Node) error { + return node.Shutdown(ctx) + }}, + {name: "Close", close: func(ctx context.Context, node *Node) error { + return node.Close(ctx) + }}, + } { + t.Run(test.name, func(t *testing.T) { + ctx := ptesting.NewTestContext(t) + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + node := newTestNode(t, ctx, rdb, strings.ReplaceAll(t.Name(), "/", "_")) + worker := newTestWorker(t, ctx, node) + producer := newTestProducer("stale-resumed", func() (*JobPlan, error) { + return &JobPlan{}, nil + }) + require.NoError(t, node.Schedule(ctx, producer, time.Millisecond)) + require.NoError(t, rdb.HSet( + ctx, + poolCleanupGenerationsKey(node.PoolName), + "state", poolCleanupCompleteState, + "generation", node.resources.generation, + ).Err()) + require.NoError(t, node.poolStream.Destroy(ctx)) + + require.NoError(t, test.close(ctx, node)) + require.True(t, node.IsClosed()) + require.True(t, node.IsShutdown()) + require.True(t, worker.IsStopped()) + require.True(t, node.poolSink.IsClosed()) + done := make(chan struct{}) + go func() { + node.scheduleWG.Wait() + close(done) + }() + select { + case <-done: + case <-time.After(time.Second): + require.Fail(t, "schedule goroutine was not joined") + } + }) + } +} + +func TestCompletedCleanupPreservesSettlementErrorAfterLocalTeardown(t *testing.T) { + ctx := ptesting.NewTestContext(t) + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + node := newTestNode(t, ctx, rdb, strings.ReplaceAll(t.Name(), "/", "_")) + worker := newTestWorker(t, ctx, node) + settlementErr := errors.New("injected terminal settlement failure") + node.settlements.begin(worker.ID)(settlementErr) + require.NoError(t, rdb.HSet( + ctx, + poolCleanupGenerationsKey(node.PoolName), + "state", poolCleanupCompleteState, + "generation", node.resources.generation, + ).Err()) + require.NoError(t, node.poolStream.Destroy(ctx)) + + firstErr := node.Close(ctx) + require.ErrorIs(t, firstErr, settlementErr) + require.True(t, node.IsClosed()) + require.True(t, node.IsShutdown()) + require.True(t, worker.IsStopped()) + require.True(t, node.poolSink.IsClosed()) + secondErr := node.Close(ctx) + require.ErrorIs(t, secondErr, settlementErr) + require.Equal(t, firstErr.Error(), secondErr.Error()) +} + +func TestLateShutdownCannotCleanNewPoolGeneration(t *testing.T) { + ctx := ptesting.NewTestContext(t) + testName := strings.ReplaceAll(t.Name(), "/", "_") + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, true, testName) + oldNode := newTestNode(t, ctx, rdb, testName) + oldGeneration := oldNode.poolStream.Generation() + require.NoError(t, oldNode.Shutdown(ctx)) + + newNode := newTestNode(t, ctx, rdb, testName) + require.NotEqual(t, oldGeneration, newNode.poolStream.Generation()) + require.NoError(t, oldNode.Shutdown(ctx)) + require.False(t, newNode.IsClosed()) + require.NoError(t, newNode.Shutdown(ctx)) +} + +func TestOldGenerationMutationCannotRecreateDeletedMap(t *testing.T) { + ctx := ptesting.NewTestContext(t) + testName := strings.ReplaceAll(t.Name(), "/", "_") + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, true, testName) + oldNode := newTestNode(t, ctx, rdb, testName) + oldMapKey := rmapContentKey(oldNode.resources.workerKeepAlive) + require.NoError(t, oldNode.Shutdown(ctx)) + require.EqualValues(t, 0, rdb.Exists(ctx, oldMapKey).Val()) + newNode := newTestNode(t, ctx, rdb, testName) + + err := oldNode.setPoolMap(ctx, oldNode.resources.workerKeepAlive, "stale", "1") + require.ErrorIs(t, err, ErrPoolGenerationLost) + require.EqualValues(t, 0, rdb.Exists(ctx, oldMapKey).Val()) + require.NotEqual(t, oldNode.resources.workerKeepAlive, newNode.resources.workerKeepAlive) + + require.NoError(t, newNode.Shutdown(ctx)) +} + +func TestAddNodePostRegistrationShutdownCheck(t *testing.T) { + ctx := ptesting.NewTestContext(t) + testName := strings.ReplaceAll(t.Name(), "/", "_") + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, true, testName) + hook := &poolRedisHook{ + shutdownKey: rmapContentKey(nodeShutdownMapName(testName)), + shutdownCheckStart: make(chan struct{}), + releaseShutdownCheck: make(chan struct{}), + } + rdb.AddHook(hook) + first, err := AddNode( + ctx, + testName, + rdb, + WithWorkerTTL(2*time.Second), + WithRequeueTimeout(100*time.Millisecond), + WithDispatchTimeout(time.Second), + WithRecoveryGrace(500*time.Millisecond), + WithJobSinkBlockDuration(100*time.Millisecond), + ) + require.NoError(t, err) + hook.blockShutdownCheck.Store(true) + + type addResult struct { + node *Node + err error + } + added := make(chan addResult, 1) + go func() { + node, err := AddNode( + ctx, + testName, + rdb, + WithWorkerTTL(2*time.Second), + WithRequeueTimeout(100*time.Millisecond), + WithDispatchTimeout(time.Second), + WithRecoveryGrace(500*time.Millisecond), + WithJobSinkBlockDuration(100*time.Millisecond), + ) + added <- addResult{node: node, err: err} + }() + + select { + case <-hook.shutdownCheckStart: + case <-time.After(max): + t.Fatal("AddNode did not reach post-registration shutdown check") + } + require.NoError(t, rdb.HSet(ctx, hook.shutdownKey, "shutdown", first.ID).Err()) + close(hook.releaseShutdownCheck) + result := <-added + require.NoError(t, result.err) + require.Eventually(t, result.node.IsClosed, max, delay) + require.True(t, result.node.IsShutdown()) + + require.NoError(t, rdb.HDel(ctx, hook.shutdownKey, "shutdown").Err()) + require.NoError(t, first.Shutdown(ctx)) +} + +func TestPeerShutdownDetachFailureIsRetriedAndSurfaced(t *testing.T) { + ctx := ptesting.NewTestContext(t) + testName := strings.ReplaceAll(t.Name(), "/", "_") + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, true, testName) + failure := errors.New("injected peer detach failure") + hook := &poolRedisHook{ + detachKey: "map:stream:" + poolStreamName(testName) + ":generation:", + failure: failure, + } + rdb.AddHook(hook) + first := newTestNode(t, ctx, rdb, testName) + peer := newTestNode(t, ctx, rdb, testName) + require.NoError(t, first.close(ctx, true)) + hook.failDetach.Store(true) + require.NoError(t, rdb.HSet( + ctx, + rmapContentKey(nodeShutdownMapName(testName)), + "shutdown", + first.ID, + ).Err()) + peer.ownShutdown(context.Background()) + + require.Eventually(t, func() bool { + return rdb.HExists( + ctx, + rmapContentKey(nodeShutdownMapName(testName)), + shutdownErrorKey(peer.ID), + ).Val() + }, peer.workerTTL, delay) + start := time.Now() + err := first.waitForPoolNodes(ctx) + require.Error(t, err) + require.Contains(t, err.Error(), "failed to detach pool sink") + require.Less(t, time.Since(start), peer.workerTTL) + require.False(t, peer.IsClosed()) + + hook.failDetach.Store(false) + require.Eventually(t, peer.IsClosed, peer.workerTTL, delay) + require.NoError(t, first.Shutdown(ctx)) +} + func TestTwoNodeJobDispatchAndAck(t *testing.T) { testName := strings.Replace(t.Name(), "/", "_", -1) ctx := ptesting.NewTestContext(t) @@ -882,7 +1593,7 @@ func TestNodeCloseAndRequeue(t *testing.T) { select { case <-jobRequeued: // Job successfully requeued - case <-time.After(max): + case <-time.After(2 * testAckGracePeriod): t.Error("Timeout: job was not requeued within expected time") } @@ -921,71 +1632,6 @@ func TestAckWorkerEventWithMissingPendingEvent(t *testing.T) { assert.True(t, true, "ackWorkerEvent should complete without panic") } -func TestStaleEventsAreRemoved(t *testing.T) { - // Setup - ctx := ptesting.NewTestContext(t) - testName := strings.Replace(t.Name(), "/", "_", -1) - rdb := ptesting.NewRedisClient(t) - defer ptesting.CleanupRedis(t, rdb, true, testName) - node := newTestNode(t, ctx, rdb, testName) - defer func() { assert.NoError(t, node.Shutdown(ctx)) }() - - // Add a stale event manually - staleEventID := fmt.Sprintf("%d-0", time.Now().Add(-2*pendingEventTTL).UnixNano()/int64(time.Millisecond)) - staleEvent := &streaming.Event{ - ID: staleEventID, - EventName: "test-event", - Payload: []byte("test-payload"), - Acker: &mockAcker{ - XAckFunc: func(ctx context.Context, streamKey, sinkName string, ids ...string) *redis.IntCmd { - return redis.NewIntCmd(ctx, 0) - }, - }, - } - node.pendingEvents.Store(pendingEventKey("worker", staleEventID), staleEvent) - - // Add a fresh event - freshEventID := fmt.Sprintf("%d-0", time.Now().Add(-time.Second).UnixNano()/int64(time.Millisecond)) - freshEvent := &streaming.Event{ - ID: freshEventID, - EventName: "test-event", - Payload: []byte("test-payload"), - Acker: &mockAcker{ - XAckFunc: func(ctx context.Context, streamKey, sinkName string, ids ...string) *redis.IntCmd { - return redis.NewIntCmd(ctx, 0) - }, - }, - } - node.pendingEvents.Store(pendingEventKey("worker", freshEventID), freshEvent) - - // Create a mock event to trigger the ackWorkerEvent function - mockEventID := "mock-event-id" - mockEvent := &streaming.Event{ - ID: mockEventID, - EventName: evAck, - Payload: marshalEnvelope("worker", marshalAck(&ack{EventID: mockEventID})), - Acker: &mockAcker{ - XAckFunc: func(ctx context.Context, streamKey, sinkName string, ids ...string) *redis.IntCmd { - return redis.NewIntCmd(ctx, 0) - }, - }, - } - node.pendingEvents.Store(pendingEventKey("worker", mockEventID), mockEvent) - - // Call ackWorkerEvent to trigger the stale event cleanup - node.ackWorkerEvent(mockEvent) - - assert.Eventually(t, func() bool { - _, ok := node.pendingEvents.Load(pendingEventKey("worker", staleEventID)) - return !ok - }, max, delay, "Stale event should have been removed") - - assert.Eventually(t, func() bool { - _, ok := node.pendingEvents.Load(pendingEventKey("worker", freshEventID)) - return ok - }, max, delay, "Fresh event should still be present") -} - func TestStaleNodeStreamCleanup(t *testing.T) { var ( ctx = ptesting.NewTestContext(t) @@ -1020,19 +1666,17 @@ func TestStaleNodeStreamCleanup(t *testing.T) { assert.NoError(t, node1.DispatchJob(ctx, "job1", []byte("payload1"))) assert.NoError(t, node2.DispatchJob(ctx, "job2", []byte("payload2"))) - // Verify both streams exist initially - name1 := "pulse:stream:" + nodeStreamName(node1.PoolName, node1.ID) - name2 := "pulse:stream:" + nodeStreamName(node2.PoolName, node2.ID) + // Verify both generation-qualified streams exist initially. + var name1, name2 string assert.Eventually(t, func() bool { - exists1, err1 := rdb.Exists(ctx, name1).Result() - exists2, err2 := rdb.Exists(ctx, name2).Result() - return err1 == nil && err2 == nil && exists1 == 1 && exists2 == 1 + name1 = generationStreamKey(ctx, rdb, nodeStreamName(node1.PoolName, node1.ID)) + name2 = generationStreamKey(ctx, rdb, nodeStreamName(node2.PoolName, node2.ID)) + return name1 != "" && name2 != "" }, max, delay, "Node streams should exist initially") // Set node2's last seen time to a stale value close(node2.stop) - _, err := node2.nodeKeepAliveMap.Set(ctx, node2.ID, - strconv.FormatInt(time.Now().Add(-3*node2.workerTTL).UnixNano(), 10)) + _, err := node2.nodeKeepAliveMap.Set(ctx, node2.ID, "0") assert.NoError(t, err) node2.wg.Wait() node2.stop = make(chan struct{}) // so we can close @@ -1054,12 +1698,58 @@ func TestStaleNodeStreamCleanup(t *testing.T) { _, exists := node1.nodeKeepAliveMap.Get(node2.ID) return !exists }, max, delay, "Stale node should have been removed from keep-alive map") - - // Clean up - assert.NoError(t, node2.Close(ctx)) + lifecycle, err := rdb.HGetAll(ctx, "pulse:stream:"+nodeStreamName(node2.PoolName, node2.ID)+":lifecycle").Result() + require.NoError(t, err) + require.Equal(t, node2.nodeStream.Generation(), lifecycle["generation"]) + require.Equal(t, "destroyed", lifecycle["state"]) + + // A stale node that resumes observes the cleanup fence before mutating + // ownership and performs complete local teardown. + err = node2.ensureGenerationActive(ctx) + require.ErrorIs(t, err, ErrPoolGenerationLost) + require.Eventually(t, node2.IsClosed, max, delay) + require.NoError(t, node2.Close(ctx)) assert.NoError(t, node1.Shutdown(ctx)) } +func TestInactiveNodeDiscoveryRemainsUntilStreamDestroySucceeds(t *testing.T) { + ctx := ptesting.NewTestContext(t) + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + staleID := "stale-node" + hook := &poolRedisHook{ + failure: errors.New("destroy failed"), + key: "pulse:stream:" + nodeStreamName(t.Name(), staleID) + ":lifecycle", + } + rdb.AddHook(hook) + node := newTestNode(t, ctx, rdb, t.Name()) + staleStream, err := streaming.NewStream(nodeStreamName(node.PoolName, staleID), rdb) + require.NoError(t, err) + _, err = staleStream.Add(ctx, evInit, []byte(staleID)) + require.NoError(t, err) + now, err := rdb.Time(ctx).Result() + require.NoError(t, err) + _, err = node.nodeKeepAliveMap.SetAndWait( + ctx, + staleID, + strconv.FormatInt(now.Add(-2*node.workerTTL).UnixNano(), 10), + ) + require.NoError(t, err) + hook.failDestroy.Store(true) + + node.cleanupInactiveNodes() + _, exists := node.nodeKeepAliveMap.Get(staleID) + require.True(t, exists) + + hook.failDestroy.Store(false) + node.cleanupInactiveNodes() + require.Eventually(t, func() bool { + _, exists = node.nodeKeepAliveMap.Get(staleID) + return !exists + }, max, delay) + require.NoError(t, node.Shutdown(ctx)) +} + func TestShutdownStopsAllJobs(t *testing.T) { testName := strings.Replace(t.Name(), "/", "_", -1) ctx := ptesting.NewTestContext(t) @@ -1147,7 +1837,8 @@ func TestWorkerAckStreams(t *testing.T) { assert.Same(t, stream1, stream2, "Expected same stream instance to be returned") // Verify stream exists before shutdown - streamKey := "pulse:stream:" + nodeStreamName(testName, node.ID) + streamKey := generationStreamKey(ctx, rdb, nodeStreamName(testName, node.ID)) + require.NotEmpty(t, streamKey) exists, err := rdb.Exists(ctx, streamKey).Result() assert.NoError(t, err) assert.Equal(t, int64(1), exists, "Expected stream to exist before shutdown") @@ -1187,8 +1878,7 @@ func TestStaleWorkerCleanupAfterJobRequeue(t *testing.T) { // Make the worker stale by stopping it and setting an old keepalive staleWorker.stop(ctx) - _, err := node.workerKeepAliveMap.Set(ctx, staleWorker.ID, - strconv.FormatInt(time.Now().Add(-2*node.workerTTL).UnixNano(), 10)) + _, err := node.workerKeepAliveMap.Set(ctx, staleWorker.ID, "0") require.NoError(t, err) // Create a new worker to receive requeued jobs @@ -1197,7 +1887,7 @@ func TestStaleWorkerCleanupAfterJobRequeue(t *testing.T) { // Wait for cleanup to happen and jobs to be requeued require.Eventually(t, func() bool { return len(newWorker.Jobs()) == 3 - }, max, delay, "Jobs were not requeued to new worker") + }, 2*node.workerTTL, delay, "Jobs were not requeued to new worker") // Verify stale worker was deleted require.Eventually(t, func() bool { @@ -1325,7 +2015,7 @@ func TestRequeueOrphanedPayloads(t *testing.T) { testName := strings.Replace(t.Name(), "/", "_", -1) ctx := ptesting.NewTestContext(t) rdb := ptesting.NewRedisClient(t) - node := newFastCleanupTestNode(t, ctx, rdb, testName) + node := newTestNode(t, ctx, rdb, testName) worker := newTestWorker(t, ctx, node) defer ptesting.CleanupRedis(t, rdb, true, testName) @@ -1363,7 +2053,7 @@ func TestRequeueOrphanedPayloads(t *testing.T) { // The requeued jobs may end up on this worker or another (if present); // in this test there is only one worker, so they should all reappear here. return len(jobs) == len(tt.setupJobs) - }, max, delay, fmt.Sprintf("Orphaned payload requeue did not restore job keys; expected %d jobs in jobMap", len(tt.setupJobs))) + }, 15*time.Second, delay, fmt.Sprintf("Orphaned payload requeue did not restore job keys; expected %d jobs in jobMap", len(tt.setupJobs))) assert.NoError(t, node.Shutdown(ctx)) }) @@ -1383,7 +2073,7 @@ func requireActiveWorkerRing(t *testing.T, nodes []*Node, workerIDs ...string) { } } return true - }, max, delay, "active worker ring did not converge") + }, 5*time.Second, delay, "active worker ring did not converge") } // orphanedPayloadGrace mirrors the recovery grace used by @@ -1391,8 +2081,8 @@ func requireActiveWorkerRing(t *testing.T, nodes []*Node, workerIDs ...string) { // an unrelated timing constant. func orphanedPayloadGrace(node *Node) time.Duration { grace := 2 * node.workerTTL - if grace < node.ackGracePeriod { - return node.ackGracePeriod + if grace < node.recoveryGrace { + return node.recoveryGrace } return grace } diff --git a/pool/pool_map.go b/pool/pool_map.go new file mode 100644 index 0000000..30ad9ff --- /dev/null +++ b/pool/pool_map.go @@ -0,0 +1,248 @@ +// Pool map mutations share the pool stream's Redis-owned generation fence. +// Worker and node operations use these recipes so cleanup and stale processes +// cannot interleave a lifecycle check with recreation of a deleted old map. +package pool + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/redis/go-redis/v9" + + "goa.design/pulse/rmap" +) + +// mutatePoolMapScript applies one rmap mutation only while the exact pool +// stream generation is active. It emits the canonical rmap wire update in the +// same Redis operation, so paused old-generation nodes cannot recreate deleted +// map keys after cleanup. +var mutatePoolMapScript = redis.NewScript(` +if redis.call("HGET", KEYS[1], "state") ~= ARGV[1] or + redis.call("HGET", KEYS[1], "generation") ~= ARGV[2] then + return redis.error_reply("POOLGENERATIONLOST") +end +if redis.call("HGET", KEYS[4], ARGV[6]) then + return redis.error_reply("NODECLEANUPLOST") +end + +local operation = ARGV[3] +local key = ARGV[4] +local value = ARGV[5] +local changed = false +local update = "" + +if operation == "set" then + redis.call("HSET", KEYS[2], key, value) + changed = true + update = value +elseif operation == "delete" then + changed = redis.call("HDEL", KEYS[2], key) == 1 +elseif operation == "append" then + local values = {} + local current = redis.call("HGET", KEYS[2], key) + if current then + local ok, decoded = pcall(cjson.decode, current) + if not ok or type(decoded) ~= "table" then + return redis.error_reply("POOLMAPINVALID") + end + values = decoded + end + for _, item in ipairs(values) do + if item == value then + return 0 + end + end + table.insert(values, value) + update = cjson.encode(values) + redis.call("HSET", KEYS[2], key, update) + changed = true +elseif operation == "remove" then + local current = redis.call("HGET", KEYS[2], key) + if not current then + return 0 + end + local ok, values = pcall(cjson.decode, current) + if not ok or type(values) ~= "table" then + return redis.error_reply("POOLMAPINVALID") + end + local remaining = {} + for _, item in ipairs(values) do + if item ~= value then + table.insert(remaining, item) + else + changed = true + end + end + if not changed then + return 0 + end + if #remaining == 0 then + redis.call("HDEL", KEYS[2], key) + else + update = cjson.encode(remaining) + redis.call("HSET", KEYS[2], key, update) + end +else + return redis.error_reply("POOLMAPOPERATION") +end + +if not changed then + return 0 +end +local rev = tostring(redis.call("HINCRBY", KEYS[2], "=rev", 1)) +if operation == "delete" or (operation == "remove" and update == "") then + redis.call("HSET", KEYS[2], "=kind", "del") + local msg = struct.pack("ic0ic0", string.len(key), key, string.len(rev), rev) + redis.call("PUBLISH", KEYS[3], "del:" .. msg) +else + redis.call("HSET", KEYS[2], "=kind", "set") + local msg = struct.pack( + "ic0ic0ic0", + string.len(key), key, + string.len(update), update, + string.len(rev), rev + ) + redis.call("PUBLISH", KEYS[3], "set:" .. msg) +end +return 1 +`) + +// testAndSetPoolMapScript atomically advances a distributed ticker only while +// its exact pool generation remains active. +var testAndSetPoolMapScript = redis.NewScript(` +if redis.call("HGET", KEYS[1], "state") ~= ARGV[1] or + redis.call("HGET", KEYS[1], "generation") ~= ARGV[2] then + return redis.error_reply("POOLGENERATIONLOST") +end +if redis.call("HGET", KEYS[4], ARGV[6]) then + return redis.error_reply("NODECLEANUPLOST") +end +local current = redis.call("HGET", KEYS[2], ARGV[3]) +if (ARGV[4] == "" and current) or + (ARGV[4] ~= "" and current ~= ARGV[4]) then + return current or "" +end +redis.call("HSET", KEYS[2], ARGV[3], ARGV[5]) +local rev = tostring(redis.call("HINCRBY", KEYS[2], "=rev", 1)) +redis.call("HSET", KEYS[2], "=kind", "set") +local msg = struct.pack( + "ic0ic0ic0", + string.len(ARGV[3]), ARGV[3], + string.len(ARGV[5]), ARGV[5], + string.len(rev), rev +) +redis.call("PUBLISH", KEYS[3], "set:" .. msg) +return ARGV[4] +`) + +// setPoolMap stores one value under the node's exact pool-generation fence. +func (node *Node) setPoolMap(ctx context.Context, name, key, value string) error { + return node.mutatePoolMap(ctx, name, "set", key, value) +} + +// setPoolMapAndWait stores one value and waits until the joined local replica +// has observed the Redis-published revision. +func (node *Node) setPoolMapAndWait( + ctx context.Context, + m *rmap.Map, + name, key, value string, +) error { + if err := node.setPoolMap(ctx, name, key, value); err != nil { + return err + } + return waitPoolMapValue(ctx, m, key, value) +} + +// waitPoolMapValue waits until the joined replica observes an already +// committed scripted map mutation. +func waitPoolMapValue(ctx context.Context, m *rmap.Map, key, value string) error { + ticker := time.NewTicker(time.Millisecond) + defer ticker.Stop() + for { + if current, ok := m.Get(key); ok && current == value { + return nil + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-ticker.C: + } + } +} + +// deletePoolMap removes one value under the node's exact pool-generation fence. +func (node *Node) deletePoolMap(ctx context.Context, name, key string) error { + return node.mutatePoolMap(ctx, name, "delete", key, "") +} + +// testAndSetPoolMap replaces expected with value and returns the prior value +// under the exact generation fence. +func (node *Node) testAndSetPoolMap( + ctx context.Context, + name, key, expected, value string, +) (string, error) { + previous, err := testAndSetPoolMapScript.Run( + ctx, + node.rdb, + []string{ + fmt.Sprintf("pulse:stream:%s:lifecycle", node.poolStream.Name), + rmapContentKey(name), + rmapUpdateChannel(name), + rmapContentKey(node.resources.nodeKeepAlive), + }, + "active", + node.resources.generation, + key, + expected, + value, + nodeCleanupField(node.ID), + ).Text() + if err != nil { + return "", poolBoundaryError(err) + } + return previous, nil +} + +// appendPoolMapValue appends one unique array value under the exact generation +// fence used by worker ownership maps. +func (node *Node) appendPoolMapValue(ctx context.Context, name, key, value string) error { + return node.mutatePoolMap(ctx, name, "append", key, value) +} + +// removePoolMapValue removes one array value under the exact generation fence. +func (node *Node) removePoolMapValue(ctx context.Context, name, key, value string) error { + return node.mutatePoolMap(ctx, name, "remove", key, value) +} + +// mutatePoolMap verifies the active lifecycle record and publishes one +// generation-owned map mutation atomically. +func (node *Node) mutatePoolMap(ctx context.Context, name, operation, key, value string) error { + err := mutatePoolMapScript.Run( + ctx, + node.rdb, + []string{ + fmt.Sprintf("pulse:stream:%s:lifecycle", node.poolStream.Name), + rmapContentKey(name), + rmapUpdateChannel(name), + rmapContentKey(node.resources.nodeKeepAlive), + }, + "active", + node.resources.generation, + operation, + key, + value, + nodeCleanupField(node.ID), + ).Err() + if err == nil { + return nil + } + boundaryErr := poolBoundaryError(err) + if errors.Is(boundaryErr, ErrPoolGenerationLost) { + if generationErr := node.ensureGenerationActive(ctx); generationErr != nil { + return generationErr + } + } + return boundaryErr +} diff --git a/pool/resources.go b/pool/resources.go new file mode 100644 index 0000000..488f4f0 --- /dev/null +++ b/pool/resources.go @@ -0,0 +1,379 @@ +// Package pool binds every shared Redis resource to one pool-stream +// incarnation. Existing deployments are adopted in place once; resources +// created after explicit cleanup are generation-qualified. +package pool + +import ( + "context" + "fmt" + "strconv" + "time" + + redis "github.com/redis/go-redis/v9" +) + +type ( + // poolResources is the immutable Redis namespace selected for one pool + // stream generation. + poolResources struct { + pool string + generation string + nodeKeepAlive string + nodeShutdown string + workers string + workerKeepAlive string + workerCleanup string + jobs string + jobPending string + dispatches string + jobPayloads string + tickers string + schedulerJobs string + maxQueuedJobs int + workerTTL time.Duration + cleanupLease time.Duration + dispatchResultRetention time.Duration + } +) + +const ( + poolResourceStateActive = "active" + poolResourceStateDestroyed = "destroyed" +) + +var ( + // establishPoolResourcesScript adopts flat pre-generation resources when no + // manifest exists. Once cleanup marks a manifest destroyed, the next stream + // generation receives the supplied qualified names. + establishPoolResourcesScript = redis.NewScript(` +if redis.call("HGET", KEYS[2], "state") ~= ARGV[4] +or redis.call("HGET", KEYS[2], "generation") ~= ARGV[1] then + return redis.error_reply("POOLGENERATIONLOST") +end +local state = redis.call("HGET", KEYS[1], "state") +local generation = redis.call("HGET", KEYS[1], "generation") +local function has_values(name) + for _, key in ipairs(redis.call("HKEYS", "map:" .. name .. ":content")) do + if string.sub(key, 1, 1) ~= "=" then + return true + end + end + return false +end +if not redis.call("HGET", KEYS[1], "format_version") then + for i = 10, 20 do + if has_values(ARGV[i]) then + return redis.error_reply("POOLQUIESCENCEREQUIRED") + end + end + local pool = string.sub(ARGV[10], 1, string.len(ARGV[10]) - string.len(":node-keepalive")) + local map_prefix = "map:" .. pool .. ":" + local cursor = "0" + repeat + local scan = redis.call("SCAN", cursor, "MATCH", "map:*:content", "COUNT", 100) + cursor = scan[1] + for _, key in ipairs(scan[2]) do + if string.sub(key, 1, string.len(map_prefix)) == map_prefix then + for _, field in ipairs(redis.call("HKEYS", key)) do + if string.sub(field, 1, 1) ~= "=" then + return redis.error_reply("POOLQUIESCENCEREQUIRED") + end + end + end + end + until cursor == "0" + local physical = redis.call("HGET", KEYS[2], "physical_key") + if physical and redis.call("EXISTS", physical) == 1 then + if redis.call("XLEN", physical) > 0 or #redis.call("XINFO", "GROUPS", physical) > 0 then + return redis.error_reply("POOLQUIESCENCEREQUIRED") + end + end + local node_prefix = "pulse:stream:" .. pool .. ":node:" + cursor = "0" + repeat + local scan = redis.call("SCAN", cursor, "MATCH", "pulse:stream:*", "COUNT", 100) + cursor = scan[1] + for _, key in ipairs(scan[2]) do + if string.sub(key, 1, string.len(node_prefix)) == node_prefix then + local suffix = string.sub(key, string.len(node_prefix) + 1) + if not string.find(suffix, ":", 1, true) then + redis.call("DEL", key) + end + end + end + until cursor == "0" +end +if generation and not redis.call("HGET", KEYS[1], "format_version") then + local keepalive = redis.call("HGET", KEYS[1], "node_keepalive") + if keepalive then + for _, key in ipairs(redis.call("HKEYS", "map:" .. keepalive .. ":content")) do + if string.sub(key, 1, 1) ~= "=" then + return redis.error_reply("POOLQUIESCENCEREQUIRED") + end + end + end +end +if state == ARGV[2] and generation == ARGV[1] then + local format = redis.call("HGET", KEYS[1], "format_version") + if format and (format ~= ARGV[9] + or redis.call("HGET", KEYS[1], "max_queued_jobs") ~= ARGV[5] + or redis.call("HGET", KEYS[1], "worker_ttl_ms") ~= ARGV[6] + or redis.call("HGET", KEYS[1], "cleanup_lease_ms") ~= ARGV[7] + or redis.call("HGET", KEYS[1], "dispatch_result_retention_ms") ~= ARGV[8]) then + return redis.error_reply("POOLCONFIGMISMATCH") + end + if format then + return redis.call("HMGET", KEYS[1], + "generation", "node_keepalive", "node_shutdown", "workers", + "worker_keepalive", "worker_cleanup", "jobs", "job_pending", + "dispatches", "job_payloads", "tickers", "scheduler_jobs", + "max_queued_jobs", "worker_ttl_ms", "cleanup_lease_ms", + "dispatch_result_retention_ms", "format_version") + end +end +local use_qualified = state == ARGV[3] +local offset = use_qualified and 21 or 10 +redis.call("HSET", KEYS[1], + "state", ARGV[2], + "generation", ARGV[1], + "node_keepalive", ARGV[offset], + "node_shutdown", ARGV[offset + 1], + "workers", ARGV[offset + 2], + "worker_keepalive", ARGV[offset + 3], + "worker_cleanup", ARGV[offset + 4], + "jobs", ARGV[offset + 5], + "job_pending", ARGV[offset + 6], + "dispatches", ARGV[offset + 7], + "job_payloads", ARGV[offset + 8], + "tickers", ARGV[offset + 9], + "scheduler_jobs", ARGV[offset + 10], + "max_queued_jobs", ARGV[5], + "worker_ttl_ms", ARGV[6], + "cleanup_lease_ms", ARGV[7], + "dispatch_result_retention_ms", ARGV[8], + "format_version", ARGV[9]) +return redis.call("HMGET", KEYS[1], + "generation", "node_keepalive", "node_shutdown", "workers", + "worker_keepalive", "worker_cleanup", "jobs", "job_pending", + "dispatches", "job_payloads", "tickers", "scheduler_jobs", + "max_queued_jobs", "worker_ttl_ms", "cleanup_lease_ms", + "dispatch_result_retention_ms", "format_version") +`) +) + +// establishPoolResources records or loads the exact resource names selected +// for generation. The first observed generation adopts legacy flat names. +func establishPoolResources( + ctx context.Context, + rdb *redis.Client, + pool, generation string, + maxQueuedJobs int, + workerTTL, cleanupLease, dispatchResultRetention time.Duration, +) (poolResources, error) { + flat := flatPoolResources(pool, generation) + qualified := qualifiedPoolResources(pool, generation) + values, err := establishPoolResourcesScript.Run( + ctx, + rdb, + []string{ + poolResourcesKey(pool), + fmt.Sprintf("pulse:stream:%s:lifecycle", poolStreamName(pool)), + }, + append( + []any{ + generation, + poolResourceStateActive, + poolResourceStateDestroyed, + "active", + fmt.Sprintf("%d", maxQueuedJobs), + strconv.FormatInt(workerTTL.Milliseconds(), 10), + strconv.FormatInt(cleanupLease.Milliseconds(), 10), + strconv.FormatInt(dispatchResultRetention.Milliseconds(), 10), + "7", + }, + append(flat.names(), qualified.names()...)..., + )..., + ).Slice() + if err != nil { + return poolResources{}, fmt.Errorf("establish pool %q resources: %w", pool, poolBoundaryError(err)) + } + resources, err := parsePoolResources(values) + resources.pool = pool + return resources, err +} + +// loadPoolResources loads the exact resource manifest for generation. +func loadPoolResources(ctx context.Context, rdb *redis.Client, pool, generation string) (poolResources, error) { + values, err := rdb.HMGet( + ctx, + poolResourcesKey(pool), + "generation", + "node_keepalive", + "node_shutdown", + "workers", + "worker_keepalive", + "worker_cleanup", + "jobs", + "job_pending", + "dispatches", + "job_payloads", + "tickers", + "scheduler_jobs", + "max_queued_jobs", + "worker_ttl_ms", + "cleanup_lease_ms", + "dispatch_result_retention_ms", + "format_version", + ).Result() + if err != nil { + return poolResources{}, fmt.Errorf("load pool %q resources: %w", pool, err) + } + resources, err := parsePoolResources(values) + if err != nil { + return poolResources{}, fmt.Errorf("load pool %q resources: %w", pool, err) + } + if resources.generation != generation { + return poolResources{}, fmt.Errorf( + "pool %q resource generation mismatch: have %q, need %q", + pool, + resources.generation, + generation, + ) + } + resources.pool = pool + return resources, nil +} + +// parsePoolResources validates the Redis-owned resource manifest. +func parsePoolResources(values []any) (poolResources, error) { + if len(values) != 17 { + return poolResources{}, fmt.Errorf("invalid resource manifest length %d", len(values)) + } + decoded := make([]string, len(values)) + for i, value := range values { + name, ok := value.(string) + if !ok || name == "" { + return poolResources{}, fmt.Errorf("invalid resource manifest field %d: %T", i, value) + } + decoded[i] = name + } + maxQueuedJobs, err := strconv.Atoi(decoded[12]) + if err != nil || maxQueuedJobs <= 0 { + return poolResources{}, fmt.Errorf("invalid resource manifest capacity %q", decoded[12]) + } + workerTTLMillis, err := strconv.ParseInt(decoded[13], 10, 64) + if err != nil || workerTTLMillis <= 0 { + return poolResources{}, fmt.Errorf("invalid resource manifest worker TTL %q", decoded[13]) + } + cleanupLeaseMillis, err := strconv.ParseInt(decoded[14], 10, 64) + if err != nil || cleanupLeaseMillis <= 0 { + return poolResources{}, fmt.Errorf("invalid resource manifest cleanup lease %q", decoded[14]) + } + dispatchRetentionMillis, err := strconv.ParseInt(decoded[15], 10, 64) + if err != nil || dispatchRetentionMillis <= 0 { + return poolResources{}, fmt.Errorf( + "invalid resource manifest dispatch result retention %q", + decoded[15], + ) + } + if decoded[16] != "7" { + return poolResources{}, fmt.Errorf("unsupported resource manifest format %q", decoded[16]) + } + return poolResources{ + generation: decoded[0], + nodeKeepAlive: decoded[1], + nodeShutdown: decoded[2], + workers: decoded[3], + workerKeepAlive: decoded[4], + workerCleanup: decoded[5], + jobs: decoded[6], + jobPending: decoded[7], + dispatches: decoded[8], + jobPayloads: decoded[9], + tickers: decoded[10], + schedulerJobs: decoded[11], + maxQueuedJobs: maxQueuedJobs, + workerTTL: time.Duration(workerTTLMillis) * time.Millisecond, + cleanupLease: time.Duration(cleanupLeaseMillis) * time.Millisecond, + dispatchResultRetention: time.Duration(dispatchRetentionMillis) * time.Millisecond, + }, nil +} + +// flatPoolResources returns the pre-generation resource layout. +func flatPoolResources(pool, generation string) poolResources { + return poolResources{ + pool: pool, + generation: generation, + nodeKeepAlive: nodeKeepAliveMapName(pool), + nodeShutdown: nodeShutdownMapName(pool), + workers: workerMapName(pool), + workerKeepAlive: workerKeepAliveMapName(pool), + workerCleanup: workerCleanupMapName(pool), + jobs: jobMapName(pool), + jobPending: jobPendingMapName(pool), + dispatches: dispatchMapName(pool), + jobPayloads: jobPayloadMapName(pool), + tickers: tickerMapName(pool), + schedulerJobs: schedulerJobMapName(pool), + } +} + +// qualifiedPoolResources returns the namespace used after explicit cleanup. +func qualifiedPoolResources(pool, generation string) poolResources { + suffix := fmt.Sprintf(":generation:%s", generation) + resources := flatPoolResources(pool, generation) + resources.nodeKeepAlive += suffix + resources.nodeShutdown += suffix + resources.workers += suffix + resources.workerKeepAlive += suffix + resources.workerCleanup += suffix + resources.jobs += suffix + resources.jobPending += suffix + resources.dispatches += suffix + resources.jobPayloads += suffix + resources.tickers += suffix + resources.schedulerJobs += suffix + return resources +} + +// names returns the manifest names in the Lua contract order. +func (r poolResources) names() []any { + return []any{ + r.nodeKeepAlive, + r.nodeShutdown, + r.workers, + r.workerKeepAlive, + r.workerCleanup, + r.jobs, + r.jobPending, + r.dispatches, + r.jobPayloads, + r.tickers, + r.schedulerJobs, + } +} + +// mapNames returns every generation-owned rmap name, including the dispatch +// map name whose rmap-format keys only legacy layouts populate: cleanup must +// delete those too or an adopted deployment retains them forever. +func (r poolResources) mapNames() []string { + return []string{ + r.nodeKeepAlive, + r.workers, + r.dispatches, + r.workerKeepAlive, + r.workerCleanup, + r.jobs, + r.jobPending, + r.jobPayloads, + r.tickers, + r.schedulerJobs, + r.nodeShutdown, + } +} + +// poolResourcesKey stores the exact resource manifest for the active or most +// recently destroyed pool generation. +func poolResourcesKey(pool string) string { + return fmt.Sprintf("pulse:pool:%s:resources", pool) +} diff --git a/pool/scheduler.go b/pool/scheduler.go index 1ef1119..56a4519 100644 --- a/pool/scheduler.go +++ b/pool/scheduler.go @@ -2,11 +2,16 @@ package pool import ( "context" + "encoding/hex" + "errors" "fmt" + "strings" "time" + "github.com/oklog/ulid/v2" + redis "github.com/redis/go-redis/v9" + "goa.design/pulse/pulse" - "goa.design/pulse/rmap" ) type ( @@ -19,10 +24,19 @@ type ( Name() string // Plan computes the list of jobs to start and job keys to stop. // Returning ErrScheduleStop indicates that the recurring - // schedule should be stopped. + // schedule should be stopped. Legacy Plan calls cannot be cancelled; + // implement ContextJobProducer when planning may block. Plan() (*JobPlan, error) } + // ContextJobProducer adds cancellable planning without breaking the v1 + // JobProducer contract. The scheduler prefers PlanContext when implemented; + // Node.Close cancels its context and joins the in-flight transition. + ContextJobProducer interface { + JobProducer + PlanContext(ctx context.Context) (*JobPlan, error) + } + // JobPlan represents a list of jobs to start and job keys to stop. JobPlan struct { // Jobs to start. @@ -53,104 +67,268 @@ type ( producer JobProducer // node is the node running the scheduler. node *Node - // ticker is the ticker used to run the scheduler. - ticker *Ticker - // jobMap is the map of jobs keyed by job key. - jobMap *rmap.Map + // keyPrefix scopes ownership records in the generation-owned scheduler + // map. + keyPrefix string + // transitionPrefix scopes Redis-owned due time and lease fields. + transitionPrefix string + // owner is this local schedule's unique transition owner token. + owner string + // lease is the renewable Redis-time transition lease. + lease time.Duration // logger is the logger used by the scheduler. logger pulse.Logger } ) -// ErrScheduleStop is returned by JobProducer.Plan to indicate that the -// corresponding schedule should be stopped. -var ErrScheduleStop = fmt.Errorf("stop") +// ErrScheduleStop is returned by JobProducer.Plan or +// ContextJobProducer.PlanContext to stop the corresponding schedule. +var ( + ErrScheduleStop = fmt.Errorf("stop") -// Schedule calls the producer Plan method on the given interval and starts and -// stops jobs accordingly. The schedule stops when the producer Plan method -// returns ErrScheduleStop. Plan is called on only one of the nodes that -// scheduled the same producer. + // stopSchedulerJobScript atomically verifies exact scheduler ownership, + // publishes the durable stop request, and removes the ownership record. + stopSchedulerJobScript = redis.NewScript(` +if redis.call("HGET", KEYS[1], "state") ~= ARGV[1] +or redis.call("HGET", KEYS[1], "generation") ~= ARGV[2] then + return redis.error_reply("POOLGENERATIONLOST") +end` + nodeLivenessFenceLua + ` +local clock = redis.call("TIME") +local now = tonumber(clock[1]) * 1000 + math.floor(tonumber(clock[2]) / 1000) +if redis.call("HGET", KEYS[2], ARGV[7] .. "owner") ~= ARGV[8] +or redis.call("HGET", KEYS[2], ARGV[7] .. "active_fence") ~= ARGV[9] +or tonumber(redis.call("HGET", KEYS[2], ARGV[7] .. "lease_until") or "0") <= now then + return redis.error_reply("SCHEDULERLEASELOST") +end +if redis.call("HGET", KEYS[2], ARGV[3]) ~= ARGV[4] then + return 0 +end +local stream = redis.call("HGET", KEYS[1], "physical_key") +if not stream then + return redis.error_reply("POOLGENERATIONLOST") +end +redis.call("XADD", stream, "*", "n", ARGV[5], "p", ARGV[6]) +redis.call("HDEL", KEYS[2], ARGV[3]) +local rev = tostring(redis.call("HINCRBY", KEYS[2], "=rev", 1)) +redis.call("HSET", KEYS[2], "=kind", "del") +local message = struct.pack( + "ic0ic0", + string.len(ARGV[3]), ARGV[3], + string.len(rev), rev +) +redis.call("PUBLISH", KEYS[3], "del:" .. message) +return 1 +`) + + // releaseSchedulerOwnershipScript removes only the exact scheduler + // capability, without publishing a stop request for a foreign job. + releaseSchedulerOwnershipScript = redis.NewScript(` +if redis.call("HGET", KEYS[1], "state") ~= ARGV[1] +or redis.call("HGET", KEYS[1], "generation") ~= ARGV[2] then + return redis.error_reply("POOLGENERATIONLOST") +end` + nodeLivenessFenceLua + ` +local clock = redis.call("TIME") +local now = tonumber(clock[1]) * 1000 + math.floor(tonumber(clock[2]) / 1000) +if redis.call("HGET", KEYS[2], ARGV[5] .. "owner") ~= ARGV[6] +or redis.call("HGET", KEYS[2], ARGV[5] .. "active_fence") ~= ARGV[7] +or tonumber(redis.call("HGET", KEYS[2], ARGV[5] .. "lease_until") or "0") <= now then + return redis.error_reply("SCHEDULERLEASELOST") +end +if redis.call("HGET", KEYS[2], ARGV[3]) ~= ARGV[4] then + return 0 +end +redis.call("HDEL", KEYS[2], ARGV[3]) +local rev = tostring(redis.call("HINCRBY", KEYS[2], "=rev", 1)) +redis.call("HSET", KEYS[2], "=kind", "del") +local message = struct.pack( + "ic0ic0", + string.len(ARGV[3]), ARGV[3], + string.len(rev), rev +) +redis.call("PUBLISH", KEYS[3], "del:" .. message) +return 1 +`) +) + +// Schedule starts a distributed schedule. The shared ticker establishes +// per-producer ownership before every Plan call, including the initial call, so +// only one node computes or applies each transition. The node owns the schedule +// goroutine and ticker: caller cancellation stops this schedule, and Node.Close +// cancels and joins every remaining schedule before returning. func (node *Node) Schedule(ctx context.Context, producer JobProducer, interval time.Duration) error { - name := node.PoolName + ":" + producer.Name() - jobMap, err := rmap.Join(ctx, name, node.rdb, rmap.WithLogger(node.logger)) - if err != nil { - return fmt.Errorf("failed to join job map %s: %w", name, err) + node.lock.RLock() + defer node.lock.RUnlock() + if node.closing { + return fmt.Errorf("schedule %q: pool %q is closed", producer.Name(), node.PoolName) } - ticker, err := node.NewTicker(ctx, producer.Name(), interval) - if err != nil { - return fmt.Errorf("failed to create ticker %s: %w", name, err) + if err := node.ensureGenerationActive(ctx); err != nil { + return fmt.Errorf("schedule %q: %w", producer.Name(), err) } - sched := &scheduler{ - name: name, - interval: interval, - producer: producer, - node: node, - ticker: ticker, - jobMap: jobMap, - logger: node.logger, - } - plan, err := producer.Plan() - if err != nil { - return fmt.Errorf("failed to compute schedule: %w", err) - } - if err := sched.startJobs(ctx, plan.Start); err != nil { - return fmt.Errorf("failed to start jobs: %w", err) + if interval < time.Millisecond { + return fmt.Errorf("schedule %q: interval must be at least 1ms", producer.Name()) } - if err := sched.stopJobs(ctx, plan); err != nil { - return fmt.Errorf("failed to stop jobs: %w", err) + name := node.PoolName + ":" + producer.Name() + encodedName := hex.EncodeToString([]byte(producer.Name())) + sched := &scheduler{ + name: name, + interval: interval, + producer: producer, + node: node, + keyPrefix: encodedName + ":", + transitionPrefix: "=transition:" + encodedName + ":", + owner: "scheduler-transition-" + ulid.Make().String(), + lease: node.workerTTL, + logger: node.logger, } - - pulse.Go(sched.logger, func() { sched.scheduleJobs(ctx, ticker, producer) }) - pulse.Go(sched.logger, func() { sched.handleStop() }) + scheduleCtx, cancel := context.WithCancel(ctx) + stopNodeCancellation := context.AfterFunc(node.scheduleCtx, cancel) + node.scheduleWG.Add(1) + pulse.Go(sched.logger, func() { + defer node.scheduleWG.Done() + defer stopNodeCancellation() + defer cancel() + sched.scheduleJobs(scheduleCtx) + }) return nil } -// scheduleJobs calls Plan on ticks and starts and stops jobs as needed. -func (sched *scheduler) scheduleJobs(ctx context.Context, ticker *Ticker, producer JobProducer) { - for range ticker.C { - plan, err := producer.Plan() +// scheduleJobs claims, renews, and commits one canonical transition at a time. +func (sched *scheduler) scheduleJobs(ctx context.Context) { + var wait time.Duration + for { + if !waitForScheduler(ctx, wait) { + return + } + claim, err := sched.claimTransition(ctx) if err != nil { - if err == ErrScheduleStop { - if err := sched.jobMap.Reset(ctx); err != nil { - sched.logger.Error(err, "failed to reset job map", "scheduler", sched.name) - continue - } - return - } - sched.logger.Error(err, "failed to compute schedule", "scheduler", sched.name) + sched.logger.Error(err, "scheduler", sched.name) + wait = min(sched.interval, time.Second) continue } - sched.logger.Info("scheduling jobs", "scheduler", sched.name, "start", len(plan.Start), "stop", len(plan.Stop), "stopAll", plan.StopAll) - if err := sched.startJobs(ctx, plan.Start); err != nil { - sched.logger.Error(err, "failed to start jobs", "scheduler", sched.name) + if claim.stopped { + return + } + if !claim.owned { + wait = claim.wait + continue + } + stop, err := sched.runTransition(ctx, claim.fence) + if err != nil { + sched.logger.Error(err, "scheduler", sched.name) + wait = min(sched.interval, 100*time.Millisecond) + continue + } + if stop { + return } - if err := sched.stopJobs(ctx, plan); err != nil { - sched.logger.Error(err, "failed to stop jobs", "scheduler", sched.name) + wait = 0 + } +} + +// applyTransition computes and fully applies one owner-held scheduler +// transition. A failure leaves canonical ownership records unchanged wherever +// the corresponding side effect did not complete. +func (sched *scheduler) applyTransition(ctx context.Context, fence string) (bool, error) { + var ( + plan *JobPlan + err error + ) + if producer, ok := sched.producer.(ContextJobProducer); ok { + plan, err = producer.PlanContext(ctx) + } else { + plan, err = sched.producer.Plan() + } + if err != nil { + if errors.Is(err, ErrScheduleStop) { + if err := sched.clearJobs(ctx, fence); err != nil { + return false, fmt.Errorf("clear scheduler jobs: %w", err) + } + return true, nil } + return false, fmt.Errorf("compute schedule: %w", err) } + sched.logger.Info( + "scheduling jobs", + "scheduler", + sched.name, + "start", + len(plan.Start), + "stop", + len(plan.Stop), + "stopAll", + plan.StopAll, + ) + if err := sched.startJobs(ctx, fence, plan.Start); err != nil { + return false, fmt.Errorf("start jobs: %w", err) + } + if err := sched.stopJobs(ctx, fence, plan); err != nil { + return false, fmt.Errorf("stop jobs: %w", err) + } + return false, nil } // startJobs dispatches the given jobs. -func (sched *scheduler) startJobs(ctx context.Context, jobs []*JobParam) error { +func (sched *scheduler) startJobs(ctx context.Context, fence string, jobs []*JobParam) error { for _, job := range jobs { - err := sched.node.DispatchJob(ctx, job.Key, job.Payload) + field := sched.keyPrefix + job.Key + dispatchID, err := sched.schedulerOwnership(ctx, field) if err != nil { - sched.logger.Error(fmt.Errorf("failed to dispatch job: %w", err), "job", job.Key) - continue + return fmt.Errorf("read job %q ownership: %w", job.Key, err) } - if _, err := sched.jobMap.Set(ctx, job.Key, time.Now().String()); err != nil { - sched.logger.Error(fmt.Errorf("failed to store job: %w", err), "job", job.Key) - continue + if dispatchID == "" { + proposed := "scheduler-" + ulid.Make().String() + previous, err := sched.claimJobOwnership(ctx, fence, field, proposed) + if err != nil { + return fmt.Errorf("store job %q ownership: %w", job.Key, err) + } + dispatchID = proposed + if previous != "" { + dispatchID = previous + } + } + dispatched := &Job{ + Key: job.Key, + Payload: job.Payload, + CreatedAt: time.Now(), + NodeID: sched.node.ID, + dispatchID: dispatchID, + } + if _, err := sched.dispatchJob(ctx, fence, dispatchID, dispatched); err != nil { + release := errors.Is(err, ErrJobExists) + if !release { + identity, identityErr := dispatchIdentity(job.Key, job.Payload) + if identityErr != nil { + return fmt.Errorf("encode job %q identity: %w", job.Key, identityErr) + } + record, readErr := sched.node.readDispatchRecord(ctx, dispatchID, identity) + release = readErr == nil && record.status == dispatchTerminal + } + if release { + if releaseErr := sched.releaseOwnership(ctx, fence, field, dispatchID); releaseErr != nil { + return errors.Join( + fmt.Errorf("dispatch job %q as %q: %w", job.Key, dispatchID, err), + releaseErr, + ) + } + } + return fmt.Errorf("dispatch job %q as %q: %w", job.Key, dispatchID, err) } } return nil } // stopJobs stops jobs according to the given schedule. -func (sched *scheduler) stopJobs(ctx context.Context, plan *JobPlan) error { +func (sched *scheduler) stopJobs(ctx context.Context, fence string, plan *JobPlan) error { var toStop []string if plan.StopAll { - toStop = sched.jobMap.Keys() + ownership, err := sched.jobOwnership(ctx) + if err != nil { + return err + } + toStop = make([]string, 0, len(ownership)) + for key := range ownership { + toStop = append(toStop, key) + } for _, j := range plan.Start { for i, k := range toStop { if k == j.Key { @@ -163,26 +341,157 @@ func (sched *scheduler) stopJobs(ctx context.Context, plan *JobPlan) error { toStop = plan.Stop } for _, key := range toStop { - err := sched.node.StopJob(ctx, key) + field := sched.keyPrefix + key + dispatchID, err := sched.schedulerOwnership(ctx, field) if err != nil { - sched.logger.Error(fmt.Errorf("failed to stop job: %w", err), "job", key) + return fmt.Errorf("read job %q ownership: %w", key, err) + } + if dispatchID == "" { continue } - if _, err := sched.jobMap.Delete(ctx, key); err != nil { - sched.logger.Error(fmt.Errorf("failed to delete job: %w", err), "job", key) + stopped, err := sched.stopOwnedJob(ctx, fence, field, key, dispatchID) + if err != nil { + return fmt.Errorf("stop job %q: %w", key, err) + } + if !stopped { + continue } } return nil } -// handleStop handles the scheduler stop signal. -func (sched *scheduler) handleStop() { - ch := sched.jobMap.Subscribe() - for ev := range ch { - if ev == rmap.EventReset { - sched.logger.Info("stopping scheduler", "scheduler", sched.name) - sched.ticker.Stop() - return +// jobOwnership scans Redis rather than the eventually consistent local rmap +// cache. Each later mutation rechecks the exact field and dispatch ID. +func (sched *scheduler) jobOwnership(ctx context.Context) (map[string]string, error) { + ownership := make(map[string]string) + var cursor uint64 + for { + values, next, err := sched.node.rdb.HScan( + ctx, + rmapContentKey(sched.node.resources.schedulerJobs), + cursor, + sched.keyPrefix+"*", + 100, + ).Result() + if err != nil { + return nil, fmt.Errorf("scan scheduler ownership: %w", err) } + for index := 0; index < len(values); index += 2 { + ownership[strings.TrimPrefix(values[index], sched.keyPrefix)] = values[index+1] + } + cursor = next + if cursor == 0 { + return ownership, nil + } + } +} + +// clearJobs durably stops every Redis-authoritative job owned by this schedule. +func (sched *scheduler) clearJobs(ctx context.Context, fence string) error { + ownership, err := sched.jobOwnership(ctx) + if err != nil { + return err + } + for key, dispatchID := range ownership { + if _, err := sched.stopOwnedJob(ctx, fence, sched.keyPrefix+key, key, dispatchID); err != nil { + return err + } + } + return nil +} + +// schedulerOwnership reads one exact ownership capability from Redis. +func (sched *scheduler) schedulerOwnership(ctx context.Context, field string) (string, error) { + value, err := sched.node.rdb.HGet( + ctx, + rmapContentKey(sched.node.resources.schedulerJobs), + field, + ).Result() + if errors.Is(err, redis.Nil) { + return "", nil + } + if err != nil { + return "", err + } + return value, nil +} + +// stopOwnedJob publishes a stop request only if the exact scheduler dispatch +// capability remains current at the Redis linearization point. +func (sched *scheduler) stopOwnedJob( + ctx context.Context, + fence, field, key, dispatchID string, +) (bool, error) { + stopped, err := stopSchedulerJobScript.Run( + ctx, + sched.node.rdb, + []string{ + fmt.Sprintf("pulse:stream:%s:lifecycle", sched.node.poolStream.Name), + rmapContentKey(sched.node.resources.schedulerJobs), + rmapUpdateChannel(sched.node.resources.schedulerJobs), + rmapContentKey(sched.node.resources.nodeKeepAlive), + }, + "active", + sched.node.resources.generation, + field, + dispatchID, + evStopJob, + marshalJobKey(key), + sched.transitionPrefix, + sched.owner, + fence, + sched.node.ID, + nodeCleanupField(sched.node.ID), + ).Int64() + if err != nil { + return false, poolBoundaryError(err) + } + return stopped == 1, nil +} + +// releaseOwnership removes only the exact persisted scheduler capability. +func (sched *scheduler) releaseOwnership( + ctx context.Context, + fence, field, dispatchID string, +) error { + _, err := releaseSchedulerOwnershipScript.Run( + ctx, + sched.node.rdb, + []string{ + fmt.Sprintf("pulse:stream:%s:lifecycle", sched.node.poolStream.Name), + rmapContentKey(sched.node.resources.schedulerJobs), + rmapUpdateChannel(sched.node.resources.schedulerJobs), + rmapContentKey(sched.node.resources.nodeKeepAlive), + }, + "active", + sched.node.resources.generation, + field, + dispatchID, + sched.transitionPrefix, + sched.owner, + fence, + sched.node.ID, + nodeCleanupField(sched.node.ID), + ).Int64() + return poolBoundaryError(err) +} + +// waitForScheduler waits without allocating a ticker for the zero-delay path. +func waitForScheduler(ctx context.Context, wait time.Duration) bool { + if wait <= 0 { + select { + case <-ctx.Done(): + return false + default: + return true + } + } + timer := time.NewTimer(wait) + defer timer.Stop() + select { + case <-ctx.Done(): + return false + case <-timer.C: + return true } } diff --git a/pool/scheduler_test.go b/pool/scheduler_test.go index 66e623b..7b7ad7c 100644 --- a/pool/scheduler_test.go +++ b/pool/scheduler_test.go @@ -1,7 +1,11 @@ package pool import ( + "context" + "encoding/hex" + "errors" "sync" + "sync/atomic" "testing" "time" @@ -9,7 +13,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "goa.design/pulse/rmap" ptesting "goa.design/pulse/testing" ) @@ -70,46 +73,355 @@ func TestSchedule(t *testing.T) { return nil, nil }) - // Observe call to reset - jobMap, err := rmap.Join(ctx, testName+":"+testName, rdb) + err := node.Schedule(ctx, producer, d) require.NoError(t, err) - var reset bool - c := jobMap.Subscribe() - defer jobMap.Unsubscribe(c) - done := make(chan struct{}) - go func() { - defer close(done) - for ev := range c { - if ev == rmap.EventReset { - reset = true - return - } + + assert.Eventually(t, func() bool { return it() == 7 }, 5*time.Second, delay, "schedule should have stopped, got %d", it()) + assert.Eventually(t, func() bool { + return len(node.schedulerJobMap.Keys()) == 0 + }, time.Second, delay, "scheduler ownership should be cleared") + assert.NotContains(t, buf.String(), "level=error", "unexpected logged error") +} + +func TestSchedulePlansOnceAcrossNodes(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + poolName := ulid.Make().String() + first := newTestNode(t, ctx, rdb, poolName) + second := newTestNode(t, ctx, rdb, poolName) + var plans atomic.Int64 + newProducer := func() JobProducer { + return newTestProducer("shared", func() (*JobPlan, error) { + plans.Add(1) + return nil, ErrScheduleStop + }) + } + + require.NoError(t, first.Schedule(ctx, newProducer(), 20*time.Millisecond)) + require.NoError(t, second.Schedule(ctx, newProducer(), 20*time.Millisecond)) + require.Eventually(t, func() bool { + return plans.Load() == 1 + }, time.Second, time.Millisecond) + require.Never(t, func() bool { + return plans.Load() > 1 + }, 100*time.Millisecond, 5*time.Millisecond) + require.NoError(t, first.Shutdown(ctx)) +} + +func TestScheduleRenewsOwnershipAcrossSlowPlan(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + poolName := ulid.Make().String() + first := newTestNode(t, ctx, rdb, poolName) + second := newTestNode(t, ctx, rdb, poolName) + entered := make(chan struct{}) + release := make(chan struct{}) + var ( + plans atomic.Int64 + once sync.Once + ) + newProducer := func() JobProducer { + return &testProducer{ + name: "slow-shared", + compute: func(ctx context.Context) (*JobPlan, error) { + plans.Add(1) + once.Do(func() { + close(entered) + }) + select { + case <-release: + return nil, ErrScheduleStop + case <-ctx.Done(): + return nil, ctx.Err() + } + }, } - }() + } + require.NoError(t, first.Schedule(ctx, newProducer(), time.Millisecond)) + require.NoError(t, second.Schedule(ctx, newProducer(), time.Millisecond)) + <-entered + time.Sleep(3 * first.workerTTL) + require.EqualValues(t, 1, plans.Load()) + close(release) + require.Never(t, func() bool { + return plans.Load() > 1 + }, 100*time.Millisecond, 5*time.Millisecond) + require.NoError(t, first.Shutdown(ctx)) +} - err = node.Schedule(ctx, producer, d) - require.NoError(t, err) +func TestScheduleRetriesFailedStartTransition(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + node := newTestNode(t, ctx, rdb, ulid.Make().String()) + worker := newTestWorker(t, ctx, node) + var starts atomic.Int64 + worker.handler.(*mockHandler).startFunc = func(*Job) error { + if starts.Add(1) == 1 { + return errors.New("injected start failure") + } + return nil + } + producer := newTestProducer("retry", func() (*JobPlan, error) { + if starts.Load() >= 2 { + return nil, ErrScheduleStop + } + return &JobPlan{ + Start: []*JobParam{{Key: "job", Payload: []byte("payload")}}, + }, nil + }) + + require.NoError(t, node.Schedule(ctx, producer, 20*time.Millisecond)) + require.Eventually(t, func() bool { + return starts.Load() == 2 && len(worker.Jobs()) == 1 + }, 2*time.Second, 5*time.Millisecond) + require.NoError(t, node.Shutdown(ctx)) +} + +func TestNodeCloseCancelsAndJoinsSchedules(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + node := newTestNode(t, ctx, rdb, ulid.Make().String()) + entered := make(chan struct{}) + var once sync.Once + producer := &testProducer{ + name: "owned", + compute: func(ctx context.Context) (*JobPlan, error) { + once.Do(func() { + close(entered) + }) + <-ctx.Done() + return nil, ctx.Err() + }, + } + require.NoError(t, node.Schedule(ctx, producer, time.Millisecond)) + <-entered - assert.Eventually(t, func() bool { return it() == 7 }, max, delay, "schedule should have stopped, got %d", it()) + closed := make(chan error, 1) + go func() { + closed <- node.Close(ctx) + }() + select { + case err := <-closed: + require.Failf(t, "Close returned before schedule exited", "error: %v", err) + case <-time.After(20 * time.Millisecond): + } + require.NoError(t, <-closed) + require.True(t, node.IsClosed()) + require.NoError(t, node.poolStream.Destroy(ctx)) +} + +func TestScheduleCallerCancellationStopsOwnedSchedule(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + node := newTestNode(t, ctx, rdb, ulid.Make().String()) + scheduleCtx, cancel := context.WithCancel(ctx) + var plans atomic.Int64 + producer := newTestProducer("caller", func() (*JobPlan, error) { + plans.Add(1) + return &JobPlan{}, nil + }) + require.NoError(t, node.Schedule(scheduleCtx, producer, time.Millisecond)) + require.Eventually(t, func() bool { + return plans.Load() > 0 + }, time.Second, time.Millisecond) + cancel() + done := make(chan struct{}) + go func() { + node.scheduleWG.Wait() + close(done) + }() select { case <-done: - reset = true case <-time.After(time.Second): - break + require.Fail(t, "caller cancellation did not join schedule") } - assert.True(t, reset, "job map should have been reset") - assert.NotContains(t, buf.String(), "level=error", "unexpected logged error") + require.False(t, node.IsClosed()) + require.NoError(t, node.Close(ctx)) + require.NoError(t, node.poolStream.Destroy(ctx)) +} + +func TestSchedulerDispatchOwnershipIsExactAcrossNodes(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + poolName := ulid.Make().String() + first := newTestNode(t, ctx, rdb, poolName) + second := newTestNode(t, ctx, rdb, poolName) + worker := newTestWorker(t, ctx, first) + var starts atomic.Int64 + worker.handler.(*mockHandler).startFunc = func(*Job) error { + starts.Add(1) + return nil + } + producer := newTestProducer("shared", func() (*JobPlan, error) { + return &JobPlan{}, nil + }) + newScheduler := func(node *Node) *scheduler { + encodedName := hex.EncodeToString([]byte(producer.Name())) + return &scheduler{ + name: poolName + ":shared", + interval: 20 * time.Millisecond, + producer: producer, + node: node, + keyPrefix: encodedName + ":", + transitionPrefix: "=transition:" + encodedName + ":", + owner: "test-" + ulid.Make().String(), + lease: node.workerTTL, + logger: node.logger, + } + } + firstScheduler := newScheduler(first) + secondScheduler := newScheduler(second) + job := &JobParam{Key: "scheduled", Payload: []byte("payload")} + firstFence := claimTestSchedulerTransition(t, ctx, firstScheduler) + require.NoError(t, firstScheduler.startJobs(ctx, firstFence, []*JobParam{job})) + require.NoError(t, firstScheduler.releaseTransition(ctx, firstFence)) + secondFence := claimTestSchedulerTransition(t, ctx, secondScheduler) + require.NoError(t, secondScheduler.startJobs(ctx, secondFence, []*JobParam{job})) + require.EqualValues(t, 1, starts.Load()) + field := firstScheduler.keyPrefix + job.Key + dispatchID, err := firstScheduler.schedulerOwnership(ctx, field) + require.NoError(t, err) + require.Contains(t, dispatchID, "scheduler-") + + require.NoError(t, secondScheduler.stopJobs(ctx, secondFence, &JobPlan{Stop: []string{job.Key}})) + require.Eventually(t, func() bool { + return len(worker.Jobs()) == 0 + }, time.Second, time.Millisecond) + require.NoError(t, first.Shutdown(ctx)) +} + +func TestSchedulerStopAllRequiresCurrentTransitionOwner(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + poolName := ulid.Make().String() + first := newTestNode(t, ctx, rdb, poolName) + second := newTestNode(t, ctx, rdb, poolName) + worker := newTestWorker(t, ctx, first) + producer := newTestProducer("shared", func() (*JobPlan, error) { + return &JobPlan{}, nil + }) + newScheduler := func(node *Node) *scheduler { + encodedName := hex.EncodeToString([]byte(producer.Name())) + return &scheduler{ + name: poolName + ":shared", + interval: 20 * time.Millisecond, + producer: producer, + node: node, + keyPrefix: encodedName + ":", + transitionPrefix: "=transition:" + encodedName + ":", + owner: "test-" + ulid.Make().String(), + lease: node.workerTTL, + logger: node.logger, + } + } + owner := newScheduler(first) + contender := newScheduler(second) + job := &JobParam{Key: "scheduled", Payload: []byte("payload")} + fence := claimTestSchedulerTransition(t, ctx, owner) + require.NoError(t, owner.startJobs(ctx, fence, []*JobParam{job})) + + claim, err := contender.claimTransition(ctx) + require.NoError(t, err) + require.False(t, claim.owned) + err = contender.stopJobs(ctx, "not-owner", &JobPlan{StopAll: true}) + require.ErrorContains(t, err, "SCHEDULERLEASELOST") + require.Len(t, worker.Jobs(), 1) + dispatchID, err := owner.schedulerOwnership(ctx, owner.keyPrefix+job.Key) + require.NoError(t, err) + require.NotEmpty(t, dispatchID) + + require.NoError(t, owner.stopJobs(ctx, fence, &JobPlan{StopAll: true})) + require.Eventually(t, func() bool { + return len(worker.Jobs()) == 0 + }, time.Second, time.Millisecond) + require.NoError(t, owner.releaseTransition(ctx, fence)) + require.NoError(t, first.Shutdown(ctx)) +} + +func TestSchedulerCollisionNeverOwnsForeignJob(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + node := newTestNode(t, ctx, rdb, ulid.Make().String()) + worker := newTestWorker(t, ctx, node) + require.NoError(t, node.DispatchJob(ctx, "collision", []byte("foreign"))) + require.Eventually(t, func() bool { + return len(worker.Jobs()) == 1 + }, time.Second, time.Millisecond) + producer := newTestProducer("schedule", func() (*JobPlan, error) { + return &JobPlan{}, nil + }) + encodedName := hex.EncodeToString([]byte(producer.Name())) + sched := &scheduler{ + name: node.PoolName + ":schedule", + interval: 20 * time.Millisecond, + producer: producer, + node: node, + keyPrefix: encodedName + ":", + transitionPrefix: "=transition:" + encodedName + ":", + owner: "test-" + ulid.Make().String(), + lease: node.workerTTL, + logger: node.logger, + } + fence := claimTestSchedulerTransition(t, ctx, sched) + + err := sched.startJobs(ctx, fence, []*JobParam{{Key: "collision", Payload: []byte("scheduled")}}) + require.ErrorIs(t, err, ErrJobExists) + ownership, err := sched.schedulerOwnership(ctx, sched.keyPrefix+"collision") + require.NoError(t, err) + require.Empty(t, ownership) + require.NoError(t, sched.clearJobs(ctx, fence)) + require.Len(t, worker.Jobs(), 1) + require.Equal(t, []byte("foreign"), worker.Jobs()[0].Payload) + require.NoError(t, node.Shutdown(ctx)) } type testProducer struct { name string - compute func() (*JobPlan, error) + compute func(context.Context) (*JobPlan, error) } // newTestProducer returns a producer with the given name and compute schedule // function. func newTestProducer(name string, compute func() (*JobPlan, error)) JobProducer { - return &testProducer{name: name, compute: compute} + return &testProducer{ + name: name, + compute: func(context.Context) (*JobPlan, error) { + return compute() + }, + } +} +func (p *testProducer) Name() string { return p.name } +func (p *testProducer) Plan() (*JobPlan, error) { + return p.compute(context.Background()) +} +func (p *testProducer) PlanContext(ctx context.Context) (*JobPlan, error) { + return p.compute(ctx) +} + +// claimTestSchedulerTransition makes one constructed scheduler immediately due +// and returns its exact Redis fence. +func claimTestSchedulerTransition(t *testing.T, ctx context.Context, sched *scheduler) string { + t.Helper() + claim, err := sched.claimTransition(ctx) + require.NoError(t, err) + if !claim.owned { + require.NoError(t, sched.node.rdb.HSet( + ctx, + rmapContentKey(sched.node.resources.schedulerJobs), + sched.transitionPrefix+"next_ms", + "1", + ).Err()) + claim, err = sched.claimTransition(ctx) + require.NoError(t, err) + } + require.True(t, claim.owned) + return claim.fence } -func (p *testProducer) Name() string { return p.name } -func (p *testProducer) Plan() (*JobPlan, error) { return p.compute() } diff --git a/pool/scheduler_transition.go b/pool/scheduler_transition.go new file mode 100644 index 0000000..c33ce81 --- /dev/null +++ b/pool/scheduler_transition.go @@ -0,0 +1,555 @@ +// Scheduler transitions use one Redis-time lease from due-time claim through +// Plan, job mutations, canonical ownership scans, and next-time commit. Every +// mutating script verifies the exact pool generation, owner token, fence, and +// unexpired lease so a paused predecessor cannot apply work after takeover. +package pool + +import ( + "context" + "errors" + "fmt" + "strconv" + "sync" + "time" + + redis "github.com/redis/go-redis/v9" +) + +type ( + // schedulerClaim is the Redis-owned result of attempting one due transition. + schedulerClaim struct { + owned bool + stopped bool + fence string + wait time.Duration + } +) + +var ( + // claimSchedulerTransitionScript initializes canonical timing or acquires a + // due transition after verifying the pool generation. + claimSchedulerTransitionScript = redis.NewScript(` +if redis.call("HGET", KEYS[1], "state") ~= ARGV[1] +or redis.call("HGET", KEYS[1], "generation") ~= ARGV[2] then + return redis.error_reply("POOLGENERATIONLOST") +end` + nodeLivenessFenceLua + ` +local clock = redis.call("TIME") +local now = tonumber(clock[1]) * 1000 + math.floor(tonumber(clock[2]) / 1000) +if redis.call("HGET", KEYS[2], ARGV[3] .. "stopped") == "1" then + return {2, "", 0} +end +local interval = redis.call("HGET", KEYS[2], ARGV[3] .. "interval_ms") +if interval and interval ~= ARGV[5] then + return redis.error_reply("SCHEDULERCONFIGMISMATCH") +end +local next_at = tonumber(redis.call("HGET", KEYS[2], ARGV[3] .. "next_ms") or "0") +if next_at == 0 then + next_at = now + tonumber(ARGV[5]) + redis.call("HSET", KEYS[2], + ARGV[3] .. "interval_ms", ARGV[5], + ARGV[3] .. "next_ms", tostring(next_at)) + return {0, "", tonumber(ARGV[5])} +end +local owner = redis.call("HGET", KEYS[2], ARGV[3] .. "owner") +local lease_until = tonumber(redis.call("HGET", KEYS[2], ARGV[3] .. "lease_until") or "0") +if owner and owner ~= ARGV[4] and lease_until > now then + return {0, "", math.max(1, math.min(lease_until - now, math.max(1, next_at - now)))} +end +if next_at > now then + return {0, "", math.max(1, next_at - now)} +end +local fence = tostring(redis.call("HINCRBY", KEYS[2], ARGV[3] .. "fence", 1)) +redis.call("HSET", KEYS[2], + ARGV[3] .. "owner", ARGV[4], + ARGV[3] .. "active_fence", fence, + ARGV[3] .. "lease_until", tostring(now + tonumber(ARGV[6]))) +return {1, fence, 0} +`) + + // renewSchedulerTransitionScript extends only the exact live transition. + renewSchedulerTransitionScript = redis.NewScript(` +if redis.call("HGET", KEYS[1], "state") ~= ARGV[1] +or redis.call("HGET", KEYS[1], "generation") ~= ARGV[2] then + return redis.error_reply("POOLGENERATIONLOST") +end` + nodeLivenessFenceLua + ` +local clock = redis.call("TIME") +local now = tonumber(clock[1]) * 1000 + math.floor(tonumber(clock[2]) / 1000) +if redis.call("HGET", KEYS[2], ARGV[3] .. "owner") ~= ARGV[4] +or redis.call("HGET", KEYS[2], ARGV[3] .. "active_fence") ~= ARGV[5] +or tonumber(redis.call("HGET", KEYS[2], ARGV[3] .. "lease_until") or "0") <= now then + return redis.error_reply("SCHEDULERLEASELOST") +end +redis.call("HSET", KEYS[2], ARGV[3] .. "lease_until", tostring(now + tonumber(ARGV[6]))) +return 1 +`) + + // commitSchedulerTransitionScript advances canonical time and releases only + // the exact live transition after every planned mutation completed. + commitSchedulerTransitionScript = redis.NewScript(` +if redis.call("HGET", KEYS[1], "state") ~= ARGV[1] +or redis.call("HGET", KEYS[1], "generation") ~= ARGV[2] then + return redis.error_reply("POOLGENERATIONLOST") +end` + nodeLivenessFenceLua + ` +local clock = redis.call("TIME") +local now = tonumber(clock[1]) * 1000 + math.floor(tonumber(clock[2]) / 1000) +if redis.call("HGET", KEYS[2], ARGV[3] .. "owner") ~= ARGV[4] +or redis.call("HGET", KEYS[2], ARGV[3] .. "active_fence") ~= ARGV[5] +or tonumber(redis.call("HGET", KEYS[2], ARGV[3] .. "lease_until") or "0") <= now then + return redis.error_reply("SCHEDULERLEASELOST") +end +local interval = tonumber(redis.call("HGET", KEYS[2], ARGV[3] .. "interval_ms")) +local next_at = tonumber(redis.call("HGET", KEYS[2], ARGV[3] .. "next_ms")) +repeat + next_at = next_at + interval +until next_at > now +redis.call("HSET", KEYS[2], ARGV[3] .. "next_ms", tostring(next_at)) +redis.call("HDEL", KEYS[2], + ARGV[3] .. "owner", + ARGV[3] .. "active_fence", + ARGV[3] .. "lease_until") +return math.max(1, next_at - now) +`) + + // stopSchedulerTransitionScript removes canonical scheduling state only + // after the exact owner has stopped every scheduler-owned job. + stopSchedulerTransitionScript = redis.NewScript(` +if redis.call("HGET", KEYS[1], "state") ~= ARGV[1] +or redis.call("HGET", KEYS[1], "generation") ~= ARGV[2] then + return redis.error_reply("POOLGENERATIONLOST") +end` + nodeLivenessFenceLua + ` +local clock = redis.call("TIME") +local now = tonumber(clock[1]) * 1000 + math.floor(tonumber(clock[2]) / 1000) +if redis.call("HGET", KEYS[2], ARGV[3] .. "owner") ~= ARGV[4] +or redis.call("HGET", KEYS[2], ARGV[3] .. "active_fence") ~= ARGV[5] +or tonumber(redis.call("HGET", KEYS[2], ARGV[3] .. "lease_until") or "0") <= now then + return redis.error_reply("SCHEDULERLEASELOST") +end +redis.call("HDEL", KEYS[2], + ARGV[3] .. "interval_ms", + ARGV[3] .. "next_ms", + ARGV[3] .. "owner", + ARGV[3] .. "active_fence", + ARGV[3] .. "lease_until") +redis.call("HSET", KEYS[2], ARGV[3] .. "stopped", "1") +return 1 +`) + + // releaseSchedulerTransitionScript leaves the due time unchanged after a + // failed attempt so another owner can retry the same transition. + releaseSchedulerTransitionScript = redis.NewScript(` +if redis.call("HGET", KEYS[1], "state") ~= ARGV[1] +or redis.call("HGET", KEYS[1], "generation") ~= ARGV[2] then + return redis.error_reply("POOLGENERATIONLOST") +end` + nodeLivenessFenceLua + ` +local clock = redis.call("TIME") +local now = tonumber(clock[1]) * 1000 + math.floor(tonumber(clock[2]) / 1000) +if redis.call("HGET", KEYS[2], ARGV[3] .. "owner") ~= ARGV[4] +or redis.call("HGET", KEYS[2], ARGV[3] .. "active_fence") ~= ARGV[5] +or tonumber(redis.call("HGET", KEYS[2], ARGV[3] .. "lease_until") or "0") <= now then + return redis.error_reply("SCHEDULERLEASELOST") +end +redis.call("HDEL", KEYS[2], + ARGV[3] .. "owner", + ARGV[3] .. "active_fence", + ARGV[3] .. "lease_until") +return 1 +`) + + // claimSchedulerJobScript stores a scheduler dispatch capability only while + // the transition lease is current and publishes the matching rmap update. + claimSchedulerJobScript = redis.NewScript(` +if redis.call("HGET", KEYS[1], "state") ~= ARGV[1] +or redis.call("HGET", KEYS[1], "generation") ~= ARGV[2] then + return redis.error_reply("POOLGENERATIONLOST") +end` + nodeLivenessFenceLua + ` +local clock = redis.call("TIME") +local now = tonumber(clock[1]) * 1000 + math.floor(tonumber(clock[2]) / 1000) +if redis.call("HGET", KEYS[2], ARGV[3] .. "owner") ~= ARGV[4] +or redis.call("HGET", KEYS[2], ARGV[3] .. "active_fence") ~= ARGV[5] +or tonumber(redis.call("HGET", KEYS[2], ARGV[3] .. "lease_until") or "0") <= now then + return redis.error_reply("SCHEDULERLEASELOST") +end +local current = redis.call("HGET", KEYS[2], ARGV[6]) +if current then + return current +end +redis.call("HSET", KEYS[2], ARGV[6], ARGV[7]) +local rev = tostring(redis.call("HINCRBY", KEYS[2], "=rev", 1)) +redis.call("HSET", KEYS[2], "=kind", "set") +local message = struct.pack( + "ic0ic0ic0", + string.len(ARGV[6]), ARGV[6], + string.len(ARGV[7]), ARGV[7], + string.len(rev), rev +) +redis.call("PUBLISH", KEYS[3], "set:" .. message) +return "" +`) + + // dispatchScheduledJobScript admits the exact persisted scheduler dispatch + // only while the transition owner/fence is live. + dispatchScheduledJobScript = redis.NewScript(` +if redis.call("HGET", KEYS[1], "state") ~= ARGV[1] +or redis.call("HGET", KEYS[1], "generation") ~= ARGV[2] then + return redis.error_reply("POOLGENERATIONLOST") +end` + nodeLivenessFenceLua + ` +local clock = redis.call("TIME") +local now = tonumber(clock[1]) * 1000 + math.floor(tonumber(clock[2]) / 1000) +if redis.call("HGET", KEYS[2], ARGV[3] .. "owner") ~= ARGV[4] +or redis.call("HGET", KEYS[2], ARGV[3] .. "active_fence") ~= ARGV[5] +or tonumber(redis.call("HGET", KEYS[2], ARGV[3] .. "lease_until") or "0") <= now then + return redis.error_reply("SCHEDULERLEASELOST") +end +local identity = redis.call("HGET", KEYS[6], "identity") +if identity then + if identity ~= ARGV[12] then + return redis.error_reply("DISPATCHIDEMPOTENCYCONFLICT") + end + local state = redis.call("HGET", KEYS[6], "state") + return { + state == "terminal" and 2 or 1, + redis.call("HGET", KEYS[6], "event"), + redis.call("HGET", KEYS[6], "result") or "", + redis.call("HGET", KEYS[6], "error") or "" + } +end +if redis.call("HGET", KEYS[3], ARGV[6]) then + return {4, "", "", ""} +end +if redis.call("HGET", KEYS[4], ARGV[6]) then + return {3, "", "", ""} +end +local count = 0 +for _, key in ipairs(redis.call("HKEYS", KEYS[4])) do + if string.sub(key, 1, 1) ~= "=" then + count = count + 1 + end +end +if count >= tonumber(ARGV[8]) then + return {5, "", "", ""} +end +local stream = redis.call("HGET", KEYS[1], ARGV[11]) +if not stream then + return redis.error_reply("POOLGENERATIONLOST") +end +local event_id = redis.call("XADD", stream, "*", "n", ARGV[9], "p", ARGV[10]) +redis.call("HSET", KEYS[6], + "id", ARGV[7], + "identity", ARGV[12], + "key", ARGV[6], + "event", event_id, + "state", "pending", + "result", "", + "error", "") +redis.call("SADD", KEYS[7], KEYS[6]) +redis.call("HSET", KEYS[4], ARGV[6], ARGV[7]) +local rev = tostring(redis.call("HINCRBY", KEYS[4], "=rev", 1)) +redis.call("HSET", KEYS[4], "=kind", "set") +local msg = struct.pack( + "ic0ic0ic0", + string.len(ARGV[6]), ARGV[6], + string.len(ARGV[7]), ARGV[7], + string.len(rev), rev +) +redis.call("PUBLISH", KEYS[5], "set:" .. msg) +return {1, event_id, "", ""} +`) +) + +// claimTransition claims one due transition or returns the Redis-derived wait. +func (sched *scheduler) claimTransition(ctx context.Context) (schedulerClaim, error) { + raw, err := claimSchedulerTransitionScript.Run( + ctx, + sched.node.rdb, + sched.transitionKeys(), + "active", + sched.node.resources.generation, + sched.transitionPrefix, + sched.owner, + strconv.FormatInt(sched.interval.Milliseconds(), 10), + strconv.FormatInt(sched.lease.Milliseconds(), 10), + sched.node.ID, + nodeCleanupField(sched.node.ID), + ).Slice() + if err != nil { + return schedulerClaim{}, poolBoundaryError(err) + } + if len(raw) != 3 { + return schedulerClaim{}, fmt.Errorf("scheduler claim returned %d fields", len(raw)) + } + status, ok := raw[0].(int64) + if !ok || status < 0 || status > 2 { + return schedulerClaim{}, fmt.Errorf("scheduler claim returned invalid status %T(%v)", raw[0], raw[0]) + } + fence, ok := raw[1].(string) + if !ok || (status == 1 && fence == "") { + return schedulerClaim{}, fmt.Errorf("scheduler claim returned invalid fence %T", raw[1]) + } + waitMillis, ok := raw[2].(int64) + if !ok || waitMillis < 0 { + return schedulerClaim{}, fmt.Errorf("scheduler claim returned invalid wait %T(%v)", raw[2], raw[2]) + } + return schedulerClaim{ + owned: status == 1, + stopped: status == 2, + fence: fence, + wait: time.Duration(waitMillis) * time.Millisecond, + }, nil +} + +// runTransition renews ownership while Plan and all side effects run, then +// commits canonical next time or releases the unchanged due transition. +func (sched *scheduler) runTransition(ctx context.Context, fence string) (bool, error) { + transitionCtx, cancel := context.WithCancel(ctx) + var renewWG sync.WaitGroup + renewErr := make(chan error, 1) + renewWG.Add(1) + go sched.renewTransition(transitionCtx, cancel, fence, renewErr, &renewWG) + + stop, applyErr := sched.applyTransition(transitionCtx, fence) + cancel() + renewWG.Wait() + select { + case err := <-renewErr: + applyErr = errors.Join(applyErr, err) + default: + } + if applyErr != nil { + releaseCtx, releaseCancel := context.WithTimeout(context.WithoutCancel(ctx), time.Second) + releaseErr := sched.releaseTransition(releaseCtx, fence) + releaseCancel() + return false, errors.Join(applyErr, releaseErr) + } + if stop { + return true, sched.stopTransition(ctx, fence) + } + return false, sched.commitTransition(ctx, fence) +} + +// renewTransition keeps a slow cancellable Plan and its apply phase fenced. +func (sched *scheduler) renewTransition( + ctx context.Context, + cancel context.CancelFunc, + fence string, + result chan<- error, + wg *sync.WaitGroup, +) { + defer wg.Done() + renewEvery := sched.lease / 3 + if renewEvery < time.Millisecond { + renewEvery = time.Millisecond + } + ticker := time.NewTicker(renewEvery) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + if err := sched.renewTransitionLease(ctx, fence); err != nil { + if ctx.Err() != nil { + return + } + select { + case result <- err: + default: + } + cancel() + return + } + } + } +} + +// renewTransitionLease extends the exact owner/fence using Redis time. +func (sched *scheduler) renewTransitionLease(ctx context.Context, fence string) error { + return poolBoundaryError(renewSchedulerTransitionScript.Run( + ctx, + sched.node.rdb, + sched.transitionKeys(), + "active", + sched.node.resources.generation, + sched.transitionPrefix, + sched.owner, + fence, + strconv.FormatInt(sched.lease.Milliseconds(), 10), + sched.node.ID, + nodeCleanupField(sched.node.ID), + ).Err()) +} + +// commitTransition advances canonical next time and releases this transition. +func (sched *scheduler) commitTransition(ctx context.Context, fence string) error { + return poolBoundaryError(commitSchedulerTransitionScript.Run( + ctx, + sched.node.rdb, + sched.transitionKeys(), + "active", + sched.node.resources.generation, + sched.transitionPrefix, + sched.owner, + fence, + sched.node.ID, + nodeCleanupField(sched.node.ID), + ).Err()) +} + +// stopTransition removes canonical timing after all owned jobs are stopped. +func (sched *scheduler) stopTransition(ctx context.Context, fence string) error { + return poolBoundaryError(stopSchedulerTransitionScript.Run( + ctx, + sched.node.rdb, + sched.transitionKeys(), + "active", + sched.node.resources.generation, + sched.transitionPrefix, + sched.owner, + fence, + sched.node.ID, + nodeCleanupField(sched.node.ID), + ).Err()) +} + +// releaseTransition makes a failed due transition immediately retryable. +func (sched *scheduler) releaseTransition(ctx context.Context, fence string) error { + return poolBoundaryError(releaseSchedulerTransitionScript.Run( + ctx, + sched.node.rdb, + sched.transitionKeys(), + "active", + sched.node.resources.generation, + sched.transitionPrefix, + sched.owner, + fence, + sched.node.ID, + nodeCleanupField(sched.node.ID), + ).Err()) +} + +// claimJobOwnership persists one exact scheduler dispatch under the transition +// lease. Existing ownership is returned and never overwritten. +func (sched *scheduler) claimJobOwnership( + ctx context.Context, + fence, field, proposed string, +) (string, error) { + return claimSchedulerJobScript.Run( + ctx, + sched.node.rdb, + []string{ + fmt.Sprintf("pulse:stream:%s:lifecycle", sched.node.poolStream.Name), + rmapContentKey(sched.node.resources.schedulerJobs), + rmapUpdateChannel(sched.node.resources.schedulerJobs), + rmapContentKey(sched.node.resources.nodeKeepAlive), + }, + "active", + sched.node.resources.generation, + sched.transitionPrefix, + sched.owner, + fence, + field, + proposed, + sched.node.ID, + nodeCleanupField(sched.node.ID), + ).Text() +} + +// dispatchJob publishes and waits for one exact scheduler-owned dispatch while +// the transition renewer keeps the admission fence live. +func (sched *scheduler) dispatchJob( + ctx context.Context, + fence, dispatchID string, + job *Job, +) (string, error) { + waiter := sched.node.acquireDispatchWaiter(dispatchID) + defer sched.node.releaseDispatchWaiter(dispatchID, waiter) + record, err := sched.publishDispatchRecord(ctx, fence, dispatchID, job) + if err != nil { + return "", err + } + if record.status == dispatchTerminal { + return record.eventID, dispatchTerminalError(record) + } + identity, err := dispatchIdentity(job.Key, job.Payload) + if err != nil { + return record.eventID, err + } + record, err = sched.node.awaitDispatch(ctx, waiter, dispatchID, identity, record) + if err != nil { + return record.eventID, err + } + return record.eventID, dispatchTerminalError(record) +} + +// publishDispatchRecord atomically verifies transition ownership and admits +// the exact scheduler dispatch. +func (sched *scheduler) publishDispatchRecord( + ctx context.Context, + fence, dispatchID string, + job *Job, +) (dispatchRecord, error) { + identity, err := dispatchIdentity(job.Key, job.Payload) + if err != nil { + return dispatchRecord{}, err + } + raw, err := dispatchScheduledJobScript.Run( + ctx, + sched.node.rdb, + []string{ + fmt.Sprintf("pulse:stream:%s:lifecycle", sched.node.poolStream.Name), + rmapContentKey(sched.node.resources.schedulerJobs), + rmapContentKey(sched.node.resources.jobPayloads), + rmapContentKey(sched.node.resources.jobPending), + rmapUpdateChannel(sched.node.resources.jobPending), + dispatchRecordKey(sched.node.resources.dispatches, dispatchID), + dispatchActiveKey(sched.node.resources.dispatches), + rmapContentKey(sched.node.resources.nodeKeepAlive), + }, + "active", + sched.node.resources.generation, + sched.transitionPrefix, + sched.owner, + fence, + job.Key, + dispatchID, + sched.node.maxQueuedJobs, + evStartJob, + marshalJob(job), + "physical_key", + identity, + sched.node.ID, + nodeCleanupField(sched.node.ID), + ).Result() + if err != nil { + if redis.HasErrorPrefix(err, "DISPATCHIDEMPOTENCYCONFLICT") { + return dispatchRecord{}, fmt.Errorf("%w: dispatch %q", ErrDispatchConflict, dispatchID) + } + return dispatchRecord{}, poolBoundaryError(err) + } + record, err := parseDispatchRecord(raw) + if err != nil { + return dispatchRecord{}, err + } + switch record.status { + case dispatchClaimed, dispatchTerminal: + return record, nil + case dispatchAlreadyPending, dispatchAlreadyRunning: + return dispatchRecord{}, fmt.Errorf("%w: job %q", ErrJobExists, job.Key) + case dispatchCapacityReached: + return dispatchRecord{}, fmt.Errorf( + "%w: maximum %d pending jobs", + ErrPoolCapacity, + sched.node.maxQueuedJobs, + ) + default: + return dispatchRecord{}, fmt.Errorf("unexpected scheduler dispatch status %d", record.status) + } +} + +// transitionKeys returns the lifecycle and generation-owned scheduler state. +func (sched *scheduler) transitionKeys() []string { + return []string{ + fmt.Sprintf("pulse:stream:%s:lifecycle", sched.node.poolStream.Name), + rmapContentKey(sched.node.resources.schedulerJobs), + rmapContentKey(sched.node.resources.nodeKeepAlive), + } +} diff --git a/pool/scripts.go b/pool/scripts.go index f4b6e85..9e08653 100644 --- a/pool/scripts.go +++ b/pool/scripts.go @@ -7,69 +7,371 @@ import redis "github.com/redis/go-redis/v9" const ( dispatchClaimed int64 = iota + 1 + dispatchTerminal dispatchAlreadyPending dispatchAlreadyRunning - dispatchMalformedPending + dispatchCapacityReached ) +// nodeLivenessFenceLua rejects pool mutations from a stale node: the acting +// node must still hold its keep-alive registration and no stale-node cleanup +// fence may be installed. Composing scripts append the node keep-alive content +// key as the last KEYS entry and the node heartbeat field plus node-cleanup +// field as the last two ARGV entries. +const nodeLivenessFenceLua = ` +local node_keepalive = KEYS[#KEYS] +if not redis.call("HGET", node_keepalive, ARGV[#ARGV - 1]) +or redis.call("HGET", node_keepalive, ARGV[#ARGV]) then + return redis.error_reply("NODECLEANUPLOST") +end +` + var ( - // luaClaimDispatch atomically admits a new external dispatch. A job key may - // be claimed only when no durable payload exists and no active pending guard - // exists. Stale pending guards are replaced by the caller's new guard and - // published through the pending rmap channel; malformed guards are rejected - // because they indicate corrupted coordination state. - luaClaimDispatch = redis.NewScript(` -local payload = redis.call("HGET", KEYS[1], ARGV[1]) + // luaDispatchJob atomically resolves an exact dispatch retry or admits one + // new dispatch by writing its durable record, job-key index, and stream + // event at one Redis linearization point. + luaDispatchJob = redis.NewScript(` +if redis.call("HGET", KEYS[1], "state") ~= ARGV[3] +or redis.call("HGET", KEYS[1], "generation") ~= ARGV[4] then + return redis.error_reply("POOLGENERATIONLOST") +end` + nodeLivenessFenceLua + ` +local identity = redis.call("HGET", KEYS[5], "identity") +if identity then + if identity ~= ARGV[9] then + return redis.error_reply("DISPATCHIDEMPOTENCYCONFLICT") + end + local state = redis.call("HGET", KEYS[5], "state") + local event_id = redis.call("HGET", KEYS[5], "event") + return { + state == "terminal" and 2 or 1, + event_id, + redis.call("HGET", KEYS[5], "result") or "", + redis.call("HGET", KEYS[5], "error") or "" + } +end +local payload = redis.call("HGET", KEYS[2], ARGV[1]) if payload then - return {3, ""} + return {4, "", "", ""} end -local function all_digits(value) - return string.match(value, "^%d+$") ~= nil +local pending = redis.call("HGET", KEYS[3], ARGV[1]) +if pending then + return {3, "", "", ""} end -local function active_until(value, now) - if not all_digits(value) then - return false - end - if string.len(value) ~= string.len(now) then - return string.len(value) > string.len(now) - end - return value >= now +local count = 0 +for _, key in ipairs(redis.call("HKEYS", KEYS[3])) do + if string.sub(key, 1, 1) ~= "=" then + count = count + 1 + end +end +if count >= tonumber(ARGV[5]) then + return {5, "", "", ""} end -local pending = redis.call("HGET", KEYS[2], ARGV[1]) -if pending then - if not all_digits(pending) then - return {4, pending} - end - if active_until(pending, ARGV[2]) then - return {2, pending} - end +local stream = redis.call("HGET", KEYS[1], ARGV[8]) +if not stream then + return redis.error_reply("POOLGENERATIONLOST") end +local event_id = redis.call("XADD", stream, "*", "n", ARGV[6], "p", ARGV[7]) +redis.call("HSET", KEYS[5], + "id", ARGV[2], + "identity", ARGV[9], + "key", ARGV[1], + "event", event_id, + "state", "pending", + "result", "", + "error", "") +redis.call("SADD", KEYS[6], KEYS[5]) +redis.call("HSET", KEYS[3], ARGV[1], ARGV[2]) +local rev = tostring(redis.call("HINCRBY", KEYS[3], "=rev", 1)) +redis.call("HSET", KEYS[3], "=kind", "set") +local msg = struct.pack("ic0ic0ic0", string.len(ARGV[1]), ARGV[1], string.len(ARGV[2]), ARGV[2], string.len(rev), rev) +redis.call("PUBLISH", KEYS[4], "set:" .. msg) +return {1, event_id, "", ""} +`) -redis.call("HSET", KEYS[2], ARGV[1], ARGV[3]) + // luaSettleDispatch atomically records the immutable terminal outcome, + // clears admission, acknowledges the sink event, advances its durable + // recovery cursor, and deletes the settled event. + luaSettleDispatch = redis.NewScript(` +if redis.call("HGET", KEYS[1], "state") ~= ARGV[3] +or redis.call("HGET", KEYS[1], "generation") ~= ARGV[4] then + return redis.error_reply("POOLGENERATIONLOST") +end +local identity = redis.call("HGET", KEYS[4], "identity") +if not identity +or redis.call("HGET", KEYS[4], "id") ~= ARGV[2] +or redis.call("HGET", KEYS[4], "key") ~= ARGV[1] then + return redis.error_reply("DISPATCHLOST") +end +local state = redis.call("HGET", KEYS[4], "state") +if state == "terminal" then + return { + redis.call("HGET", KEYS[4], "event"), + redis.call("HGET", KEYS[4], "result") or "", + redis.call("HGET", KEYS[4], "error") or "" + } +end +if redis.call("HGET", KEYS[2], ARGV[1]) ~= ARGV[2] then + return redis.error_reply("DISPATCHLOST") +end +local event_id = redis.call("HGET", KEYS[4], "event") +local stream = redis.call("HGET", KEYS[1], ARGV[5]) +if not stream then + return redis.error_reply("POOLGENERATIONLOST") +end + +redis.call("HSET", KEYS[4], + "state", "terminal", + "result", ARGV[6], + "error", ARGV[7]) +redis.call("SREM", KEYS[5], KEYS[4]) +redis.call("PEXPIRE", KEYS[4], ARGV[9]) +redis.call("HDEL", KEYS[2], ARGV[1]) local rev = tostring(redis.call("HINCRBY", KEYS[2], "=rev", 1)) -redis.call("HSET", KEYS[2], "=kind", "set") -local msg = struct.pack("ic0ic0ic0", string.len(ARGV[1]), ARGV[1], string.len(ARGV[3]), ARGV[3], string.len(rev), rev) -redis.call("PUBLISH", KEYS[3], "set:" .. msg) -return {1, ARGV[3]} +redis.call("HSET", KEYS[2], "=kind", "del") +local msg = struct.pack("ic0ic0", string.len(ARGV[1]), ARGV[1], string.len(rev), rev) +redis.call("PUBLISH", KEYS[3], "del:" .. msg) +redis.call("XACK", stream, ARGV[8], event_id) +local pending = redis.call("XPENDING", stream, ARGV[8]) +local cursor +if pending[1] == 0 then + local groups = redis.call("XINFO", "GROUPS", stream) + for _, group in ipairs(groups) do + local name + local delivered + for i = 1, #group, 2 do + if group[i] == "name" then + name = group[i + 1] + elseif group[i] == "last-delivered-id" then + delivered = group[i + 1] + end + end + if name == ARGV[8] then + cursor = delivered + break + end + end +else + local previous = redis.call("XREVRANGE", stream, "(" .. pending[2], "-", "COUNT", 1) + cursor = #previous == 0 and "0-0" or previous[1][1] +end +if cursor then + local recovery = stream .. ":sink-recovery:" .. ARGV[4] + redis.call("HSET", recovery, ARGV[8], cursor) +end +redis.call("XDEL", stream, event_id) +return {event_id, ARGV[6], ARGV[7]} `) - // luaReleaseDispatch removes the pending guard only if it still belongs to - // this dispatch attempt. It publishes a delete notification so rmap replicas - // converge without relying on a later cleanup sweep. - luaReleaseDispatch = redis.NewScript(` -local pending = redis.call("HGET", KEYS[1], ARGV[1]) -if pending ~= ARGV[2] then - return 0 + // readDispatchRecordScript authoritatively reads one exact dispatch after + // verifying the pool generation and event identity. Local notifications are + // only wake-up hints; callers use this result as the completion truth. + readDispatchRecordScript = redis.NewScript(` +if redis.call("HGET", KEYS[1], "state") ~= ARGV[1] +or redis.call("HGET", KEYS[1], "generation") ~= ARGV[2] then + return redis.error_reply("POOLGENERATIONLOST") +end +local identity = redis.call("HGET", KEYS[2], "identity") +if not identity then + return redis.error_reply("DISPATCHRECORDNOTFOUND") +end +if identity ~= ARGV[3] then + return redis.error_reply("DISPATCHIDEMPOTENCYCONFLICT") end +local state = redis.call("HGET", KEYS[2], "state") +return { + state == "terminal" and 2 or 1, + redis.call("HGET", KEYS[2], "event") or "", + redis.call("HGET", KEYS[2], "result") or "", + redis.call("HGET", KEYS[2], "error") or "" +} +`) -redis.call("HDEL", KEYS[1], ARGV[1]) -local rev = tostring(redis.call("HINCRBY", KEYS[1], "=rev", 1)) -redis.call("HSET", KEYS[1], "=kind", "del") -local msg = struct.pack("ic0ic0", string.len(ARGV[1]), ARGV[1], string.len(rev), rev) -redis.call("PUBLISH", KEYS[2], "del:" .. msg) + // cleanupFailedStartScript atomically verifies this pool generation, removes + // the worker's ownership, deletes the durable payload, and publishes both + // rmap updates. A worker may publish terminal Start failure only after this + // idempotent recipe succeeds. + cleanupFailedStartScript = redis.NewScript(` +if redis.call("HGET", KEYS[1], "state") ~= ARGV[4] +or redis.call("HGET", KEYS[1], "generation") ~= ARGV[1] then + return redis.error_reply("POOLGENERATIONLOST") +end + +local worker = ARGV[2] +local job = ARGV[3] +local encoded = redis.call("HGET", KEYS[2], worker) +if encoded then + local ok, values = pcall(cjson.decode, encoded) + if not ok or type(values) ~= "table" then + return redis.error_reply("INVALIDJOBOWNERSHIP") + end + local remaining = {} + local removed = false + for _, value in ipairs(values) do + if value == job then + removed = true + else + table.insert(remaining, value) + end + end + if removed then + local rev + if #remaining == 0 then + redis.call("HDEL", KEYS[2], worker) + rev = tostring(redis.call("HINCRBY", KEYS[2], "=rev", 1)) + redis.call("HSET", KEYS[2], "=kind", "del") + local msg = struct.pack("ic0ic0", string.len(worker), worker, string.len(rev), rev) + redis.call("PUBLISH", KEYS[3], "del:" .. msg) + else + local value = cjson.encode(remaining) + redis.call("HSET", KEYS[2], worker, value) + rev = tostring(redis.call("HINCRBY", KEYS[2], "=rev", 1)) + redis.call("HSET", KEYS[2], "=kind", "set") + local msg = struct.pack("ic0ic0ic0", string.len(worker), worker, string.len(value), value, string.len(rev), rev) + redis.call("PUBLISH", KEYS[3], "set:" .. msg) + end + end +end + +if redis.call("HDEL", KEYS[4], job) == 1 then + local rev = tostring(redis.call("HINCRBY", KEYS[4], "=rev", 1)) + redis.call("HSET", KEYS[4], "=kind", "del") + local msg = struct.pack("ic0ic0", string.len(job), job, string.len(rev), rev) + redis.call("PUBLISH", KEYS[5], "del:" .. msg) +end +return 1 +`) + + // releaseCrashedDispatchStartScript removes only the stale worker + // ownership and payload for an exact dispatch that is still active. It + // leaves the original dispatch event, ID, admission index, and record + // untouched so sink recovery reclaims the same dispatch. + releaseCrashedDispatchStartScript = redis.NewScript(` +if redis.call("HGET", KEYS[1], "state") ~= ARGV[5] +or redis.call("HGET", KEYS[1], "generation") ~= ARGV[1] then + return redis.error_reply("POOLGENERATIONLOST") +end +if ARGV[2] ~= "" then + local clock = redis.call("TIME") + local now = tonumber(clock[1]) * 1000 + math.floor(tonumber(clock[2]) / 1000) + local cleanup = redis.call("HGET", KEYS[8], ARGV[2]) + local owner, fence, lease_until = + string.match(cleanup or "", "^([^|]+)|([^|]+)|(%d+)$") + if owner ~= ARGV[6] or fence ~= ARGV[7] + or tonumber(lease_until or "0") <= now then + return redis.error_reply("WORKERCLEANUPLOST") + end +end +if redis.call("HGET", KEYS[2], ARGV[3]) ~= ARGV[4] +or redis.call("HGET", KEYS[3], "id") ~= ARGV[4] +or redis.call("HGET", KEYS[3], "key") ~= ARGV[3] +or redis.call("HGET", KEYS[3], "state") ~= "pending" then + return 0 +end + +local worker = ARGV[2] +local job = ARGV[3] +if worker ~= "" then + local encoded = redis.call("HGET", KEYS[4], worker) + if encoded then + local ok, values = pcall(cjson.decode, encoded) + if not ok or type(values) ~= "table" then + return redis.error_reply("INVALIDJOBOWNERSHIP") + end + local remaining = {} + local removed = false + for _, value in ipairs(values) do + if value == job then + removed = true + else + table.insert(remaining, value) + end + end + if removed then + local rev + if #remaining == 0 then + redis.call("HDEL", KEYS[4], worker) + rev = tostring(redis.call("HINCRBY", KEYS[4], "=rev", 1)) + redis.call("HSET", KEYS[4], "=kind", "del") + local msg = struct.pack("ic0ic0", string.len(worker), worker, string.len(rev), rev) + redis.call("PUBLISH", KEYS[5], "del:" .. msg) + else + local value = cjson.encode(remaining) + redis.call("HSET", KEYS[4], worker, value) + rev = tostring(redis.call("HINCRBY", KEYS[4], "=rev", 1)) + redis.call("HSET", KEYS[4], "=kind", "set") + local msg = struct.pack("ic0ic0ic0", string.len(worker), worker, string.len(value), value, string.len(rev), rev) + redis.call("PUBLISH", KEYS[5], "set:" .. msg) + end + end + end +end + +if redis.call("HDEL", KEYS[6], job) == 1 then + local rev = tostring(redis.call("HINCRBY", KEYS[6], "=rev", 1)) + redis.call("HSET", KEYS[6], "=kind", "del") + local msg = struct.pack("ic0ic0", string.len(job), job, string.len(rev), rev) + redis.call("PUBLISH", KEYS[7], "del:" .. msg) +end +return 1 +`) + + // claimWorkerStartScript validates the exact dispatch guard and atomically + // creates durable worker ownership plus payload before handler execution. + // The claiming worker must still be registered with no cleanup fence + // installed: heartbeat runs before the claim, so only this check makes a + // worker paused past WorkerTTL unable to claim after authoritative takeover. + // The claimed payload must match the dispatch record's canonical identity, + // so a worker can never start a job whose bytes diverged from admission. + claimWorkerStartScript = redis.NewScript(` +if redis.call("HGET", KEYS[1], "state") ~= ARGV[2] +or redis.call("HGET", KEYS[1], "generation") ~= ARGV[1] then + return redis.error_reply("POOLGENERATIONLOST") +end +local registration = redis.call("HGET", KEYS[8], ARGV[5]) +if not registration or registration == "-" +or redis.call("HGET", KEYS[9], ARGV[5]) then + return redis.error_reply("WORKERCLEANUPLOST") +end +local pending = redis.call("HGET", KEYS[2], ARGV[3]) +if pending ~= ARGV[4] then + return redis.error_reply("DISPATCHLOST") +end +if redis.call("HGET", KEYS[3], "id") ~= ARGV[4] +or redis.call("HGET", KEYS[3], "key") ~= ARGV[3] +or redis.call("HGET", KEYS[3], "state") ~= "pending" then + return redis.error_reply("DISPATCHLOST") +end +if redis.call("HGET", KEYS[3], "identity") ~= ARGV[7] then + return redis.error_reply("DISPATCHIDENTITYMISMATCH") +end +if redis.call("HEXISTS", KEYS[4], ARGV[3]) == 1 then + return 0 +end +local jobs = {} +local encoded = redis.call("HGET", KEYS[5], ARGV[5]) +if encoded then + local ok, decoded = pcall(cjson.decode, encoded) + if not ok or type(decoded) ~= "table" then + return redis.error_reply("INVALIDJOBOWNERSHIP") + end + jobs = decoded +end +table.insert(jobs, ARGV[3]) +local jobs_value = cjson.encode(jobs) +redis.call("HSET", KEYS[5], ARGV[5], jobs_value) +local jobs_rev = tostring(redis.call("HINCRBY", KEYS[5], "=rev", 1)) +redis.call("HSET", KEYS[5], "=kind", "set") +local jobs_msg = struct.pack("ic0ic0ic0", string.len(ARGV[5]), ARGV[5], string.len(jobs_value), jobs_value, string.len(jobs_rev), jobs_rev) +redis.call("PUBLISH", KEYS[6], "set:" .. jobs_msg) + +redis.call("HSET", KEYS[4], ARGV[3], ARGV[6]) +local payload_rev = tostring(redis.call("HINCRBY", KEYS[4], "=rev", 1)) +redis.call("HSET", KEYS[4], "=kind", "set") +local payload_msg = struct.pack("ic0ic0ic0", string.len(ARGV[3]), ARGV[3], string.len(ARGV[6]), ARGV[6], string.len(payload_rev), payload_rev) +redis.call("PUBLISH", KEYS[7], "set:" .. payload_msg) return 1 `) ) diff --git a/pool/settlement.go b/pool/settlement.go new file mode 100644 index 0000000..0f7681f --- /dev/null +++ b/pool/settlement.go @@ -0,0 +1,223 @@ +// Node-owned settlement separates durable handler outcomes from worker intake. +// Once a handler returns, worker removal may stop Redis intake but cannot cancel +// the exact dispatch obligation that atomically records the result and settles +// the original pool event. +package pool + +import ( + "context" + "errors" + "fmt" + "sync" + "time" + + redis "github.com/redis/go-redis/v9" + + "goa.design/pulse/pulse" +) + +type ( + // dispatchSettlements tracks node-owned terminal outcomes by worker so + // graceful worker and node closure can join the exact obligations. + dispatchSettlements struct { + mu sync.Mutex + changed chan struct{} + total int + workers map[string]int + errs map[string]error + } +) + +// newDispatchSettlements creates an empty obligation tracker. +func newDispatchSettlements() *dispatchSettlements { + return &dispatchSettlements{ + changed: make(chan struct{}), + workers: make(map[string]int), + errs: make(map[string]error), + } +} + +// ownDispatchSettlement transfers one known handler outcome from worker +// intake to a node-owned retry loop before intake can observe cancellation. +func (node *Node) ownDispatchSettlement( + worker *Worker, + routingNodeID, workerEventID string, + job *Job, + resultErr error, +) { + finish := node.settlements.begin(worker.ID) + pulse.Go(node.logger, func() { + err := node.retryDispatchSettlement(job, resultErr) + if err == nil { + worker.markDispatchSettled(job.Key, job.dispatchID) + if routingNodeID == node.ID { + node.pendingEvents.Delete(pendingEventKey(worker.ID, workerEventID)) + } else { + ackCtx, cancel := context.WithTimeout( + context.Background(), + min(node.workerTTL, time.Second), + ) + worker.ackPoolEvent(ackCtx, routingNodeID, workerEventID, resultErr) + cancel() + } + } + finish(err) + }) +} + +// retryDispatchSettlement retries transient Redis failures independently of +// worker cancellation. Generation loss is terminal and remains observable to +// RemoveWorker and Close. +func (node *Node) retryDispatchSettlement(job *Job, resultErr error) error { + delay := 100 * time.Millisecond + for { + _, err := node.settleDispatch(context.Background(), job.Key, job.dispatchID, resultErr) + if err == nil { + return nil + } + node.logger.Error(err, "job", job.Key, "dispatch", job.dispatchID, "retry_in", delay) + if errors.Is(err, ErrPoolGenerationLost) { + return err + } + time.Sleep(delay) + delay = min(delay*2, 5*time.Second) + } +} + +// releaseCrashedDispatchStart removes stale execution ownership only while the +// exact dispatch remains active. A false result means settlement won the race, +// so normal settled-job recovery may proceed. +func (node *Node) releaseCrashedDispatchStart( + ctx context.Context, + lease *workerCleanupLease, + key, dispatchID string, +) (bool, error) { + var workerID, owner, fence string + if lease != nil { + workerID = lease.workerID + owner = lease.owner + fence = lease.fence + } + result, err := releaseCrashedDispatchStartScript.Run( + ctx, + node.rdb, + []string{ + fmt.Sprintf("pulse:stream:%s:lifecycle", node.poolStream.Name), + rmapContentKey(node.resources.jobPending), + dispatchRecordKey(node.resources.dispatches, dispatchID), + rmapContentKey(node.resources.jobs), + rmapUpdateChannel(node.resources.jobs), + rmapContentKey(node.resources.jobPayloads), + rmapUpdateChannel(node.resources.jobPayloads), + rmapContentKey(node.resources.workerCleanup), + }, + node.resources.generation, + workerID, + key, + dispatchID, + "active", + owner, + fence, + ).Int64() + if err != nil { + return false, fmt.Errorf( + "release crashed exact dispatch %q: %w", + dispatchID, + poolBoundaryError(err), + ) + } + return result == 1, nil +} + +// activeDispatchID reads the authoritative job-key admission index. Stale +// worker recovery cannot rely on the eventually updated local rmap projection. +func (node *Node) activeDispatchID(ctx context.Context, key string) (string, error) { + dispatchID, err := node.rdb.HGet( + ctx, + rmapContentKey(node.resources.jobPending), + key, + ).Result() + if errors.Is(err, redis.Nil) { + return "", nil + } + if err != nil { + return "", fmt.Errorf("read active dispatch for job %q: %w", key, err) + } + return dispatchID, nil +} + +// begin registers an obligation synchronously and returns its completion +// callback. Registration precedes the worker loop's next cancellation point. +func (s *dispatchSettlements) begin(workerID string) func(error) { + s.mu.Lock() + s.total++ + s.workers[workerID]++ + s.signalLocked() + s.mu.Unlock() + return func(err error) { + s.finish(workerID, err) + } +} + +// waitWorker joins all outcomes observed by one worker before its jobs or +// distributed registration can be removed. +func (s *dispatchSettlements) waitWorker(ctx context.Context, workerID string) error { + return s.wait(ctx, workerID) +} + +// waitAll joins all node-owned outcomes before graceful node closure proceeds +// to detach the pool sink or destroy node resources. +func (s *dispatchSettlements) waitAll(ctx context.Context) error { + return s.wait(ctx, "") +} + +// finish records terminal settlement failure and wakes closure waiters. +func (s *dispatchSettlements) finish(workerID string, err error) { + s.mu.Lock() + defer s.mu.Unlock() + s.total-- + s.workers[workerID]-- + if s.workers[workerID] == 0 { + delete(s.workers, workerID) + } + if err != nil { + s.errs[workerID] = errors.Join(s.errs[workerID], err) + } + s.signalLocked() +} + +// wait blocks on tracker state changes without coupling the obligation to the +// caller's cancellation. A cancelled caller receives an explicit pending error +// and may retry closure with a fresh context. +func (s *dispatchSettlements) wait(ctx context.Context, workerID string) error { + for { + s.mu.Lock() + count := s.total + if workerID != "" { + count = s.workers[workerID] + } + if count == 0 { + err := s.errs[workerID] + if workerID == "" { + for _, workerErr := range s.errs { + err = errors.Join(err, workerErr) + } + } + s.mu.Unlock() + return err + } + changed := s.changed + s.mu.Unlock() + select { + case <-changed: + case <-ctx.Done(): + return fmt.Errorf("%d terminal dispatch settlements still pending: %w", count, ctx.Err()) + } + } +} + +// signalLocked broadcasts a tracker state change. The caller must hold mu. +func (s *dispatchSettlements) signalLocked() { + close(s.changed) + s.changed = make(chan struct{}) +} diff --git a/pool/testing.go b/pool/testing.go index 25db450..196ff7f 100644 --- a/pool/testing.go +++ b/pool/testing.go @@ -31,27 +31,40 @@ type mockMessageHandler struct { } const ( - testWorkerShutdownTTL = 100 * time.Millisecond - testJobSinkBlockDuration = 100 * time.Millisecond - testWorkerTTL = 2 * time.Second - testFastWorkerTTL = 150 * time.Millisecond - testFastWorkerShutdownTTL = 100 * time.Millisecond + testRequeueTimeout = 100 * time.Millisecond + testJobSinkBlockDuration = 100 * time.Millisecond + testWorkerTTL = 2 * time.Second + testFastWorkerTTL = 500 * time.Millisecond + testFastRequeueTimeout = 100 * time.Millisecond // testAckGracePeriod should cover scheduler jitter under -race; tests that // need fast stale-worker detection use the worker TTLs above instead. - testAckGracePeriod = 500 * time.Millisecond + testAckGracePeriod = 4 * time.Second ) // newTestNode creates a new Node instance for testing purposes. // It configures the node with specific TTL and block duration settings // suitable for testing, and uses the provided Redis client and name. func newTestNode(t *testing.T, ctx context.Context, rdb *redis.Client, name string) *Node { + t.Helper() + return newTestNodeWithLogger(t, ctx, rdb, name, pulse.NoopLogger()) +} + +// newTestNodeWithLogger creates a regular test node with the supplied logger. +func newTestNodeWithLogger( + t *testing.T, + ctx context.Context, + rdb *redis.Client, + name string, + logger pulse.Logger, +) *Node { t.Helper() node, err := AddNode(ctx, name, rdb, - WithLogger(pulse.ClueLogger(ctx)), - WithWorkerShutdownTTL(testWorkerShutdownTTL), + WithLogger(logger), + WithRequeueTimeout(testRequeueTimeout), WithJobSinkBlockDuration(testJobSinkBlockDuration), WithWorkerTTL(testWorkerTTL), - WithAckGracePeriod(testAckGracePeriod)) + WithDispatchTimeout(2*testAckGracePeriod), + WithRecoveryGrace(testAckGracePeriod)) require.NoError(t, err) return node } @@ -63,11 +76,12 @@ func newTestNode(t *testing.T, ctx context.Context, rdb *redis.Client, name stri func newFastCleanupTestNode(t *testing.T, ctx context.Context, rdb *redis.Client, name string) *Node { t.Helper() node, err := AddNode(ctx, name, rdb, - WithLogger(pulse.ClueLogger(ctx)), - WithWorkerShutdownTTL(testFastWorkerShutdownTTL), + WithLogger(pulse.NoopLogger()), + WithRequeueTimeout(testFastRequeueTimeout), WithJobSinkBlockDuration(testJobSinkBlockDuration), WithWorkerTTL(testFastWorkerTTL), - WithAckGracePeriod(testAckGracePeriod)) + WithDispatchTimeout(2*testAckGracePeriod), + WithRecoveryGrace(testAckGracePeriod)) require.NoError(t, err) return node } diff --git a/pool/ticker.go b/pool/ticker.go index 42b3c2f..a71706f 100644 --- a/pool/ticker.go +++ b/pool/ticker.go @@ -2,6 +2,7 @@ package pool import ( "context" + "errors" "fmt" "strconv" "strings" @@ -19,6 +20,7 @@ type ( C <-chan time.Time c chan time.Time name string + node *Node lock sync.Mutex tickerMap *rmap.Map timer *time.Timer @@ -43,6 +45,12 @@ func (node *Node) NewTicker(ctx context.Context, name string, d time.Duration, o if node.clientOnly { return nil, fmt.Errorf("cannot create ticker on client-only node") } + if err := node.ensureGenerationActive(ctx); err != nil { + return nil, fmt.Errorf("create ticker: %w", err) + } + if d < time.Millisecond { + return nil, fmt.Errorf("create ticker: duration must be at least 1ms") + } name = node.PoolName + ":" + name o := parseTickerOptions(opts...) logger := o.logger @@ -54,25 +62,33 @@ func (node *Node) NewTicker(ctx context.Context, name string, d time.Duration, o C: c, c: c, name: name, + node: node, tickerMap: node.tickerMap, mapch: node.tickerMap.Subscribe(), wg: &sync.WaitGroup{}, logger: logger, } if current, ok := node.tickerMap.Get(name); ok { - _, curd := deserialize(current) + _, curd, err := deserialize(current) + if err != nil { + node.tickerMap.Unsubscribe(t.mapch) + return nil, fmt.Errorf("create ticker: decode shared state: %w", err) + } if d == curd { t.next = current } } if t.next == "" { next := serialize(time.Now().Add(d), d) - if _, err := t.tickerMap.SetAndWait(ctx, t.name, next); err != nil { + if err := node.setPoolMapAndWait(ctx, node.tickerMap, node.resources.tickers, t.name, next); err != nil { return nil, fmt.Errorf("failed to store tick and duration: %s", err) } t.next = next } - t.initTimer() + if err := t.initTimer(); err != nil { + node.tickerMap.Unsubscribe(t.mapch) + return nil, fmt.Errorf("create ticker: %w", err) + } t.wg.Add(1) pulse.Go(logger, func() { t.handleEvents() }) return t, nil @@ -103,19 +119,29 @@ func (t *Ticker) Close() { // not close the channel, to prevent a concurrent goroutine reading from the // channel from seeing an erroneous "tick". func (t *Ticker) Stop() { + if err := t.stop(context.Background()); err != nil { + t.logger.Error(err, "msg", "failed to stop ticker") + } +} + +// stop deletes the canonical shared ticker before stopping this local replica. +// A deletion failure leaves the ticker live so its owner can retry. +func (t *Ticker) stop(ctx context.Context) error { t.lock.Lock() + if err := t.node.deletePoolMap(ctx, t.node.resources.tickers, t.name); err != nil { + t.lock.Unlock() + return fmt.Errorf("delete shared ticker %q: %w", t.name, err) + } if t.timer != nil { t.timer.Stop() } - if _, err := t.tickerMap.Delete(context.Background(), t.name); err != nil { - t.logger.Error(err, "msg", "failed to delete ticker") - } if t.mapch != nil { t.tickerMap.Unsubscribe(t.mapch) } t.mapch = nil t.lock.Unlock() t.wg.Wait() + return nil } // handleEvents handles events from the ticker timer and map. @@ -153,7 +179,12 @@ func (t *Ticker) handleEvents() { continue } t.next = next - t.initTimer() + if err := t.initTimer(); err != nil { + t.logger.Error(err, "msg", "invalid shared ticker state") + t.stopInvalidStateLocked() + t.lock.Unlock() + return + } t.lock.Unlock() case <-t.timer.C: t.handleTick() @@ -165,13 +196,24 @@ func (t *Ticker) handleEvents() { func (t *Ticker) handleTick() { t.lock.Lock() defer t.lock.Unlock() - ts, d := deserialize(t.next) + ts, d, err := deserialize(t.next) + if err != nil { + t.logger.Error(err, "msg", "invalid shared ticker state") + t.stopInvalidStateLocked() + return + } ts = ts.Add(d) for ts.Before(time.Now()) { ts = ts.Add(d) } next := serialize(ts, d) - prev, err := t.tickerMap.TestAndSet(context.Background(), t.name, t.next, next) + prev, err := t.node.testAndSetPoolMap( + context.Background(), + t.node.resources.tickers, + t.name, + t.next, + next, + ) if err != nil { t.handleAdvanceFailureLocked(err, d) return @@ -179,21 +221,45 @@ func (t *Ticker) handleTick() { if prev != t.next { // Another node already updated the ticker, restart the timer. t.next = prev - t.initTimer() + if err := t.initTimer(); err != nil { + t.logger.Error(err, "msg", "invalid shared ticker state") + t.stopInvalidStateLocked() + } return } t.next = next - t.initTimer() + if err := t.initTimer(); err != nil { + t.logger.Error(err, "msg", "invalid shared ticker state") + t.stopInvalidStateLocked() + return + } select { case t.c <- time.Now(): default: } } -// initTimer sets the timer to fire at the next tick. -func (t *Ticker) initTimer() { - next, _ := deserialize(t.next) +// initTimer sets the timer to fire at the next strictly decoded tick. +func (t *Ticker) initTimer() error { + next, _, err := deserialize(t.next) + if err != nil { + return err + } t.resetTimerLocked(time.Until(next)) + return nil +} + +// stopInvalidStateLocked stops this replica after Redis returned malformed +// canonical state. The caller holds lock; another explicit NewTicker may repair +// the state only through the normal construction contract. +func (t *Ticker) stopInvalidStateLocked() { + if t.mapch != nil { + t.tickerMap.Unsubscribe(t.mapch) + t.mapch = nil + } + if t.timer != nil { + t.timer.Stop() + } } // handleAdvanceFailureLocked logs a transient Redis advance failure and rearms @@ -201,6 +267,14 @@ func (t *Ticker) initTimer() { // caller must hold t.lock. func (t *Ticker) handleAdvanceFailureLocked(err error, interval time.Duration) { t.logger.Error(err, "msg", "failed to update next tick") + if errors.Is(err, ErrPoolGenerationLost) { + if t.mapch != nil { + t.tickerMap.Unsubscribe(t.mapch) + t.mapch = nil + } + t.timer.Stop() + return + } t.resetTimerLocked(min(interval, tickerRetryMaxInterval)) } @@ -230,13 +304,22 @@ func serialize(t time.Time, d time.Duration) string { return ts + "|" + ds } -// deserialize returns the time and duration represented by the given serialized -// string. s must be a value returned by serialize, the behavior is undefined -// otherwise. -func deserialize(s string) (time.Time, time.Duration) { +// deserialize validates and returns one canonical shared ticker state. +func deserialize(s string) (time.Time, time.Duration, error) { parts := strings.Split(s, "|") - ts, _ := strconv.ParseInt(parts[0], 10, 64) - t := time.UnixMicro(ts) - d, _ := time.ParseDuration(parts[1]) - return t, d + if len(parts) != 2 { + return time.Time{}, 0, fmt.Errorf("ticker state %q must contain timestamp and duration", s) + } + ts, err := strconv.ParseInt(parts[0], 10, 64) + if err != nil { + return time.Time{}, 0, fmt.Errorf("ticker state %q has invalid timestamp: %w", s, err) + } + d, err := time.ParseDuration(parts[1]) + if err != nil { + return time.Time{}, 0, fmt.Errorf("ticker state %q has invalid duration: %w", s, err) + } + if d < time.Millisecond { + return time.Time{}, 0, fmt.Errorf("ticker state %q has duration below 1ms", s) + } + return time.UnixMicro(ts), d, nil } diff --git a/pool/ticker_test.go b/pool/ticker_test.go index 88a3988..90efdce 100644 --- a/pool/ticker_test.go +++ b/pool/ticker_test.go @@ -1,6 +1,7 @@ package pool import ( + "errors" "io" "strings" "testing" @@ -10,7 +11,6 @@ import ( "github.com/stretchr/testify/require" "goa.design/clue/log" "goa.design/pulse/pulse" - "goa.design/pulse/rmap" ptesting "goa.design/pulse/testing" ) @@ -35,7 +35,8 @@ func TestNewTicker(t *testing.T) { // Verify next tick time and duration ticker.lock.Lock() - nextTickTime, tickerDuration := deserialize(ticker.next) + nextTickTime, tickerDuration, err := deserialize(ticker.next) + require.NoError(t, err) ticker.lock.Unlock() assert.WithinDuration(t, startTime.Add(tickDuration), nextTickTime, time.Second, "Next tick time should be approximately one tick duration from start") assert.Equal(t, tickDuration, tickerDuration, "Ticker duration should match the specified duration") @@ -53,6 +54,23 @@ func TestNewTicker(t *testing.T) { assert.NoError(t, node.Shutdown(ctx), "Failed to shutdown node") } +func TestDeserializeTickerStateRejectsMalformedValues(t *testing.T) { + for _, value := range []string{ + "", + "1", + "not-a-time|1s", + "1|not-a-duration", + "1|0s", + "1|500us", + "1|1s|extra", + } { + t.Run(value, func(t *testing.T) { + _, _, err := deserialize(value) + require.Error(t, err) + }) + } +} + func TestReplaceTickerTimer(t *testing.T) { rdb := ptesting.NewRedisClient(t) defer ptesting.CleanupRedis(t, rdb, true, t.Name()) @@ -71,7 +89,8 @@ func TestReplaceTickerTimer(t *testing.T) { require.NotNil(t, ticker1) // Verify first ticker properties - nextTick, tickDuration := deserialize(ticker1.next) + nextTick, tickDuration, err := deserialize(ticker1.next) + require.NoError(t, err) assert.WithinDuration(t, now.Add(shortDuration), nextTick, time.Second, "First ticker: invalid next tick time") assert.Equal(t, shortDuration, tickDuration, "First ticker: invalid duration") @@ -82,7 +101,8 @@ func TestReplaceTickerTimer(t *testing.T) { // Verify second ticker properties ticker2.lock.Lock() - nextTick, tickDuration = deserialize(ticker2.next) + nextTick, tickDuration, err = deserialize(ticker2.next) + require.NoError(t, err) ticker2.lock.Unlock() assert.WithinDuration(t, now.Add(longDuration), nextTick, time.Second, "Second ticker: invalid next tick time") assert.Equal(t, longDuration, tickDuration, "Second ticker: invalid duration") @@ -105,20 +125,25 @@ func TestHandleTickRetriesAfterMapWriteError(t *testing.T) { ctx := log.Context(ptesting.NewTestContext(t), log.WithOutput(io.Discard)) testName := strings.Replace(t.Name(), "/", "_", -1) - tickerMap, err := rmap.Join(ctx, "ticker-map-"+testName, rdb) - require.NoError(t, err) + hook := &ambiguousDispatchHook{ + err: errors.New("ticker write failed"), + scriptHash: testAndSetPoolMapScript.Hash(), + } + rdb.AddHook(hook) + node := newTestNode(t, ctx, rdb, testName) + require.NoError(t, testAndSetPoolMapScript.Load(ctx, rdb).Err()) tickDuration := 10 * time.Millisecond next := serialize(time.Now().Add(tickDuration), tickDuration) - _, err = tickerMap.Set(ctx, testName, next) - require.NoError(t, err) + require.NoError(t, node.setPoolMap(ctx, node.resources.tickers, testName, next)) c := make(chan time.Time, 1) ticker := &Ticker{ C: c, c: c, name: testName, - tickerMap: tickerMap, + node: node, + tickerMap: node.tickerMap, next: next, timer: time.NewTimer(time.Hour), logger: pulse.NoopLogger(), @@ -127,8 +152,7 @@ func TestHandleTickRetriesAfterMapWriteError(t *testing.T) { ticker.timer.Stop() }) - tickerMap.Close() - + hook.fail.Store(true) start := time.Now() ticker.handleTick() diff --git a/pool/worker.go b/pool/worker.go index 671b893..a900b4e 100644 --- a/pool/worker.go +++ b/pool/worker.go @@ -10,6 +10,7 @@ import ( "time" "github.com/oklog/ulid/v2" + redis "github.com/redis/go-redis/v9" "goa.design/clue/log" "goa.design/pulse/pulse" @@ -26,26 +27,29 @@ type ( // Time worker was created. CreatedAt time.Time - node *Node - handler JobHandler - stream *streaming.Stream - reader *streaming.Reader - done chan struct{} - jobsMap *rmap.Map - jobPayloadsMap *rmap.Map - keepAliveMap *rmap.Map - shutdownMap *rmap.Map - workerTTL time.Duration - workerShutdownTTL time.Duration - pendingJobTTL time.Duration - logger pulse.Logger - wg sync.WaitGroup + node *Node + handler JobHandler + stream *streaming.Stream + reader *streaming.Reader + done chan struct{} + jobsMap *rmap.Map + jobPayloadsMap *rmap.Map + keepAliveMap *rmap.Map + shutdownMap *rmap.Map + workerTTL time.Duration + requeueTimeout time.Duration + logger pulse.Logger + wg sync.WaitGroup + rebalanceLock sync.Mutex jobs sync.Map // jobs being handled by the worker indexed by job key nodeStreams sync.Map lock sync.RWMutex stopped bool + // streamDestroyed records completion of the retryable distributed stop + // side effect after local worker goroutines have stopped. + streamDestroyed bool } // Job is a job that can be added to a worker. @@ -63,6 +67,15 @@ type ( Worker *Worker // NodeID is the ID of the node that created the job. NodeID string + // dispatchID correlates an admitted dispatch before its event is + // published. It is intentionally not part of the public job contract. + dispatchID string + } + + // requeueResult reports one concurrent handoff attempt. + requeueResult struct { + key string + err error } // JobHandler starts and stops jobs. @@ -91,6 +104,8 @@ type ( ack struct { // EventID is the ID of the event being acknowledged. EventID string + // JobKey is the singleton admission key completed by this ack. + JobKey string // Error is the error that occurred while handling the event if any. Error string } @@ -100,48 +115,83 @@ var errJobNotOwned = errors.New("job not owned by worker") // newWorker creates a new worker. func newWorker(ctx context.Context, node *Node, h JobHandler) (*Worker, error) { - wid := ulid.Make().String() - createdAt := time.Now() - if _, err := node.workerMap.SetAndWait(ctx, wid, strconv.FormatInt(createdAt.UnixNano(), 10)); err != nil { - return nil, fmt.Errorf("failed to add worker %q to pool %q: %w", wid, node.PoolName, err) + if err := node.ensureGenerationActive(ctx); err != nil { + return nil, err } - now := strconv.FormatInt(time.Now().UnixNano(), 10) - if _, err := node.workerKeepAliveMap.SetAndWait(ctx, wid, now); err != nil { - return nil, fmt.Errorf("failed to update worker keep-alive: %w", err) + wid := ulid.Make().String() + createdAt, err := node.rdb.Time(ctx).Result() + if err != nil { + return nil, fmt.Errorf("failed to read Redis time for worker %q: %w", wid, err) } stream, err := streaming.NewStream(workerStreamName(wid), node.rdb, options.WithStreamLogger(node.logger)) if err != nil { return nil, fmt.Errorf("failed to create jobs stream for worker %q: %w", wid, err) } if _, err := stream.Add(ctx, evInit, marshalEnvelope(node.ID, []byte(wid))); err != nil { - return nil, fmt.Errorf("failed to add init event to worker stream %q: %w", workerStreamName(wid), err) + destroyErr := stream.Destroy(context.WithoutCancel(ctx)) + return nil, errors.Join( + fmt.Errorf("failed to add init event to worker stream %q: %w", workerStreamName(wid), err), + destroyErr, + ) } reader, err := stream.NewReader(ctx, options.WithReaderBlockDuration(node.workerTTL/2), options.WithReaderStartAtOldest()) if err != nil { - return nil, fmt.Errorf("failed to create reader for worker %q: %w", wid, err) + destroyErr := stream.Destroy(context.WithoutCancel(ctx)) + return nil, errors.Join( + fmt.Errorf("failed to create reader for worker %q: %w", wid, err), + destroyErr, + ) + } + if err := node.setPoolMapAndWait( + ctx, + node.workerMap, + node.resources.workers, + wid, + strconv.FormatInt(createdAt.UnixNano(), 10), + ); err != nil { + reader.Close() + destroyErr := stream.Destroy(context.WithoutCancel(ctx)) + return nil, errors.Join( + fmt.Errorf("failed to add worker %q to pool %q: %w", wid, node.PoolName, err), + destroyErr, + ) + } + now, err := node.updateWorkerHeartbeat(ctx, wid) + if err == nil { + err = waitPoolMapValue(ctx, node.workerKeepAliveMap, wid, now) + } + if err != nil { + removeErr := node.deletePoolMap(context.WithoutCancel(ctx), node.resources.workers, wid) + reader.Close() + destroyErr := stream.Destroy(context.WithoutCancel(ctx)) + return nil, errors.Join( + fmt.Errorf("failed to update worker keep-alive: %w", err), + removeErr, + destroyErr, + ) } w := &Worker{ - ID: wid, - node: node, - handler: h, - CreatedAt: time.Now(), - stream: stream, - reader: reader, - done: make(chan struct{}), - jobsMap: node.jobMap, - jobPayloadsMap: node.jobPayloadMap, - keepAliveMap: node.workerKeepAliveMap, - shutdownMap: node.nodeShutdownMap, - workerTTL: node.workerTTL, - workerShutdownTTL: node.workerShutdownTTL, - logger: node.logger.WithPrefix("worker", wid), - jobs: sync.Map{}, - nodeStreams: sync.Map{}, + ID: wid, + node: node, + handler: h, + CreatedAt: createdAt, + stream: stream, + reader: reader, + done: make(chan struct{}), + jobsMap: node.jobMap, + jobPayloadsMap: node.jobPayloadMap, + keepAliveMap: node.workerKeepAliveMap, + shutdownMap: node.nodeShutdownMap, + workerTTL: node.workerTTL, + requeueTimeout: node.requeueTimeout, + logger: node.logger.WithPrefix("worker", wid), + jobs: sync.Map{}, + nodeStreams: sync.Map{}, } w.logger.Info("created", "worker_ttl", w.workerTTL, - "worker_shutdown_ttl", w.workerShutdownTTL) + "worker_requeue_timeout", w.requeueTimeout) w.wg.Add(2) @@ -149,7 +199,8 @@ func newWorker(ctx context.Context, node *Node, h JobHandler) (*Worker, error) { // not cancel the worker. logCtx := context.Background() logCtx = log.WithContext(logCtx, ctx) - pulse.Go(w.logger, func() { w.handleEvents(logCtx, reader.Subscribe()) }) + events := reader.Subscribe() + pulse.Go(w.logger, func() { w.handleEvents(logCtx, events) }) pulse.Go(w.logger, func() { w.keepAlive(logCtx) }) return w, nil @@ -176,6 +227,7 @@ func (w *Worker) Jobs() []*Job { CreatedAt: job.CreatedAt, Worker: &Worker{ID: w.ID, node: w.node, CreatedAt: w.CreatedAt}, NodeID: job.NodeID, + Requeued: job.Requeued, }) } return jobs @@ -198,37 +250,74 @@ func (w *Worker) handleEvents(ctx context.Context, c <-chan *streaming.Event) { if !ok { return } - nodeID, payload := unmarshalEnvelope(ev.Payload) - var err error + if err := w.refreshHeartbeat(ctx); err != nil { + w.logger.Error(fmt.Errorf("worker intake heartbeat failed: %w", err)) + if redis.HasErrorPrefix(err, "WORKERCLEANUPLOST") { + return + } + continue + } + nodeID, payload, err := unmarshalEnvelope(ev.Payload) + if err != nil { + w.dropMalformedEvent(ctx, ev, fmt.Errorf("decode worker event envelope: %w", err)) + continue + } + var dispatched *Job switch ev.EventName { case evInit: w.logger.Debug("handleEvents: received init", "event", ev.EventName, "id", ev.ID) continue case evStartJob: w.logger.Debug("handleEvents: received start job", "event", ev.EventName, "id", ev.ID) - err = w.startJob(ctx, unmarshalJob(payload)) + dispatched, err = unmarshalJob(payload) + if err == nil { + err = w.startJob(ctx, dispatched) + } case evMessage: w.logger.Debug("handleEvents: received message", "event", ev.EventName, "id", ev.ID) - key, payload := unmarshalKeyedPayload(payload) - err = w.message(key, payload) + var key string + key, payload, err = unmarshalKeyedPayload(payload) + if err == nil { + err = w.message(key, payload) + } case evStopJob: w.logger.Debug("handleEvents: received stop job", "event", ev.EventName, "id", ev.ID) - err = w.stopJob(ctx, unmarshalJobKey(payload)) + var key string + key, err = unmarshalJobKey(payload) + if err == nil { + err = w.stopJob(ctx, key) + } case evNotify: w.logger.Debug("handleEvents: received notify", "event", ev.EventName, "id", ev.ID) - key, payload := unmarshalKeyedPayload(payload) - err = w.notify(ctx, key, payload) + var key string + key, payload, err = unmarshalKeyedPayload(payload) + if err == nil { + err = w.notify(ctx, key, payload) + } + default: + err = fmt.Errorf("unknown worker event %q", ev.EventName) } if err != nil { + if redis.HasErrorPrefix(err, "WORKERCLEANUPLOST") { + return + } if errors.Is(err, ErrRequeue) { - w.logger.Info("requeue", "event", ev.EventName, "id", ev.ID, "after", w.pendingJobTTL) + w.logger.Info("requeue", "event", ev.EventName, "id", ev.ID) continue } - w.ackPoolEvent(ctx, nodeID, ev.ID, err) + if dispatched != nil && dispatched.dispatchID != "" { + w.node.ownDispatchSettlement(w, nodeID, ev.ID, dispatched, err) + } else { + w.ackPoolEvent(ctx, nodeID, ev.ID, err) + } w.logger.Error(fmt.Errorf("handler failed: %w", err), "event", ev.EventName, "id", ev.ID) continue } - w.ackPoolEvent(ctx, nodeID, ev.ID, nil) + if dispatched != nil && dispatched.dispatchID != "" { + w.node.ownDispatchSettlement(w, nodeID, ev.ID, dispatched, nil) + } else { + w.ackPoolEvent(ctx, nodeID, ev.ID, nil) + } case <-w.done: w.logger.Debug("handleEvents: done") return @@ -236,21 +325,56 @@ func (w *Worker) handleEvents(ctx context.Context, c <-chan *streaming.Event) { } } +// dropMalformedEvent logs and removes an envelope that cannot identify its +// sender, so a permanent worker loop never reprocesses the poison entry. +func (w *Worker) dropMalformedEvent(ctx context.Context, event *streaming.Event, decodeErr error) { + w.logger.Error(decodeErr, "event", event.EventName, "id", event.ID) + if err := w.stream.Remove(ctx, event.ID); err != nil { + w.logger.Error(fmt.Errorf("drop malformed worker event %s: %w", event.ID, err)) + } +} + // stop stops the reader, destroys the stream and closes the worker. -func (w *Worker) stop(ctx context.Context) { +func (w *Worker) stop(ctx context.Context) error { + w.stopLocal() + w.lock.RLock() + destroyed := w.streamDestroyed + w.lock.RUnlock() + if destroyed { + return nil + } + if err := w.stream.Destroy(ctx); err != nil { + return fmt.Errorf("failed to destroy stream for worker: %w", err) + } w.lock.Lock() - if w.stopped { - w.lock.Unlock() - return + w.streamDestroyed = true + w.lock.Unlock() + return nil +} + +// stopLocal stops and joins worker intake without mutating Redis. It is used +// when pool cleanup already destroyed the worker's generation-owned resources. +func (w *Worker) stopLocal() { + firstAttempt := w.stopIntake() + if firstAttempt { + w.wg.Wait() + } +} + +// stopIntake closes worker-owned input without joining the calling goroutine. +func (w *Worker) stopIntake() bool { + w.lock.Lock() + firstAttempt := !w.stopped + if firstAttempt { + w.stopped = true } - w.stopped = true w.lock.Unlock() - w.reader.Close() - if err := w.stream.Destroy(ctx); err != nil { - w.logger.Error(fmt.Errorf("failed to destroy stream for worker: %w", err)) + + if firstAttempt { + close(w.done) + w.reader.Close() } - close(w.done) - w.wg.Wait() + return firstAttempt } // startJob starts a job. @@ -258,27 +382,42 @@ func (w *Worker) startJob(ctx context.Context, job *Job) error { if w.IsStopped() { return fmt.Errorf("worker %q stopped", w.ID) } - if _, err := w.jobsMap.AppendUniqueValues(ctx, w.ID, job.Key); err != nil { - w.logger.Error(fmt.Errorf("failed to add job %q to jobs map: %w, requeueing", job.Key, err)) - return ErrRequeue + if err := w.refreshHeartbeat(ctx); err != nil { + return err + } + if err := w.node.ensureGenerationActive(ctx); err != nil { + return err } - if _, err := w.jobPayloadsMap.Set(ctx, job.Key, string(job.Payload)); err != nil { - w.logger.Error(fmt.Errorf("failed to add job payload %q to job payloads map: %w, requeueing", job.Key, err)) - if _, _, removeErr := w.jobsMap.RemoveValues(ctx, w.ID, job.Key); removeErr != nil { - w.logger.Error(fmt.Errorf("start failure handling: failed to remove job %q from jobs map: %w", job.Key, removeErr)) + if job.dispatchID != "" { + claimed, err := w.claimDispatchedStart(ctx, job) + if err != nil { + return errors.Join(ErrRequeue, err) + } + if !claimed { + return ErrRequeue + } + } else { + if err := w.node.appendPoolMapValue(ctx, w.node.resources.jobs, w.ID, job.Key); err != nil { + w.logger.Error(fmt.Errorf("failed to add job %q to jobs map: %w, requeueing", job.Key, err)) + return ErrRequeue + } + if err := w.node.setPoolMap(ctx, w.node.resources.jobPayloads, job.Key, string(job.Payload)); err != nil { + w.logger.Error(fmt.Errorf("failed to add job payload %q to job payloads map: %w, requeueing", job.Key, err)) + if cleanupErr := w.cleanupFailedStart(ctx, job.Key); cleanupErr != nil { + return errors.Join( + ErrRequeue, + fmt.Errorf("persist job payload %q: %w", job.Key, err), + cleanupErr, + ) + } + return ErrRequeue } - return ErrRequeue } job.Worker = w if err := w.handler.Start(job); err != nil { w.logger.Debug("handler failed to start job", "job", job.Key, "error", err) - if _, _, err := w.jobsMap.RemoveValues(ctx, w.ID, job.Key); err != nil { - w.logger.Error(fmt.Errorf("start failure handling: failed to remove job %q from jobs map: %w", job.Key, err)) - } - if !job.Requeued { - if _, err := w.jobPayloadsMap.Delete(ctx, job.Key); err != nil { - w.logger.Error(fmt.Errorf("start failure handling: failed to remove job payload %q from job payloads map: %w", job.Key, err)) - } + if cleanupErr := w.cleanupFailedStart(ctx, job.Key); cleanupErr != nil { + return errors.Join(ErrRequeue, err, cleanupErr) } return err } @@ -287,6 +426,85 @@ func (w *Worker) startJob(ctx context.Context, job *Job) error { return nil } +// claimDispatchedStart creates durable ownership exactly once for the pending +// dispatch capability before the handler can run. +func (w *Worker) claimDispatchedStart(ctx context.Context, job *Job) (bool, error) { + identity, err := dispatchIdentity(job.Key, job.Payload) + if err != nil { + return false, fmt.Errorf("claim dispatched start for job %q: %w", job.Key, err) + } + result, err := claimWorkerStartScript.Run( + ctx, + w.node.rdb, + []string{ + fmt.Sprintf("pulse:stream:%s:lifecycle", w.node.poolStream.Name), + rmapContentKey(w.node.resources.jobPending), + dispatchRecordKey(w.node.resources.dispatches, job.dispatchID), + rmapContentKey(w.node.resources.jobPayloads), + rmapContentKey(w.node.resources.jobs), + rmapUpdateChannel(w.node.resources.jobs), + rmapUpdateChannel(w.node.resources.jobPayloads), + rmapContentKey(w.node.resources.workers), + rmapContentKey(w.node.resources.workerCleanup), + }, + w.node.resources.generation, + "active", + job.Key, + job.dispatchID, + w.ID, + job.Payload, + identity, + ).Int64() + if err != nil { + return false, fmt.Errorf("claim dispatched start for job %q: %w", job.Key, err) + } + return result == 1, nil +} + +// cleanupFailedStart atomically removes durable ownership and payload under the +// exact pool-stream generation. Callers must retry the event while this recipe +// fails and may report terminal handler failure only after it succeeds. +func (w *Worker) cleanupFailedStart(ctx context.Context, key string) error { + err := cleanupFailedStartScript.Run( + ctx, + w.node.rdb, + []string{ + fmt.Sprintf("pulse:stream:%s:lifecycle", w.node.poolStream.Name), + rmapContentKey(w.node.resources.jobs), + rmapUpdateChannel(w.node.resources.jobs), + rmapContentKey(w.node.resources.jobPayloads), + rmapUpdateChannel(w.node.resources.jobPayloads), + }, + w.node.resources.generation, + w.ID, + key, + "active", + ).Err() + if err != nil { + return fmt.Errorf("clean failed start for job %q: %w", key, err) + } + return nil +} + +// markDispatchSettled makes a successfully started exact-dispatch job eligible +// for ordinary running-job rebalancing only after its original dispatch event +// and durable terminal record have settled atomically. +func (w *Worker) markDispatchSettled(key, dispatchID string) { + w.lock.Lock() + defer w.lock.Unlock() + value, ok := w.jobs.Load(key) + if !ok { + return + } + job := value.(*Job) + if job.dispatchID != dispatchID { + return + } + settled := *job + settled.dispatchID = "" + w.jobs.Store(key, &settled) +} + // stopJob stops a job. func (w *Worker) stopJob(ctx context.Context, key string) error { if err := w.releaseJob(ctx, key); err != nil { @@ -295,7 +513,7 @@ func (w *Worker) stopJob(ctx context.Context, key string) error { } return err } - if _, err := w.jobPayloadsMap.Delete(ctx, key); err != nil { + if err := w.node.deletePoolMap(ctx, w.node.resources.jobPayloads, key); err != nil { w.logger.Error(fmt.Errorf("stop job: failed to remove job payload %q from job payloads map: %w", key, err)) } w.logger.Info("stopped job", "job", key) @@ -308,12 +526,15 @@ func (w *Worker) releaseJob(ctx context.Context, key string) error { if _, ok := w.jobs.Load(key); !ok { return fmt.Errorf("%w: %s", errJobNotOwned, key) } + if err := w.node.ensureGenerationActive(ctx); err != nil { + return err + } if err := w.handler.Stop(key); err != nil { return fmt.Errorf("failed to stop job %q: %w", key, err) } w.logger.Debug("stopped job", "job", key) w.jobs.Delete(key) - if _, _, err := w.jobsMap.RemoveValues(ctx, w.ID, key); err != nil { + if err := w.node.removePoolMapValue(ctx, w.node.resources.jobs, w.ID, key); err != nil { return fmt.Errorf("failed to release job %q from jobs map: %w", key, err) } return nil @@ -351,22 +572,46 @@ func (w *Worker) message(key string, payload []byte) error { return mh.HandleMessage(key, payload) } -// ackPoolEvent acknowledges the pool event that originated from the node with -// the given ID. +// ackPoolEvent publishes the worker outcome to the originating node. It retries +// while that node stream exists; a vanished exact generation leaves the +// original pool event to sink recovery, while dispatched starts are already +// durably terminal through settleDispatchedStart. func (w *Worker) ackPoolEvent(ctx context.Context, nodeID, eventID string, ackerr error) { - stream, err := w.node.getNodeStream(nodeID) - if err != nil { - w.logger.Error(fmt.Errorf("failed to get ack stream for node %q: %w", nodeID, err)) - return - } - var msg string if ackerr != nil { msg = ackerr.Error() } ack := &ack{EventID: eventID, Error: msg} - if _, err := stream.Add(ctx, evAck, marshalEnvelope(w.ID, marshalAck(ack)), options.WithOnlyIfStreamExists()); err != nil { - w.logger.Error(fmt.Errorf("failed to ack event %q from node %q: %w", eventID, nodeID, err)) + payload := marshalEnvelope(w.ID, marshalAck(ack)) + delay := 100 * time.Millisecond + for { + stream, err := w.node.getNodeStream(nodeID) + if err == nil { + _, err = stream.Add(ctx, evAck, payload, options.WithOnlyIfStreamExists()) + } + if err == nil { + return + } + if errors.Is(err, streaming.ErrStreamNotFound) || + errors.Is(err, streaming.ErrStreamDestroyed) { + return + } + w.logger.Error( + fmt.Errorf("failed to ack event %q from node %q: %w", eventID, nodeID, err), + "retry_in", + delay, + ) + timer := time.NewTimer(delay) + select { + case <-timer.C: + delay = min(delay*2, 5*time.Second) + case <-ctx.Done(): + timer.Stop() + return + case <-w.done: + timer.Stop() + return + } } } @@ -382,9 +627,11 @@ func (w *Worker) keepAlive(ctx context.Context) { if w.IsStopped() { return // Let's not recreate the map if we just deleted it } - now := strconv.FormatInt(time.Now().UnixNano(), 10) - if _, err := w.keepAliveMap.Set(ctx, w.ID, now); err != nil { + if err := w.refreshHeartbeat(ctx); err != nil { w.logger.Error(fmt.Errorf("failed to update worker keep-alive: %w", err)) + if redis.HasErrorPrefix(err, "WORKERCLEANUPLOST") { + return + } } case <-w.done: w.logger.Debug("keepAlive: done") @@ -393,12 +640,28 @@ func (w *Worker) keepAlive(ctx context.Context) { } } +// refreshHeartbeat renews the Redis-time worker liveness proof. A cleanup +// fence is terminal and closes intake before further handler or ownership work. +func (w *Worker) refreshHeartbeat(ctx context.Context) error { + _, err := w.node.updateWorkerHeartbeat(ctx, w.ID) + if redis.HasErrorPrefix(err, "WORKERCLEANUPLOST") { + w.stopIntake() + } + return err +} + // rebalance rebalances the jobs handled by the worker. func (w *Worker) rebalance(ctx context.Context, activeWorkers []string) { + w.rebalanceLock.Lock() + defer w.rebalanceLock.Unlock() + w.logger.Debug("rebalance") rebalanced := make(map[string]*Job) w.jobs.Range(func(key, value any) bool { job := value.(*Job) + if job.dispatchID != "" { + return true + } wid := activeWorkers[w.node.h.Hash(job.Key, int64(len(activeWorkers)))] if wid != w.ID { rebalanced[job.Key] = job @@ -411,19 +674,21 @@ func (w *Worker) rebalance(ctx context.Context, activeWorkers []string) { return } for key, job := range rebalanced { - job.Requeued = true + requeue := *job + requeue.Requeued = true + requeue.dispatchID = "" if err := w.releaseJob(ctx, key); err != nil { w.logger.Error(fmt.Errorf("rebalance: failed to release job: %w", err), "job", key) if _, ok := w.jobs.Load(key); !ok { - if err := w.startJob(ctx, job); err != nil { + if err := w.startJob(ctx, &requeue); err != nil { w.logger.Error(fmt.Errorf("rebalance: failed to restart job: %w", err), "job", key) } } continue } - if _, err := w.node.poolStream.Add(ctx, evStartJob, marshalJob(job)); err != nil { + if _, err := w.node.poolStream.Add(ctx, evStartJob, marshalJob(&requeue)); err != nil { w.logger.Error(fmt.Errorf("rebalance: failed to requeue job: %w", err), "job", key) - if err := w.startJob(ctx, job); err != nil { + if err := w.startJob(ctx, &requeue); err != nil { w.logger.Error(fmt.Errorf("rebalance: failed to restart job: %w", err), "job", key) continue } @@ -437,37 +702,42 @@ func (w *Worker) rebalance(ctx context.Context, activeWorkers []string) { // This should be done after the worker is stopped. func (w *Worker) requeueJobs(ctx context.Context) error { jobsToRequeue := make(map[string]*Job) + var unsettled []string jobCount := 0 w.jobs.Range(func(key, value any) bool { job := value.(*Job) + if job.dispatchID != "" { + unsettled = append(unsettled, job.dispatchID) + return true + } jobsToRequeue[key.(string)] = job jobCount++ return true }) + if len(unsettled) > 0 { + sort.Strings(unsettled) + return fmt.Errorf("requeueJobs: exact dispatch settlements still pending: %v", unsettled) + } if jobCount == 0 { w.logger.Debug("requeueJobs: no jobs to requeue") return nil } - createdAt := strconv.FormatInt(w.CreatedAt.UnixNano(), 10) w.logger.Debug("requeueJobs: requeuing", "jobs", jobCount) - // First mark the worker as inactive so that requeued jobs are not assigned to this worker - // Use optimistic locking to avoid race conditions. - prev, err := w.node.workerMap.TestAndSet(ctx, w.ID, createdAt, "-") + // Mark the worker inactive behind the cleanup fence so requeued jobs are + // not assigned to this worker and exactly one party requeues: losing + // deactivation means stale-worker cleanup owns (or already completed) the + // requeue, so this worker must not publish duplicates. + won, prev, err := w.node.deactivateWorker(ctx, w.ID) if err != nil { return fmt.Errorf("requeueJobs: failed to mark worker as inactive: %w", err) } - if prev == "-" { - w.logger.Debug("requeueJobs: jobs already requeued, skipping requeue") + if !won { + w.logger.Debug("requeueJobs: requeue owned elsewhere, skipping", "registration", prev) return nil } - // If the optimistic lock failed (unexpected value), force the worker to - // inactive anyway. This worker is stopped and must not receive requeued jobs. - if prev != createdAt { - w.logger.Error(fmt.Errorf("requeueJobs: failed optimistic lock for worker inactive mark"), "worker", w.ID, "expected", createdAt, "got", prev) - if _, err := w.node.workerMap.Set(ctx, w.ID, "-"); err != nil { - return fmt.Errorf("requeueJobs: failed to force worker inactive state: %w", err) - } + if createdAt := strconv.FormatInt(w.CreatedAt.UnixNano(), 10); prev != createdAt { + w.logger.Error(fmt.Errorf("requeueJobs: unexpected worker registration"), "worker", w.ID, "expected", createdAt, "got", prev) } retryUntil := time.Now().Add(w.workerTTL) @@ -491,61 +761,77 @@ func (w *Worker) requeueJobs(ctx context.Context) error { // attemptRequeue attempts to requeue the jobs in the given map. // It returns any job that failed to be requeued. func (w *Worker) attemptRequeue(ctx context.Context, jobsToRequeue map[string]*Job) map[string]*Job { + return w.attemptRequeueWith(ctx, jobsToRequeue, w.requeueJob) +} + +// attemptRequeueWith runs every handoff concurrently under one timeout, joins +// all senders, and removes only jobs whose send completed successfully. +func (w *Worker) attemptRequeueWith( + ctx context.Context, + jobsToRequeue map[string]*Job, + send func(context.Context, *Job) error, +) map[string]*Job { var wg sync.WaitGroup - type result struct { - key string - err error + resultChan := make(chan requeueResult, len(jobsToRequeue)) + remainingJobs := make(map[string]*Job, len(jobsToRequeue)) + for key, job := range jobsToRequeue { + remainingJobs[key] = job } - resultChan := make(chan result, len(jobsToRequeue)) - defer close(resultChan) + attemptCtx, cancel := context.WithTimeout(ctx, w.requeueTimeout) + defer cancel() wg.Add(len(jobsToRequeue)) for key, job := range jobsToRequeue { pulse.Go(w.logger, func() { defer wg.Done() - err := w.requeueJob(ctx, job) + err := send(attemptCtx, job) if err != nil { w.logger.Error(fmt.Errorf("failed to requeue job: %w", err), "job", key) } else { w.logger.Debug("requeueJobs: requeued", "job", key) } - resultChan <- result{key: key, err: err} + resultChan <- requeueResult{key: key, err: err} }) } - wg.Wait() - remainingJobs := make(map[string]*Job) - for { + timedOut := false + for processed := 0; processed < len(jobsToRequeue); processed++ { select { case res := <-resultChan: if res.err != nil { w.logger.Error(fmt.Errorf("requeueJobs: failed to requeue job %q: %w", res.key, res.err)) - remainingJobs[res.key] = jobsToRequeue[res.key] continue } delete(remainingJobs, res.key) w.logger.Info("requeued", "job", res.key) - if len(remainingJobs) == 0 { - w.logger.Debug("requeueJobs: all jobs requeued") - return remainingJobs - } - case <-time.After(w.workerShutdownTTL): - w.logger.Error(fmt.Errorf("requeueJobs: timeout reached, some jobs may not have been processed")) - return remainingJobs + case <-attemptCtx.Done(): + timedOut = true + cancel() + processed = len(jobsToRequeue) } } + wg.Wait() + close(resultChan) + for res := range resultChan { + if res.err == nil { + delete(remainingJobs, res.key) + w.logger.Info("requeued", "job", res.key) + } + } + if timedOut { + w.logger.Error(fmt.Errorf("requeueJobs: timeout reached with %d jobs not handed off", len(remainingJobs))) + } + return remainingJobs } // requeueJob requeues a job. func (w *Worker) requeueJob(ctx context.Context, job *Job) error { job.Requeued = true - eventID, err := w.node.poolStream.Add(ctx, evStartJob, marshalJob(job)) + job.dispatchID = "" + _, err := w.node.poolStream.Add(ctx, evStartJob, marshalJob(job)) if err != nil { return fmt.Errorf("requeueJob: failed to add job to pool stream: %w", err) } - // Mark this event as a "requeue" so any node waiting for a dispatch return (if any) - // can simply clean up without blocking. - w.node.pendingJobChannels.Store(eventID, nil) // Stop locally, but do not touch the replicated job/payload maps: we want the // payload to remain available for distributed recovery until the job is diff --git a/pool/worker_cleanup.go b/pool/worker_cleanup.go new file mode 100644 index 0000000..eeb555a --- /dev/null +++ b/pool/worker_cleanup.go @@ -0,0 +1,656 @@ +// Stale-worker recovery is owned by one Redis-time lease capability. Requeue +// publication verifies generation, owner token, fence, and lease in the same +// Lua operation and records a stable worker/job publication key, preventing +// overlap and ABA duplicates after takeover. +package pool + +import ( + "context" + "encoding/hex" + "errors" + "fmt" + "strconv" + "strings" + "time" + + "github.com/oklog/ulid/v2" + redis "github.com/redis/go-redis/v9" + + "goa.design/pulse/streaming" +) + +type ( + // workerCleanupLease is the exact capability returned by Redis. + workerCleanupLease struct { + workerID string + owner string + fence string + } +) + +var ( + // updateWorkerHeartbeatScript refreshes the authoritative Redis-time + // heartbeat only while the worker remains registered and no cleanup fence + // has been installed. A resumed stale worker therefore cannot resurrect + // itself after cleanup begins. + updateWorkerHeartbeatScript = redis.NewScript(` +if redis.call("HGET", KEYS[1], "state") ~= ARGV[1] +or redis.call("HGET", KEYS[1], "generation") ~= ARGV[2] then + return redis.error_reply("POOLGENERATIONLOST") +end +local registration = redis.call("HGET", KEYS[4], ARGV[3]) +if not registration or registration == "-" +or redis.call("HGET", KEYS[3], ARGV[3]) then + return redis.error_reply("WORKERCLEANUPLOST") +end +local clock = redis.call("TIME") +local timestamp = clock[1] .. string.format("%06d", clock[2]) .. "000" +redis.call("HSET", KEYS[2], ARGV[3], timestamp) +local rev = tostring(redis.call("HINCRBY", KEYS[2], "=rev", 1)) +redis.call("HSET", KEYS[2], "=kind", "set") +local message = struct.pack( + "ic0ic0ic0", + string.len(ARGV[3]), ARGV[3], + string.len(timestamp), timestamp, + string.len(rev), rev +) +redis.call("PUBLISH", KEYS[5], "set:" .. message) +return timestamp +`) + + // deactivateWorkerScript marks one registered worker inactive for graceful + // requeue. It refuses while a cleanup fence is installed and never + // recreates a registration that stale-worker cleanup already removed, so + // exactly one party — the worker or the cleanup owner — requeues its jobs. + deactivateWorkerScript = redis.NewScript(` +if redis.call("HGET", KEYS[1], "state") ~= ARGV[1] +or redis.call("HGET", KEYS[1], "generation") ~= ARGV[2] then + return redis.error_reply("POOLGENERATIONLOST") +end +if redis.call("HGET", KEYS[3], ARGV[3]) then + return {0, "cleanup"} +end +local registration = redis.call("HGET", KEYS[2], ARGV[3]) +if not registration then + return {0, "absent"} +end +if registration == "-" then + return {0, "-"} +end +local inactive = "-" +redis.call("HSET", KEYS[2], ARGV[3], inactive) +local rev = tostring(redis.call("HINCRBY", KEYS[2], "=rev", 1)) +redis.call("HSET", KEYS[2], "=kind", "set") +local message = struct.pack( + "ic0ic0ic0", + string.len(ARGV[3]), ARGV[3], + string.len(inactive), inactive, + string.len(rev), rev +) +redis.call("PUBLISH", KEYS[4], "set:" .. message) +return {1, registration} +`) + + // acquireWorkerCleanupScript acquires, renews, or steals an expired cleanup + // lease only after atomically proving the authoritative heartbeat expired. + acquireWorkerCleanupScript = redis.NewScript(` +if redis.call("HGET", KEYS[1], "state") ~= ARGV[1] +or redis.call("HGET", KEYS[1], "generation") ~= ARGV[2] then + return redis.error_reply("POOLGENERATIONLOST") +end +local clock = redis.call("TIME") +local now = tonumber(clock[1]) * 1000 + math.floor(tonumber(clock[2]) / 1000) +local now_ns = tonumber(clock[1]) * 1000000000 + tonumber(clock[2]) * 1000 +local heartbeat = redis.call("HGET", KEYS[2], ARGV[3]) +if heartbeat then + local heartbeat_ns = tonumber(heartbeat) + if not heartbeat_ns then + return redis.error_reply("WORKERHEARTBEATINVALID") + end + if heartbeat_ns + tonumber(ARGV[7]) >= now_ns then + return {0, "live"} + end +end +local current = redis.call("HGET", KEYS[3], ARGV[3]) +if current then + local current_owner, current_fence, current_until = + string.match(current, "^([^|]+)|([^|]+)|(%d+)$") + if not current_owner then + return redis.error_reply("WORKERCLEANUPINVALID") + end + if current_owner ~= ARGV[4] and tonumber(current_until) > now then + return {0, current_fence} + end + if current_owner == ARGV[4] and tonumber(current_until) > now then + redis.call("HSET", KEYS[3], ARGV[3], + current_owner .. "|" .. current_fence .. "|" .. tostring(now + tonumber(ARGV[5]))) + return {1, current_fence} + end +end +local fence_field = ARGV[6] +local fence = tostring(redis.call("HINCRBY", KEYS[3], fence_field, 1)) +local value = ARGV[4] .. "|" .. fence .. "|" .. tostring(now + tonumber(ARGV[5])) +redis.call("HSET", KEYS[3], ARGV[3], value) +local rev = tostring(redis.call("HINCRBY", KEYS[3], "=rev", 1)) +redis.call("HSET", KEYS[3], "=kind", "set") +local message = struct.pack( + "ic0ic0ic0", + string.len(ARGV[3]), ARGV[3], + string.len(value), value, + string.len(rev), rev +) +redis.call("PUBLISH", KEYS[4], "set:" .. message) +return {1, fence} +`) + + // renewWorkerCleanupScript proves the exact owner still holds an unexpired + // lease before a non-publication cleanup step begins. + renewWorkerCleanupScript = redis.NewScript(` +if redis.call("HGET", KEYS[1], "state") ~= ARGV[1] +or redis.call("HGET", KEYS[1], "generation") ~= ARGV[2] then + return redis.error_reply("POOLGENERATIONLOST") +end +local clock = redis.call("TIME") +local now = tonumber(clock[1]) * 1000 + math.floor(tonumber(clock[2]) / 1000) +local current = redis.call("HGET", KEYS[2], ARGV[3]) +local owner, fence, lease_until = string.match(current or "", "^([^|]+)|([^|]+)|(%d+)$") +if owner ~= ARGV[4] or fence ~= ARGV[5] or tonumber(lease_until or "0") <= now then + return redis.error_reply("WORKERCLEANUPLOST") +end +redis.call("HSET", KEYS[2], ARGV[3], + owner .. "|" .. fence .. "|" .. tostring(now + tonumber(ARGV[6]))) +return 1 +`) + + // publishWorkerRequeueScript atomically fences and deduplicates one stale + // worker/job handoff before appending it to the pool stream. + publishWorkerRequeueScript = redis.NewScript(` +if redis.call("HGET", KEYS[1], "state") ~= ARGV[1] +or redis.call("HGET", KEYS[1], "generation") ~= ARGV[2] then + return redis.error_reply("POOLGENERATIONLOST") +end +local clock = redis.call("TIME") +local now = tonumber(clock[1]) * 1000 + math.floor(tonumber(clock[2]) / 1000) +local current = redis.call("HGET", KEYS[2], ARGV[3]) +local owner, fence, lease_until = string.match(current or "", "^([^|]+)|([^|]+)|(%d+)$") +if owner ~= ARGV[4] or fence ~= ARGV[5] or tonumber(lease_until or "0") <= now then + return redis.error_reply("WORKERCLEANUPLOST") +end +redis.call("HSET", KEYS[2], ARGV[3], + owner .. "|" .. fence .. "|" .. tostring(now + tonumber(ARGV[6]))) +local existing = redis.call("HGET", KEYS[2], ARGV[9]) +if existing then + return {0, existing} +end +local encoded = redis.call("HGET", KEYS[4], ARGV[3]) +if not encoded then + return {2, ""} +end +local ok, jobs = pcall(cjson.decode, encoded) +if not ok or type(jobs) ~= "table" then + return redis.error_reply("POOLMAPINVALID") +end +local owned = false +for _, job in ipairs(jobs) do + if job == ARGV[7] then + owned = true + break + end +end +if not owned then + return {2, ""} +end +if not redis.call("HGET", KEYS[5], ARGV[7]) then + return {3, ""} +end +local stream = redis.call("HGET", KEYS[1], "physical_key") +if not stream then + return redis.error_reply("POOLGENERATIONLOST") +end +local event_id = redis.call("XADD", stream, "*", "n", ARGV[8], "p", ARGV[10]) +redis.call("HSET", KEYS[2], ARGV[9], event_id) +return {1, event_id} +`) + + // removeStaleWorkerJobScript removes payload-less ownership only while the + // exact cleanup lease is live and the authoritative payload remains absent. + removeStaleWorkerJobScript = redis.NewScript(` +if redis.call("HGET", KEYS[1], "state") ~= ARGV[1] +or redis.call("HGET", KEYS[1], "generation") ~= ARGV[2] then + return redis.error_reply("POOLGENERATIONLOST") +end +local clock = redis.call("TIME") +local now = tonumber(clock[1]) * 1000 + math.floor(tonumber(clock[2]) / 1000) +local current = redis.call("HGET", KEYS[2], ARGV[3]) +local owner, fence, lease_until = string.match(current or "", "^([^|]+)|([^|]+)|(%d+)$") +if owner ~= ARGV[4] or fence ~= ARGV[5] or tonumber(lease_until or "0") <= now then + return redis.error_reply("WORKERCLEANUPLOST") +end +if redis.call("HGET", KEYS[5], ARGV[6]) then + return 0 +end +local encoded = redis.call("HGET", KEYS[3], ARGV[3]) +if not encoded then + return 1 +end +local ok, jobs = pcall(cjson.decode, encoded) +if not ok or type(jobs) ~= "table" then + return redis.error_reply("POOLMAPINVALID") +end +local remaining = {} +local changed = false +for _, job in ipairs(jobs) do + if job == ARGV[6] then + changed = true + else + table.insert(remaining, job) + end +end +if not changed then + return 1 +end +local update = "" +if #remaining == 0 then + redis.call("HDEL", KEYS[3], ARGV[3]) +else + update = cjson.encode(remaining) + redis.call("HSET", KEYS[3], ARGV[3], update) +end +local rev = tostring(redis.call("HINCRBY", KEYS[3], "=rev", 1)) +if update == "" then + redis.call("HSET", KEYS[3], "=kind", "del") + local message = struct.pack( + "ic0ic0", + string.len(ARGV[3]), ARGV[3], + string.len(rev), rev + ) + redis.call("PUBLISH", KEYS[4], "del:" .. message) +else + redis.call("HSET", KEYS[3], "=kind", "set") + local message = struct.pack( + "ic0ic0ic0", + string.len(ARGV[3]), ARGV[3], + string.len(update), update, + string.len(rev), rev + ) + redis.call("PUBLISH", KEYS[4], "set:" .. message) +end +return 1 +`) + + // deleteStaleWorkerScript destroys the exact worker stream incarnation and + // removes worker discovery only while the cleanup capability is current. + deleteStaleWorkerScript = redis.NewScript(` +if redis.call("HGET", KEYS[1], "state") ~= ARGV[1] +or redis.call("HGET", KEYS[1], "generation") ~= ARGV[2] then + return redis.error_reply("POOLGENERATIONLOST") +end +local clock = redis.call("TIME") +local now = tonumber(clock[1]) * 1000 + math.floor(tonumber(clock[2]) / 1000) +local current = redis.call("HGET", KEYS[2], ARGV[3]) +local owner, fence, lease_until = string.match(current or "", "^([^|]+)|([^|]+)|(%d+)$") +if owner ~= ARGV[4] or fence ~= ARGV[5] or tonumber(lease_until or "0") <= now then + return redis.error_reply("WORKERCLEANUPLOST") +end +redis.call("HSET", KEYS[2], ARGV[3], + owner .. "|" .. fence .. "|" .. tostring(now + tonumber(ARGV[6]))) + +if redis.call("HGET", KEYS[3], "generation") == ARGV[7] then + local state = redis.call("HGET", KEYS[3], "state") + if state == "active" then + local physical = redis.call("HGET", KEYS[3], "physical_key") + if not physical then + return redis.error_reply("STREAMDESTROYED") + end + redis.call("HSET", KEYS[3], "state", "destroyed") + local resources = redis.call("SMEMBERS", KEYS[4]) + if #resources > 0 then + redis.call("DEL", unpack(resources)) + end + redis.call("DEL", physical, physical .. ":sink-recovery:" .. ARGV[7], KEYS[4]) + elseif state ~= "destroyed" then + return redis.error_reply("STREAMDESTROYED") + end +end + +local function delete_field(content, channel, field) + if redis.call("HDEL", content, field) == 0 then + return + end + local rev = tostring(redis.call("HINCRBY", content, "=rev", 1)) + redis.call("HSET", content, "=kind", "del") + local message = struct.pack( + "ic0ic0", + string.len(field), field, + string.len(rev), rev + ) + redis.call("PUBLISH", channel, "del:" .. message) +end +delete_field(KEYS[5], KEYS[6], ARGV[3]) +delete_field(KEYS[7], KEYS[8], ARGV[3]) +delete_field(KEYS[9], KEYS[10], ARGV[3]) +return 1 +`) + + // releaseWorkerCleanupScript releases only the exact capability. Completed + // cleanup also removes its stable publication and fence metadata. + releaseWorkerCleanupScript = redis.NewScript(` +if redis.call("HGET", KEYS[1], "state") ~= ARGV[1] +or redis.call("HGET", KEYS[1], "generation") ~= ARGV[2] then + return redis.error_reply("POOLGENERATIONLOST") +end +local current = redis.call("HGET", KEYS[2], ARGV[3]) +local owner, fence = string.match(current or "", "^([^|]+)|([^|]+)|%d+$") +if owner ~= ARGV[4] or fence ~= ARGV[5] then + return 0 +end +redis.call("HDEL", KEYS[2], ARGV[3]) +if ARGV[6] == "1" then + local fields = redis.call("HKEYS", KEYS[2]) + for _, field in ipairs(fields) do + if field == ARGV[7] or string.sub(field, 1, string.len(ARGV[8])) == ARGV[8] then + redis.call("HDEL", KEYS[2], field) + end + end +end +local rev = tostring(redis.call("HINCRBY", KEYS[2], "=rev", 1)) +redis.call("HSET", KEYS[2], "=kind", "del") +local message = struct.pack( + "ic0ic0", + string.len(ARGV[3]), ARGV[3], + string.len(rev), rev +) +redis.call("PUBLISH", KEYS[3], "del:" .. message) +return 1 +`) +) + +// acquireWorkerCleanup returns an exact lease or nil while another owner is live. +func (node *Node) acquireWorkerCleanup( + ctx context.Context, + workerID string, +) (*workerCleanupLease, error) { + owner := node.ID + "-" + ulid.Make().String() + raw, err := acquireWorkerCleanupScript.Run( + ctx, + node.rdb, + []string{ + fmt.Sprintf("pulse:stream:%s:lifecycle", node.poolStream.Name), + rmapContentKey(node.resources.workerKeepAlive), + rmapContentKey(node.resources.workerCleanup), + rmapUpdateChannel(node.resources.workerCleanup), + }, + "active", + node.resources.generation, + workerID, + owner, + strconv.FormatInt(node.workerTTL.Milliseconds(), 10), + workerCleanupFenceField(workerID), + strconv.FormatInt(node.resources.workerTTL.Nanoseconds(), 10), + ).Slice() + if err != nil { + return nil, poolBoundaryError(err) + } + if len(raw) != 2 { + return nil, fmt.Errorf("worker cleanup claim returned %d fields", len(raw)) + } + status, ok := raw[0].(int64) + if !ok || (status != 0 && status != 1) { + return nil, fmt.Errorf("worker cleanup claim returned invalid status %T(%v)", raw[0], raw[0]) + } + fence, ok := raw[1].(string) + if !ok || fence == "" { + return nil, fmt.Errorf("worker cleanup claim returned invalid fence %T", raw[1]) + } + if status == 0 { + return nil, nil + } + return &workerCleanupLease{workerID: workerID, owner: owner, fence: fence}, nil +} + +// updateWorkerHeartbeat atomically proves workerID is still registered and not +// fenced, then records and returns Redis TIME from the authoritative map. +func (node *Node) updateWorkerHeartbeat(ctx context.Context, workerID string) (string, error) { + timestamp, err := updateWorkerHeartbeatScript.Run( + ctx, + node.rdb, + []string{ + fmt.Sprintf("pulse:stream:%s:lifecycle", node.poolStream.Name), + rmapContentKey(node.resources.workerKeepAlive), + rmapContentKey(node.resources.workerCleanup), + rmapContentKey(node.resources.workers), + rmapUpdateChannel(node.resources.workerKeepAlive), + }, + "active", + node.resources.generation, + workerID, + ).Text() + return timestamp, poolBoundaryError(err) +} + +// deactivateWorker atomically marks workerID inactive unless stale-worker +// cleanup already fenced or removed it. It returns true when this caller won +// deactivation and therefore owns requeueing the worker's jobs, along with the +// prior registration value for diagnostics. +func (node *Node) deactivateWorker(ctx context.Context, workerID string) (bool, string, error) { + result, err := deactivateWorkerScript.Run( + ctx, + node.rdb, + []string{ + fmt.Sprintf("pulse:stream:%s:lifecycle", node.poolStream.Name), + rmapContentKey(node.resources.workers), + rmapContentKey(node.resources.workerCleanup), + rmapUpdateChannel(node.resources.workers), + }, + "active", + node.resources.generation, + workerID, + ).Slice() + if err != nil { + return false, "", poolBoundaryError(err) + } + if len(result) != 2 { + return false, "", fmt.Errorf("deactivate worker returned %d values", len(result)) + } + won, ok := result[0].(int64) + if !ok { + return false, "", fmt.Errorf("deactivate worker returned invalid status %T", result[0]) + } + prev, ok := result[1].(string) + if !ok { + return false, "", fmt.Errorf("deactivate worker returned invalid registration %T", result[1]) + } + return won == 1, prev, nil +} + +// renewWorkerCleanup extends the exact stale-worker cleanup lease. +func (node *Node) renewWorkerCleanup(ctx context.Context, lease *workerCleanupLease) error { + return poolBoundaryError(renewWorkerCleanupScript.Run( + ctx, + node.rdb, + node.workerCleanupKeys()[:2], + "active", + node.resources.generation, + lease.workerID, + lease.owner, + lease.fence, + strconv.FormatInt(node.workerTTL.Milliseconds(), 10), + ).Err()) +} + +// publishWorkerRequeue publishes one stable worker/job handoff under the exact +// lease. Status 2 means ownership already moved; status 3 means stale metadata +// has no payload and must be removed by the lease owner. +func (node *Node) publishWorkerRequeue( + ctx context.Context, + lease *workerCleanupLease, + job *Job, +) (int64, error) { + raw, err := publishWorkerRequeueScript.Run( + ctx, + node.rdb, + []string{ + fmt.Sprintf("pulse:stream:%s:lifecycle", node.poolStream.Name), + rmapContentKey(node.resources.workerCleanup), + rmapUpdateChannel(node.resources.workerCleanup), + rmapContentKey(node.resources.jobs), + rmapContentKey(node.resources.jobPayloads), + }, + "active", + node.resources.generation, + lease.workerID, + lease.owner, + lease.fence, + strconv.FormatInt(node.workerTTL.Milliseconds(), 10), + job.Key, + evStartJob, + workerRequeueField(lease.workerID, job.Key), + marshalJob(job), + ).Slice() + if err != nil { + return 0, poolBoundaryError(err) + } + if len(raw) != 2 { + return 0, fmt.Errorf("worker requeue returned %d fields", len(raw)) + } + status, ok := raw[0].(int64) + if !ok || status < 0 || status > 3 { + return 0, fmt.Errorf("worker requeue returned invalid status %T(%v)", raw[0], raw[0]) + } + return status, nil +} + +// removeStaleWorkerJob removes one payload-less worker ownership under the +// exact cleanup lease. A false result means a payload appeared concurrently. +func (node *Node) removeStaleWorkerJob( + ctx context.Context, + lease *workerCleanupLease, + jobKey string, +) (bool, error) { + removed, err := removeStaleWorkerJobScript.Run( + ctx, + node.rdb, + []string{ + fmt.Sprintf("pulse:stream:%s:lifecycle", node.poolStream.Name), + rmapContentKey(node.resources.workerCleanup), + rmapContentKey(node.resources.jobs), + rmapUpdateChannel(node.resources.jobs), + rmapContentKey(node.resources.jobPayloads), + }, + "active", + node.resources.generation, + lease.workerID, + lease.owner, + lease.fence, + jobKey, + ).Int64() + if err != nil { + return false, poolBoundaryError(err) + } + return removed == 1, nil +} + +// deleteStaleWorker destroys the worker stream and removes worker-owned map +// entries under the exact Redis-time cleanup fence. +func (node *Node) deleteStaleWorker( + ctx context.Context, + lease *workerCleanupLease, +) error { + stream, err := node.getWorkerStream(lease.workerID) + if err != nil { + return err + } + if err := stream.Open(ctx); err != nil { + if !errors.Is(err, streaming.ErrStreamNotFound) && + !errors.Is(err, streaming.ErrStreamDestroyed) { + return err + } + } + generation := stream.Generation() + return poolBoundaryError(deleteStaleWorkerScript.Run( + ctx, + node.rdb, + []string{ + fmt.Sprintf("pulse:stream:%s:lifecycle", node.poolStream.Name), + rmapContentKey(node.resources.workerCleanup), + fmt.Sprintf("pulse:stream:%s:lifecycle", stream.Name), + fmt.Sprintf("pulse:stream:%s:generation:%s:resources", stream.Name, generation), + rmapContentKey(node.resources.workers), + rmapUpdateChannel(node.resources.workers), + rmapContentKey(node.resources.workerKeepAlive), + rmapUpdateChannel(node.resources.workerKeepAlive), + rmapContentKey(node.resources.jobs), + rmapUpdateChannel(node.resources.jobs), + }, + "active", + node.resources.generation, + lease.workerID, + lease.owner, + lease.fence, + strconv.FormatInt(node.workerTTL.Milliseconds(), 10), + generation, + ).Err()) +} + +// releaseWorkerCleanup releases the exact lease. Complete cleanup compacts all +// per-worker idempotency metadata. +func (node *Node) releaseWorkerCleanup( + ctx context.Context, + lease *workerCleanupLease, + complete bool, +) error { + completeValue := "0" + if complete { + completeValue = "1" + } + return poolBoundaryError(releaseWorkerCleanupScript.Run( + ctx, + node.rdb, + node.workerCleanupKeys(), + "active", + node.resources.generation, + lease.workerID, + lease.owner, + lease.fence, + completeValue, + workerCleanupFenceField(lease.workerID), + workerRequeuePrefix(lease.workerID), + ).Err()) +} + +// workerCleanupLeaseActive strictly decodes one Redis-owned lease projection. +func workerCleanupLeaseActive(value string, now time.Time) (bool, error) { + parts := strings.Split(value, "|") + if len(parts) != 3 || parts[0] == "" || parts[1] == "" { + return false, fmt.Errorf("invalid worker cleanup lease %q", value) + } + leaseUntil, err := strconv.ParseInt(parts[2], 10, 64) + if err != nil { + return false, fmt.Errorf("invalid worker cleanup lease %q: %w", value, err) + } + return leaseUntil > now.UnixMilli(), nil +} + +// workerCleanupKeys returns lifecycle, content, and rmap update keys. +func (node *Node) workerCleanupKeys() []string { + return []string{ + fmt.Sprintf("pulse:stream:%s:lifecycle", node.poolStream.Name), + rmapContentKey(node.resources.workerCleanup), + rmapUpdateChannel(node.resources.workerCleanup), + } +} + +// workerCleanupFenceField scopes the ABA counter for one worker. +func workerCleanupFenceField(workerID string) string { + return "=cleanup-fence:" + hex.EncodeToString([]byte(workerID)) +} + +// workerRequeuePrefix scopes stable publication records for one stale worker. +func workerRequeuePrefix(workerID string) string { + return "=requeue:" + hex.EncodeToString([]byte(workerID)) + ":" +} + +// workerRequeueField identifies one stable stale-worker/job publication. +func workerRequeueField(workerID, jobKey string) string { + return workerRequeuePrefix(workerID) + hex.EncodeToString([]byte(jobKey)) +} diff --git a/pool/worker_cleanup_test.go b/pool/worker_cleanup_test.go new file mode 100644 index 0000000..4f0b769 --- /dev/null +++ b/pool/worker_cleanup_test.go @@ -0,0 +1,256 @@ +package pool + +import ( + "context" + "fmt" + "strconv" + "testing" + "time" + + "github.com/stretchr/testify/require" + + ptesting "goa.design/pulse/testing" +) + +func TestWorkerCleanupLeaseFencesAndDeduplicatesTakeover(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + node := newTestNode(t, ctx, rdb, t.Name()) + workerID := "stale-worker" + job := &Job{ + Key: "job", + Payload: []byte("payload"), + CreatedAt: time.Unix(1, 0), + NodeID: node.ID, + Requeued: true, + } + require.NoError(t, node.appendPoolMapValue(ctx, node.resources.jobs, workerID, job.Key)) + require.NoError(t, node.setPoolMap(ctx, node.resources.jobPayloads, job.Key, string(job.Payload))) + + first, err := node.acquireWorkerCleanup(ctx, workerID) + require.NoError(t, err) + require.NotNil(t, first) + require.NoError(t, rdb.HSet( + ctx, + rmapContentKey(node.resources.workerCleanup), + workerID, + fmt.Sprintf("%s|%s|0", first.owner, first.fence), + ).Err()) + second, err := node.acquireWorkerCleanup(ctx, workerID) + require.NoError(t, err) + require.NotNil(t, second) + require.NotEqual(t, first.fence, second.fence) + + _, err = node.publishWorkerRequeue(ctx, first, job) + require.ErrorContains(t, err, "WORKERCLEANUPLOST") + status, err := node.publishWorkerRequeue(ctx, second, job) + require.NoError(t, err) + require.EqualValues(t, 1, status) + status, err = node.publishWorkerRequeue(ctx, second, job) + require.NoError(t, err) + require.EqualValues(t, 0, status) + physical := rdb.HGet( + ctx, + fmt.Sprintf("pulse:stream:%s:lifecycle", node.poolStream.Name), + "physical_key", + ).Val() + require.EqualValues(t, 1, rdb.XLen(ctx, physical).Val()) + + require.NoError(t, node.releaseWorkerCleanup(ctx, second, true)) + require.False(t, rdb.HExists( + ctx, + rmapContentKey(node.resources.workerCleanup), + workerRequeueField(workerID, job.Key), + ).Val()) + require.NoError(t, node.Shutdown(context.Background())) +} + +func TestWorkerCleanupAcquisitionLinearizesWithHeartbeat(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + node := newTestNode(t, ctx, rdb, t.Name()) + workerID := "paused-worker" + require.NoError(t, node.setPoolMap( + ctx, + node.resources.workers, + workerID, + strconv.FormatInt(time.Now().UnixNano(), 10), + )) + + _, err := node.updateWorkerHeartbeat(ctx, workerID) + require.NoError(t, err) + lease, err := node.acquireWorkerCleanup(ctx, workerID) + require.NoError(t, err) + require.Nil(t, lease, "a heartbeat committed before acquisition must win") + + redisNow, err := rdb.Time(ctx).Result() + require.NoError(t, err) + require.NoError(t, rdb.HSet( + ctx, + rmapContentKey(node.resources.workerKeepAlive), + workerID, + strconv.FormatInt(redisNow.Add(-2*node.workerTTL).UnixNano(), 10), + ).Err()) + lease, err = node.acquireWorkerCleanup(ctx, workerID) + require.NoError(t, err) + require.NotNil(t, lease) + + _, err = node.updateWorkerHeartbeat(ctx, workerID) + require.ErrorContains(t, err, "WORKERCLEANUPLOST") + require.NoError(t, node.releaseWorkerCleanup(ctx, lease, false)) + require.NoError(t, node.Shutdown(context.Background())) +} + +// runClaimWorkerStart mirrors Worker.claimDispatchedStart for a synthetic +// worker so tests can drive the claim script against injected fence states. +func runClaimWorkerStart( + ctx context.Context, + node *Node, + workerID, key, dispatchID string, + payload, identity []byte, +) (int64, error) { + return claimWorkerStartScript.Run( + ctx, + node.rdb, + []string{ + fmt.Sprintf("pulse:stream:%s:lifecycle", node.poolStream.Name), + rmapContentKey(node.resources.jobPending), + dispatchRecordKey(node.resources.dispatches, dispatchID), + rmapContentKey(node.resources.jobPayloads), + rmapContentKey(node.resources.jobs), + rmapUpdateChannel(node.resources.jobs), + rmapUpdateChannel(node.resources.jobPayloads), + rmapContentKey(node.resources.workers), + rmapContentKey(node.resources.workerCleanup), + }, + node.resources.generation, + "active", + key, + dispatchID, + workerID, + payload, + identity, + ).Int64() +} + +// seedPendingDispatch installs the admission guard and durable record one +// exact dispatch needs before a worker may claim its start. +func seedPendingDispatch( + t *testing.T, + ctx context.Context, + node *Node, + key, dispatchID string, + payload []byte, +) []byte { + t.Helper() + identity, err := dispatchIdentity(key, payload) + require.NoError(t, err) + require.NoError(t, node.setPoolMap(ctx, node.resources.jobPending, key, dispatchID)) + require.NoError(t, node.rdb.HSet( + ctx, + dispatchRecordKey(node.resources.dispatches, dispatchID), + "id", dispatchID, + "identity", identity, + "key", key, + "state", "pending", + ).Err()) + return identity +} + +func TestStaleWorkerCannotClaimAfterCleanupFence(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + node := newTestNode(t, ctx, rdb, t.Name()) + workerID := "claiming-worker" + payload := []byte("payload") + createdAt := strconv.FormatInt(time.Now().UnixNano(), 10) + require.NoError(t, node.setPoolMap(ctx, node.resources.workers, workerID, createdAt)) + + // A live registered worker claims exactly once. + identity := seedPendingDispatch(t, ctx, node, "job-live", "dispatch-live", payload) + claimed, err := runClaimWorkerStart(ctx, node, workerID, "job-live", "dispatch-live", payload, identity) + require.NoError(t, err) + require.EqualValues(t, 1, claimed) + + // A payload that diverged from the admitted identity is rejected: the + // claim carries the identity derived from the bytes it is about to run. + seedPendingDispatch(t, ctx, node, "job-identity", "dispatch-identity", payload) + mutated := []byte("mutated") + mutatedIdentity, err := dispatchIdentity("job-identity", mutated) + require.NoError(t, err) + _, err = runClaimWorkerStart( + ctx, node, workerID, "job-identity", "dispatch-identity", mutated, mutatedIdentity, + ) + require.ErrorContains(t, err, "DISPATCHIDENTITYMISMATCH") + + // An installed cleanup fence rejects the claim atomically. + redisNow, err := rdb.Time(ctx).Result() + require.NoError(t, err) + require.NoError(t, rdb.HSet( + ctx, + rmapContentKey(node.resources.workerCleanup), + workerID, + fmt.Sprintf("owner|1|%d", redisNow.Add(time.Minute).UnixMilli()), + ).Err()) + identity = seedPendingDispatch(t, ctx, node, "job-fenced", "dispatch-fenced", payload) + _, err = runClaimWorkerStart(ctx, node, workerID, "job-fenced", "dispatch-fenced", payload, identity) + require.ErrorContains(t, err, "WORKERCLEANUPLOST") + + // A deregistered worker cannot claim even after the fence is gone. + require.NoError(t, rdb.HDel(ctx, rmapContentKey(node.resources.workerCleanup), workerID).Err()) + require.NoError(t, node.setPoolMap(ctx, node.resources.workers, workerID, "-")) + _, err = runClaimWorkerStart(ctx, node, workerID, "job-fenced", "dispatch-fenced", payload, identity) + require.ErrorContains(t, err, "WORKERCLEANUPLOST") + + require.NoError(t, node.Shutdown(context.Background())) +} + +func TestDeactivateWorkerFencesGracefulRequeue(t *testing.T) { + rdb := ptesting.NewRedisClient(t) + defer ptesting.CleanupRedis(t, rdb, false, "") + ctx := ptesting.NewTestContext(t) + node := newTestNode(t, ctx, rdb, t.Name()) + workerID := "graceful-worker" + createdAt := strconv.FormatInt(time.Now().UnixNano(), 10) + require.NoError(t, node.setPoolMap(ctx, node.resources.workers, workerID, createdAt)) + + won, prev, err := node.deactivateWorker(ctx, workerID) + require.NoError(t, err) + require.True(t, won) + require.Equal(t, createdAt, prev) + + won, prev, err = node.deactivateWorker(ctx, workerID) + require.NoError(t, err) + require.False(t, won, "an inactive worker must not win requeue twice") + require.Equal(t, "-", prev) + + // Cleanup removed the registration: deactivation must not recreate it. + require.NoError(t, rdb.HDel(ctx, rmapContentKey(node.resources.workers), workerID).Err()) + won, prev, err = node.deactivateWorker(ctx, workerID) + require.NoError(t, err) + require.False(t, won) + require.Equal(t, "absent", prev) + require.False(t, rdb.HExists(ctx, rmapContentKey(node.resources.workers), workerID).Val()) + + // An installed cleanup fence cedes requeue to the cleanup owner. + require.NoError(t, node.setPoolMap(ctx, node.resources.workers, workerID, createdAt)) + redisNow, err := rdb.Time(ctx).Result() + require.NoError(t, err) + require.NoError(t, rdb.HSet( + ctx, + rmapContentKey(node.resources.workerCleanup), + workerID, + fmt.Sprintf("owner|1|%d", redisNow.Add(time.Minute).UnixMilli()), + ).Err()) + won, prev, err = node.deactivateWorker(ctx, workerID) + require.NoError(t, err) + require.False(t, won) + require.Equal(t, "cleanup", prev) + require.Equal(t, createdAt, rdb.HGet(ctx, rmapContentKey(node.resources.workers), workerID).Val()) + + require.NoError(t, rdb.HDel(ctx, rmapContentKey(node.resources.workerCleanup), workerID).Err()) + require.NoError(t, node.Shutdown(context.Background())) +} diff --git a/pool/worker_test.go b/pool/worker_test.go index 3c1929e..bea8cd8 100644 --- a/pool/worker_test.go +++ b/pool/worker_test.go @@ -6,25 +6,94 @@ import ( "sort" "strconv" "strings" + "sync/atomic" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "goa.design/pulse/pulse" "goa.design/pulse/rmap" ptesting "goa.design/pulse/testing" ) +func TestAttemptRequeueAccountsForEveryConcurrentHandoff(t *testing.T) { + worker := &Worker{ + requeueTimeout: 30 * time.Millisecond, + logger: pulse.NoopLogger(), + } + jobs := map[string]*Job{ + "success": {Key: "success"}, + "late": {Key: "late"}, + "failed": {Key: "failed"}, + "blocked": {Key: "blocked"}, + } + started := make(chan string, len(jobs)) + release := make(chan struct{}) + var running atomic.Int64 + send := func(ctx context.Context, job *Job) error { + running.Add(1) + defer running.Add(-1) + started <- job.Key + switch job.Key { + case "success": + <-release + return nil + case "late": + <-release + time.Sleep(5 * time.Millisecond) + return nil + case "failed": + <-release + return errors.New("injected handoff failure") + case "blocked": + <-ctx.Done() + return ctx.Err() + default: + panic("unexpected job") + } + } + result := make(chan map[string]*Job, 1) + go func() { + result <- worker.attemptRequeueWith(context.Background(), jobs, send) + }() + for range jobs { + select { + case <-started: + case <-time.After(time.Second): + require.Fail(t, "handoffs did not start concurrently") + } + } + close(release) + remaining := <-result + require.Equal(t, map[string]*Job{ + "failed": jobs["failed"], + "blocked": jobs["blocked"], + }, remaining) + require.Zero(t, running.Load(), "attempt returned before all handoff goroutines joined") +} + func TestWorkerRequeueJobs(t *testing.T) { var ( ctx = ptesting.NewTestContext(t) testName = strings.Replace(t.Name(), "/", "_", -1) rdb = ptesting.NewRedisClient(t) - node = newFastCleanupTestNode(t, ctx, rdb, testName) ) defer ptesting.CleanupRedis(t, rdb, false, testName) - ctx, cancel := context.WithTimeout(ctx, time.Second) + node, err := AddNode( + ctx, + testName, + rdb, + WithLogger(pulse.NoopLogger()), + WithRequeueTimeout(testFastRequeueTimeout), + WithJobSinkBlockDuration(testJobSinkBlockDuration), + WithWorkerTTL(testFastWorkerTTL), + WithDispatchTimeout(16*time.Second), + WithRecoveryGrace(8*time.Second), + ) + require.NoError(t, err) + ctx, cancel := context.WithTimeout(ctx, 30*time.Second) defer cancel() // Create a worker and dispatch a job @@ -36,7 +105,7 @@ func TestWorkerRequeueJobs(t *testing.T) { // Emulate the worker failing by preventing it from refreshing its keepalive // This means we can't cleanup cleanly, hence "false" in CleanupRedis - worker.stop(ctx) + require.NoError(t, worker.stop(ctx)) // Create a new worker to pick up the requeued job newWorker := newTestWorker(t, ctx, node) @@ -51,7 +120,7 @@ func TestWorkerRequeueJobs(t *testing.T) { // Increase 'max' to cover the time until requeue happens require.Eventually(t, func() bool { return len(newWorker.Jobs()) == 2 - }, time.Second, delay, "job was not requeued") + }, 10*time.Second, delay, "job was not requeued") // Cleanup assert.NoError(t, node.Shutdown(ctx)) @@ -95,6 +164,20 @@ func TestWorkerRebalanceReleasesPreviousJobOwner(t *testing.T) { assert.NoError(t, node.Shutdown(ctx)) } +func TestWorkerJobsPreservesRequeuedState(t *testing.T) { + worker := &Worker{ID: "worker", CreatedAt: time.Unix(1, 0)} + worker.jobs.Store("job", &Job{ + Key: "job", + Payload: []byte("payload"), + CreatedAt: time.Unix(2, 0), + Requeued: true, + }) + + jobs := worker.Jobs() + require.Len(t, jobs, 1) + assert.True(t, jobs[0].Requeued) +} + func TestWorkerStartFailurePayloadOwnership(t *testing.T) { var ( ctx = ptesting.NewTestContext(t) @@ -130,13 +213,88 @@ func TestWorkerStartFailurePayloadOwnership(t *testing.T) { }) assert.ErrorIs(t, err, errStart) assert.Empty(t, snapshotJobOwners(t, ctx, node, "requeued-job")) - gotPayload, ok := snapshotValue(t, ctx, node, node.jobPayloadMap, "requeued-job") - require.True(t, ok) - assert.Equal(t, "requeued payload", gotPayload) + _, ok = snapshotValue(t, ctx, node, node.jobPayloadMap, "requeued-job") + assert.False(t, ok) assert.NoError(t, node.Shutdown(ctx)) } +func TestWorkerStartFailureRetriesDurableCleanup(t *testing.T) { + ctx := ptesting.NewTestContext(t) + testName := strings.Replace(t.Name(), "/", "_", -1) + rdb := ptesting.NewRedisClient(t) + failure := errors.New("injected start cleanup failure") + hook := &poolRedisHook{ + failure: failure, + startCleanupSHA: cleanupFailedStartScript.Hash(), + } + rdb.AddHook(hook) + node := newTestNode(t, ctx, rdb, testName) + defer ptesting.CleanupRedis(t, rdb, true, testName) + worker := newTestWorker(t, ctx, node) + startFailure := errors.New("start failed") + worker.handler.(*mockHandler).startFunc = func(job *Job) error { + return startFailure + } + job := &Job{ + Key: "job", + Payload: []byte("payload"), + CreatedAt: time.Now(), + NodeID: node.ID, + } + + hook.failStartCleanup.Store(true) + err := worker.startJob(ctx, job) + require.ErrorIs(t, err, ErrRequeue) + require.ErrorIs(t, err, failure) + require.Equal(t, []string{worker.ID}, snapshotJobOwners(t, ctx, node, job.Key)) + payload, ok := snapshotValue(t, ctx, node, node.jobPayloadMap, job.Key) + require.True(t, ok) + require.Equal(t, "payload", payload) + + hook.failStartCleanup.Store(false) + err = worker.startJob(ctx, job) + require.ErrorIs(t, err, startFailure) + require.Empty(t, snapshotJobOwners(t, ctx, node, job.Key)) + _, ok = snapshotValue(t, ctx, node, node.jobPayloadMap, job.Key) + require.False(t, ok) + + require.NoError(t, node.Shutdown(ctx)) +} + +func TestWorkerLoopDropsMalformedEventAndContinues(t *testing.T) { + ctx := ptesting.NewTestContext(t) + testName := strings.Replace(t.Name(), "/", "_", -1) + rdb := ptesting.NewRedisClient(t) + node := newTestNode(t, ctx, rdb, testName) + defer ptesting.CleanupRedis(t, rdb, true, testName) + worker := newTestWorker(t, ctx, node) + started := make(chan string, 1) + worker.handler.(*mockHandler).startFunc = func(job *Job) error { + started <- job.Key + return nil + } + + malformedID, err := worker.stream.Add(ctx, evStartJob, []byte{1, 2, 3}) + require.NoError(t, err) + job := &Job{ + Key: "valid", + Payload: []byte("payload"), + CreatedAt: time.Now(), + NodeID: node.ID, + } + _, err = worker.stream.Add(ctx, evStartJob, marshalEnvelope(node.ID, marshalJob(job))) + require.NoError(t, err) + require.Equal(t, job.Key, <-started) + streamKey := generationStreamKey(ctx, rdb, worker.stream.Name) + require.Eventually(t, func() bool { + events, rangeErr := rdb.XRange(ctx, streamKey, malformedID, malformedID).Result() + return rangeErr == nil && len(events) == 0 + }, max, delay) + + require.NoError(t, node.Shutdown(ctx)) +} + func TestWorkerControlEventsRequireLocalOwnership(t *testing.T) { var ( ctx = ptesting.NewTestContext(t) @@ -184,7 +342,7 @@ func TestStaleWorkerCleanupInNode(t *testing.T) { staleWorkers := make([]*Worker, 5) for i := 0; i < 5; i++ { staleWorkers[i] = newTestWorker(t, ctx, node) - staleWorkers[i].stop(ctx) + require.NoError(t, staleWorkers[i].stop(ctx)) // Set the last seen time to a past time _, err := node.workerKeepAliveMap.Set(ctx, staleWorkers[i].ID, strconv.FormatInt(time.Now().Add(-2*node.workerTTL).UnixNano(), 10)) assert.NoError(t, err) diff --git a/scripts/run-examples b/scripts/run-examples index 371bfba..34f05c0 100755 --- a/scripts/run-examples +++ b/scripts/run-examples @@ -10,7 +10,7 @@ fi # Load environment variables from .env file in the Git root if [ -f "$git_root/.env" ]; then - export $(grep -v '^#' "$git_root/.env" | xargs) + source "$git_root/.env" fi # Update file paths to use absolute paths from Git root @@ -36,15 +36,15 @@ args=( "--write" ) -# Pool worker can take up to 15 seconds to shutdown -timeout_sec=20 +# Worker and pool readers use independent finite blocking reads during shutdown. +timeout_sec=40 run_example() { local file=$1 shift local args=("$@") echo "Running: $file ${args[*]}" - output=$(cd "$git_root" && timeout "$timeout_sec" go run -v "$file" "${args[@]}" 2>&1) + output=$(cd "$git_root" && TIMEOUT=5s timeout "$timeout_sec" go run -v "$file" "${args[@]}" 2>&1) local exit_status=$? if [[ $exit_status -ne 0 ]]; then echo "Example '${file}' exited with an error or exceeded the timeout:" From 356c0bc96b849d7f0f9622a1dc10e91ecfdc7a79 Mon Sep 17 00:00:00 2001 From: "Raphael (manual office deploy after cloud-state fix)" Date: Fri, 24 Jul 2026 22:05:26 -0700 Subject: [PATCH 2/3] pool: unify graceful requeue with stale-worker takeover on one lease Graceful shutdown now self-acquires the worker requeue lease - the same Redis-time capability stale-worker cleanup uses - atomically deactivating the worker's registration, and republishes jobs through the shared lease-fenced stable publication records. Exactly one party requeues each job, a duplicate start event can no longer be published by concurrent graceful and takeover requeuers, and an interrupted graceful requeue is finished by takeover after lease expiry. The requeue deadline is WithRequeueTimeout, matching its v1 WithWorkerShutdownTTL meaning. A dispatch payload identity mismatch is now classified as a terminal invariant violation: the event is acknowledged with the error instead of being redelivered forever. --- pool/README.md | 15 +-- pool/node.go | 61 +++++++++--- pool/worker.go | 178 ++++++++++++------------------------ pool/worker_cleanup.go | 141 +++++++++++++--------------- pool/worker_cleanup_test.go | 69 +++++++++----- pool/worker_test.go | 57 ------------ 6 files changed, 227 insertions(+), 294 deletions(-) diff --git a/pool/README.md b/pool/README.md index b03520b..9dc02b4 100644 --- a/pool/README.md +++ b/pool/README.md @@ -279,12 +279,15 @@ operation reads the authoritative heartbeat, compares it with immutable `WorkerTTL` using Redis time, and installs an exact owner/fence. Heartbeat scripts reject that fence, and every mutation a resumed stale process could attempt re-verifies liveness at its own Redis linearization point: a worker -start claim re-checks the worker's registration and cleanup fence, graceful -requeue deactivation refuses under a fence and never recreates a removed -registration, and job dispatch plus every scheduler transition and ownership -script re-check the node's keep-alive registration and node-cleanup field. -Requeue, dispatch-release, stream destruction, and discovery removal verify -the same unexpired owner token. +start claim re-checks the worker's registration and cleanup fence, and job +dispatch plus every scheduler transition and ownership script re-check the +node's keep-alive registration and node-cleanup field. Graceful shutdown and +stale-worker takeover share one requeue lease: a stopping worker self-acquires +it (atomically deactivating its registration) and republishes through the same +lease-fenced stable publication records the cleanup owner uses, so exactly one +party requeues each job and an interrupted graceful requeue is finished by +takeover after lease expiry. Requeue, dispatch-release, stream destruction, +and discovery removal verify the same unexpired owner token. The pool stream generation selects every shared map and stream as one immutable resource manifest. The first deployment seen by this version adopts existing diff --git a/pool/node.go b/pool/node.go index a31daa1..5ad41e3 100644 --- a/pool/node.go +++ b/pool/node.go @@ -2283,17 +2283,41 @@ func (node *Node) cleanupWorker(ctx context.Context, workerID string) { node.logger.Error(fmt.Errorf("cleanupWorker: release lease: %w", err), "worker", workerID) } }() + complete = node.requeueWorkerJobs(ctx, lease, nil) +} + +// requeueWorkerJobs republishes every job owned by the leased worker through +// the lease-fenced stable publication records and deletes the worker once all +// jobs are processed. onProcessed, when non-nil, observes each job key that +// left the worker's ownership so a gracefully stopping worker can stop its +// local handler; an onProcessed error leaves the job for the next attempt. +// It returns true when the worker was completely requeued and deleted. +func (node *Node) requeueWorkerJobs( + ctx context.Context, + lease *workerCleanupLease, + onProcessed func(key string) error, +) bool { + workerID := lease.workerID + processKey := func(key string) bool { + if onProcessed == nil { + return true + } + if err := onProcessed(key); err != nil { + node.logger.Error(fmt.Errorf("requeueWorkerJobs: local stop failed: %w", err), "job", key, "worker", workerID) + return false + } + return true + } // Get the worker's jobs keys, ok := node.jobMap.GetValues(workerID) if !ok || len(keys) == 0 { if err := node.deleteStaleWorker(ctx, lease); err != nil { - node.logger.Error(fmt.Errorf("cleanupWorkerJobs: failed to delete worker: %w", err), "worker", workerID) - return + node.logger.Error(fmt.Errorf("requeueWorkerJobs: failed to delete worker: %w", err), "worker", workerID) + return false } - complete = true node.logger.Info("cleaned up worker with no jobs", "worker", workerID) - return + return true } // Requeue jobs and process them @@ -2323,6 +2347,9 @@ func (node *Node) cleanupWorker(ctx context.Context, workerID string) { "dispatch", dispatchID, ) + if !processKey(key) { + continue + } processed++ continue } @@ -2331,13 +2358,16 @@ func (node *Node) cleanupWorker(ctx context.Context, workerID string) { if !ok { removed, err := node.removeStaleWorkerJob(ctx, lease, key) if err != nil { - node.logger.Error(fmt.Errorf("cleanupWorker: failed to remove stale job from jobs map: %w", err), "job", key, "worker", workerID) + node.logger.Error(fmt.Errorf("requeueWorkerJobs: failed to remove stale job from jobs map: %w", err), "job", key, "worker", workerID) continue } if !removed { continue } - node.logger.Info("cleanupWorker: removed stale job key with missing payload", "job", key, "worker", workerID) + node.logger.Info("requeueWorkerJobs: removed stale job key with missing payload", "job", key, "worker", workerID) + if !processKey(key) { + continue + } processed++ continue } @@ -2351,36 +2381,45 @@ func (node *Node) cleanupWorker(ctx context.Context, workerID string) { continue } if status == 2 { + if !processKey(key) { + continue + } processed++ continue } if status == 3 { removed, err := node.removeStaleWorkerJob(ctx, lease, key) if err != nil { - node.logger.Error(fmt.Errorf("cleanupWorker: failed to remove stale job from jobs map: %w", err), "job", key, "worker", workerID) + node.logger.Error(fmt.Errorf("requeueWorkerJobs: failed to remove stale job from jobs map: %w", err), "job", key, "worker", workerID) continue } if !removed { continue } + if !processKey(key) { + continue + } processed++ continue } + if !processKey(key) { + continue + } requeued++ processed++ } if len(keys) != processed { node.logger.Info("partially processed stale worker jobs", "requeued", requeued, "processed", processed, "jobs", len(keys), "worker", workerID) - return + return false } // Delete worker node.logger.Info("cleaned up worker", "worker", workerID, "requeued", requeued) if err := node.deleteStaleWorker(ctx, lease); err != nil { - node.logger.Error(fmt.Errorf("cleanupWorkerJobs: failed to delete worker: %w", err), "worker", workerID) - return + node.logger.Error(fmt.Errorf("requeueWorkerJobs: failed to delete worker: %w", err), "worker", workerID) + return false } - complete = true + return true } // isWithinTTL checks if a timestamp is within a TTL. If lastSeen is not a valid diff --git a/pool/worker.go b/pool/worker.go index a900b4e..f37d07e 100644 --- a/pool/worker.go +++ b/pool/worker.go @@ -72,12 +72,6 @@ type ( dispatchID string } - // requeueResult reports one concurrent handoff attempt. - requeueResult struct { - key string - err error - } - // JobHandler starts and stops jobs. JobHandler interface { // Start starts a job. @@ -111,7 +105,15 @@ type ( } ) -var errJobNotOwned = errors.New("job not owned by worker") +var ( + errJobNotOwned = errors.New("job not owned by worker") + + // errDispatchIdentityMismatch reports that a start event's payload + // diverged from the admitted dispatch record. The identity is derived + // from immutable event bytes, so this is an invariant violation that no + // retry can repair: the event must be acknowledged with a terminal error. + errDispatchIdentityMismatch = errors.New("dispatch payload identity mismatch") +) // newWorker creates a new worker. func newWorker(ctx context.Context, node *Node, h JobHandler) (*Worker, error) { @@ -391,6 +393,11 @@ func (w *Worker) startJob(ctx context.Context, job *Job) error { if job.dispatchID != "" { claimed, err := w.claimDispatchedStart(ctx, job) if err != nil { + if errors.Is(err, errDispatchIdentityMismatch) { + // Invariant violation: fail the event terminally instead of + // redelivering bytes that can never match their admission. + return err + } return errors.Join(ErrRequeue, err) } if !claimed { @@ -456,6 +463,9 @@ func (w *Worker) claimDispatchedStart(ctx context.Context, job *Job) (bool, erro identity, ).Int64() if err != nil { + if redis.HasErrorPrefix(err, "DISPATCHIDENTITYMISMATCH") { + return false, fmt.Errorf("%w: job %q dispatch %q", errDispatchIdentityMismatch, job.Key, job.dispatchID) + } return false, fmt.Errorf("claim dispatched start for job %q: %w", job.Key, err) } return result == 1, nil @@ -698,19 +708,20 @@ func (w *Worker) rebalance(ctx context.Context, activeWorkers []string) { } } -// requeueJobs requeues the jobs handled by the worker. -// This should be done after the worker is stopped. +// requeueJobs requeues the jobs handled by the worker during graceful +// shutdown. It self-acquires the worker requeue lease — the same capability +// stale-worker cleanup uses — so exactly one party republishes the jobs, and +// every publication flows through the lease-fenced stable dedup records. When +// this worker loses the lease, the winning cleanup owner owns the requeue. func (w *Worker) requeueJobs(ctx context.Context) error { - jobsToRequeue := make(map[string]*Job) var unsettled []string jobCount := 0 - w.jobs.Range(func(key, value any) bool { + w.jobs.Range(func(_, value any) bool { job := value.(*Job) if job.dispatchID != "" { unsettled = append(unsettled, job.dispatchID) return true } - jobsToRequeue[key.(string)] = job jobCount++ return true }) @@ -724,124 +735,53 @@ func (w *Worker) requeueJobs(ctx context.Context) error { } w.logger.Debug("requeueJobs: requeuing", "jobs", jobCount) - // Mark the worker inactive behind the cleanup fence so requeued jobs are - // not assigned to this worker and exactly one party requeues: losing - // deactivation means stale-worker cleanup owns (or already completed) the - // requeue, so this worker must not publish duplicates. - won, prev, err := w.node.deactivateWorker(ctx, w.ID) + lease, err := w.node.acquireGracefulRequeue(ctx, w.ID) if err != nil { - return fmt.Errorf("requeueJobs: failed to mark worker as inactive: %w", err) + return fmt.Errorf("requeueJobs: failed to acquire requeue lease: %w", err) } - if !won { - w.logger.Debug("requeueJobs: requeue owned elsewhere, skipping", "registration", prev) + if lease == nil { + w.logger.Debug("requeueJobs: requeue owned elsewhere, skipping") return nil } - if createdAt := strconv.FormatInt(w.CreatedAt.UnixNano(), 10); prev != createdAt { - w.logger.Error(fmt.Errorf("requeueJobs: unexpected worker registration"), "worker", w.ID, "expected", createdAt, "got", prev) - } - - retryUntil := time.Now().Add(w.workerTTL) - for retryUntil.After(time.Now()) { - remainingJobs := w.attemptRequeue(ctx, jobsToRequeue) - jobsToRequeue = remainingJobs - if len(remainingJobs) == 0 { - break + complete := false + defer func() { + releaseCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), time.Second) + defer cancel() + if err := w.node.releaseWorkerCleanup(releaseCtx, lease, complete); err != nil { + w.logger.Error(fmt.Errorf("requeueJobs: release lease: %w", err)) } - } - - failedCount := len(jobsToRequeue) - w.logger.Info("requeued", "jobs", jobCount, "failed", failedCount) - if failedCount > 0 { - return fmt.Errorf("requeueJobs: failed to requeue %d/%d jobs after retrying for %v", failedCount, jobCount, w.workerTTL) - } - - return nil -} - -// attemptRequeue attempts to requeue the jobs in the given map. -// It returns any job that failed to be requeued. -func (w *Worker) attemptRequeue(ctx context.Context, jobsToRequeue map[string]*Job) map[string]*Job { - return w.attemptRequeueWith(ctx, jobsToRequeue, w.requeueJob) -} - -// attemptRequeueWith runs every handoff concurrently under one timeout, joins -// all senders, and removes only jobs whose send completed successfully. -func (w *Worker) attemptRequeueWith( - ctx context.Context, - jobsToRequeue map[string]*Job, - send func(context.Context, *Job) error, -) map[string]*Job { - var wg sync.WaitGroup - resultChan := make(chan requeueResult, len(jobsToRequeue)) - remainingJobs := make(map[string]*Job, len(jobsToRequeue)) - for key, job := range jobsToRequeue { - remainingJobs[key] = job - } - - attemptCtx, cancel := context.WithTimeout(ctx, w.requeueTimeout) - defer cancel() - wg.Add(len(jobsToRequeue)) - for key, job := range jobsToRequeue { - pulse.Go(w.logger, func() { - defer wg.Done() - err := send(attemptCtx, job) - if err != nil { - w.logger.Error(fmt.Errorf("failed to requeue job: %w", err), "job", key) - } else { - w.logger.Debug("requeueJobs: requeued", "job", key) - } - resultChan <- requeueResult{key: key, err: err} - }) - } + }() - timedOut := false - for processed := 0; processed < len(jobsToRequeue); processed++ { - select { - case res := <-resultChan: - if res.err != nil { - w.logger.Error(fmt.Errorf("requeueJobs: failed to requeue job %q: %w", res.key, res.err)) - continue - } - delete(remainingJobs, res.key) - w.logger.Info("requeued", "job", res.key) - case <-attemptCtx.Done(): - timedOut = true - cancel() - processed = len(jobsToRequeue) + stopLocal := func(key string) error { + if _, ok := w.jobs.Load(key); !ok { + return nil } - } - wg.Wait() - close(resultChan) - for res := range resultChan { - if res.err == nil { - delete(remainingJobs, res.key) - w.logger.Info("requeued", "job", res.key) + if err := w.handler.Stop(key); err != nil { + return fmt.Errorf("requeueJobs: failed to stop job %q: %w", key, err) } + w.jobs.Delete(key) + w.logger.Debug("requeueJobs: stopped", "job", key) + return nil } - if timedOut { - w.logger.Error(fmt.Errorf("requeueJobs: timeout reached with %d jobs not handed off", len(remainingJobs))) - } - return remainingJobs -} - -// requeueJob requeues a job. -func (w *Worker) requeueJob(ctx context.Context, job *Job) error { - job.Requeued = true - job.dispatchID = "" - _, err := w.node.poolStream.Add(ctx, evStartJob, marshalJob(job)) - if err != nil { - return fmt.Errorf("requeueJob: failed to add job to pool stream: %w", err) - } - - // Stop locally, but do not touch the replicated job/payload maps: we want the - // payload to remain available for distributed recovery until the job is - // confirmed running elsewhere. - if _, ok := w.jobs.Load(job.Key); ok { - if err := w.handler.Stop(job.Key); err != nil { - return fmt.Errorf("requeueJob: failed to stop job %q: %w", job.Key, err) + retryUntil := time.Now().Add(w.requeueTimeout) + for { + complete = w.node.requeueWorkerJobs(ctx, lease, stopLocal) + if complete || !retryUntil.After(time.Now()) { + break } - w.jobs.Delete(job.Key) + if err := w.node.renewWorkerCleanup(ctx, lease); err != nil { + return fmt.Errorf("requeueJobs: requeue lease lost: %w", err) + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(100 * time.Millisecond): + } + } + if !complete { + return fmt.Errorf("requeueJobs: failed to requeue %d jobs after retrying for %v", jobCount, w.requeueTimeout) } + w.logger.Info("requeued", "jobs", jobCount) return nil } diff --git a/pool/worker_cleanup.go b/pool/worker_cleanup.go index eeb555a..ea5de1f 100644 --- a/pool/worker_cleanup.go +++ b/pool/worker_cleanup.go @@ -58,41 +58,12 @@ redis.call("PUBLISH", KEYS[5], "set:" .. message) return timestamp `) - // deactivateWorkerScript marks one registered worker inactive for graceful - // requeue. It refuses while a cleanup fence is installed and never - // recreates a registration that stale-worker cleanup already removed, so - // exactly one party — the worker or the cleanup owner — requeues its jobs. - deactivateWorkerScript = redis.NewScript(` -if redis.call("HGET", KEYS[1], "state") ~= ARGV[1] -or redis.call("HGET", KEYS[1], "generation") ~= ARGV[2] then - return redis.error_reply("POOLGENERATIONLOST") -end -if redis.call("HGET", KEYS[3], ARGV[3]) then - return {0, "cleanup"} -end -local registration = redis.call("HGET", KEYS[2], ARGV[3]) -if not registration then - return {0, "absent"} -end -if registration == "-" then - return {0, "-"} -end -local inactive = "-" -redis.call("HSET", KEYS[2], ARGV[3], inactive) -local rev = tostring(redis.call("HINCRBY", KEYS[2], "=rev", 1)) -redis.call("HSET", KEYS[2], "=kind", "set") -local message = struct.pack( - "ic0ic0ic0", - string.len(ARGV[3]), ARGV[3], - string.len(inactive), inactive, - string.len(rev), rev -) -redis.call("PUBLISH", KEYS[4], "set:" .. message) -return {1, registration} -`) - - // acquireWorkerCleanupScript acquires, renews, or steals an expired cleanup - // lease only after atomically proving the authoritative heartbeat expired. + // acquireWorkerCleanupScript acquires, renews, or steals an expired + // requeue lease. Foreign acquisition must atomically prove the + // authoritative heartbeat expired; graceful self-acquisition (ARGV[8]) + // instead requires the worker's own live registration and atomically + // marks it inactive, so the lease is the single fence deciding which + // party — the worker or a cleanup owner — requeues the jobs. acquireWorkerCleanupScript = redis.NewScript(` if redis.call("HGET", KEYS[1], "state") ~= ARGV[1] or redis.call("HGET", KEYS[1], "generation") ~= ARGV[2] then @@ -101,14 +72,16 @@ end local clock = redis.call("TIME") local now = tonumber(clock[1]) * 1000 + math.floor(tonumber(clock[2]) / 1000) local now_ns = tonumber(clock[1]) * 1000000000 + tonumber(clock[2]) * 1000 -local heartbeat = redis.call("HGET", KEYS[2], ARGV[3]) -if heartbeat then - local heartbeat_ns = tonumber(heartbeat) - if not heartbeat_ns then - return redis.error_reply("WORKERHEARTBEATINVALID") - end - if heartbeat_ns + tonumber(ARGV[7]) >= now_ns then - return {0, "live"} +if ARGV[8] ~= "1" then + local heartbeat = redis.call("HGET", KEYS[2], ARGV[3]) + if heartbeat then + local heartbeat_ns = tonumber(heartbeat) + if not heartbeat_ns then + return redis.error_reply("WORKERHEARTBEATINVALID") + end + if heartbeat_ns + tonumber(ARGV[7]) >= now_ns then + return {0, "live"} + end end end local current = redis.call("HGET", KEYS[3], ARGV[3]) @@ -127,6 +100,23 @@ if current then return {1, current_fence} end end +if ARGV[8] == "1" then + local registration = redis.call("HGET", KEYS[5], ARGV[3]) + if not registration or registration == "-" then + return {0, "inactive"} + end + local inactive = "-" + redis.call("HSET", KEYS[5], ARGV[3], inactive) + local registration_rev = tostring(redis.call("HINCRBY", KEYS[5], "=rev", 1)) + redis.call("HSET", KEYS[5], "=kind", "set") + local registration_message = struct.pack( + "ic0ic0ic0", + string.len(ARGV[3]), ARGV[3], + string.len(inactive), inactive, + string.len(registration_rev), registration_rev + ) + redis.call("PUBLISH", KEYS[6], "set:" .. registration_message) +end local fence_field = ARGV[6] local fence = tostring(redis.call("HINCRBY", KEYS[3], fence_field, 1)) local value = ARGV[4] .. "|" .. fence .. "|" .. tostring(now + tonumber(ARGV[5])) @@ -365,12 +355,37 @@ return 1 `) ) -// acquireWorkerCleanup returns an exact lease or nil while another owner is live. +// acquireWorkerCleanup returns an exact lease or nil while another owner is +// live. It proves the worker heartbeat expired before fencing it. func (node *Node) acquireWorkerCleanup( ctx context.Context, workerID string, +) (*workerCleanupLease, error) { + return node.runWorkerCleanupAcquire(ctx, workerID, false) +} + +// acquireGracefulRequeue returns the requeue lease for this worker's own +// graceful shutdown, atomically marking its registration inactive, or nil +// when another owner already holds (or completed) the requeue. +func (node *Node) acquireGracefulRequeue( + ctx context.Context, + workerID string, +) (*workerCleanupLease, error) { + return node.runWorkerCleanupAcquire(ctx, workerID, true) +} + +// runWorkerCleanupAcquire runs the shared lease acquisition; graceful skips +// the heartbeat-expiry proof and deactivates the worker's registration. +func (node *Node) runWorkerCleanupAcquire( + ctx context.Context, + workerID string, + graceful bool, ) (*workerCleanupLease, error) { owner := node.ID + "-" + ulid.Make().String() + self := "" + if graceful { + self = "1" + } raw, err := acquireWorkerCleanupScript.Run( ctx, node.rdb, @@ -379,6 +394,8 @@ func (node *Node) acquireWorkerCleanup( rmapContentKey(node.resources.workerKeepAlive), rmapContentKey(node.resources.workerCleanup), rmapUpdateChannel(node.resources.workerCleanup), + rmapContentKey(node.resources.workers), + rmapUpdateChannel(node.resources.workers), }, "active", node.resources.generation, @@ -387,6 +404,7 @@ func (node *Node) acquireWorkerCleanup( strconv.FormatInt(node.workerTTL.Milliseconds(), 10), workerCleanupFenceField(workerID), strconv.FormatInt(node.resources.workerTTL.Nanoseconds(), 10), + self, ).Slice() if err != nil { return nil, poolBoundaryError(err) @@ -428,41 +446,6 @@ func (node *Node) updateWorkerHeartbeat(ctx context.Context, workerID string) (s return timestamp, poolBoundaryError(err) } -// deactivateWorker atomically marks workerID inactive unless stale-worker -// cleanup already fenced or removed it. It returns true when this caller won -// deactivation and therefore owns requeueing the worker's jobs, along with the -// prior registration value for diagnostics. -func (node *Node) deactivateWorker(ctx context.Context, workerID string) (bool, string, error) { - result, err := deactivateWorkerScript.Run( - ctx, - node.rdb, - []string{ - fmt.Sprintf("pulse:stream:%s:lifecycle", node.poolStream.Name), - rmapContentKey(node.resources.workers), - rmapContentKey(node.resources.workerCleanup), - rmapUpdateChannel(node.resources.workers), - }, - "active", - node.resources.generation, - workerID, - ).Slice() - if err != nil { - return false, "", poolBoundaryError(err) - } - if len(result) != 2 { - return false, "", fmt.Errorf("deactivate worker returned %d values", len(result)) - } - won, ok := result[0].(int64) - if !ok { - return false, "", fmt.Errorf("deactivate worker returned invalid status %T", result[0]) - } - prev, ok := result[1].(string) - if !ok { - return false, "", fmt.Errorf("deactivate worker returned invalid registration %T", result[1]) - } - return won == 1, prev, nil -} - // renewWorkerCleanup extends the exact stale-worker cleanup lease. func (node *Node) renewWorkerCleanup(ctx context.Context, lease *workerCleanupLease) error { return poolBoundaryError(renewWorkerCleanupScript.Run( diff --git a/pool/worker_cleanup_test.go b/pool/worker_cleanup_test.go index 4f0b769..e0c2abd 100644 --- a/pool/worker_cleanup_test.go +++ b/pool/worker_cleanup_test.go @@ -9,6 +9,8 @@ import ( "github.com/stretchr/testify/require" + "goa.design/pulse/pulse" + ptesting "goa.design/pulse/testing" ) @@ -186,6 +188,16 @@ func TestStaleWorkerCannotClaimAfterCleanupFence(t *testing.T) { ) require.ErrorContains(t, err, "DISPATCHIDENTITYMISMATCH") + // The worker classifies the mismatch as a terminal invariant violation, + // not a requeue: retrying immutable bytes can never repair it. + mismatchWorker := &Worker{ID: workerID, node: node, logger: pulse.NoopLogger()} + _, err = mismatchWorker.claimDispatchedStart(ctx, &Job{ + Key: "job-identity", + Payload: mutated, + dispatchID: "dispatch-identity", + }) + require.ErrorIs(t, err, errDispatchIdentityMismatch) + // An installed cleanup fence rejects the claim atomically. redisNow, err := rdb.Time(ctx).Result() require.NoError(t, err) @@ -208,7 +220,7 @@ func TestStaleWorkerCannotClaimAfterCleanupFence(t *testing.T) { require.NoError(t, node.Shutdown(context.Background())) } -func TestDeactivateWorkerFencesGracefulRequeue(t *testing.T) { +func TestGracefulRequeueLeaseArbitratesWithCleanup(t *testing.T) { rdb := ptesting.NewRedisClient(t) defer ptesting.CleanupRedis(t, rdb, false, "") ctx := ptesting.NewTestContext(t) @@ -217,40 +229,53 @@ func TestDeactivateWorkerFencesGracefulRequeue(t *testing.T) { createdAt := strconv.FormatInt(time.Now().UnixNano(), 10) require.NoError(t, node.setPoolMap(ctx, node.resources.workers, workerID, createdAt)) - won, prev, err := node.deactivateWorker(ctx, workerID) + // Graceful self-acquisition wins the lease despite a live heartbeat and + // atomically deactivates the registration. + _, err := node.updateWorkerHeartbeat(ctx, workerID) require.NoError(t, err) - require.True(t, won) - require.Equal(t, createdAt, prev) - - won, prev, err = node.deactivateWorker(ctx, workerID) + lease, err := node.acquireGracefulRequeue(ctx, workerID) require.NoError(t, err) - require.False(t, won, "an inactive worker must not win requeue twice") - require.Equal(t, "-", prev) + require.NotNil(t, lease) + require.Equal(t, "-", rdb.HGet(ctx, rmapContentKey(node.resources.workers), workerID).Val()) - // Cleanup removed the registration: deactivation must not recreate it. - require.NoError(t, rdb.HDel(ctx, rmapContentKey(node.resources.workers), workerID).Err()) - won, prev, err = node.deactivateWorker(ctx, workerID) + // While the graceful lease is live, neither a foreign cleanup owner nor a + // second graceful attempt can win. + foreign, err := node.acquireWorkerCleanup(ctx, workerID) require.NoError(t, err) - require.False(t, won) - require.Equal(t, "absent", prev) - require.False(t, rdb.HExists(ctx, rmapContentKey(node.resources.workers), workerID).Val()) + require.Nil(t, foreign, "foreign cleanup must not steal a live graceful lease") + second, err := node.acquireGracefulRequeue(ctx, workerID) + require.NoError(t, err) + require.Nil(t, second, "a second graceful attempt must not win a live lease") - // An installed cleanup fence cedes requeue to the cleanup owner. - require.NoError(t, node.setPoolMap(ctx, node.resources.workers, workerID, createdAt)) + // After the graceful owner disappears (lease expired, heartbeat stale), + // foreign cleanup recovers the half-requeued worker: the deactivated + // registration is not a shield against takeover. redisNow, err := rdb.Time(ctx).Result() require.NoError(t, err) + require.NoError(t, rdb.HSet( + ctx, + rmapContentKey(node.resources.workerKeepAlive), + workerID, + strconv.FormatInt(redisNow.Add(-2*node.workerTTL).UnixNano(), 10), + ).Err()) require.NoError(t, rdb.HSet( ctx, rmapContentKey(node.resources.workerCleanup), workerID, - fmt.Sprintf("owner|1|%d", redisNow.Add(time.Minute).UnixMilli()), + fmt.Sprintf("%s|%s|0", lease.owner, lease.fence), ).Err()) - won, prev, err = node.deactivateWorker(ctx, workerID) + foreign, err = node.acquireWorkerCleanup(ctx, workerID) require.NoError(t, err) - require.False(t, won) - require.Equal(t, "cleanup", prev) - require.Equal(t, createdAt, rdb.HGet(ctx, rmapContentKey(node.resources.workers), workerID).Val()) + require.NotNil(t, foreign, "expired graceful lease must be recoverable by cleanup") + + // Once cleanup deleted the registration, graceful acquisition refuses and + // never recreates it. + require.NoError(t, node.releaseWorkerCleanup(ctx, foreign, true)) + require.NoError(t, rdb.HDel(ctx, rmapContentKey(node.resources.workers), workerID).Err()) + late, err := node.acquireGracefulRequeue(ctx, workerID) + require.NoError(t, err) + require.Nil(t, late) + require.False(t, rdb.HExists(ctx, rmapContentKey(node.resources.workers), workerID).Val()) - require.NoError(t, rdb.HDel(ctx, rmapContentKey(node.resources.workerCleanup), workerID).Err()) require.NoError(t, node.Shutdown(context.Background())) } diff --git a/pool/worker_test.go b/pool/worker_test.go index bea8cd8..419f4ac 100644 --- a/pool/worker_test.go +++ b/pool/worker_test.go @@ -6,7 +6,6 @@ import ( "sort" "strconv" "strings" - "sync/atomic" "testing" "time" @@ -18,62 +17,6 @@ import ( ptesting "goa.design/pulse/testing" ) -func TestAttemptRequeueAccountsForEveryConcurrentHandoff(t *testing.T) { - worker := &Worker{ - requeueTimeout: 30 * time.Millisecond, - logger: pulse.NoopLogger(), - } - jobs := map[string]*Job{ - "success": {Key: "success"}, - "late": {Key: "late"}, - "failed": {Key: "failed"}, - "blocked": {Key: "blocked"}, - } - started := make(chan string, len(jobs)) - release := make(chan struct{}) - var running atomic.Int64 - send := func(ctx context.Context, job *Job) error { - running.Add(1) - defer running.Add(-1) - started <- job.Key - switch job.Key { - case "success": - <-release - return nil - case "late": - <-release - time.Sleep(5 * time.Millisecond) - return nil - case "failed": - <-release - return errors.New("injected handoff failure") - case "blocked": - <-ctx.Done() - return ctx.Err() - default: - panic("unexpected job") - } - } - result := make(chan map[string]*Job, 1) - go func() { - result <- worker.attemptRequeueWith(context.Background(), jobs, send) - }() - for range jobs { - select { - case <-started: - case <-time.After(time.Second): - require.Fail(t, "handoffs did not start concurrently") - } - } - close(release) - remaining := <-result - require.Equal(t, map[string]*Job{ - "failed": jobs["failed"], - "blocked": jobs["blocked"], - }, remaining) - require.Zero(t, running.Load(), "attempt returned before all handoff goroutines joined") -} - func TestWorkerRequeueJobs(t *testing.T) { var ( ctx = ptesting.NewTestContext(t) From 4c9822cbb22263420d2651a9510aacdb6718840f Mon Sep 17 00:00:00 2001 From: "Raphael (manual office deploy after cloud-state fix)" Date: Sat, 25 Jul 2026 09:28:38 -0700 Subject: [PATCH 3/3] streaming: poll replica membership after atomic consumer rotation Registration now writes membership atomically in Redis, so the writing process's rmap replica converges through the update channel instead of synchronously; the rotation test must poll the replica like the other membership assertions. --- streaming/sink_recovery_test.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/streaming/sink_recovery_test.go b/streaming/sink_recovery_test.go index 0ac71bc..3b08446 100644 --- a/streaming/sink_recovery_test.go +++ b/streaming/sink_recovery_test.go @@ -6,6 +6,7 @@ package streaming import ( "context" "errors" + "slices" "testing" "time" @@ -349,15 +350,19 @@ func TestSinkConsumerRotationRegistersEveryStreamOrRollsBack(t *testing.T) { require.NoError(t, err) require.NotEqual(t, original, replacement) for _, stream := range []*Stream{mainStream, addedStream} { - members, ok := sink.streams[stream.key].consumers.GetValues(sink.Name) - require.True(t, ok) - require.Contains(t, members, replacement) + // Registration writes membership in Redis atomically; the local rmap + // replica converges through the update channel, so poll it. + require.Eventually(t, func() bool { + members, ok := sink.streams[stream.key].consumers.GetValues(sink.Name) + return ok && slices.Contains(members, replacement) + }, max, delay) consumers, err := rdb.XInfoConsumers(ctx, stream.key, sink.Name).Result() require.NoError(t, err) assert.Contains(t, consumerNames(consumers), replacement) } } + func TestSinkStreamMutationRollback(t *testing.T) { rdb := ptesting.NewRedisClient(t) defer ptesting.CleanupRedis(t, rdb, false, "")