From a19284ef33cfce3c5bcdbb22a68ad76a43798f6c Mon Sep 17 00:00:00 2001 From: John Nunemaker Date: Wed, 8 Jul 2026 12:33:52 -0600 Subject: [PATCH 01/15] Fix atomic sync gates --- lib/flipper/adapters/poll.rb | 55 ++++++- .../adapters/sync/interval_synchronizer.rb | 43 ++++- spec/flipper/adapters/poll_spec.rb | 155 ++++++++++++++++++ .../sync/interval_synchronizer_spec.rb | 66 ++++++++ 4 files changed, 311 insertions(+), 8 deletions(-) diff --git a/lib/flipper/adapters/poll.rb b/lib/flipper/adapters/poll.rb index 20cd9e920..7b810ae88 100644 --- a/lib/flipper/adapters/poll.rb +++ b/lib/flipper/adapters/poll.rb @@ -17,7 +17,11 @@ class Poll def initialize(poller, adapter) @adapter = adapter @poller = poller + @pid = Process.pid @last_synced_at = 0 + @syncing = false + @sync_mutex = Mutex.new + @sync_condition = ConditionVariable.new # If the adapter is empty, we need to sync before starting the poller. # Yes, this will block the main thread, but that's better than thinking @@ -39,14 +43,59 @@ def initialize(poller, adapter) private def synced_adapter + reset_sync_state_if_forked @poller.start poller_last_synced_at = @poller.last_synced_at.value - if poller_last_synced_at > @last_synced_at - Flipper::Adapters::Sync::Synchronizer.new(@adapter, @poller.adapter).call - @last_synced_at = poller_last_synced_at + if claim_sync(poller_last_synced_at) + begin + Flipper::Adapters::Sync::Synchronizer.new(@adapter, @poller.adapter).call + complete_sync(poller_last_synced_at) + rescue + release_sync + raise + end end @adapter end + + def reset_sync_state_if_forked + return if @pid == Process.pid + + @pid = Process.pid + @syncing = false + @sync_mutex = Mutex.new + @sync_condition = ConditionVariable.new + end + + def claim_sync(poller_last_synced_at) + @sync_mutex.synchronize do + loop do + return false unless poller_last_synced_at > @last_synced_at + + unless @syncing + @syncing = true + return true + end + + @sync_condition.wait(@sync_mutex) + end + end + end + + def complete_sync(poller_last_synced_at) + @sync_mutex.synchronize do + @last_synced_at = poller_last_synced_at + @syncing = false + @sync_condition.broadcast + end + end + + def release_sync + @sync_mutex.synchronize do + @syncing = false + @sync_condition.broadcast + end + end end end end diff --git a/lib/flipper/adapters/sync/interval_synchronizer.rb b/lib/flipper/adapters/sync/interval_synchronizer.rb index f84309d74..6d91b37c4 100644 --- a/lib/flipper/adapters/sync/interval_synchronizer.rb +++ b/lib/flipper/adapters/sync/interval_synchronizer.rb @@ -21,22 +21,55 @@ def initialize(synchronizer, interval: nil) @interval = interval || DEFAULT_INTERVAL # TODO: add jitter to this so all processes booting at the same time # don't phone home at the same time. + @pid = Process.pid @last_sync_at = 0 + @syncing = false + @sync_mutex = Mutex.new end def call - return unless time_to_sync? + reset_sync_state_if_forked + return unless sync_needed? - @last_sync_at = now - @synchronizer.call + begin + @synchronizer.call + ensure + complete_sync + end nil end private - def time_to_sync? - seconds_since_last_sync = now - @last_sync_at + def reset_sync_state_if_forked + return if @pid == Process.pid + + @pid = Process.pid + @syncing = false + @sync_mutex = Mutex.new + end + + def sync_needed? + @sync_mutex.synchronize do + current_time = now + return false unless time_to_sync?(current_time) + return false if @syncing + + @last_sync_at = current_time + @syncing = true + true + end + end + + def complete_sync + @sync_mutex.synchronize do + @syncing = false + end + end + + def time_to_sync?(current_time) + seconds_since_last_sync = current_time - @last_sync_at seconds_since_last_sync >= @interval end diff --git a/spec/flipper/adapters/poll_spec.rb b/spec/flipper/adapters/poll_spec.rb index 2fe08fe75..0239b8738 100644 --- a/spec/flipper/adapters/poll_spec.rb +++ b/spec/flipper/adapters/poll_spec.rb @@ -38,4 +38,159 @@ expect(local_adapter.features).to eq(remote_adapter.features) end + + it "only synchronizes once per poller update when called concurrently" do + flipper = Flipper.new(local_adapter) + flipper.enable(:existing) + + get_all_calls = Concurrent::AtomicFixnum.new(0) + slow_remote_adapter = Class.new do + def initialize(result, get_all_calls) + @result = result + @get_all_calls = get_all_calls + end + + def get_all(**kwargs) + @get_all_calls.increment + sleep 0.05 + @result + end + end.new(local_adapter.get_all, get_all_calls) + + fake_poller = Struct.new(:last_synced_at, :adapter) do + def start + end + + def sync + raise "sync should not be called when the local adapter is not empty" + end + end.new(Concurrent::AtomicFixnum.new(1), slow_remote_adapter) + + instance = described_class.new(fake_poller, local_adapter) + threads = 10.times.map { Thread.new { instance.features } } + threads.each(&:join) + + expect(get_all_calls.value).to eq(1) + end + + it "waits for an in-flight poller update before returning the adapter" do + flipper = Flipper.new(local_adapter) + flipper.enable(:existing) + + remote = Flipper::Adapters::Memory.new(threadsafe: true) + Flipper.new(remote).enable(:updated) + + entered = Queue.new + release = Queue.new + slow_remote_adapter = Class.new do + def initialize(result, entered, release) + @result = result + @entered = entered + @release = release + end + + def get_all(**kwargs) + @entered << true + @release.pop + @result + end + end.new(remote.get_all, entered, release) + + fake_poller = Struct.new(:last_synced_at, :adapter) do + def start + end + + def sync + raise "sync should not be called when the local adapter is not empty" + end + end.new(Concurrent::AtomicFixnum.new(1), slow_remote_adapter) + + instance = described_class.new(fake_poller, local_adapter) + first_thread = Thread.new { instance.features } + entered.pop + + completed = Queue.new + second_thread = Thread.new { completed << instance.features } + sleep 0.05 + + expect(completed).to be_empty + + release << true + expect(first_thread.value).to eq(Set["updated"]) + expect(completed.pop).to eq(Set["updated"]) + second_thread.join + end + + it "retries a poller update after synchronization fails" do + flipper = Flipper.new(local_adapter) + flipper.enable(:existing) + + get_all_calls = Concurrent::AtomicFixnum.new(0) + flaky_remote_adapter = Class.new do + def initialize(result, get_all_calls) + @result = result + @get_all_calls = get_all_calls + end + + def get_all(**kwargs) + raise "transient failure" if @get_all_calls.increment == 1 + + @result + end + end.new(local_adapter.get_all, get_all_calls) + + fake_poller = Struct.new(:last_synced_at, :adapter) do + def start + end + + def sync + raise "sync should not be called when the local adapter is not empty" + end + end.new(Concurrent::AtomicFixnum.new(1), flaky_remote_adapter) + + instance = described_class.new(fake_poller, local_adapter) + + expect { instance.features }.to raise_error("transient failure") + instance.features + + expect(get_all_calls.value).to eq(2) + end + + it "resets in-flight synchronization state after a fork" do + flipper = Flipper.new(local_adapter) + flipper.enable(:existing) + + remote = Flipper::Adapters::Memory.new(threadsafe: true) + Flipper.new(remote).enable(:updated) + + get_all_calls = Concurrent::AtomicFixnum.new(0) + counting_remote_adapter = Class.new do + def initialize(result, get_all_calls) + @result = result + @get_all_calls = get_all_calls + end + + def get_all(**kwargs) + @get_all_calls.increment + @result + end + end.new(remote.get_all, get_all_calls) + + fake_poller = Struct.new(:last_synced_at, :adapter) do + def start + end + + def sync + raise "sync should not be called when the local adapter is not empty" + end + end.new(Concurrent::AtomicFixnum.new(1), counting_remote_adapter) + + instance = described_class.new(fake_poller, local_adapter) + instance.instance_variable_set(:@syncing, true) + + allow(Process).to receive(:pid).and_return(instance.instance_variable_get(:@pid) + 1) + + expect(instance.features).to eq(Set["updated"]) + expect(get_all_calls.value).to eq(1) + end end diff --git a/spec/flipper/adapters/sync/interval_synchronizer_spec.rb b/spec/flipper/adapters/sync/interval_synchronizer_spec.rb index e2076c26f..7408aeac2 100644 --- a/spec/flipper/adapters/sync/interval_synchronizer_spec.rb +++ b/spec/flipper/adapters/sync/interval_synchronizer_spec.rb @@ -30,4 +30,70 @@ subject.call expect(events.size).to be(1) end + + it "does not synchronize again while a claimed interval sync is in flight" do + entered = Queue.new + release = Queue.new + synchronizer = -> do + events << now + entered << true + release.pop + end + instance = described_class.new(synchronizer, interval: interval) + + allow(instance).to receive(:now).and_return(interval) + + first_thread = Thread.new { instance.call } + entered.pop + + threads = 10.times.map { Thread.new { instance.call } } + sleep 0.05 + + expect(events.size).to eq(1) + + release << true + ([first_thread] + threads).each(&:join) + + expect(events.size).to eq(1) + end + + it "does not synchronize again when the interval passes during an in-flight sync" do + current_time = interval + entered = Queue.new + release = Queue.new + synchronizer = -> do + events << current_time + entered << true + release.pop + end + instance = described_class.new(synchronizer, interval: interval) + + allow(instance).to receive(:now) { current_time } + + first_thread = Thread.new { instance.call } + entered.pop + + current_time += interval + second_thread = Thread.new { instance.call } + sleep 0.05 + + expect(events.size).to eq(1) + + release << true + [first_thread, second_thread].each(&:join) + + expect(events.size).to eq(1) + end + + it "resets in-flight synchronization state after a fork" do + instance = described_class.new(synchronizer, interval: interval) + instance.instance_variable_set(:@syncing, true) + + allow(instance).to receive(:now).and_return(interval) + allow(Process).to receive(:pid).and_return(instance.instance_variable_get(:@pid) + 1) + + instance.call + + expect(events.size).to eq(1) + end end From 4920865988ca834b75320e084a181f960c518235 Mon Sep 17 00:00:00 2001 From: John Nunemaker Date: Tue, 21 Jul 2026 16:47:58 -0500 Subject: [PATCH 02/15] Clear sync state in ensure to avoid deadlock on non-StandardError The bare rescue in synced_adapter only cleared @syncing for StandardError, so a non-StandardError from the Synchronizer (Interrupt, SignalException, etc.) would leave @syncing true permanently, deadlocking all subsequent reads waiting on @sync_condition. Use an ensure with a synced flag so state is always cleared, bumping @last_synced_at only on success. Mirrors the ensure-based cleanup already in IntervalSynchronizer#call. Co-Authored-By: Claude Opus 4.8 --- lib/flipper/adapters/poll.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/flipper/adapters/poll.rb b/lib/flipper/adapters/poll.rb index 7b810ae88..a83c5435a 100644 --- a/lib/flipper/adapters/poll.rb +++ b/lib/flipper/adapters/poll.rb @@ -47,12 +47,12 @@ def synced_adapter @poller.start poller_last_synced_at = @poller.last_synced_at.value if claim_sync(poller_last_synced_at) + synced = false begin Flipper::Adapters::Sync::Synchronizer.new(@adapter, @poller.adapter).call - complete_sync(poller_last_synced_at) - rescue - release_sync - raise + synced = true + ensure + synced ? complete_sync(poller_last_synced_at) : release_sync end end @adapter From 0b6ad9e94bae13854c73f1a770850eee6070f922 Mon Sep 17 00:00:00 2001 From: John Nunemaker Date: Wed, 5 Aug 2026 17:01:13 -0400 Subject: [PATCH 03/15] Don't block requests waiting on an in-flight poll sync Threads that lose the sync claim now return the local adapter immediately instead of waiting on the syncing thread. The data they serve is at most one poll interval stale, which is the contract already, and strictly better than the pre-fix behavior where they read partially applied mid-sync state. Co-Authored-By: Claude Opus 5 (1M context) --- lib/flipper/adapters/poll.rb | 26 ++++++++++---------------- spec/flipper/adapters/poll_spec.rb | 10 +++++----- 2 files changed, 15 insertions(+), 21 deletions(-) diff --git a/lib/flipper/adapters/poll.rb b/lib/flipper/adapters/poll.rb index a83c5435a..ac547dcec 100644 --- a/lib/flipper/adapters/poll.rb +++ b/lib/flipper/adapters/poll.rb @@ -21,7 +21,6 @@ def initialize(poller, adapter) @last_synced_at = 0 @syncing = false @sync_mutex = Mutex.new - @sync_condition = ConditionVariable.new # If the adapter is empty, we need to sync before starting the poller. # Yes, this will block the main thread, but that's better than thinking @@ -64,21 +63,20 @@ def reset_sync_state_if_forked @pid = Process.pid @syncing = false @sync_mutex = Mutex.new - @sync_condition = ConditionVariable.new end + # Internal: Attempts to claim the right to sync. Returns true if this + # caller should sync. Returns false if a sync is unnecessary or if + # another thread is already syncing. Never blocks. Callers that lose the + # claim serve the local adapter as is, which is at most one poll interval + # stale, rather than waiting on a sync in the middle of a request. def claim_sync(poller_last_synced_at) @sync_mutex.synchronize do - loop do - return false unless poller_last_synced_at > @last_synced_at + return false if @syncing + return false unless poller_last_synced_at > @last_synced_at - unless @syncing - @syncing = true - return true - end - - @sync_condition.wait(@sync_mutex) - end + @syncing = true + true end end @@ -86,15 +84,11 @@ def complete_sync(poller_last_synced_at) @sync_mutex.synchronize do @last_synced_at = poller_last_synced_at @syncing = false - @sync_condition.broadcast end end def release_sync - @sync_mutex.synchronize do - @syncing = false - @sync_condition.broadcast - end + @sync_mutex.synchronize { @syncing = false } end end end diff --git a/spec/flipper/adapters/poll_spec.rb b/spec/flipper/adapters/poll_spec.rb index 0239b8738..9bed58dc3 100644 --- a/spec/flipper/adapters/poll_spec.rb +++ b/spec/flipper/adapters/poll_spec.rb @@ -73,7 +73,7 @@ def sync expect(get_all_calls.value).to eq(1) end - it "waits for an in-flight poller update before returning the adapter" do + it "does not wait for an in-flight poller update before returning the adapter" do flipper = Flipper.new(local_adapter) flipper.enable(:existing) @@ -111,14 +111,14 @@ def sync completed = Queue.new second_thread = Thread.new { completed << instance.features } - sleep 0.05 + second_thread.join(1) - expect(completed).to be_empty + # The second thread serves the local adapter as is rather than blocking on + # the sync the first thread is running. + expect(completed.pop(true)).to eq(Set["existing"]) release << true expect(first_thread.value).to eq(Set["updated"]) - expect(completed.pop).to eq(Set["updated"]) - second_thread.join end it "retries a poller update after synchronization fails" do From d35fb2fd7f093d622a0203505f6467c9e5db7b2d Mon Sep 17 00:00:00 2001 From: John Nunemaker Date: Sun, 9 Aug 2026 19:21:04 -0400 Subject: [PATCH 04/15] Harden atomic sync gates Use PID-scoped atomic gate state so forked child threads converge on one claim, keep Poll losers nonblocking with a coherent pre-sync snapshot, and cover contention and failure recovery paths. --- lib/flipper/adapters/poll.rb | 95 +++++++++++++------ .../adapters/sync/interval_synchronizer.rb | 52 +++++----- spec/flipper/adapters/poll_spec.rb | 91 +++++++++++++++++- .../sync/interval_synchronizer_spec.rb | 36 ++++++- 4 files changed, 214 insertions(+), 60 deletions(-) diff --git a/lib/flipper/adapters/poll.rb b/lib/flipper/adapters/poll.rb index ac547dcec..109fd51d1 100644 --- a/lib/flipper/adapters/poll.rb +++ b/lib/flipper/adapters/poll.rb @@ -1,5 +1,7 @@ require 'flipper/adapters/sync/synchronizer' +require 'flipper/adapters/memory' require 'flipper/poller' +require 'concurrent/atomic/atomic_reference' module Flipper module Adapters @@ -7,6 +9,20 @@ class Poll extend Forwardable include ::Flipper::Adapter + SyncState = Struct.new(:pid, :mutex, :syncing, :last_synced_at, :snapshot) + + class InFlightAdapter + extend Forwardable + + def_delegators :@snapshot, :features, :get, :get_multi, :get_all + def_delegators :@adapter, :add, :remove, :clear, :enable, :disable + + def initialize(snapshot, adapter) + @snapshot = snapshot + @adapter = adapter + end + end + # Deprecated Poller = ::Flipper::Poller @@ -17,10 +33,9 @@ class Poll def initialize(poller, adapter) @adapter = adapter @poller = poller - @pid = Process.pid - @last_synced_at = 0 - @syncing = false - @sync_mutex = Mutex.new + @sync_state = Concurrent::AtomicReference.new( + SyncState.new(Process.pid, Mutex.new, false, 0, nil) + ) # If the adapter is empty, we need to sync before starting the poller. # Yes, this will block the main thread, but that's better than thinking @@ -42,53 +57,71 @@ def initialize(poller, adapter) private def synced_adapter - reset_sync_state_if_forked + state = sync_state @poller.start poller_last_synced_at = @poller.last_synced_at.value - if claim_sync(poller_last_synced_at) + case claim_sync(state, poller_last_synced_at) + when :claimed synced = false begin Flipper::Adapters::Sync::Synchronizer.new(@adapter, @poller.adapter).call synced = true ensure - synced ? complete_sync(poller_last_synced_at) : release_sync + synced ? complete_sync(state, poller_last_synced_at) : release_sync(state) end + @adapter + when :syncing + state.snapshot + else + @adapter end - @adapter end - def reset_sync_state_if_forked - return if @pid == Process.pid + def sync_state + pid = Process.pid + loop do + state = @sync_state.get + return state if state.pid == pid - @pid = Process.pid - @syncing = false - @sync_mutex = Mutex.new + replacement = SyncState.new(pid, Mutex.new, false, state.last_synced_at, nil) + return replacement if @sync_state.compare_and_set(state, replacement) + end end - # Internal: Attempts to claim the right to sync. Returns true if this - # caller should sync. Returns false if a sync is unnecessary or if - # another thread is already syncing. Never blocks. Callers that lose the - # claim serve the local adapter as is, which is at most one poll interval - # stale, rather than waiting on a sync in the middle of a request. - def claim_sync(poller_last_synced_at) - @sync_mutex.synchronize do - return false if @syncing - return false unless poller_last_synced_at > @last_synced_at - - @syncing = true - true + # Internal: Attempts to claim the right to sync. Returns :claimed if this + # caller should sync, :syncing if another caller owns the sync, or another + # status when no sync should run. Never blocks. Callers that lose an + # in-flight claim read from the coherent pre-sync snapshot rather than + # waiting on a sync in the middle of a request. + def claim_sync(state, poller_last_synced_at) + return :contended unless state.mutex.try_lock + + begin + return :syncing if state.syncing + return :not_needed unless poller_last_synced_at > state.last_synced_at + + snapshot = Flipper::Adapters::Memory.new(@adapter.get_all) + state.snapshot = InFlightAdapter.new(snapshot, @adapter) + state.syncing = true + :claimed + ensure + state.mutex.unlock end end - def complete_sync(poller_last_synced_at) - @sync_mutex.synchronize do - @last_synced_at = poller_last_synced_at - @syncing = false + def complete_sync(state, poller_last_synced_at) + state.mutex.synchronize do + state.last_synced_at = poller_last_synced_at + state.syncing = false + state.snapshot = nil end end - def release_sync - @sync_mutex.synchronize { @syncing = false } + def release_sync(state) + state.mutex.synchronize do + state.syncing = false + state.snapshot = nil + end end end end diff --git a/lib/flipper/adapters/sync/interval_synchronizer.rb b/lib/flipper/adapters/sync/interval_synchronizer.rb index 6d91b37c4..752e650d6 100644 --- a/lib/flipper/adapters/sync/interval_synchronizer.rb +++ b/lib/flipper/adapters/sync/interval_synchronizer.rb @@ -1,9 +1,13 @@ +require 'concurrent/atomic/atomic_reference' + module Flipper module Adapters class Sync # Internal: Wraps a Synchronizer instance and only invokes it every # N seconds. class IntervalSynchronizer + SyncState = Struct.new(:pid, :mutex, :syncing, :last_sync_at) + # Private: Number of seconds between syncs (default: 10). DEFAULT_INTERVAL = 10 @@ -21,20 +25,19 @@ def initialize(synchronizer, interval: nil) @interval = interval || DEFAULT_INTERVAL # TODO: add jitter to this so all processes booting at the same time # don't phone home at the same time. - @pid = Process.pid - @last_sync_at = 0 - @syncing = false - @sync_mutex = Mutex.new + @sync_state = Concurrent::AtomicReference.new( + SyncState.new(Process.pid, Mutex.new, false, 0) + ) end def call - reset_sync_state_if_forked - return unless sync_needed? + state = sync_state + return unless sync_needed?(state) begin @synchronizer.call ensure - complete_sync + complete_sync(state) end nil @@ -42,34 +45,37 @@ def call private - def reset_sync_state_if_forked - return if @pid == Process.pid + def sync_state + pid = Process.pid + loop do + state = @sync_state.get + return state if state.pid == pid - @pid = Process.pid - @syncing = false - @sync_mutex = Mutex.new + replacement = SyncState.new(pid, Mutex.new, false, state.last_sync_at) + return replacement if @sync_state.compare_and_set(state, replacement) + end end - def sync_needed? - @sync_mutex.synchronize do + def sync_needed?(state) + state.mutex.synchronize do current_time = now - return false unless time_to_sync?(current_time) - return false if @syncing + return false unless time_to_sync?(state, current_time) + return false if state.syncing - @last_sync_at = current_time - @syncing = true + state.last_sync_at = current_time + state.syncing = true true end end - def complete_sync - @sync_mutex.synchronize do - @syncing = false + def complete_sync(state) + state.mutex.synchronize do + state.syncing = false end end - def time_to_sync?(current_time) - seconds_since_last_sync = current_time - @last_sync_at + def time_to_sync?(state, current_time) + seconds_since_last_sync = current_time - state.last_sync_at seconds_since_last_sync >= @interval end diff --git a/spec/flipper/adapters/poll_spec.rb b/spec/flipper/adapters/poll_spec.rb index 9bed58dc3..b963bb519 100644 --- a/spec/flipper/adapters/poll_spec.rb +++ b/spec/flipper/adapters/poll_spec.rb @@ -121,6 +121,89 @@ def sync expect(first_thread.value).to eq(Set["updated"]) end + it "serves a coherent snapshot while a poller update is being applied" do + entered = Queue.new + release = Queue.new + pausing_local_adapter = Class.new(Flipper::Adapters::Memory) do + def initialize(entered, release) + super(nil, threadsafe: true) + @entered = entered + @release = release + @pause = true + end + + def disable(feature, gate, thing) + result = super + if @pause + @pause = false + @entered << true + @release.pop + end + result + end + end.new(entered, release) + Flipper.new(pausing_local_adapter).enable(:existing) + + remote = Flipper::Adapters::Memory.new(threadsafe: true) + Flipper.new(remote).disable(:existing) + Flipper.new(remote).enable(:updated) + + fake_poller = Struct.new(:last_synced_at, :adapter) do + def start + end + + def sync + raise "sync should not be called when the local adapter is not empty" + end + end.new(Concurrent::AtomicFixnum.new(1), remote) + + instance = described_class.new(fake_poller, pausing_local_adapter) + first_thread = Thread.new { instance.features } + entered.pop + + expect(Flipper.new(pausing_local_adapter).enabled?(:existing)).to be(false) + expect(Flipper.new(instance).enabled?(:existing)).to be(true) + + release << true + expect(first_thread.value).to eq(Set["existing", "updated"]) + expect(Flipper.new(instance).enabled?(:existing)).to be(false) + end + + it "does not wait for the sync claim mutex" do + Flipper.new(local_adapter).enable(:existing) + + fake_poller = Struct.new(:last_synced_at, :adapter) do + def start + end + + def sync + raise "sync should not be called when the local adapter is not empty" + end + end.new(Concurrent::AtomicFixnum.new(1), remote_adapter) + + instance = described_class.new(fake_poller, local_adapter) + state = instance.instance_variable_get(:@sync_state).get + locked = Queue.new + release = Queue.new + holder = Thread.new do + state.mutex.lock + locked << true + release.pop + state.mutex.unlock + end + locked.pop + + completed = Queue.new + caller = Thread.new { completed << instance.features } + caller.join(1) + + expect(completed.pop(true)).to eq(Set["existing"]) + ensure + release << true if release + holder&.join + caller&.join + end + it "retries a poller update after synchronization fails" do flipper = Flipper.new(local_adapter) flipper.enable(:existing) @@ -186,11 +269,13 @@ def sync end.new(Concurrent::AtomicFixnum.new(1), counting_remote_adapter) instance = described_class.new(fake_poller, local_adapter) - instance.instance_variable_set(:@syncing, true) + stale_state = instance.instance_variable_get(:@sync_state).get + stale_state.syncing = true - allow(Process).to receive(:pid).and_return(instance.instance_variable_get(:@pid) + 1) + allow(Process).to receive(:pid).and_return(stale_state.pid + 1) - expect(instance.features).to eq(Set["updated"]) + threads = 10.times.map { Thread.new { instance.features } } + expect(threads.map(&:value)).to all(eq(Set["updated"])) expect(get_all_calls.value).to eq(1) end end diff --git a/spec/flipper/adapters/sync/interval_synchronizer_spec.rb b/spec/flipper/adapters/sync/interval_synchronizer_spec.rb index 7408aeac2..ff2bc5d71 100644 --- a/spec/flipper/adapters/sync/interval_synchronizer_spec.rb +++ b/spec/flipper/adapters/sync/interval_synchronizer_spec.rb @@ -85,14 +85,44 @@ expect(events.size).to eq(1) end + it "releases a failed sync for the next interval" do + current_time = interval + calls = 0 + synchronizer = -> do + calls += 1 + raise "transient failure" if calls == 1 + end + instance = described_class.new(synchronizer, interval: interval) + allow(instance).to receive(:now) { current_time } + + expect { instance.call }.to raise_error("transient failure") + instance.call + expect(calls).to eq(1) + + current_time += interval + instance.call + expect(calls).to eq(2) + end + it "resets in-flight synchronization state after a fork" do + entered = Queue.new + release = Queue.new + synchronizer = -> do + events << now + entered << true + release.pop + end instance = described_class.new(synchronizer, interval: interval) - instance.instance_variable_set(:@syncing, true) + stale_state = instance.instance_variable_get(:@sync_state).get + stale_state.syncing = true allow(instance).to receive(:now).and_return(interval) - allow(Process).to receive(:pid).and_return(instance.instance_variable_get(:@pid) + 1) + allow(Process).to receive(:pid).and_return(stale_state.pid + 1) - instance.call + threads = 10.times.map { Thread.new { instance.call } } + entered.pop + release << true + threads.each(&:join) expect(events.size).to eq(1) end From aba63826e913f3491cc251d11e39e84679d9f5a9 Mon Sep 17 00:00:00 2001 From: John Nunemaker Date: Sun, 9 Aug 2026 20:21:52 -0400 Subject: [PATCH 05/15] Close poll snapshot handoff race --- lib/flipper/adapters/poll.rb | 32 +++++--- lib/flipper/adapters/sync/synchronizer.rb | 4 +- spec/flipper/adapters/poll_spec.rb | 82 +++++++++++++++++++ .../adapters/sync/synchronizer_spec.rb | 11 +++ 4 files changed, 115 insertions(+), 14 deletions(-) diff --git a/lib/flipper/adapters/poll.rb b/lib/flipper/adapters/poll.rb index 109fd51d1..11f20a1be 100644 --- a/lib/flipper/adapters/poll.rb +++ b/lib/flipper/adapters/poll.rb @@ -60,18 +60,23 @@ def synced_adapter state = sync_state @poller.start poller_last_synced_at = @poller.last_synced_at.value - case claim_sync(state, poller_last_synced_at) + claim, value = claim_sync(state, poller_last_synced_at) + case claim when :claimed synced = false begin - Flipper::Adapters::Sync::Synchronizer.new(@adapter, @poller.adapter).call + Flipper::Adapters::Sync::Synchronizer.new( + @adapter, + @poller.adapter, + local_get_all: value + ).call synced = true ensure synced ? complete_sync(state, poller_last_synced_at) : release_sync(state) end @adapter when :syncing - state.snapshot + value else @adapter end @@ -88,22 +93,23 @@ def sync_state end end - # Internal: Attempts to claim the right to sync. Returns :claimed if this - # caller should sync, :syncing if another caller owns the sync, or another - # status when no sync should run. Never blocks. Callers that lose an - # in-flight claim read from the coherent pre-sync snapshot rather than - # waiting on a sync in the middle of a request. + # Internal: Attempts to claim the right to sync. Returns the status and + # the data captured while holding the mutex: the local state for :claimed + # or the coherent pre-sync adapter for :syncing. Never blocks. Callers + # that lose an in-flight claim read from the snapshot rather than waiting + # on a sync in the middle of a request. def claim_sync(state, poller_last_synced_at) - return :contended unless state.mutex.try_lock + return [:contended, nil] unless state.mutex.try_lock begin - return :syncing if state.syncing - return :not_needed unless poller_last_synced_at > state.last_synced_at + return [:syncing, state.snapshot] if state.syncing + return [:not_needed, nil] unless poller_last_synced_at > state.last_synced_at - snapshot = Flipper::Adapters::Memory.new(@adapter.get_all) + local_get_all = @adapter.get_all + snapshot = Flipper::Adapters::Memory.new(local_get_all) state.snapshot = InFlightAdapter.new(snapshot, @adapter) state.syncing = true - :claimed + [:claimed, local_get_all] ensure state.mutex.unlock end diff --git a/lib/flipper/adapters/sync/synchronizer.rb b/lib/flipper/adapters/sync/synchronizer.rb index 7ce58a471..035e6160c 100644 --- a/lib/flipper/adapters/sync/synchronizer.rb +++ b/lib/flipper/adapters/sync/synchronizer.rb @@ -19,12 +19,14 @@ class Synchronizer # :instrumenter - The instrumenter used to instrument. # :raise - Should errors be raised (default: true). # :cache_bust - Should cache busting be used for remote get_all (default: false). + # :local_get_all - Optional pre-fetched local adapter state. def initialize(local, remote, options = {}) @local = local @remote = remote @instrumenter = options.fetch(:instrumenter, Instrumenters::Noop) @raise = options.fetch(:raise, true) @cache_bust = options.fetch(:cache_bust, false) + @local_get_all = options[:local_get_all] end # Public: Forces a sync. @@ -39,7 +41,7 @@ def call private def sync - local_get_all = @local.get_all + local_get_all = @local_get_all || @local.get_all remote_get_all = @remote.get_all(cache_bust: @cache_bust) # Sync all the gate values. diff --git a/spec/flipper/adapters/poll_spec.rb b/spec/flipper/adapters/poll_spec.rb index b963bb519..6252bf695 100644 --- a/spec/flipper/adapters/poll_spec.rb +++ b/spec/flipper/adapters/poll_spec.rb @@ -169,6 +169,88 @@ def sync expect(Flipper.new(instance).enabled?(:existing)).to be(false) end + it "keeps the claimed snapshot after the poller update completes" do + flipper = Flipper.new(local_adapter) + flipper.enable(:existing) + + remote = Flipper::Adapters::Memory.new(threadsafe: true) + Flipper.new(remote).enable(:updated) + + sync_entered = Queue.new + release_sync = Queue.new + slow_remote_adapter = Class.new do + def initialize(result, entered, release) + @result = result + @entered = entered + @release = release + end + + def get_all(**kwargs) + @entered << true + @release.pop + @result + end + end.new(remote.get_all, sync_entered, release_sync) + + fake_poller = Struct.new(:last_synced_at, :adapter) do + def start + end + + def sync + raise "sync should not be called when the local adapter is not empty" + end + end.new(Concurrent::AtomicFixnum.new(1), slow_remote_adapter) + + instance = described_class.new(fake_poller, local_adapter) + loser_claimed = Queue.new + release_loser = Queue.new + allow(instance).to receive(:claim_sync).and_wrap_original do |method, *args| + result = method.call(*args) + if Thread.current[:poll_loser] + loser_claimed << true + release_loser.pop + end + result + end + + winner = Thread.new { instance.features } + sync_entered.pop + loser = Thread.new do + Thread.current[:poll_loser] = true + instance.features + end + loser_claimed.pop + + release_sync << true + expect(winner.value).to eq(Set["updated"]) + release_loser << true + + expect(loser.value).to eq(Set["existing"]) + ensure + release_sync << true if release_sync + release_loser << true if release_loser + winner&.join + loser&.join + end + + it "reads the local adapter once for a claimed poller update" do + Flipper.new(local_adapter).enable(:existing) + + fake_poller = Struct.new(:last_synced_at, :adapter) do + def start + end + + def sync + raise "sync should not be called when the local adapter is not empty" + end + end.new(Concurrent::AtomicFixnum.new(1), remote_adapter) + + instance = described_class.new(fake_poller, local_adapter) + expect(local_adapter).to receive(:get_all).once.and_call_original + + instance.features + end + it "does not wait for the sync claim mutex" do Flipper.new(local_adapter).enable(:existing) diff --git a/spec/flipper/adapters/sync/synchronizer_spec.rb b/spec/flipper/adapters/sync/synchronizer_spec.rb index 6e6c19a99..d6713b1c5 100644 --- a/spec/flipper/adapters/sync/synchronizer_spec.rb +++ b/spec/flipper/adapters/sync/synchronizer_spec.rb @@ -56,6 +56,17 @@ expect(instrumenter.events_by_name("synchronizer_exception.flipper").size).to be(0) end + it 'uses pre-fetched local adapter state when provided' do + local_flipper.enable(:existing) + prefetched = local.get_all + remote_flipper.enable(:updated) + expect(local).not_to receive(:get_all) + + described_class.new(local, remote, local_get_all: prefetched).call + + expect(local_flipper.features.map(&:key)).to eq(["updated"]) + end + it 'syncs each remote feature to local' do remote_flipper.enable(:search) remote_flipper.enable_percentage_of_time(:logging, 10) From 16bc64377d940f945af289fbec33c9bb62061336 Mon Sep 17 00:00:00 2001 From: John Nunemaker Date: Sun, 9 Aug 2026 20:33:01 -0400 Subject: [PATCH 06/15] fix(review): preserve contended poll snapshots --- lib/flipper/adapters/poll.rb | 25 +++++------ spec/flipper/adapters/poll_spec.rb | 71 ++++++++++++++++++++++++++++++ 2 files changed, 83 insertions(+), 13 deletions(-) diff --git a/lib/flipper/adapters/poll.rb b/lib/flipper/adapters/poll.rb index 11f20a1be..be0009c58 100644 --- a/lib/flipper/adapters/poll.rb +++ b/lib/flipper/adapters/poll.rb @@ -33,10 +33,6 @@ def initialize(snapshot, adapter) def initialize(poller, adapter) @adapter = adapter @poller = poller - @sync_state = Concurrent::AtomicReference.new( - SyncState.new(Process.pid, Mutex.new, false, 0, nil) - ) - # If the adapter is empty, we need to sync before starting the poller. # Yes, this will block the main thread, but that's better than thinking # nothing is enabled. @@ -51,6 +47,11 @@ def initialize(poller, adapter) end end + snapshot = InFlightAdapter.new(Flipper::Adapters::Memory.new(adapter.get_all), adapter) + @sync_state = Concurrent::AtomicReference.new( + SyncState.new(Process.pid, Mutex.new, false, 0, snapshot) + ) + @poller.start end @@ -75,7 +76,7 @@ def synced_adapter synced ? complete_sync(state, poller_last_synced_at) : release_sync(state) end @adapter - when :syncing + when :syncing, :contended value else @adapter @@ -88,18 +89,18 @@ def sync_state state = @sync_state.get return state if state.pid == pid - replacement = SyncState.new(pid, Mutex.new, false, state.last_synced_at, nil) + replacement = SyncState.new(pid, Mutex.new, false, state.last_synced_at, state.snapshot) return replacement if @sync_state.compare_and_set(state, replacement) end end # Internal: Attempts to claim the right to sync. Returns the status and - # the data captured while holding the mutex: the local state for :claimed - # or the coherent pre-sync adapter for :syncing. Never blocks. Callers - # that lose an in-flight claim read from the snapshot rather than waiting - # on a sync in the middle of a request. + # the data associated with that status: the local state for :claimed or + # the latest coherent adapter for :syncing and :contended. Never blocks. + # Callers that lose an in-flight claim read from the snapshot rather than + # waiting on a sync in the middle of a request. def claim_sync(state, poller_last_synced_at) - return [:contended, nil] unless state.mutex.try_lock + return [:contended, state.snapshot] unless state.mutex.try_lock begin return [:syncing, state.snapshot] if state.syncing @@ -119,14 +120,12 @@ def complete_sync(state, poller_last_synced_at) state.mutex.synchronize do state.last_synced_at = poller_last_synced_at state.syncing = false - state.snapshot = nil end end def release_sync(state) state.mutex.synchronize do state.syncing = false - state.snapshot = nil end end end diff --git a/spec/flipper/adapters/poll_spec.rb b/spec/flipper/adapters/poll_spec.rb index 6252bf695..479a5f49e 100644 --- a/spec/flipper/adapters/poll_spec.rb +++ b/spec/flipper/adapters/poll_spec.rb @@ -233,6 +233,77 @@ def sync loser&.join end + it "keeps a coherent snapshot when the sync claim mutex is contended" do + get_all_entered = Queue.new + release_get_all = Queue.new + pausing_local_adapter = Class.new(Flipper::Adapters::Memory) do + def initialize(entered, release) + super(nil, threadsafe: true) + @entered = entered + @release = release + @pause_next_get_all = false + end + + def pause_next_get_all + @pause_next_get_all = true + end + + def get_all(**kwargs) + if @pause_next_get_all + @pause_next_get_all = false + @entered << true + @release.pop + end + super + end + end.new(get_all_entered, release_get_all) + Flipper.new(pausing_local_adapter).enable(:existing) + + remote = Flipper::Adapters::Memory.new(threadsafe: true) + Flipper.new(remote).enable(:updated) + fake_poller = Struct.new(:last_synced_at, :adapter) do + def start + end + + def sync + raise "sync should not be called when the local adapter is not empty" + end + end.new(Concurrent::AtomicFixnum.new(1), remote) + + instance = described_class.new(fake_poller, pausing_local_adapter) + pausing_local_adapter.pause_next_get_all + + loser_claimed = Queue.new + release_loser = Queue.new + allow(instance).to receive(:claim_sync).and_wrap_original do |method, *args| + result = method.call(*args) + if Thread.current[:poll_contender] + loser_claimed << true + release_loser.pop + end + result + end + + winner = Thread.new { instance.features } + get_all_entered.pop + loser = Thread.new do + Thread.current[:poll_contender] = true + instance.features + end + loser_claimed.pop + + release_get_all << true + expect(winner.value).to eq(Set["updated"]) + release_loser << true + + expect(loser.value).to eq(Set["existing"]) + ensure + release_get_all << true if release_get_all + release_loser << true if release_loser + winner&.join + loser&.join + end + it "reads the local adapter once for a claimed poller update" do Flipper.new(local_adapter).enable(:existing) From 272be0a6959e126e80f14cbc6220cd13fbb9dab2 Mon Sep 17 00:00:00 2001 From: John Nunemaker Date: Sun, 9 Aug 2026 20:37:27 -0400 Subject: [PATCH 07/15] fix(review): preserve poll initialization fallback --- lib/flipper/adapters/poll.rb | 15 ++++++++++++-- spec/flipper/adapters/poll_spec.rb | 32 ++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 2 deletions(-) diff --git a/lib/flipper/adapters/poll.rb b/lib/flipper/adapters/poll.rb index be0009c58..95d6ab12d 100644 --- a/lib/flipper/adapters/poll.rb +++ b/lib/flipper/adapters/poll.rb @@ -47,7 +47,12 @@ def initialize(poller, adapter) end end - snapshot = InFlightAdapter.new(Flipper::Adapters::Memory.new(adapter.get_all), adapter) + snapshot = begin + InFlightAdapter.new(Flipper::Adapters::Memory.new(adapter.get_all), adapter) + rescue + # Preserve the existing fail-open initialization behavior. The first + # successful request establishes the snapshot before a sync can run. + end @sync_state = Concurrent::AtomicReference.new( SyncState.new(Process.pid, Mutex.new, false, 0, snapshot) ) @@ -100,10 +105,16 @@ def sync_state # Callers that lose an in-flight claim read from the snapshot rather than # waiting on a sync in the middle of a request. def claim_sync(state, poller_last_synced_at) - return [:contended, state.snapshot] unless state.mutex.try_lock + return [:contended, state.snapshot || @adapter] unless state.mutex.try_lock begin return [:syncing, state.snapshot] if state.syncing + unless state.snapshot + local_get_all = @adapter.get_all + snapshot = Flipper::Adapters::Memory.new(local_get_all) + state.snapshot = InFlightAdapter.new(snapshot, @adapter) + return [:snapshot_established, nil] + end return [:not_needed, nil] unless poller_last_synced_at > state.last_synced_at local_get_all = @adapter.get_all diff --git a/spec/flipper/adapters/poll_spec.rb b/spec/flipper/adapters/poll_spec.rb index 479a5f49e..d388a6496 100644 --- a/spec/flipper/adapters/poll_spec.rb +++ b/spec/flipper/adapters/poll_spec.rb @@ -39,6 +39,38 @@ expect(local_adapter.features).to eq(remote_adapter.features) end + it "establishes a snapshot after a local get_all initialization failure" do + flaky_local_adapter = Class.new(Flipper::Adapters::Memory) do + def initialize + super + @get_all_calls = 0 + end + + def get_all(**kwargs) + @get_all_calls += 1 + raise "transient local failure" if @get_all_calls == 1 + + super + end + end.new + Flipper.new(flaky_local_adapter).enable(:existing) + + fake_poller = Struct.new(:last_synced_at, :adapter) do + def start + end + + def sync + raise "sync should not be called when the local adapter is not empty" + end + end.new(Concurrent::AtomicFixnum.new(1), remote_adapter) + + instance = nil + expect { instance = described_class.new(fake_poller, flaky_local_adapter) }.not_to raise_error + + expect(instance.features).to eq(Set["existing"]) + expect(instance.features).to eq(Set["analytics", "search"]) + end + it "only synchronizes once per poller update when called concurrently" do flipper = Flipper.new(local_adapter) flipper.enable(:existing) From 0c160f1ea1abd733f7aeb2e5949e1bc370721ef2 Mon Sep 17 00:00:00 2001 From: John Nunemaker Date: Sun, 9 Aug 2026 20:41:27 -0400 Subject: [PATCH 08/15] fix(review): refresh completed poll snapshots --- lib/flipper/adapters/poll.rb | 14 ++++++- spec/flipper/adapters/poll_spec.rb | 61 +++++++++++++++++++++++++++++- 2 files changed, 71 insertions(+), 4 deletions(-) diff --git a/lib/flipper/adapters/poll.rb b/lib/flipper/adapters/poll.rb index 95d6ab12d..c4caad55e 100644 --- a/lib/flipper/adapters/poll.rb +++ b/lib/flipper/adapters/poll.rb @@ -70,15 +70,24 @@ def synced_adapter case claim when :claimed synced = false + completed_snapshot = nil begin Flipper::Adapters::Sync::Synchronizer.new( @adapter, @poller.adapter, local_get_all: value ).call + completed_snapshot = InFlightAdapter.new( + Flipper::Adapters::Memory.new(@adapter.get_all), + @adapter + ) synced = true ensure - synced ? complete_sync(state, poller_last_synced_at) : release_sync(state) + if synced + complete_sync(state, poller_last_synced_at, completed_snapshot) + else + release_sync(state) + end end @adapter when :syncing, :contended @@ -127,9 +136,10 @@ def claim_sync(state, poller_last_synced_at) end end - def complete_sync(state, poller_last_synced_at) + def complete_sync(state, poller_last_synced_at, completed_snapshot) state.mutex.synchronize do state.last_synced_at = poller_last_synced_at + state.snapshot = completed_snapshot state.syncing = false end end diff --git a/spec/flipper/adapters/poll_spec.rb b/spec/flipper/adapters/poll_spec.rb index d388a6496..547ca51bf 100644 --- a/spec/flipper/adapters/poll_spec.rb +++ b/spec/flipper/adapters/poll_spec.rb @@ -336,7 +336,64 @@ def sync loser&.join end - it "reads the local adapter once for a claimed poller update" do + it "retains the completed snapshot for contention during the next poll" do + get_all_entered = Queue.new + release_get_all = Queue.new + pausing_local_adapter = Class.new(Flipper::Adapters::Memory) do + def initialize(entered, release) + super(nil, threadsafe: true) + @entered = entered + @release = release + @pause_next_get_all = false + end + + def pause_next_get_all + @pause_next_get_all = true + end + + def get_all(**kwargs) + if @pause_next_get_all + @pause_next_get_all = false + @entered << true + @release.pop + end + super + end + end.new(get_all_entered, release_get_all) + Flipper.new(pausing_local_adapter).enable(:original) + + remote = Flipper::Adapters::Memory.new(threadsafe: true) + Flipper.new(remote).enable(:first_update) + fake_poller = Struct.new(:last_synced_at, :adapter) do + def start + end + + def sync + raise "sync should not be called when the local adapter is not empty" + end + end.new(Concurrent::AtomicFixnum.new(1), remote) + + instance = described_class.new(fake_poller, pausing_local_adapter) + expect(instance.features).to eq(Set["first_update"]) + + Flipper.new(remote).enable(:second_update) + fake_poller.last_synced_at.value = 2 + pausing_local_adapter.pause_next_get_all + + winner = Thread.new { instance.features } + get_all_entered.pop + contender = Thread.new { instance.features } + + expect(contender.value).to eq(Set["first_update"]) + release_get_all << true + expect(winner.value).to eq(Set["first_update", "second_update"]) + ensure + release_get_all << true if release_get_all + winner&.join + contender&.join + end + + it "reads the local adapter before and after a claimed poller update" do Flipper.new(local_adapter).enable(:existing) fake_poller = Struct.new(:last_synced_at, :adapter) do @@ -349,7 +406,7 @@ def sync end.new(Concurrent::AtomicFixnum.new(1), remote_adapter) instance = described_class.new(fake_poller, local_adapter) - expect(local_adapter).to receive(:get_all).once.and_call_original + expect(local_adapter).to receive(:get_all).twice.and_call_original instance.features end From 4c0cafe6f1ed1dcedc18dd9f4c51f20b94d2c432 Mon Sep 17 00:00:00 2001 From: John Nunemaker Date: Sun, 9 Aug 2026 20:50:33 -0400 Subject: [PATCH 09/15] fix(review): preserve trusted poll snapshots --- lib/flipper/adapters/poll.rb | 53 ++++++++++--- spec/flipper/adapters/poll_spec.rb | 117 +++++++++++++++++++++++++++++ 2 files changed, 160 insertions(+), 10 deletions(-) diff --git a/lib/flipper/adapters/poll.rb b/lib/flipper/adapters/poll.rb index c4caad55e..eae4d7294 100644 --- a/lib/flipper/adapters/poll.rb +++ b/lib/flipper/adapters/poll.rb @@ -9,7 +9,7 @@ class Poll extend Forwardable include ::Flipper::Adapter - SyncState = Struct.new(:pid, :mutex, :syncing, :last_synced_at, :snapshot) + SyncState = Struct.new(:pid, :mutex, :syncing, :last_synced_at, :snapshot, :sync_failed) class InFlightAdapter extend Forwardable @@ -23,6 +23,24 @@ def initialize(snapshot, adapter) end end + class PendingSnapshotAdapter + extend Forwardable + + def_delegators :read_adapter, :features, :get, :get_multi, :get_all + def_delegators :@adapter, :add, :remove, :clear, :enable, :disable + + def initialize(state, adapter) + @state = state + @adapter = adapter + end + + private + + def read_adapter + @state.snapshot || @adapter + end + end + # Deprecated Poller = ::Flipper::Poller @@ -54,7 +72,7 @@ def initialize(poller, adapter) # successful request establishes the snapshot before a sync can run. end @sync_state = Concurrent::AtomicReference.new( - SyncState.new(Process.pid, Mutex.new, false, 0, snapshot) + SyncState.new(Process.pid, Mutex.new, false, 0, snapshot, false) ) @poller.start @@ -90,7 +108,7 @@ def synced_adapter end end @adapter - when :syncing, :contended + when :syncing, :contended, :snapshot_established value else @adapter @@ -103,32 +121,45 @@ def sync_state state = @sync_state.get return state if state.pid == pid - replacement = SyncState.new(pid, Mutex.new, false, state.last_synced_at, state.snapshot) + replacement = SyncState.new( + pid, + Mutex.new, + false, + state.last_synced_at, + state.snapshot, + state.sync_failed + ) return replacement if @sync_state.compare_and_set(state, replacement) end end # Internal: Attempts to claim the right to sync. Returns the status and # the data associated with that status: the local state for :claimed or - # the latest coherent adapter for :syncing and :contended. Never blocks. + # the latest coherent adapter for all other read statuses. Never blocks. # Callers that lose an in-flight claim read from the snapshot rather than # waiting on a sync in the middle of a request. def claim_sync(state, poller_last_synced_at) - return [:contended, state.snapshot || @adapter] unless state.mutex.try_lock + unless state.mutex.try_lock + adapter = state.snapshot || PendingSnapshotAdapter.new(state, @adapter) + return [:contended, adapter] + end begin return [:syncing, state.snapshot] if state.syncing unless state.snapshot local_get_all = @adapter.get_all snapshot = Flipper::Adapters::Memory.new(local_get_all) - state.snapshot = InFlightAdapter.new(snapshot, @adapter) - return [:snapshot_established, nil] + established_snapshot = InFlightAdapter.new(snapshot, @adapter) + state.snapshot = established_snapshot + return [:snapshot_established, established_snapshot] end return [:not_needed, nil] unless poller_last_synced_at > state.last_synced_at local_get_all = @adapter.get_all - snapshot = Flipper::Adapters::Memory.new(local_get_all) - state.snapshot = InFlightAdapter.new(snapshot, @adapter) + unless state.sync_failed + snapshot = Flipper::Adapters::Memory.new(local_get_all) + state.snapshot = InFlightAdapter.new(snapshot, @adapter) + end state.syncing = true [:claimed, local_get_all] ensure @@ -140,12 +171,14 @@ def complete_sync(state, poller_last_synced_at, completed_snapshot) state.mutex.synchronize do state.last_synced_at = poller_last_synced_at state.snapshot = completed_snapshot + state.sync_failed = false state.syncing = false end end def release_sync(state) state.mutex.synchronize do + state.sync_failed = true state.syncing = false end end diff --git a/spec/flipper/adapters/poll_spec.rb b/spec/flipper/adapters/poll_spec.rb index 547ca51bf..15d862ca8 100644 --- a/spec/flipper/adapters/poll_spec.rb +++ b/spec/flipper/adapters/poll_spec.rb @@ -71,6 +71,57 @@ def sync expect(instance.features).to eq(Set["analytics", "search"]) end + it "serves the established snapshot when initialization recovery races with a sync" do + flaky_local_adapter = Class.new(Flipper::Adapters::Memory) do + def initialize + super + @get_all_calls = 0 + end + + def get_all(**kwargs) + @get_all_calls += 1 + raise "transient local failure" if @get_all_calls == 1 + + super + end + end.new + Flipper.new(flaky_local_adapter).enable(:existing) + + fake_poller = Struct.new(:last_synced_at, :adapter) do + def start + end + + def sync + raise "sync should not be called when the local adapter is not empty" + end + end.new(Concurrent::AtomicFixnum.new(1), remote_adapter) + + instance = described_class.new(fake_poller, flaky_local_adapter) + snapshot_established = Queue.new + release_recovery = Queue.new + allow(instance).to receive(:claim_sync).and_wrap_original do |method, *args| + result = method.call(*args) + if Thread.current[:poll_recovery] + snapshot_established << true + release_recovery.pop + end + result + end + + recovery = Thread.new do + Thread.current[:poll_recovery] = true + instance.features + end + snapshot_established.pop + + expect(instance.features).to eq(Set["analytics", "search"]) + release_recovery << true + expect(recovery.value).to eq(Set["existing"]) + ensure + release_recovery << true if release_recovery + recovery&.join + end + it "only synchronizes once per poller update when called concurrently" do flipper = Flipper.new(local_adapter) flipper.enable(:existing) @@ -481,6 +532,72 @@ def sync expect(get_all_calls.value).to eq(2) end + it "retains the last trusted snapshot while retrying a partially failed update" do + failing_local_adapter = Class.new(Flipper::Adapters::Memory) do + def initialize + super(nil, threadsafe: true) + @fail_next_disable = true + end + + def disable(feature, gate, thing) + result = super + if @fail_next_disable + @fail_next_disable = false + raise "partial local failure" + end + result + end + end.new + Flipper.new(failing_local_adapter).enable(:existing) + + remote = Flipper::Adapters::Memory.new(threadsafe: true) + Flipper.new(remote).disable(:existing) + Flipper.new(remote).enable(:updated) + retry_entered = Queue.new + release_retry = Queue.new + pausing_remote_adapter = Class.new do + def initialize(result, entered, release) + @result = result + @entered = entered + @release = release + @get_all_calls = 0 + end + + def get_all(**kwargs) + @get_all_calls += 1 + if @get_all_calls == 2 + @entered << true + @release.pop + end + @result + end + end.new(remote.get_all, retry_entered, release_retry) + + fake_poller = Struct.new(:last_synced_at, :adapter) do + def start + end + + def sync + raise "sync should not be called when the local adapter is not empty" + end + end.new(Concurrent::AtomicFixnum.new(1), pausing_remote_adapter) + + instance = described_class.new(fake_poller, failing_local_adapter) + expect { instance.features }.to raise_error("partial local failure") + expect(Flipper.new(failing_local_adapter).enabled?(:existing)).to be(false) + + retrying = Thread.new { instance.features } + retry_entered.pop + + expect(Flipper.new(instance).enabled?(:existing)).to be(true) + release_retry << true + expect(retrying.value).to eq(Set["existing", "updated"]) + expect(Flipper.new(instance).enabled?(:existing)).to be(false) + ensure + release_retry << true if release_retry + retrying&.join + end + it "resets in-flight synchronization state after a fork" do flipper = Flipper.new(local_adapter) flipper.enable(:existing) From 274489f9c494321307222507fdd4ba079f6bfe13 Mon Sep 17 00:00:00 2001 From: John Nunemaker Date: Sun, 9 Aug 2026 20:52:44 -0400 Subject: [PATCH 10/15] fix(review): tolerate poll snapshot capture failures --- lib/flipper/adapters/poll.rb | 18 ++++++++++------- spec/flipper/adapters/poll_spec.rb | 31 ++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 7 deletions(-) diff --git a/lib/flipper/adapters/poll.rb b/lib/flipper/adapters/poll.rb index eae4d7294..d6201c683 100644 --- a/lib/flipper/adapters/poll.rb +++ b/lib/flipper/adapters/poll.rb @@ -87,7 +87,6 @@ def synced_adapter claim, value = claim_sync(state, poller_last_synced_at) case claim when :claimed - synced = false completed_snapshot = nil begin Flipper::Adapters::Sync::Synchronizer.new( @@ -95,13 +94,18 @@ def synced_adapter @poller.adapter, local_get_all: value ).call - completed_snapshot = InFlightAdapter.new( - Flipper::Adapters::Memory.new(@adapter.get_all), - @adapter - ) - synced = true + begin + completed_snapshot = InFlightAdapter.new( + Flipper::Adapters::Memory.new(@adapter.get_all), + @adapter + ) + rescue + # The adapter is synchronized, but its completed state could not + # be captured. Keep serving the previous trusted snapshot to + # contenders and retry publication on the next request. + end ensure - if synced + if completed_snapshot complete_sync(state, poller_last_synced_at, completed_snapshot) else release_sync(state) diff --git a/spec/flipper/adapters/poll_spec.rb b/spec/flipper/adapters/poll_spec.rb index 15d862ca8..632371702 100644 --- a/spec/flipper/adapters/poll_spec.rb +++ b/spec/flipper/adapters/poll_spec.rb @@ -462,6 +462,37 @@ def sync instance.features end + it "does not fail a successful update when its completed snapshot cannot be captured" do + flaky_local_adapter = Class.new(Flipper::Adapters::Memory) do + def initialize + super + @get_all_calls = 0 + end + + def get_all(**kwargs) + @get_all_calls += 1 + raise "completed snapshot failure" if @get_all_calls == 3 + + super + end + end.new + Flipper.new(flaky_local_adapter).enable(:existing) + + fake_poller = Struct.new(:last_synced_at, :adapter) do + def start + end + + def sync + raise "sync should not be called when the local adapter is not empty" + end + end.new(Concurrent::AtomicFixnum.new(1), remote_adapter) + + instance = described_class.new(fake_poller, flaky_local_adapter) + + expect { instance.features }.not_to raise_error + expect(instance.features).to eq(Set["analytics", "search"]) + end + it "does not wait for the sync claim mutex" do Flipper.new(local_adapter).enable(:existing) From 1fdb3f3983a2c7dc1753f3b177619bbd4256d059 Mon Sep 17 00:00:00 2001 From: John Nunemaker Date: Sun, 9 Aug 2026 20:53:43 -0400 Subject: [PATCH 11/15] fix(review): distrust forked poll updates --- lib/flipper/adapters/poll.rb | 2 +- spec/flipper/adapters/poll_spec.rb | 49 ++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 1 deletion(-) diff --git a/lib/flipper/adapters/poll.rb b/lib/flipper/adapters/poll.rb index d6201c683..ae6d56d4d 100644 --- a/lib/flipper/adapters/poll.rb +++ b/lib/flipper/adapters/poll.rb @@ -131,7 +131,7 @@ def sync_state false, state.last_synced_at, state.snapshot, - state.sync_failed + state.sync_failed || state.syncing ) return replacement if @sync_state.compare_and_set(state, replacement) end diff --git a/spec/flipper/adapters/poll_spec.rb b/spec/flipper/adapters/poll_spec.rb index 632371702..d861e2332 100644 --- a/spec/flipper/adapters/poll_spec.rb +++ b/spec/flipper/adapters/poll_spec.rb @@ -668,4 +668,53 @@ def sync expect(threads.map(&:value)).to all(eq(Set["updated"])) expect(get_all_calls.value).to eq(1) end + + it "retains the trusted snapshot after forking during a partial update" do + Flipper.new(local_adapter).enable(:existing) + + remote = Flipper::Adapters::Memory.new(threadsafe: true) + Flipper.new(remote).disable(:existing) + Flipper.new(remote).enable(:updated) + retry_entered = Queue.new + release_retry = Queue.new + pausing_remote_adapter = Class.new do + def initialize(result, entered, release) + @result = result + @entered = entered + @release = release + end + + def get_all(**kwargs) + @entered << true + @release.pop + @result + end + end.new(remote.get_all, retry_entered, release_retry) + + fake_poller = Struct.new(:last_synced_at, :adapter) do + def start + end + + def sync + raise "sync should not be called when the local adapter is not empty" + end + end.new(Concurrent::AtomicFixnum.new(1), pausing_remote_adapter) + + instance = described_class.new(fake_poller, local_adapter) + stale_state = instance.instance_variable_get(:@sync_state).get + Flipper.new(local_adapter).disable(:existing) + stale_state.syncing = true + allow(Process).to receive(:pid).and_return(stale_state.pid + 1) + + retrying = Thread.new { instance.features } + retry_entered.pop + + expect(Flipper.new(instance).enabled?(:existing)).to be(true) + release_retry << true + expect(retrying.value).to eq(Set["existing", "updated"]) + expect(Flipper.new(instance).enabled?(:existing)).to be(false) + ensure + release_retry << true if release_retry + retrying&.join + end end From c93284bd549aae0f1acff3746b4449a1d56b16cb Mon Sep 17 00:00:00 2001 From: John Nunemaker Date: Wed, 12 Aug 2026 15:31:33 -0400 Subject: [PATCH 12/15] Fix fork-safe synchronization gates Retain stable Ruby mutexes across fork, reset only process-owned state under lock, and preserve nonblocking interval and trusted-snapshot behavior. Add real-fork regression coverage for Memory, Poller, Poll, and IntervalSynchronizer. --- lib/flipper/adapters/memory.rb | 11 - lib/flipper/adapters/poll.rb | 129 ++++--- .../adapters/sync/interval_synchronizer.rb | 58 ++- lib/flipper/poller.rb | 33 +- spec/flipper/adapters/memory_spec.rb | 66 ++++ spec/flipper/adapters/poll_spec.rb | 362 +++++++++++------- .../sync/interval_synchronizer_spec.rb | 117 +++++- spec/flipper/poller_spec.rb | 176 +++++++++ 8 files changed, 701 insertions(+), 251 deletions(-) diff --git a/lib/flipper/adapters/memory.rb b/lib/flipper/adapters/memory.rb index 67a312af9..3b267dad4 100644 --- a/lib/flipper/adapters/memory.rb +++ b/lib/flipper/adapters/memory.rb @@ -12,7 +12,6 @@ class Memory def initialize(source = nil, threadsafe: true) @source = Typecast.features_hash(source) @lock = Mutex.new if threadsafe - reset end # Public: The set of known features. @@ -122,18 +121,8 @@ def import(source) private - def reset - @pid = Process.pid - @lock&.unlock if @lock&.locked? - end - - def forked? - @pid != Process.pid - end - def synchronize(&block) if @lock - reset if forked? @lock.synchronize(&block) else block.call diff --git a/lib/flipper/adapters/poll.rb b/lib/flipper/adapters/poll.rb index ae6d56d4d..32aa07fdb 100644 --- a/lib/flipper/adapters/poll.rb +++ b/lib/flipper/adapters/poll.rb @@ -1,7 +1,6 @@ require 'flipper/adapters/sync/synchronizer' require 'flipper/adapters/memory' require 'flipper/poller' -require 'concurrent/atomic/atomic_reference' module Flipper module Adapters @@ -9,8 +8,6 @@ class Poll extend Forwardable include ::Flipper::Adapter - SyncState = Struct.new(:pid, :mutex, :syncing, :last_synced_at, :snapshot, :sync_failed) - class InFlightAdapter extend Forwardable @@ -29,15 +26,15 @@ class PendingSnapshotAdapter def_delegators :read_adapter, :features, :get, :get_multi, :get_all def_delegators :@adapter, :add, :remove, :clear, :enable, :disable - def initialize(state, adapter) - @state = state + def initialize(snapshot, adapter) + @snapshot = snapshot @adapter = adapter end private def read_adapter - @state.snapshot || @adapter + @snapshot.call || @adapter end end @@ -51,6 +48,11 @@ def read_adapter def initialize(poller, adapter) @adapter = adapter @poller = poller + @mutex = Mutex.new + @pid = Process.pid + @syncing = false + @last_synced_at = 0 + @sync_failed = false # If the adapter is empty, we need to sync before starting the poller. # Yes, this will block the main thread, but that's better than thinking # nothing is enabled. @@ -66,14 +68,12 @@ def initialize(poller, adapter) end snapshot = begin - InFlightAdapter.new(Flipper::Adapters::Memory.new(adapter.get_all), adapter) + build_snapshot(adapter.get_all) rescue # Preserve the existing fail-open initialization behavior. The first # successful request establishes the snapshot before a sync can run. end - @sync_state = Concurrent::AtomicReference.new( - SyncState.new(Process.pid, Mutex.new, false, 0, snapshot, false) - ) + @snapshot = snapshot @poller.start end @@ -81,10 +81,9 @@ def initialize(poller, adapter) private def synced_adapter - state = sync_state @poller.start poller_last_synced_at = @poller.last_synced_at.value - claim, value = claim_sync(state, poller_last_synced_at) + claim, value = claim_sync(poller_last_synced_at) case claim when :claimed completed_snapshot = nil @@ -95,10 +94,7 @@ def synced_adapter local_get_all: value ).call begin - completed_snapshot = InFlightAdapter.new( - Flipper::Adapters::Memory.new(@adapter.get_all), - @adapter - ) + completed_snapshot = build_snapshot(@adapter.get_all) rescue # The adapter is synchronized, but its completed state could not # be captured. Keep serving the previous trusted snapshot to @@ -106,9 +102,9 @@ def synced_adapter end ensure if completed_snapshot - complete_sync(state, poller_last_synced_at, completed_snapshot) + complete_sync(poller_last_synced_at, completed_snapshot) else - release_sync(state) + release_sync end end @adapter @@ -119,71 +115,84 @@ def synced_adapter end end - def sync_state - pid = Process.pid - loop do - state = @sync_state.get - return state if state.pid == pid - - replacement = SyncState.new( - pid, - Mutex.new, - false, - state.last_synced_at, - state.snapshot, - state.sync_failed || state.syncing - ) - return replacement if @sync_state.compare_and_set(state, replacement) - end - end - # Internal: Attempts to claim the right to sync. Returns the status and # the data associated with that status: the local state for :claimed or # the latest coherent adapter for all other read statuses. Never blocks. # Callers that lose an in-flight claim read from the snapshot rather than # waiting on a sync in the middle of a request. - def claim_sync(state, poller_last_synced_at) - unless state.mutex.try_lock - adapter = state.snapshot || PendingSnapshotAdapter.new(state, @adapter) + def claim_sync(poller_last_synced_at) + unless @mutex.try_lock + adapter = @snapshot || PendingSnapshotAdapter.new(-> { @snapshot }, @adapter) return [:contended, adapter] end begin - return [:syncing, state.snapshot] if state.syncing - unless state.snapshot + reset_if_forked + return [:syncing, @snapshot] if @syncing + unless @snapshot local_get_all = @adapter.get_all - snapshot = Flipper::Adapters::Memory.new(local_get_all) - established_snapshot = InFlightAdapter.new(snapshot, @adapter) - state.snapshot = established_snapshot + established_snapshot = build_snapshot(local_get_all) + @snapshot = established_snapshot return [:snapshot_established, established_snapshot] end - return [:not_needed, nil] unless poller_last_synced_at > state.last_synced_at + return [:not_needed, nil] unless poller_last_synced_at > @last_synced_at local_get_all = @adapter.get_all - unless state.sync_failed - snapshot = Flipper::Adapters::Memory.new(local_get_all) - state.snapshot = InFlightAdapter.new(snapshot, @adapter) + unless @sync_failed + @snapshot = build_snapshot(local_get_all) end - state.syncing = true + @syncing = true [:claimed, local_get_all] ensure - state.mutex.unlock + @mutex.unlock + end + end + + def complete_sync(poller_last_synced_at, completed_snapshot) + @mutex.synchronize do + @last_synced_at = poller_last_synced_at + @snapshot = completed_snapshot + @sync_failed = false + @syncing = false end end - def complete_sync(state, poller_last_synced_at, completed_snapshot) - state.mutex.synchronize do - state.last_synced_at = poller_last_synced_at - state.snapshot = completed_snapshot - state.sync_failed = false - state.syncing = false + def build_snapshot(local_get_all) + snapshot = Flipper::Adapters::Memory.new(snapshot_copy(local_get_all)) + InFlightAdapter.new(snapshot, @adapter) + end + + def snapshot_copy(value) + case value + when Hash + value.each_with_object({}) do |(key, nested_value), copy| + copy[key] = snapshot_copy(nested_value) + end + when Set + Set.new(value.map { |nested_value| snapshot_copy(nested_value) }) + when Array + value.map { |nested_value| snapshot_copy(nested_value) } + when String + value.dup + else + value end end - def release_sync(state) - state.mutex.synchronize do - state.sync_failed = true - state.syncing = false + def release_sync + @mutex.synchronize do + @sync_failed = true + @syncing = false + end + end + + def reset_if_forked + return if @pid == Process.pid + + @pid = Process.pid + if @syncing + @syncing = false + @sync_failed = true end end end diff --git a/lib/flipper/adapters/sync/interval_synchronizer.rb b/lib/flipper/adapters/sync/interval_synchronizer.rb index 752e650d6..f717a481c 100644 --- a/lib/flipper/adapters/sync/interval_synchronizer.rb +++ b/lib/flipper/adapters/sync/interval_synchronizer.rb @@ -1,13 +1,9 @@ -require 'concurrent/atomic/atomic_reference' - module Flipper module Adapters class Sync # Internal: Wraps a Synchronizer instance and only invokes it every # N seconds. class IntervalSynchronizer - SyncState = Struct.new(:pid, :mutex, :syncing, :last_sync_at) - # Private: Number of seconds between syncs (default: 10). DEFAULT_INTERVAL = 10 @@ -25,19 +21,19 @@ def initialize(synchronizer, interval: nil) @interval = interval || DEFAULT_INTERVAL # TODO: add jitter to this so all processes booting at the same time # don't phone home at the same time. - @sync_state = Concurrent::AtomicReference.new( - SyncState.new(Process.pid, Mutex.new, false, 0) - ) + @mutex = Mutex.new + @pid = Process.pid + @syncing = false + @last_sync_at = 0 end def call - state = sync_state - return unless sync_needed?(state) + return unless sync_needed? begin @synchronizer.call ensure - complete_sync(state) + complete_sync end nil @@ -45,40 +41,42 @@ def call private - def sync_state - pid = Process.pid - loop do - state = @sync_state.get - return state if state.pid == pid + def sync_needed? + return false unless @mutex.try_lock - replacement = SyncState.new(pid, Mutex.new, false, state.last_sync_at) - return replacement if @sync_state.compare_and_set(state, replacement) - end - end + begin + reset_if_forked + return false if @syncing - def sync_needed?(state) - state.mutex.synchronize do current_time = now - return false unless time_to_sync?(state, current_time) - return false if state.syncing + return false unless time_to_sync?(current_time) - state.last_sync_at = current_time - state.syncing = true + @last_sync_at = current_time + @syncing = true true + ensure + @mutex.unlock end end - def complete_sync(state) - state.mutex.synchronize do - state.syncing = false + def complete_sync + @mutex.synchronize do + @syncing = false end end - def time_to_sync?(state, current_time) - seconds_since_last_sync = current_time - state.last_sync_at + def time_to_sync?(current_time) + seconds_since_last_sync = current_time - @last_sync_at seconds_since_last_sync >= @interval end + def reset_if_forked + return if @pid == Process.pid + + @pid = Process.pid + @syncing = false + end + def now Process.clock_gettime(Process::CLOCK_MONOTONIC, :second) end diff --git a/lib/flipper/poller.rb b/lib/flipper/poller.rb index 69f48ddbc..6683378dd 100644 --- a/lib/flipper/poller.rb +++ b/lib/flipper/poller.rb @@ -2,11 +2,10 @@ require 'concurrent/utility/monotonic_time' require 'concurrent/map' require 'concurrent/atomic/atomic_fixnum' -require 'concurrent/atomic/atomic_boolean' module Flipper class Poller - attr_reader :adapter, :thread, :pid, :mutex, :interval, :last_synced_at + attr_reader :adapter, :thread, :interval, :last_synced_at def self.instances @instances ||= Concurrent::Map.new @@ -34,7 +33,7 @@ def initialize(options = {}) @remote_adapter = options.fetch(:remote_adapter) @last_synced_at = Concurrent::AtomicFixnum.new(0) @adapter = Adapters::Memory.new(nil, threadsafe: true) - @shutdown_requested = Concurrent::AtomicBoolean.new(false) + @shutdown_requested = false self.interval = options.fetch(:interval, 10) @initial_interval = @interval @@ -47,8 +46,6 @@ def initialize(options = {}) end def start - reset if forked? - return if @shutdown_requested.true? ensure_worker_running end @@ -102,19 +99,17 @@ def jitter rand end - def forked? - pid != Process.pid - end - def ensure_worker_running # Return early if thread is alive and avoid the mutex lock and unlock. return if thread_alive? # If another thread is starting worker thread, then return early so this # thread can enqueue and move on with life. - return unless mutex.try_lock + return unless @mutex.try_lock begin + reset_if_forked + return if @shutdown_requested return if thread_alive? @thread = Thread.new { run } @thread&.report_on_exception = false @@ -122,7 +117,7 @@ def ensure_worker_running operation: :thread_start, }) ensure - mutex.unlock + @mutex.unlock end end @@ -130,10 +125,18 @@ def thread_alive? @thread && @thread.alive? end - def reset + def reset_if_forked + return if @pid == Process.pid + @pid = Process.pid - @shutdown_requested.make_false - mutex.unlock if mutex.locked? + @shutdown_requested = false + end + + def request_shutdown + @mutex.synchronize do + reset_if_forked + @shutdown_requested = true + end end def apply_response_headers @@ -142,7 +145,7 @@ def apply_response_headers if response = @remote_adapter.last_get_all_response # shutdown based on response header if Flipper::Typecast.to_boolean(response["poll-shutdown"]) - @shutdown_requested.make_true + request_shutdown @instrumenter.instrument("poller.#{InstrumentationNamespace}", { operation: :shutdown_requested, }) diff --git a/spec/flipper/adapters/memory_spec.rb b/spec/flipper/adapters/memory_spec.rb index 5477bc490..cce5487b2 100644 --- a/spec/flipper/adapters/memory_spec.rb +++ b/spec/flipper/adapters/memory_spec.rb @@ -1,3 +1,6 @@ +require "open3" +require "rbconfig" + RSpec.describe Flipper::Adapters::Memory do let(:source) { {} } @@ -33,4 +36,67 @@ "following" => subject.default_config.merge(actors: Set["1", "3"], groups: Set["staff"]), }) end + + it "uses its inherited Mutex safely after a fork" do + skip "Process.fork is not supported" unless Process.respond_to?(:fork) + + script = <<~'RUBY' + require "flipper" + + adapter = Flipper::Adapters::Memory.new({}, threadsafe: true) + mutex = adapter.instance_variable_get(:@lock) + raise "memory adapter does not use Mutex" unless mutex.instance_of?(Mutex) + + locked = Queue.new + release = Queue.new + holder = Thread.new do + mutex.lock + locked << true + release.pop + mutex.unlock + end + locked.pop + + begin + child_pid = fork do + begin + adapter.features + exit! 0 + rescue => error + warn error.message + exit! 1 + end + end + + deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + 2 + status = nil + until status + if result = Process.wait2(child_pid, Process::WNOHANG) + _, status = result + elsif Process.clock_gettime(Process::CLOCK_MONOTONIC) >= deadline + Process.kill("KILL", child_pid) + Process.wait(child_pid) + raise "forked child timed out" + else + sleep 0.01 + end + end + ensure + release << true + holder.join(1) + end + + exit(status.success? ? 0 : 1) + RUBY + + _, stderr, status = Open3.capture3( + RbConfig.ruby, + "-Ilib", + "-e", + script, + chdir: File.expand_path("../../..", __dir__) + ) + + expect(status).to be_success, stderr + end end diff --git a/spec/flipper/adapters/poll_spec.rb b/spec/flipper/adapters/poll_spec.rb index d861e2332..53721ef23 100644 --- a/spec/flipper/adapters/poll_spec.rb +++ b/spec/flipper/adapters/poll_spec.rb @@ -1,6 +1,17 @@ require 'flipper/adapters/poll' +require 'open3' +require 'rbconfig' RSpec.describe Flipper::Adapters::Poll do + FakePoller = Struct.new(:last_synced_at, :adapter) do + def start + end + + def sync + raise "sync should not be called when the local adapter is not empty" + end + end + let(:remote_adapter) { adapter = Flipper::Adapters::Memory.new(threadsafe: true) flipper = Flipper.new(adapter) @@ -16,6 +27,10 @@ }) } + def build_poller(adapter) + FakePoller.new(Concurrent::AtomicFixnum.new(1), adapter) + end + it "syncs in main thread if local adapter is empty" do instance = described_class.new(poller, local_adapter) instance.features # call something to force sync @@ -55,14 +70,7 @@ def get_all(**kwargs) end.new Flipper.new(flaky_local_adapter).enable(:existing) - fake_poller = Struct.new(:last_synced_at, :adapter) do - def start - end - - def sync - raise "sync should not be called when the local adapter is not empty" - end - end.new(Concurrent::AtomicFixnum.new(1), remote_adapter) + fake_poller = build_poller(remote_adapter) instance = nil expect { instance = described_class.new(fake_poller, flaky_local_adapter) }.not_to raise_error @@ -87,14 +95,7 @@ def get_all(**kwargs) end.new Flipper.new(flaky_local_adapter).enable(:existing) - fake_poller = Struct.new(:last_synced_at, :adapter) do - def start - end - - def sync - raise "sync should not be called when the local adapter is not empty" - end - end.new(Concurrent::AtomicFixnum.new(1), remote_adapter) + fake_poller = build_poller(remote_adapter) instance = described_class.new(fake_poller, flaky_local_adapter) snapshot_established = Queue.new @@ -140,14 +141,7 @@ def get_all(**kwargs) end end.new(local_adapter.get_all, get_all_calls) - fake_poller = Struct.new(:last_synced_at, :adapter) do - def start - end - - def sync - raise "sync should not be called when the local adapter is not empty" - end - end.new(Concurrent::AtomicFixnum.new(1), slow_remote_adapter) + fake_poller = build_poller(slow_remote_adapter) instance = described_class.new(fake_poller, local_adapter) threads = 10.times.map { Thread.new { instance.features } } @@ -156,7 +150,7 @@ def sync expect(get_all_calls.value).to eq(1) end - it "does not wait for an in-flight poller update before returning the adapter" do + it "serves a coherent snapshot without waiting for an in-flight poller update" do flipper = Flipper.new(local_adapter) flipper.enable(:existing) @@ -179,14 +173,7 @@ def get_all(**kwargs) end end.new(remote.get_all, entered, release) - fake_poller = Struct.new(:last_synced_at, :adapter) do - def start - end - - def sync - raise "sync should not be called when the local adapter is not empty" - end - end.new(Concurrent::AtomicFixnum.new(1), slow_remote_adapter) + fake_poller = build_poller(slow_remote_adapter) instance = described_class.new(fake_poller, local_adapter) first_thread = Thread.new { instance.features } @@ -196,8 +183,8 @@ def sync second_thread = Thread.new { completed << instance.features } second_thread.join(1) - # The second thread serves the local adapter as is rather than blocking on - # the sync the first thread is running. + # The second thread reads the pre-sync snapshot without waiting for the + # first thread's sync to finish. expect(completed.pop(true)).to eq(Set["existing"]) release << true @@ -231,14 +218,7 @@ def disable(feature, gate, thing) Flipper.new(remote).disable(:existing) Flipper.new(remote).enable(:updated) - fake_poller = Struct.new(:last_synced_at, :adapter) do - def start - end - - def sync - raise "sync should not be called when the local adapter is not empty" - end - end.new(Concurrent::AtomicFixnum.new(1), remote) + fake_poller = build_poller(remote) instance = described_class.new(fake_poller, pausing_local_adapter) first_thread = Thread.new { instance.features } @@ -252,6 +232,49 @@ def sync expect(Flipper.new(instance).enabled?(:existing)).to be(false) end + it "keeps mutable gate values isolated in the trusted snapshot" do + entered = Queue.new + release = Queue.new + pausing_local_adapter = Class.new(Flipper::Adapters::Memory) do + def initialize(entered, release) + super(nil, threadsafe: true) + @entered = entered + @release = release + @pause = true + end + + def disable(feature, gate, thing) + result = super + if @pause && gate.data_type == :set + @pause = false + @entered << true + @release.pop + end + result + end + end.new(entered, release) + actor = Flipper::Actor.new("User;1") + Flipper.new(pausing_local_adapter).enable_actor(:search, actor) + + remote = Flipper::Adapters::Memory.new(threadsafe: true) + Flipper.new(remote).enable_percentage_of_actors(:search, 1) + fake_poller = build_poller(remote) + + instance = described_class.new(fake_poller, pausing_local_adapter) + winner = Thread.new { instance.features } + entered.pop + + expect(Flipper.new(pausing_local_adapter).enabled?(:search, actor)).to be(false) + expect(Flipper.new(instance).enabled?(:search, actor)).to be(true) + + release << true + expect(winner.value).to eq(Set["search"]) + expect(Flipper.new(instance).enabled?(:search, actor)).to be(false) + ensure + release << true if release + winner&.join + end + it "keeps the claimed snapshot after the poller update completes" do flipper = Flipper.new(local_adapter) flipper.enable(:existing) @@ -275,14 +298,7 @@ def get_all(**kwargs) end end.new(remote.get_all, sync_entered, release_sync) - fake_poller = Struct.new(:last_synced_at, :adapter) do - def start - end - - def sync - raise "sync should not be called when the local adapter is not empty" - end - end.new(Concurrent::AtomicFixnum.new(1), slow_remote_adapter) + fake_poller = build_poller(slow_remote_adapter) instance = described_class.new(fake_poller, local_adapter) loser_claimed = Queue.new @@ -344,14 +360,7 @@ def get_all(**kwargs) remote = Flipper::Adapters::Memory.new(threadsafe: true) Flipper.new(remote).enable(:updated) - fake_poller = Struct.new(:last_synced_at, :adapter) do - def start - end - - def sync - raise "sync should not be called when the local adapter is not empty" - end - end.new(Concurrent::AtomicFixnum.new(1), remote) + fake_poller = build_poller(remote) instance = described_class.new(fake_poller, pausing_local_adapter) pausing_local_adapter.pause_next_get_all @@ -415,14 +424,7 @@ def get_all(**kwargs) remote = Flipper::Adapters::Memory.new(threadsafe: true) Flipper.new(remote).enable(:first_update) - fake_poller = Struct.new(:last_synced_at, :adapter) do - def start - end - - def sync - raise "sync should not be called when the local adapter is not empty" - end - end.new(Concurrent::AtomicFixnum.new(1), remote) + fake_poller = build_poller(remote) instance = described_class.new(fake_poller, pausing_local_adapter) expect(instance.features).to eq(Set["first_update"]) @@ -447,14 +449,7 @@ def sync it "reads the local adapter before and after a claimed poller update" do Flipper.new(local_adapter).enable(:existing) - fake_poller = Struct.new(:last_synced_at, :adapter) do - def start - end - - def sync - raise "sync should not be called when the local adapter is not empty" - end - end.new(Concurrent::AtomicFixnum.new(1), remote_adapter) + fake_poller = build_poller(remote_adapter) instance = described_class.new(fake_poller, local_adapter) expect(local_adapter).to receive(:get_all).twice.and_call_original @@ -478,14 +473,7 @@ def get_all(**kwargs) end.new Flipper.new(flaky_local_adapter).enable(:existing) - fake_poller = Struct.new(:last_synced_at, :adapter) do - def start - end - - def sync - raise "sync should not be called when the local adapter is not empty" - end - end.new(Concurrent::AtomicFixnum.new(1), remote_adapter) + fake_poller = build_poller(remote_adapter) instance = described_class.new(fake_poller, flaky_local_adapter) @@ -496,24 +484,17 @@ def sync it "does not wait for the sync claim mutex" do Flipper.new(local_adapter).enable(:existing) - fake_poller = Struct.new(:last_synced_at, :adapter) do - def start - end - - def sync - raise "sync should not be called when the local adapter is not empty" - end - end.new(Concurrent::AtomicFixnum.new(1), remote_adapter) + fake_poller = build_poller(remote_adapter) instance = described_class.new(fake_poller, local_adapter) - state = instance.instance_variable_get(:@sync_state).get + mutex = instance.instance_variable_get(:@mutex) locked = Queue.new release = Queue.new holder = Thread.new do - state.mutex.lock + mutex.lock locked << true release.pop - state.mutex.unlock + mutex.unlock end locked.pop @@ -546,14 +527,7 @@ def get_all(**kwargs) end end.new(local_adapter.get_all, get_all_calls) - fake_poller = Struct.new(:last_synced_at, :adapter) do - def start - end - - def sync - raise "sync should not be called when the local adapter is not empty" - end - end.new(Concurrent::AtomicFixnum.new(1), flaky_remote_adapter) + fake_poller = build_poller(flaky_remote_adapter) instance = described_class.new(fake_poller, local_adapter) @@ -604,14 +578,7 @@ def get_all(**kwargs) end end.new(remote.get_all, retry_entered, release_retry) - fake_poller = Struct.new(:last_synced_at, :adapter) do - def start - end - - def sync - raise "sync should not be called when the local adapter is not empty" - end - end.new(Concurrent::AtomicFixnum.new(1), pausing_remote_adapter) + fake_poller = build_poller(pausing_remote_adapter) instance = described_class.new(fake_poller, failing_local_adapter) expect { instance.features }.to raise_error("partial local failure") @@ -649,24 +616,162 @@ def get_all(**kwargs) end end.new(remote.get_all, get_all_calls) - fake_poller = Struct.new(:last_synced_at, :adapter) do - def start - end - - def sync - raise "sync should not be called when the local adapter is not empty" - end - end.new(Concurrent::AtomicFixnum.new(1), counting_remote_adapter) + fake_poller = build_poller(counting_remote_adapter) instance = described_class.new(fake_poller, local_adapter) - stale_state = instance.instance_variable_get(:@sync_state).get - stale_state.syncing = true + mutex = instance.instance_variable_get(:@mutex) + parent_pid = instance.instance_variable_get(:@pid) + instance.instance_variable_set(:@syncing, true) - allow(Process).to receive(:pid).and_return(stale_state.pid + 1) + allow(Process).to receive(:pid).and_return(parent_pid + 1) threads = 10.times.map { Thread.new { instance.features } } expect(threads.map(&:value)).to all(eq(Set["updated"])) expect(get_all_calls.value).to eq(1) + expect(instance.instance_variable_get(:@pid)).to eq(parent_pid + 1) + expect(instance.instance_variable_get(:@mutex)).to equal(mutex) + end + + it "keeps its mutex and trusted snapshot in a real forked child" do + skip "Process.fork is not supported" unless Process.respond_to?(:fork) + + script = <<~'RUBY' + require "flipper" + require "flipper/adapters/poll" + + def wait_for(queue, timeout: 2) + deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + timeout + loop do + return queue.pop(true) + rescue ThreadError + raise "queue wait timed out" if Process.clock_gettime(Process::CLOCK_MONOTONIC) >= deadline + Thread.pass + end + end + + def join_thread(thread, timeout: 2) + deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + timeout + until thread.join(0.01) + raise "thread join timed out" if Process.clock_gettime(Process::CLOCK_MONOTONIC) >= deadline + end + end + + Poller = Struct.new(:last_synced_at, :adapter) do + def start + end + + def sync + raise "unexpected bootstrap sync" + end + end + + entered = Queue.new + release_sync = Queue.new + remote_calls = 0 + remote = Class.new do + def initialize(result, entered, release_sync, calls) + @result = result + @entered = entered + @release_sync = release_sync + @calls = calls + end + + def get_all(**kwargs) + @calls[0] += 1 + @entered << true + @release_sync.pop + @result + end + end + + local_adapter = Flipper::Adapters::Memory.new(threadsafe: true) + Flipper.new(local_adapter).enable(:existing) + remote_adapter = Flipper::Adapters::Memory.new(threadsafe: true) + Flipper.new(remote_adapter).enable(:updated) + calls = [remote_calls] + poller = Poller.new( + Concurrent::AtomicFixnum.new(1), + remote.new(remote_adapter.get_all, entered, release_sync, calls) + ) + instance = Flipper::Adapters::Poll.new(poller, local_adapter) + mutex = instance.instance_variable_get(:@mutex) + raise "poll does not use a stable Mutex" unless mutex.instance_of?(Mutex) + + Flipper.new(local_adapter).disable(:existing) + instance.instance_variable_set(:@syncing, true) + instance.instance_variable_set(:@sync_failed, false) + + mutex_locked = Queue.new + release_mutex = Queue.new + holder = Thread.new do + mutex.lock + mutex_locked << true + release_mutex.pop + mutex.unlock + end + wait_for(mutex_locked) + + begin + child_pid = fork do + success = false + winner = nil + losers = [] + begin + winner = Thread.new { instance.features } + wait_for(entered) + losers = 8.times.map { Thread.new { instance.features } } + losers.each { |thread| join_thread(thread) } + + raise "inherited mutex was replaced" unless instance.instance_variable_get(:@mutex).equal?(mutex) + raise "loser did not receive trusted snapshot" unless losers.map(&:value).all? { |features| features == Set["existing"] } + raise "duplicate synchronization" unless calls[0] == 1 + + release_sync << true + join_thread(winner) + raise "winner did not publish synchronized state" unless winner.value == Set["updated"] + raise "inherited syncing was not cleared" if instance.instance_variable_get(:@syncing) + raise "successful publication stayed failed" if instance.instance_variable_get(:@sync_failed) + success = true + rescue => error + warn error.full_message + ensure + release_sync << true + winner&.join(1) + losers.each { |thread| thread.join(1) } + end + exit!(success ? 0 : 1) + end + + deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + 5 + status = nil + until status + if result = Process.wait2(child_pid, Process::WNOHANG) + _, status = result + elsif Process.clock_gettime(Process::CLOCK_MONOTONIC) >= deadline + Process.kill("KILL", child_pid) + Process.wait(child_pid) + raise "forked child timed out" + else + sleep 0.01 + end + end + ensure + release_mutex << true + holder.join(1) + end + + exit(status.success? ? 0 : 1) + RUBY + + _, stderr, status = Open3.capture3( + RbConfig.ruby, + "-Ilib", + "-e", + script, + chdir: File.expand_path("../../..", __dir__) + ) + + expect(status).to be_success, stderr end it "retains the trusted snapshot after forking during a partial update" do @@ -691,20 +796,13 @@ def get_all(**kwargs) end end.new(remote.get_all, retry_entered, release_retry) - fake_poller = Struct.new(:last_synced_at, :adapter) do - def start - end - - def sync - raise "sync should not be called when the local adapter is not empty" - end - end.new(Concurrent::AtomicFixnum.new(1), pausing_remote_adapter) + fake_poller = build_poller(pausing_remote_adapter) instance = described_class.new(fake_poller, local_adapter) - stale_state = instance.instance_variable_get(:@sync_state).get + parent_pid = instance.instance_variable_get(:@pid) Flipper.new(local_adapter).disable(:existing) - stale_state.syncing = true - allow(Process).to receive(:pid).and_return(stale_state.pid + 1) + instance.instance_variable_set(:@syncing, true) + allow(Process).to receive(:pid).and_return(parent_pid + 1) retrying = Thread.new { instance.features } retry_entered.pop diff --git a/spec/flipper/adapters/sync/interval_synchronizer_spec.rb b/spec/flipper/adapters/sync/interval_synchronizer_spec.rb index ff2bc5d71..50ba3d26d 100644 --- a/spec/flipper/adapters/sync/interval_synchronizer_spec.rb +++ b/spec/flipper/adapters/sync/interval_synchronizer_spec.rb @@ -1,4 +1,6 @@ require "flipper/adapters/sync/interval_synchronizer" +require "open3" +require "rbconfig" RSpec.describe Flipper::Adapters::Sync::IntervalSynchronizer do let(:events) { [] } @@ -113,11 +115,12 @@ release.pop end instance = described_class.new(synchronizer, interval: interval) - stale_state = instance.instance_variable_get(:@sync_state).get - stale_state.syncing = true + mutex = instance.instance_variable_get(:@mutex) + parent_pid = instance.instance_variable_get(:@pid) + instance.instance_variable_set(:@syncing, true) allow(instance).to receive(:now).and_return(interval) - allow(Process).to receive(:pid).and_return(stale_state.pid + 1) + allow(Process).to receive(:pid).and_return(parent_pid + 1) threads = 10.times.map { Thread.new { instance.call } } entered.pop @@ -125,5 +128,113 @@ threads.each(&:join) expect(events.size).to eq(1) + expect(instance.instance_variable_get(:@pid)).to eq(parent_pid + 1) + expect(instance.instance_variable_get(:@mutex)).to equal(mutex) + end + + it "keeps its mutex and elects one winner in a real forked child" do + skip "Process.fork is not supported" unless Process.respond_to?(:fork) + + script = <<~'RUBY' + require "flipper/adapters/sync/interval_synchronizer" + + def wait_for(queue, timeout: 2) + deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + timeout + loop do + return queue.pop(true) + rescue ThreadError + raise "queue wait timed out" if Process.clock_gettime(Process::CLOCK_MONOTONIC) >= deadline + Thread.pass + end + end + + def join_thread(thread, timeout: 2) + deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + timeout + until thread.join(0.01) + raise "thread join timed out" if Process.clock_gettime(Process::CLOCK_MONOTONIC) >= deadline + end + end + + entered = Queue.new + release_sync = Queue.new + calls = [0] + synchronizer = lambda do + calls[0] += 1 + entered << true + release_sync.pop + end + instance = Flipper::Adapters::Sync::IntervalSynchronizer.new(synchronizer, interval: 10) + mutex = instance.instance_variable_get(:@mutex) + raise "interval synchronizer does not use a stable Mutex" unless mutex.instance_of?(Mutex) + instance.instance_variable_set(:@syncing, true) + + mutex_locked = Queue.new + release_mutex = Queue.new + holder = Thread.new do + mutex.lock + mutex_locked << true + release_mutex.pop + mutex.unlock + end + wait_for(mutex_locked) + + begin + child_pid = fork do + success = false + winner = nil + losers = [] + begin + winner = Thread.new { instance.call } + wait_for(entered) + losers = 8.times.map { Thread.new { instance.call } } + losers.each { |thread| join_thread(thread) } + + raise "inherited mutex was replaced" unless instance.instance_variable_get(:@mutex).equal?(mutex) + raise "duplicate synchronization" unless calls[0] == 1 + + release_sync << true + join_thread(winner) + raise "inherited syncing was not cleared" if instance.instance_variable_get(:@syncing) + success = true + rescue => error + warn error.full_message + ensure + release_sync << true + winner&.join(1) + losers.each { |thread| thread.join(1) } + end + exit!(success ? 0 : 1) + end + + deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + 5 + status = nil + until status + if result = Process.wait2(child_pid, Process::WNOHANG) + _, status = result + elsif Process.clock_gettime(Process::CLOCK_MONOTONIC) >= deadline + Process.kill("KILL", child_pid) + Process.wait(child_pid) + raise "forked child timed out" + else + sleep 0.01 + end + end + ensure + release_mutex << true + holder.join(1) + end + + exit(status.success? ? 0 : 1) + RUBY + + _, stderr, status = Open3.capture3( + RbConfig.ruby, + "-Ilib", + "-e", + script, + chdir: File.expand_path("../../../..", __dir__) + ) + + expect(status).to be_success, stderr end end diff --git a/spec/flipper/poller_spec.rb b/spec/flipper/poller_spec.rb index 44843647f..6d0acefc1 100644 --- a/spec/flipper/poller_spec.rb +++ b/spec/flipper/poller_spec.rb @@ -1,5 +1,8 @@ require "flipper/poller" require "flipper/adapters/http" +require "open3" +require "rbconfig" +require "timeout" RSpec.describe Flipper::Poller do let(:url) { "http://app.com/flipper" } @@ -356,6 +359,27 @@ subject.start end + it "keeps using its Mutex after a fork" do + mutex = subject.instance_variable_get(:@mutex) + allow(Process).to receive(:pid).and_return(Process.pid + 1) + + expect(mutex).to be_instance_of(Mutex) + expect { subject.start }.not_to raise_error + expect(subject.instance_variable_get(:@mutex)).to equal(mutex) + end + + it "updates its process state after a fork" do + parent_pid = subject.instance_variable_get(:@pid) + allow(Process).to receive(:pid).and_return(parent_pid + 1) + + subject.start + expect(subject.instance_variable_get(:@pid)).to eq(parent_pid + 1) + end + + it "does not expose the raw mutex" do + expect(subject).not_to respond_to(:mutex) + end + context "after shutdown_requested" do before do stub_request(:get, "#{url}/features?exclude_gate_names=true") @@ -385,6 +409,158 @@ expect(subject).to receive(:sync) subject.start end + + it "serializes a shutdown request with fork reset" do + allow(Thread).to receive(:new).and_call_original + allow(Process).to receive(:pid).and_return(Process.pid + 1) + mutex = subject.instance_variable_get(:@mutex) + started = Queue.new + mutex.lock + + thread = Thread.new do + started << true + subject.send(:request_shutdown) + end + started.pop + + Timeout.timeout(2) do + Thread.pass until thread.status == "sleep" + end + expect(thread).to be_alive + + mutex.unlock + thread.join(1) + + expect(subject.instance_variable_get(:@pid)).to eq(Process.pid) + expect(subject.instance_variable_get(:@shutdown_requested)).to be(true) + ensure + mutex&.unlock if mutex&.owned? + thread&.join(1) + end + + it "starts one fresh worker in a real forked child" do + skip "Process.fork is not supported" unless Process.respond_to?(:fork) + allow(Thread).to receive(:new).and_call_original + + script = <<~'RUBY' + require "flipper" + + class ForkTestPoller < Flipper::Poller + def run + sleep + end + + def pause_next_start(acquired, release) + @start_acquired = acquired + @release_start = release + end + + private + + def reset_if_forked + super + return unless start_acquired = @start_acquired + + @start_acquired = nil + start_acquired << true + @release_start.pop + end + end + + poller = ForkTestPoller.new( + remote_adapter: Object.new, + start_automatically: false, + shutdown_automatically: false + ) + poller.start + parent_worker = poller.thread + poller.send(:request_shutdown) + inherited_mutex = poller.instance_variable_get(:@mutex) + raise "poller does not use Mutex" unless inherited_mutex.instance_of?(Mutex) + mutex_locked = Queue.new + release_mutex = Queue.new + mutex_holder = Thread.new do + inherited_mutex.lock + mutex_locked << true + release_mutex.pop + inherited_mutex.unlock + end + mutex_locked.pop + + begin + child_pid = fork do + success = false + release_start = nil + first_start = nil + second_start = nil + + begin + inherited_worker = poller.thread + raise "inherited worker is unexpectedly alive" if inherited_worker.alive? + + start_acquired = Queue.new + release_start = Queue.new + poller.pause_next_start(start_acquired, release_start) + first_start = Thread.new { poller.start } + start_acquired.pop + + second_start = Thread.new { poller.start } + raise "competing start did not return" unless second_start.join(1) + + release_start << true + raise "first start did not return" unless first_start.join(1) + + child_worker = poller.thread + raise "inherited mutex was replaced" unless poller.instance_variable_get(:@mutex).equal?(inherited_mutex) + raise "worker was not replaced" if child_worker.equal?(inherited_worker) + raise "replacement worker is not alive" unless child_worker.alive? + success = true + rescue => error + warn error.message + ensure + release_start << true if release_start + first_start&.join(1) + second_start&.join(1) + poller.stop + poller.thread&.join(1) + end + + exit!(success ? 0 : 1) + end + + deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + 5 + status = nil + until status + if result = Process.wait2(child_pid, Process::WNOHANG) + _, status = result + elsif Process.clock_gettime(Process::CLOCK_MONOTONIC) >= deadline + Process.kill("KILL", child_pid) + Process.wait(child_pid) + raise "forked child timed out" + else + sleep 0.01 + end + end + ensure + release_mutex << true + mutex_holder.join(1) + poller.stop + parent_worker.join(1) + end + + exit(status.success? ? 0 : 1) + RUBY + + _, stderr, status = Open3.capture3( + RbConfig.ruby, + "-Ilib", + "-e", + script, + chdir: File.expand_path("../..", __dir__) + ) + + expect(status).to be_success, stderr + end end end end From a4c2e56d6a9fec648bc40409c5fc82203e6f4218 Mon Sep 17 00:00:00 2001 From: John Nunemaker Date: Wed, 12 Aug 2026 17:37:32 -0400 Subject: [PATCH 13/15] fix(review): harden interval concurrency specs --- .../sync/interval_synchronizer_spec.rb | 45 ++++++++++++++++--- 1 file changed, 39 insertions(+), 6 deletions(-) diff --git a/spec/flipper/adapters/sync/interval_synchronizer_spec.rb b/spec/flipper/adapters/sync/interval_synchronizer_spec.rb index 50ba3d26d..726c5e0e5 100644 --- a/spec/flipper/adapters/sync/interval_synchronizer_spec.rb +++ b/spec/flipper/adapters/sync/interval_synchronizer_spec.rb @@ -3,6 +3,23 @@ require "rbconfig" RSpec.describe Flipper::Adapters::Sync::IntervalSynchronizer do + def wait_for(queue, timeout: 2) + deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + timeout + loop do + return queue.pop(true) + rescue ThreadError + raise "queue wait timed out" if Process.clock_gettime(Process::CLOCK_MONOTONIC) >= deadline + Thread.pass + end + end + + def join_thread(thread, timeout: 2) + deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + timeout + until thread.join(0.01) + raise "thread join timed out" if Process.clock_gettime(Process::CLOCK_MONOTONIC) >= deadline + end + end + let(:events) { [] } let(:synchronizer) { -> { events << now } } let(:interval) { 10 } @@ -48,15 +65,24 @@ first_thread = Thread.new { instance.call } entered.pop - threads = 10.times.map { Thread.new { instance.call } } - sleep 0.05 + completed = Queue.new + threads = 10.times.map do + Thread.new do + instance.call + completed << true + end + end + threads.size.times { wait_for(completed) } expect(events.size).to eq(1) release << true - ([first_thread] + threads).each(&:join) + ([first_thread] + threads).each { |thread| join_thread(thread) } expect(events.size).to eq(1) + ensure + 11.times { release << true } if release + ([first_thread] + Array(threads)).compact.each { |thread| thread.join(1) } end it "does not synchronize again when the interval passes during an in-flight sync" do @@ -76,15 +102,22 @@ entered.pop current_time += interval - second_thread = Thread.new { instance.call } - sleep 0.05 + completed = Queue.new + second_thread = Thread.new do + instance.call + completed << true + end + wait_for(completed) expect(events.size).to eq(1) release << true - [first_thread, second_thread].each(&:join) + [first_thread, second_thread].each { |thread| join_thread(thread) } expect(events.size).to eq(1) + ensure + 2.times { release << true } if release + [first_thread, second_thread].compact.each { |thread| thread.join(1) } end it "releases a failed sync for the next interval" do From 691ac782503ba5e0adc9e330937fb48a03aa3880 Mon Sep 17 00:00:00 2001 From: John Nunemaker Date: Wed, 12 Aug 2026 17:54:58 -0400 Subject: [PATCH 14/15] Test concurrent poll synchronization behavior --- spec/flipper/adapters/poll_spec.rb | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/spec/flipper/adapters/poll_spec.rb b/spec/flipper/adapters/poll_spec.rb index 53721ef23..dd414acd0 100644 --- a/spec/flipper/adapters/poll_spec.rb +++ b/spec/flipper/adapters/poll_spec.rb @@ -603,18 +603,24 @@ def get_all(**kwargs) remote = Flipper::Adapters::Memory.new(threadsafe: true) Flipper.new(remote).enable(:updated) + sync_entered = Queue.new + release_sync = Queue.new get_all_calls = Concurrent::AtomicFixnum.new(0) counting_remote_adapter = Class.new do - def initialize(result, get_all_calls) + def initialize(result, get_all_calls, sync_entered, release_sync) @result = result @get_all_calls = get_all_calls + @sync_entered = sync_entered + @release_sync = release_sync end def get_all(**kwargs) @get_all_calls.increment + @sync_entered << true + @release_sync.pop @result end - end.new(remote.get_all, get_all_calls) + end.new(remote.get_all, get_all_calls, sync_entered, release_sync) fake_poller = build_poller(counting_remote_adapter) @@ -625,11 +631,22 @@ def get_all(**kwargs) allow(Process).to receive(:pid).and_return(parent_pid + 1) - threads = 10.times.map { Thread.new { instance.features } } - expect(threads.map(&:value)).to all(eq(Set["updated"])) + winner = Thread.new { instance.features } + sync_entered.pop + + losers = 9.times.map { Thread.new { instance.features } } + losers.each { |thread| expect(thread.join(1)).to equal(thread) } + expect(losers.map(&:value)).to all(eq(Set["existing"])) expect(get_all_calls.value).to eq(1) + + release_sync << true + expect(winner.value).to eq(Set["updated"]) expect(instance.instance_variable_get(:@pid)).to eq(parent_pid + 1) expect(instance.instance_variable_get(:@mutex)).to equal(mutex) + ensure + release_sync << true if release_sync + winner&.join(1) + losers&.each { |thread| thread.join(1) } end it "keeps its mutex and trusted snapshot in a real forked child" do From fccbb4781e594264d3f38b7d6175888fe702f3c0 Mon Sep 17 00:00:00 2001 From: John Nunemaker Date: Thu, 13 Aug 2026 11:58:59 -0400 Subject: [PATCH 15/15] Re-run CI after main sync