diff --git a/Engine/TransactionHandlers/BacktestingTransactionHandler.cs b/Engine/TransactionHandlers/BacktestingTransactionHandler.cs index 8d26ff193315..411c49833e08 100644 --- a/Engine/TransactionHandlers/BacktestingTransactionHandler.cs +++ b/Engine/TransactionHandlers/BacktestingTransactionHandler.cs @@ -33,7 +33,6 @@ public class BacktestingTransactionHandler : BrokerageTransactionHandler private BacktestingBrokerage _brokerage; private IAlgorithm _algorithm; private Delistings _lastestDelistings; - private bool _enableConcurrency; /// /// Gets current time UTC. This is here to facilitate testing @@ -55,26 +54,25 @@ public override void Initialize(IAlgorithm algorithm, IBrokerage brokerage, IRes _brokerage = (BacktestingBrokerage)brokerage; _algorithm = algorithm; - _enableConcurrency = _brokerage.ConcurrencyEnabled && _algorithm.LiveMode; base.Initialize(algorithm, brokerage, resultHandler); - - if (!_enableConcurrency) - { - // non blocking implementation - _orderRequestQueues = new() { new BusyCollection() }; - } } + /// + /// For backtesting order requests are processed synchronously by the algorithm thread, only live + /// deployments with a concurrency enabled brokerage use background transaction threads + /// + protected override bool SynchronousProcessing => !(ConcurrencyEnabled && _algorithm.LiveMode); + /// /// Processes all synchronous events that must take place before the next time loop for the algorithm /// public override void ProcessSynchronousEvents() { - if (!_enableConcurrency) + if (SynchronousProcessing) { // we process pending order requests our selves - Run(0); + ProcessPendingRequests(); } base.ProcessSynchronousEvents(); @@ -105,7 +103,7 @@ public override void ProcessAsynchronousEvents() /// The expecting to be submitted protected override void WaitForOrderSubmission(OrderTicket ticket) { - if (_enableConcurrency) + if (!SynchronousProcessing) { // let the base class handle this base.WaitForOrderSubmission(ticket); @@ -113,7 +111,7 @@ protected override void WaitForOrderSubmission(OrderTicket ticket) } // we submit the order request our selves - Run(0); + ProcessPendingRequests(); if (!ticket.OrderSet.WaitOne(0)) { @@ -124,18 +122,5 @@ protected override void WaitForOrderSubmission(OrderTicket ticket) "See the OrderRequest.Response for more information"); } } - - /// - /// For backtesting order requests will be processed by the algorithm thread - /// sequentially at and - /// - protected override void InitializeTransactionThread() - { - if (_enableConcurrency) - { - // let the base class handle this - base.InitializeTransactionThread(); - } - } } } diff --git a/Engine/TransactionHandlers/BrokerageTransactionHandler.cs b/Engine/TransactionHandlers/BrokerageTransactionHandler.cs index 53bf641716c7..08e84ecf1178 100644 --- a/Engine/TransactionHandlers/BrokerageTransactionHandler.cs +++ b/Engine/TransactionHandlers/BrokerageTransactionHandler.cs @@ -70,13 +70,10 @@ public class BrokerageTransactionHandler : ITransactionHandler private int _failedCashSyncAttempts; /// - /// OrderQueue holds the newly updated orders from the user algorithm waiting to be processed. Once - /// orders are processed they are moved into the Orders queue awaiting the brokerage response. + /// Runs order requests on worker threads that pull from a single shared queue, keeping each order's + /// requests in order while growing the pool on demand as the threads get saturated. /// - protected List> _orderRequestQueues { get; set; } - - private List _processingThreads; - private readonly CancellationTokenSource _cancellationTokenSource = new CancellationTokenSource(); + private OrderRequestProcessingPool _threadPool; private readonly ConcurrentQueue _orderEvents = new ConcurrentQueue(); @@ -210,8 +207,6 @@ public virtual void Initialize(IAlgorithm algorithm, IBrokerage brokerage, IResu HandleOrderUpdated(e); }; - IsActive = true; - if (_algorithm is QCAlgorithm qcAlgorithm) { _qcAlgorithmInstance = qcAlgorithm; @@ -230,35 +225,58 @@ public virtual void Initialize(IAlgorithm algorithm, IBrokerage brokerage, IResu InitializeTransactionThread(); } + /// + /// Whether the transaction thread pool can grow on demand to process order requests concurrently. + /// When false a single worker thread is used. + /// + protected virtual bool ConcurrencyEnabled => _brokerage.ConcurrencyEnabled; + + /// + /// Whether order requests are drained synchronously by the algorithm thread instead of by background + /// worker threads. Used by backtesting deployments. + /// + protected virtual bool SynchronousProcessing => false; + + /// + /// The maximum number of transaction threads the pool can grow to + /// + protected virtual int MaximumTransactionThreads => Config.GetInt("maximum-transaction-threads", 10); + + /// + /// The number of transaction threads the pool starts with + /// + protected virtual int MinimumTransactionThreads => Config.GetInt("minimum-transaction-threads", 2); + + /// + /// The number of transaction threads currently running + /// + protected int ProcessingThreadsCount => _threadPool?.ThreadCount ?? 0; + + /// + /// Boolean flag indicating the transaction threads are busy. + /// False indicates they are completely finished processing and ready to be terminated. + /// + public bool IsActive => _threadPool?.IsActive ?? false; + /// /// Create and start the transaction thread, who will be in charge of processing /// the order requests /// protected virtual void InitializeTransactionThread() { - // multi threaded queue, used for live deployments - var processingThreadsCount = _brokerage.ConcurrencyEnabled - ? Config.GetInt("maximum-transaction-threads", 4) - : 1; - _orderRequestQueues = new(processingThreadsCount); - _processingThreads = new(processingThreadsCount); - for (var i = 0; i < processingThreadsCount; i++) + Action processRequest = request => { - _orderRequestQueues.Add(new BusyBlockingCollection()); - var threadId = i; // avoid modified closure - _processingThreads.Add(new Thread(() => Run(threadId)) { IsBackground = true, Name = $"Transaction Thread {i}" }); - } - foreach (var thread in _processingThreads) - { - thread.Start(); - } - } + HandleOrderRequest(request); + ProcessAsynchronousEvents(); + }; + Action onError = error => _algorithm.SetRuntimeError(error, "HandleOrderRequest"); - /// - /// Boolean flag indicating the Run thread method is busy. - /// False indicates it is completely finished processing and ready to be terminated. - /// - public bool IsActive { get; private set; } + // backtesting drains a single queue synchronously on the algorithm thread, live deployments use + // background worker threads: a single one, or growing on demand up to the maximum when concurrent. + _threadPool = SynchronousProcessing + ? OrderRequestProcessingPool.Synchronous(processRequest, onError) + : new OrderRequestProcessingPool(ConcurrencyEnabled, MinimumTransactionThreads, MaximumTransactionThreads, processRequest, onError); + } #region Order Request Processing @@ -338,7 +356,7 @@ public OrderTicket AddOrder(SubmitOrderRequest request) order.OrderSubmissionData = new OrderSubmissionData(security.BidPrice, security.AskPrice, security.Close); _openOrders[order.Id] = new OpenOrderState(order, ticket, security); - EnqueueOrderRequest(request, order); + _threadPool.Dispatch(request, order); WaitForOrderSubmission(ticket); } @@ -366,7 +384,7 @@ public OrderTicket AddOrder(SubmitOrderRequest request) } /// - /// Wait for the order to be handled by the + /// Wait for the order to be handled by the /// /// The expecting to be submitted protected virtual void WaitForOrderSubmission(OrderTicket ticket) @@ -454,7 +472,7 @@ public OrderTicket UpdateOrder(UpdateOrderRequest request) else { request.SetResponse(OrderResponse.Success(request), OrderRequestStatus.Processing); - EnqueueOrderRequest(request, order); + _threadPool.Dispatch(request, order); } } catch (Exception err) @@ -526,7 +544,7 @@ public OrderTicket CancelOrder(CancelOrderRequest request) // send the request to be processed request.SetResponse(OrderResponse.Success(request), OrderRequestStatus.Processing); - EnqueueOrderRequest(request, order); + _threadPool.Dispatch(request, order); } } catch (Exception err) @@ -674,29 +692,12 @@ public List GetOpenOrders(Func filter = null) } /// - /// Primary thread entry point to launch the transaction thread. + /// Drains the pending order requests on the calling thread. Used by synchronous (non concurrent) + /// deployments, where the algorithm thread pumps the request queue itself. /// - protected void Run(int threadId) + protected void ProcessPendingRequests() { - try - { - foreach (var request in _orderRequestQueues[threadId].GetConsumingEnumerable(_cancellationTokenSource.Token)) - { - HandleOrderRequest(request); - ProcessAsynchronousEvents(); - } - } - catch (Exception err) - { - // unexpected error, we need to close down shop - _algorithm.SetRuntimeError(err, "HandleOrderRequest"); - } - - if (_processingThreads != null) - { - Log.Trace($"BrokerageTransactionHandler.Run(): Ending Thread {threadId}..."); - IsActive = false; - } + _threadPool.ProcessPending(); } /// @@ -717,7 +718,7 @@ public virtual void ProcessSynchronousEvents() // in backtesting we need to wait for orders to be removed from the queue and finished processing if (!_algorithm.LiveMode) { - if (_orderRequestQueues.Any(queue => queue.IsBusy && !queue.WaitHandle.WaitOne(Time.OneSecond, _cancellationTokenSource.Token))) + if (_threadPool.WaitForProcessing(Time.OneSecond)) { Log.Error("BrokerageTransactionHandler.ProcessSynchronousEvents(): Timed out waiting for request queue to finish processing."); } @@ -799,27 +800,8 @@ public void AddOpenOrder(Order order, IAlgorithm algorithm) /// public void Exit() { - var timeout = TimeSpan.FromSeconds(60); - if (_processingThreads != null) - { - // only wait if the processing thread is running - if (_orderRequestQueues.Any(queue => queue.IsBusy && !queue.WaitHandle.WaitOne(timeout))) - { - Log.Error("BrokerageTransactionHandler.Exit(): Exceed timeout: " + (int)(timeout.TotalSeconds) + " seconds."); - } - - foreach (var queue in _orderRequestQueues) - { - queue.CompleteAdding(); - } - - foreach (var thread in _processingThreads) - { - thread?.StopSafely(timeout, _cancellationTokenSource); - } - } - IsActive = false; - _cancellationTokenSource.DisposeSafely(); + // Dispose drains the queued requests (CompleteAdding) and waits for the threads before stopping + _threadPool.DisposeSafely(); } /// @@ -1937,16 +1919,6 @@ private string GetShortableErrorMessage(Symbol symbol, decimal quantity) return $"Order exceeds shortable quantity {shortableQuantity} for Symbol {symbol} requested {quantity})"; } - private void EnqueueOrderRequest(OrderRequest request, Order order) - { - var queueKey = request.OrderId; - if (order.GroupOrderManager?.Id > 0) - { - queueKey = order.GroupOrderManager.Id; - } - _orderRequestQueues[queueKey % _orderRequestQueues.Count].Add(request); - } - /// /// Holds an order and its state /// diff --git a/Engine/TransactionHandlers/OrderRequestProcessingPool.cs b/Engine/TransactionHandlers/OrderRequestProcessingPool.cs new file mode 100644 index 000000000000..1174e6fe3ca7 --- /dev/null +++ b/Engine/TransactionHandlers/OrderRequestProcessingPool.cs @@ -0,0 +1,412 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using System; +using System.Collections.Generic; +using System.Threading; +using QuantConnect.Interfaces; +using QuantConnect.Logging; +using QuantConnect.Orders; +using QuantConnect.Util; + +namespace QuantConnect.Lean.Engine.TransactionHandlers +{ + /// + /// Runs order requests on background worker threads that pull from a single shared queue. The pool grows on + /// demand when the workers get saturated and keeps every request of an order processed in order. + /// + /// + /// Workers pull from one shared queue, so the load spreads across them instead of pinning each order to a thread + /// up front. To keep a single order (or combo group) in order, only one of its requests runs at a time. While one + /// runs the rest wait parked, and the same worker takes them next in arrival order. This state only exists while + /// an order has requests in flight, so nothing needs releasing once the order closes. When a single consumer + /// drains the queue, a lone fixed worker or the caller itself in synchronous mode (through + /// ), arrival order is already preserved so the per-order bookkeeping is skipped. + /// + public class OrderRequestProcessingPool : IDisposable + { + // maximum time to wait for each worker thread to stop when disposing the pool + private static readonly TimeSpan ShutdownTimeout = TimeSpan.FromSeconds(60); + // the shared queue of requests cleared to run. every worker pulls from here so the load stays balanced + private readonly IBusyCollection _readyQueue; + private readonly List _threads; + // for each order (or combo group) being processed, the follow up requests waiting their turn in arrival order, + // or null until a second request actually needs parking. while the key is here the order is already running + private readonly Dictionary<(bool IsGroup, int Id), Queue> _inFlight = new(); + // guards the in flight map, the threads list and the growth/shutdown flags + private readonly Lock _lock = new(); + // maximum number of worker threads the pool can grow to on demand + private readonly int _maximumThreads; + // true when there are no worker threads and the caller drains the single queue itself + private readonly bool _synchronous; + // true when a single consumer drains the queue (synchronous or a single fixed worker), which already + // preserves arrival order across all orders so the per-order serialization is skipped entirely + private readonly bool _singleConsumer; + // set under the lock when shutting down so the pool stops growing while the queue drains, before the + // cancellation token is cancelled as the final hard stop + private bool _shuttingDown; + // number of workers currently processing a request, used to decide when the pool is saturated + private int _busyWorkers; + private readonly Action _processRequest; + private readonly Action _onError; + private readonly CancellationTokenSource _cancellationTokenSource = new CancellationTokenSource(); + + /// + /// True while the pool is processing order requests, false once it has been shut down. + /// + public bool IsActive { get; private set; } + + /// + /// The number of worker threads currently running. + /// + public int ThreadCount + { + get + { + lock (_lock) + { + return _threads.Count; + } + } + } + + /// + /// Creates a threaded pool and starts its initial worker threads. When concurrency is enabled the pool + /// starts at and grows on demand up to , + /// otherwise it runs a single fixed worker thread. + /// + /// True to grow the pool on demand, false to run a single worker thread + /// The number of worker threads the pool starts with when growing + /// The maximum number of worker threads the pool can grow to on demand + /// Handles a single order request + /// Invoked when processing fails unexpectedly + public OrderRequestProcessingPool(bool concurrencyEnabled, int minimumThreads, int maximumThreads, + Action processRequest, Action onError) + { + _synchronous = false; + _processRequest = processRequest; + _onError = onError; + // concurrency grows the pool minimum..maximum on demand, otherwise a single fixed thread is used + _maximumThreads = concurrencyEnabled ? Math.Max(1, maximumThreads) : 1; + _singleConsumer = _maximumThreads == 1; + var initialThreadsCount = concurrencyEnabled ? Math.Min(Math.Max(1, minimumThreads), _maximumThreads) : 1; + + _readyQueue = new BusyBlockingCollection(); + _threads = new(_maximumThreads); + IsActive = true; + for (var i = 0; i < initialThreadsCount; i++) + { + AddThread().Start(); + } + } + + /// + /// Private constructor for the synchronous pool, a single non blocking queue and no worker threads. + /// + private OrderRequestProcessingPool(Action processRequest, Action onError) + { + _synchronous = true; + _processRequest = processRequest; + _onError = onError; + _maximumThreads = 1; + _singleConsumer = true; + + _readyQueue = new BusyCollection(); + _threads = new(0); + IsActive = true; + } + + /// + /// Creates a synchronous pool with no worker threads. Its single queue is drained on the caller thread + /// via . + /// + /// Handles a single order request + /// Invoked when processing fails unexpectedly + public static OrderRequestProcessingPool Synchronous(Action processRequest, Action onError) + { + return new OrderRequestProcessingPool(processRequest, onError); + } + + /// + /// Dispatches an order request to be processed. If the order already has a request in flight, the new one + /// waits parked so its worker runs it next and the order stays in arrival order. Otherwise it is queued for + /// any worker to pick up, growing the pool first when every worker is already busy. + /// + /// The order request to process + /// The order the request belongs to, used to keep its requests ordered + public void Dispatch(OrderRequest request, Order order) + { + // a single consumer drains in arrival order across all orders, no need to serialize per order + if (_singleConsumer) + { + _readyQueue.Add(new WorkItem(request, default)); + return; + } + + var key = GetRoutingKey(order); + WorkItem readyItem = default; + Thread newThread = null; + var run = false; + lock (_lock) + { + if (_inFlight.TryGetValue(key, out var parked)) + { + // the order is already being processed, park this request so its worker runs it next in order, + // allocating the queue only now that a second request has actually arrived + if (parked == null) + { + _inFlight[key] = parked = new Queue(); + } + parked.Enqueue(request); + } + else + { + // claim the order without a queue, most orders never get a second request. grow the pool if + // every worker is already busy so this request would wait + _inFlight[key] = null; + newThread = TryExpand(); + readyItem = new WorkItem(request, key); + run = true; + } + } + + // start the new worker and add outside the lock: starting an OS thread and a potentially blocking + // add on a bounded queue shouldn't stall other dispatchers + if (run) + { + newThread?.Start(); + _readyQueue.Add(readyItem); + } + } + + /// + /// Drains the pending order requests on the calling thread. Only used in synchronous mode, where there + /// are no worker threads and the caller pumps the single queue itself. + /// + public void ProcessPending() + { + Drain(item => _processRequest(item.Request)); + } + + /// + /// Waits until no order has requests in flight, up to the given timeout. In practice only the synchronous + /// early return runs. The threaded branch below is defensive, since its callers only reach it in backtesting + /// where the pool is synchronous, so it never runs in a live deployment. + /// + /// The maximum time to wait + /// True if the pool was still processing when the timeout elapsed + public bool WaitForProcessing(TimeSpan timeout) + { + // synchronous mode has no worker thread to drain the queue, the caller pumps it via ProcessPending + if (_synchronous) + { + return false; + } + + // re-check each pass since the shared queue signals idle as soon as a worker finds it empty, even if + // another worker is still processing or a request is parked + while (IsProcessing()) + { + if (!_readyQueue.WaitHandle.WaitOne(timeout, _cancellationTokenSource.Token)) + { + return true; + } + } + return false; + } + + /// + /// Whether any order still has a request in flight, either queued, being processed or parked. + /// + private bool IsProcessing() + { + lock (_lock) + { + return _inFlight.Count > 0 || _readyQueue.IsBusy; + } + } + + /// + /// Stops every worker thread and waits for them to terminate, then releases the pool resources. + /// + public void Dispose() + { + lock (_lock) + { + // already disposed, nothing else to do + if (_shuttingDown) + { + return; + } + // stop growing so the threads list is frozen and safe to iterate without taking a snapshot + _shuttingDown = true; + } + + // let the workers drain whatever is queued and parked: once adding is complete their consuming + // enumerables finish naturally when the queue empties, so join before cancelling anything. Only + // escalate to StopSafely, which cancels the shared token and drops pending requests, on timeout + _readyQueue.CompleteAdding(); + foreach (var thread in _threads) + { + try + { + if (thread != null && !thread.Join(ShutdownTimeout)) + { + Log.Error($"OrderRequestProcessingPool.Dispose(): Exceeded timeout: {(int)ShutdownTimeout.TotalSeconds} seconds waiting for '{thread.Name}' to finish processing"); + thread.StopSafely(ShutdownTimeout, _cancellationTokenSource); + } + } + catch (ThreadStateException) + { + // registered by a concurrent Dispatch but not started yet, nothing to drain on it + } + } + + IsActive = false; + _readyQueue.DisposeSafely(); + _cancellationTokenSource.DisposeSafely(); + } + + /// + /// Creates and registers a worker thread without starting it, so callers can start it outside the lock. + /// Callers growing the pool on demand must hold . + /// + /// The new worker thread, for the caller to start + private Thread AddThread() + { + var thread = new Thread(Run) { IsBackground = true, Name = $"Transaction Thread {_threads.Count}" }; + _threads.Add(thread); + return thread; + } + + /// + /// Grows the pool by one worker when every existing worker is already busy, up to the maximum. + /// Caller must hold and start the returned thread, if any, outside of it. + /// + /// The new worker thread to start, null when the pool doesn't need to grow + private Thread TryExpand() + { + if (_shuttingDown || _threads.Count >= _maximumThreads) + { + return null; + } + + // only grow when every worker is already busy, so the request being enqueued would have to wait + if (Volatile.Read(ref _busyWorkers) >= _threads.Count) + { + Log.Trace($"OrderRequestProcessingPool.TryExpand(): adding new thread, current count {_threads.Count}"); + return AddThread(); + } + return null; + } + + /// + /// Worker thread loop that consumes ready requests until the pool is shut down. A single fixed worker + /// already consumes in arrival order so it skips the per-order bookkeeping. + /// + private void Run() + { + if (_singleConsumer) + { + Drain(item => _processRequest(item.Request)); + } + else + { + Drain(ProcessInOrder); + } + } + + /// + /// Consumes ready requests on the calling thread until the queue completes adding or the pool is shut down. + /// + private void Drain(Action process) + { + try + { + foreach (var item in _readyQueue.GetConsumingEnumerable(_cancellationTokenSource.Token)) + { + process(item); + } + } + catch (Exception err) + { + // unexpected error, we need to close down shop + _onError(err); + } + } + + /// + /// Processes a request and then drains, in arrival order, every follow up request parked for the same order, + /// so a single worker handles the whole order in sequence before moving on to other work. + /// + private void ProcessInOrder(WorkItem item) + { + var request = item.Request; + Interlocked.Increment(ref _busyWorkers); + try + { + while (request != null) + { + _processRequest(request); + + lock (_lock) + { + var parked = _inFlight[item.Key]; + if (parked != null && parked.Count > 0) + { + request = parked.Dequeue(); + } + else + { + // no more requests for this order in flight, drop its bookkeeping + _inFlight.Remove(item.Key); + request = null; + } + } + } + } + finally + { + Interlocked.Decrement(ref _busyWorkers); + } + } + + /// + /// Builds the routing key that ties an order's requests together, the combo group when it has one, otherwise + /// the order itself. Order ids and group ids are separate counters that can share a value, so the flag keeps + /// a simple order and a combo group from colliding. + /// + private static (bool IsGroup, int Id) GetRoutingKey(Order order) + { + var group = order.GroupOrderManager; + return group?.Id > 0 ? (true, group.Id) : (false, order.Id); + } + + /// + /// Pairs a request with its routing key so the worker can drain the rest of the order without re-deriving it. + /// + private readonly struct WorkItem + { + public OrderRequest Request { get; } + public (bool IsGroup, int Id) Key { get; } + + public WorkItem(OrderRequest request, (bool IsGroup, int Id) key) + { + Request = request; + Key = key; + } + } + } +} diff --git a/Launcher/config.json b/Launcher/config.json index f0498a0996ab..03af8dd9c5f9 100644 --- a/Launcher/config.json +++ b/Launcher/config.json @@ -58,7 +58,9 @@ "ignore-unknown-asset-holdings": true, // The maximum amount of transaction threads for concurrent order submissions if the brokerage supports it. - //"maximum-transaction-threads": 4, + // The pool starts at the minimum and grows up to the maximum on demand. + //"minimum-transaction-threads": 2, + //"maximum-transaction-threads": 10, // log missing data files, useful for debugging "show-missing-data-logs": false, diff --git a/Tests/Engine/BrokerageTransactionHandlerTests/BrokerageTransactionHandlerTests.cs b/Tests/Engine/BrokerageTransactionHandlerTests/BrokerageTransactionHandlerTests.cs index ffb9fc557da2..e5ce75ff1999 100644 --- a/Tests/Engine/BrokerageTransactionHandlerTests/BrokerageTransactionHandlerTests.cs +++ b/Tests/Engine/BrokerageTransactionHandlerTests/BrokerageTransactionHandlerTests.cs @@ -2467,7 +2467,7 @@ public void ProcessesOrdersConcurrently() } [Test] - public void ProcessesComboRequestsOnSameThreadWhenConcurrencyIsEnabled() + public void ProcessesComboRequestsWhenConcurrencyIsEnabled() { var algorithm = new TestAlgorithm(); using var brokerage = new TestingConcurrentBrokerage(); @@ -2504,9 +2504,9 @@ public void ProcessesComboRequestsOnSameThreadWhenConcurrencyIsEnabled() Assert.IsTrue(finishedEvent.Wait(10000)); - Assert.IsTrue(transactionHandler.RequestProcessingThreads.TryGetValue(orderRequest1.OrderId, out var order1Thread)); - Assert.IsTrue(transactionHandler.RequestProcessingThreads.TryGetValue(orderRequest2.OrderId, out var order2Thread)); - Assert.AreEqual(order1Thread, order2Thread); + // both legs of the combo must be processed + Assert.IsTrue(transactionHandler.RequestProcessingThreads.ContainsKey(orderRequest1.OrderId)); + Assert.IsTrue(transactionHandler.RequestProcessingThreads.ContainsKey(orderRequest2.OrderId)); } finally { @@ -2514,6 +2514,217 @@ public void ProcessesComboRequestsOnSameThreadWhenConcurrencyIsEnabled() } } + [Test] + public void TransactionThreadPoolStartsAtMinimumThreads() + { + var algorithm = new TestAlgorithm(); + using var brokerage = new TestingConcurrentBrokerage(); + using var finishedEvent = new ManualResetEventSlim(false); + var transactionHandler = new TestableConcurrentBrokerageTransactionHandler(1, finishedEvent); + transactionHandler.Initialize(algorithm, brokerage, new BacktestingResultHandler()); + + try + { + // the pool starts with the minimum number of threads and grows only on demand + Assert.AreEqual(2, transactionHandler.ActiveThreadCount); + } + finally + { + transactionHandler.Exit(); + } + } + + [TestCase(10)] + [TestCase(3)] + public void TransactionThreadPoolGrowsUnderBacklogUpToMaximum(int maximumThreads) + { + var algorithm = new TestAlgorithm(); + using var brokerage = new TestingConcurrentBrokerage(); + + using var finishedEvent = new ManualResetEventSlim(false); + using var gate = new ManualResetEventSlim(false); + var transactionHandler = new TestableConcurrentBrokerageTransactionHandler(int.MaxValue, finishedEvent) + { + Gate = gate, + MaxThreadsOverride = maximumThreads + }; + transactionHandler.Initialize(algorithm, brokerage, new BacktestingResultHandler()); + + try + { + algorithm.Transactions.SetOrderProcessor(transactionHandler); + + var security = (Security)algorithm.AddEquity("SPY"); + algorithm.SetFinishedWarmingUp(); + + var reference = new DateTime(2025, 07, 03, 10, 0, 0); + security.SetMarketPrice(new Tick(reference, security.Symbol, 300, 300)); + + // starts at the minimum + Assert.AreEqual(2, transactionHandler.ActiveThreadCount); + + // keep feeding orders while threads stay blocked on the gate, forcing the pool to grow to the max + var orderId = 0; + var reachedMax = SpinWait.SpinUntil(() => + { + if (orderId < 1000) + { + var request = MakeAsyncMarketRequest(security, reference); + request.SetOrderId(++orderId); + transactionHandler.Process(request); + } + return transactionHandler.ActiveThreadCount >= maximumThreads; + }, 10000); + + Assert.IsTrue(reachedMax, $"Pool did not grow to the maximum, current size: {transactionHandler.ActiveThreadCount}"); + // never grows beyond the configured maximum + Assert.AreEqual(maximumThreads, transactionHandler.ActiveThreadCount); + } + finally + { + gate.Set(); + transactionHandler.Exit(); + } + } + + [Test] + public void ProcessesAnOrdersRequestsInOrderAsThePoolGrows() + { + // the requests of a single order must be processed in arrival order even when the pool grows between them + using var gate = new ManualResetEventSlim(false); + var processed = new ConcurrentQueue<(int OrderId, OrderRequestType Type)>(); + Exception processingError = null; + var pool = new OrderRequestProcessingPool(concurrencyEnabled: true, minimumThreads: 1, maximumThreads: 10, + request => + { + // block first so the requests pile up, then record the order they run in + gate.Wait(); + processed.Enqueue((request.OrderId, request.OrderRequestType)); + }, + exception => processingError = exception); + + try + { + var symbol = Symbols.SPY; + var reference = new DateTime(2025, 07, 03, 10, 0, 0); + + // the order we track, its submit claims a worker and blocks on the gate + var submit = new SubmitOrderRequest(OrderType.Market, symbol.SecurityType, symbol, 1, 0, 0, reference, ""); + submit.SetOrderId(1); + var order = Order.CreateOrder(submit); + pool.Dispatch(submit, order); + + // saturate the pool with unrelated orders so it grows while the submit is still in flight + var fillerId = 1000; + var grew = SpinWait.SpinUntil(() => + { + var filler = new SubmitOrderRequest(OrderType.Market, symbol.SecurityType, symbol, 1, 0, 0, 0, 0, false, reference, "", + asynchronous: true); + filler.SetOrderId(++fillerId); + pool.Dispatch(filler, Order.CreateOrder(filler)); + return pool.ThreadCount >= 3; + }, 10000); + Assert.IsTrue(grew, $"the pool did not grow, current size: {pool.ThreadCount}"); + + // the update and cancel arrive after the pool grew, they must still run after the submit and in order + pool.Dispatch(new UpdateOrderRequest(reference, order.Id, new UpdateOrderFields()), order); + pool.Dispatch(new CancelOrderRequest(reference, order.Id, ""), order); + + gate.Set(); + + Assert.IsTrue(SpinWait.SpinUntil(() => processed.Count(x => x.OrderId == 1) >= 3, 10000), + "the order's requests were not all processed"); + var sequence = processed.Where(x => x.OrderId == 1).Select(x => x.Type).ToList(); + Assert.AreEqual(new[] { OrderRequestType.Submit, OrderRequestType.Update, OrderRequestType.Cancel }, sequence); + Assert.IsNull(processingError, $"the pool reported an error: {processingError}"); + } + finally + { + gate.Set(); + pool.DisposeSafely(); + } + } + + [Test] + public void DoesNotGrowWhenOnlyOneOrderIsBusy() + { + using var gate = new ManualResetEventSlim(false); + var pool = new OrderRequestProcessingPool(concurrencyEnabled: true, minimumThreads: 2, maximumThreads: 10, + request => gate.Wait(), + exception => { }); + + try + { + var symbol = Symbols.SPY; + var reference = new DateTime(2025, 07, 03, 10, 0, 0); + + var submit = new SubmitOrderRequest(OrderType.Market, symbol.SecurityType, symbol, 1, 0, 0, reference, ""); + submit.SetOrderId(1); + var order = Order.CreateOrder(submit); + pool.Dispatch(submit, order); + + // every follow up request is for the same order, so they are parked behind the one busy worker while + // the other stays idle, so the pool must not grow no matter how many pile up + for (var i = 0; i < 20; i++) + { + pool.Dispatch(new UpdateOrderRequest(reference, order.Id, new UpdateOrderFields()), order); + } + + Assert.AreEqual(2, pool.ThreadCount); + } + finally + { + gate.Set(); + pool.DisposeSafely(); + } + } + + [Test] + public void ProcessesManyOrdersWithUpdatesAndCancelsQuickly() + { + // lots of submit/update/cancel at once, the handler does nothing so this only measures the pool + const int orderCount = 1000; + const int requestsPerOrder = 3; + var processedCount = 0; + Exception processingError = null; + + var pool = new OrderRequestProcessingPool(concurrencyEnabled: true, minimumThreads: 2, maximumThreads: 10, + _ => Interlocked.Increment(ref processedCount), + exception => processingError = exception); + + try + { + var symbol = Symbols.SPY; + var reference = new DateTime(2025, 07, 03, 10, 0, 0); + + for (var i = 1; i <= orderCount; i++) + { + var submit = new SubmitOrderRequest(OrderType.Market, symbol.SecurityType, symbol, 1, 0, 0, reference, ""); + submit.SetOrderId(i); + var order = Order.CreateOrder(submit); + + pool.Dispatch(submit, order); + pool.Dispatch(new UpdateOrderRequest(reference, order.Id, new UpdateOrderFields()), order); + pool.Dispatch(new CancelOrderRequest(reference, order.Id, ""), order); + } + + // if the pool ever hangs or falls behind this wait times out instead of finishing in a few ms + var expectedRequests = orderCount * requestsPerOrder; + Assert.IsTrue(SpinWait.SpinUntil(() => processedCount >= expectedRequests, 10000)); + Assert.IsNull(processingError); + } + finally + { + pool.DisposeSafely(); + } + } + + private static SubmitOrderRequest MakeAsyncMarketRequest(Security security, DateTime date) + { + return new SubmitOrderRequest(OrderType.Market, security.Type, security.Symbol, 1, 0, 0, 0, 0, false, date, "", + asynchronous: true); + } + [TestCase("OnAccountChanged")] [TestCase("OnOptionNotification")] [TestCase("OnNewBrokerageOrderNotification")] @@ -2765,6 +2976,9 @@ public class TestBrokerageTransactionHandler : BrokerageTransactionHandler protected override TimeSpan TimeSinceLastFill => TestTimeSinceLastFill; + // no worker thread: these tests drive HandleOrderRequest manually + protected override bool SynchronousProcessing => true; + public override void Initialize(IAlgorithm algorithm, IBrokerage brokerage, IResultHandler resultHandler) { _brokerage = brokerage; @@ -2777,11 +2991,6 @@ public DateTime GetLastSyncDate() return _brokerage.LastSyncDateTimeUtc.ConvertFromUtc(TimeZones.NewYork); } - protected override void InitializeTransactionThread() - { - _orderRequestQueues = new() { new BusyCollection() }; - } - public new void RoundOrderPrices(Order order, Security security) { base.RoundOrderPrices(order, security); @@ -2875,6 +3084,15 @@ private class TestableConcurrentBrokerageTransactionHandler : BrokerageTransacti public ConcurrentDictionary RequestProcessingThreads = new(); + // blocks threads so requests pile up and force the pool to grow + public ManualResetEventSlim Gate; + + public int ActiveThreadCount => ProcessingThreadsCount; + + // overrides the pool maximum without touching the global Config + public int? MaxThreadsOverride { get; set; } + protected override int MaximumTransactionThreads => MaxThreadsOverride ?? base.MaximumTransactionThreads; + public TestableConcurrentBrokerageTransactionHandler(int expectedOrdersCount, ManualResetEventSlim finishedEvent) { _expectedOrdersCount = expectedOrdersCount; @@ -2883,6 +3101,8 @@ public TestableConcurrentBrokerageTransactionHandler(int expectedOrdersCount, Ma public override void HandleOrderRequest(OrderRequest request) { + Gate?.Wait(); + base.HandleOrderRequest(request); // Capture the thread name for debugging purposes diff --git a/Tests/Engine/DataFeeds/InternalSubscriptionManagerTests.cs b/Tests/Engine/DataFeeds/InternalSubscriptionManagerTests.cs index 8d7501ab44f5..a5deddcd405e 100644 --- a/Tests/Engine/DataFeeds/InternalSubscriptionManagerTests.cs +++ b/Tests/Engine/DataFeeds/InternalSubscriptionManagerTests.cs @@ -238,9 +238,9 @@ public void PreMarketDataSetsCache() .Any(config => config.IsInternalFeed && config.Resolution == Resolution.Second)); first = false; } - else if(_algorithm.Securities["AAPL"].Price != 0 && _algorithm.Securities["IBM"].Price != 0) + else if (_algorithm.Securities["AAPL"].Price != 0 && _algorithm.Securities["IBM"].Price != 0) { - #pragma warning disable CS0618 +#pragma warning disable CS0618 _algorithm.SetHoldings("AAPL", 0.01); _algorithm.SetHoldings("IBM", 0.01); @@ -250,7 +250,7 @@ public void PreMarketDataSetsCache() Assert.AreEqual(OrderStatus.Submitted, orders[0].Status); orders = _algorithm.Transactions.GetOpenOrders("IBM"); - #pragma warning restore CS0618 +#pragma warning restore CS0618 Assert.AreEqual(1, orders.Count); Assert.AreEqual(Symbols.IBM, orders[0].Symbol); Assert.AreEqual(OrderStatus.Submitted, orders[0].Status); @@ -423,7 +423,7 @@ private void SetupImpl(IDataQueueHandler dataQueueHandler, Synchronizer synchron new DataChannelProvider()); _algorithm.SubscriptionManager.SetDataManager(_dataManager); _algorithm.Securities.SetSecurityService(securityService); - var backtestingTransactionHandler = new BacktestingTransactionHandler(); + var backtestingTransactionHandler = new SynchronousBacktestingTransactionHandler(); _paperBrokerage = new PaperBrokerage(_algorithm, new LiveNodePacket()); backtestingTransactionHandler.Initialize(_algorithm, _paperBrokerage, _resultHandler); _algorithm.Transactions.SetOrderProcessor(backtestingTransactionHandler); @@ -434,6 +434,12 @@ private void SetupImpl(IDataQueueHandler dataQueueHandler, Synchronizer synchron } _transactionHandler = backtestingTransactionHandler; } + + private class SynchronousBacktestingTransactionHandler : BacktestingTransactionHandler + { + protected override bool SynchronousProcessing => true; + } + private class TestAggregationManager : AggregationManager { public TestAggregationManager(ITimeProvider timeProvider)