diff --git a/Detectors/ITSMFT/ITS/tracking/CMakeLists.txt b/Detectors/ITSMFT/ITS/tracking/CMakeLists.txt index 17420d47a2732..6560f25966a63 100644 --- a/Detectors/ITSMFT/ITS/tracking/CMakeLists.txt +++ b/Detectors/ITSMFT/ITS/tracking/CMakeLists.txt @@ -13,7 +13,6 @@ o2_add_library(ITStracking TARGETVARNAME targetName SOURCES src/ClusterLines.cxx src/Cluster.cxx - src/CapacityEstimator.cxx src/Configuration.cxx src/FastMultEstConfig.cxx src/FastMultEst.cxx @@ -22,7 +21,6 @@ o2_add_library(ITStracking src/IOUtils.cxx src/Tracker.cxx src/TrackerTraits.cxx - src/TrackingConfigParam.cxx src/Vertexer.cxx src/VertexerTraits.cxx PUBLIC_LINK_LIBRARIES @@ -36,9 +34,11 @@ o2_add_library(ITStracking O2::ITSReconstruction O2::ITSMFTReconstruction O2::DataFormatsITS + O2::ITSMFTTracking PRIVATE_LINK_LIBRARIES O2::Steer TBB::tbb) + # target_compile_options(${targetName} PRIVATE -O0 -g -fPIC -fno-omit-frame-pointer) o2_add_library(ITSTrackingInterface @@ -55,7 +55,6 @@ o2_target_root_dictionary(ITStracking include/ITStracking/Cluster.h include/ITStracking/Definitions.h include/ITStracking/FastMultEstConfig.h - include/ITStracking/TrackingConfigParam.h LINKDEF src/TrackingLinkDef.h) if(CUDA_ENABLED OR HIP_ENABLED) diff --git a/Detectors/ITSMFT/ITS/tracking/GPU/ITStrackingGPU/TimeFrameGPU.h b/Detectors/ITSMFT/ITS/tracking/GPU/ITStrackingGPU/TimeFrameGPU.h index 0d70158b9bdb8..9d80edfaf2b92 100644 --- a/Detectors/ITSMFT/ITS/tracking/GPU/ITStrackingGPU/TimeFrameGPU.h +++ b/Detectors/ITSMFT/ITS/tracking/GPU/ITStrackingGPU/TimeFrameGPU.h @@ -16,7 +16,7 @@ #include #include -#include "ITStracking/BoundedAllocator.h" +#include "ITSMFTTracking/BoundedAllocator.h" #include "ITStracking/TimeFrame.h" #include "ITStracking/Configuration.h" #include "ITStracking/TrackExtensionHypothesis.h" diff --git a/Detectors/ITSMFT/ITS/tracking/GPU/ITStrackingGPU/TrackingKernels.h b/Detectors/ITSMFT/ITS/tracking/GPU/ITStrackingGPU/TrackingKernels.h index 94950c04877b9..3a7c5dbd6b510 100644 --- a/Detectors/ITSMFT/ITS/tracking/GPU/ITStrackingGPU/TrackingKernels.h +++ b/Detectors/ITSMFT/ITS/tracking/GPU/ITStrackingGPU/TrackingKernels.h @@ -16,9 +16,9 @@ #include #include -#include "ITStracking/BoundedAllocator.h" -#include "ITStracking/CapacityEstimator.h" -#include "ITStracking/ROFLookupTables.h" +#include "ITSMFTTracking/BoundedAllocator.h" +#include "ITSMFTTracking/CapacityEstimator.h" +#include "ITSMFTTracking/ROFLookupTables.h" #include "ITStracking/TrackingTopology.h" #include "ITStracking/TrackExtensionHypothesis.h" #include "ITStrackingGPU/Utils.h" @@ -26,6 +26,9 @@ namespace o2::its { +using o2::itsmft::tracking::bounded_vector; +using o2::itsmft::tracking::CapacityEstimator; + class CellSeed; struct CellNeighbour; template diff --git a/Detectors/ITSMFT/ITS/tracking/GPU/ITStrackingGPU/Utils.h b/Detectors/ITSMFT/ITS/tracking/GPU/ITStrackingGPU/Utils.h index e6909b28a687a..8f5baecd80aa1 100644 --- a/Detectors/ITSMFT/ITS/tracking/GPU/ITStrackingGPU/Utils.h +++ b/Detectors/ITSMFT/ITS/tracking/GPU/ITStrackingGPU/Utils.h @@ -20,7 +20,7 @@ #include #include -#include "ITStracking/MathUtils.h" +#include "ITSMFTTracking/MathUtils.h" #include "ITStracking/ExternalAllocator.h" #include "GPUCommonDef.h" diff --git a/Detectors/ITSMFT/ITS/tracking/GPU/cuda/TimeFrameGPU.cu b/Detectors/ITSMFT/ITS/tracking/GPU/cuda/TimeFrameGPU.cu index 593f5529cf8ea..c610ffe011171 100644 --- a/Detectors/ITSMFT/ITS/tracking/GPU/cuda/TimeFrameGPU.cu +++ b/Detectors/ITSMFT/ITS/tracking/GPU/cuda/TimeFrameGPU.cu @@ -20,8 +20,8 @@ #include #include "ITStrackingGPU/TimeFrameGPU.h" -#include "ITStracking/Constants.h" -#include "ITStracking/BoundedAllocator.h" +#include "ITSMFTTracking/Constants.h" +#include "ITSMFTTracking/BoundedAllocator.h" #include "ITStrackingGPU/Utils.h" #include "GPUCommonDef.h" diff --git a/Detectors/ITSMFT/ITS/tracking/GPU/cuda/TrackerTraitsGPU.cxx b/Detectors/ITSMFT/ITS/tracking/GPU/cuda/TrackerTraitsGPU.cxx index b80fb6452f8a9..767c02c646293 100644 --- a/Detectors/ITSMFT/ITS/tracking/GPU/cuda/TrackerTraitsGPU.cxx +++ b/Detectors/ITSMFT/ITS/tracking/GPU/cuda/TrackerTraitsGPU.cxx @@ -20,6 +20,9 @@ namespace o2::its { +using o2::itsmft::tracking::runOnSlab; +using o2::itsmft::tracking::SlabSite; + template void TrackerTraitsGPU::initialiseTimeFrame(const int iteration) { diff --git a/Detectors/ITSMFT/ITS/tracking/GPU/cuda/TrackingKernels.cu b/Detectors/ITSMFT/ITS/tracking/GPU/cuda/TrackingKernels.cu index a179fd0962dd3..f2be0158ea343 100644 --- a/Detectors/ITSMFT/ITS/tracking/GPU/cuda/TrackingKernels.cu +++ b/Detectors/ITSMFT/ITS/tracking/GPU/cuda/TrackingKernels.cu @@ -29,11 +29,11 @@ #include #include "DataFormatsITS/TrackITS.h" -#include "ITStracking/Constants.h" +#include "ITSMFTTracking/Constants.h" #include "ITStracking/Definitions.h" #include "ITStracking/IndexTableUtils.h" #include "ITStrackingGPU/LaunchGeometry.h" -#include "ITStracking/MathUtils.h" +#include "ITSMFTTracking/MathUtils.h" #include "ITStracking/ExternalAllocator.h" #include "ITStracking/Tracklet.h" #include "ITStracking/Cluster.h" @@ -52,6 +52,9 @@ using namespace o2::track; namespace o2::its { + +using o2::itsmft::tracking::runOnSlab; +using o2::itsmft::tracking::SlabSite; namespace gpu { diff --git a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/BoundedAllocator.h b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/BoundedAllocator.h deleted file mode 100644 index 2e47664ee78ea..0000000000000 --- a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/BoundedAllocator.h +++ /dev/null @@ -1,320 +0,0 @@ -// Copyright 2019-2020 CERN and copyright holders of ALICE O2. -// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. -// All rights not expressly granted are reserved. -// -// This software is distributed under the terms of the GNU General Public -// License v3 (GPL Version 3), copied verbatim in the file "COPYING". -// -// In applying this license CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. -/// -/// \file BoundedAllocator.h -/// \brief -/// - -#ifndef TRACKINGITSU_INCLUDE_BOUNDEDALLOCATOR_H_ -#define TRACKINGITSU_INCLUDE_BOUNDEDALLOCATOR_H_ - -#include -#include -#include -#include -#include - -#if !defined(__HIPCC__) && !defined(__CUDACC__) -#include -#include -#include "GPUCommonLogger.h" -#endif -#include "ITStracking/ExternalAllocator.h" -#include "ITStracking/Constants.h" - -namespace o2::its -{ - -// #define BOUNDED_MR_STATS -class BoundedMemoryResource final : public std::pmr::memory_resource -{ - public: - class MemoryLimitExceeded final : public std::bad_alloc - { - public: - MemoryLimitExceeded(size_t attempted, size_t used, size_t max) - { - char buf[256]; - if (attempted != 0) { - (void)snprintf(buf, sizeof(buf), "Reached set memory limit (attempted: %zu, used: %zu, max: %zu)", attempted, used, max); - } else { - (void)snprintf(buf, sizeof(buf), "New set maximum below current used (newMax: %zu, used: %zu)", max, used); - } - mMsg = buf; - } - const char* what() const noexcept final { return mMsg.c_str(); } - - private: - std::string mMsg; - }; - - static std::pmr::memory_resource* cachingUpstream() - { - static std::pmr::synchronized_pool_resource pool{std::pmr::get_default_resource()}; - return &pool; - } - - BoundedMemoryResource(size_t maxBytes = std::numeric_limits::max(), - std::pmr::memory_resource* upstream = nullptr) - : mMaxMemory(maxBytes), mUpstream(upstream != nullptr ? upstream : cachingUpstream()) {} - - BoundedMemoryResource(ExternalAllocator* alloc, - size_t maxBytes = std::numeric_limits::max()) - : mMaxMemory(maxBytes), - mAdaptor(std::make_unique(alloc)), - mUpstream(mAdaptor.get()) {} - - void* do_allocate(size_t bytes, size_t alignment) final - { - size_t new_used{0}; - size_t current_used{mUsedMemory.load(std::memory_order_relaxed)}; - do { - new_used = current_used + bytes; - if (new_used > mMaxMemory.load(std::memory_order_relaxed)) { - mCountThrow.fetch_add(1, std::memory_order_relaxed); - throw MemoryLimitExceeded(new_used, current_used, - mMaxMemory.load(std::memory_order_relaxed)); - } - } while (!mUsedMemory.compare_exchange_weak(current_used, new_used, - std::memory_order_acq_rel, - std::memory_order_relaxed)); - - void* p{nullptr}; - try { - p = mUpstream->allocate(bytes, alignment); - } catch (...) { - mUsedMemory.fetch_sub(bytes, std::memory_order_relaxed); -#ifdef BOUNDED_MR_STATS - mStats.upstreamFailures.fetch_add(1, std::memory_order_relaxed); -#endif - throw; - } - - size_t peak = mPeakUsedMemory.load(std::memory_order_relaxed); - while (new_used > peak && - !mPeakUsedMemory.compare_exchange_weak(peak, new_used, - std::memory_order_relaxed)) { - } - -#ifdef BOUNDED_MR_STATS - size_t statsPeak = mStats.peak.load(std::memory_order_relaxed); - while (new_used > statsPeak && - !mStats.peak.compare_exchange_weak(statsPeak, new_used, - std::memory_order_relaxed)) { - } - mStats.live.fetch_add(1, std::memory_order_relaxed); - mStats.nAlloc.fetch_add(1, std::memory_order_relaxed); - mStats.totalAlloc.fetch_add(bytes, std::memory_order_relaxed); - - size_t ma = mStats.maxAlign.load(std::memory_order_relaxed); - while (alignment > ma && !mStats.maxAlign.compare_exchange_weak(ma, alignment, std::memory_order_relaxed)) { - } -#endif - return p; - } - - void do_deallocate(void* p, size_t bytes, size_t alignment) final - { - mUpstream->deallocate(p, bytes, alignment); - mUsedMemory.fetch_sub(bytes, std::memory_order_relaxed); -#ifdef BOUNDED_MR_STATS - mStats.live.fetch_sub(1, std::memory_order_relaxed); - mStats.nFree.fetch_add(1, std::memory_order_relaxed); - mStats.totalFreed.fetch_add(bytes, std::memory_order_relaxed); -#endif - } - - bool do_is_equal(const std::pmr::memory_resource& other) const noexcept final - { - return this == &other; - } - - [[nodiscard]] size_t getUsedMemory() const noexcept - { - return mUsedMemory.load(std::memory_order_relaxed); - } - [[nodiscard]] size_t getMaxMemory() const noexcept - { - return mMaxMemory.load(std::memory_order_relaxed); - } - [[nodiscard]] size_t getThrowCount() const noexcept - { - return mCountThrow.load(std::memory_order_relaxed); - } - [[nodiscard]] size_t getPeakMemory() const noexcept - { - return mPeakUsedMemory.load(std::memory_order_relaxed); - } - [[nodiscard]] size_t getPeakMemoryDelta() const noexcept - { - const size_t peak = mPeakUsedMemory.load(std::memory_order_relaxed); - const size_t baseline = mPeakBaselineMemory.load(std::memory_order_relaxed); - return peak > baseline ? peak - baseline : 0; - } - - void resetPeakMemory() noexcept - { - const size_t used = mUsedMemory.load(std::memory_order_acquire); - mPeakBaselineMemory.store(used, std::memory_order_release); - mPeakUsedMemory.store(used, std::memory_order_release); - } - - void setMaxMemory(size_t max) - { - size_t current = mMaxMemory.load(std::memory_order_relaxed); - if (max == current) { - return; - } - for (;;) { - size_t used = mUsedMemory.load(std::memory_order_acquire); - if (used > max) { - mCountThrow.fetch_add(1, std::memory_order_relaxed); - throw MemoryLimitExceeded(0, used, max); - } - if (mMaxMemory.compare_exchange_weak(current, max, - std::memory_order_release, - std::memory_order_relaxed)) { - return; - } - if (current == max) { - return; - } - } - } - -#if !defined(__HIPCC__) && !defined(__CUDACC__) - std::string asString() const - { - const auto throw_ = mCountThrow.load(std::memory_order_relaxed); - const auto used = static_cast(mUsedMemory.load(std::memory_order_relaxed)); - const auto peak = static_cast(mPeakUsedMemory.load(std::memory_order_relaxed)); - const auto peakDelta = static_cast(getPeakMemoryDelta()); - const auto maxm = mMaxMemory.load(std::memory_order_relaxed); - std::string ret; - if (maxm == std::numeric_limits::max()) { - ret += std::format("maxthrow={} maxmem=unbounded used={:.2f} GB stagepeak={:.2f} GB stagealloc={:.2f} GB", throw_, used / constants::GB, peak / constants::GB, peakDelta / constants::GB); - } else { - ret += std::format("maxthrow={} maxmem={:.2f} GB used={:.2f} GB ({:.2f}%) stagepeak={:.2f} GB stagealloc={:.2f} GB", throw_, (double)maxm / constants::GB, used / constants::GB, 100.0 * used / (double)maxm, peak / constants::GB, peakDelta / constants::GB); - } -#ifdef BOUNDED_MR_STATS - ret += std::format(" peak={:.2f} GB live={} nAlloc={} nFree={} totalAlloc={:.2f} GB totalFreed={:.2f} GB maxAlign={} upstreamFail={}", - (float)mStats.peak.load(std::memory_order_relaxed) / constants::GB, - mStats.live.load(std::memory_order_relaxed), - mStats.nAlloc.load(std::memory_order_relaxed), - mStats.nFree.load(std::memory_order_relaxed), - (float)mStats.totalAlloc.load(std::memory_order_relaxed) / constants::GB, - (float)mStats.totalFreed.load(std::memory_order_relaxed) / constants::GB, - mStats.maxAlign.load(std::memory_order_relaxed), - mStats.upstreamFailures.load(std::memory_order_relaxed)); -#endif - return ret; - } - - void print() const - { - LOGP(info, "{}", asString()); - } -#endif - - private: - std::atomic mMaxMemory{std::numeric_limits::max()}; - std::atomic mCountThrow{0}; - std::atomic mUsedMemory{0}; - std::atomic mPeakUsedMemory{0}; - std::atomic mPeakBaselineMemory{0}; - std::unique_ptr mAdaptor{nullptr}; - std::pmr::memory_resource* mUpstream{nullptr}; - -#ifdef BOUNDED_MR_STATS - struct Stats { - std::atomic peak{0}; - std::atomic live{0}; - std::atomic nAlloc{0}; - std::atomic nFree{0}; - std::atomic totalAlloc{0}; - std::atomic totalFreed{0}; - std::atomic maxAlign{0}; - std::atomic upstreamFailures{0}; - }; - Stats mStats{}; -#endif -}; - -template -using bounded_vector = std::pmr::vector; - -template -inline void deepVectorClear(std::vector& vec) -{ - std::vector().swap(vec); -} - -template -inline void deepVectorClear(bounded_vector& vec, std::pmr::memory_resource* mr = nullptr) -{ - std::pmr::memory_resource* tmr = (mr != nullptr) ? mr : vec.get_allocator().resource(); - vec.~bounded_vector(); - new (&vec) bounded_vector(std::pmr::polymorphic_allocator{tmr}); -} - -template -inline void deepVectorClear(std::vector>& vec, std::pmr::memory_resource* mr = nullptr) -{ - for (auto& v : vec) { - deepVectorClear(v, mr); - } -} - -template -inline void deepVectorClear(std::array, S>& arr, std::pmr::memory_resource* mr = nullptr) -{ - for (size_t i{0}; i < S; ++i) { - deepVectorClear(arr[i], mr); - } -} - -template -inline void clearResizeBoundedVector(bounded_vector& vec, size_t sz, std::pmr::memory_resource* mr = nullptr, T def = T()) -{ - std::pmr::memory_resource* tmr = (mr != nullptr) ? mr : vec.get_allocator().resource(); - vec.~bounded_vector(); - new (&vec) bounded_vector(sz, def, std::pmr::polymorphic_allocator{tmr}); -} - -template -inline void clearResizeBoundedVector(std::vector>& vec, size_t size, std::pmr::memory_resource* mr) -{ - vec.clear(); - vec.reserve(size); - for (size_t i = 0; i < size; ++i) { - vec.emplace_back(std::pmr::polymorphic_allocator>{mr}); - } -} - -template -inline void clearResizeBoundedArray(std::array, S>& arr, size_t size, std::pmr::memory_resource* mr = nullptr, T def = T()) -{ - for (size_t i{0}; i < S; ++i) { - clearResizeBoundedVector(arr[i], size, mr, def); - } -} - -template -inline std::vector toSTDVector(const bounded_vector& b) -{ - std::vector t(b.size()); - std::copy(b.cbegin(), b.cend(), t.begin()); - return t; -} - -} // namespace o2::its - -#endif diff --git a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/Cell.h b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/Cell.h index 4706977d08ba6..ad3b11d3c1eec 100644 --- a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/Cell.h +++ b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/Cell.h @@ -18,7 +18,7 @@ #include -#include "ITStracking/Constants.h" +#include "ITSMFTTracking/Constants.h" #include "ITStracking/LayerMask.h" #include "DataFormatsITS/TimeEstBC.h" #include "ReconstructionDataFormats/Track.h" diff --git a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/Cluster.h b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/Cluster.h index fb5f1a13ef3d2..7187c5f50cb4e 100644 --- a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/Cluster.h +++ b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/Cluster.h @@ -17,8 +17,8 @@ #define TRACKINGITSU_INCLUDE_CACLUSTER_H_ #include -#include "ITStracking/Constants.h" -#include "ITStracking/MathUtils.h" +#include "ITSMFTTracking/Constants.h" +#include "ITSMFTTracking/MathUtils.h" #include "GPUCommonRtypes.h" #include "GPUCommonDef.h" diff --git a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/ClusterLines.h b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/ClusterLines.h index bcb8a98a62cab..8110b2bdfb384 100644 --- a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/ClusterLines.h +++ b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/ClusterLines.h @@ -18,7 +18,7 @@ #include #include #include "ITStracking/Cluster.h" -#include "ITStracking/Constants.h" +#include "ITSMFTTracking/Constants.h" #include "ITStracking/Tracklet.h" #include "GPUCommonRtypes.h" diff --git a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/Configuration.h b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/Configuration.h index 8e1c68fc31c6c..20a497c3b8a58 100644 --- a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/Configuration.h +++ b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/Configuration.h @@ -25,7 +25,7 @@ #include "CommonUtils/EnumFlags.h" #include "DetectorsBase/Propagator.h" -#include "ITStracking/Constants.h" +#include "ITSMFTTracking/Constants.h" #include "ITStracking/LayerMask.h" namespace o2::its diff --git a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/FastMultEst.h b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/FastMultEst.h index f94c7c2034b46..d283d6eed2d45 100644 --- a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/FastMultEst.h +++ b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/FastMultEst.h @@ -22,7 +22,7 @@ #include "DataFormatsITSMFT/CompCluster.h" #include "DataFormatsITSMFT/PhysTrigger.h" #include "ITStracking/FastMultEstConfig.h" -#include "ITStracking/ROFLookupTables.h" +#include "ITSMFTTracking/ROFLookupTables.h" #include #include diff --git a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/IndexTableUtils.h b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/IndexTableUtils.h index a7b44e91d9093..427abd9b876a5 100644 --- a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/IndexTableUtils.h +++ b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/IndexTableUtils.h @@ -19,7 +19,7 @@ #include #include "ITStracking/Cluster.h" -#include "ITStracking/MathUtils.h" +#include "ITSMFTTracking/MathUtils.h" #include "CommonConstants/MathConstants.h" #include "GPUCommonMath.h" #include "GPUCommonDef.h" diff --git a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/LayerMask.h b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/LayerMask.h index 9fe9894b3b457..1a3854f4a6d2b 100644 --- a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/LayerMask.h +++ b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/LayerMask.h @@ -22,7 +22,7 @@ #include "GPUCommonDef.h" #include "GPUCommonMath.h" -#include "ITStracking/Constants.h" +#include "ITSMFTTracking/Constants.h" namespace o2::its { diff --git a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/LineVertexerHelpers.h b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/LineVertexerHelpers.h index 0e3807aba8efb..9e0f0d2eed02c 100644 --- a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/LineVertexerHelpers.h +++ b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/LineVertexerHelpers.h @@ -15,12 +15,15 @@ #include #include -#include "ITStracking/BoundedAllocator.h" +#include "ITSMFTTracking/BoundedAllocator.h" #include "ITStracking/ClusterLines.h" namespace o2::its::line_vertexer { +using o2::itsmft::tracking::bounded_vector; +using o2::itsmft::tracking::BoundedMemoryResource; + struct Settings { float beamX = 0.f; float beamY = 0.f; diff --git a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TimeFrame.h b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TimeFrame.h index 11246fa0ee3b0..e4bc7f045f64f 100644 --- a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TimeFrame.h +++ b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TimeFrame.h @@ -23,7 +23,7 @@ #include "DataFormatsITS/TrackITS.h" #include "DataFormatsITS/Vertex.h" -#include "ITStracking/CapacityEstimator.h" +#include "ITSMFTTracking/CapacityEstimator.h" #include "ITStracking/Cell.h" #include "ITStracking/Cluster.h" #include "ITStracking/Configuration.h" @@ -31,8 +31,8 @@ #include "ITStracking/Tracklet.h" #include "ITStracking/IndexTableUtils.h" #include "ITStracking/ExternalAllocator.h" -#include "ITStracking/BoundedAllocator.h" -#include "ITStracking/ROFLookupTables.h" +#include "ITSMFTTracking/BoundedAllocator.h" +#include "ITSMFTTracking/ROFLookupTables.h" #include "ITStracking/TrackingTopology.h" #include "SimulationDataFormat/MCCompLabel.h" #include "SimulationDataFormat/MCTruthContainer.h" @@ -57,6 +57,10 @@ class ROFRecord; namespace its { +using o2::itsmft::tracking::bounded_vector; +using o2::itsmft::tracking::BoundedMemoryResource; +using o2::itsmft::tracking::CapacityEstimator; + namespace gpu { template diff --git a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TrackExtensionHypothesis.h b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TrackExtensionHypothesis.h index afa24c34afd4f..a3ebd47ff54aa 100644 --- a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TrackExtensionHypothesis.h +++ b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TrackExtensionHypothesis.h @@ -16,7 +16,7 @@ #include "GPUCommonDef.h" #include "DataFormatsITS/TimeEstBC.h" -#include "ITStracking/Constants.h" +#include "ITSMFTTracking/Constants.h" #include "ITStracking/TrackITSInternal.h" #include "ReconstructionDataFormats/Track.h" diff --git a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TrackFollower.h b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TrackFollower.h index 703b01e014dd7..cd3194807225f 100644 --- a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TrackFollower.h +++ b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TrackFollower.h @@ -22,10 +22,10 @@ #include "DetectorsBase/Propagator.h" #include "ITStracking/Cluster.h" -#include "ITStracking/Constants.h" +#include "ITSMFTTracking/Constants.h" #include "ITStracking/IndexTableUtils.h" -#include "ITStracking/MathUtils.h" -#include "ITStracking/ROFLookupTables.h" +#include "ITSMFTTracking/MathUtils.h" +#include "ITSMFTTracking/ROFLookupTables.h" #include "ITStracking/TrackExtensionHypothesis.h" #include "ITStracking/TrackHelpers.h" diff --git a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TrackHelpers.h b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TrackHelpers.h index b9924a8197d4c..1ed68342fc160 100644 --- a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TrackHelpers.h +++ b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TrackHelpers.h @@ -20,9 +20,9 @@ #include "DataFormatsITS/TrackITS.h" #include "ITStracking/Cell.h" #include "ITStracking/Cluster.h" -#include "ITStracking/Constants.h" +#include "ITSMFTTracking/Constants.h" #include "ITStracking/LayerMask.h" -#include "ITStracking/MathUtils.h" +#include "ITSMFTTracking/MathUtils.h" #include "ITStracking/TrackITSInternal.h" #include "DetectorsBase/Propagator.h" #include "ReconstructionDataFormats/Track.h" diff --git a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TrackITSInternal.h b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TrackITSInternal.h index 28a1117324550..2a0cbac70c870 100644 --- a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TrackITSInternal.h +++ b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TrackITSInternal.h @@ -17,7 +17,7 @@ #include "GPUCommonDef.h" #include "DataFormatsITS/TrackITS.h" #include "DataFormatsITS/TimeEstBC.h" -#include "ITStracking/Constants.h" +#include "ITSMFTTracking/Constants.h" #include "ReconstructionDataFormats/Track.h" namespace o2::its diff --git a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/Tracker.h b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/Tracker.h index c60f280de5307..53c32be166f06 100644 --- a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/Tracker.h +++ b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/Tracker.h @@ -33,7 +33,7 @@ #include "ITStracking/Definitions.h" #include "ITStracking/TimeFrame.h" #include "ITStracking/TrackerTraits.h" -#include "ITStracking/BoundedAllocator.h" +#include "ITSMFTTracking/BoundedAllocator.h" namespace o2 { diff --git a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TrackerTraits.h b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TrackerTraits.h index 1e3b9e7d7fbf7..bcf865e7d34b3 100644 --- a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TrackerTraits.h +++ b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TrackerTraits.h @@ -23,10 +23,10 @@ #include "DetectorsBase/Propagator.h" #include "ITStracking/Configuration.h" #include "ITStracking/IndexTableUtils.h" -#include "ITStracking/CapacityEstimator.h" +#include "ITSMFTTracking/CapacityEstimator.h" #include "ITStracking/TimeFrame.h" #include "ITStracking/Cell.h" -#include "ITStracking/BoundedAllocator.h" +#include "ITSMFTTracking/BoundedAllocator.h" #include "ITStracking/TrackExtensionHypothesis.h" #include "ITStracking/TrackFollower.h" #include "ITStracking/TrackITSInternal.h" diff --git a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TrackingInterface.h b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TrackingInterface.h index 14c5d6a62e0ad..f3fd4c26aeacd 100644 --- a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TrackingInterface.h +++ b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TrackingInterface.h @@ -19,7 +19,7 @@ #include "ITStracking/TrackerTraits.h" #include "ITStracking/Vertexer.h" #include "ITStracking/VertexerTraits.h" -#include "ITStracking/BoundedAllocator.h" +#include "ITSMFTTracking/BoundedAllocator.h" #include "DataFormatsParameters/GRPObject.h" #include "DataFormatsITSMFT/TopologyDictionary.h" #include "DataFormatsCalibration/MeanVertexObject.h" diff --git a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/Tracklet.h b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/Tracklet.h index be9d8840d88f5..ab55c77e373e7 100644 --- a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/Tracklet.h +++ b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/Tracklet.h @@ -16,7 +16,7 @@ #ifndef TRACKINGITS_INCLUDE_TRACKLET_H_ #define TRACKINGITS_INCLUDE_TRACKLET_H_ -#include "ITStracking/Constants.h" +#include "ITSMFTTracking/Constants.h" #include "DataFormatsITS/TimeEstBC.h" #include "ITStracking/Cluster.h" #include "MathUtils/Utils.h" diff --git a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/Vertexer.h b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/Vertexer.h index 59c22b505bc94..76647d923e2aa 100644 --- a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/Vertexer.h +++ b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/Vertexer.h @@ -26,11 +26,11 @@ #include -#include "ITStracking/Constants.h" +#include "ITSMFTTracking/Constants.h" #include "ITStracking/Configuration.h" #include "ITStracking/TimeFrame.h" #include "ITStracking/VertexerTraits.h" -#include "ITStracking/BoundedAllocator.h" +#include "ITSMFTTracking/BoundedAllocator.h" namespace o2::its { diff --git a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/VertexerTraits.h b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/VertexerTraits.h index daf8d708e1e23..0182e30cbf4ed 100644 --- a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/VertexerTraits.h +++ b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/VertexerTraits.h @@ -21,7 +21,7 @@ #include #include -#include "ITStracking/BoundedAllocator.h" +#include "ITSMFTTracking/BoundedAllocator.h" #include "ITStracking/Cluster.h" #include "ITStracking/ClusterLines.h" #include "ITStracking/Configuration.h" @@ -29,7 +29,7 @@ #include "ITStracking/IndexTableUtils.h" #include "ITStracking/TimeFrame.h" #include "ITStracking/Tracklet.h" -#include "ITStracking/MathUtils.h" +#include "ITSMFTTracking/MathUtils.h" #include "GPUCommonDef.h" #include "GPUCommonMath.h" diff --git a/Detectors/ITSMFT/ITS/tracking/src/CapacityEstimator.cxx b/Detectors/ITSMFT/ITS/tracking/src/CapacityEstimator.cxx deleted file mode 100644 index a1171fbdba326..0000000000000 --- a/Detectors/ITSMFT/ITS/tracking/src/CapacityEstimator.cxx +++ /dev/null @@ -1,179 +0,0 @@ -// Copyright 2019-2020 CERN and copyright holders of ALICE O2. -// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. -// All rights not expressly granted are reserved. -// -// This software is distributed under the terms of the GNU General Public -// License v3 (GPL Version 3), copied verbatim in the file "COPYING". -// -// In applying this license CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -#include "ITStracking/CapacityEstimator.h" - -#include -#include -#include -#include -#include -#include -#include - -#include "Framework/Logger.h" - -namespace o2::its -{ - -struct CapacityEstimator::Impl { - struct Entry { - float ratio{0.f}; - float margin{0.f}; - size_t maxEmitted{0}; - uint32_t nSamples{0}; - uint32_t nLowStreak{0}; - uint32_t nOverflows{0}; - }; - - explicit Impl(Config config) : cfg{config} {} - - Config cfg; - mutable std::mutex mutex; - std::unordered_map entries; -}; - -CapacityEstimator::CapacityEstimator() : CapacityEstimator{Config{}} {} - -CapacityEstimator::CapacityEstimator(Config cfg) : mImpl{std::make_unique(cfg)} {} - -CapacityEstimator::~CapacityEstimator() = default; - -void CapacityEstimator::reset() -{ - std::lock_guard lock{mImpl->mutex}; - mImpl->entries.clear(); -} - -size_t CapacityEstimator::capacity(uint64_t key, double scale) const -{ - if (!(scale > 0.)) { - return 0; - } - std::lock_guard lock{mImpl->mutex}; - const auto it = mImpl->entries.find(key); - if (it == mImpl->entries.end() || it->second.nSamples == 0) { - return mImpl->cfg.floorSlots; - } - const auto& e = it->second; - const double raw = double(e.ratio) * scale * double(e.margin); - if (!std::isfinite(raw) || raw < 0.) { - return mImpl->cfg.floorSlots; - } - // A ratio is only meaningful at the scale it was measured at. Learned on a handful of inputs it - // can be arbitrarily large, and applying it to a scale orders of magnitude bigger asks for a slab - // nobody can allocate. Bound the request by what this site has ever actually emitted: overshooting - // burns memory that a bump allocator cannot give back, undershooting only costs one retry. - const size_t ceiling = std::max(mImpl->cfg.floorSlots, static_cast(double(e.maxEmitted) * double(mImpl->cfg.marginMax))); - if (raw >= static_cast(ceiling)) { - return ceiling; - } - return std::max(mImpl->cfg.floorSlots, static_cast(std::ceil(raw))); -} - -size_t CapacityEstimator::peakCapacity(uint64_t key) const -{ - std::lock_guard lock{mImpl->mutex}; - const auto it = mImpl->entries.find(key); - if (it == mImpl->entries.end() || it->second.maxEmitted == 0) { - return mImpl->cfg.floorSlots; - } - const auto& e = it->second; - const double raw = double(e.maxEmitted) * double(e.margin); - if (!std::isfinite(raw) || raw >= static_cast(std::numeric_limits::max())) { - return std::numeric_limits::max(); - } - return std::max(mImpl->cfg.floorSlots, static_cast(std::ceil(raw))); -} - -double CapacityEstimator::expected(uint64_t key, double scale) const -{ - if (!(scale > 0.)) { - return 0.; - } - std::lock_guard lock{mImpl->mutex}; - const auto it = mImpl->entries.find(key); - if (it == mImpl->entries.end() || it->second.nSamples == 0) { - return 0.; - } - const double raw = double(it->second.ratio) * scale; - return std::isfinite(raw) && raw > 0. ? raw : 0.; -} - -void CapacityEstimator::update(uint64_t key, double scale, size_t emitted, size_t capacityUsed, bool overflowed, bool memoryLimited) -{ - if (!(scale > 0.)) { - return; - } - std::lock_guard lock{mImpl->mutex}; - auto& e = mImpl->entries[key]; - const auto& cfg = mImpl->cfg; - - const bool firstSample = e.nSamples == 0; - if (firstSample) { - e.margin = cfg.marginInit; - } - const auto sample = static_cast(double(emitted) / scale); - e.ratio = firstSample ? sample : (cfg.alpha * sample) + ((1.f - cfg.alpha) * e.ratio); - e.maxEmitted = std::max(e.maxEmitted, emitted); - ++e.nSamples; - - if (memoryLimited) { - e.nLowStreak = 0; - e.margin = std::max(cfg.marginMin, e.margin * cfg.marginDown); - return; - } - if (overflowed) { - ++e.nOverflows; - e.nLowStreak = 0; - if (!firstSample) { - const float shortfall = capacityUsed ? static_cast(double(emitted) / double(capacityUsed)) : cfg.marginUp; - e.margin = std::min(cfg.marginMax, e.margin * std::clamp(shortfall * cfg.marginOverflowSlack, 1.02f, cfg.marginUp)); - } - return; - } - const float util = capacityUsed ? float(double(emitted) / double(capacityUsed)) : 1.f; - if (util < cfg.lowWatermark) { - if (++e.nLowStreak >= cfg.decayAfter) { - e.margin = std::max(cfg.marginMin, e.margin * cfg.marginDown); - e.nLowStreak = 0; - } - } else if (e.nLowStreak > 0) { - --e.nLowStreak; - } -} - -void CapacityEstimator::print() const -{ - std::lock_guard lock{mImpl->mutex}; - std::vector keys; - keys.reserve(mImpl->entries.size()); - for (const auto& [key, _] : mImpl->entries) { - keys.push_back(key); - } - std::sort(keys.begin(), keys.end(), [](KeyType a, KeyType b) { - const auto da = decodeKey(a); - const auto db = decodeKey(b); - return std::tie(da.site, da.iteration, da.variant, da.slot) < - std::tie(db.site, db.iteration, db.variant, db.slot); - }); - if (keys.empty()) { - return; - } - LOGP(info, "Printing CapacityEstimators:"); - for (const auto key : keys) { - const auto& value = mImpl->entries.at(key); - const auto decoded = decodeKey(key); - LOGP(info, "\tSite:{} | iter:{} | var:({},{}) | slot:{} | ratio:{} | margin:{} | maxEmitted:{} | sam:{} | low:{} | overflows:{}", SlabSiteNames[decoded.site], decoded.iteration, getVariantHigh(decoded.variant), getVariantLow(decoded.variant), decoded.slot, value.ratio, value.margin, value.maxEmitted, value.nSamples, value.nLowStreak, value.nOverflows); - } -} - -} // namespace o2::its diff --git a/Detectors/ITSMFT/ITS/tracking/src/Configuration.cxx b/Detectors/ITSMFT/ITS/tracking/src/Configuration.cxx index eb4d90ee9d15f..5e8e24664ed78 100644 --- a/Detectors/ITSMFT/ITS/tracking/src/Configuration.cxx +++ b/Detectors/ITSMFT/ITS/tracking/src/Configuration.cxx @@ -16,9 +16,9 @@ #include #include "Framework/Logger.h" -#include "ITStracking/Constants.h" +#include "ITSMFTTracking/Constants.h" #include "ITStracking/Configuration.h" -#include "ITStracking/TrackingConfigParam.h" +#include "ITSMFTTracking/ITSTrackingConfigParam.h" using namespace o2::its; diff --git a/Detectors/ITSMFT/ITS/tracking/src/IOUtils.cxx b/Detectors/ITSMFT/ITS/tracking/src/IOUtils.cxx index e2ce374ed1600..9b1f9836a1053 100644 --- a/Detectors/ITSMFT/ITS/tracking/src/IOUtils.cxx +++ b/Detectors/ITSMFT/ITS/tracking/src/IOUtils.cxx @@ -20,7 +20,7 @@ #include #include "ITSBase/GeometryTGeo.h" -#include "ITStracking/TrackingConfigParam.h" +#include "ITSMFTTracking/ITSTrackingConfigParam.h" #include "ITSMFTReconstruction/ChipMappingITS.h" namespace diff --git a/Detectors/ITSMFT/ITS/tracking/src/LineVertexerHelpers.cxx b/Detectors/ITSMFT/ITS/tracking/src/LineVertexerHelpers.cxx index cbb8d52571ec9..807863fc54895 100644 --- a/Detectors/ITSMFT/ITS/tracking/src/LineVertexerHelpers.cxx +++ b/Detectors/ITSMFT/ITS/tracking/src/LineVertexerHelpers.cxx @@ -23,8 +23,8 @@ #include #include -#include "ITStracking/Constants.h" -#include "ITStracking/MathUtils.h" +#include "ITSMFTTracking/Constants.h" +#include "ITSMFTTracking/MathUtils.h" #include "ITStracking/LineVertexerHelpers.h" namespace o2::its::line_vertexer diff --git a/Detectors/ITSMFT/ITS/tracking/src/TimeFrame.cxx b/Detectors/ITSMFT/ITS/tracking/src/TimeFrame.cxx index 08c0164288388..9502f55bda9db 100644 --- a/Detectors/ITSMFT/ITS/tracking/src/TimeFrame.cxx +++ b/Detectors/ITSMFT/ITS/tracking/src/TimeFrame.cxx @@ -20,13 +20,13 @@ #include #include "ITStracking/TimeFrame.h" -#include "ITStracking/MathUtils.h" +#include "ITSMFTTracking/MathUtils.h" #include "DataFormatsITSMFT/CompCluster.h" #include "DataFormatsITSMFT/ROFRecord.h" #include "DataFormatsITSMFT/TopologyDictionary.h" #include "ITSBase/GeometryTGeo.h" #include "ITSMFTBase/SegmentationAlpide.h" -#include "ITStracking/BoundedAllocator.h" +#include "ITSMFTTracking/BoundedAllocator.h" namespace { @@ -41,6 +41,9 @@ struct ClusterHelper { namespace o2::its { +using o2::itsmft::tracking::clearResizeBoundedVector; +using o2::itsmft::tracking::deepVectorClear; + constexpr float DefClusErrorRow = o2::itsmft::SegmentationAlpide::PitchRow * 0.5; constexpr float DefClusErrorCol = o2::itsmft::SegmentationAlpide::PitchCol * 0.5; constexpr float DefClusError2Row = DefClusErrorRow * DefClusErrorRow; @@ -498,7 +501,7 @@ template void TimeFrame::setFrameworkAllocator(ExternalAllocator* ext) { mExternalAllocator = ext; - mExtMemoryPool = std::make_shared(mExternalAllocator); + mExtMemoryPool = std::make_shared(std::make_unique(mExternalAllocator)); } template diff --git a/Detectors/ITSMFT/ITS/tracking/src/Tracker.cxx b/Detectors/ITSMFT/ITS/tracking/src/Tracker.cxx index d8ff8442f908f..08d61ce23bf26 100644 --- a/Detectors/ITSMFT/ITS/tracking/src/Tracker.cxx +++ b/Detectors/ITSMFT/ITS/tracking/src/Tracker.cxx @@ -14,10 +14,10 @@ /// #include "ITStracking/Tracker.h" -#include "ITStracking/BoundedAllocator.h" -#include "ITStracking/Constants.h" +#include "ITSMFTTracking/BoundedAllocator.h" +#include "ITSMFTTracking/Constants.h" #include "ITStracking/TrackerTraits.h" -#include "ITStracking/TrackingConfigParam.h" +#include "ITSMFTTracking/ITSTrackingConfigParam.h" #include #include diff --git a/Detectors/ITSMFT/ITS/tracking/src/TrackerTraits.cxx b/Detectors/ITSMFT/ITS/tracking/src/TrackerTraits.cxx index 79511e6a9add5..f7caab856d8fd 100644 --- a/Detectors/ITSMFT/ITS/tracking/src/TrackerTraits.cxx +++ b/Detectors/ITSMFT/ITS/tracking/src/TrackerTraits.cxx @@ -30,13 +30,13 @@ #include "DetectorsBase/Propagator.h" #include "GPUCommonMath.h" -#include "ITStracking/BoundedAllocator.h" +#include "ITSMFTTracking/BoundedAllocator.h" #include "ITStracking/Cell.h" -#include "ITStracking/Constants.h" +#include "ITSMFTTracking/Constants.h" #include "ITStracking/IndexTableUtils.h" #include "ITStracking/LayerMask.h" -#include "ITStracking/ROFLookupTables.h" -#include "ITStracking/SlabBumpAllocator.h" +#include "ITSMFTTracking/ROFLookupTables.h" +#include "ITSMFTTracking/SlabBumpAllocator.h" #include "ITStracking/TrackerTraits.h" #include "ITStracking/TrackFollower.h" #include "ITStracking/TrackHelpers.h" @@ -45,6 +45,11 @@ namespace o2::its { +using o2::itsmft::tracking::deepVectorClear; +using o2::itsmft::tracking::GroupedSlabSink; +using o2::itsmft::tracking::SlabSite; +using o2::itsmft::tracking::UnorderedSlabSink; + template void TrackerTraits::initialiseTimeFrame(const int iteration) { @@ -200,7 +205,8 @@ void TrackerTraits::computeLayerTracklets(const int iteration, int iVer }); const auto st = sink.stats(); sink.finalizeUnordered(tracklets); - mTimeFrame->getCapacityEstimator().update(key, scale, st.emitted, st.capacity, st.overflowed, st.memoryLimited); + mTimeFrame->getCapacityEstimator().update(key, scale, st.requested, st.capacity, st.emitted, st.spilled, + st.overflowed, st.memoryLimited); }); } @@ -399,7 +405,8 @@ void TrackerTraits::computeLayerCells(const int iteration) }); const auto st = sink.stats(); sink.finalizeGrouped(size_t(currentLayerTrackletsNum), lut, layerCells); - mTimeFrame->getCapacityEstimator().update(key, scale, st.emitted, st.capacity, st.overflowed, st.memoryLimited); + mTimeFrame->getCapacityEstimator().update(key, scale, st.requested, st.capacity, st.emitted, st.spilled, + st.overflowed, st.memoryLimited); } else { lut.resize(currentLayerTrackletsNum + 1); for (int iTracklet{0}; iTracklet < currentLayerTrackletsNum; ++iTracklet) { @@ -549,7 +556,8 @@ void TrackerTraits::findCellsNeighbours(const int iteration) }); const auto st = sink.stats(); sink.finalizeUnordered(waveNeighbours); - mTimeFrame->getCapacityEstimator().update(key, scale, st.emitted, st.capacity, st.overflowed, st.memoryLimited); + mTimeFrame->getCapacityEstimator().update(key, scale, st.requested, st.capacity, st.emitted, st.spilled, + st.overflowed, st.memoryLimited); tbb::parallel_sort(waveNeighbours.begin(), waveNeighbours.end(), neighbourLess); } else { for (const int cellTopologyId : activeTopologies) { @@ -750,7 +758,8 @@ void TrackerTraits::processNeighbours(int iteration, int defaultCellTop }); const auto st = sink.stats(); sink.finalizeUnordered(updatedSeeds); - mTimeFrame->getCapacityEstimator().update(capacityKey, scale, st.emitted, st.capacity, st.overflowed, st.memoryLimited); + mTimeFrame->getCapacityEstimator().update(capacityKey, scale, st.requested, st.capacity, st.emitted, st.spilled, + st.overflowed, st.memoryLimited); } }); } diff --git a/Detectors/ITSMFT/ITS/tracking/src/TrackingInterface.cxx b/Detectors/ITSMFT/ITS/tracking/src/TrackingInterface.cxx index 83a1086ec5263..a0e8d708cffa2 100644 --- a/Detectors/ITSMFT/ITS/tracking/src/TrackingInterface.cxx +++ b/Detectors/ITSMFT/ITS/tracking/src/TrackingInterface.cxx @@ -22,8 +22,8 @@ #include "ITStracking/FastMultEstConfig.h" #include "ITStracking/FastMultEst.h" -#include "ITStracking/ROFLookupTables.h" -#include "ITStracking/TrackingConfigParam.h" +#include "ITSMFTTracking/ROFLookupTables.h" +#include "ITSMFTTracking/ITSTrackingConfigParam.h" #include "ITStracking/TrackingInterface.h" #include "DataFormatsITSMFT/ROFRecord.h" @@ -31,7 +31,7 @@ #include "DataFormatsTRD/TriggerRecord.h" #include "CommonDataFormat/IRFrame.h" #include "DetectorsBase/GRPGeomHelper.h" -#include "ITStracking/BoundedAllocator.h" +#include "ITSMFTTracking/BoundedAllocator.h" #include "Framework/InputRecordWalker.h" #include "Framework/DataRefUtils.h" #include "Framework/DeviceSpec.h" diff --git a/Detectors/ITSMFT/ITS/tracking/src/TrackingLinkDef.h b/Detectors/ITSMFT/ITS/tracking/src/TrackingLinkDef.h index 46af692fe0c15..5ba4e36dda875 100644 --- a/Detectors/ITSMFT/ITS/tracking/src/TrackingLinkDef.h +++ b/Detectors/ITSMFT/ITS/tracking/src/TrackingLinkDef.h @@ -33,12 +33,6 @@ #pragma link C++ class o2::its::ClusterLines + ; #pragma link C++ class std::vector < o2::its::ClusterLines> + ; -#pragma link C++ class o2::its::VertexerParamConfig + ; -#pragma link C++ class o2::conf::ConfigurableParamHelper < o2::its::VertexerParamConfig> + ; - -#pragma link C++ class o2::its::TrackerParamConfig + ; -#pragma link C++ class o2::conf::ConfigurableParamHelper < o2::its::TrackerParamConfig> + ; - #pragma link C++ class o2::its::FastMultEstConfig + ; #pragma link C++ class o2::conf::ConfigurableParamHelper < o2::its::FastMultEstConfig> + ; diff --git a/Detectors/ITSMFT/ITS/tracking/src/Vertexer.cxx b/Detectors/ITSMFT/ITS/tracking/src/Vertexer.cxx index d25d5efbec262..356229bc201c5 100644 --- a/Detectors/ITSMFT/ITS/tracking/src/Vertexer.cxx +++ b/Detectors/ITSMFT/ITS/tracking/src/Vertexer.cxx @@ -14,9 +14,9 @@ /// #include "ITStracking/Vertexer.h" -#include "ITStracking/BoundedAllocator.h" +#include "ITSMFTTracking/BoundedAllocator.h" #include "ITStracking/VertexerTraits.h" -#include "ITStracking/TrackingConfigParam.h" +#include "ITSMFTTracking/ITSTrackingConfigParam.h" namespace o2::its { diff --git a/Detectors/ITSMFT/ITS/tracking/src/VertexerTraits.cxx b/Detectors/ITSMFT/ITS/tracking/src/VertexerTraits.cxx index 15c26bc0a952d..de39210cc74eb 100644 --- a/Detectors/ITSMFT/ITS/tracking/src/VertexerTraits.cxx +++ b/Detectors/ITSMFT/ITS/tracking/src/VertexerTraits.cxx @@ -21,7 +21,7 @@ #include #include "ITStracking/VertexerTraits.h" -#include "ITStracking/BoundedAllocator.h" +#include "ITSMFTTracking/BoundedAllocator.h" #include "ITStracking/ClusterLines.h" #include "ITStracking/Definitions.h" #include "ITStracking/LineVertexerHelpers.h" @@ -34,6 +34,8 @@ namespace o2::its { +using o2::itsmft::tracking::deepVectorClear; + namespace { diff --git a/Detectors/ITSMFT/ITS/tracking/test/CMakeLists.txt b/Detectors/ITSMFT/ITS/tracking/test/CMakeLists.txt index c7c4d6dc101a2..b8858dda5ea92 100644 --- a/Detectors/ITSMFT/ITS/tracking/test/CMakeLists.txt +++ b/Detectors/ITSMFT/ITS/tracking/test/CMakeLists.txt @@ -9,24 +9,6 @@ # granted to it by virtue of its status as an Intergovernmental Organization # or submit itself to any jurisdiction. -o2_add_test(boundedmemoryresource - SOURCES testBoundedMemoryResource.cxx - COMPONENT_NAME its-tracking - LABELS "its;tracking" - PUBLIC_LINK_LIBRARIES O2::ITStracking) - -o2_add_test(slabbumpallocator - SOURCES testSlabBumpAllocator.cxx - COMPONENT_NAME its-tracking - LABELS "its;tracking" - PUBLIC_LINK_LIBRARIES O2::ITStracking TBB::tbb) - -o2_add_test(roflookuptables - SOURCES testROFLookupTables.cxx - COMPONENT_NAME its-tracking - LABELS "its;tracking" - PUBLIC_LINK_LIBRARIES O2::ITStracking) - o2_add_test(trackingtopology SOURCES testTrackingTopology.cxx COMPONENT_NAME its-tracking diff --git a/Detectors/ITSMFT/ITS/workflow/src/RecoWorkflow.cxx b/Detectors/ITSMFT/ITS/workflow/src/RecoWorkflow.cxx index 06b3f019a6be7..3a9c28b4935ec 100644 --- a/Detectors/ITSMFT/ITS/workflow/src/RecoWorkflow.cxx +++ b/Detectors/ITSMFT/ITS/workflow/src/RecoWorkflow.cxx @@ -16,7 +16,7 @@ #include "ITSMFTWorkflow/ClusterWriterSpec.h" #include "ITSWorkflow/TrackerSpec.h" #include "ITSWorkflow/TrackWriterSpec.h" -#include "ITStracking/TrackingConfigParam.h" +#include "ITSMFTTracking/ITSTrackingConfigParam.h" #include "ITSMFTWorkflow/DigitReaderSpec.h" #include "GlobalTrackingWorkflowWriters/IRFrameWriterSpec.h" #include "GPUWorkflow/GPUWorkflowSpec.h" diff --git a/Detectors/ITSMFT/ITS/workflow/src/TrackerSpec.cxx b/Detectors/ITSMFT/ITS/workflow/src/TrackerSpec.cxx index a198e638c5bac..cb53dae1ed905 100644 --- a/Detectors/ITSMFT/ITS/workflow/src/TrackerSpec.cxx +++ b/Detectors/ITSMFT/ITS/workflow/src/TrackerSpec.cxx @@ -19,7 +19,7 @@ #include "DataFormatsITSMFT/DPLAlpideParam.h" #include "ITSWorkflow/TrackerSpec.h" #include "ITStracking/Definitions.h" -#include "ITStracking/TrackingConfigParam.h" +#include "ITSMFTTracking/ITSTrackingConfigParam.h" namespace o2 { diff --git a/Detectors/ITSMFT/ITS/workflow/src/its-threshold-calib-workflow.cxx b/Detectors/ITSMFT/ITS/workflow/src/its-threshold-calib-workflow.cxx index a7d252d59b9f0..fe4176bea5269 100644 --- a/Detectors/ITSMFT/ITS/workflow/src/its-threshold-calib-workflow.cxx +++ b/Detectors/ITSMFT/ITS/workflow/src/its-threshold-calib-workflow.cxx @@ -11,7 +11,7 @@ #include "ITSWorkflow/ThresholdCalibratorSpec.h" #include "CommonUtils/ConfigurableParam.h" -#include "ITStracking/TrackingConfigParam.h" +#include "ITSMFTTracking/ITSTrackingConfigParam.h" #include "ITStracking/Configuration.h" #include "Framework/ConfigParamSpec.h" diff --git a/Detectors/ITSMFT/common/CMakeLists.txt b/Detectors/ITSMFT/common/CMakeLists.txt index 3991f3e67a82b..92b934020f109 100644 --- a/Detectors/ITSMFT/common/CMakeLists.txt +++ b/Detectors/ITSMFT/common/CMakeLists.txt @@ -12,5 +12,6 @@ add_subdirectory(base) add_subdirectory(simulation) add_subdirectory(reconstruction) +add_subdirectory(tracking) add_subdirectory(workflow) -add_subdirectory(data) \ No newline at end of file +add_subdirectory(data) diff --git a/Detectors/ITSMFT/common/tracking/CMakeLists.txt b/Detectors/ITSMFT/common/tracking/CMakeLists.txt new file mode 100644 index 0000000000000..af69c29a8583c --- /dev/null +++ b/Detectors/ITSMFT/common/tracking/CMakeLists.txt @@ -0,0 +1,31 @@ +# Copyright 2019-2020 CERN and copyright holders of ALICE O2. +# See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +# All rights not expressly granted are reserved. +# +# This software is distributed under the terms of the GNU General Public +# License v3 (GPL Version 3), copied verbatim in the file "COPYING". +# +# In applying this license CERN does not waive the privileges and immunities +# granted to it by virtue of its status as an Intergovernmental Organization +# or submit itself to any jurisdiction. + +o2_add_library(ITSMFTTracking + SOURCES src/BoundedAllocator.cxx + src/CapacityEstimator.cxx + src/ITSTrackingConfigParam.cxx + src/SlabBumpAllocator.cxx + PUBLIC_LINK_LIBRARIES O2::CommonConstants + O2::CommonDataFormat + O2::CommonUtils + O2::DataFormatsITS + O2::FrameworkLogger + O2::GPUCommon + O2::MathUtils + PRIVATE_LINK_LIBRARIES + TBB::tbb) + +o2_target_root_dictionary(ITSMFTTracking + HEADERS include/ITSMFTTracking/ITSTrackingConfigParam.h + LINKDEF src/ITSTrackingLinkDef.h) + +add_subdirectory(test) diff --git a/Detectors/ITSMFT/common/tracking/include/ITSMFTTracking/BoundedAllocator.h b/Detectors/ITSMFT/common/tracking/include/ITSMFTTracking/BoundedAllocator.h new file mode 100644 index 0000000000000..4f5c634a6f50b --- /dev/null +++ b/Detectors/ITSMFT/common/tracking/include/ITSMFTTracking/BoundedAllocator.h @@ -0,0 +1,166 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. +/// +/// \file BoundedAllocator.h +/// \brief +/// + +#ifndef ALICEO2_ITSMFT_TRACKING_BOUNDEDALLOCATOR_H_ +#define ALICEO2_ITSMFT_TRACKING_BOUNDEDALLOCATOR_H_ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace o2::itsmft::tracking +{ + +// #define BOUNDED_MR_STATS +class BoundedMemoryResource final : public std::pmr::memory_resource +{ + public: + class MemoryLimitExceeded final : public std::bad_alloc + { + public: + MemoryLimitExceeded(size_t attempted, size_t used, size_t max); + const char* what() const noexcept final; + + private: + std::string mMsg; + }; + + static std::pmr::memory_resource* cachingUpstream(); + + BoundedMemoryResource(size_t maxBytes = std::numeric_limits::max(), + std::pmr::memory_resource* upstream = nullptr); + + BoundedMemoryResource(std::unique_ptr upstream, + size_t maxBytes = std::numeric_limits::max()); + + [[nodiscard]] size_t getUsedMemory() const noexcept; + [[nodiscard]] size_t getMaxMemory() const noexcept; + [[nodiscard]] size_t getThrowCount() const noexcept; + [[nodiscard]] size_t getPeakMemory() const noexcept; + [[nodiscard]] size_t getPeakMemoryDelta() const noexcept; + + void resetPeakMemory() noexcept; + void setMaxMemory(size_t max); + +#if !defined(__HIPCC__) && !defined(__CUDACC__) + std::string asString() const; + void print() const; +#endif + + private: + void* do_allocate(size_t bytes, size_t alignment) final; + void do_deallocate(void* p, size_t bytes, size_t alignment) final; + bool do_is_equal(const std::pmr::memory_resource& other) const noexcept final; + + std::atomic mMaxMemory{std::numeric_limits::max()}; + std::atomic mCountThrow{0}; + std::atomic mUsedMemory{0}; + std::atomic mPeakUsedMemory{0}; + std::atomic mPeakBaselineMemory{0}; + std::unique_ptr mOwnedUpstream; + std::pmr::memory_resource* mUpstream{nullptr}; + +#ifdef BOUNDED_MR_STATS + struct Stats { + std::atomic peak{0}; + std::atomic live{0}; + std::atomic nAlloc{0}; + std::atomic nFree{0}; + std::atomic totalAlloc{0}; + std::atomic totalFreed{0}; + std::atomic maxAlign{0}; + std::atomic upstreamFailures{0}; + }; + Stats mStats{}; +#endif +}; + +template +using bounded_vector = std::pmr::vector; + +template +inline void deepVectorClear(std::vector& vec) +{ + std::vector().swap(vec); +} + +template +inline void deepVectorClear(bounded_vector& vec, std::pmr::memory_resource* mr = nullptr) +{ + std::pmr::memory_resource* tmr = (mr != nullptr) ? mr : vec.get_allocator().resource(); + vec.~bounded_vector(); + new (&vec) bounded_vector(std::pmr::polymorphic_allocator{tmr}); +} + +template +inline void deepVectorClear(std::vector>& vec, std::pmr::memory_resource* mr = nullptr) +{ + for (auto& v : vec) { + deepVectorClear(v, mr); + } +} + +template +inline void deepVectorClear(std::array, S>& arr, std::pmr::memory_resource* mr = nullptr) +{ + for (size_t i{0}; i < S; ++i) { + deepVectorClear(arr[i], mr); + } +} + +template +inline void clearResizeBoundedVector(bounded_vector& vec, size_t sz, std::pmr::memory_resource* mr = nullptr, T def = T()) +{ + std::pmr::memory_resource* tmr = (mr != nullptr) ? mr : vec.get_allocator().resource(); + vec.~bounded_vector(); + new (&vec) bounded_vector(sz, def, std::pmr::polymorphic_allocator{tmr}); +} + +template +inline void clearResizeBoundedVector(std::vector>& vec, size_t size, std::pmr::memory_resource* mr) +{ + vec.clear(); + vec.reserve(size); + for (size_t i = 0; i < size; ++i) { + vec.emplace_back(std::pmr::polymorphic_allocator>{mr}); + } +} + +template +inline void clearResizeBoundedArray(std::array, S>& arr, size_t size, std::pmr::memory_resource* mr = nullptr, T def = T()) +{ + for (size_t i{0}; i < S; ++i) { + clearResizeBoundedVector(arr[i], size, mr, def); + } +} + +template +inline std::vector toSTDVector(const bounded_vector& b) +{ + std::vector t(b.size()); + std::copy(b.cbegin(), b.cend(), t.begin()); + return t; +} + +} // namespace o2::itsmft::tracking + +#endif /* ALICEO2_ITSMFT_TRACKING_BOUNDEDALLOCATOR_H_ */ diff --git a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/CapacityEstimator.h b/Detectors/ITSMFT/common/tracking/include/ITSMFTTracking/CapacityEstimator.h similarity index 83% rename from Detectors/ITSMFT/ITS/tracking/include/ITStracking/CapacityEstimator.h rename to Detectors/ITSMFT/common/tracking/include/ITSMFTTracking/CapacityEstimator.h index de1ed88c516f3..43b4e277fc290 100644 --- a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/CapacityEstimator.h +++ b/Detectors/ITSMFT/common/tracking/include/ITSMFTTracking/CapacityEstimator.h @@ -13,8 +13,8 @@ /// \brief Cross-timeframe output-size prediction. /// -#ifndef TRACKINGITSU_INCLUDE_CAPACITYESTIMATOR_H_ -#define TRACKINGITSU_INCLUDE_CAPACITYESTIMATOR_H_ +#ifndef ALICEO2_ITSMFT_TRACKING_CAPACITYESTIMATOR_H_ +#define ALICEO2_ITSMFT_TRACKING_CAPACITYESTIMATOR_H_ #include #include @@ -22,7 +22,7 @@ #include #include -namespace o2::its +namespace o2::itsmft::tracking { enum SlabSite : uint8_t { @@ -63,6 +63,17 @@ class CapacityEstimator int slot; }; + struct Statistics { + size_t requested{0}; + size_t granted{0}; + size_t emitted{0}; + size_t spilled{0}; + size_t maxEmitted{0}; + uint32_t samples{0}; + uint32_t overflowEvents{0}; + uint32_t nLowStreak{0}; + }; + static constexpr KeyType makeKey(SlabSite site, int iteration, int variant, int slot) noexcept { return (static_cast(site) << 56) | @@ -102,10 +113,16 @@ class CapacityEstimator CapacityEstimator& operator=(const CapacityEstimator&) = delete; void reset(); + void beginTransaction(); + void commitTransaction() noexcept; + void rollbackTransaction() noexcept; size_t capacity(uint64_t key, double scale) const; size_t peakCapacity(uint64_t key) const; double expected(uint64_t key, double scale) const; + Statistics statistics(uint64_t key) const; void update(uint64_t key, double scale, size_t emitted, size_t capacityUsed, bool overflowed, bool memoryLimited); + void update(uint64_t key, double scale, size_t requested, size_t granted, size_t emitted, + size_t spilled, bool overflowed, bool memoryLimited); void print() const; private: @@ -133,6 +150,6 @@ int runOnSlab(CapacityEstimator& estimator, const CapacityEstimator::KeyType key return emitted; } -} // namespace o2::its +} // namespace o2::itsmft::tracking -#endif /* TRACKINGITSU_INCLUDE_CAPACITYESTIMATOR_H_ */ +#endif /* ALICEO2_ITSMFT_TRACKING_CAPACITYESTIMATOR_H_ */ diff --git a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/Constants.h b/Detectors/ITSMFT/common/tracking/include/ITSMFTTracking/Constants.h similarity index 100% rename from Detectors/ITSMFT/ITS/tracking/include/ITStracking/Constants.h rename to Detectors/ITSMFT/common/tracking/include/ITSMFTTracking/Constants.h diff --git a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TrackingConfigParam.h b/Detectors/ITSMFT/common/tracking/include/ITSMFTTracking/ITSTrackingConfigParam.h similarity index 99% rename from Detectors/ITSMFT/ITS/tracking/include/ITStracking/TrackingConfigParam.h rename to Detectors/ITSMFT/common/tracking/include/ITSMFTTracking/ITSTrackingConfigParam.h index ee67b7234450d..39aae6cb8330e 100644 --- a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TrackingConfigParam.h +++ b/Detectors/ITSMFT/common/tracking/include/ITSMFTTracking/ITSTrackingConfigParam.h @@ -15,7 +15,7 @@ #include #include "CommonUtils/ConfigurableParam.h" #include "CommonUtils/ConfigurableParamHelper.h" -#include "ITStracking/Constants.h" +#include "ITSMFTTracking/Constants.h" namespace o2::its { diff --git a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/MathUtils.h b/Detectors/ITSMFT/common/tracking/include/ITSMFTTracking/MathUtils.h similarity index 99% rename from Detectors/ITSMFT/ITS/tracking/include/ITStracking/MathUtils.h rename to Detectors/ITSMFT/common/tracking/include/ITSMFTTracking/MathUtils.h index fc903d32e5d5e..fd1595d425573 100644 --- a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/MathUtils.h +++ b/Detectors/ITSMFT/common/tracking/include/ITSMFTTracking/MathUtils.h @@ -17,7 +17,7 @@ #define O2_ITS_TRACKING_MATHUTILS_H_ #include "CommonConstants/MathConstants.h" -#include "ITStracking/Constants.h" +#include "ITSMFTTracking/Constants.h" #include "MathUtils/Utils.h" #include "GPUCommonMath.h" #include "GPUCommonDef.h" diff --git a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/ROFLookupTables.h b/Detectors/ITSMFT/common/tracking/include/ITSMFTTracking/ROFLookupTables.h similarity index 100% rename from Detectors/ITSMFT/ITS/tracking/include/ITStracking/ROFLookupTables.h rename to Detectors/ITSMFT/common/tracking/include/ITSMFTTracking/ROFLookupTables.h diff --git a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/SlabBumpAllocator.h b/Detectors/ITSMFT/common/tracking/include/ITSMFTTracking/SlabBumpAllocator.h similarity index 78% rename from Detectors/ITSMFT/ITS/tracking/include/ITStracking/SlabBumpAllocator.h rename to Detectors/ITSMFT/common/tracking/include/ITSMFTTracking/SlabBumpAllocator.h index e32516ea1e0e0..d55353938eee4 100644 --- a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/SlabBumpAllocator.h +++ b/Detectors/ITSMFT/common/tracking/include/ITSMFTTracking/SlabBumpAllocator.h @@ -13,28 +13,60 @@ /// \brief Lock-free slot allocator and single-pass sink. /// -#ifndef TRACKINGITSU_INCLUDE_SLABBUMPALLOCATOR_H_ -#define TRACKINGITSU_INCLUDE_SLABBUMPALLOCATOR_H_ +#ifndef ALICEO2_ITSMFT_TRACKING_SLABBUMPALLOCATOR_H_ +#define ALICEO2_ITSMFT_TRACKING_SLABBUMPALLOCATOR_H_ #include #include #include #include #include +#include #include #include #include #include #include +#include -#include -#include -#include +#include "ITSMFTTracking/BoundedAllocator.h" -#include "ITStracking/BoundedAllocator.h" +namespace o2::itsmft::tracking +{ + +namespace detail +{ -namespace o2::its +class ThreadLocalStorage { + public: + using Factory = void* (*)(void*); + using Deleter = void (*)(void*); + + ThreadLocalStorage(void* context, Factory factory, Deleter deleter); + ~ThreadLocalStorage(); + ThreadLocalStorage(const ThreadLocalStorage&) = delete; + ThreadLocalStorage& operator=(const ThreadLocalStorage&) = delete; + + void* local(); + std::vector values() const; + + private: + struct Impl; + std::unique_ptr mImpl; +}; + +using ParallelForBody = void (*)(void*, size_t, size_t); +void parallelFor(size_t begin, size_t end, size_t grainSize, void* context, ParallelForBody body); + +template +struct MoveContext { + T* staging; + int32_t* producerOf; + bounded_vector* destination; +}; + +} // namespace detail class SlabBumpAllocator { @@ -45,44 +77,17 @@ class SlabBumpAllocator bool valid() const noexcept { return n != 0; } }; - SlabBumpAllocator(size_t capacity, size_t slab) noexcept - : mCapacity{capacity}, mSlab{slab ? slab : size_t{1}} {} + SlabBumpAllocator(size_t capacity, size_t slab) noexcept; - Range grab() noexcept - { - if (mExhausted.load(std::memory_order_relaxed)) { - return {}; - } - const size_t base = mCursor.fetch_add(mSlab, std::memory_order_relaxed); - if (base >= mCapacity) { - mExhausted.store(true, std::memory_order_relaxed); - return {}; - } - return {.base = base, .n = std::min(mSlab, mCapacity - base)}; - } + Range grab() noexcept; [[nodiscard]] size_t capacity() const noexcept { return mCapacity; } [[nodiscard]] size_t slab() const noexcept { return mSlab; } - [[nodiscard]] size_t watermark() const noexcept - { - return std::min(mCursor.load(std::memory_order_relaxed), mCapacity); - } + [[nodiscard]] size_t watermark() const noexcept; - static size_t suggestSlab(size_t capacity, int nThreads, size_t minSlab = 256, size_t maxSlab = 4096) noexcept - { - const size_t t = static_cast(std::max(1, nThreads)); - const size_t fairShare = std::max(1, capacity / t); - return std::clamp(std::max(1, capacity / (8 * t)), - std::min(minSlab, fairShare), - std::min(maxSlab, fairShare)); - } + static size_t suggestSlab(size_t capacity, int nThreads, size_t minSlab = 256, size_t maxSlab = 4096) noexcept; - void resetCapacity(size_t capacity) noexcept - { - assert(mCursor.load(std::memory_order_relaxed) == 0); - mCapacity = capacity; - mExhausted.store(capacity == 0, std::memory_order_relaxed); - } + void resetCapacity(size_t capacity) noexcept; private: std::atomic mCursor{0}; @@ -204,7 +209,7 @@ class SlabSink SlabSink& operator=(const SlabSink&) = delete; ~SlabSink() = default; - Handle& local() { return mHandles.local(); } + Handle& local() { return *static_cast(mHandles.local()); } [[nodiscard]] std::pmr::memory_resource* memoryResource() const noexcept { return mMR; } @@ -214,7 +219,8 @@ class SlabSink s.requested = mRequested; s.capacity = mAlloc.capacity(); s.memoryLimited = s.capacity < s.requested; - for (const auto& h : mHandles) { + for (const void* value : mHandles.values()) { + const auto& h = *static_cast(value); s.emitted += h.emitted(); s.spilled += h.spilled(); } @@ -231,12 +237,15 @@ class SlabSink bounded_vector runs{mMR}; size_t nRuns{0}; - for (auto& h : mHandles) { + const auto handles = mHandles.values(); + for (void* value : handles) { + auto& h = *static_cast(value); h.closeRun(); nRuns += h.mRuns.size(); } runs.reserve(nRuns); - for (const auto& h : mHandles) { + for (const void* value : handles) { + const auto& h = *static_cast(value); runs.insert(runs.end(), h.mRuns.begin(), h.mRuns.end()); } std::sort(runs.begin(), runs.end(), [](const Run& a, const Run& b) { return a.begin < b.begin; }); @@ -255,7 +264,8 @@ class SlabSink mStaging.resize(outputSize); dest.swap(mStaging); - for (auto& h : mHandles) { + for (void* value : handles) { + auto& h = *static_cast(value); dest.insert(dest.end(), std::make_move_iterator(h.mSpill.begin()), std::make_move_iterator(h.mSpill.end())); deepVectorClear(h.mSpill, mMR); } @@ -278,7 +288,9 @@ class SlabSink ++lut[p + 1]; } } - for (const auto& h : mHandles) { + const auto handles = mHandles.values(); + for (const void* value : handles) { + const auto& h = *static_cast(value); for (const int32_t p : h.mSpillProducer) { ++lut[p + 1]; } @@ -293,7 +305,8 @@ class SlabSink const auto total = static_cast(lut.back()); dest.resize(total); - for (auto& h : mHandles) { + for (void* value : handles) { + auto& h = *static_cast(value); for (size_t i = 0; i < h.mSpill.size(); ++i) { dest[cursor[h.mSpillProducer[i]]++] = std::move(h.mSpill[i]); } @@ -302,14 +315,15 @@ class SlabSink } deepVectorClear(cursor, mMR); - T* const staging = mStaging.data(); - tbb::parallel_for(tbb::blocked_range(0, wm, 4096), [&](const tbb::blocked_range& r) { - for (size_t s = r.begin(); s != r.end(); ++s) { - const int d = mProducerOf[s]; + detail::MoveContext context{mStaging.data(), mProducerOf.data(), &dest}; + detail::parallelFor(0, wm, 4096, &context, [](void* opaque, size_t begin, size_t end) { + auto& ctx = *static_cast*>(opaque); + for (size_t s = begin; s != end; ++s) { + const int d = ctx.producerOf[s]; if (d < 0) { continue; } - dest[d] = std::move(staging[s]); + (*ctx.destination)[d] = std::move(ctx.staging[s]); } }); @@ -324,7 +338,7 @@ class SlabSink mAlloc{granted, cfg.slabOverride ? cfg.slabOverride : SlabBumpAllocator::suggestSlab(granted, cfg.nThreads)}, mStaging{mr}, mProducerOf{mr}, - mHandles{[this]() { return Handle{this}; }} + mHandles{this, &SlabSink::createHandle, &SlabSink::deleteHandle} { try { mStaging.resize(granted); @@ -377,12 +391,22 @@ class SlabSink } } + static void* createHandle(void* sink) + { + return new Handle{static_cast(sink)}; + } + + static void deleteHandle(void* handle) + { + delete static_cast(handle); + } + std::pmr::memory_resource* mMR{nullptr}; size_t mRequested{0}; SlabBumpAllocator mAlloc; bounded_vector mStaging; bounded_vector mProducerOf; - tbb::enumerable_thread_specific mHandles; + detail::ThreadLocalStorage mHandles; bool mFinalized{false}; }; @@ -392,6 +416,6 @@ using UnorderedSlabSink = SlabSink; template using GroupedSlabSink = SlabSink; -} // namespace o2::its +} // namespace o2::itsmft::tracking -#endif /* TRACKINGITSU_INCLUDE_SLABBUMPALLOCATOR_H_ */ +#endif /* ALICEO2_ITSMFT_TRACKING_SLABBUMPALLOCATOR_H_ */ diff --git a/Detectors/ITSMFT/common/tracking/src/BoundedAllocator.cxx b/Detectors/ITSMFT/common/tracking/src/BoundedAllocator.cxx new file mode 100644 index 0000000000000..152d3c8d7dd51 --- /dev/null +++ b/Detectors/ITSMFT/common/tracking/src/BoundedAllocator.cxx @@ -0,0 +1,200 @@ +// Copyright 2019-2026 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +#include "ITSMFTTracking/BoundedAllocator.h" + +#include +#include + +#include "GPUCommonLogger.h" +#include "ITSMFTTracking/Constants.h" + +namespace o2::itsmft::tracking +{ + +BoundedMemoryResource::MemoryLimitExceeded::MemoryLimitExceeded(size_t attempted, size_t used, size_t max) +{ + char buf[256]; + if (attempted != 0) { + (void)snprintf(buf, sizeof(buf), "Reached set memory limit (attempted: %zu, used: %zu, max: %zu)", attempted, used, max); + } else { + (void)snprintf(buf, sizeof(buf), "New set maximum below current used (newMax: %zu, used: %zu)", max, used); + } + mMsg = buf; +} + +const char* BoundedMemoryResource::MemoryLimitExceeded::what() const noexcept +{ + return mMsg.c_str(); +} + +std::pmr::memory_resource* BoundedMemoryResource::cachingUpstream() +{ + static std::pmr::synchronized_pool_resource pool{std::pmr::get_default_resource()}; + return &pool; +} + +BoundedMemoryResource::BoundedMemoryResource(size_t maxBytes, std::pmr::memory_resource* upstream) + : mMaxMemory(maxBytes), mUpstream(upstream != nullptr ? upstream : cachingUpstream()) +{ +} + +BoundedMemoryResource::BoundedMemoryResource(std::unique_ptr upstream, size_t maxBytes) + : mMaxMemory(maxBytes), mOwnedUpstream(std::move(upstream)), mUpstream(mOwnedUpstream.get()) +{ +} + +void* BoundedMemoryResource::do_allocate(size_t bytes, size_t alignment) +{ + size_t newUsed{0}; + size_t currentUsed{mUsedMemory.load(std::memory_order_relaxed)}; + do { + newUsed = currentUsed + bytes; + if (newUsed > mMaxMemory.load(std::memory_order_relaxed)) { + mCountThrow.fetch_add(1, std::memory_order_relaxed); + throw MemoryLimitExceeded(newUsed, currentUsed, mMaxMemory.load(std::memory_order_relaxed)); + } + } while (!mUsedMemory.compare_exchange_weak(currentUsed, newUsed, std::memory_order_acq_rel, std::memory_order_relaxed)); + + void* p{nullptr}; + try { + p = mUpstream->allocate(bytes, alignment); + } catch (...) { + mUsedMemory.fetch_sub(bytes, std::memory_order_relaxed); +#ifdef BOUNDED_MR_STATS + mStats.upstreamFailures.fetch_add(1, std::memory_order_relaxed); +#endif + throw; + } + + size_t peak = mPeakUsedMemory.load(std::memory_order_relaxed); + while (newUsed > peak && !mPeakUsedMemory.compare_exchange_weak(peak, newUsed, std::memory_order_relaxed)) { + } + +#ifdef BOUNDED_MR_STATS + size_t statsPeak = mStats.peak.load(std::memory_order_relaxed); + while (newUsed > statsPeak && !mStats.peak.compare_exchange_weak(statsPeak, newUsed, std::memory_order_relaxed)) { + } + mStats.live.fetch_add(1, std::memory_order_relaxed); + mStats.nAlloc.fetch_add(1, std::memory_order_relaxed); + mStats.totalAlloc.fetch_add(bytes, std::memory_order_relaxed); + + size_t maxAlignment = mStats.maxAlign.load(std::memory_order_relaxed); + while (alignment > maxAlignment && !mStats.maxAlign.compare_exchange_weak(maxAlignment, alignment, std::memory_order_relaxed)) { + } +#endif + return p; +} + +void BoundedMemoryResource::do_deallocate(void* p, size_t bytes, size_t alignment) +{ + mUpstream->deallocate(p, bytes, alignment); + mUsedMemory.fetch_sub(bytes, std::memory_order_relaxed); +#ifdef BOUNDED_MR_STATS + mStats.live.fetch_sub(1, std::memory_order_relaxed); + mStats.nFree.fetch_add(1, std::memory_order_relaxed); + mStats.totalFreed.fetch_add(bytes, std::memory_order_relaxed); +#endif +} + +bool BoundedMemoryResource::do_is_equal(const std::pmr::memory_resource& other) const noexcept +{ + return this == &other; +} + +size_t BoundedMemoryResource::getUsedMemory() const noexcept +{ + return mUsedMemory.load(std::memory_order_relaxed); +} + +size_t BoundedMemoryResource::getMaxMemory() const noexcept +{ + return mMaxMemory.load(std::memory_order_relaxed); +} + +size_t BoundedMemoryResource::getThrowCount() const noexcept +{ + return mCountThrow.load(std::memory_order_relaxed); +} + +size_t BoundedMemoryResource::getPeakMemory() const noexcept +{ + return mPeakUsedMemory.load(std::memory_order_relaxed); +} + +size_t BoundedMemoryResource::getPeakMemoryDelta() const noexcept +{ + const size_t peak = mPeakUsedMemory.load(std::memory_order_relaxed); + const size_t baseline = mPeakBaselineMemory.load(std::memory_order_relaxed); + return peak > baseline ? peak - baseline : 0; +} + +void BoundedMemoryResource::resetPeakMemory() noexcept +{ + const size_t used = mUsedMemory.load(std::memory_order_acquire); + mPeakBaselineMemory.store(used, std::memory_order_release); + mPeakUsedMemory.store(used, std::memory_order_release); +} + +void BoundedMemoryResource::setMaxMemory(size_t max) +{ + size_t current = mMaxMemory.load(std::memory_order_relaxed); + if (max == current) { + return; + } + for (;;) { + const size_t used = mUsedMemory.load(std::memory_order_acquire); + if (used > max) { + mCountThrow.fetch_add(1, std::memory_order_relaxed); + throw MemoryLimitExceeded(0, used, max); + } + if (mMaxMemory.compare_exchange_weak(current, max, std::memory_order_release, std::memory_order_relaxed)) { + return; + } + if (current == max) { + return; + } + } +} + +std::string BoundedMemoryResource::asString() const +{ + const auto throwCount = mCountThrow.load(std::memory_order_relaxed); + const auto used = static_cast(mUsedMemory.load(std::memory_order_relaxed)); + const auto peak = static_cast(mPeakUsedMemory.load(std::memory_order_relaxed)); + const auto peakDelta = static_cast(getPeakMemoryDelta()); + const auto maxMemory = mMaxMemory.load(std::memory_order_relaxed); + std::string result; + if (maxMemory == std::numeric_limits::max()) { + result += std::format("maxthrow={} maxmem=unbounded used={:.2f} GB stagepeak={:.2f} GB stagealloc={:.2f} GB", throwCount, used / o2::its::constants::GB, peak / o2::its::constants::GB, peakDelta / o2::its::constants::GB); + } else { + result += std::format("maxthrow={} maxmem={:.2f} GB used={:.2f} GB ({:.2f}%) stagepeak={:.2f} GB stagealloc={:.2f} GB", throwCount, static_cast(maxMemory) / o2::its::constants::GB, used / o2::its::constants::GB, 100.0 * used / static_cast(maxMemory), peak / o2::its::constants::GB, peakDelta / o2::its::constants::GB); + } +#ifdef BOUNDED_MR_STATS + result += std::format(" peak={:.2f} GB live={} nAlloc={} nFree={} totalAlloc={:.2f} GB totalFreed={:.2f} GB maxAlign={} upstreamFail={}", + static_cast(mStats.peak.load(std::memory_order_relaxed)) / o2::its::constants::GB, + mStats.live.load(std::memory_order_relaxed), + mStats.nAlloc.load(std::memory_order_relaxed), + mStats.nFree.load(std::memory_order_relaxed), + static_cast(mStats.totalAlloc.load(std::memory_order_relaxed)) / o2::its::constants::GB, + static_cast(mStats.totalFreed.load(std::memory_order_relaxed)) / o2::its::constants::GB, + mStats.maxAlign.load(std::memory_order_relaxed), + mStats.upstreamFailures.load(std::memory_order_relaxed)); +#endif + return result; +} + +void BoundedMemoryResource::print() const +{ + LOGP(info, "{}", asString()); +} + +} // namespace o2::itsmft::tracking diff --git a/Detectors/ITSMFT/common/tracking/src/CapacityEstimator.cxx b/Detectors/ITSMFT/common/tracking/src/CapacityEstimator.cxx new file mode 100644 index 0000000000000..f54680ae9df71 --- /dev/null +++ b/Detectors/ITSMFT/common/tracking/src/CapacityEstimator.cxx @@ -0,0 +1,274 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +#include "ITSMFTTracking/CapacityEstimator.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "Framework/Logger.h" + +namespace o2::itsmft::tracking +{ + +struct CapacityEstimator::Impl { + struct Entry { + float ratio{0.f}; + float margin{0.f}; + Statistics statistics{}; + }; + + struct UndoRecord { + bool existed{false}; + Entry previous{}; + }; + + explicit Impl(Config config) : cfg{config} {} + + Config cfg; + mutable std::mutex mutex; + std::unordered_map entries; + std::unordered_map undo; + bool transactionActive{false}; + + void checkpointBeforeUpdate(KeyType key) + { + if (!transactionActive || undo.find(key) != undo.end()) { + return; + } + const auto current = entries.find(key); + if (current == entries.end()) { + undo.emplace(key, UndoRecord{}); + } else { + undo.emplace(key, UndoRecord{.existed = true, .previous = current->second}); + } + } + + void observe(KeyType key, double scale, size_t requested, size_t granted, size_t emitted, + size_t spilled, bool overflowed, bool memoryLimited) + { + // Record the first-touch undo state before entries[key] can insert or the + // existing live entry can be modified. If undo insertion throws, the live + // estimator remains unchanged and Tracker's failure path can roll back the + // transaction without observing a partial update. + checkpointBeforeUpdate(key); + auto& e = entries[key]; + auto& statistics = e.statistics; + statistics.requested += requested; + statistics.granted += granted; + statistics.emitted += emitted; + statistics.spilled += spilled; + + const bool firstSample = statistics.samples == 0; + if (firstSample) { + e.margin = cfg.marginInit; + } + const auto sample = static_cast(double(emitted) / scale); + e.ratio = firstSample ? sample : (cfg.alpha * sample) + ((1.f - cfg.alpha) * e.ratio); + statistics.maxEmitted = std::max(statistics.maxEmitted, emitted); + ++statistics.samples; + + if (memoryLimited) { + statistics.nLowStreak = 0; + e.margin = std::max(cfg.marginMin, e.margin * cfg.marginDown); + return; + } + if (overflowed) { + ++statistics.overflowEvents; + statistics.nLowStreak = 0; + if (!firstSample) { + const float shortfall = granted ? static_cast(double(emitted) / double(granted)) : cfg.marginUp; + e.margin = std::min(cfg.marginMax, e.margin * std::clamp(shortfall * cfg.marginOverflowSlack, 1.02f, cfg.marginUp)); + } + return; + } + const float util = granted ? float(double(emitted) / double(granted)) : 1.f; + if (util < cfg.lowWatermark) { + if (++statistics.nLowStreak >= cfg.decayAfter) { + e.margin = std::max(cfg.marginMin, e.margin * cfg.marginDown); + statistics.nLowStreak = 0; + } + } else if (statistics.nLowStreak > 0) { + --statistics.nLowStreak; + } + } +}; + +CapacityEstimator::CapacityEstimator() : CapacityEstimator{Config{}} {} + +CapacityEstimator::CapacityEstimator(Config cfg) : mImpl{std::make_unique(cfg)} {} + +CapacityEstimator::~CapacityEstimator() = default; + +void CapacityEstimator::reset() +{ + std::lock_guard lock{mImpl->mutex}; + mImpl->entries.clear(); + mImpl->undo.clear(); + mImpl->transactionActive = false; +} + +void CapacityEstimator::beginTransaction() +{ + std::lock_guard lock{mImpl->mutex}; + if (mImpl->transactionActive) { + throw std::logic_error{"CapacityEstimator transaction already active"}; + } + assert(mImpl->undo.empty()); + mImpl->transactionActive = true; +} + +void CapacityEstimator::commitTransaction() noexcept +{ + std::lock_guard lock{mImpl->mutex}; + mImpl->undo.clear(); + mImpl->transactionActive = false; +} + +void CapacityEstimator::rollbackTransaction() noexcept +{ + std::lock_guard lock{mImpl->mutex}; + if (!mImpl->transactionActive) { + return; + } + for (const auto& [key, record] : mImpl->undo) { + if (record.existed) { + const auto current = mImpl->entries.find(key); + assert(current != mImpl->entries.end()); + current->second = record.previous; + } else { + mImpl->entries.erase(key); + } + } + mImpl->undo.clear(); + mImpl->transactionActive = false; +} + +size_t CapacityEstimator::capacity(uint64_t key, double scale) const +{ + if (!(scale > 0.)) { + return 0; + } + std::lock_guard lock{mImpl->mutex}; + const auto it = mImpl->entries.find(key); + if (it == mImpl->entries.end() || it->second.statistics.samples == 0) { + return mImpl->cfg.floorSlots; + } + const auto& e = it->second; + const double raw = double(e.ratio) * scale * double(e.margin); + if (!std::isfinite(raw) || raw < 0.) { + return mImpl->cfg.floorSlots; + } + // A ratio is only meaningful at the scale it was measured at. Learned on a handful of inputs it + // can be arbitrarily large, and applying it to a scale orders of magnitude bigger asks for a slab + // nobody can allocate. Bound the request by what this site has ever actually emitted: overshooting + // burns memory that a bump allocator cannot give back, undershooting only costs one retry. + const size_t ceiling = std::max(mImpl->cfg.floorSlots, static_cast(double(e.statistics.maxEmitted) * double(mImpl->cfg.marginMax))); + if (raw >= static_cast(ceiling)) { + return ceiling; + } + return std::max(mImpl->cfg.floorSlots, static_cast(std::ceil(raw))); +} + +size_t CapacityEstimator::peakCapacity(uint64_t key) const +{ + std::lock_guard lock{mImpl->mutex}; + const auto it = mImpl->entries.find(key); + if (it == mImpl->entries.end() || it->second.statistics.maxEmitted == 0) { + return mImpl->cfg.floorSlots; + } + const auto& e = it->second; + const double raw = double(e.statistics.maxEmitted) * double(e.margin); + if (!std::isfinite(raw) || raw >= static_cast(std::numeric_limits::max())) { + return std::numeric_limits::max(); + } + return std::max(mImpl->cfg.floorSlots, static_cast(std::ceil(raw))); +} + +double CapacityEstimator::expected(uint64_t key, double scale) const +{ + if (!(scale > 0.)) { + return 0.; + } + std::lock_guard lock{mImpl->mutex}; + const auto it = mImpl->entries.find(key); + if (it == mImpl->entries.end() || it->second.statistics.samples == 0) { + return 0.; + } + const double raw = double(it->second.ratio) * scale; + return std::isfinite(raw) && raw > 0. ? raw : 0.; +} + +CapacityEstimator::Statistics CapacityEstimator::statistics(uint64_t key) const +{ + std::lock_guard lock{mImpl->mutex}; + const auto it = mImpl->entries.find(key); + if (it == mImpl->entries.end()) { + return {}; + } + return it->second.statistics; +} + +void CapacityEstimator::update(uint64_t key, double scale, size_t emitted, size_t capacityUsed, bool overflowed, bool memoryLimited) +{ + if (!(scale > 0.)) { + return; + } + std::lock_guard lock{mImpl->mutex}; + mImpl->observe(key, scale, capacityUsed, capacityUsed, emitted, + overflowed && emitted > capacityUsed ? emitted - capacityUsed : 0, + overflowed, memoryLimited); +} + +void CapacityEstimator::update(uint64_t key, double scale, size_t requested, size_t granted, + size_t emitted, size_t spilled, bool overflowed, bool memoryLimited) +{ + if (!(scale > 0.)) { + return; + } + std::lock_guard lock{mImpl->mutex}; + mImpl->observe(key, scale, requested, granted, emitted, spilled, overflowed, memoryLimited); +} + +void CapacityEstimator::print() const +{ + std::lock_guard lock{mImpl->mutex}; + std::vector keys; + keys.reserve(mImpl->entries.size()); + for (const auto& [key, _] : mImpl->entries) { + keys.push_back(key); + } + std::sort(keys.begin(), keys.end(), [](KeyType a, KeyType b) { + const auto da = decodeKey(a); + const auto db = decodeKey(b); + return std::tie(da.site, da.iteration, da.variant, da.slot) < + std::tie(db.site, db.iteration, db.variant, db.slot); + }); + if (keys.empty()) { + return; + } + LOGP(info, "Printing CapacityEstimators:"); + for (const auto key : keys) { + const auto& value = mImpl->entries.at(key); + const auto& statistics = value.statistics; + const auto decoded = decodeKey(key); + LOGP(info, "\tSite:{} | iter:{} | var:({},{}) | slot:{} | ratio:{} | margin:{} | maxEmitted:{} | samples:{} | low:{} | requested:{} | granted:{} | emitted:{} | spilled:{} | overflows:{}", SlabSiteNames[decoded.site], decoded.iteration, getVariantHigh(decoded.variant), getVariantLow(decoded.variant), decoded.slot, value.ratio, value.margin, statistics.maxEmitted, statistics.samples, statistics.nLowStreak, statistics.requested, statistics.granted, statistics.emitted, statistics.spilled, statistics.overflowEvents); + } +} + +} // namespace o2::itsmft::tracking diff --git a/Detectors/ITSMFT/ITS/tracking/src/TrackingConfigParam.cxx b/Detectors/ITSMFT/common/tracking/src/ITSTrackingConfigParam.cxx similarity index 92% rename from Detectors/ITSMFT/ITS/tracking/src/TrackingConfigParam.cxx rename to Detectors/ITSMFT/common/tracking/src/ITSTrackingConfigParam.cxx index 47b5f8ffffdb1..063f51636894f 100644 --- a/Detectors/ITSMFT/ITS/tracking/src/TrackingConfigParam.cxx +++ b/Detectors/ITSMFT/common/tracking/src/ITSTrackingConfigParam.cxx @@ -9,6 +9,6 @@ // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. -#include "ITStracking/TrackingConfigParam.h" +#include "ITSMFTTracking/ITSTrackingConfigParam.h" O2ParamImpl(o2::its::VertexerParamConfig); O2ParamImpl(o2::its::TrackerParamConfig); diff --git a/Detectors/ITSMFT/common/tracking/src/ITSTrackingLinkDef.h b/Detectors/ITSMFT/common/tracking/src/ITSTrackingLinkDef.h new file mode 100644 index 0000000000000..630684d8bfda4 --- /dev/null +++ b/Detectors/ITSMFT/common/tracking/src/ITSTrackingLinkDef.h @@ -0,0 +1,24 @@ +// Copyright 2019-2026 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +#ifdef __CLING__ + +#pragma link off all globals; +#pragma link off all classes; +#pragma link off all functions; + +#pragma link C++ class o2::its::VertexerParamConfig + ; +#pragma link C++ class o2::conf::ConfigurableParamHelper < o2::its::VertexerParamConfig> + ; + +#pragma link C++ class o2::its::TrackerParamConfig + ; +#pragma link C++ class o2::conf::ConfigurableParamHelper < o2::its::TrackerParamConfig> + ; + +#endif diff --git a/Detectors/ITSMFT/common/tracking/src/SlabBumpAllocator.cxx b/Detectors/ITSMFT/common/tracking/src/SlabBumpAllocator.cxx new file mode 100644 index 0000000000000..9856d6bc888fa --- /dev/null +++ b/Detectors/ITSMFT/common/tracking/src/SlabBumpAllocator.cxx @@ -0,0 +1,108 @@ +// Copyright 2019-2026 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +#include "ITSMFTTracking/SlabBumpAllocator.h" + +#include +#include +#include + +namespace o2::itsmft::tracking +{ + +namespace detail +{ + +struct ThreadLocalStorage::Impl { + Impl(void* context_, Factory factory_, Deleter deleter_) + : context{context_}, factory{factory_}, deleter{deleter_}, values{[this] { return factory(context); }} + { + } + + ~Impl() + { + for (void* value : values) { + deleter(value); + } + } + + void* context; + Factory factory; + Deleter deleter; + tbb::enumerable_thread_specific values; +}; + +ThreadLocalStorage::ThreadLocalStorage(void* context, Factory factory, Deleter deleter) + : mImpl{std::make_unique(context, factory, deleter)} +{ +} + +ThreadLocalStorage::~ThreadLocalStorage() = default; + +void* ThreadLocalStorage::local() +{ + return mImpl->values.local(); +} + +std::vector ThreadLocalStorage::values() const +{ + return {mImpl->values.begin(), mImpl->values.end()}; +} + +void parallelFor(size_t begin, size_t end, size_t grainSize, void* context, ParallelForBody body) +{ + tbb::parallel_for(tbb::blocked_range{begin, end, grainSize}, [context, body](const tbb::blocked_range& range) { + body(context, range.begin(), range.end()); + }); +} + +} // namespace detail + +SlabBumpAllocator::SlabBumpAllocator(size_t capacity, size_t slab) noexcept + : mCapacity{capacity}, mSlab{slab ? slab : size_t{1}} +{ +} + +SlabBumpAllocator::Range SlabBumpAllocator::grab() noexcept +{ + if (mExhausted.load(std::memory_order_relaxed)) { + return {}; + } + const size_t base = mCursor.fetch_add(mSlab, std::memory_order_relaxed); + if (base >= mCapacity) { + mExhausted.store(true, std::memory_order_relaxed); + return {}; + } + return {.base = base, .n = std::min(mSlab, mCapacity - base)}; +} + +size_t SlabBumpAllocator::watermark() const noexcept +{ + return std::min(mCursor.load(std::memory_order_relaxed), mCapacity); +} + +size_t SlabBumpAllocator::suggestSlab(size_t capacity, int nThreads, size_t minSlab, size_t maxSlab) noexcept +{ + const size_t threads = static_cast(std::max(1, nThreads)); + const size_t fairShare = std::max(1, capacity / threads); + return std::clamp(std::max(1, capacity / (8 * threads)), + std::min(minSlab, fairShare), + std::min(maxSlab, fairShare)); +} + +void SlabBumpAllocator::resetCapacity(size_t capacity) noexcept +{ + assert(mCursor.load(std::memory_order_relaxed) == 0); + mCapacity = capacity; + mExhausted.store(capacity == 0, std::memory_order_relaxed); +} + +} // namespace o2::itsmft::tracking diff --git a/Detectors/ITSMFT/common/tracking/test/CMakeLists.txt b/Detectors/ITSMFT/common/tracking/test/CMakeLists.txt new file mode 100644 index 0000000000000..e7f6d20e32773 --- /dev/null +++ b/Detectors/ITSMFT/common/tracking/test/CMakeLists.txt @@ -0,0 +1,30 @@ +# Copyright 2019-2026 CERN and copyright holders of ALICE O2. +# See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +# All rights not expressly granted are reserved. +# +# This software is distributed under the terms of the GNU General Public +# License v3 (GPL Version 3), copied verbatim in the file "COPYING". +# +# In applying this license CERN does not waive the privileges and immunities +# granted to it by virtue of its status as an Intergovernmental Organization +# or submit itself to any jurisdiction. + +o2_add_test(roflookuptables + SOURCES testROFLookupTables.cxx + COMPONENT_NAME itsmft-tracking + LABELS "itsmft;tracking" + PUBLIC_LINK_LIBRARIES O2::ITSMFTTracking) + +o2_add_test(slabbumpallocator + SOURCES testSlabBumpAllocator.cxx + COMPONENT_NAME itsmft-tracking + LABELS "itsmft;tracking" + PUBLIC_LINK_LIBRARIES O2::GPUCommon + O2::ITSMFTTracking + TBB::tbb) + +o2_add_test(boundedmemoryresource + SOURCES testBoundedMemoryResource.cxx + COMPONENT_NAME itsmft-tracking + LABELS "itsmft;tracking" + PUBLIC_LINK_LIBRARIES O2::ITSMFTTracking) diff --git a/Detectors/ITSMFT/ITS/tracking/test/testBoundedMemoryResource.cxx b/Detectors/ITSMFT/common/tracking/test/testBoundedMemoryResource.cxx similarity index 97% rename from Detectors/ITSMFT/ITS/tracking/test/testBoundedMemoryResource.cxx rename to Detectors/ITSMFT/common/tracking/test/testBoundedMemoryResource.cxx index aae28f5cbc36e..81157568dadc6 100644 --- a/Detectors/ITSMFT/ITS/tracking/test/testBoundedMemoryResource.cxx +++ b/Detectors/ITSMFT/common/tracking/test/testBoundedMemoryResource.cxx @@ -9,15 +9,15 @@ // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. -#define BOOST_TEST_MODULE Test Flags +#define BOOST_TEST_MODULE Test BoundedMemoryResource #define BOOST_TEST_MAIN #define BOOST_TEST_DYN_LINK #include #include -#include "ITStracking/BoundedAllocator.h" +#include "ITSMFTTracking/BoundedAllocator.h" -using namespace o2::its; +using namespace o2::itsmft::tracking; using Vec = bounded_vector; auto getRandomInt(int min = -100, int max = 100) { diff --git a/Detectors/ITSMFT/ITS/tracking/test/testROFLookupTables.cxx b/Detectors/ITSMFT/common/tracking/test/testROFLookupTables.cxx similarity index 99% rename from Detectors/ITSMFT/ITS/tracking/test/testROFLookupTables.cxx rename to Detectors/ITSMFT/common/tracking/test/testROFLookupTables.cxx index 9626e42efd547..486af25ee72cb 100644 --- a/Detectors/ITSMFT/ITS/tracking/test/testROFLookupTables.cxx +++ b/Detectors/ITSMFT/common/tracking/test/testROFLookupTables.cxx @@ -10,12 +10,12 @@ // or submit itself to any jurisdiction. #include -#define BOOST_TEST_MODULE ITS ROFLookupTables +#define BOOST_TEST_MODULE ITSMFT ROFLookupTables #define BOOST_TEST_MAIN #define BOOST_TEST_DYN_LINK #include -#include "ITStracking/ROFLookupTables.h" +#include "ITSMFTTracking/ROFLookupTables.h" /// -------- Tests -------- // LayerTiming diff --git a/Detectors/ITSMFT/ITS/tracking/test/testSlabBumpAllocator.cxx b/Detectors/ITSMFT/common/tracking/test/testSlabBumpAllocator.cxx similarity index 75% rename from Detectors/ITSMFT/ITS/tracking/test/testSlabBumpAllocator.cxx rename to Detectors/ITSMFT/common/tracking/test/testSlabBumpAllocator.cxx index d72ede1a01c09..f12e1b3d2c1fd 100644 --- a/Detectors/ITSMFT/ITS/tracking/test/testSlabBumpAllocator.cxx +++ b/Detectors/ITSMFT/common/tracking/test/testSlabBumpAllocator.cxx @@ -21,15 +21,17 @@ #include #include #include +#include #include +#include #include -#include "ITStracking/BoundedAllocator.h" -#include "ITStracking/CapacityEstimator.h" -#include "ITStracking/SlabBumpAllocator.h" +#include "ITSMFTTracking/BoundedAllocator.h" +#include "ITSMFTTracking/CapacityEstimator.h" +#include "ITSMFTTracking/SlabBumpAllocator.h" -using namespace o2::its; +using namespace o2::itsmft::tracking; namespace { @@ -103,6 +105,36 @@ std::vector> reference(int nProducers, uint32_t seed) return out; } +struct EstimatorSnapshot { + size_t capacity{0}; + size_t peakCapacity{0}; + double expected{0.}; + CapacityEstimator::Statistics statistics{}; +}; + +EstimatorSnapshot snapshot(const CapacityEstimator& estimator, CapacityEstimator::KeyType key, double scale) +{ + return {.capacity = estimator.capacity(key, scale), + .peakCapacity = estimator.peakCapacity(key), + .expected = estimator.expected(key, scale), + .statistics = estimator.statistics(key)}; +} + +void checkSnapshot(const EstimatorSnapshot& actual, const EstimatorSnapshot& expected) +{ + BOOST_TEST(actual.capacity == expected.capacity); + BOOST_TEST(actual.peakCapacity == expected.peakCapacity); + BOOST_TEST(actual.expected == expected.expected); + BOOST_TEST(actual.statistics.requested == expected.statistics.requested); + BOOST_TEST(actual.statistics.granted == expected.statistics.granted); + BOOST_TEST(actual.statistics.emitted == expected.statistics.emitted); + BOOST_TEST(actual.statistics.spilled == expected.statistics.spilled); + BOOST_TEST(actual.statistics.maxEmitted == expected.statistics.maxEmitted); + BOOST_TEST(actual.statistics.samples == expected.statistics.samples); + BOOST_TEST(actual.statistics.overflowEvents == expected.statistics.overflowEvents); + BOOST_TEST(actual.statistics.nLowStreak == expected.statistics.nLowStreak); +} + void checkGrouped(int nProducers, size_t capacity, size_t slab, size_t maxMemory = std::numeric_limits::max()) { constexpr uint32_t seed = 7u; @@ -580,6 +612,106 @@ BOOST_AUTO_TEST_CASE(estimator_reset_forgets_inflated_margins) BOOST_TEST(est.capacity(key, scale) == 1024u); } +BOOST_AUTO_TEST_CASE(estimator_updates_immediately_and_commit_retains_updates) +{ + CapacityEstimator est; + const auto key = CapacityEstimator::makeKey(SlabSite::Neighbours, 2, 0, 4); + est.update(key, 100., 120, 100, 95, 7, true, false); + const auto immediate = est.statistics(key); + BOOST_TEST(immediate.requested == 120u); + BOOST_TEST(immediate.granted == 100u); + BOOST_TEST(immediate.emitted == 95u); + BOOST_TEST(immediate.spilled == 7u); + BOOST_TEST(immediate.maxEmitted == 95u); + BOOST_TEST(immediate.samples == 1u); + BOOST_TEST(immediate.overflowEvents == 1u); + BOOST_TEST(immediate.nLowStreak == 0u); + + est.beginTransaction(); + est.update(key, 100., 80, 80, 70, 0, false, false); + const auto beforeCommit = snapshot(est, key, 100.); + est.commitTransaction(); + checkSnapshot(snapshot(est, key, 100.), beforeCommit); +} + +BOOST_AUTO_TEST_CASE(estimator_rollback_restores_the_first_touch_state_exactly) +{ + CapacityEstimator est; + const auto key = CapacityEstimator::makeKey(SlabSite::Neighbours, 2, 0, 4); + constexpr double scale = 100.; + est.update(key, scale, 120, 100, 95, 7, true, false); + const auto before = snapshot(est, key, scale); + + est.beginTransaction(); + est.update(key, scale, 8000, 6000, 5500, 500, true, false); + est.update(key, scale, 40, 400, 20, 0, false, false); + const auto during = snapshot(est, key, scale); + BOOST_TEST(during.statistics.samples == before.statistics.samples + 2u); + BOOST_TEST(during.statistics.requested == before.statistics.requested + 8040u); + BOOST_TEST(during.peakCapacity > before.peakCapacity); + + est.rollbackTransaction(); + checkSnapshot(snapshot(est, key, scale), before); +} + +BOOST_AUTO_TEST_CASE(estimator_rollback_removes_a_transaction_created_key) +{ + CapacityEstimator est; + const auto key = CapacityEstimator::makeKey(SlabSite::Cells, 3, 0, 5); + constexpr double scale = 50.; + const auto absent = snapshot(est, key, scale); + + est.beginTransaction(); + est.update(key, scale, 90, 80, 75, 4, true, false); + BOOST_TEST(est.statistics(key).samples == 1u); + BOOST_TEST(est.expected(key, scale) > 0.); + est.rollbackTransaction(); + + checkSnapshot(snapshot(est, key, scale), absent); +} + +BOOST_AUTO_TEST_CASE(estimator_nested_transaction_rejection_preserves_the_active_transaction) +{ + CapacityEstimator est; + const auto key = CapacityEstimator::makeKey(SlabSite::Roads, 1, 0, 2); + constexpr double scale = 100.; + est.update(key, scale, 50, 50, 40, 0, false, false); + const auto before = snapshot(est, key, scale); + + est.beginTransaction(); + est.update(key, scale, 200, 180, 160, 5, true, false); + const auto beforeRejectedBegin = snapshot(est, key, scale); + BOOST_CHECK_THROW(est.beginTransaction(), std::logic_error); + checkSnapshot(snapshot(est, key, scale), beforeRejectedBegin); + est.rollbackTransaction(); + checkSnapshot(snapshot(est, key, scale), before); + + BOOST_CHECK_NO_THROW(est.beginTransaction()); + est.commitTransaction(); +} + +BOOST_AUTO_TEST_CASE(estimator_reset_clears_active_transaction_and_learning) +{ + CapacityEstimator est; + const auto existing = CapacityEstimator::makeKey(SlabSite::Tracklets, 1, 0, 2); + const auto created = CapacityEstimator::makeKey(SlabSite::Tracklets, 1, 0, 3); + constexpr double scale = 100.; + est.update(existing, scale, 200, 180, 170, 3, true, false); + est.beginTransaction(); + est.update(existing, scale, 300, 250, 240, 5, true, false); + est.update(created, scale, 100, 90, 80, 2, true, false); + + est.reset(); + BOOST_TEST(est.statistics(existing).samples == 0u); + BOOST_TEST(est.statistics(created).samples == 0u); + BOOST_TEST(est.capacity(existing, scale) == 1024u); + BOOST_TEST(est.expected(existing, scale) == 0.); + BOOST_CHECK_NO_THROW(est.beginTransaction()); + est.update(existing, scale, 60, 60, 50, 0, false, false); + est.commitTransaction(); + BOOST_TEST(est.statistics(existing).samples == 1u); +} + BOOST_AUTO_TEST_CASE(estimator_keys_separate_the_road_walk_steps) { const auto a = CapacityEstimator::makeKey(SlabSite::Roads, 0, CapacityEstimator::makeVariant(6, 4), 1); @@ -589,3 +721,16 @@ BOOST_AUTO_TEST_CASE(estimator_keys_separate_the_road_walk_steps) BOOST_TEST(a != c); BOOST_TEST(b != c); } + +BOOST_AUTO_TEST_CASE(estimator_keys_separate_stage_iteration_and_site) +{ + const auto edge0 = CapacityEstimator::makeKey(SlabSite::Tracklets, 0, 0, 0); + const auto edge1 = CapacityEstimator::makeKey(SlabSite::Tracklets, 0, 0, 1); + const auto nextIteration = CapacityEstimator::makeKey(SlabSite::Tracklets, 1, 0, 0); + const auto path0 = CapacityEstimator::makeKey(SlabSite::Cells, 0, 0, 0); + const auto path1 = CapacityEstimator::makeKey(SlabSite::Cells, 0, 0, 1); + BOOST_TEST(edge0 != edge1); + BOOST_TEST(edge0 != nextIteration); + BOOST_TEST(edge0 != path0); + BOOST_TEST(path0 != path1); +} diff --git a/Detectors/Upgrades/ALICE3/GlobalReconstruction/reconstruction/include/ALICE3GlobalReconstruction/TimeFrameMixin.h b/Detectors/Upgrades/ALICE3/GlobalReconstruction/reconstruction/include/ALICE3GlobalReconstruction/TimeFrameMixin.h index da1a80b77772b..4e08b460d999e 100644 --- a/Detectors/Upgrades/ALICE3/GlobalReconstruction/reconstruction/include/ALICE3GlobalReconstruction/TimeFrameMixin.h +++ b/Detectors/Upgrades/ALICE3/GlobalReconstruction/reconstruction/include/ALICE3GlobalReconstruction/TimeFrameMixin.h @@ -19,7 +19,7 @@ #include "CommonDataFormat/InteractionRecord.h" #include "DataFormatsTRKFT3/Cluster.h" #include "DataFormatsTRKFT3/ROFRecord.h" -#include "ITStracking/ROFLookupTables.h" +#include "ITSMFTTracking/ROFLookupTables.h" #include "ITStracking/TimeFrame.h" #include "SimulationDataFormat/MCCompLabel.h" #include "SimulationDataFormat/MCEventHeader.h" diff --git a/Detectors/Upgrades/ALICE3/GlobalReconstruction/workflow/include/ALICE3GlobalReconstructionWorkflow/TrackerSpec.h b/Detectors/Upgrades/ALICE3/GlobalReconstruction/workflow/include/ALICE3GlobalReconstructionWorkflow/TrackerSpec.h index 7bd85c0862ae7..d3e70bcc9c5a1 100644 --- a/Detectors/Upgrades/ALICE3/GlobalReconstruction/workflow/include/ALICE3GlobalReconstructionWorkflow/TrackerSpec.h +++ b/Detectors/Upgrades/ALICE3/GlobalReconstruction/workflow/include/ALICE3GlobalReconstructionWorkflow/TrackerSpec.h @@ -21,7 +21,7 @@ #include -#include "ITStracking/BoundedAllocator.h" +#include "ITSMFTTracking/BoundedAllocator.h" #include "ITStracking/ExternalAllocator.h" #include "ITStracking/TrackingInterface.h" #include "GPUDataTypesConfig.h" @@ -69,7 +69,7 @@ class TrackerDPL : public framework::Task bool mIsMC{true}; gpu::gpudatatypes::DeviceType mDeviceType{gpu::gpudatatypes::DeviceType::CPU}; int mTrackingThreads{1}; - std::shared_ptr mMemoryPool; + std::shared_ptr mMemoryPool; std::shared_ptr mGPUAllocator; std::shared_ptr mTaskArena; std::vector mTrackingParams; diff --git a/Detectors/Upgrades/ALICE3/GlobalReconstruction/workflow/src/TrackerSpec.cxx b/Detectors/Upgrades/ALICE3/GlobalReconstruction/workflow/src/TrackerSpec.cxx index f588b07c598bf..8e841000d483c 100644 --- a/Detectors/Upgrades/ALICE3/GlobalReconstruction/workflow/src/TrackerSpec.cxx +++ b/Detectors/Upgrades/ALICE3/GlobalReconstruction/workflow/src/TrackerSpec.cxx @@ -29,7 +29,7 @@ #include "Framework/ControlService.h" #include "Framework/ConfigParamRegistry.h" #include "Framework/CCDBParamSpec.h" -#include "ITStracking/TrackingConfigParam.h" +#include "ITSMFTTracking/ITSTrackingConfigParam.h" #include "SimulationDataFormat/MCEventHeader.h" #include "SimulationDataFormat/MCCompLabel.h" #include "SimulationDataFormat/MCTruthContainer.h" @@ -294,7 +294,7 @@ std::vector TrackerDPL::createTrackingParamsFromCon void TrackerDPL::run(ProcessingContext& pc) { if (mMemoryPool.get() == nullptr) { - mMemoryPool = std::make_shared(); + mMemoryPool = std::make_shared(); } if (mTaskArena.get() == nullptr) { mTaskArena = std::make_shared(mTrackingThreads); diff --git a/Detectors/Upgrades/ITS3/alignment/src/AlignmentSpec.cxx b/Detectors/Upgrades/ITS3/alignment/src/AlignmentSpec.cxx index d50eba24327ee..edba6718ab268 100644 --- a/Detectors/Upgrades/ITS3/alignment/src/AlignmentSpec.cxx +++ b/Detectors/Upgrades/ITS3/alignment/src/AlignmentSpec.cxx @@ -40,7 +40,7 @@ #include "ReconstructionDataFormats/VtxTrackRef.h" #include "ITS3Reconstruction/TopologyDictionary.h" #include "DataFormatsITSMFT/TopologyDictionary.h" -#include "ITStracking/MathUtils.h" +#include "ITSMFTTracking/MathUtils.h" #include "ITStracking/IOUtils.h" #include "ITS3Reconstruction/IOUtils.h" #include "ITS3Align/TrackFit.h" diff --git a/Detectors/Upgrades/ITS3/reconstruction/src/IOUtils.cxx b/Detectors/Upgrades/ITS3/reconstruction/src/IOUtils.cxx index 1a7bb8adb1d58..95f868bdaa615 100644 --- a/Detectors/Upgrades/ITS3/reconstruction/src/IOUtils.cxx +++ b/Detectors/Upgrades/ITS3/reconstruction/src/IOUtils.cxx @@ -11,12 +11,12 @@ #include "ITS3Reconstruction/IOUtils.h" #include "ITStracking/TimeFrame.h" -#include "ITStracking/BoundedAllocator.h" +#include "ITSMFTTracking/BoundedAllocator.h" #include "DataFormatsITSMFT/CompCluster.h" #include "DataFormatsITSMFT/ROFRecord.h" #include "ITS3Reconstruction/TopologyDictionary.h" #include "ITSBase/GeometryTGeo.h" -#include "ITStracking/TrackingConfigParam.h" +#include "ITSMFTTracking/ITSTrackingConfigParam.h" namespace o2::its3::ioutils { @@ -77,7 +77,7 @@ int loadROFrameDataITS3(its::TimeFrame<7>* tf, LOGP(fatal, "Received inconsistent number of rofs on layer:{} expected:{} received:{}", layer, timing.mNROFsTF, rofs.size()); } - its::bounded_vector clusterSizeVec(clusters.size(), 0, tf->getMemoryPool().get()); + itsmft::tracking::bounded_vector clusterSizeVec(clusters.size(), 0, tf->getMemoryPool().get()); for (size_t iRof{0}; iRof < rofs.size(); ++iRof) { const auto& rof = rofs[iRof]; diff --git a/Detectors/Upgrades/ITS3/workflow/src/TrackerSpec.cxx b/Detectors/Upgrades/ITS3/workflow/src/TrackerSpec.cxx index 94e711a05a2d6..7c1a87a778c8d 100644 --- a/Detectors/Upgrades/ITS3/workflow/src/TrackerSpec.cxx +++ b/Detectors/Upgrades/ITS3/workflow/src/TrackerSpec.cxx @@ -22,7 +22,7 @@ #include "DataFormatsITSMFT/ROFRecord.h" #include "DataFormatsITSMFT/PhysTrigger.h" -#include "ITStracking/TrackingConfigParam.h" +#include "ITSMFTTracking/ITSTrackingConfigParam.h" #include "DataFormatsITSMFT/DPLAlpideParam.h" #include "ITSBase/GeometryTGeo.h" diff --git a/Detectors/Upgrades/ITS3/workflow/src/its3-reco-workflow.cxx b/Detectors/Upgrades/ITS3/workflow/src/its3-reco-workflow.cxx index beb6815bc9bcd..ad76b60da5a06 100644 --- a/Detectors/Upgrades/ITS3/workflow/src/its3-reco-workflow.cxx +++ b/Detectors/Upgrades/ITS3/workflow/src/its3-reco-workflow.cxx @@ -11,7 +11,7 @@ #include "ITS3Workflow/RecoWorkflow.h" #include "CommonUtils/ConfigurableParam.h" -#include "ITStracking/TrackingConfigParam.h" +#include "ITSMFTTracking/ITSTrackingConfigParam.h" #include "ITStracking/Configuration.h" #include "DetectorsRaw/HBFUtilsInitializer.h" #include "Framework/CallbacksPolicy.h" diff --git a/GPU/Workflow/src/GPUWorkflowITS.cxx b/GPU/Workflow/src/GPUWorkflowITS.cxx index 2a0e36d65bb7a..794f7fc3fda22 100644 --- a/GPU/Workflow/src/GPUWorkflowITS.cxx +++ b/GPU/Workflow/src/GPUWorkflowITS.cxx @@ -22,7 +22,7 @@ #include "CommonUtils/ConfigurableParam.h" #include "CommonUtils/NameConf.h" #include "ITStracking/TrackingInterface.h" -#include "ITStracking/TrackingConfigParam.h" +#include "ITSMFTTracking/ITSTrackingConfigParam.h" #include #include diff --git a/GPU/Workflow/src/GPUWorkflowSpec.cxx b/GPU/Workflow/src/GPUWorkflowSpec.cxx index 5edf72eabd9b7..990873f053fe3 100644 --- a/GPU/Workflow/src/GPUWorkflowSpec.cxx +++ b/GPU/Workflow/src/GPUWorkflowSpec.cxx @@ -78,7 +78,7 @@ #include "GPUReconstructionConvert.h" #include "DetectorsRaw/RDHUtils.h" #include "ITStracking/TrackingInterface.h" -#include "ITStracking/TrackingConfigParam.h" +#include "ITSMFTTracking/ITSTrackingConfigParam.h" #include "GPUWorkflowInternal.h" #include "GPUDataTypesQA.h" // #include "Framework/ThreadPool.h"