From 24d008ecde88f59ff1da784f19763372f955d956 Mon Sep 17 00:00:00 2001 From: Maximiliano Puccio Date: Thu, 27 Aug 2026 09:02:21 +0200 Subject: [PATCH 1/7] ITSMFT: share tracking slab allocation primitives --- Detectors/ITSMFT/ITS/tracking/CMakeLists.txt | 6 +- .../GPU/ITStrackingGPU/TrackingKernels.h | 4 +- .../tracking/GPU/cuda/TrackerTraitsGPU.cxx | 4 + .../ITS/tracking/GPU/cuda/TrackingKernels.cu | 4 + .../include/ITStracking/BoundedAllocator.h | 302 +--------------- .../tracking/include/ITStracking/TimeFrame.h | 8 +- .../include/ITStracking/TrackerTraits.h | 2 +- .../ITS/tracking/src/CapacityEstimator.cxx | 179 ---------- .../ITSMFT/ITS/tracking/src/TimeFrame.cxx | 2 +- .../ITSMFT/ITS/tracking/src/TrackerTraits.cxx | 15 +- .../ITSMFT/ITS/tracking/test/CMakeLists.txt | 6 - Detectors/ITSMFT/common/CMakeLists.txt | 3 +- .../ITSMFT/common/tracking/CMakeLists.txt | 18 + .../include/ITSMFTTracking/BoundedAllocator.h | 324 ++++++++++++++++++ .../ITSMFTTracking}/CapacityEstimator.h | 31 +- .../ITSMFTTracking}/SlabBumpAllocator.h | 12 +- .../common/tracking/src/CapacityEstimator.cxx | 285 +++++++++++++++ .../common/tracking/test/CMakeLists.txt | 16 + .../tracking/test/testSlabBumpAllocator.cxx | 148 +++++++- 19 files changed, 860 insertions(+), 509 deletions(-) delete mode 100644 Detectors/ITSMFT/ITS/tracking/src/CapacityEstimator.cxx create mode 100644 Detectors/ITSMFT/common/tracking/CMakeLists.txt create mode 100644 Detectors/ITSMFT/common/tracking/include/ITSMFTTracking/BoundedAllocator.h rename Detectors/ITSMFT/{ITS/tracking/include/ITStracking => common/tracking/include/ITSMFTTracking}/CapacityEstimator.h (80%) rename Detectors/ITSMFT/{ITS/tracking/include/ITStracking => common/tracking/include/ITSMFTTracking}/SlabBumpAllocator.h (97%) create mode 100644 Detectors/ITSMFT/common/tracking/src/CapacityEstimator.cxx create mode 100644 Detectors/ITSMFT/common/tracking/test/CMakeLists.txt rename Detectors/ITSMFT/{ITS => common}/tracking/test/testSlabBumpAllocator.cxx (76%) diff --git a/Detectors/ITSMFT/ITS/tracking/CMakeLists.txt b/Detectors/ITSMFT/ITS/tracking/CMakeLists.txt index 17420d47a2732..07d40764d0498 100644 --- a/Detectors/ITSMFT/ITS/tracking/CMakeLists.txt +++ b/Detectors/ITSMFT/ITS/tracking/CMakeLists.txt @@ -13,7 +13,6 @@ o2_add_library(ITStracking TARGETVARNAME targetName SOURCES src/ClusterLines.cxx src/Cluster.cxx - src/CapacityEstimator.cxx src/Configuration.cxx src/FastMultEstConfig.cxx src/FastMultEst.cxx @@ -36,9 +35,14 @@ o2_add_library(ITStracking O2::ITSReconstruction O2::ITSMFTReconstruction O2::DataFormatsITS + O2::ITSMFTTrackingAllocation PRIVATE_LINK_LIBRARIES O2::Steer TBB::tbb) + +target_include_directories(${targetName} PUBLIC + $ + $) # target_compile_options(${targetName} PRIVATE -O0 -g -fPIC -fno-omit-frame-pointer) o2_add_library(ITSTrackingInterface diff --git a/Detectors/ITSMFT/ITS/tracking/GPU/ITStrackingGPU/TrackingKernels.h b/Detectors/ITSMFT/ITS/tracking/GPU/ITStrackingGPU/TrackingKernels.h index 94950c04877b9..00b2911a75458 100644 --- a/Detectors/ITSMFT/ITS/tracking/GPU/ITStrackingGPU/TrackingKernels.h +++ b/Detectors/ITSMFT/ITS/tracking/GPU/ITStrackingGPU/TrackingKernels.h @@ -17,7 +17,7 @@ #include #include "ITStracking/BoundedAllocator.h" -#include "ITStracking/CapacityEstimator.h" +#include "ITSMFTTracking/CapacityEstimator.h" #include "ITStracking/ROFLookupTables.h" #include "ITStracking/TrackingTopology.h" #include "ITStracking/TrackExtensionHypothesis.h" @@ -118,7 +118,7 @@ struct TrackingKernels { TrackSeed* seedsDevice, const int seedsCapacity, int& seedsCursor, - CapacityEstimator& estimator, + o2::itsmft::tracking::CapacityEstimator& estimator, const int iteration, const float bz, const float MaxChi2ClusterAttachment, diff --git a/Detectors/ITSMFT/ITS/tracking/GPU/cuda/TrackerTraitsGPU.cxx b/Detectors/ITSMFT/ITS/tracking/GPU/cuda/TrackerTraitsGPU.cxx index b80fb6452f8a9..96443cc6b2668 100644 --- a/Detectors/ITSMFT/ITS/tracking/GPU/cuda/TrackerTraitsGPU.cxx +++ b/Detectors/ITSMFT/ITS/tracking/GPU/cuda/TrackerTraitsGPU.cxx @@ -20,6 +20,10 @@ namespace o2::its { +using o2::itsmft::tracking::CapacityEstimator; +using o2::itsmft::tracking::runOnSlab; +using o2::itsmft::tracking::SlabSite; + template void TrackerTraitsGPU::initialiseTimeFrame(const int iteration) { diff --git a/Detectors/ITSMFT/ITS/tracking/GPU/cuda/TrackingKernels.cu b/Detectors/ITSMFT/ITS/tracking/GPU/cuda/TrackingKernels.cu index a179fd0962dd3..c44638466c01f 100644 --- a/Detectors/ITSMFT/ITS/tracking/GPU/cuda/TrackingKernels.cu +++ b/Detectors/ITSMFT/ITS/tracking/GPU/cuda/TrackingKernels.cu @@ -52,6 +52,10 @@ using namespace o2::track; namespace o2::its { + +using o2::itsmft::tracking::CapacityEstimator; +using o2::itsmft::tracking::runOnSlab; +using o2::itsmft::tracking::SlabSite; namespace gpu { diff --git a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/BoundedAllocator.h b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/BoundedAllocator.h index 2e47664ee78ea..70d6aed207092 100644 --- a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/BoundedAllocator.h +++ b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/BoundedAllocator.h @@ -16,305 +16,19 @@ #ifndef TRACKINGITSU_INCLUDE_BOUNDEDALLOCATOR_H_ #define TRACKINGITSU_INCLUDE_BOUNDEDALLOCATOR_H_ -#include -#include -#include -#include -#include - -#if !defined(__HIPCC__) && !defined(__CUDACC__) -#include -#include -#include "GPUCommonLogger.h" -#endif -#include "ITStracking/ExternalAllocator.h" -#include "ITStracking/Constants.h" +#include "ITSMFTTracking/BoundedAllocator.h" namespace o2::its { -// #define BOUNDED_MR_STATS -class BoundedMemoryResource final : public std::pmr::memory_resource -{ - public: - class MemoryLimitExceeded final : public std::bad_alloc - { - public: - MemoryLimitExceeded(size_t attempted, size_t used, size_t max) - { - char buf[256]; - if (attempted != 0) { - (void)snprintf(buf, sizeof(buf), "Reached set memory limit (attempted: %zu, used: %zu, max: %zu)", attempted, used, max); - } else { - (void)snprintf(buf, sizeof(buf), "New set maximum below current used (newMax: %zu, used: %zu)", max, used); - } - mMsg = buf; - } - const char* what() const noexcept final { return mMsg.c_str(); } - - private: - std::string mMsg; - }; - - static std::pmr::memory_resource* cachingUpstream() - { - static std::pmr::synchronized_pool_resource pool{std::pmr::get_default_resource()}; - return &pool; - } - - BoundedMemoryResource(size_t maxBytes = std::numeric_limits::max(), - std::pmr::memory_resource* upstream = nullptr) - : mMaxMemory(maxBytes), mUpstream(upstream != nullptr ? upstream : cachingUpstream()) {} - - BoundedMemoryResource(ExternalAllocator* alloc, - size_t maxBytes = std::numeric_limits::max()) - : mMaxMemory(maxBytes), - mAdaptor(std::make_unique(alloc)), - mUpstream(mAdaptor.get()) {} - - void* do_allocate(size_t bytes, size_t alignment) final - { - size_t new_used{0}; - size_t current_used{mUsedMemory.load(std::memory_order_relaxed)}; - do { - new_used = current_used + bytes; - if (new_used > mMaxMemory.load(std::memory_order_relaxed)) { - mCountThrow.fetch_add(1, std::memory_order_relaxed); - throw MemoryLimitExceeded(new_used, current_used, - mMaxMemory.load(std::memory_order_relaxed)); - } - } while (!mUsedMemory.compare_exchange_weak(current_used, new_used, - std::memory_order_acq_rel, - std::memory_order_relaxed)); - - void* p{nullptr}; - try { - p = mUpstream->allocate(bytes, alignment); - } catch (...) { - mUsedMemory.fetch_sub(bytes, std::memory_order_relaxed); -#ifdef BOUNDED_MR_STATS - mStats.upstreamFailures.fetch_add(1, std::memory_order_relaxed); -#endif - throw; - } - - size_t peak = mPeakUsedMemory.load(std::memory_order_relaxed); - while (new_used > peak && - !mPeakUsedMemory.compare_exchange_weak(peak, new_used, - std::memory_order_relaxed)) { - } - -#ifdef BOUNDED_MR_STATS - size_t statsPeak = mStats.peak.load(std::memory_order_relaxed); - while (new_used > statsPeak && - !mStats.peak.compare_exchange_weak(statsPeak, new_used, - std::memory_order_relaxed)) { - } - mStats.live.fetch_add(1, std::memory_order_relaxed); - mStats.nAlloc.fetch_add(1, std::memory_order_relaxed); - mStats.totalAlloc.fetch_add(bytes, std::memory_order_relaxed); - - size_t ma = mStats.maxAlign.load(std::memory_order_relaxed); - while (alignment > ma && !mStats.maxAlign.compare_exchange_weak(ma, alignment, std::memory_order_relaxed)) { - } -#endif - return p; - } - - void do_deallocate(void* p, size_t bytes, size_t alignment) final - { - mUpstream->deallocate(p, bytes, alignment); - mUsedMemory.fetch_sub(bytes, std::memory_order_relaxed); -#ifdef BOUNDED_MR_STATS - mStats.live.fetch_sub(1, std::memory_order_relaxed); - mStats.nFree.fetch_add(1, std::memory_order_relaxed); - mStats.totalFreed.fetch_add(bytes, std::memory_order_relaxed); -#endif - } - - bool do_is_equal(const std::pmr::memory_resource& other) const noexcept final - { - return this == &other; - } - - [[nodiscard]] size_t getUsedMemory() const noexcept - { - return mUsedMemory.load(std::memory_order_relaxed); - } - [[nodiscard]] size_t getMaxMemory() const noexcept - { - return mMaxMemory.load(std::memory_order_relaxed); - } - [[nodiscard]] size_t getThrowCount() const noexcept - { - return mCountThrow.load(std::memory_order_relaxed); - } - [[nodiscard]] size_t getPeakMemory() const noexcept - { - return mPeakUsedMemory.load(std::memory_order_relaxed); - } - [[nodiscard]] size_t getPeakMemoryDelta() const noexcept - { - const size_t peak = mPeakUsedMemory.load(std::memory_order_relaxed); - const size_t baseline = mPeakBaselineMemory.load(std::memory_order_relaxed); - return peak > baseline ? peak - baseline : 0; - } - - void resetPeakMemory() noexcept - { - const size_t used = mUsedMemory.load(std::memory_order_acquire); - mPeakBaselineMemory.store(used, std::memory_order_release); - mPeakUsedMemory.store(used, std::memory_order_release); - } - - void setMaxMemory(size_t max) - { - size_t current = mMaxMemory.load(std::memory_order_relaxed); - if (max == current) { - return; - } - for (;;) { - size_t used = mUsedMemory.load(std::memory_order_acquire); - if (used > max) { - mCountThrow.fetch_add(1, std::memory_order_relaxed); - throw MemoryLimitExceeded(0, used, max); - } - if (mMaxMemory.compare_exchange_weak(current, max, - std::memory_order_release, - std::memory_order_relaxed)) { - return; - } - if (current == max) { - return; - } - } - } - -#if !defined(__HIPCC__) && !defined(__CUDACC__) - std::string asString() const - { - const auto throw_ = mCountThrow.load(std::memory_order_relaxed); - const auto used = static_cast(mUsedMemory.load(std::memory_order_relaxed)); - const auto peak = static_cast(mPeakUsedMemory.load(std::memory_order_relaxed)); - const auto peakDelta = static_cast(getPeakMemoryDelta()); - const auto maxm = mMaxMemory.load(std::memory_order_relaxed); - std::string ret; - if (maxm == std::numeric_limits::max()) { - ret += std::format("maxthrow={} maxmem=unbounded used={:.2f} GB stagepeak={:.2f} GB stagealloc={:.2f} GB", throw_, used / constants::GB, peak / constants::GB, peakDelta / constants::GB); - } else { - ret += std::format("maxthrow={} maxmem={:.2f} GB used={:.2f} GB ({:.2f}%) stagepeak={:.2f} GB stagealloc={:.2f} GB", throw_, (double)maxm / constants::GB, used / constants::GB, 100.0 * used / (double)maxm, peak / constants::GB, peakDelta / constants::GB); - } -#ifdef BOUNDED_MR_STATS - ret += std::format(" peak={:.2f} GB live={} nAlloc={} nFree={} totalAlloc={:.2f} GB totalFreed={:.2f} GB maxAlign={} upstreamFail={}", - (float)mStats.peak.load(std::memory_order_relaxed) / constants::GB, - mStats.live.load(std::memory_order_relaxed), - mStats.nAlloc.load(std::memory_order_relaxed), - mStats.nFree.load(std::memory_order_relaxed), - (float)mStats.totalAlloc.load(std::memory_order_relaxed) / constants::GB, - (float)mStats.totalFreed.load(std::memory_order_relaxed) / constants::GB, - mStats.maxAlign.load(std::memory_order_relaxed), - mStats.upstreamFailures.load(std::memory_order_relaxed)); -#endif - return ret; - } - - void print() const - { - LOGP(info, "{}", asString()); - } -#endif - - private: - std::atomic mMaxMemory{std::numeric_limits::max()}; - std::atomic mCountThrow{0}; - std::atomic mUsedMemory{0}; - std::atomic mPeakUsedMemory{0}; - std::atomic mPeakBaselineMemory{0}; - std::unique_ptr mAdaptor{nullptr}; - std::pmr::memory_resource* mUpstream{nullptr}; - -#ifdef BOUNDED_MR_STATS - struct Stats { - std::atomic peak{0}; - std::atomic live{0}; - std::atomic nAlloc{0}; - std::atomic nFree{0}; - std::atomic totalAlloc{0}; - std::atomic totalFreed{0}; - std::atomic maxAlign{0}; - std::atomic upstreamFailures{0}; - }; - Stats mStats{}; -#endif -}; - +using o2::itsmft::tracking::BoundedMemoryResource; template -using bounded_vector = std::pmr::vector; - -template -inline void deepVectorClear(std::vector& vec) -{ - std::vector().swap(vec); -} - -template -inline void deepVectorClear(bounded_vector& vec, std::pmr::memory_resource* mr = nullptr) -{ - std::pmr::memory_resource* tmr = (mr != nullptr) ? mr : vec.get_allocator().resource(); - vec.~bounded_vector(); - new (&vec) bounded_vector(std::pmr::polymorphic_allocator{tmr}); -} - -template -inline void deepVectorClear(std::vector>& vec, std::pmr::memory_resource* mr = nullptr) -{ - for (auto& v : vec) { - deepVectorClear(v, mr); - } -} - -template -inline void deepVectorClear(std::array, S>& arr, std::pmr::memory_resource* mr = nullptr) -{ - for (size_t i{0}; i < S; ++i) { - deepVectorClear(arr[i], mr); - } -} - -template -inline void clearResizeBoundedVector(bounded_vector& vec, size_t sz, std::pmr::memory_resource* mr = nullptr, T def = T()) -{ - std::pmr::memory_resource* tmr = (mr != nullptr) ? mr : vec.get_allocator().resource(); - vec.~bounded_vector(); - new (&vec) bounded_vector(sz, def, std::pmr::polymorphic_allocator{tmr}); -} - -template -inline void clearResizeBoundedVector(std::vector>& vec, size_t size, std::pmr::memory_resource* mr) -{ - vec.clear(); - vec.reserve(size); - for (size_t i = 0; i < size; ++i) { - vec.emplace_back(std::pmr::polymorphic_allocator>{mr}); - } -} - -template -inline void clearResizeBoundedArray(std::array, S>& arr, size_t size, std::pmr::memory_resource* mr = nullptr, T def = T()) -{ - for (size_t i{0}; i < S; ++i) { - clearResizeBoundedVector(arr[i], size, mr, def); - } -} - -template -inline std::vector toSTDVector(const bounded_vector& b) -{ - std::vector t(b.size()); - std::copy(b.cbegin(), b.cend(), t.begin()); - return t; -} +using bounded_vector = o2::itsmft::tracking::bounded_vector; +using o2::itsmft::tracking::clearResizeBoundedArray; +using o2::itsmft::tracking::clearResizeBoundedVector; +using o2::itsmft::tracking::deepVectorClear; +using o2::itsmft::tracking::toSTDVector; } // namespace o2::its -#endif +#endif /* TRACKINGITSU_INCLUDE_BOUNDEDALLOCATOR_H_ */ diff --git a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TimeFrame.h b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TimeFrame.h index 11246fa0ee3b0..db237c6e6bc61 100644 --- a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TimeFrame.h +++ b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TimeFrame.h @@ -23,7 +23,7 @@ #include "DataFormatsITS/TrackITS.h" #include "DataFormatsITS/Vertex.h" -#include "ITStracking/CapacityEstimator.h" +#include "ITSMFTTracking/CapacityEstimator.h" #include "ITStracking/Cell.h" #include "ITStracking/Cluster.h" #include "ITStracking/Configuration.h" @@ -231,8 +231,8 @@ struct TimeFrame { /// staggering void setIsStaggered(bool b) noexcept { mIsStaggered = b; } - CapacityEstimator& getCapacityEstimator() noexcept { return mCapacityEstimator; } - const CapacityEstimator& getCapacityEstimator() const noexcept { return mCapacityEstimator; } + o2::itsmft::tracking::CapacityEstimator& getCapacityEstimator() noexcept { return mCapacityEstimator; } + const o2::itsmft::tracking::CapacityEstimator& getCapacityEstimator() const noexcept { return mCapacityEstimator; } // Vertexer void computeTrackletsPerROFScans(); @@ -325,7 +325,7 @@ struct TimeFrame { std::vector> mCellsNeighboursLUT; bounded_vector mBogusClusters; /// keep track of clusters with wild coordinates - CapacityEstimator mCapacityEstimator; + o2::itsmft::tracking::CapacityEstimator mCapacityEstimator; // Vertexer bounded_vector mPrimaryVertices; diff --git a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TrackerTraits.h b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TrackerTraits.h index 1e3b9e7d7fbf7..6c6159f8eee2b 100644 --- a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TrackerTraits.h +++ b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TrackerTraits.h @@ -23,7 +23,7 @@ #include "DetectorsBase/Propagator.h" #include "ITStracking/Configuration.h" #include "ITStracking/IndexTableUtils.h" -#include "ITStracking/CapacityEstimator.h" +#include "ITSMFTTracking/CapacityEstimator.h" #include "ITStracking/TimeFrame.h" #include "ITStracking/Cell.h" #include "ITStracking/BoundedAllocator.h" diff --git a/Detectors/ITSMFT/ITS/tracking/src/CapacityEstimator.cxx b/Detectors/ITSMFT/ITS/tracking/src/CapacityEstimator.cxx deleted file mode 100644 index a1171fbdba326..0000000000000 --- a/Detectors/ITSMFT/ITS/tracking/src/CapacityEstimator.cxx +++ /dev/null @@ -1,179 +0,0 @@ -// Copyright 2019-2020 CERN and copyright holders of ALICE O2. -// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. -// All rights not expressly granted are reserved. -// -// This software is distributed under the terms of the GNU General Public -// License v3 (GPL Version 3), copied verbatim in the file "COPYING". -// -// In applying this license CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -#include "ITStracking/CapacityEstimator.h" - -#include -#include -#include -#include -#include -#include -#include - -#include "Framework/Logger.h" - -namespace o2::its -{ - -struct CapacityEstimator::Impl { - struct Entry { - float ratio{0.f}; - float margin{0.f}; - size_t maxEmitted{0}; - uint32_t nSamples{0}; - uint32_t nLowStreak{0}; - uint32_t nOverflows{0}; - }; - - explicit Impl(Config config) : cfg{config} {} - - Config cfg; - mutable std::mutex mutex; - std::unordered_map entries; -}; - -CapacityEstimator::CapacityEstimator() : CapacityEstimator{Config{}} {} - -CapacityEstimator::CapacityEstimator(Config cfg) : mImpl{std::make_unique(cfg)} {} - -CapacityEstimator::~CapacityEstimator() = default; - -void CapacityEstimator::reset() -{ - std::lock_guard lock{mImpl->mutex}; - mImpl->entries.clear(); -} - -size_t CapacityEstimator::capacity(uint64_t key, double scale) const -{ - if (!(scale > 0.)) { - return 0; - } - std::lock_guard lock{mImpl->mutex}; - const auto it = mImpl->entries.find(key); - if (it == mImpl->entries.end() || it->second.nSamples == 0) { - return mImpl->cfg.floorSlots; - } - const auto& e = it->second; - const double raw = double(e.ratio) * scale * double(e.margin); - if (!std::isfinite(raw) || raw < 0.) { - return mImpl->cfg.floorSlots; - } - // A ratio is only meaningful at the scale it was measured at. Learned on a handful of inputs it - // can be arbitrarily large, and applying it to a scale orders of magnitude bigger asks for a slab - // nobody can allocate. Bound the request by what this site has ever actually emitted: overshooting - // burns memory that a bump allocator cannot give back, undershooting only costs one retry. - const size_t ceiling = std::max(mImpl->cfg.floorSlots, static_cast(double(e.maxEmitted) * double(mImpl->cfg.marginMax))); - if (raw >= static_cast(ceiling)) { - return ceiling; - } - return std::max(mImpl->cfg.floorSlots, static_cast(std::ceil(raw))); -} - -size_t CapacityEstimator::peakCapacity(uint64_t key) const -{ - std::lock_guard lock{mImpl->mutex}; - const auto it = mImpl->entries.find(key); - if (it == mImpl->entries.end() || it->second.maxEmitted == 0) { - return mImpl->cfg.floorSlots; - } - const auto& e = it->second; - const double raw = double(e.maxEmitted) * double(e.margin); - if (!std::isfinite(raw) || raw >= static_cast(std::numeric_limits::max())) { - return std::numeric_limits::max(); - } - return std::max(mImpl->cfg.floorSlots, static_cast(std::ceil(raw))); -} - -double CapacityEstimator::expected(uint64_t key, double scale) const -{ - if (!(scale > 0.)) { - return 0.; - } - std::lock_guard lock{mImpl->mutex}; - const auto it = mImpl->entries.find(key); - if (it == mImpl->entries.end() || it->second.nSamples == 0) { - return 0.; - } - const double raw = double(it->second.ratio) * scale; - return std::isfinite(raw) && raw > 0. ? raw : 0.; -} - -void CapacityEstimator::update(uint64_t key, double scale, size_t emitted, size_t capacityUsed, bool overflowed, bool memoryLimited) -{ - if (!(scale > 0.)) { - return; - } - std::lock_guard lock{mImpl->mutex}; - auto& e = mImpl->entries[key]; - const auto& cfg = mImpl->cfg; - - const bool firstSample = e.nSamples == 0; - if (firstSample) { - e.margin = cfg.marginInit; - } - const auto sample = static_cast(double(emitted) / scale); - e.ratio = firstSample ? sample : (cfg.alpha * sample) + ((1.f - cfg.alpha) * e.ratio); - e.maxEmitted = std::max(e.maxEmitted, emitted); - ++e.nSamples; - - if (memoryLimited) { - e.nLowStreak = 0; - e.margin = std::max(cfg.marginMin, e.margin * cfg.marginDown); - return; - } - if (overflowed) { - ++e.nOverflows; - e.nLowStreak = 0; - if (!firstSample) { - const float shortfall = capacityUsed ? static_cast(double(emitted) / double(capacityUsed)) : cfg.marginUp; - e.margin = std::min(cfg.marginMax, e.margin * std::clamp(shortfall * cfg.marginOverflowSlack, 1.02f, cfg.marginUp)); - } - return; - } - const float util = capacityUsed ? float(double(emitted) / double(capacityUsed)) : 1.f; - if (util < cfg.lowWatermark) { - if (++e.nLowStreak >= cfg.decayAfter) { - e.margin = std::max(cfg.marginMin, e.margin * cfg.marginDown); - e.nLowStreak = 0; - } - } else if (e.nLowStreak > 0) { - --e.nLowStreak; - } -} - -void CapacityEstimator::print() const -{ - std::lock_guard lock{mImpl->mutex}; - std::vector keys; - keys.reserve(mImpl->entries.size()); - for (const auto& [key, _] : mImpl->entries) { - keys.push_back(key); - } - std::sort(keys.begin(), keys.end(), [](KeyType a, KeyType b) { - const auto da = decodeKey(a); - const auto db = decodeKey(b); - return std::tie(da.site, da.iteration, da.variant, da.slot) < - std::tie(db.site, db.iteration, db.variant, db.slot); - }); - if (keys.empty()) { - return; - } - LOGP(info, "Printing CapacityEstimators:"); - for (const auto key : keys) { - const auto& value = mImpl->entries.at(key); - const auto decoded = decodeKey(key); - LOGP(info, "\tSite:{} | iter:{} | var:({},{}) | slot:{} | ratio:{} | margin:{} | maxEmitted:{} | sam:{} | low:{} | overflows:{}", SlabSiteNames[decoded.site], decoded.iteration, getVariantHigh(decoded.variant), getVariantLow(decoded.variant), decoded.slot, value.ratio, value.margin, value.maxEmitted, value.nSamples, value.nLowStreak, value.nOverflows); - } -} - -} // namespace o2::its diff --git a/Detectors/ITSMFT/ITS/tracking/src/TimeFrame.cxx b/Detectors/ITSMFT/ITS/tracking/src/TimeFrame.cxx index 08c0164288388..a652e3aaecd75 100644 --- a/Detectors/ITSMFT/ITS/tracking/src/TimeFrame.cxx +++ b/Detectors/ITSMFT/ITS/tracking/src/TimeFrame.cxx @@ -498,7 +498,7 @@ template void TimeFrame::setFrameworkAllocator(ExternalAllocator* ext) { mExternalAllocator = ext; - mExtMemoryPool = std::make_shared(mExternalAllocator); + mExtMemoryPool = std::make_shared(std::make_unique(mExternalAllocator)); } template diff --git a/Detectors/ITSMFT/ITS/tracking/src/TrackerTraits.cxx b/Detectors/ITSMFT/ITS/tracking/src/TrackerTraits.cxx index 79511e6a9add5..f086ae33a722f 100644 --- a/Detectors/ITSMFT/ITS/tracking/src/TrackerTraits.cxx +++ b/Detectors/ITSMFT/ITS/tracking/src/TrackerTraits.cxx @@ -36,7 +36,7 @@ #include "ITStracking/IndexTableUtils.h" #include "ITStracking/LayerMask.h" #include "ITStracking/ROFLookupTables.h" -#include "ITStracking/SlabBumpAllocator.h" +#include "ITSMFTTracking/SlabBumpAllocator.h" #include "ITStracking/TrackerTraits.h" #include "ITStracking/TrackFollower.h" #include "ITStracking/TrackHelpers.h" @@ -45,6 +45,11 @@ namespace o2::its { +using o2::itsmft::tracking::CapacityEstimator; +using o2::itsmft::tracking::GroupedSlabSink; +using o2::itsmft::tracking::SlabSite; +using o2::itsmft::tracking::UnorderedSlabSink; + template void TrackerTraits::initialiseTimeFrame(const int iteration) { @@ -200,7 +205,7 @@ void TrackerTraits::computeLayerTracklets(const int iteration, int iVer }); const auto st = sink.stats(); sink.finalizeUnordered(tracklets); - mTimeFrame->getCapacityEstimator().update(key, scale, st.emitted, st.capacity, st.overflowed, st.memoryLimited); + mTimeFrame->getCapacityEstimator().update(key, scale, st); }); } @@ -399,7 +404,7 @@ void TrackerTraits::computeLayerCells(const int iteration) }); const auto st = sink.stats(); sink.finalizeGrouped(size_t(currentLayerTrackletsNum), lut, layerCells); - mTimeFrame->getCapacityEstimator().update(key, scale, st.emitted, st.capacity, st.overflowed, st.memoryLimited); + mTimeFrame->getCapacityEstimator().update(key, scale, st); } else { lut.resize(currentLayerTrackletsNum + 1); for (int iTracklet{0}; iTracklet < currentLayerTrackletsNum; ++iTracklet) { @@ -549,7 +554,7 @@ void TrackerTraits::findCellsNeighbours(const int iteration) }); const auto st = sink.stats(); sink.finalizeUnordered(waveNeighbours); - mTimeFrame->getCapacityEstimator().update(key, scale, st.emitted, st.capacity, st.overflowed, st.memoryLimited); + mTimeFrame->getCapacityEstimator().update(key, scale, st); tbb::parallel_sort(waveNeighbours.begin(), waveNeighbours.end(), neighbourLess); } else { for (const int cellTopologyId : activeTopologies) { @@ -750,7 +755,7 @@ void TrackerTraits::processNeighbours(int iteration, int defaultCellTop }); const auto st = sink.stats(); sink.finalizeUnordered(updatedSeeds); - mTimeFrame->getCapacityEstimator().update(capacityKey, scale, st.emitted, st.capacity, st.overflowed, st.memoryLimited); + mTimeFrame->getCapacityEstimator().update(capacityKey, scale, st); } }); } diff --git a/Detectors/ITSMFT/ITS/tracking/test/CMakeLists.txt b/Detectors/ITSMFT/ITS/tracking/test/CMakeLists.txt index c7c4d6dc101a2..f8fce10b78602 100644 --- a/Detectors/ITSMFT/ITS/tracking/test/CMakeLists.txt +++ b/Detectors/ITSMFT/ITS/tracking/test/CMakeLists.txt @@ -15,12 +15,6 @@ o2_add_test(boundedmemoryresource LABELS "its;tracking" PUBLIC_LINK_LIBRARIES O2::ITStracking) -o2_add_test(slabbumpallocator - SOURCES testSlabBumpAllocator.cxx - COMPONENT_NAME its-tracking - LABELS "its;tracking" - PUBLIC_LINK_LIBRARIES O2::ITStracking TBB::tbb) - o2_add_test(roflookuptables SOURCES testROFLookupTables.cxx COMPONENT_NAME its-tracking diff --git a/Detectors/ITSMFT/common/CMakeLists.txt b/Detectors/ITSMFT/common/CMakeLists.txt index 3991f3e67a82b..92b934020f109 100644 --- a/Detectors/ITSMFT/common/CMakeLists.txt +++ b/Detectors/ITSMFT/common/CMakeLists.txt @@ -12,5 +12,6 @@ add_subdirectory(base) add_subdirectory(simulation) add_subdirectory(reconstruction) +add_subdirectory(tracking) add_subdirectory(workflow) -add_subdirectory(data) \ No newline at end of file +add_subdirectory(data) diff --git a/Detectors/ITSMFT/common/tracking/CMakeLists.txt b/Detectors/ITSMFT/common/tracking/CMakeLists.txt new file mode 100644 index 0000000000000..78ad324ef0d9b --- /dev/null +++ b/Detectors/ITSMFT/common/tracking/CMakeLists.txt @@ -0,0 +1,18 @@ +# Copyright 2019-2020 CERN and copyright holders of ALICE O2. +# See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +# All rights not expressly granted are reserved. +# +# This software is distributed under the terms of the GNU General Public +# License v3 (GPL Version 3), copied verbatim in the file "COPYING". +# +# In applying this license CERN does not waive the privileges and immunities +# granted to it by virtue of its status as an Intergovernmental Organization +# or submit itself to any jurisdiction. + +o2_add_library(ITSMFTTrackingAllocation + SOURCES src/CapacityEstimator.cxx + PUBLIC_LINK_LIBRARIES O2::GPUCommon + TBB::tbb + PRIVATE_LINK_LIBRARIES O2::FrameworkLogger) + +add_subdirectory(test) diff --git a/Detectors/ITSMFT/common/tracking/include/ITSMFTTracking/BoundedAllocator.h b/Detectors/ITSMFT/common/tracking/include/ITSMFTTracking/BoundedAllocator.h new file mode 100644 index 0000000000000..e37715324be86 --- /dev/null +++ b/Detectors/ITSMFT/common/tracking/include/ITSMFTTracking/BoundedAllocator.h @@ -0,0 +1,324 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. +/// +/// \file BoundedAllocator.h +/// \brief +/// + +#ifndef ALICEO2_ITSMFT_TRACKING_BOUNDEDALLOCATOR_H_ +#define ALICEO2_ITSMFT_TRACKING_BOUNDEDALLOCATOR_H_ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if !defined(__HIPCC__) && !defined(__CUDACC__) +#include +#include "GPUCommonLogger.h" +#endif + +namespace o2::itsmft::tracking +{ + +// #define BOUNDED_MR_STATS +class BoundedMemoryResource final : public std::pmr::memory_resource +{ + public: + class MemoryLimitExceeded final : public std::bad_alloc + { + public: + MemoryLimitExceeded(size_t attempted, size_t used, size_t max) + { + char buf[256]; + if (attempted != 0) { + (void)snprintf(buf, sizeof(buf), "Reached set memory limit (attempted: %zu, used: %zu, max: %zu)", attempted, used, max); + } else { + (void)snprintf(buf, sizeof(buf), "New set maximum below current used (newMax: %zu, used: %zu)", max, used); + } + mMsg = buf; + } + const char* what() const noexcept final { return mMsg.c_str(); } + + private: + std::string mMsg; + }; + + static std::pmr::memory_resource* cachingUpstream() + { + static std::pmr::synchronized_pool_resource pool{std::pmr::get_default_resource()}; + return &pool; + } + + BoundedMemoryResource(size_t maxBytes = std::numeric_limits::max(), + std::pmr::memory_resource* upstream = nullptr) + : mMaxMemory(maxBytes), mUpstream(upstream != nullptr ? upstream : cachingUpstream()) {} + + BoundedMemoryResource(std::unique_ptr upstream, + size_t maxBytes = std::numeric_limits::max()) + : mMaxMemory(maxBytes), + mOwnedUpstream(std::move(upstream)), + mUpstream(mOwnedUpstream.get()) {} + + void* do_allocate(size_t bytes, size_t alignment) final + { + size_t new_used{0}; + size_t current_used{mUsedMemory.load(std::memory_order_relaxed)}; + do { + new_used = current_used + bytes; + if (new_used > mMaxMemory.load(std::memory_order_relaxed)) { + mCountThrow.fetch_add(1, std::memory_order_relaxed); + throw MemoryLimitExceeded(new_used, current_used, + mMaxMemory.load(std::memory_order_relaxed)); + } + } while (!mUsedMemory.compare_exchange_weak(current_used, new_used, + std::memory_order_acq_rel, + std::memory_order_relaxed)); + + void* p{nullptr}; + try { + p = mUpstream->allocate(bytes, alignment); + } catch (...) { + mUsedMemory.fetch_sub(bytes, std::memory_order_relaxed); +#ifdef BOUNDED_MR_STATS + mStats.upstreamFailures.fetch_add(1, std::memory_order_relaxed); +#endif + throw; + } + + size_t peak = mPeakUsedMemory.load(std::memory_order_relaxed); + while (new_used > peak && + !mPeakUsedMemory.compare_exchange_weak(peak, new_used, + std::memory_order_relaxed)) { + } + +#ifdef BOUNDED_MR_STATS + size_t statsPeak = mStats.peak.load(std::memory_order_relaxed); + while (new_used > statsPeak && + !mStats.peak.compare_exchange_weak(statsPeak, new_used, + std::memory_order_relaxed)) { + } + mStats.live.fetch_add(1, std::memory_order_relaxed); + mStats.nAlloc.fetch_add(1, std::memory_order_relaxed); + mStats.totalAlloc.fetch_add(bytes, std::memory_order_relaxed); + + size_t ma = mStats.maxAlign.load(std::memory_order_relaxed); + while (alignment > ma && !mStats.maxAlign.compare_exchange_weak(ma, alignment, std::memory_order_relaxed)) { + } +#endif + return p; + } + + void do_deallocate(void* p, size_t bytes, size_t alignment) final + { + mUpstream->deallocate(p, bytes, alignment); + mUsedMemory.fetch_sub(bytes, std::memory_order_relaxed); +#ifdef BOUNDED_MR_STATS + mStats.live.fetch_sub(1, std::memory_order_relaxed); + mStats.nFree.fetch_add(1, std::memory_order_relaxed); + mStats.totalFreed.fetch_add(bytes, std::memory_order_relaxed); +#endif + } + + bool do_is_equal(const std::pmr::memory_resource& other) const noexcept final + { + return this == &other; + } + + [[nodiscard]] size_t getUsedMemory() const noexcept + { + return mUsedMemory.load(std::memory_order_relaxed); + } + [[nodiscard]] size_t getMaxMemory() const noexcept + { + return mMaxMemory.load(std::memory_order_relaxed); + } + [[nodiscard]] size_t getThrowCount() const noexcept + { + return mCountThrow.load(std::memory_order_relaxed); + } + [[nodiscard]] size_t getPeakMemory() const noexcept + { + return mPeakUsedMemory.load(std::memory_order_relaxed); + } + [[nodiscard]] size_t getPeakMemoryDelta() const noexcept + { + const size_t peak = mPeakUsedMemory.load(std::memory_order_relaxed); + const size_t baseline = mPeakBaselineMemory.load(std::memory_order_relaxed); + return peak > baseline ? peak - baseline : 0; + } + + void resetPeakMemory() noexcept + { + const size_t used = mUsedMemory.load(std::memory_order_acquire); + mPeakBaselineMemory.store(used, std::memory_order_release); + mPeakUsedMemory.store(used, std::memory_order_release); + } + + void setMaxMemory(size_t max) + { + size_t current = mMaxMemory.load(std::memory_order_relaxed); + if (max == current) { + return; + } + for (;;) { + size_t used = mUsedMemory.load(std::memory_order_acquire); + if (used > max) { + mCountThrow.fetch_add(1, std::memory_order_relaxed); + throw MemoryLimitExceeded(0, used, max); + } + if (mMaxMemory.compare_exchange_weak(current, max, + std::memory_order_release, + std::memory_order_relaxed)) { + return; + } + if (current == max) { + return; + } + } + } + +#if !defined(__HIPCC__) && !defined(__CUDACC__) + std::string asString() const + { + constexpr double gigabyte = 1024. * 1024. * 1024.; + const auto throw_ = mCountThrow.load(std::memory_order_relaxed); + const auto used = static_cast(mUsedMemory.load(std::memory_order_relaxed)); + const auto peak = static_cast(mPeakUsedMemory.load(std::memory_order_relaxed)); + const auto peakDelta = static_cast(getPeakMemoryDelta()); + const auto maxm = mMaxMemory.load(std::memory_order_relaxed); + std::string ret; + if (maxm == std::numeric_limits::max()) { + ret += std::format("maxthrow={} maxmem=unbounded used={:.2f} GB stagepeak={:.2f} GB stagealloc={:.2f} GB", throw_, used / gigabyte, peak / gigabyte, peakDelta / gigabyte); + } else { + ret += std::format("maxthrow={} maxmem={:.2f} GB used={:.2f} GB ({:.2f}%) stagepeak={:.2f} GB stagealloc={:.2f} GB", throw_, (double)maxm / gigabyte, used / gigabyte, 100.0 * used / (double)maxm, peak / gigabyte, peakDelta / gigabyte); + } +#ifdef BOUNDED_MR_STATS + ret += std::format(" peak={:.2f} GB live={} nAlloc={} nFree={} totalAlloc={:.2f} GB totalFreed={:.2f} GB maxAlign={} upstreamFail={}", + (float)mStats.peak.load(std::memory_order_relaxed) / gigabyte, + mStats.live.load(std::memory_order_relaxed), + mStats.nAlloc.load(std::memory_order_relaxed), + mStats.nFree.load(std::memory_order_relaxed), + (float)mStats.totalAlloc.load(std::memory_order_relaxed) / gigabyte, + (float)mStats.totalFreed.load(std::memory_order_relaxed) / gigabyte, + mStats.maxAlign.load(std::memory_order_relaxed), + mStats.upstreamFailures.load(std::memory_order_relaxed)); +#endif + return ret; + } + + void print() const + { + LOGP(info, "{}", asString()); + } +#endif + + private: + std::atomic mMaxMemory{std::numeric_limits::max()}; + std::atomic mCountThrow{0}; + std::atomic mUsedMemory{0}; + std::atomic mPeakUsedMemory{0}; + std::atomic mPeakBaselineMemory{0}; + std::unique_ptr mOwnedUpstream; + std::pmr::memory_resource* mUpstream{nullptr}; + +#ifdef BOUNDED_MR_STATS + struct Stats { + std::atomic peak{0}; + std::atomic live{0}; + std::atomic nAlloc{0}; + std::atomic nFree{0}; + std::atomic totalAlloc{0}; + std::atomic totalFreed{0}; + std::atomic maxAlign{0}; + std::atomic upstreamFailures{0}; + }; + Stats mStats{}; +#endif +}; + +template +using bounded_vector = std::pmr::vector; + +template +inline void deepVectorClear(std::vector& vec) +{ + std::vector().swap(vec); +} + +template +inline void deepVectorClear(bounded_vector& vec, std::pmr::memory_resource* mr = nullptr) +{ + std::pmr::memory_resource* tmr = (mr != nullptr) ? mr : vec.get_allocator().resource(); + vec.~bounded_vector(); + new (&vec) bounded_vector(std::pmr::polymorphic_allocator{tmr}); +} + +template +inline void deepVectorClear(std::vector>& vec, std::pmr::memory_resource* mr = nullptr) +{ + for (auto& v : vec) { + deepVectorClear(v, mr); + } +} + +template +inline void deepVectorClear(std::array, S>& arr, std::pmr::memory_resource* mr = nullptr) +{ + for (size_t i{0}; i < S; ++i) { + deepVectorClear(arr[i], mr); + } +} + +template +inline void clearResizeBoundedVector(bounded_vector& vec, size_t sz, std::pmr::memory_resource* mr = nullptr, T def = T()) +{ + std::pmr::memory_resource* tmr = (mr != nullptr) ? mr : vec.get_allocator().resource(); + vec.~bounded_vector(); + new (&vec) bounded_vector(sz, def, std::pmr::polymorphic_allocator{tmr}); +} + +template +inline void clearResizeBoundedVector(std::vector>& vec, size_t size, std::pmr::memory_resource* mr) +{ + vec.clear(); + vec.reserve(size); + for (size_t i = 0; i < size; ++i) { + vec.emplace_back(std::pmr::polymorphic_allocator>{mr}); + } +} + +template +inline void clearResizeBoundedArray(std::array, S>& arr, size_t size, std::pmr::memory_resource* mr = nullptr, T def = T()) +{ + for (size_t i{0}; i < S; ++i) { + clearResizeBoundedVector(arr[i], size, mr, def); + } +} + +template +inline std::vector toSTDVector(const bounded_vector& b) +{ + std::vector t(b.size()); + std::copy(b.cbegin(), b.cend(), t.begin()); + return t; +} + +} // namespace o2::itsmft::tracking + +#endif /* ALICEO2_ITSMFT_TRACKING_BOUNDEDALLOCATOR_H_ */ diff --git a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/CapacityEstimator.h b/Detectors/ITSMFT/common/tracking/include/ITSMFTTracking/CapacityEstimator.h similarity index 80% rename from Detectors/ITSMFT/ITS/tracking/include/ITStracking/CapacityEstimator.h rename to Detectors/ITSMFT/common/tracking/include/ITSMFTTracking/CapacityEstimator.h index de1ed88c516f3..d352613b397d1 100644 --- a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/CapacityEstimator.h +++ b/Detectors/ITSMFT/common/tracking/include/ITSMFTTracking/CapacityEstimator.h @@ -13,8 +13,8 @@ /// \brief Cross-timeframe output-size prediction. /// -#ifndef TRACKINGITSU_INCLUDE_CAPACITYESTIMATOR_H_ -#define TRACKINGITSU_INCLUDE_CAPACITYESTIMATOR_H_ +#ifndef ALICEO2_ITSMFT_TRACKING_CAPACITYESTIMATOR_H_ +#define ALICEO2_ITSMFT_TRACKING_CAPACITYESTIMATOR_H_ #include #include @@ -22,7 +22,7 @@ #include #include -namespace o2::its +namespace o2::itsmft::tracking { enum SlabSite : uint8_t { @@ -63,6 +63,15 @@ class CapacityEstimator int slot; }; + struct Statistics { + size_t requested{0}; + size_t granted{0}; + size_t emitted{0}; + size_t spilled{0}; + uint32_t samples{0}; + uint32_t overflowEvents{0}; + }; + static constexpr KeyType makeKey(SlabSite site, int iteration, int variant, int slot) noexcept { return (static_cast(site) << 56) | @@ -102,10 +111,22 @@ class CapacityEstimator CapacityEstimator& operator=(const CapacityEstimator&) = delete; void reset(); + void beginTransaction(); + void commitTransaction() noexcept; + void rollbackTransaction() noexcept; size_t capacity(uint64_t key, double scale) const; size_t peakCapacity(uint64_t key) const; double expected(uint64_t key, double scale) const; + Statistics statistics(uint64_t key) const; void update(uint64_t key, double scale, size_t emitted, size_t capacityUsed, bool overflowed, bool memoryLimited); + void update(uint64_t key, double scale, size_t requested, size_t granted, size_t emitted, + size_t spilled, bool overflowed, bool memoryLimited); + template + void update(uint64_t key, double scale, const Stats& stats) + { + update(key, scale, stats.requested, stats.capacity, stats.emitted, stats.spilled, + stats.overflowed, stats.memoryLimited); + } void print() const; private: @@ -133,6 +154,6 @@ int runOnSlab(CapacityEstimator& estimator, const CapacityEstimator::KeyType key return emitted; } -} // namespace o2::its +} // namespace o2::itsmft::tracking -#endif /* TRACKINGITSU_INCLUDE_CAPACITYESTIMATOR_H_ */ +#endif /* ALICEO2_ITSMFT_TRACKING_CAPACITYESTIMATOR_H_ */ diff --git a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/SlabBumpAllocator.h b/Detectors/ITSMFT/common/tracking/include/ITSMFTTracking/SlabBumpAllocator.h similarity index 97% rename from Detectors/ITSMFT/ITS/tracking/include/ITStracking/SlabBumpAllocator.h rename to Detectors/ITSMFT/common/tracking/include/ITSMFTTracking/SlabBumpAllocator.h index e32516ea1e0e0..71f7f93bff59a 100644 --- a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/SlabBumpAllocator.h +++ b/Detectors/ITSMFT/common/tracking/include/ITSMFTTracking/SlabBumpAllocator.h @@ -13,8 +13,8 @@ /// \brief Lock-free slot allocator and single-pass sink. /// -#ifndef TRACKINGITSU_INCLUDE_SLABBUMPALLOCATOR_H_ -#define TRACKINGITSU_INCLUDE_SLABBUMPALLOCATOR_H_ +#ifndef ALICEO2_ITSMFT_TRACKING_SLABBUMPALLOCATOR_H_ +#define ALICEO2_ITSMFT_TRACKING_SLABBUMPALLOCATOR_H_ #include #include @@ -31,9 +31,9 @@ #include #include -#include "ITStracking/BoundedAllocator.h" +#include "ITSMFTTracking/BoundedAllocator.h" -namespace o2::its +namespace o2::itsmft::tracking { class SlabBumpAllocator @@ -392,6 +392,6 @@ using UnorderedSlabSink = SlabSink; template using GroupedSlabSink = SlabSink; -} // namespace o2::its +} // namespace o2::itsmft::tracking -#endif /* TRACKINGITSU_INCLUDE_SLABBUMPALLOCATOR_H_ */ +#endif /* ALICEO2_ITSMFT_TRACKING_SLABBUMPALLOCATOR_H_ */ diff --git a/Detectors/ITSMFT/common/tracking/src/CapacityEstimator.cxx b/Detectors/ITSMFT/common/tracking/src/CapacityEstimator.cxx new file mode 100644 index 0000000000000..6cfcd662e3db9 --- /dev/null +++ b/Detectors/ITSMFT/common/tracking/src/CapacityEstimator.cxx @@ -0,0 +1,285 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +#include "ITSMFTTracking/CapacityEstimator.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "Framework/Logger.h" + +namespace o2::itsmft::tracking +{ + +struct CapacityEstimator::Impl { + struct Entry { + float ratio{0.f}; + float margin{0.f}; + size_t maxEmitted{0}; + uint32_t nSamples{0}; + uint32_t nLowStreak{0}; + uint32_t nOverflows{0}; + size_t requested{0}; + size_t granted{0}; + size_t emitted{0}; + size_t spilled{0}; + }; + + struct UndoRecord { + bool existed{false}; + Entry previous{}; + }; + + explicit Impl(Config config) : cfg{config} {} + + Config cfg; + mutable std::mutex mutex; + std::unordered_map entries; + std::unordered_map undo; + bool transactionActive{false}; + + void checkpointBeforeUpdate(KeyType key) + { + if (!transactionActive || undo.find(key) != undo.end()) { + return; + } + const auto current = entries.find(key); + if (current == entries.end()) { + undo.emplace(key, UndoRecord{}); + } else { + undo.emplace(key, UndoRecord{.existed = true, .previous = current->second}); + } + } + + void observe(KeyType key, double scale, size_t requested, size_t granted, size_t emitted, + size_t spilled, bool overflowed, bool memoryLimited) + { + // Record the first-touch undo state before entries[key] can insert or the + // existing live entry can be modified. If undo insertion throws, the live + // estimator remains unchanged and Tracker's failure path can roll back the + // transaction without observing a partial update. + checkpointBeforeUpdate(key); + auto& e = entries[key]; + e.requested += requested; + e.granted += granted; + e.emitted += emitted; + e.spilled += spilled; + + const bool firstSample = e.nSamples == 0; + if (firstSample) { + e.margin = cfg.marginInit; + } + const auto sample = static_cast(double(emitted) / scale); + e.ratio = firstSample ? sample : (cfg.alpha * sample) + ((1.f - cfg.alpha) * e.ratio); + e.maxEmitted = std::max(e.maxEmitted, emitted); + ++e.nSamples; + + if (memoryLimited) { + e.nLowStreak = 0; + e.margin = std::max(cfg.marginMin, e.margin * cfg.marginDown); + return; + } + if (overflowed) { + ++e.nOverflows; + e.nLowStreak = 0; + if (!firstSample) { + const float shortfall = granted ? static_cast(double(emitted) / double(granted)) : cfg.marginUp; + e.margin = std::min(cfg.marginMax, e.margin * std::clamp(shortfall * cfg.marginOverflowSlack, 1.02f, cfg.marginUp)); + } + return; + } + const float util = granted ? float(double(emitted) / double(granted)) : 1.f; + if (util < cfg.lowWatermark) { + if (++e.nLowStreak >= cfg.decayAfter) { + e.margin = std::max(cfg.marginMin, e.margin * cfg.marginDown); + e.nLowStreak = 0; + } + } else if (e.nLowStreak > 0) { + --e.nLowStreak; + } + } +}; + +CapacityEstimator::CapacityEstimator() : CapacityEstimator{Config{}} {} + +CapacityEstimator::CapacityEstimator(Config cfg) : mImpl{std::make_unique(cfg)} {} + +CapacityEstimator::~CapacityEstimator() = default; + +void CapacityEstimator::reset() +{ + std::lock_guard lock{mImpl->mutex}; + mImpl->entries.clear(); + mImpl->undo.clear(); + mImpl->transactionActive = false; +} + +void CapacityEstimator::beginTransaction() +{ + std::lock_guard lock{mImpl->mutex}; + if (mImpl->transactionActive) { + throw std::logic_error{"CapacityEstimator transaction already active"}; + } + assert(mImpl->undo.empty()); + mImpl->transactionActive = true; +} + +void CapacityEstimator::commitTransaction() noexcept +{ + std::lock_guard lock{mImpl->mutex}; + mImpl->undo.clear(); + mImpl->transactionActive = false; +} + +void CapacityEstimator::rollbackTransaction() noexcept +{ + std::lock_guard lock{mImpl->mutex}; + if (!mImpl->transactionActive) { + return; + } + for (const auto& [key, record] : mImpl->undo) { + if (record.existed) { + const auto current = mImpl->entries.find(key); + assert(current != mImpl->entries.end()); + current->second = record.previous; + } else { + mImpl->entries.erase(key); + } + } + mImpl->undo.clear(); + mImpl->transactionActive = false; +} + +size_t CapacityEstimator::capacity(uint64_t key, double scale) const +{ + if (!(scale > 0.)) { + return 0; + } + std::lock_guard lock{mImpl->mutex}; + const auto it = mImpl->entries.find(key); + if (it == mImpl->entries.end() || it->second.nSamples == 0) { + return mImpl->cfg.floorSlots; + } + const auto& e = it->second; + const double raw = double(e.ratio) * scale * double(e.margin); + if (!std::isfinite(raw) || raw < 0.) { + return mImpl->cfg.floorSlots; + } + // A ratio is only meaningful at the scale it was measured at. Learned on a handful of inputs it + // can be arbitrarily large, and applying it to a scale orders of magnitude bigger asks for a slab + // nobody can allocate. Bound the request by what this site has ever actually emitted: overshooting + // burns memory that a bump allocator cannot give back, undershooting only costs one retry. + const size_t ceiling = std::max(mImpl->cfg.floorSlots, static_cast(double(e.maxEmitted) * double(mImpl->cfg.marginMax))); + if (raw >= static_cast(ceiling)) { + return ceiling; + } + return std::max(mImpl->cfg.floorSlots, static_cast(std::ceil(raw))); +} + +size_t CapacityEstimator::peakCapacity(uint64_t key) const +{ + std::lock_guard lock{mImpl->mutex}; + const auto it = mImpl->entries.find(key); + if (it == mImpl->entries.end() || it->second.maxEmitted == 0) { + return mImpl->cfg.floorSlots; + } + const auto& e = it->second; + const double raw = double(e.maxEmitted) * double(e.margin); + if (!std::isfinite(raw) || raw >= static_cast(std::numeric_limits::max())) { + return std::numeric_limits::max(); + } + return std::max(mImpl->cfg.floorSlots, static_cast(std::ceil(raw))); +} + +double CapacityEstimator::expected(uint64_t key, double scale) const +{ + if (!(scale > 0.)) { + return 0.; + } + std::lock_guard lock{mImpl->mutex}; + const auto it = mImpl->entries.find(key); + if (it == mImpl->entries.end() || it->second.nSamples == 0) { + return 0.; + } + const double raw = double(it->second.ratio) * scale; + return std::isfinite(raw) && raw > 0. ? raw : 0.; +} + +CapacityEstimator::Statistics CapacityEstimator::statistics(uint64_t key) const +{ + std::lock_guard lock{mImpl->mutex}; + const auto it = mImpl->entries.find(key); + if (it == mImpl->entries.end()) { + return {}; + } + const auto& e = it->second; + return {.requested = e.requested, + .granted = e.granted, + .emitted = e.emitted, + .spilled = e.spilled, + .samples = e.nSamples, + .overflowEvents = e.nOverflows}; +} + +void CapacityEstimator::update(uint64_t key, double scale, size_t emitted, size_t capacityUsed, bool overflowed, bool memoryLimited) +{ + if (!(scale > 0.)) { + return; + } + std::lock_guard lock{mImpl->mutex}; + mImpl->observe(key, scale, capacityUsed, capacityUsed, emitted, + overflowed && emitted > capacityUsed ? emitted - capacityUsed : 0, + overflowed, memoryLimited); +} + +void CapacityEstimator::update(uint64_t key, double scale, size_t requested, size_t granted, + size_t emitted, size_t spilled, bool overflowed, bool memoryLimited) +{ + if (!(scale > 0.)) { + return; + } + std::lock_guard lock{mImpl->mutex}; + mImpl->observe(key, scale, requested, granted, emitted, spilled, overflowed, memoryLimited); +} + +void CapacityEstimator::print() const +{ + std::lock_guard lock{mImpl->mutex}; + std::vector keys; + keys.reserve(mImpl->entries.size()); + for (const auto& [key, _] : mImpl->entries) { + keys.push_back(key); + } + std::sort(keys.begin(), keys.end(), [](KeyType a, KeyType b) { + const auto da = decodeKey(a); + const auto db = decodeKey(b); + return std::tie(da.site, da.iteration, da.variant, da.slot) < + std::tie(db.site, db.iteration, db.variant, db.slot); + }); + if (keys.empty()) { + return; + } + LOGP(info, "Printing CapacityEstimators:"); + for (const auto key : keys) { + const auto& value = mImpl->entries.at(key); + const auto decoded = decodeKey(key); + LOGP(info, "\tSite:{} | iter:{} | var:({},{}) | slot:{} | ratio:{} | margin:{} | maxEmitted:{} | samples:{} | low:{} | requested:{} | granted:{} | emitted:{} | spilled:{} | overflows:{}", SlabSiteNames[decoded.site], decoded.iteration, getVariantHigh(decoded.variant), getVariantLow(decoded.variant), decoded.slot, value.ratio, value.margin, value.maxEmitted, value.nSamples, value.nLowStreak, value.requested, value.granted, value.emitted, value.spilled, value.nOverflows); + } +} + +} // namespace o2::itsmft::tracking diff --git a/Detectors/ITSMFT/common/tracking/test/CMakeLists.txt b/Detectors/ITSMFT/common/tracking/test/CMakeLists.txt new file mode 100644 index 0000000000000..164506401bd38 --- /dev/null +++ b/Detectors/ITSMFT/common/tracking/test/CMakeLists.txt @@ -0,0 +1,16 @@ +# Copyright 2019-2026 CERN and copyright holders of ALICE O2. +# See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +# All rights not expressly granted are reserved. +# +# This software is distributed under the terms of the GNU General Public +# License v3 (GPL Version 3), copied verbatim in the file "COPYING". +# +# In applying this license CERN does not waive the privileges and immunities +# granted to it by virtue of its status as an Intergovernmental Organization +# or submit itself to any jurisdiction. + +o2_add_test(slabbumpallocator + SOURCES testSlabBumpAllocator.cxx + COMPONENT_NAME itsmft-tracking + LABELS "itsmft;tracking" + PUBLIC_LINK_LIBRARIES O2::ITSMFTTrackingAllocation TBB::tbb) diff --git a/Detectors/ITSMFT/ITS/tracking/test/testSlabBumpAllocator.cxx b/Detectors/ITSMFT/common/tracking/test/testSlabBumpAllocator.cxx similarity index 76% rename from Detectors/ITSMFT/ITS/tracking/test/testSlabBumpAllocator.cxx rename to Detectors/ITSMFT/common/tracking/test/testSlabBumpAllocator.cxx index d72ede1a01c09..656d65eb77e45 100644 --- a/Detectors/ITSMFT/ITS/tracking/test/testSlabBumpAllocator.cxx +++ b/Detectors/ITSMFT/common/tracking/test/testSlabBumpAllocator.cxx @@ -21,15 +21,16 @@ #include #include #include +#include #include #include -#include "ITStracking/BoundedAllocator.h" -#include "ITStracking/CapacityEstimator.h" -#include "ITStracking/SlabBumpAllocator.h" +#include "ITSMFTTracking/BoundedAllocator.h" +#include "ITSMFTTracking/CapacityEstimator.h" +#include "ITSMFTTracking/SlabBumpAllocator.h" -using namespace o2::its; +using namespace o2::itsmft::tracking; namespace { @@ -103,6 +104,34 @@ std::vector> reference(int nProducers, uint32_t seed) return out; } +struct EstimatorSnapshot { + size_t capacity{0}; + size_t peakCapacity{0}; + double expected{0.}; + CapacityEstimator::Statistics statistics{}; +}; + +EstimatorSnapshot snapshot(const CapacityEstimator& estimator, CapacityEstimator::KeyType key, double scale) +{ + return {.capacity = estimator.capacity(key, scale), + .peakCapacity = estimator.peakCapacity(key), + .expected = estimator.expected(key, scale), + .statistics = estimator.statistics(key)}; +} + +void checkSnapshot(const EstimatorSnapshot& actual, const EstimatorSnapshot& expected) +{ + BOOST_TEST(actual.capacity == expected.capacity); + BOOST_TEST(actual.peakCapacity == expected.peakCapacity); + BOOST_TEST(actual.expected == expected.expected); + BOOST_TEST(actual.statistics.requested == expected.statistics.requested); + BOOST_TEST(actual.statistics.granted == expected.statistics.granted); + BOOST_TEST(actual.statistics.emitted == expected.statistics.emitted); + BOOST_TEST(actual.statistics.spilled == expected.statistics.spilled); + BOOST_TEST(actual.statistics.samples == expected.statistics.samples); + BOOST_TEST(actual.statistics.overflowEvents == expected.statistics.overflowEvents); +} + void checkGrouped(int nProducers, size_t capacity, size_t slab, size_t maxMemory = std::numeric_limits::max()) { constexpr uint32_t seed = 7u; @@ -580,6 +609,104 @@ BOOST_AUTO_TEST_CASE(estimator_reset_forgets_inflated_margins) BOOST_TEST(est.capacity(key, scale) == 1024u); } +BOOST_AUTO_TEST_CASE(estimator_updates_immediately_and_commit_retains_updates) +{ + CapacityEstimator est; + const auto key = CapacityEstimator::makeKey(SlabSite::Neighbours, 2, 0, 4); + est.update(key, 100., 120, 100, 95, 7, true, false); + const auto immediate = est.statistics(key); + BOOST_TEST(immediate.requested == 120u); + BOOST_TEST(immediate.granted == 100u); + BOOST_TEST(immediate.emitted == 95u); + BOOST_TEST(immediate.spilled == 7u); + BOOST_TEST(immediate.samples == 1u); + BOOST_TEST(immediate.overflowEvents == 1u); + + est.beginTransaction(); + est.update(key, 100., 80, 80, 70, 0, false, false); + const auto beforeCommit = snapshot(est, key, 100.); + est.commitTransaction(); + checkSnapshot(snapshot(est, key, 100.), beforeCommit); +} + +BOOST_AUTO_TEST_CASE(estimator_rollback_restores_the_first_touch_state_exactly) +{ + CapacityEstimator est; + const auto key = CapacityEstimator::makeKey(SlabSite::Neighbours, 2, 0, 4); + constexpr double scale = 100.; + est.update(key, scale, 120, 100, 95, 7, true, false); + const auto before = snapshot(est, key, scale); + + est.beginTransaction(); + est.update(key, scale, 8000, 6000, 5500, 500, true, false); + est.update(key, scale, 40, 400, 20, 0, false, false); + const auto during = snapshot(est, key, scale); + BOOST_TEST(during.statistics.samples == before.statistics.samples + 2u); + BOOST_TEST(during.statistics.requested == before.statistics.requested + 8040u); + BOOST_TEST(during.peakCapacity > before.peakCapacity); + + est.rollbackTransaction(); + checkSnapshot(snapshot(est, key, scale), before); +} + +BOOST_AUTO_TEST_CASE(estimator_rollback_removes_a_transaction_created_key) +{ + CapacityEstimator est; + const auto key = CapacityEstimator::makeKey(SlabSite::Cells, 3, 0, 5); + constexpr double scale = 50.; + const auto absent = snapshot(est, key, scale); + + est.beginTransaction(); + est.update(key, scale, 90, 80, 75, 4, true, false); + BOOST_TEST(est.statistics(key).samples == 1u); + BOOST_TEST(est.expected(key, scale) > 0.); + est.rollbackTransaction(); + + checkSnapshot(snapshot(est, key, scale), absent); +} + +BOOST_AUTO_TEST_CASE(estimator_nested_transaction_rejection_preserves_the_active_transaction) +{ + CapacityEstimator est; + const auto key = CapacityEstimator::makeKey(SlabSite::Roads, 1, 0, 2); + constexpr double scale = 100.; + est.update(key, scale, 50, 50, 40, 0, false, false); + const auto before = snapshot(est, key, scale); + + est.beginTransaction(); + est.update(key, scale, 200, 180, 160, 5, true, false); + const auto beforeRejectedBegin = snapshot(est, key, scale); + BOOST_CHECK_THROW(est.beginTransaction(), std::logic_error); + checkSnapshot(snapshot(est, key, scale), beforeRejectedBegin); + est.rollbackTransaction(); + checkSnapshot(snapshot(est, key, scale), before); + + BOOST_CHECK_NO_THROW(est.beginTransaction()); + est.commitTransaction(); +} + +BOOST_AUTO_TEST_CASE(estimator_reset_clears_active_transaction_and_learning) +{ + CapacityEstimator est; + const auto existing = CapacityEstimator::makeKey(SlabSite::Tracklets, 1, 0, 2); + const auto created = CapacityEstimator::makeKey(SlabSite::Tracklets, 1, 0, 3); + constexpr double scale = 100.; + est.update(existing, scale, 200, 180, 170, 3, true, false); + est.beginTransaction(); + est.update(existing, scale, 300, 250, 240, 5, true, false); + est.update(created, scale, 100, 90, 80, 2, true, false); + + est.reset(); + BOOST_TEST(est.statistics(existing).samples == 0u); + BOOST_TEST(est.statistics(created).samples == 0u); + BOOST_TEST(est.capacity(existing, scale) == 1024u); + BOOST_TEST(est.expected(existing, scale) == 0.); + BOOST_CHECK_NO_THROW(est.beginTransaction()); + est.update(existing, scale, 60, 60, 50, 0, false, false); + est.commitTransaction(); + BOOST_TEST(est.statistics(existing).samples == 1u); +} + BOOST_AUTO_TEST_CASE(estimator_keys_separate_the_road_walk_steps) { const auto a = CapacityEstimator::makeKey(SlabSite::Roads, 0, CapacityEstimator::makeVariant(6, 4), 1); @@ -589,3 +716,16 @@ BOOST_AUTO_TEST_CASE(estimator_keys_separate_the_road_walk_steps) BOOST_TEST(a != c); BOOST_TEST(b != c); } + +BOOST_AUTO_TEST_CASE(estimator_keys_separate_stage_iteration_and_site) +{ + const auto edge0 = CapacityEstimator::makeKey(SlabSite::Tracklets, 0, 0, 0); + const auto edge1 = CapacityEstimator::makeKey(SlabSite::Tracklets, 0, 0, 1); + const auto nextIteration = CapacityEstimator::makeKey(SlabSite::Tracklets, 1, 0, 0); + const auto path0 = CapacityEstimator::makeKey(SlabSite::Cells, 0, 0, 0); + const auto path1 = CapacityEstimator::makeKey(SlabSite::Cells, 0, 0, 1); + BOOST_TEST(edge0 != edge1); + BOOST_TEST(edge0 != nextIteration); + BOOST_TEST(edge0 != path0); + BOOST_TEST(path0 != path1); +} From 6229d42177ac625490dfee912abb4f153e233f5d Mon Sep 17 00:00:00 2001 From: Maximiliano Puccio Date: Thu, 27 Aug 2026 22:06:45 +0200 Subject: [PATCH 2/7] ITSMFT: complete tracking allocation migration --- .../GPU/ITStrackingGPU/TimeFrameGPU.h | 6 +- .../GPU/ITStrackingGPU/TrackingKernels.h | 8 +- .../ITS/tracking/GPU/cuda/TimeFrameGPU.cu | 6 +- .../tracking/GPU/cuda/TrackerTraitsGPU.cxx | 4 +- .../ITS/tracking/GPU/cuda/TrackingKernels.cu | 6 +- .../include/ITStracking/BoundedAllocator.h | 34 --- .../include/ITStracking/LineVertexerHelpers.h | 6 +- .../tracking/include/ITStracking/TimeFrame.h | 80 +++---- .../tracking/include/ITStracking/Tracker.h | 6 +- .../include/ITStracking/TrackerTraits.h | 14 +- .../include/ITStracking/TrackingInterface.h | 4 +- .../tracking/include/ITStracking/Vertexer.h | 6 +- .../include/ITStracking/VertexerTraits.h | 8 +- .../ITS/tracking/src/LineVertexerHelpers.cxx | 100 ++++----- .../ITSMFT/ITS/tracking/src/TimeFrame.cxx | 176 +++++++-------- Detectors/ITSMFT/ITS/tracking/src/Tracker.cxx | 10 +- .../ITSMFT/ITS/tracking/src/TrackerTraits.cxx | 72 +++---- .../ITS/tracking/src/TrackingInterface.cxx | 4 +- .../ITSMFT/ITS/tracking/src/Vertexer.cxx | 10 +- .../ITS/tracking/src/VertexerTraits.cxx | 20 +- .../ITSMFT/ITS/tracking/test/CMakeLists.txt | 6 - .../ITSMFT/common/tracking/CMakeLists.txt | 10 +- .../include/ITSMFTTracking/BoundedAllocator.h | 194 ++--------------- .../ITSMFTTracking/SlabBumpAllocator.h | 121 ++++++----- .../common/tracking/src/BoundedAllocator.cxx | 200 ++++++++++++++++++ .../common/tracking/src/SlabBumpAllocator.cxx | 108 ++++++++++ .../common/tracking/test/CMakeLists.txt | 10 +- .../test/testBoundedMemoryResource.cxx | 6 +- .../tracking/test/testSlabBumpAllocator.cxx | 1 + .../TrackerSpec.h | 4 +- .../workflow/src/TrackerSpec.cxx | 2 +- .../ITS3/reconstruction/src/IOUtils.cxx | 4 +- 32 files changed, 694 insertions(+), 552 deletions(-) delete mode 100644 Detectors/ITSMFT/ITS/tracking/include/ITStracking/BoundedAllocator.h create mode 100644 Detectors/ITSMFT/common/tracking/src/BoundedAllocator.cxx create mode 100644 Detectors/ITSMFT/common/tracking/src/SlabBumpAllocator.cxx rename Detectors/ITSMFT/{ITS => common}/tracking/test/testBoundedMemoryResource.cxx (97%) diff --git a/Detectors/ITSMFT/ITS/tracking/GPU/ITStrackingGPU/TimeFrameGPU.h b/Detectors/ITSMFT/ITS/tracking/GPU/ITStrackingGPU/TimeFrameGPU.h index 0d70158b9bdb8..ab27cc5b1804d 100644 --- a/Detectors/ITSMFT/ITS/tracking/GPU/ITStrackingGPU/TimeFrameGPU.h +++ b/Detectors/ITSMFT/ITS/tracking/GPU/ITStrackingGPU/TimeFrameGPU.h @@ -16,7 +16,7 @@ #include #include -#include "ITStracking/BoundedAllocator.h" +#include "ITSMFTTracking/BoundedAllocator.h" #include "ITStracking/TimeFrame.h" #include "ITStracking/Configuration.h" #include "ITStracking/TrackExtensionHypothesis.h" @@ -250,8 +250,8 @@ class TimeFrameGPU : public TimeFrame std::bitset mPinnedTrackingFrameInfo{0}; // Temporary buffer for storing output tracks from GPU tracking - bounded_vector mTrackITSExt; - bounded_vector mTrackIndices; + o2::itsmft::tracking::bounded_vector mTrackITSExt; + o2::itsmft::tracking::bounded_vector mTrackIndices; }; template diff --git a/Detectors/ITSMFT/ITS/tracking/GPU/ITStrackingGPU/TrackingKernels.h b/Detectors/ITSMFT/ITS/tracking/GPU/ITStrackingGPU/TrackingKernels.h index 00b2911a75458..33fac990ee086 100644 --- a/Detectors/ITSMFT/ITS/tracking/GPU/ITStrackingGPU/TrackingKernels.h +++ b/Detectors/ITSMFT/ITS/tracking/GPU/ITStrackingGPU/TrackingKernels.h @@ -16,7 +16,7 @@ #include #include -#include "ITStracking/BoundedAllocator.h" +#include "ITSMFTTracking/BoundedAllocator.h" #include "ITSMFTTracking/CapacityEstimator.h" #include "ITStracking/ROFLookupTables.h" #include "ITStracking/TrackingTopology.h" @@ -62,13 +62,13 @@ struct TrackingKernels { const bool selectUPCVertices, const float NSigmaCut, const typename TrackingTopology::View topology, - bounded_vector& linkPhiCuts, + o2::itsmft::tracking::bounded_vector& linkPhiCuts, const float resolutionPV, std::array& minR, std::array& maxR, - bounded_vector& resolutions, + o2::itsmft::tracking::bounded_vector& resolutions, std::vector& radii, - bounded_vector& linkMSAngles, + o2::itsmft::tracking::bounded_vector& linkMSAngles, o2::its::ExternalAllocator* alloc, gpu::Streams& streams); diff --git a/Detectors/ITSMFT/ITS/tracking/GPU/cuda/TimeFrameGPU.cu b/Detectors/ITSMFT/ITS/tracking/GPU/cuda/TimeFrameGPU.cu index 593f5529cf8ea..38661c8d136df 100644 --- a/Detectors/ITSMFT/ITS/tracking/GPU/cuda/TimeFrameGPU.cu +++ b/Detectors/ITSMFT/ITS/tracking/GPU/cuda/TimeFrameGPU.cu @@ -21,7 +21,7 @@ #include "ITStrackingGPU/TimeFrameGPU.h" #include "ITStracking/Constants.h" -#include "ITStracking/BoundedAllocator.h" +#include "ITSMFTTracking/BoundedAllocator.h" #include "ITStrackingGPU/Utils.h" #include "GPUCommonDef.h" @@ -491,8 +491,8 @@ void TimeFrameGPU::createTrackITSExtHost(const size_t nTracks) { GPUTimer timer("reserving host tracks"); mNTracks = nTracks; - mTrackITSExt = bounded_vector(nTracks, {}, this->getMemoryPool().get()); - mTrackIndices = bounded_vector(nTracks, 0, this->getMemoryPool().get()); + mTrackITSExt = o2::itsmft::tracking::bounded_vector(nTracks, {}, this->getMemoryPool().get()); + mTrackIndices = o2::itsmft::tracking::bounded_vector(nTracks, 0, this->getMemoryPool().get()); std::iota(mTrackIndices.begin(), mTrackIndices.end(), 0); } diff --git a/Detectors/ITSMFT/ITS/tracking/GPU/cuda/TrackerTraitsGPU.cxx b/Detectors/ITSMFT/ITS/tracking/GPU/cuda/TrackerTraitsGPU.cxx index 96443cc6b2668..a76fc258a06b5 100644 --- a/Detectors/ITSMFT/ITS/tracking/GPU/cuda/TrackerTraitsGPU.cxx +++ b/Detectors/ITSMFT/ITS/tracking/GPU/cuda/TrackerTraitsGPU.cxx @@ -199,7 +199,7 @@ template void TrackerTraitsGPU::findCellsNeighbours(const int iteration) { const auto hostTopology = mTimeFrameGPU->getTrackingTopologyView(); - bounded_vector sourceTopologies(this->getMemoryPool().get()); + o2::itsmft::tracking::bounded_vector sourceTopologies(this->getMemoryPool().get()); sourceTopologies.reserve(hostTopology.nCells); for (int outerLayer{0}; outerLayer < NLayers; ++outerLayer) { for (int targetCellTopologyId{0}; targetCellTopologyId < hostTopology.nCells; ++targetCellTopologyId) { @@ -266,7 +266,7 @@ void TrackerTraitsGPU::findCellsNeighbours(const int iteration) template void TrackerTraitsGPU::findRoads(const int iteration) { - bounded_vector> firstClusters(this->mTrkParams[iteration].NLayers, bounded_vector(this->getMemoryPool().get()), this->getMemoryPool().get()); + o2::itsmft::tracking::bounded_vector> firstClusters(this->mTrkParams[iteration].NLayers, o2::itsmft::tracking::bounded_vector(this->getMemoryPool().get()), this->getMemoryPool().get()); firstClusters.resize(this->mTrkParams[iteration].NLayers); const auto hostTopology = mTimeFrameGPU->getTrackingTopologyView(); const bool extendTop = this->mTrkParams[iteration].PassFlags[IterationStep::TrackFollowerTop]; diff --git a/Detectors/ITSMFT/ITS/tracking/GPU/cuda/TrackingKernels.cu b/Detectors/ITSMFT/ITS/tracking/GPU/cuda/TrackingKernels.cu index c44638466c01f..4d6deb124451a 100644 --- a/Detectors/ITSMFT/ITS/tracking/GPU/cuda/TrackingKernels.cu +++ b/Detectors/ITSMFT/ITS/tracking/GPU/cuda/TrackingKernels.cu @@ -789,13 +789,13 @@ int TrackingKernels::computeTrackletsInROFsHandler(const IndexTableUtil const bool selectUPCVertices, const float NSigmaCut, const typename TrackingTopology::View topology, - bounded_vector& linkPhiCuts, + o2::itsmft::tracking::bounded_vector& linkPhiCuts, const float resolutionPV, std::array& minRs, std::array& maxRs, - bounded_vector& resolutions, + o2::itsmft::tracking::bounded_vector& resolutions, std::vector& radii, - bounded_vector& linkMSAngles, + o2::itsmft::tracking::bounded_vector& linkMSAngles, o2::its::ExternalAllocator* alloc, gpu::Streams& streams) { diff --git a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/BoundedAllocator.h b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/BoundedAllocator.h deleted file mode 100644 index 70d6aed207092..0000000000000 --- a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/BoundedAllocator.h +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright 2019-2020 CERN and copyright holders of ALICE O2. -// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. -// All rights not expressly granted are reserved. -// -// This software is distributed under the terms of the GNU General Public -// License v3 (GPL Version 3), copied verbatim in the file "COPYING". -// -// In applying this license CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. -/// -/// \file BoundedAllocator.h -/// \brief -/// - -#ifndef TRACKINGITSU_INCLUDE_BOUNDEDALLOCATOR_H_ -#define TRACKINGITSU_INCLUDE_BOUNDEDALLOCATOR_H_ - -#include "ITSMFTTracking/BoundedAllocator.h" - -namespace o2::its -{ - -using o2::itsmft::tracking::BoundedMemoryResource; -template -using bounded_vector = o2::itsmft::tracking::bounded_vector; -using o2::itsmft::tracking::clearResizeBoundedArray; -using o2::itsmft::tracking::clearResizeBoundedVector; -using o2::itsmft::tracking::deepVectorClear; -using o2::itsmft::tracking::toSTDVector; - -} // namespace o2::its - -#endif /* TRACKINGITSU_INCLUDE_BOUNDEDALLOCATOR_H_ */ diff --git a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/LineVertexerHelpers.h b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/LineVertexerHelpers.h index 0e3807aba8efb..04b93a849718c 100644 --- a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/LineVertexerHelpers.h +++ b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/LineVertexerHelpers.h @@ -15,7 +15,7 @@ #include #include -#include "ITStracking/BoundedAllocator.h" +#include "ITSMFTTracking/BoundedAllocator.h" #include "ITStracking/ClusterLines.h" namespace o2::its::line_vertexer @@ -36,10 +36,10 @@ struct Settings { float maxZ = 0.f; int seedMemberRadiusTime = 1; int seedMemberRadiusZ = 2; - std::shared_ptr memoryPool; + std::shared_ptr memoryPool; }; -bounded_vector buildClusters(std::span lines, const Settings& settings); +o2::itsmft::tracking::bounded_vector buildClusters(std::span lines, const Settings& settings); } // namespace o2::its::line_vertexer diff --git a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TimeFrame.h b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TimeFrame.h index db237c6e6bc61..ea11b6b5dd95a 100644 --- a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TimeFrame.h +++ b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TimeFrame.h @@ -31,7 +31,7 @@ #include "ITStracking/Tracklet.h" #include "ITStracking/IndexTableUtils.h" #include "ITStracking/ExternalAllocator.h" -#include "ITStracking/BoundedAllocator.h" +#include "ITSMFTTracking/BoundedAllocator.h" #include "ITStracking/ROFLookupTables.h" #include "ITStracking/TrackingTopology.h" #include "SimulationDataFormat/MCCompLabel.h" @@ -179,7 +179,7 @@ struct TimeFrame { gsl::span getClusterLabels(int layerId, const int clId) const { return mClusterLabels[((mIsStaggered) ? layerId : 0)]->getLabels(mClusterExternalIndices[layerId][clId]); } int getClusterExternalIndex(int layerId, const int clId) const { return mClusterExternalIndices[layerId][clId]; } int getClusterSize(int layer, int clusterId) const { return mClusterSize[layer][clusterId]; } - void setClusterSize(int layer, bounded_vector& v) { mClusterSize[layer] = std::move(v); } + void setClusterSize(int layer, o2::itsmft::tracking::bounded_vector& v) { mClusterSize[layer] = std::move(v); } auto& getTrackletsLabel(int layer) { return mTrackletLabels[layer]; } auto& getCellsLabel(int layer) { return mCellLabels[layer]; } @@ -222,7 +222,7 @@ struct TimeFrame { size_t getNExtendedClusters() const { return mNExtendedClusters; } /// memory management - void setMemoryPool(std::shared_ptr pool); + void setMemoryPool(std::shared_ptr pool); auto& getMemoryPool() const noexcept { return mMemoryPool; } bool checkMemory(unsigned long max) { return getArtefactsMemory() < max; } unsigned long getArtefactsMemory() const; @@ -258,7 +258,7 @@ struct TimeFrame { /// State if memory will be externally managed by the GPU framework ExternalAllocator* mExternalAllocator{nullptr}; - std::shared_ptr mExtMemoryPool; // host memory pool managed by the framework + std::shared_ptr mExtMemoryPool; // host memory pool managed by the framework auto getFrameworkAllocator() { return mExternalAllocator; }; void setFrameworkAllocator(ExternalAllocator* ext); bool hasFrameworkAllocator() const noexcept { return mExternalAllocator != nullptr; } @@ -274,28 +274,28 @@ struct TimeFrame { void addTrackingFrameInfoToLayer(int layer, T&&... args); void addClusterExternalIndexToLayer(int layer, const int idx) { mClusterExternalIndices[layer].push_back(idx); } - std::array, NLayers> mClusters; - std::array, NLayers> mTrackingFrameInfo; - std::array, NLayers> mClusterExternalIndices; - std::array, NLayers> mROFramesClusters; + std::array, NLayers> mClusters; + std::array, NLayers> mTrackingFrameInfo; + std::array, NLayers> mClusterExternalIndices; + std::array, NLayers> mROFramesClusters; std::array*, NLayers> mClusterLabels{nullptr}; - std::array, 2> mNTrackletsPerCluster; - std::array, 2> mNTrackletsPerClusterSum; - std::array, NLayers> mNClustersPerROF; - std::array, NLayers> mIndexTables; - std::vector> mTrackletsLookupTable; - std::array, NLayers> mUsedClusters; - - std::array, NLayers> mUnsortedClusters; - std::vector> mTracklets; - std::vector> mCells; - bounded_vector mTracks; - bounded_vector mTracksLabel; + std::array, 2> mNTrackletsPerCluster; + std::array, 2> mNTrackletsPerClusterSum; + std::array, NLayers> mNClustersPerROF; + std::array, NLayers> mIndexTables; + std::vector> mTrackletsLookupTable; + std::array, NLayers> mUsedClusters; + + std::array, NLayers> mUnsortedClusters; + std::vector> mTracklets; + std::vector> mCells; + o2::itsmft::tracking::bounded_vector mTracks; + o2::itsmft::tracking::bounded_vector mTracksLabel; size_t mNExtendedTracks = 0; size_t mNExtendedClusters = 0; - std::vector> mCellsNeighbours; - std::vector> mCellsNeighboursTopology; - std::vector> mCellsLookupTable; + std::vector> mCellsNeighbours; + std::vector> mCellsNeighboursTopology; + std::vector> mCellsLookupTable; const o2::base::PropagatorImpl* mPropagatorDevice = nullptr; // Needed only for GPU @@ -314,27 +314,27 @@ struct TimeFrame { bool isBeamPositionOverridden = false; std::array mMinR; std::array mMaxR; - bounded_vector mLinkPhiCuts; - bounded_vector mLinkMSAngles; - bounded_vector mPositionResolution; - std::array, NLayers> mClusterSize; + o2::itsmft::tracking::bounded_vector mLinkPhiCuts; + o2::itsmft::tracking::bounded_vector mLinkMSAngles; + o2::itsmft::tracking::bounded_vector mPositionResolution; + std::array, NLayers> mClusterSize; - bounded_vector> mPValphaX; /// PV x and alpha for track propagation - std::vector> mTrackletLabels; - std::vector> mCellLabels; - std::vector> mCellsNeighboursLUT; - bounded_vector mBogusClusters; /// keep track of clusters with wild coordinates + o2::itsmft::tracking::bounded_vector> mPValphaX; /// PV x and alpha for track propagation + std::vector> mTrackletLabels; + std::vector> mCellLabels; + std::vector> mCellsNeighboursLUT; + o2::itsmft::tracking::bounded_vector mBogusClusters; /// keep track of clusters with wild coordinates o2::itsmft::tracking::CapacityEstimator mCapacityEstimator; // Vertexer - bounded_vector mPrimaryVertices; - bounded_vector mPrimaryVerticesLabels; - std::vector> mNTrackletsPerROF; - std::vector> mLines; - std::vector> mTrackletClusters; - std::array, 2> mTrackletsIndexROF; - std::vector> mLinesLabels; + o2::itsmft::tracking::bounded_vector mPrimaryVertices; + o2::itsmft::tracking::bounded_vector mPrimaryVerticesLabels; + std::vector> mNTrackletsPerROF; + std::vector> mLines; + std::vector> mTrackletClusters; + std::array, 2> mTrackletsIndexROF; + std::vector> mLinesLabels; std::array mTotalTracklets = {0, 0}; uint32_t mTotalLines = 0; // \Vertexer @@ -356,7 +356,7 @@ struct TimeFrame { bool mIsStaggered{false}; - std::shared_ptr mMemoryPool; + std::shared_ptr mMemoryPool; }; template diff --git a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/Tracker.h b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/Tracker.h index c60f280de5307..569f4e6e913a6 100644 --- a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/Tracker.h +++ b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/Tracker.h @@ -33,7 +33,7 @@ #include "ITStracking/Definitions.h" #include "ITStracking/TimeFrame.h" #include "ITStracking/TrackerTraits.h" -#include "ITStracking/BoundedAllocator.h" +#include "ITSMFTTracking/BoundedAllocator.h" namespace o2 { @@ -60,7 +60,7 @@ class Tracker const LogFunc& = [](const std::string& s) { std::cerr << s << '\n'; }); void setParameters(const std::vector& p) { mTrkParams = p; } - void setMemoryPool(std::shared_ptr pool) { mMemoryPool = pool; } + void setMemoryPool(std::shared_ptr pool) { mMemoryPool = pool; } std::vector& getParameters() { return mTrkParams; } void setBz(float bz) { mTraits->setBz(bz); } void setTimeSlice(size_t slice) noexcept { mTimeSlice = slice; } @@ -91,7 +91,7 @@ class Tracker unsigned int mNumberOfDroppedTFs{0}; unsigned int mTimeFrameCounter{0}; double mTotalTime{0}; - std::shared_ptr mMemoryPool; + std::shared_ptr mMemoryPool; enum Steps { TFInit = 0, diff --git a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TrackerTraits.h b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TrackerTraits.h index 6c6159f8eee2b..edfe908aaabff 100644 --- a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TrackerTraits.h +++ b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TrackerTraits.h @@ -26,7 +26,7 @@ #include "ITSMFTTracking/CapacityEstimator.h" #include "ITStracking/TimeFrame.h" #include "ITStracking/Cell.h" -#include "ITStracking/BoundedAllocator.h" +#include "ITSMFTTracking/BoundedAllocator.h" #include "ITStracking/TrackExtensionHypothesis.h" #include "ITStracking/TrackFollower.h" #include "ITStracking/TrackITSInternal.h" @@ -72,9 +72,9 @@ class TrackerTraits virtual void findRoads(const int iteration); template - void processNeighbours(int iteration, int defaultCellTopologyId, int iLevel, uint64_t capacityKey, const bounded_vector& currentSeeds, bounded_vector& updatedSeeds); + void processNeighbours(int iteration, int defaultCellTopologyId, int iLevel, uint64_t capacityKey, const o2::itsmft::tracking::bounded_vector& currentSeeds, o2::itsmft::tracking::bounded_vector& updatedSeeds); - void acceptTracks(int iteration, bounded_vector& tracks, const bounded_vector& trackIndices, bounded_vector>& firstClusters); + void acceptTracks(int iteration, o2::itsmft::tracking::bounded_vector& tracks, const o2::itsmft::tracking::bounded_vector& trackIndices, o2::itsmft::tracking::bounded_vector>& firstClusters); void markTracks(int iteration); void updateTrackingParameters(const std::vector& trkPars) @@ -86,7 +86,7 @@ class TrackerTraits float getBz() const { return mBz; } virtual const char* getName() const noexcept { return "CPU"; } virtual bool isGPU() const noexcept { return false; } - void setMemoryPool(std::shared_ptr pool) noexcept { mMemoryPool = pool; } + void setMemoryPool(std::shared_ptr pool) noexcept { mMemoryPool = pool; } auto getMemoryPool() const noexcept { return mMemoryPool; } // Others @@ -99,7 +99,7 @@ class TrackerTraits virtual int getTFNumberOfCells() const { return mTimeFrame->getNumberOfCells(); } private: - std::shared_ptr mMemoryPool; + std::shared_ptr mMemoryPool; protected: std::shared_ptr mTaskArena; @@ -110,8 +110,8 @@ class TrackerTraits { } - bounded_vector> activeHypotheses; - bounded_vector> nextHypotheses; + o2::itsmft::tracking::bounded_vector> activeHypotheses; + o2::itsmft::tracking::bounded_vector> nextHypotheses; }; bool finaliseTrackSeed(const TrackSeedN& seed, diff --git a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TrackingInterface.h b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TrackingInterface.h index 14c5d6a62e0ad..4ccc1259a33bf 100644 --- a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TrackingInterface.h +++ b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TrackingInterface.h @@ -19,7 +19,7 @@ #include "ITStracking/TrackerTraits.h" #include "ITStracking/Vertexer.h" #include "ITStracking/VertexerTraits.h" -#include "ITStracking/BoundedAllocator.h" +#include "ITSMFTTracking/BoundedAllocator.h" #include "DataFormatsParameters/GRPObject.h" #include "DataFormatsITSMFT/TopologyDictionary.h" #include "DataFormatsCalibration/MeanVertexObject.h" @@ -101,7 +101,7 @@ class ITSTrackingInterface std::unique_ptr mTracker = nullptr; std::unique_ptr mVertexer = nullptr; const o2::dataformats::MeanVertexObject* mMeanVertex{}; - std::shared_ptr mMemoryPool; + std::shared_ptr mMemoryPool; std::shared_ptr mTaskArena; }; diff --git a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/Vertexer.h b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/Vertexer.h index 59c22b505bc94..a932d0b7b35c5 100644 --- a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/Vertexer.h +++ b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/Vertexer.h @@ -30,7 +30,7 @@ #include "ITStracking/Configuration.h" #include "ITStracking/TimeFrame.h" #include "ITStracking/VertexerTraits.h" -#include "ITStracking/BoundedAllocator.h" +#include "ITSMFTTracking/BoundedAllocator.h" namespace o2::its { @@ -52,7 +52,7 @@ class Vertexer auto& getVertParameters() const { return mTraits->getVertexingParameters(); } void setParameters(const std::vector& vertParams) { mVertParams = vertParams; } const auto& getParameters() const noexcept { return mVertParams; } - void setMemoryPool(std::shared_ptr pool) { mMemoryPool = pool; } + void setMemoryPool(std::shared_ptr pool) { mMemoryPool = pool; } float clustersToVertices(LogFunc = [](const std::string& s) { std::cout << s << '\n'; }); void filterMCTracklets(); @@ -105,7 +105,7 @@ class Vertexer TimeFrameN* mTimeFrame = nullptr; /// Observer pointer, not owned by this class std::vector mVertParams; - std::shared_ptr mMemoryPool; + std::shared_ptr mMemoryPool; enum Steps { Init = 0, diff --git a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/VertexerTraits.h b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/VertexerTraits.h index daf8d708e1e23..23d3934a8764e 100644 --- a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/VertexerTraits.h +++ b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/VertexerTraits.h @@ -21,7 +21,7 @@ #include #include -#include "ITStracking/BoundedAllocator.h" +#include "ITSMFTTracking/BoundedAllocator.h" #include "ITStracking/Cluster.h" #include "ITStracking/ClusterLines.h" #include "ITStracking/Configuration.h" @@ -73,9 +73,9 @@ class VertexerTraits virtual bool isGPU() const noexcept { return false; } virtual const char* getName() const noexcept { return "CPU"; } virtual bool usesMemoryPool() const noexcept { return true; } - void setMemoryPool(std::shared_ptr pool) { mMemoryPool = pool; } + void setMemoryPool(std::shared_ptr pool) { mMemoryPool = pool; } - static VertexLabel computeMain(const bounded_vector& elements) + static VertexLabel computeMain(const o2::itsmft::tracking::bounded_vector& elements) { // we only care about the source&event of the tracks, not the trackId auto composeVtxLabel = [](const o2::MCCompLabel& lbl) -> o2::MCCompLabel { @@ -108,7 +108,7 @@ class VertexerTraits private: bool skipROF(int iteration, int rof) const; - std::shared_ptr mMemoryPool; + std::shared_ptr mMemoryPool; std::shared_ptr mTaskArena; }; diff --git a/Detectors/ITSMFT/ITS/tracking/src/LineVertexerHelpers.cxx b/Detectors/ITSMFT/ITS/tracking/src/LineVertexerHelpers.cxx index cbb8d52571ec9..de70d07f8c362 100644 --- a/Detectors/ITSMFT/ITS/tracking/src/LineVertexerHelpers.cxx +++ b/Detectors/ITSMFT/ITS/tracking/src/LineVertexerHelpers.cxx @@ -80,13 +80,13 @@ struct LineRef { }; struct VertexSeed { - explicit VertexSeed(const std::shared_ptr& mr) : contributors(mr.get()), assigned(mr.get()) {} + explicit VertexSeed(const std::shared_ptr& mr) : contributors(mr.get()), assigned(mr.get()) {} std::array vertex = {}; TimeEstBC time; float scale2 = InitialScale2; - bounded_vector contributors; - bounded_vector assigned; + o2::itsmft::tracking::bounded_vector contributors; + o2::itsmft::tracking::bounded_vector assigned; bool valid = false; bool isUsableSeed() const noexcept { @@ -94,7 +94,7 @@ struct VertexSeed { } }; -void compactSeeds(bounded_vector& seeds) +void compactSeeds(o2::itsmft::tracking::bounded_vector& seeds) { seeds.erase(std::remove_if(seeds.begin(), seeds.end(), [](const VertexSeed& seed) { return !seed.isUsableSeed(); @@ -103,7 +103,7 @@ void compactSeeds(bounded_vector& seeds) } struct Histogram2D { - explicit Histogram2D(const std::shared_ptr& mr) : bins(mr.get()) {} + explicit Histogram2D(const std::shared_ptr& mr) : bins(mr.get()) {} int nTimeBins = 0; int nZBins = 0; @@ -111,7 +111,7 @@ struct Histogram2D { float zMin = 0.f; float timeBinSize = 1.f; float zBinSize = 1.f; - bounded_vector bins; + o2::itsmft::tracking::bounded_vector bins; int getIndex(const int tBin, const int zBin) const noexcept { @@ -313,9 +313,9 @@ class SeedHistogram return mHistogram.getNeighborhoodSum(peakIndex, mSeedMemberRadiusTime, mSeedMemberRadiusZ); } - bounded_vector collectLocalMembers(const int peakIndex, const int radiusTime, const int radiusZ) const + o2::itsmft::tracking::bounded_vector collectLocalMembers(const int peakIndex, const int radiusTime, const int radiusZ) const { - bounded_vector localMembers(mMemoryPool.get()); + o2::itsmft::tracking::bounded_vector localMembers(mMemoryPool.get()); localMembers.reserve(mMembers.size()); const auto [timeBin, zBin] = mHistogram.decodeIndex(peakIndex); for (const auto lineRefIdx : mMembers) { @@ -358,7 +358,7 @@ class SeedHistogram private: float medianTimeError(std::span lines) const { - bounded_vector errors(mMemoryPool.get()); + o2::itsmft::tracking::bounded_vector errors(mMemoryPool.get()); errors.reserve(mMembers.size()); for (const auto lineRefIdx : mMembers) { errors.push_back(static_cast(lines[mLineRefs[lineRefIdx].lineIndex].mTime.getTimeStampError())); @@ -371,17 +371,17 @@ class SeedHistogram std::span mLineRefs; int mSeedMemberRadiusTime = 1; int mSeedMemberRadiusZ = 2; - std::shared_ptr mMemoryPool; + std::shared_ptr mMemoryPool; Histogram2D mHistogram; }; -float updateScale2(const std::span chi2s, const std::shared_ptr& mr) noexcept +float updateScale2(const std::span chi2s, const std::shared_ptr& mr) noexcept { if (chi2s.empty()) { return MinScale2; } - bounded_vector sorted(chi2s.begin(), chi2s.end(), mr.get()); + o2::itsmft::tracking::bounded_vector sorted(chi2s.begin(), chi2s.end(), mr.get()); std::sort(sorted.begin(), sorted.end()); const auto median = sorted[sorted.size() / 2]; @@ -442,7 +442,7 @@ VertexSeed fitSeed(const VertexSeed& initialSeed, std::span members, std::span lineRefs, std::span lines, - const std::shared_ptr& mr, + const std::shared_ptr& mr, const float pairCut2) { VertexSeed seed{mr}; @@ -460,7 +460,7 @@ VertexSeed fitSeed(const VertexSeed& initialSeed, VertexFit vertexFit; TimeEstBC commonTime{}; bool hasCommonTime = false; - bounded_vector contributors{mr.get()}; + o2::itsmft::tracking::bounded_vector contributors{mr.get()}; const auto scale2 = std::max(seed.scale2, MinScale2); const auto tukeyFactor = 1.f / (scale2 * TukeyC2); @@ -511,7 +511,7 @@ VertexSeed fitSeed(const VertexSeed& initialSeed, seed.vertex = updatedVertex; seed.time = commonTime; - bounded_vector updatedChi2s{mr.get()}; + o2::itsmft::tracking::bounded_vector updatedChi2s{mr.get()}; updatedChi2s.reserve(contributors.size()); for (const auto lineRefIx : contributors) { updatedChi2s.push_back(Line::getDistance2FromPoint(lines[lineRefs[lineRefIx].lineIndex], seed.vertex) / pairCut2); @@ -547,14 +547,14 @@ size_t countSharedContributors(std::span lhs, std::span rh return shared; } -bounded_vector collectCompatibleContributors(const VertexSeed& seed, - std::span members, - std::span lineRefs, - std::span lines, - const std::shared_ptr& mr, - const float pairCut2) +o2::itsmft::tracking::bounded_vector collectCompatibleContributors(const VertexSeed& seed, + std::span members, + std::span lineRefs, + std::span lines, + const std::shared_ptr& mr, + const float pairCut2) { - bounded_vector contributors{mr.get()}; + o2::itsmft::tracking::bounded_vector contributors{mr.get()}; contributors.reserve(members.size()); for (const auto lineRefIdx : members) { const auto lineIdx = lineRefs[lineRefIdx].lineIndex; @@ -571,7 +571,7 @@ bounded_vector collectCompatibleContributors(const VertexSeed& seed, return contributors; } -void deduplicateSeeds(bounded_vector& seeds, const Settings& settings) +void deduplicateSeeds(o2::itsmft::tracking::bounded_vector& seeds, const Settings& settings) { if (seeds.size() < 2) { return; @@ -622,7 +622,7 @@ void deduplicateSeeds(bounded_vector& seeds, const Settings& setting compactSeeds(seeds); } -void deduplicateRefittedSeeds(bounded_vector& seeds, const Settings& settings) +void deduplicateRefittedSeeds(o2::itsmft::tracking::bounded_vector& seeds, const Settings& settings) { if (seeds.size() < 2) { return; @@ -675,21 +675,21 @@ void deduplicateRefittedSeeds(bounded_vector& seeds, const Settings& } struct OrderedComponent { - explicit OrderedComponent(const std::shared_ptr& mr) : members(mr.get()) {} + explicit OrderedComponent(const std::shared_ptr& mr) : members(mr.get()) {} float center = 0.f; - bounded_vector members; + o2::itsmft::tracking::bounded_vector members; }; -bounded_vector> buildCoarseClusters(std::span lineRefs, - std::span lines, - const Settings& settings) +o2::itsmft::tracking::bounded_vector> buildCoarseClusters(std::span lineRefs, + std::span lines, + const Settings& settings) { - bounded_vector> clusters(settings.memoryPool.get()); + o2::itsmft::tracking::bounded_vector> clusters(settings.memoryPool.get()); if (lineRefs.size() < 2) { return clusters; } - bounded_vector sortedByLower(lineRefs.size(), settings.memoryPool.get()); + o2::itsmft::tracking::bounded_vector sortedByLower(lineRefs.size(), settings.memoryPool.get()); std::iota(sortedByLower.begin(), sortedByLower.end(), 0); std::sort(sortedByLower.begin(), sortedByLower.end(), [&](const int lhs, const int rhs) { const auto lhsLower = lines[lineRefs[lhs].lineIndex].mTime.lower(); @@ -701,8 +701,8 @@ bounded_vector> buildCoarseClusters(std::span }); const auto coarseZWindow = settings.coarseZWindow > 0.f ? settings.coarseZWindow : settings.clusterCut; - bounded_vector parent(lineRefs.size(), settings.memoryPool.get()); - bounded_vector componentSize(lineRefs.size(), 1, settings.memoryPool.get()); + o2::itsmft::tracking::bounded_vector parent(lineRefs.size(), settings.memoryPool.get()); + o2::itsmft::tracking::bounded_vector componentSize(lineRefs.size(), 1, settings.memoryPool.get()); std::iota(parent.begin(), parent.end(), 0); float minZ = std::numeric_limits::max(); float maxZ = std::numeric_limits::lowest(); @@ -742,10 +742,10 @@ bounded_vector> buildCoarseClusters(std::span }; using ActiveEntry = std::pair; - bounded_vector activeEntries(settings.memoryPool.get()); - std::priority_queue, std::greater<>> activeByUpper(std::greater<>{}, std::move(activeEntries)); - bounded_vector activeMask(lineRefs.size(), 0, settings.memoryPool.get()); - bounded_vector> activeByZBin(settings.memoryPool.get()); + o2::itsmft::tracking::bounded_vector activeEntries(settings.memoryPool.get()); + std::priority_queue, std::greater<>> activeByUpper(std::greater<>{}, std::move(activeEntries)); + o2::itsmft::tracking::bounded_vector activeMask(lineRefs.size(), 0, settings.memoryPool.get()); + o2::itsmft::tracking::bounded_vector> activeByZBin(settings.memoryPool.get()); activeByZBin.reserve(nZBins); for (int iBin = 0; iBin < nZBins; ++iBin) { activeByZBin.emplace_back(); @@ -787,7 +787,7 @@ bounded_vector> buildCoarseClusters(std::span activeByZBin[zBin].push_back(lineRefIdx); } - std::unordered_map> components; + std::unordered_map> components; components.reserve(lineRefs.size()); for (int lineRefIdx = 0; lineRefIdx < static_cast(lineRefs.size()); ++lineRefIdx) { const auto root = findRoot(lineRefIdx); @@ -796,7 +796,7 @@ bounded_vector> buildCoarseClusters(std::span it->second.push_back(lineRefIdx); } - bounded_vector orderedComponents(settings.memoryPool.get()); + o2::itsmft::tracking::bounded_vector orderedComponents(settings.memoryPool.get()); orderedComponents.reserve(components.size()); for (auto& [root, members] : components) { (void)root; @@ -829,13 +829,13 @@ bounded_vector> buildCoarseClusters(std::span return clusters; } -bounded_vector buildSeeds(std::span members, - std::span lineRefs, - std::span lines, - const Settings& settings) +o2::itsmft::tracking::bounded_vector buildSeeds(std::span members, + std::span lineRefs, + std::span lines, + const Settings& settings) { SeedHistogram histogram(members, lineRefs, lines, settings); - bounded_vector seeds(settings.memoryPool.get()); + o2::itsmft::tracking::bounded_vector seeds(settings.memoryPool.get()); seeds.reserve(MaxSeedsPerCluster); float leadingPeakSupport = 0.f; @@ -879,7 +879,7 @@ bounded_vector buildSeeds(std::span members, return seeds; } -void assignLinesToSeeds(bounded_vector& seeds, +void assignLinesToSeeds(o2::itsmft::tracking::bounded_vector& seeds, std::span members, std::span lineRefs, std::span lines, @@ -937,9 +937,9 @@ void assignLinesToSeeds(bounded_vector& seeds, ClusterLines materializeCluster(const VertexSeed& seed, std::span lineRefs, std::span lines, - const std::shared_ptr& mr) + const std::shared_ptr& mr) { - bounded_vector lineIndices{mr.get()}; + o2::itsmft::tracking::bounded_vector lineIndices{mr.get()}; lineIndices.reserve(seed.contributors.size()); for (const auto lineRefIdx : seed.contributors) { lineIndices.push_back(lineRefs[lineRefIdx].lineIndex); @@ -956,14 +956,14 @@ ClusterLines materializeCluster(const VertexSeed& seed, } // namespace -bounded_vector buildClusters(std::span lines, const Settings& settings) +o2::itsmft::tracking::bounded_vector buildClusters(std::span lines, const Settings& settings) { - bounded_vector clusters(settings.memoryPool.get()); + o2::itsmft::tracking::bounded_vector clusters(settings.memoryPool.get()); if (lines.size() < 2) { return clusters; } - bounded_vector refs(settings.memoryPool.get()); + o2::itsmft::tracking::bounded_vector refs(settings.memoryPool.get()); refs.reserve(lines.size()); for (int lineIdx = 0; lineIdx < static_cast(lines.size()); ++lineIdx) { LineRef ref(lines[lineIdx], lineIdx, settings.beamX, settings.beamY, settings.maxZ); diff --git a/Detectors/ITSMFT/ITS/tracking/src/TimeFrame.cxx b/Detectors/ITSMFT/ITS/tracking/src/TimeFrame.cxx index a652e3aaecd75..2ce0b7dabb070 100644 --- a/Detectors/ITSMFT/ITS/tracking/src/TimeFrame.cxx +++ b/Detectors/ITSMFT/ITS/tracking/src/TimeFrame.cxx @@ -26,7 +26,7 @@ #include "DataFormatsITSMFT/TopologyDictionary.h" #include "ITSBase/GeometryTGeo.h" #include "ITSMFTBase/SegmentationAlpide.h" -#include "ITStracking/BoundedAllocator.h" +#include "ITSMFTTracking/BoundedAllocator.h" namespace { @@ -150,16 +150,16 @@ template void TimeFrame::resetROFrameData(int layer) { if (layer >= 0) { - deepVectorClear(mUnsortedClusters[layer], getMaybeFrameworkHostResource()); - deepVectorClear(mTrackingFrameInfo[layer], getMaybeFrameworkHostResource()); - deepVectorClear(mClusterExternalIndices[layer], mMemoryPool.get()); - clearResizeBoundedVector(mROFramesClusters[layer], mROFOverlapTableView.getLayer(layer).mNROFsTF + 1, getMaybeFrameworkHostResource()); + o2::itsmft::tracking::deepVectorClear(mUnsortedClusters[layer], getMaybeFrameworkHostResource()); + o2::itsmft::tracking::deepVectorClear(mTrackingFrameInfo[layer], getMaybeFrameworkHostResource()); + o2::itsmft::tracking::deepVectorClear(mClusterExternalIndices[layer], mMemoryPool.get()); + o2::itsmft::tracking::clearResizeBoundedVector(mROFramesClusters[layer], mROFOverlapTableView.getLayer(layer).mNROFsTF + 1, getMaybeFrameworkHostResource()); } else { for (int iLayer{0}; iLayer < NLayers; ++iLayer) { - deepVectorClear(mUnsortedClusters[iLayer], getMaybeFrameworkHostResource()); - deepVectorClear(mTrackingFrameInfo[iLayer], getMaybeFrameworkHostResource()); - deepVectorClear(mClusterExternalIndices[iLayer], mMemoryPool.get()); - clearResizeBoundedVector(mROFramesClusters[iLayer], mROFOverlapTableView.getLayer(iLayer).mNROFsTF + 1, getMaybeFrameworkHostResource()); + o2::itsmft::tracking::deepVectorClear(mUnsortedClusters[iLayer], getMaybeFrameworkHostResource()); + o2::itsmft::tracking::deepVectorClear(mTrackingFrameInfo[iLayer], getMaybeFrameworkHostResource()); + o2::itsmft::tracking::deepVectorClear(mClusterExternalIndices[iLayer], mMemoryPool.get()); + o2::itsmft::tracking::clearResizeBoundedVector(mROFramesClusters[iLayer], mROFOverlapTableView.getLayer(iLayer).mNROFsTF + 1, getMaybeFrameworkHostResource()); } } } @@ -171,10 +171,10 @@ void TimeFrame::prepareROFrameData(gsl::span clusterCountPerLayer{0}; for (const auto& cls : clusters) { ++clusterCountPerLayer[geom->getLayer(cls.getChipID())]; @@ -195,9 +195,9 @@ void TimeFrame::prepareClusters(const TrackingParameters& trkParam, con const int stopLayer = std::min(trkParam.NLayers, maxLayers); tbb::parallel_for(0, stopLayer, [&](const int iLayer) { - bounded_vector cHelper(mMemoryPool.get()); - bounded_vector clsPerBin(numBins, 0, mMemoryPool.get()); - bounded_vector lutPerBin(numBins, 0, mMemoryPool.get()); + o2::itsmft::tracking::bounded_vector cHelper(mMemoryPool.get()); + o2::itsmft::tracking::bounded_vector clsPerBin(numBins, 0, mMemoryPool.get()); + o2::itsmft::tracking::bounded_vector lutPerBin(numBins, 0, mMemoryPool.get()); float minR{mMinR[iLayer]}; float maxR{mMaxR[iLayer]}; int bogus{0}; @@ -298,29 +298,29 @@ void TimeFrame::initialise(const TrackingParameters& trkParam, const in mTrackingTopologyView = iteration != constants::UnusedIndex ? mTrackerTopologies[iteration].getView() : (maxLayers == 3 ? mVertexingTopology.getView() : mDefaultTrackingTopology.getView()); if (trkParam.PassFlags[IterationStep::FirstPass]) { - deepVectorClear(mTracks); - deepVectorClear(mTracksLabel); - deepVectorClear(mLines); - deepVectorClear(mLinesLabels); + o2::itsmft::tracking::deepVectorClear(mTracks); + o2::itsmft::tracking::deepVectorClear(mTracksLabel); + o2::itsmft::tracking::deepVectorClear(mLines); + o2::itsmft::tracking::deepVectorClear(mLinesLabels); if (trkParam.PassFlags[IterationStep::ResetVertices]) { - deepVectorClear(mPrimaryVertices); - deepVectorClear(mPrimaryVerticesLabels); + o2::itsmft::tracking::deepVectorClear(mPrimaryVertices); + o2::itsmft::tracking::deepVectorClear(mPrimaryVerticesLabels); } - clearResizeBoundedVector(mLinesLabels, getNrof(1), mMemoryPool.get()); + o2::itsmft::tracking::clearResizeBoundedVector(mLinesLabels, getNrof(1), mMemoryPool.get()); mIndexTableUtils.setTrackingParameters(trkParam); - clearResizeBoundedVector(mPositionResolution, trkParam.NLayers, mMemoryPool.get()); - clearResizeBoundedVector(mBogusClusters, trkParam.NLayers, mMemoryPool.get()); - deepVectorClear(mTrackletClusters); + o2::itsmft::tracking::clearResizeBoundedVector(mPositionResolution, trkParam.NLayers, mMemoryPool.get()); + o2::itsmft::tracking::clearResizeBoundedVector(mBogusClusters, trkParam.NLayers, mMemoryPool.get()); + o2::itsmft::tracking::deepVectorClear(mTrackletClusters); for (unsigned int iLayer{0}; iLayer < std::min((int)mClusters.size(), maxLayers); ++iLayer) { - clearResizeBoundedVector(mClusters[iLayer], mUnsortedClusters[iLayer].size(), getMaybeFrameworkHostResource(maxLayers != NLayers)); - clearResizeBoundedVector(mUsedClusters[iLayer], mUnsortedClusters[iLayer].size(), getMaybeFrameworkHostResource(maxLayers != NLayers)); + o2::itsmft::tracking::clearResizeBoundedVector(mClusters[iLayer], mUnsortedClusters[iLayer].size(), getMaybeFrameworkHostResource(maxLayers != NLayers)); + o2::itsmft::tracking::clearResizeBoundedVector(mUsedClusters[iLayer], mUnsortedClusters[iLayer].size(), getMaybeFrameworkHostResource(maxLayers != NLayers)); mPositionResolution[iLayer] = o2::gpu::CAMath::Sqrt((0.5f * (trkParam.SystErrorZ2[iLayer] + trkParam.SystErrorY2[iLayer])) + (trkParam.LayerResolution[iLayer] * trkParam.LayerResolution[iLayer])); } - clearResizeBoundedVector(mLines, getNrof(1), mMemoryPool.get()); - clearResizeBoundedVector(mTrackletClusters, getNrof(1), mMemoryPool.get()); + o2::itsmft::tracking::clearResizeBoundedVector(mLines, getNrof(1), mMemoryPool.get()); + o2::itsmft::tracking::clearResizeBoundedVector(mTrackletClusters, getNrof(1), mMemoryPool.get()); for (int iLayer{0}; iLayer < NLayers; ++iLayer) { - clearResizeBoundedVector(mIndexTables[iLayer], getNrof(iLayer) * ((trkParam.ZBins * trkParam.PhiBins) + 1), getMaybeFrameworkHostResource()); + o2::itsmft::tracking::clearResizeBoundedVector(mIndexTables[iLayer], getNrof(iLayer) * ((trkParam.ZBins * trkParam.PhiBins) + 1), getMaybeFrameworkHostResource()); } for (int iLayer{0}; iLayer < trkParam.NLayers; ++iLayer) { if (trkParam.SystErrorY2[iLayer] > 0.f || trkParam.SystErrorZ2[iLayer] > 0.f) { @@ -335,20 +335,20 @@ void TimeFrame::initialise(const TrackingParameters& trkParam, const in mMinR.fill(std::numeric_limits::max()); mMaxR.fill(std::numeric_limits::min()); } - clearResizeBoundedVector(mCells, mTrackingTopologyView.nCells, mMemoryPool.get()); - clearResizeBoundedVector(mCellsLookupTable, mTrackingTopologyView.nCells, mMemoryPool.get()); - clearResizeBoundedVector(mCellsNeighbours, mTrackingTopologyView.nCells, mMemoryPool.get()); - clearResizeBoundedVector(mCellsNeighboursTopology, mTrackingTopologyView.nCells, mMemoryPool.get()); - clearResizeBoundedVector(mCellsNeighboursLUT, mTrackingTopologyView.nCells, mMemoryPool.get()); - clearResizeBoundedVector(mCellLabels, mTrackingTopologyView.nCells, mMemoryPool.get()); - clearResizeBoundedVector(mTracklets, mTrackingTopologyView.nLinks, mMemoryPool.get()); - clearResizeBoundedVector(mTrackletLabels, mTrackingTopologyView.nLinks, mMemoryPool.get()); - clearResizeBoundedVector(mTrackletsLookupTable, mTrackingTopologyView.nLinks, mMemoryPool.get()); - clearResizeBoundedVector(mLinkPhiCuts, mTrackingTopologyView.nLinks, mMemoryPool.get()); - clearResizeBoundedVector(mLinkMSAngles, mTrackingTopologyView.nLinks, mMemoryPool.get()); + o2::itsmft::tracking::clearResizeBoundedVector(mCells, mTrackingTopologyView.nCells, mMemoryPool.get()); + o2::itsmft::tracking::clearResizeBoundedVector(mCellsLookupTable, mTrackingTopologyView.nCells, mMemoryPool.get()); + o2::itsmft::tracking::clearResizeBoundedVector(mCellsNeighbours, mTrackingTopologyView.nCells, mMemoryPool.get()); + o2::itsmft::tracking::clearResizeBoundedVector(mCellsNeighboursTopology, mTrackingTopologyView.nCells, mMemoryPool.get()); + o2::itsmft::tracking::clearResizeBoundedVector(mCellsNeighboursLUT, mTrackingTopologyView.nCells, mMemoryPool.get()); + o2::itsmft::tracking::clearResizeBoundedVector(mCellLabels, mTrackingTopologyView.nCells, mMemoryPool.get()); + o2::itsmft::tracking::clearResizeBoundedVector(mTracklets, mTrackingTopologyView.nLinks, mMemoryPool.get()); + o2::itsmft::tracking::clearResizeBoundedVector(mTrackletLabels, mTrackingTopologyView.nLinks, mMemoryPool.get()); + o2::itsmft::tracking::clearResizeBoundedVector(mTrackletsLookupTable, mTrackingTopologyView.nLinks, mMemoryPool.get()); + o2::itsmft::tracking::clearResizeBoundedVector(mLinkPhiCuts, mTrackingTopologyView.nLinks, mMemoryPool.get()); + o2::itsmft::tracking::clearResizeBoundedVector(mLinkMSAngles, mTrackingTopologyView.nLinks, mMemoryPool.get()); mNTrackletsPerROF.resize(2); for (auto& v : mNTrackletsPerROF) { - v = bounded_vector(getNrof(1) + 1, 0, mMemoryPool.get()); + v = o2::itsmft::tracking::bounded_vector(getNrof(1) + 1, 0, mMemoryPool.get()); } if (trkParam.PassFlags[IterationStep::RebuildClusterLUT]) { prepareClusters(trkParam, maxLayers); @@ -356,8 +356,8 @@ void TimeFrame::initialise(const TrackingParameters& trkParam, const in mTotalTracklets = {0, 0}; if (maxLayers < trkParam.NLayers) { // Vertexer only, but in both iterations for (size_t iLayer{0}; iLayer < maxLayers; ++iLayer) { - deepVectorClear(mUsedClusters[iLayer]); - clearResizeBoundedVector(mUsedClusters[iLayer], mUnsortedClusters[iLayer].size(), mMemoryPool.get()); + o2::itsmft::tracking::deepVectorClear(mUsedClusters[iLayer]); + o2::itsmft::tracking::clearResizeBoundedVector(mUsedClusters[iLayer], mUnsortedClusters[iLayer].size(), mMemoryPool.get()); } } @@ -390,19 +390,19 @@ void TimeFrame::initialise(const TrackingParameters& trkParam, const in mLinkPhiCuts[linkId] = o2::gpu::CAMath::Min(o2::gpu::CAMath::ASin(0.5f * x * oneOverR) + 2.f * mLinkMSAngles[linkId] + delta, o2::constants::math::PI * 0.5f); // some cleanup - deepVectorClear(mTracklets[linkId]); - deepVectorClear(mTrackletLabels[linkId]); - deepVectorClear(mTrackletsLookupTable[linkId]); + o2::itsmft::tracking::deepVectorClear(mTracklets[linkId]); + o2::itsmft::tracking::deepVectorClear(mTrackletLabels[linkId]); + o2::itsmft::tracking::deepVectorClear(mTrackletsLookupTable[linkId]); mTrackletsLookupTable[linkId].resize(mClusters[link.fromLayer].size() + 1, 0); } for (int cellId{0}; cellId < (int)mCells.size(); ++cellId) { - deepVectorClear(mCells[cellId]); - deepVectorClear(mCellsLookupTable[cellId]); - deepVectorClear(mCellsNeighbours[cellId]); - deepVectorClear(mCellsNeighboursTopology[cellId]); - deepVectorClear(mCellsNeighboursLUT[cellId]); - deepVectorClear(mCellLabels[cellId]); + o2::itsmft::tracking::deepVectorClear(mCells[cellId]); + o2::itsmft::tracking::deepVectorClear(mCellsLookupTable[cellId]); + o2::itsmft::tracking::deepVectorClear(mCellsNeighbours[cellId]); + o2::itsmft::tracking::deepVectorClear(mCellsNeighboursTopology[cellId]); + o2::itsmft::tracking::deepVectorClear(mCellsNeighboursLUT[cellId]); + o2::itsmft::tracking::deepVectorClear(mCellLabels[cellId]); } } @@ -446,13 +446,13 @@ void TimeFrame::computeTrackletsPerROFScans() } template -void TimeFrame::setMemoryPool(std::shared_ptr pool) +void TimeFrame::setMemoryPool(std::shared_ptr pool) { mMemoryPool = pool; - auto initVector = [&](bounded_vector& vec, bool useExternal = false) { + auto initVector = [&](o2::itsmft::tracking::bounded_vector& vec, bool useExternal = false) { std::pmr::memory_resource* mr = (useExternal) ? mExtMemoryPool.get() : mMemoryPool.get(); - deepVectorClear(vec, mr); + o2::itsmft::tracking::deepVectorClear(vec, mr); }; auto initContainers = [&](Container& container, bool useExternal = false) { @@ -498,51 +498,51 @@ template void TimeFrame::setFrameworkAllocator(ExternalAllocator* ext) { mExternalAllocator = ext; - mExtMemoryPool = std::make_shared(std::make_unique(mExternalAllocator)); + mExtMemoryPool = std::make_shared(std::make_unique(mExternalAllocator)); } template void TimeFrame::wipe() { resetTrackExtensionCounters(); - deepVectorClear(mTracks); - deepVectorClear(mTracklets); - deepVectorClear(mCells); - deepVectorClear(mCellsNeighbours); - deepVectorClear(mCellsNeighboursTopology); - deepVectorClear(mCellsLookupTable); - deepVectorClear(mPrimaryVertices); - deepVectorClear(mTrackletsLookupTable); - deepVectorClear(mClusterExternalIndices); - deepVectorClear(mNTrackletsPerCluster); - deepVectorClear(mNTrackletsPerClusterSum); - deepVectorClear(mNClustersPerROF); - deepVectorClear(mLinkPhiCuts); - deepVectorClear(mLinkMSAngles); - deepVectorClear(mPositionResolution); - deepVectorClear(mClusterSize); - deepVectorClear(mPValphaX); - deepVectorClear(mBogusClusters); - deepVectorClear(mTrackletsIndexROF); - deepVectorClear(mTrackletClusters); - deepVectorClear(mLines); + o2::itsmft::tracking::deepVectorClear(mTracks); + o2::itsmft::tracking::deepVectorClear(mTracklets); + o2::itsmft::tracking::deepVectorClear(mCells); + o2::itsmft::tracking::deepVectorClear(mCellsNeighbours); + o2::itsmft::tracking::deepVectorClear(mCellsNeighboursTopology); + o2::itsmft::tracking::deepVectorClear(mCellsLookupTable); + o2::itsmft::tracking::deepVectorClear(mPrimaryVertices); + o2::itsmft::tracking::deepVectorClear(mTrackletsLookupTable); + o2::itsmft::tracking::deepVectorClear(mClusterExternalIndices); + o2::itsmft::tracking::deepVectorClear(mNTrackletsPerCluster); + o2::itsmft::tracking::deepVectorClear(mNTrackletsPerClusterSum); + o2::itsmft::tracking::deepVectorClear(mNClustersPerROF); + o2::itsmft::tracking::deepVectorClear(mLinkPhiCuts); + o2::itsmft::tracking::deepVectorClear(mLinkMSAngles); + o2::itsmft::tracking::deepVectorClear(mPositionResolution); + o2::itsmft::tracking::deepVectorClear(mClusterSize); + o2::itsmft::tracking::deepVectorClear(mPValphaX); + o2::itsmft::tracking::deepVectorClear(mBogusClusters); + o2::itsmft::tracking::deepVectorClear(mTrackletsIndexROF); + o2::itsmft::tracking::deepVectorClear(mTrackletClusters); + o2::itsmft::tracking::deepVectorClear(mLines); // if we use the external host allocator then the assumption is that we // don't clear the memory ourself if (!hasFrameworkAllocator()) { - deepVectorClear(mClusters); - deepVectorClear(mUsedClusters); - deepVectorClear(mUnsortedClusters); - deepVectorClear(mIndexTables); - deepVectorClear(mTrackingFrameInfo); - deepVectorClear(mROFramesClusters); + o2::itsmft::tracking::deepVectorClear(mClusters); + o2::itsmft::tracking::deepVectorClear(mUsedClusters); + o2::itsmft::tracking::deepVectorClear(mUnsortedClusters); + o2::itsmft::tracking::deepVectorClear(mIndexTables); + o2::itsmft::tracking::deepVectorClear(mTrackingFrameInfo); + o2::itsmft::tracking::deepVectorClear(mROFramesClusters); } // only needed to clear if we have MC info if (hasMCinformation()) { - deepVectorClear(mLinesLabels); - deepVectorClear(mPrimaryVerticesLabels); - deepVectorClear(mTrackletLabels); - deepVectorClear(mCellLabels); - deepVectorClear(mTracksLabel); + o2::itsmft::tracking::deepVectorClear(mLinesLabels); + o2::itsmft::tracking::deepVectorClear(mPrimaryVerticesLabels); + o2::itsmft::tracking::deepVectorClear(mTrackletLabels); + o2::itsmft::tracking::deepVectorClear(mCellLabels); + o2::itsmft::tracking::deepVectorClear(mTracksLabel); } } diff --git a/Detectors/ITSMFT/ITS/tracking/src/Tracker.cxx b/Detectors/ITSMFT/ITS/tracking/src/Tracker.cxx index d8ff8442f908f..0da93595297f4 100644 --- a/Detectors/ITSMFT/ITS/tracking/src/Tracker.cxx +++ b/Detectors/ITSMFT/ITS/tracking/src/Tracker.cxx @@ -14,7 +14,7 @@ /// #include "ITStracking/Tracker.h" -#include "ITStracking/BoundedAllocator.h" +#include "ITSMFTTracking/BoundedAllocator.h" #include "ITStracking/Constants.h" #include "ITStracking/TrackerTraits.h" #include "ITStracking/TrackingConfigParam.h" @@ -105,7 +105,7 @@ float Tracker::clustersToTracks(const LogFunc& logger, const LogFunc& e } total += timeTracklets + timeCells + timeNeighbours + timeRoads; } - } catch (const BoundedMemoryResource::MemoryLimitExceeded& err) { + } catch (const o2::itsmft::tracking::BoundedMemoryResource::MemoryLimitExceeded& err) { handleException(err); return -1.f; } catch (const std::bad_alloc& err) { @@ -204,7 +204,7 @@ template void Tracker::sortTracks() { auto& trks = mTimeFrame->getTracks(); - bounded_vector indices(trks.size(), mMemoryPool.get()); + o2::itsmft::tracking::bounded_vector indices(trks.size(), mMemoryPool.get()); std::iota(indices.begin(), indices.end(), 0); std::sort(indices.begin(), indices.end(), [&trks](size_t i, size_t j) { // provide tracks sorted by lower-bound @@ -217,7 +217,7 @@ void Tracker::sortTracks() } return a.isBetter(b, 1e9); // then sort tracks in quality }); - bounded_vector sortedTrks(mMemoryPool.get()); + o2::itsmft::tracking::bounded_vector sortedTrks(mMemoryPool.get()); sortedTrks.reserve(trks.size()); for (size_t idx : indices) { sortedTrks.push_back(trks[idx]); @@ -225,7 +225,7 @@ void Tracker::sortTracks() trks.swap(sortedTrks); if (mTimeFrame->hasMCinformation()) { auto& trksLabels = mTimeFrame->getTracksLabel(); - bounded_vector sortedLabels(mMemoryPool.get()); + o2::itsmft::tracking::bounded_vector sortedLabels(mMemoryPool.get()); sortedLabels.reserve(trksLabels.size()); for (size_t idx : indices) { sortedLabels.push_back(trksLabels[idx]); diff --git a/Detectors/ITSMFT/ITS/tracking/src/TrackerTraits.cxx b/Detectors/ITSMFT/ITS/tracking/src/TrackerTraits.cxx index f086ae33a722f..5351a80bc8337 100644 --- a/Detectors/ITSMFT/ITS/tracking/src/TrackerTraits.cxx +++ b/Detectors/ITSMFT/ITS/tracking/src/TrackerTraits.cxx @@ -30,7 +30,7 @@ #include "DetectorsBase/Propagator.h" #include "GPUCommonMath.h" -#include "ITStracking/BoundedAllocator.h" +#include "ITSMFTTracking/BoundedAllocator.h" #include "ITStracking/Cell.h" #include "ITStracking/Constants.h" #include "ITStracking/IndexTableUtils.h" @@ -289,10 +289,10 @@ void TrackerTraits::computeLayerCells(const int iteration) mTaskArena->execute([&] { const int maxConcurrency = std::max(1, mTaskArena->max_concurrency()); auto clearTopology = [&](const int cellTopologyId) { - deepVectorClear(mTimeFrame->getCells()[cellTopologyId]); - deepVectorClear(mTimeFrame->getCellsLookupTable()[cellTopologyId]); + o2::itsmft::tracking::deepVectorClear(mTimeFrame->getCells()[cellTopologyId]); + o2::itsmft::tracking::deepVectorClear(mTimeFrame->getCellsLookupTable()[cellTopologyId]); if (createLabels) { - deepVectorClear(mTimeFrame->getCellsLabel(cellTopologyId)); + o2::itsmft::tracking::deepVectorClear(mTimeFrame->getCellsLabel(cellTopologyId)); } }; if (maxConcurrency > 1) { @@ -373,7 +373,7 @@ void TrackerTraits::computeLayerCells(const int iteration) } }; - bounded_vector activeTopologies(mMemoryPool.get()); + o2::itsmft::tracking::bounded_vector activeTopologies(mMemoryPool.get()); activeTopologies.reserve(topology.nCells); for (int cellTopologyId = 0; cellTopologyId < topology.nCells; ++cellTopologyId) { const auto& cellTopology = topology.getCell(cellTopologyId); @@ -438,8 +438,8 @@ void TrackerTraits::computeLayerCells(const int iteration) } auto clearTracklets = [&](const int linkId) { - deepVectorClear(mTimeFrame->getTracklets()[linkId]); - deepVectorClear(mTimeFrame->getTrackletsLabel(linkId)); + o2::itsmft::tracking::deepVectorClear(mTimeFrame->getTracklets()[linkId]); + o2::itsmft::tracking::deepVectorClear(mTimeFrame->getTrackletsLabel(linkId)); }; if (maxConcurrency > 1) { tbb::parallel_for(0, static_cast(topology.nLinks), clearTracklets); @@ -458,9 +458,9 @@ void TrackerTraits::findCellsNeighbours(const int iteration) mTaskArena->execute([&] { const int maxConcurrency = std::max(1, mTaskArena->max_concurrency()); auto clearNeighbours = [&](const int cellTopologyId) { - deepVectorClear(mTimeFrame->getCellsNeighbours()[cellTopologyId]); - deepVectorClear(mTimeFrame->getCellsNeighboursTopology()[cellTopologyId]); - deepVectorClear(mTimeFrame->getCellsNeighboursLUT()[cellTopologyId]); + o2::itsmft::tracking::deepVectorClear(mTimeFrame->getCellsNeighbours()[cellTopologyId]); + o2::itsmft::tracking::deepVectorClear(mTimeFrame->getCellsNeighboursTopology()[cellTopologyId]); + o2::itsmft::tracking::deepVectorClear(mTimeFrame->getCellsNeighboursLUT()[cellTopologyId]); }; if (maxConcurrency > 1) { tbb::parallel_for(0, static_cast(topology.nCells), clearNeighbours); @@ -476,7 +476,7 @@ void TrackerTraits::findCellsNeighbours(const int iteration) }; for (int outerLayer{0}; outerLayer < NLayers; ++outerLayer) { - bounded_vector activeTopologies(mMemoryPool.get()); + o2::itsmft::tracking::bounded_vector activeTopologies(mMemoryPool.get()); activeTopologies.reserve(topology.nCells); size_t sourceCellCount{0}; for (int cellTopologyId{0}; cellTopologyId < topology.nCells; ++cellTopologyId) { @@ -537,7 +537,7 @@ void TrackerTraits::findCellsNeighbours(const int iteration) } }; - bounded_vector waveNeighbours{mMemoryPool.get()}; + o2::itsmft::tracking::bounded_vector waveNeighbours{mMemoryPool.get()}; const auto key = CapacityEstimator::makeKey(SlabSite::Neighbours, iteration, 0, outerLayer); const auto scale = static_cast(sourceCellCount); if (maxConcurrency > 1) { @@ -572,7 +572,7 @@ void TrackerTraits::findCellsNeighbours(const int iteration) size_t begin; size_t end; }; - bounded_vector targetSpans{mMemoryPool.get()}; + o2::itsmft::tracking::bounded_vector targetSpans{mMemoryPool.get()}; targetSpans.reserve(topology.nCells); for (int targetTopologyId{0}; targetTopologyId < topology.nCells; ++targetTopologyId) { const auto first = std::lower_bound(waveNeighbours.begin(), waveNeighbours.end(), targetTopologyId, @@ -618,7 +618,7 @@ void TrackerTraits::findCellsNeighbours(const int iteration) // clean up LUTs auto clearCellLUT = [&](const int cellTopologyId) { - deepVectorClear(mTimeFrame->getCellsLookupTable()[cellTopologyId]); + o2::itsmft::tracking::deepVectorClear(mTimeFrame->getCellsLookupTable()[cellTopologyId]); }; if (maxConcurrency > 1) { tbb::parallel_for(0, static_cast(topology.nCells), clearCellLUT); @@ -632,7 +632,7 @@ void TrackerTraits::findCellsNeighbours(const int iteration) template template -void TrackerTraits::processNeighbours(int iteration, int defaultCellTopologyId, int iLevel, uint64_t capacityKey, const bounded_vector& currentSeeds, bounded_vector& updatedSeeds) +void TrackerTraits::processNeighbours(int iteration, int defaultCellTopologyId, int iLevel, uint64_t capacityKey, const o2::itsmft::tracking::bounded_vector& currentSeeds, o2::itsmft::tracking::bounded_vector& updatedSeeds) { constexpr bool IsInitial = std::is_same_v; static_assert(IsInitial || std::is_same_v); @@ -837,7 +837,7 @@ bool TrackerTraits::finaliseTrackSeed(const TrackSeedN& seed, template void TrackerTraits::findRoads(const int iteration) { - bounded_vector> firstClusters(mTrkParams[iteration].NLayers, bounded_vector(mMemoryPool.get()), mMemoryPool.get()); + o2::itsmft::tracking::bounded_vector> firstClusters(mTrkParams[iteration].NLayers, o2::itsmft::tracking::bounded_vector(mMemoryPool.get()), mMemoryPool.get()); firstClusters.resize(mTrkParams[iteration].NLayers); const auto propagator = o2::base::Propagator::Instance(); const TrackingFrameInfo* tfInfos[NLayers]{}; @@ -853,7 +853,7 @@ void TrackerTraits::findRoads(const int iteration) const track::TrackSeedSelector seedFilter{constants::MaxTrackSeedQ2Pt, mTrkParams[iteration].MaxChi2NDF, startLevel, mTrkParams[iteration].MaxHoles, mTrkParams[iteration].getMinSeedingClusters(), mTrkParams[iteration].HoleLayerMask, mTrkParams[iteration].getNonSeedingLayerMask()}; - bounded_vector trackSeeds(mMemoryPool.get()); + o2::itsmft::tracking::bounded_vector trackSeeds(mMemoryPool.get()); for (int startCellTopologyId{0}; startCellTopologyId < topology.nCells; ++startCellTopologyId) { const int startLayer = topology.getCell(startCellTopologyId).hitLayerMask.last(); if (!(mTrkParams[iteration].StartLayerMask.has(startLayer)) || @@ -862,7 +862,7 @@ void TrackerTraits::findRoads(const int iteration) continue; } - bounded_vector lastSeeds(mMemoryPool.get()), updatedSeeds(mMemoryPool.get()); + o2::itsmft::tracking::bounded_vector lastSeeds(mMemoryPool.get()), updatedSeeds(mMemoryPool.get()); auto roadKey = [&](int level) { return CapacityEstimator::makeKey(SlabSite::Roads, iteration, CapacityEstimator::makeVariant(startLevel, level), startCellTopologyId); @@ -873,11 +873,11 @@ void TrackerTraits::findRoads(const int iteration) int level = startLevel; while (level > 2 && !updatedSeeds.empty()) { lastSeeds.swap(updatedSeeds); - deepVectorClear(updatedSeeds); + o2::itsmft::tracking::deepVectorClear(updatedSeeds); --level; processNeighbours(iteration, constants::UnusedIndex, level, roadKey(level), lastSeeds, updatedSeeds); } - deepVectorClear(lastSeeds); + o2::itsmft::tracking::deepVectorClear(lastSeeds); if (!updatedSeeds.empty()) { trackSeeds.reserve(trackSeeds.size() + std::count_if(updatedSeeds.begin(), updatedSeeds.end(), [&](const auto& road) { return seedFilter(road.seed); })); @@ -912,7 +912,7 @@ void TrackerTraits::findRoads(const int iteration) std::max(1, mTrkParams[iteration].TrackFollowerMaxHypotheses), mTrkParams[iteration].TrackFollowerNSigmaCutPhi, mTrkParams[iteration].TrackFollowerNSigmaCutZ}; - bounded_vector tracks(mMemoryPool.get()); + o2::itsmft::tracking::bounded_vector tracks(mMemoryPool.get()); mTaskArena->execute([&] { const int nSeeds = static_cast(trackSeeds.size()); const int maxConcurrency = std::max(1, mTaskArena->max_concurrency()); @@ -920,7 +920,7 @@ void TrackerTraits::findRoads(const int iteration) // flush local track vector to global vector on reaching chunkSize std::mutex tracksMutex; - auto flushTracks = [&](bounded_vector& localTracks) { + auto flushTracks = [&](o2::itsmft::tracking::bounded_vector& localTracks) { if (localTracks.empty()) { return; } @@ -931,7 +931,7 @@ void TrackerTraits::findRoads(const int iteration) // each worker works on its own range tbb::parallel_for(tbb::blocked_range(0, nSeeds, chunkSize), [&](const auto& range) { - bounded_vector localTracks(mMemoryPool.get()); + o2::itsmft::tracking::bounded_vector localTracks(mMemoryPool.get()); localTracks.reserve(std::min(chunkSize, static_cast(range.size()))); auto& scratch = followerScratch.local(); for (int iSeed{range.begin()}; iSeed < range.end(); ++iSeed) { @@ -944,14 +944,14 @@ void TrackerTraits::findRoads(const int iteration) } } flushTracks(localTracks); // flush remaining - deepVectorClear(localTracks); + o2::itsmft::tracking::deepVectorClear(localTracks); }); - deepVectorClear(trackSeeds); + o2::itsmft::tracking::deepVectorClear(trackSeeds); }); // Sort tracks via indices to avoid moving TrackITSExt objects. - bounded_vector trackIndices(tracks.size(), mMemoryPool.get()); + o2::itsmft::tracking::bounded_vector trackIndices(tracks.size(), mMemoryPool.get()); std::iota(trackIndices.begin(), trackIndices.end(), 0); std::sort(trackIndices.begin(), trackIndices.end(), [&tracks](int a, int b) { return track::isBetter(tracks[a], tracks[b]); @@ -964,9 +964,9 @@ void TrackerTraits::findRoads(const int iteration) template void TrackerTraits::acceptTracks(int iteration, - bounded_vector& tracks, - const bounded_vector& trackIndices, - bounded_vector>& firstClusters) + o2::itsmft::tracking::bounded_vector& tracks, + const o2::itsmft::tracking::bounded_vector& trackIndices, + o2::itsmft::tracking::bounded_vector>& firstClusters) { auto& trks = mTimeFrame->getTracks(); trks.reserve(trks.size() + tracks.size()); @@ -1052,7 +1052,7 @@ void TrackerTraits::markTracks(int iteration) /// Now we have to set the shared cluster flag auto& tracks = mTimeFrame->getTracks(); - bounded_vector fclusSort(tracks.size(), mMemoryPool.get()); + o2::itsmft::tracking::bounded_vector fclusSort(tracks.size(), mMemoryPool.get()); std::iota(fclusSort.begin(), fclusSort.end(), 0); std::sort(fclusSort.begin(), fclusSort.end(), [&tracks](int a, int b) { return tracks[a].getFirstLayerClusterIndex() < tracks[b].getFirstLayerClusterIndex(); @@ -1114,16 +1114,16 @@ void TrackerTraits::setNThreads(int n, std::shared_ptr } template class TrackerTraits<7>; -template void TrackerTraits<7>::processNeighbours(int, int, int, uint64_t, const bounded_vector&, bounded_vector>&); -template void TrackerTraits<7>::processNeighbours>(int, int, int, uint64_t, const bounded_vector>&, bounded_vector>&); +template void TrackerTraits<7>::processNeighbours(int, int, int, uint64_t, const o2::itsmft::tracking::bounded_vector&, o2::itsmft::tracking::bounded_vector>&); +template void TrackerTraits<7>::processNeighbours>(int, int, int, uint64_t, const o2::itsmft::tracking::bounded_vector>&, o2::itsmft::tracking::bounded_vector>&); // ALICE3 upgrade #ifdef ENABLE_UPGRADES template class TrackerTraits<11>; -template void TrackerTraits<11>::processNeighbours(int, int, int, uint64_t, const bounded_vector&, bounded_vector>&); -template void TrackerTraits<11>::processNeighbours>(int, int, int, uint64_t, const bounded_vector>&, bounded_vector>&); +template void TrackerTraits<11>::processNeighbours(int, int, int, uint64_t, const o2::itsmft::tracking::bounded_vector&, o2::itsmft::tracking::bounded_vector>&); +template void TrackerTraits<11>::processNeighbours>(int, int, int, uint64_t, const o2::itsmft::tracking::bounded_vector>&, o2::itsmft::tracking::bounded_vector>&); template class TrackerTraits<13>; -template void TrackerTraits<13>::processNeighbours(int, int, int, uint64_t, const bounded_vector&, bounded_vector>&); -template void TrackerTraits<13>::processNeighbours>(int, int, int, uint64_t, const bounded_vector>&, bounded_vector>&); +template void TrackerTraits<13>::processNeighbours(int, int, int, uint64_t, const o2::itsmft::tracking::bounded_vector&, o2::itsmft::tracking::bounded_vector>&); +template void TrackerTraits<13>::processNeighbours>(int, int, int, uint64_t, const o2::itsmft::tracking::bounded_vector>&, o2::itsmft::tracking::bounded_vector>&); #endif } // namespace o2::its diff --git a/Detectors/ITSMFT/ITS/tracking/src/TrackingInterface.cxx b/Detectors/ITSMFT/ITS/tracking/src/TrackingInterface.cxx index 83a1086ec5263..2ee30d69f6bbf 100644 --- a/Detectors/ITSMFT/ITS/tracking/src/TrackingInterface.cxx +++ b/Detectors/ITSMFT/ITS/tracking/src/TrackingInterface.cxx @@ -31,7 +31,7 @@ #include "DataFormatsTRD/TriggerRecord.h" #include "CommonDataFormat/IRFrame.h" #include "DetectorsBase/GRPGeomHelper.h" -#include "ITStracking/BoundedAllocator.h" +#include "ITSMFTTracking/BoundedAllocator.h" #include "Framework/InputRecordWalker.h" #include "Framework/DataRefUtils.h" #include "Framework/DeviceSpec.h" @@ -487,7 +487,7 @@ void ITSTrackingInterface::setTraitsFromProvider(VertexerTraitsN* vertexerTraits // set common memory resource if (!mMemoryPool) { - mMemoryPool = std::make_shared(); + mMemoryPool = std::make_shared(); } vertexerTraits->setMemoryPool(mMemoryPool); trackerTraits->setMemoryPool(mMemoryPool); diff --git a/Detectors/ITSMFT/ITS/tracking/src/Vertexer.cxx b/Detectors/ITSMFT/ITS/tracking/src/Vertexer.cxx index d25d5efbec262..25f97b37b8e21 100644 --- a/Detectors/ITSMFT/ITS/tracking/src/Vertexer.cxx +++ b/Detectors/ITSMFT/ITS/tracking/src/Vertexer.cxx @@ -14,7 +14,7 @@ /// #include "ITStracking/Vertexer.h" -#include "ITStracking/BoundedAllocator.h" +#include "ITSMFTTracking/BoundedAllocator.h" #include "ITStracking/VertexerTraits.h" #include "ITStracking/TrackingConfigParam.h" @@ -86,7 +86,7 @@ float Vertexer::clustersToVertices(LogFunc logger) sortVertices(); } completed = true; - } catch (const BoundedMemoryResource::MemoryLimitExceeded& err) { + } catch (const o2::itsmft::tracking::BoundedMemoryResource::MemoryLimitExceeded& err) { handleException(err); } catch (const std::bad_alloc& err) { handleException(err); @@ -105,7 +105,7 @@ template void Vertexer::sortVertices() { auto& pvs = mTimeFrame->getPrimaryVertices(); - bounded_vector indices(pvs.size(), mMemoryPool.get()); + o2::itsmft::tracking::bounded_vector indices(pvs.size(), mMemoryPool.get()); std::iota(indices.begin(), indices.end(), 0); // provide vertices sorted by lower-bound std::sort(indices.begin(), indices.end(), [&pvs](size_t i, size_t j) { @@ -118,7 +118,7 @@ void Vertexer::sortVertices() } return pvs[i].getNContributors() > pvs[j].getNContributors(); }); - bounded_vector sortedVtx(mMemoryPool.get()); + o2::itsmft::tracking::bounded_vector sortedVtx(mMemoryPool.get()); sortedVtx.reserve(pvs.size()); for (const size_t idx : indices) { sortedVtx.push_back(pvs[idx]); @@ -126,7 +126,7 @@ void Vertexer::sortVertices() pvs.swap(sortedVtx); if (mTimeFrame->hasMCinformation()) { auto& mc = mTimeFrame->getPrimaryVerticesLabels(); - bounded_vector sortedMC(mMemoryPool.get()); + o2::itsmft::tracking::bounded_vector sortedMC(mMemoryPool.get()); for (const size_t idx : indices) { sortedMC.push_back(mc[idx]); } diff --git a/Detectors/ITSMFT/ITS/tracking/src/VertexerTraits.cxx b/Detectors/ITSMFT/ITS/tracking/src/VertexerTraits.cxx index 15c26bc0a952d..51cdce956f9d6 100644 --- a/Detectors/ITSMFT/ITS/tracking/src/VertexerTraits.cxx +++ b/Detectors/ITSMFT/ITS/tracking/src/VertexerTraits.cxx @@ -21,7 +21,7 @@ #include #include "ITStracking/VertexerTraits.h" -#include "ITStracking/BoundedAllocator.h" +#include "ITSMFTTracking/BoundedAllocator.h" #include "ITStracking/ClusterLines.h" #include "ITStracking/Definitions.h" #include "ITStracking/LineVertexerHelpers.h" @@ -44,7 +44,7 @@ void trackleterKernelHost( const gsl::span& usedClustersNextLayer, // 0 2 const int* indexTableNext, const float phiCut, - bounded_vector& tracklets, + o2::itsmft::tracking::bounded_vector& tracklets, gsl::span foundTracklets, const IndexTableUtils& utils, const TimeEstBC& timErr, @@ -106,12 +106,12 @@ void trackletSelectionKernelHost( gsl::span usedClusters2, // global layer 2 used clusters const gsl::span& tracklets01, const gsl::span& tracklets12, - bounded_vector& usedTracklets, + o2::itsmft::tracking::bounded_vector& usedTracklets, const gsl::span foundTracklets01, const gsl::span foundTracklets12, - bounded_vector& lines, + o2::itsmft::tracking::bounded_vector& lines, const gsl::span& trackletLabels, - bounded_vector& linesLabels, + o2::itsmft::tracking::bounded_vector& linesLabels, const int nLayer1Clusters, const float tanLambdaCut, const float phiCut, @@ -306,7 +306,7 @@ void VertexerTraits::computeTrackletMatching(const int iteration) continue; } mTimeFrame->getLines(pivotRofId).reserve(std::min(mTimeFrame->getFoundTracklets(pivotRofId, 0).size(), mTimeFrame->getNTrackletsCluster(pivotRofId, 0).size() * constants::MaxSelectedTrackletsPerCluster)); - bounded_vector usedTracklets(mTimeFrame->getFoundTracklets(pivotRofId, 0).size(), 0, mMemoryPool.get()); + o2::itsmft::tracking::bounded_vector usedTracklets(mTimeFrame->getFoundTracklets(pivotRofId, 0).size(), 0, mMemoryPool.get()); trackletSelectionKernelHost( mTimeFrame->getClusters()[0].data(), mTimeFrame->getClusters()[1].data(), @@ -331,7 +331,7 @@ void VertexerTraits::computeTrackletMatching(const int iteration) }); // from here on we do not use tracklets anymore, so let's free them - deepVectorClear(mTimeFrame->getTracklets()); + o2::itsmft::tracking::deepVectorClear(mTimeFrame->getTracklets()); } template @@ -366,7 +366,7 @@ void VertexerTraits::computeVertices(const int iteration) } auto& lines = mTimeFrame->getLines(rofId); auto clusters = line_vertexer::buildClusters(std::span{lines.data(), lines.size()}, settings); - deepVectorClear(lines); // not needed after + o2::itsmft::tracking::deepVectorClear(lines); // not needed after auto clusterBeamDistance2 = [&](const ClusterLines& cluster) { return (mTimeFrame->getBeamX() - cluster.getVertex()[0]) * (mTimeFrame->getBeamX() - cluster.getVertex()[0]) + (mTimeFrame->getBeamY() - cluster.getVertex()[1]) * (mTimeFrame->getBeamY() - cluster.getVertex()[1]); @@ -392,7 +392,7 @@ void VertexerTraits::computeVertices(const int iteration) for (const auto& cluster : clusters) { minClusterZ = std::min(minClusterZ, cluster.getVertex()[2]); } - bounded_vector deduplicated(mMemoryPool.get()); + o2::itsmft::tracking::bounded_vector deduplicated(mMemoryPool.get()); deduplicated.reserve(clusters.size()); std::unordered_map> keptByZBin; for (auto& candidate : clusters) { @@ -541,7 +541,7 @@ void VertexerTraits::computeVertices(const int iteration) rofVertices[rofId].push_back(vertex); if (mTimeFrame->hasMCinformation()) { auto& lineLabels = mTimeFrame->getLinesLabel(rofId); - bounded_vector labels(mMemoryPool.get()); + o2::itsmft::tracking::bounded_vector labels(mMemoryPool.get()); for (auto& index : cluster.getLabels()) { labels.push_back(lineLabels[index]); } diff --git a/Detectors/ITSMFT/ITS/tracking/test/CMakeLists.txt b/Detectors/ITSMFT/ITS/tracking/test/CMakeLists.txt index f8fce10b78602..39d8d3c0fec63 100644 --- a/Detectors/ITSMFT/ITS/tracking/test/CMakeLists.txt +++ b/Detectors/ITSMFT/ITS/tracking/test/CMakeLists.txt @@ -9,12 +9,6 @@ # granted to it by virtue of its status as an Intergovernmental Organization # or submit itself to any jurisdiction. -o2_add_test(boundedmemoryresource - SOURCES testBoundedMemoryResource.cxx - COMPONENT_NAME its-tracking - LABELS "its;tracking" - PUBLIC_LINK_LIBRARIES O2::ITStracking) - o2_add_test(roflookuptables SOURCES testROFLookupTables.cxx COMPONENT_NAME its-tracking diff --git a/Detectors/ITSMFT/common/tracking/CMakeLists.txt b/Detectors/ITSMFT/common/tracking/CMakeLists.txt index 78ad324ef0d9b..f2dde989c2504 100644 --- a/Detectors/ITSMFT/common/tracking/CMakeLists.txt +++ b/Detectors/ITSMFT/common/tracking/CMakeLists.txt @@ -10,9 +10,11 @@ # or submit itself to any jurisdiction. o2_add_library(ITSMFTTrackingAllocation - SOURCES src/CapacityEstimator.cxx - PUBLIC_LINK_LIBRARIES O2::GPUCommon - TBB::tbb - PRIVATE_LINK_LIBRARIES O2::FrameworkLogger) + SOURCES src/BoundedAllocator.cxx + src/CapacityEstimator.cxx + src/SlabBumpAllocator.cxx + PRIVATE_LINK_LIBRARIES O2::GPUCommon + O2::FrameworkLogger + TBB::tbb) add_subdirectory(test) diff --git a/Detectors/ITSMFT/common/tracking/include/ITSMFTTracking/BoundedAllocator.h b/Detectors/ITSMFT/common/tracking/include/ITSMFTTracking/BoundedAllocator.h index e37715324be86..4f5c634a6f50b 100644 --- a/Detectors/ITSMFT/common/tracking/include/ITSMFTTracking/BoundedAllocator.h +++ b/Detectors/ITSMFT/common/tracking/include/ITSMFTTracking/BoundedAllocator.h @@ -19,7 +19,6 @@ #include #include #include -#include #include #include #include @@ -28,11 +27,6 @@ #include #include -#if !defined(__HIPCC__) && !defined(__CUDACC__) -#include -#include "GPUCommonLogger.h" -#endif - namespace o2::itsmft::tracking { @@ -43,192 +37,40 @@ class BoundedMemoryResource final : public std::pmr::memory_resource class MemoryLimitExceeded final : public std::bad_alloc { public: - MemoryLimitExceeded(size_t attempted, size_t used, size_t max) - { - char buf[256]; - if (attempted != 0) { - (void)snprintf(buf, sizeof(buf), "Reached set memory limit (attempted: %zu, used: %zu, max: %zu)", attempted, used, max); - } else { - (void)snprintf(buf, sizeof(buf), "New set maximum below current used (newMax: %zu, used: %zu)", max, used); - } - mMsg = buf; - } - const char* what() const noexcept final { return mMsg.c_str(); } + MemoryLimitExceeded(size_t attempted, size_t used, size_t max); + const char* what() const noexcept final; private: std::string mMsg; }; - static std::pmr::memory_resource* cachingUpstream() - { - static std::pmr::synchronized_pool_resource pool{std::pmr::get_default_resource()}; - return &pool; - } + static std::pmr::memory_resource* cachingUpstream(); BoundedMemoryResource(size_t maxBytes = std::numeric_limits::max(), - std::pmr::memory_resource* upstream = nullptr) - : mMaxMemory(maxBytes), mUpstream(upstream != nullptr ? upstream : cachingUpstream()) {} + std::pmr::memory_resource* upstream = nullptr); BoundedMemoryResource(std::unique_ptr upstream, - size_t maxBytes = std::numeric_limits::max()) - : mMaxMemory(maxBytes), - mOwnedUpstream(std::move(upstream)), - mUpstream(mOwnedUpstream.get()) {} - - void* do_allocate(size_t bytes, size_t alignment) final - { - size_t new_used{0}; - size_t current_used{mUsedMemory.load(std::memory_order_relaxed)}; - do { - new_used = current_used + bytes; - if (new_used > mMaxMemory.load(std::memory_order_relaxed)) { - mCountThrow.fetch_add(1, std::memory_order_relaxed); - throw MemoryLimitExceeded(new_used, current_used, - mMaxMemory.load(std::memory_order_relaxed)); - } - } while (!mUsedMemory.compare_exchange_weak(current_used, new_used, - std::memory_order_acq_rel, - std::memory_order_relaxed)); - - void* p{nullptr}; - try { - p = mUpstream->allocate(bytes, alignment); - } catch (...) { - mUsedMemory.fetch_sub(bytes, std::memory_order_relaxed); -#ifdef BOUNDED_MR_STATS - mStats.upstreamFailures.fetch_add(1, std::memory_order_relaxed); -#endif - throw; - } - - size_t peak = mPeakUsedMemory.load(std::memory_order_relaxed); - while (new_used > peak && - !mPeakUsedMemory.compare_exchange_weak(peak, new_used, - std::memory_order_relaxed)) { - } - -#ifdef BOUNDED_MR_STATS - size_t statsPeak = mStats.peak.load(std::memory_order_relaxed); - while (new_used > statsPeak && - !mStats.peak.compare_exchange_weak(statsPeak, new_used, - std::memory_order_relaxed)) { - } - mStats.live.fetch_add(1, std::memory_order_relaxed); - mStats.nAlloc.fetch_add(1, std::memory_order_relaxed); - mStats.totalAlloc.fetch_add(bytes, std::memory_order_relaxed); - - size_t ma = mStats.maxAlign.load(std::memory_order_relaxed); - while (alignment > ma && !mStats.maxAlign.compare_exchange_weak(ma, alignment, std::memory_order_relaxed)) { - } -#endif - return p; - } + size_t maxBytes = std::numeric_limits::max()); - void do_deallocate(void* p, size_t bytes, size_t alignment) final - { - mUpstream->deallocate(p, bytes, alignment); - mUsedMemory.fetch_sub(bytes, std::memory_order_relaxed); -#ifdef BOUNDED_MR_STATS - mStats.live.fetch_sub(1, std::memory_order_relaxed); - mStats.nFree.fetch_add(1, std::memory_order_relaxed); - mStats.totalFreed.fetch_add(bytes, std::memory_order_relaxed); -#endif - } + [[nodiscard]] size_t getUsedMemory() const noexcept; + [[nodiscard]] size_t getMaxMemory() const noexcept; + [[nodiscard]] size_t getThrowCount() const noexcept; + [[nodiscard]] size_t getPeakMemory() const noexcept; + [[nodiscard]] size_t getPeakMemoryDelta() const noexcept; - bool do_is_equal(const std::pmr::memory_resource& other) const noexcept final - { - return this == &other; - } - - [[nodiscard]] size_t getUsedMemory() const noexcept - { - return mUsedMemory.load(std::memory_order_relaxed); - } - [[nodiscard]] size_t getMaxMemory() const noexcept - { - return mMaxMemory.load(std::memory_order_relaxed); - } - [[nodiscard]] size_t getThrowCount() const noexcept - { - return mCountThrow.load(std::memory_order_relaxed); - } - [[nodiscard]] size_t getPeakMemory() const noexcept - { - return mPeakUsedMemory.load(std::memory_order_relaxed); - } - [[nodiscard]] size_t getPeakMemoryDelta() const noexcept - { - const size_t peak = mPeakUsedMemory.load(std::memory_order_relaxed); - const size_t baseline = mPeakBaselineMemory.load(std::memory_order_relaxed); - return peak > baseline ? peak - baseline : 0; - } - - void resetPeakMemory() noexcept - { - const size_t used = mUsedMemory.load(std::memory_order_acquire); - mPeakBaselineMemory.store(used, std::memory_order_release); - mPeakUsedMemory.store(used, std::memory_order_release); - } - - void setMaxMemory(size_t max) - { - size_t current = mMaxMemory.load(std::memory_order_relaxed); - if (max == current) { - return; - } - for (;;) { - size_t used = mUsedMemory.load(std::memory_order_acquire); - if (used > max) { - mCountThrow.fetch_add(1, std::memory_order_relaxed); - throw MemoryLimitExceeded(0, used, max); - } - if (mMaxMemory.compare_exchange_weak(current, max, - std::memory_order_release, - std::memory_order_relaxed)) { - return; - } - if (current == max) { - return; - } - } - } + void resetPeakMemory() noexcept; + void setMaxMemory(size_t max); #if !defined(__HIPCC__) && !defined(__CUDACC__) - std::string asString() const - { - constexpr double gigabyte = 1024. * 1024. * 1024.; - const auto throw_ = mCountThrow.load(std::memory_order_relaxed); - const auto used = static_cast(mUsedMemory.load(std::memory_order_relaxed)); - const auto peak = static_cast(mPeakUsedMemory.load(std::memory_order_relaxed)); - const auto peakDelta = static_cast(getPeakMemoryDelta()); - const auto maxm = mMaxMemory.load(std::memory_order_relaxed); - std::string ret; - if (maxm == std::numeric_limits::max()) { - ret += std::format("maxthrow={} maxmem=unbounded used={:.2f} GB stagepeak={:.2f} GB stagealloc={:.2f} GB", throw_, used / gigabyte, peak / gigabyte, peakDelta / gigabyte); - } else { - ret += std::format("maxthrow={} maxmem={:.2f} GB used={:.2f} GB ({:.2f}%) stagepeak={:.2f} GB stagealloc={:.2f} GB", throw_, (double)maxm / gigabyte, used / gigabyte, 100.0 * used / (double)maxm, peak / gigabyte, peakDelta / gigabyte); - } -#ifdef BOUNDED_MR_STATS - ret += std::format(" peak={:.2f} GB live={} nAlloc={} nFree={} totalAlloc={:.2f} GB totalFreed={:.2f} GB maxAlign={} upstreamFail={}", - (float)mStats.peak.load(std::memory_order_relaxed) / gigabyte, - mStats.live.load(std::memory_order_relaxed), - mStats.nAlloc.load(std::memory_order_relaxed), - mStats.nFree.load(std::memory_order_relaxed), - (float)mStats.totalAlloc.load(std::memory_order_relaxed) / gigabyte, - (float)mStats.totalFreed.load(std::memory_order_relaxed) / gigabyte, - mStats.maxAlign.load(std::memory_order_relaxed), - mStats.upstreamFailures.load(std::memory_order_relaxed)); -#endif - return ret; - } - - void print() const - { - LOGP(info, "{}", asString()); - } + std::string asString() const; + void print() const; #endif private: + void* do_allocate(size_t bytes, size_t alignment) final; + void do_deallocate(void* p, size_t bytes, size_t alignment) final; + bool do_is_equal(const std::pmr::memory_resource& other) const noexcept final; + std::atomic mMaxMemory{std::numeric_limits::max()}; std::atomic mCountThrow{0}; std::atomic mUsedMemory{0}; diff --git a/Detectors/ITSMFT/common/tracking/include/ITSMFTTracking/SlabBumpAllocator.h b/Detectors/ITSMFT/common/tracking/include/ITSMFTTracking/SlabBumpAllocator.h index 71f7f93bff59a..a7abd16c5115d 100644 --- a/Detectors/ITSMFT/common/tracking/include/ITSMFTTracking/SlabBumpAllocator.h +++ b/Detectors/ITSMFT/common/tracking/include/ITSMFTTracking/SlabBumpAllocator.h @@ -21,21 +21,46 @@ #include #include #include +#include #include #include #include #include #include - -#include -#include -#include +#include #include "ITSMFTTracking/BoundedAllocator.h" namespace o2::itsmft::tracking { +namespace detail +{ + +class ThreadLocalStorage +{ + public: + using Factory = void* (*)(void*); + using Deleter = void (*)(void*); + + ThreadLocalStorage(void* context, Factory factory, Deleter deleter); + ~ThreadLocalStorage(); + ThreadLocalStorage(const ThreadLocalStorage&) = delete; + ThreadLocalStorage& operator=(const ThreadLocalStorage&) = delete; + + void* local(); + std::vector values() const; + + private: + struct Impl; + std::unique_ptr mImpl; +}; + +using ParallelForBody = void (*)(void*, size_t, size_t); +void parallelFor(size_t begin, size_t end, size_t grainSize, void* context, ParallelForBody body); + +} // namespace detail + class SlabBumpAllocator { public: @@ -45,44 +70,17 @@ class SlabBumpAllocator bool valid() const noexcept { return n != 0; } }; - SlabBumpAllocator(size_t capacity, size_t slab) noexcept - : mCapacity{capacity}, mSlab{slab ? slab : size_t{1}} {} + SlabBumpAllocator(size_t capacity, size_t slab) noexcept; - Range grab() noexcept - { - if (mExhausted.load(std::memory_order_relaxed)) { - return {}; - } - const size_t base = mCursor.fetch_add(mSlab, std::memory_order_relaxed); - if (base >= mCapacity) { - mExhausted.store(true, std::memory_order_relaxed); - return {}; - } - return {.base = base, .n = std::min(mSlab, mCapacity - base)}; - } + Range grab() noexcept; [[nodiscard]] size_t capacity() const noexcept { return mCapacity; } [[nodiscard]] size_t slab() const noexcept { return mSlab; } - [[nodiscard]] size_t watermark() const noexcept - { - return std::min(mCursor.load(std::memory_order_relaxed), mCapacity); - } + [[nodiscard]] size_t watermark() const noexcept; - static size_t suggestSlab(size_t capacity, int nThreads, size_t minSlab = 256, size_t maxSlab = 4096) noexcept - { - const size_t t = static_cast(std::max(1, nThreads)); - const size_t fairShare = std::max(1, capacity / t); - return std::clamp(std::max(1, capacity / (8 * t)), - std::min(minSlab, fairShare), - std::min(maxSlab, fairShare)); - } + static size_t suggestSlab(size_t capacity, int nThreads, size_t minSlab = 256, size_t maxSlab = 4096) noexcept; - void resetCapacity(size_t capacity) noexcept - { - assert(mCursor.load(std::memory_order_relaxed) == 0); - mCapacity = capacity; - mExhausted.store(capacity == 0, std::memory_order_relaxed); - } + void resetCapacity(size_t capacity) noexcept; private: std::atomic mCursor{0}; @@ -204,7 +202,7 @@ class SlabSink SlabSink& operator=(const SlabSink&) = delete; ~SlabSink() = default; - Handle& local() { return mHandles.local(); } + Handle& local() { return *static_cast(mHandles.local()); } [[nodiscard]] std::pmr::memory_resource* memoryResource() const noexcept { return mMR; } @@ -214,7 +212,8 @@ class SlabSink s.requested = mRequested; s.capacity = mAlloc.capacity(); s.memoryLimited = s.capacity < s.requested; - for (const auto& h : mHandles) { + for (const void* value : mHandles.values()) { + const auto& h = *static_cast(value); s.emitted += h.emitted(); s.spilled += h.spilled(); } @@ -231,12 +230,15 @@ class SlabSink bounded_vector runs{mMR}; size_t nRuns{0}; - for (auto& h : mHandles) { + const auto handles = mHandles.values(); + for (void* value : handles) { + auto& h = *static_cast(value); h.closeRun(); nRuns += h.mRuns.size(); } runs.reserve(nRuns); - for (const auto& h : mHandles) { + for (const void* value : handles) { + const auto& h = *static_cast(value); runs.insert(runs.end(), h.mRuns.begin(), h.mRuns.end()); } std::sort(runs.begin(), runs.end(), [](const Run& a, const Run& b) { return a.begin < b.begin; }); @@ -255,7 +257,8 @@ class SlabSink mStaging.resize(outputSize); dest.swap(mStaging); - for (auto& h : mHandles) { + for (void* value : handles) { + auto& h = *static_cast(value); dest.insert(dest.end(), std::make_move_iterator(h.mSpill.begin()), std::make_move_iterator(h.mSpill.end())); deepVectorClear(h.mSpill, mMR); } @@ -278,7 +281,9 @@ class SlabSink ++lut[p + 1]; } } - for (const auto& h : mHandles) { + const auto handles = mHandles.values(); + for (const void* value : handles) { + const auto& h = *static_cast(value); for (const int32_t p : h.mSpillProducer) { ++lut[p + 1]; } @@ -293,7 +298,8 @@ class SlabSink const auto total = static_cast(lut.back()); dest.resize(total); - for (auto& h : mHandles) { + for (void* value : handles) { + auto& h = *static_cast(value); for (size_t i = 0; i < h.mSpill.size(); ++i) { dest[cursor[h.mSpillProducer[i]]++] = std::move(h.mSpill[i]); } @@ -302,14 +308,19 @@ class SlabSink } deepVectorClear(cursor, mMR); - T* const staging = mStaging.data(); - tbb::parallel_for(tbb::blocked_range(0, wm, 4096), [&](const tbb::blocked_range& r) { - for (size_t s = r.begin(); s != r.end(); ++s) { - const int d = mProducerOf[s]; + struct MoveContext { + T* staging; + int32_t* producerOf; + bounded_vector* destination; + } context{mStaging.data(), mProducerOf.data(), &dest}; + detail::parallelFor(0, wm, 4096, &context, [](void* opaque, size_t begin, size_t end) { + auto& ctx = *static_cast(opaque); + for (size_t s = begin; s != end; ++s) { + const int d = ctx.producerOf[s]; if (d < 0) { continue; } - dest[d] = std::move(staging[s]); + (*ctx.destination)[d] = std::move(ctx.staging[s]); } }); @@ -324,7 +335,7 @@ class SlabSink mAlloc{granted, cfg.slabOverride ? cfg.slabOverride : SlabBumpAllocator::suggestSlab(granted, cfg.nThreads)}, mStaging{mr}, mProducerOf{mr}, - mHandles{[this]() { return Handle{this}; }} + mHandles{this, &SlabSink::createHandle, &SlabSink::deleteHandle} { try { mStaging.resize(granted); @@ -377,12 +388,22 @@ class SlabSink } } + static void* createHandle(void* sink) + { + return new Handle{static_cast(sink)}; + } + + static void deleteHandle(void* handle) + { + delete static_cast(handle); + } + std::pmr::memory_resource* mMR{nullptr}; size_t mRequested{0}; SlabBumpAllocator mAlloc; bounded_vector mStaging; bounded_vector mProducerOf; - tbb::enumerable_thread_specific mHandles; + detail::ThreadLocalStorage mHandles; bool mFinalized{false}; }; diff --git a/Detectors/ITSMFT/common/tracking/src/BoundedAllocator.cxx b/Detectors/ITSMFT/common/tracking/src/BoundedAllocator.cxx new file mode 100644 index 0000000000000..83d196eb18573 --- /dev/null +++ b/Detectors/ITSMFT/common/tracking/src/BoundedAllocator.cxx @@ -0,0 +1,200 @@ +// Copyright 2019-2026 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +#include "ITSMFTTracking/BoundedAllocator.h" + +#include +#include + +#include "GPUCommonLogger.h" + +namespace o2::itsmft::tracking +{ + +BoundedMemoryResource::MemoryLimitExceeded::MemoryLimitExceeded(size_t attempted, size_t used, size_t max) +{ + char buf[256]; + if (attempted != 0) { + (void)snprintf(buf, sizeof(buf), "Reached set memory limit (attempted: %zu, used: %zu, max: %zu)", attempted, used, max); + } else { + (void)snprintf(buf, sizeof(buf), "New set maximum below current used (newMax: %zu, used: %zu)", max, used); + } + mMsg = buf; +} + +const char* BoundedMemoryResource::MemoryLimitExceeded::what() const noexcept +{ + return mMsg.c_str(); +} + +std::pmr::memory_resource* BoundedMemoryResource::cachingUpstream() +{ + static std::pmr::synchronized_pool_resource pool{std::pmr::get_default_resource()}; + return &pool; +} + +BoundedMemoryResource::BoundedMemoryResource(size_t maxBytes, std::pmr::memory_resource* upstream) + : mMaxMemory(maxBytes), mUpstream(upstream != nullptr ? upstream : cachingUpstream()) +{ +} + +BoundedMemoryResource::BoundedMemoryResource(std::unique_ptr upstream, size_t maxBytes) + : mMaxMemory(maxBytes), mOwnedUpstream(std::move(upstream)), mUpstream(mOwnedUpstream.get()) +{ +} + +void* BoundedMemoryResource::do_allocate(size_t bytes, size_t alignment) +{ + size_t newUsed{0}; + size_t currentUsed{mUsedMemory.load(std::memory_order_relaxed)}; + do { + newUsed = currentUsed + bytes; + if (newUsed > mMaxMemory.load(std::memory_order_relaxed)) { + mCountThrow.fetch_add(1, std::memory_order_relaxed); + throw MemoryLimitExceeded(newUsed, currentUsed, mMaxMemory.load(std::memory_order_relaxed)); + } + } while (!mUsedMemory.compare_exchange_weak(currentUsed, newUsed, std::memory_order_acq_rel, std::memory_order_relaxed)); + + void* p{nullptr}; + try { + p = mUpstream->allocate(bytes, alignment); + } catch (...) { + mUsedMemory.fetch_sub(bytes, std::memory_order_relaxed); +#ifdef BOUNDED_MR_STATS + mStats.upstreamFailures.fetch_add(1, std::memory_order_relaxed); +#endif + throw; + } + + size_t peak = mPeakUsedMemory.load(std::memory_order_relaxed); + while (newUsed > peak && !mPeakUsedMemory.compare_exchange_weak(peak, newUsed, std::memory_order_relaxed)) { + } + +#ifdef BOUNDED_MR_STATS + size_t statsPeak = mStats.peak.load(std::memory_order_relaxed); + while (newUsed > statsPeak && !mStats.peak.compare_exchange_weak(statsPeak, newUsed, std::memory_order_relaxed)) { + } + mStats.live.fetch_add(1, std::memory_order_relaxed); + mStats.nAlloc.fetch_add(1, std::memory_order_relaxed); + mStats.totalAlloc.fetch_add(bytes, std::memory_order_relaxed); + + size_t maxAlignment = mStats.maxAlign.load(std::memory_order_relaxed); + while (alignment > maxAlignment && !mStats.maxAlign.compare_exchange_weak(maxAlignment, alignment, std::memory_order_relaxed)) { + } +#endif + return p; +} + +void BoundedMemoryResource::do_deallocate(void* p, size_t bytes, size_t alignment) +{ + mUpstream->deallocate(p, bytes, alignment); + mUsedMemory.fetch_sub(bytes, std::memory_order_relaxed); +#ifdef BOUNDED_MR_STATS + mStats.live.fetch_sub(1, std::memory_order_relaxed); + mStats.nFree.fetch_add(1, std::memory_order_relaxed); + mStats.totalFreed.fetch_add(bytes, std::memory_order_relaxed); +#endif +} + +bool BoundedMemoryResource::do_is_equal(const std::pmr::memory_resource& other) const noexcept +{ + return this == &other; +} + +size_t BoundedMemoryResource::getUsedMemory() const noexcept +{ + return mUsedMemory.load(std::memory_order_relaxed); +} + +size_t BoundedMemoryResource::getMaxMemory() const noexcept +{ + return mMaxMemory.load(std::memory_order_relaxed); +} + +size_t BoundedMemoryResource::getThrowCount() const noexcept +{ + return mCountThrow.load(std::memory_order_relaxed); +} + +size_t BoundedMemoryResource::getPeakMemory() const noexcept +{ + return mPeakUsedMemory.load(std::memory_order_relaxed); +} + +size_t BoundedMemoryResource::getPeakMemoryDelta() const noexcept +{ + const size_t peak = mPeakUsedMemory.load(std::memory_order_relaxed); + const size_t baseline = mPeakBaselineMemory.load(std::memory_order_relaxed); + return peak > baseline ? peak - baseline : 0; +} + +void BoundedMemoryResource::resetPeakMemory() noexcept +{ + const size_t used = mUsedMemory.load(std::memory_order_acquire); + mPeakBaselineMemory.store(used, std::memory_order_release); + mPeakUsedMemory.store(used, std::memory_order_release); +} + +void BoundedMemoryResource::setMaxMemory(size_t max) +{ + size_t current = mMaxMemory.load(std::memory_order_relaxed); + if (max == current) { + return; + } + for (;;) { + const size_t used = mUsedMemory.load(std::memory_order_acquire); + if (used > max) { + mCountThrow.fetch_add(1, std::memory_order_relaxed); + throw MemoryLimitExceeded(0, used, max); + } + if (mMaxMemory.compare_exchange_weak(current, max, std::memory_order_release, std::memory_order_relaxed)) { + return; + } + if (current == max) { + return; + } + } +} + +std::string BoundedMemoryResource::asString() const +{ + constexpr double gigabyte = 1024. * 1024. * 1024.; + const auto throwCount = mCountThrow.load(std::memory_order_relaxed); + const auto used = static_cast(mUsedMemory.load(std::memory_order_relaxed)); + const auto peak = static_cast(mPeakUsedMemory.load(std::memory_order_relaxed)); + const auto peakDelta = static_cast(getPeakMemoryDelta()); + const auto maxMemory = mMaxMemory.load(std::memory_order_relaxed); + std::string result; + if (maxMemory == std::numeric_limits::max()) { + result += std::format("maxthrow={} maxmem=unbounded used={:.2f} GB stagepeak={:.2f} GB stagealloc={:.2f} GB", throwCount, used / gigabyte, peak / gigabyte, peakDelta / gigabyte); + } else { + result += std::format("maxthrow={} maxmem={:.2f} GB used={:.2f} GB ({:.2f}%) stagepeak={:.2f} GB stagealloc={:.2f} GB", throwCount, static_cast(maxMemory) / gigabyte, used / gigabyte, 100.0 * used / static_cast(maxMemory), peak / gigabyte, peakDelta / gigabyte); + } +#ifdef BOUNDED_MR_STATS + result += std::format(" peak={:.2f} GB live={} nAlloc={} nFree={} totalAlloc={:.2f} GB totalFreed={:.2f} GB maxAlign={} upstreamFail={}", + static_cast(mStats.peak.load(std::memory_order_relaxed)) / gigabyte, + mStats.live.load(std::memory_order_relaxed), + mStats.nAlloc.load(std::memory_order_relaxed), + mStats.nFree.load(std::memory_order_relaxed), + static_cast(mStats.totalAlloc.load(std::memory_order_relaxed)) / gigabyte, + static_cast(mStats.totalFreed.load(std::memory_order_relaxed)) / gigabyte, + mStats.maxAlign.load(std::memory_order_relaxed), + mStats.upstreamFailures.load(std::memory_order_relaxed)); +#endif + return result; +} + +void BoundedMemoryResource::print() const +{ + LOGP(info, "{}", asString()); +} + +} // namespace o2::itsmft::tracking diff --git a/Detectors/ITSMFT/common/tracking/src/SlabBumpAllocator.cxx b/Detectors/ITSMFT/common/tracking/src/SlabBumpAllocator.cxx new file mode 100644 index 0000000000000..9856d6bc888fa --- /dev/null +++ b/Detectors/ITSMFT/common/tracking/src/SlabBumpAllocator.cxx @@ -0,0 +1,108 @@ +// Copyright 2019-2026 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +#include "ITSMFTTracking/SlabBumpAllocator.h" + +#include +#include +#include + +namespace o2::itsmft::tracking +{ + +namespace detail +{ + +struct ThreadLocalStorage::Impl { + Impl(void* context_, Factory factory_, Deleter deleter_) + : context{context_}, factory{factory_}, deleter{deleter_}, values{[this] { return factory(context); }} + { + } + + ~Impl() + { + for (void* value : values) { + deleter(value); + } + } + + void* context; + Factory factory; + Deleter deleter; + tbb::enumerable_thread_specific values; +}; + +ThreadLocalStorage::ThreadLocalStorage(void* context, Factory factory, Deleter deleter) + : mImpl{std::make_unique(context, factory, deleter)} +{ +} + +ThreadLocalStorage::~ThreadLocalStorage() = default; + +void* ThreadLocalStorage::local() +{ + return mImpl->values.local(); +} + +std::vector ThreadLocalStorage::values() const +{ + return {mImpl->values.begin(), mImpl->values.end()}; +} + +void parallelFor(size_t begin, size_t end, size_t grainSize, void* context, ParallelForBody body) +{ + tbb::parallel_for(tbb::blocked_range{begin, end, grainSize}, [context, body](const tbb::blocked_range& range) { + body(context, range.begin(), range.end()); + }); +} + +} // namespace detail + +SlabBumpAllocator::SlabBumpAllocator(size_t capacity, size_t slab) noexcept + : mCapacity{capacity}, mSlab{slab ? slab : size_t{1}} +{ +} + +SlabBumpAllocator::Range SlabBumpAllocator::grab() noexcept +{ + if (mExhausted.load(std::memory_order_relaxed)) { + return {}; + } + const size_t base = mCursor.fetch_add(mSlab, std::memory_order_relaxed); + if (base >= mCapacity) { + mExhausted.store(true, std::memory_order_relaxed); + return {}; + } + return {.base = base, .n = std::min(mSlab, mCapacity - base)}; +} + +size_t SlabBumpAllocator::watermark() const noexcept +{ + return std::min(mCursor.load(std::memory_order_relaxed), mCapacity); +} + +size_t SlabBumpAllocator::suggestSlab(size_t capacity, int nThreads, size_t minSlab, size_t maxSlab) noexcept +{ + const size_t threads = static_cast(std::max(1, nThreads)); + const size_t fairShare = std::max(1, capacity / threads); + return std::clamp(std::max(1, capacity / (8 * threads)), + std::min(minSlab, fairShare), + std::min(maxSlab, fairShare)); +} + +void SlabBumpAllocator::resetCapacity(size_t capacity) noexcept +{ + assert(mCursor.load(std::memory_order_relaxed) == 0); + mCapacity = capacity; + mExhausted.store(capacity == 0, std::memory_order_relaxed); +} + +} // namespace o2::itsmft::tracking diff --git a/Detectors/ITSMFT/common/tracking/test/CMakeLists.txt b/Detectors/ITSMFT/common/tracking/test/CMakeLists.txt index 164506401bd38..bbb78632546cd 100644 --- a/Detectors/ITSMFT/common/tracking/test/CMakeLists.txt +++ b/Detectors/ITSMFT/common/tracking/test/CMakeLists.txt @@ -13,4 +13,12 @@ o2_add_test(slabbumpallocator SOURCES testSlabBumpAllocator.cxx COMPONENT_NAME itsmft-tracking LABELS "itsmft;tracking" - PUBLIC_LINK_LIBRARIES O2::ITSMFTTrackingAllocation TBB::tbb) + PUBLIC_LINK_LIBRARIES O2::GPUCommon + O2::ITSMFTTrackingAllocation + TBB::tbb) + +o2_add_test(boundedmemoryresource + SOURCES testBoundedMemoryResource.cxx + COMPONENT_NAME itsmft-tracking + LABELS "itsmft;tracking" + PUBLIC_LINK_LIBRARIES O2::ITSMFTTrackingAllocation) diff --git a/Detectors/ITSMFT/ITS/tracking/test/testBoundedMemoryResource.cxx b/Detectors/ITSMFT/common/tracking/test/testBoundedMemoryResource.cxx similarity index 97% rename from Detectors/ITSMFT/ITS/tracking/test/testBoundedMemoryResource.cxx rename to Detectors/ITSMFT/common/tracking/test/testBoundedMemoryResource.cxx index aae28f5cbc36e..81157568dadc6 100644 --- a/Detectors/ITSMFT/ITS/tracking/test/testBoundedMemoryResource.cxx +++ b/Detectors/ITSMFT/common/tracking/test/testBoundedMemoryResource.cxx @@ -9,15 +9,15 @@ // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. -#define BOOST_TEST_MODULE Test Flags +#define BOOST_TEST_MODULE Test BoundedMemoryResource #define BOOST_TEST_MAIN #define BOOST_TEST_DYN_LINK #include #include -#include "ITStracking/BoundedAllocator.h" +#include "ITSMFTTracking/BoundedAllocator.h" -using namespace o2::its; +using namespace o2::itsmft::tracking; using Vec = bounded_vector; auto getRandomInt(int min = -100, int max = 100) { diff --git a/Detectors/ITSMFT/common/tracking/test/testSlabBumpAllocator.cxx b/Detectors/ITSMFT/common/tracking/test/testSlabBumpAllocator.cxx index 656d65eb77e45..1cbcd8feb427e 100644 --- a/Detectors/ITSMFT/common/tracking/test/testSlabBumpAllocator.cxx +++ b/Detectors/ITSMFT/common/tracking/test/testSlabBumpAllocator.cxx @@ -24,6 +24,7 @@ #include #include +#include #include #include "ITSMFTTracking/BoundedAllocator.h" diff --git a/Detectors/Upgrades/ALICE3/GlobalReconstruction/workflow/include/ALICE3GlobalReconstructionWorkflow/TrackerSpec.h b/Detectors/Upgrades/ALICE3/GlobalReconstruction/workflow/include/ALICE3GlobalReconstructionWorkflow/TrackerSpec.h index 7bd85c0862ae7..d3e70bcc9c5a1 100644 --- a/Detectors/Upgrades/ALICE3/GlobalReconstruction/workflow/include/ALICE3GlobalReconstructionWorkflow/TrackerSpec.h +++ b/Detectors/Upgrades/ALICE3/GlobalReconstruction/workflow/include/ALICE3GlobalReconstructionWorkflow/TrackerSpec.h @@ -21,7 +21,7 @@ #include -#include "ITStracking/BoundedAllocator.h" +#include "ITSMFTTracking/BoundedAllocator.h" #include "ITStracking/ExternalAllocator.h" #include "ITStracking/TrackingInterface.h" #include "GPUDataTypesConfig.h" @@ -69,7 +69,7 @@ class TrackerDPL : public framework::Task bool mIsMC{true}; gpu::gpudatatypes::DeviceType mDeviceType{gpu::gpudatatypes::DeviceType::CPU}; int mTrackingThreads{1}; - std::shared_ptr mMemoryPool; + std::shared_ptr mMemoryPool; std::shared_ptr mGPUAllocator; std::shared_ptr mTaskArena; std::vector mTrackingParams; diff --git a/Detectors/Upgrades/ALICE3/GlobalReconstruction/workflow/src/TrackerSpec.cxx b/Detectors/Upgrades/ALICE3/GlobalReconstruction/workflow/src/TrackerSpec.cxx index f588b07c598bf..41779053bd36e 100644 --- a/Detectors/Upgrades/ALICE3/GlobalReconstruction/workflow/src/TrackerSpec.cxx +++ b/Detectors/Upgrades/ALICE3/GlobalReconstruction/workflow/src/TrackerSpec.cxx @@ -294,7 +294,7 @@ std::vector TrackerDPL::createTrackingParamsFromCon void TrackerDPL::run(ProcessingContext& pc) { if (mMemoryPool.get() == nullptr) { - mMemoryPool = std::make_shared(); + mMemoryPool = std::make_shared(); } if (mTaskArena.get() == nullptr) { mTaskArena = std::make_shared(mTrackingThreads); diff --git a/Detectors/Upgrades/ITS3/reconstruction/src/IOUtils.cxx b/Detectors/Upgrades/ITS3/reconstruction/src/IOUtils.cxx index 1a7bb8adb1d58..7f153945b458b 100644 --- a/Detectors/Upgrades/ITS3/reconstruction/src/IOUtils.cxx +++ b/Detectors/Upgrades/ITS3/reconstruction/src/IOUtils.cxx @@ -11,7 +11,7 @@ #include "ITS3Reconstruction/IOUtils.h" #include "ITStracking/TimeFrame.h" -#include "ITStracking/BoundedAllocator.h" +#include "ITSMFTTracking/BoundedAllocator.h" #include "DataFormatsITSMFT/CompCluster.h" #include "DataFormatsITSMFT/ROFRecord.h" #include "ITS3Reconstruction/TopologyDictionary.h" @@ -77,7 +77,7 @@ int loadROFrameDataITS3(its::TimeFrame<7>* tf, LOGP(fatal, "Received inconsistent number of rofs on layer:{} expected:{} received:{}", layer, timing.mNROFsTF, rofs.size()); } - its::bounded_vector clusterSizeVec(clusters.size(), 0, tf->getMemoryPool().get()); + itsmft::tracking::bounded_vector clusterSizeVec(clusters.size(), 0, tf->getMemoryPool().get()); for (size_t iRof{0}; iRof < rofs.size(); ++iRof) { const auto& rof = rofs[iRof]; From cf52edd0980472c97f6c9e5285c707b69e4093a7 Mon Sep 17 00:00:00 2001 From: Maximiliano Puccio Date: Thu, 27 Aug 2026 22:10:29 +0200 Subject: [PATCH 3/7] ITSMFT: reuse capacity estimator statistics storage --- .../common/tracking/src/CapacityEstimator.cxx | 37 +++++++------------ 1 file changed, 14 insertions(+), 23 deletions(-) diff --git a/Detectors/ITSMFT/common/tracking/src/CapacityEstimator.cxx b/Detectors/ITSMFT/common/tracking/src/CapacityEstimator.cxx index 6cfcd662e3db9..ef3de7601f2f0 100644 --- a/Detectors/ITSMFT/common/tracking/src/CapacityEstimator.cxx +++ b/Detectors/ITSMFT/common/tracking/src/CapacityEstimator.cxx @@ -31,13 +31,8 @@ struct CapacityEstimator::Impl { float ratio{0.f}; float margin{0.f}; size_t maxEmitted{0}; - uint32_t nSamples{0}; uint32_t nLowStreak{0}; - uint32_t nOverflows{0}; - size_t requested{0}; - size_t granted{0}; - size_t emitted{0}; - size_t spilled{0}; + Statistics statistics{}; }; struct UndoRecord { @@ -75,19 +70,20 @@ struct CapacityEstimator::Impl { // transaction without observing a partial update. checkpointBeforeUpdate(key); auto& e = entries[key]; - e.requested += requested; - e.granted += granted; - e.emitted += emitted; - e.spilled += spilled; + auto& statistics = e.statistics; + statistics.requested += requested; + statistics.granted += granted; + statistics.emitted += emitted; + statistics.spilled += spilled; - const bool firstSample = e.nSamples == 0; + const bool firstSample = statistics.samples == 0; if (firstSample) { e.margin = cfg.marginInit; } const auto sample = static_cast(double(emitted) / scale); e.ratio = firstSample ? sample : (cfg.alpha * sample) + ((1.f - cfg.alpha) * e.ratio); e.maxEmitted = std::max(e.maxEmitted, emitted); - ++e.nSamples; + ++statistics.samples; if (memoryLimited) { e.nLowStreak = 0; @@ -95,7 +91,7 @@ struct CapacityEstimator::Impl { return; } if (overflowed) { - ++e.nOverflows; + ++statistics.overflowEvents; e.nLowStreak = 0; if (!firstSample) { const float shortfall = granted ? static_cast(double(emitted) / double(granted)) : cfg.marginUp; @@ -172,7 +168,7 @@ size_t CapacityEstimator::capacity(uint64_t key, double scale) const } std::lock_guard lock{mImpl->mutex}; const auto it = mImpl->entries.find(key); - if (it == mImpl->entries.end() || it->second.nSamples == 0) { + if (it == mImpl->entries.end() || it->second.statistics.samples == 0) { return mImpl->cfg.floorSlots; } const auto& e = it->second; @@ -213,7 +209,7 @@ double CapacityEstimator::expected(uint64_t key, double scale) const } std::lock_guard lock{mImpl->mutex}; const auto it = mImpl->entries.find(key); - if (it == mImpl->entries.end() || it->second.nSamples == 0) { + if (it == mImpl->entries.end() || it->second.statistics.samples == 0) { return 0.; } const double raw = double(it->second.ratio) * scale; @@ -227,13 +223,7 @@ CapacityEstimator::Statistics CapacityEstimator::statistics(uint64_t key) const if (it == mImpl->entries.end()) { return {}; } - const auto& e = it->second; - return {.requested = e.requested, - .granted = e.granted, - .emitted = e.emitted, - .spilled = e.spilled, - .samples = e.nSamples, - .overflowEvents = e.nOverflows}; + return it->second.statistics; } void CapacityEstimator::update(uint64_t key, double scale, size_t emitted, size_t capacityUsed, bool overflowed, bool memoryLimited) @@ -277,8 +267,9 @@ void CapacityEstimator::print() const LOGP(info, "Printing CapacityEstimators:"); for (const auto key : keys) { const auto& value = mImpl->entries.at(key); + const auto& statistics = value.statistics; const auto decoded = decodeKey(key); - LOGP(info, "\tSite:{} | iter:{} | var:({},{}) | slot:{} | ratio:{} | margin:{} | maxEmitted:{} | samples:{} | low:{} | requested:{} | granted:{} | emitted:{} | spilled:{} | overflows:{}", SlabSiteNames[decoded.site], decoded.iteration, getVariantHigh(decoded.variant), getVariantLow(decoded.variant), decoded.slot, value.ratio, value.margin, value.maxEmitted, value.nSamples, value.nLowStreak, value.requested, value.granted, value.emitted, value.spilled, value.nOverflows); + LOGP(info, "\tSite:{} | iter:{} | var:({},{}) | slot:{} | ratio:{} | margin:{} | maxEmitted:{} | samples:{} | low:{} | requested:{} | granted:{} | emitted:{} | spilled:{} | overflows:{}", SlabSiteNames[decoded.site], decoded.iteration, getVariantHigh(decoded.variant), getVariantLow(decoded.variant), decoded.slot, value.ratio, value.margin, value.maxEmitted, statistics.samples, value.nLowStreak, statistics.requested, statistics.granted, statistics.emitted, statistics.spilled, statistics.overflowEvents); } } From 591c678520740f9bbf10d79983f35fa10c04431c Mon Sep 17 00:00:00 2001 From: Maximiliano Puccio Date: Thu, 27 Aug 2026 22:39:59 +0200 Subject: [PATCH 4/7] ITSMFT: establish common tracking dependency direction --- Detectors/ITSMFT/ITS/tracking/CMakeLists.txt | 5 +---- Detectors/ITSMFT/common/tracking/CMakeLists.txt | 2 +- Detectors/ITSMFT/common/tracking/test/CMakeLists.txt | 4 ++-- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/Detectors/ITSMFT/ITS/tracking/CMakeLists.txt b/Detectors/ITSMFT/ITS/tracking/CMakeLists.txt index 07d40764d0498..a2df66a39f448 100644 --- a/Detectors/ITSMFT/ITS/tracking/CMakeLists.txt +++ b/Detectors/ITSMFT/ITS/tracking/CMakeLists.txt @@ -35,14 +35,11 @@ o2_add_library(ITStracking O2::ITSReconstruction O2::ITSMFTReconstruction O2::DataFormatsITS - O2::ITSMFTTrackingAllocation + O2::ITSMFTTracking PRIVATE_LINK_LIBRARIES O2::Steer TBB::tbb) -target_include_directories(${targetName} PUBLIC - $ - $) # target_compile_options(${targetName} PRIVATE -O0 -g -fPIC -fno-omit-frame-pointer) o2_add_library(ITSTrackingInterface diff --git a/Detectors/ITSMFT/common/tracking/CMakeLists.txt b/Detectors/ITSMFT/common/tracking/CMakeLists.txt index f2dde989c2504..a1ed4a37c83fc 100644 --- a/Detectors/ITSMFT/common/tracking/CMakeLists.txt +++ b/Detectors/ITSMFT/common/tracking/CMakeLists.txt @@ -9,7 +9,7 @@ # granted to it by virtue of its status as an Intergovernmental Organization # or submit itself to any jurisdiction. -o2_add_library(ITSMFTTrackingAllocation +o2_add_library(ITSMFTTracking SOURCES src/BoundedAllocator.cxx src/CapacityEstimator.cxx src/SlabBumpAllocator.cxx diff --git a/Detectors/ITSMFT/common/tracking/test/CMakeLists.txt b/Detectors/ITSMFT/common/tracking/test/CMakeLists.txt index bbb78632546cd..102c00841e41f 100644 --- a/Detectors/ITSMFT/common/tracking/test/CMakeLists.txt +++ b/Detectors/ITSMFT/common/tracking/test/CMakeLists.txt @@ -14,11 +14,11 @@ o2_add_test(slabbumpallocator COMPONENT_NAME itsmft-tracking LABELS "itsmft;tracking" PUBLIC_LINK_LIBRARIES O2::GPUCommon - O2::ITSMFTTrackingAllocation + O2::ITSMFTTracking TBB::tbb) o2_add_test(boundedmemoryresource SOURCES testBoundedMemoryResource.cxx COMPONENT_NAME itsmft-tracking LABELS "itsmft;tracking" - PUBLIC_LINK_LIBRARIES O2::ITSMFTTrackingAllocation) + PUBLIC_LINK_LIBRARIES O2::ITSMFTTracking) From 9f24e0b63709a03b979070f902f2b7955fbee163 Mon Sep 17 00:00:00 2001 From: Maximiliano Puccio Date: Thu, 27 Aug 2026 22:59:27 +0200 Subject: [PATCH 5/7] ITSMFT: move shared tracking support out of ITS --- Detectors/ITSMFT/ITS/tracking/CMakeLists.txt | 2 -- .../GPU/ITStrackingGPU/TrackingKernels.h | 2 +- .../ITS/tracking/GPU/ITStrackingGPU/Utils.h | 2 +- .../ITS/tracking/GPU/cuda/TimeFrameGPU.cu | 2 +- .../ITS/tracking/GPU/cuda/TrackingKernels.cu | 4 ++-- .../ITS/tracking/include/ITStracking/Cell.h | 2 +- .../tracking/include/ITStracking/Cluster.h | 4 ++-- .../include/ITStracking/ClusterLines.h | 2 +- .../include/ITStracking/Configuration.h | 2 +- .../include/ITStracking/FastMultEst.h | 2 +- .../include/ITStracking/IndexTableUtils.h | 2 +- .../tracking/include/ITStracking/LayerMask.h | 2 +- .../tracking/include/ITStracking/TimeFrame.h | 2 +- .../ITStracking/TrackExtensionHypothesis.h | 2 +- .../include/ITStracking/TrackFollower.h | 6 ++--- .../include/ITStracking/TrackHelpers.h | 4 ++-- .../include/ITStracking/TrackITSInternal.h | 2 +- .../tracking/include/ITStracking/Tracklet.h | 2 +- .../tracking/include/ITStracking/Vertexer.h | 2 +- .../include/ITStracking/VertexerTraits.h | 2 +- .../ITSMFT/ITS/tracking/src/Configuration.cxx | 4 ++-- Detectors/ITSMFT/ITS/tracking/src/IOUtils.cxx | 2 +- .../ITS/tracking/src/LineVertexerHelpers.cxx | 4 ++-- .../ITSMFT/ITS/tracking/src/TimeFrame.cxx | 2 +- Detectors/ITSMFT/ITS/tracking/src/Tracker.cxx | 4 ++-- .../ITSMFT/ITS/tracking/src/TrackerTraits.cxx | 4 ++-- .../ITS/tracking/src/TrackingInterface.cxx | 4 ++-- .../ITSMFT/ITS/tracking/src/TrackingLinkDef.h | 6 ----- .../ITSMFT/ITS/tracking/src/Vertexer.cxx | 2 +- .../ITS/tracking/test/testROFLookupTables.cxx | 2 +- .../ITSMFT/ITS/workflow/src/RecoWorkflow.cxx | 2 +- .../ITSMFT/ITS/workflow/src/TrackerSpec.cxx | 2 +- .../src/its-threshold-calib-workflow.cxx | 2 +- .../ITSMFT/common/tracking/CMakeLists.txt | 15 ++++++++++-- .../include/ITSMFTTracking}/Constants.h | 0 .../ITSMFTTracking/ITSTrackingConfigParam.h} | 2 +- .../include/ITSMFTTracking}/MathUtils.h | 2 +- .../include/ITSMFTTracking}/ROFLookupTables.h | 0 .../tracking/src/ITSTrackingConfigParam.cxx} | 2 +- .../common/tracking/src/ITSTrackingLinkDef.h | 24 +++++++++++++++++++ .../TimeFrameMixin.h | 2 +- .../workflow/src/TrackerSpec.cxx | 2 +- .../ITS3/alignment/src/AlignmentSpec.cxx | 2 +- .../ITS3/reconstruction/src/IOUtils.cxx | 2 +- .../ITS3/workflow/src/TrackerSpec.cxx | 2 +- .../ITS3/workflow/src/its3-reco-workflow.cxx | 2 +- GPU/Workflow/src/GPUWorkflowITS.cxx | 2 +- GPU/Workflow/src/GPUWorkflowSpec.cxx | 2 +- 48 files changed, 89 insertions(+), 62 deletions(-) rename Detectors/ITSMFT/{ITS/tracking/include/ITStracking => common/tracking/include/ITSMFTTracking}/Constants.h (100%) rename Detectors/ITSMFT/{ITS/tracking/include/ITStracking/TrackingConfigParam.h => common/tracking/include/ITSMFTTracking/ITSTrackingConfigParam.h} (99%) rename Detectors/ITSMFT/{ITS/tracking/include/ITStracking => common/tracking/include/ITSMFTTracking}/MathUtils.h (99%) rename Detectors/ITSMFT/{ITS/tracking/include/ITStracking => common/tracking/include/ITSMFTTracking}/ROFLookupTables.h (100%) rename Detectors/ITSMFT/{ITS/tracking/src/TrackingConfigParam.cxx => common/tracking/src/ITSTrackingConfigParam.cxx} (92%) create mode 100644 Detectors/ITSMFT/common/tracking/src/ITSTrackingLinkDef.h diff --git a/Detectors/ITSMFT/ITS/tracking/CMakeLists.txt b/Detectors/ITSMFT/ITS/tracking/CMakeLists.txt index a2df66a39f448..6560f25966a63 100644 --- a/Detectors/ITSMFT/ITS/tracking/CMakeLists.txt +++ b/Detectors/ITSMFT/ITS/tracking/CMakeLists.txt @@ -21,7 +21,6 @@ o2_add_library(ITStracking src/IOUtils.cxx src/Tracker.cxx src/TrackerTraits.cxx - src/TrackingConfigParam.cxx src/Vertexer.cxx src/VertexerTraits.cxx PUBLIC_LINK_LIBRARIES @@ -56,7 +55,6 @@ o2_target_root_dictionary(ITStracking include/ITStracking/Cluster.h include/ITStracking/Definitions.h include/ITStracking/FastMultEstConfig.h - include/ITStracking/TrackingConfigParam.h LINKDEF src/TrackingLinkDef.h) if(CUDA_ENABLED OR HIP_ENABLED) diff --git a/Detectors/ITSMFT/ITS/tracking/GPU/ITStrackingGPU/TrackingKernels.h b/Detectors/ITSMFT/ITS/tracking/GPU/ITStrackingGPU/TrackingKernels.h index 33fac990ee086..ad83d9c9a06e9 100644 --- a/Detectors/ITSMFT/ITS/tracking/GPU/ITStrackingGPU/TrackingKernels.h +++ b/Detectors/ITSMFT/ITS/tracking/GPU/ITStrackingGPU/TrackingKernels.h @@ -18,7 +18,7 @@ #include "ITSMFTTracking/BoundedAllocator.h" #include "ITSMFTTracking/CapacityEstimator.h" -#include "ITStracking/ROFLookupTables.h" +#include "ITSMFTTracking/ROFLookupTables.h" #include "ITStracking/TrackingTopology.h" #include "ITStracking/TrackExtensionHypothesis.h" #include "ITStrackingGPU/Utils.h" diff --git a/Detectors/ITSMFT/ITS/tracking/GPU/ITStrackingGPU/Utils.h b/Detectors/ITSMFT/ITS/tracking/GPU/ITStrackingGPU/Utils.h index e6909b28a687a..8f5baecd80aa1 100644 --- a/Detectors/ITSMFT/ITS/tracking/GPU/ITStrackingGPU/Utils.h +++ b/Detectors/ITSMFT/ITS/tracking/GPU/ITStrackingGPU/Utils.h @@ -20,7 +20,7 @@ #include #include -#include "ITStracking/MathUtils.h" +#include "ITSMFTTracking/MathUtils.h" #include "ITStracking/ExternalAllocator.h" #include "GPUCommonDef.h" diff --git a/Detectors/ITSMFT/ITS/tracking/GPU/cuda/TimeFrameGPU.cu b/Detectors/ITSMFT/ITS/tracking/GPU/cuda/TimeFrameGPU.cu index 38661c8d136df..25ec5b9c04928 100644 --- a/Detectors/ITSMFT/ITS/tracking/GPU/cuda/TimeFrameGPU.cu +++ b/Detectors/ITSMFT/ITS/tracking/GPU/cuda/TimeFrameGPU.cu @@ -20,7 +20,7 @@ #include #include "ITStrackingGPU/TimeFrameGPU.h" -#include "ITStracking/Constants.h" +#include "ITSMFTTracking/Constants.h" #include "ITSMFTTracking/BoundedAllocator.h" #include "ITStrackingGPU/Utils.h" diff --git a/Detectors/ITSMFT/ITS/tracking/GPU/cuda/TrackingKernels.cu b/Detectors/ITSMFT/ITS/tracking/GPU/cuda/TrackingKernels.cu index 4d6deb124451a..6fa466c6ac560 100644 --- a/Detectors/ITSMFT/ITS/tracking/GPU/cuda/TrackingKernels.cu +++ b/Detectors/ITSMFT/ITS/tracking/GPU/cuda/TrackingKernels.cu @@ -29,11 +29,11 @@ #include #include "DataFormatsITS/TrackITS.h" -#include "ITStracking/Constants.h" +#include "ITSMFTTracking/Constants.h" #include "ITStracking/Definitions.h" #include "ITStracking/IndexTableUtils.h" #include "ITStrackingGPU/LaunchGeometry.h" -#include "ITStracking/MathUtils.h" +#include "ITSMFTTracking/MathUtils.h" #include "ITStracking/ExternalAllocator.h" #include "ITStracking/Tracklet.h" #include "ITStracking/Cluster.h" diff --git a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/Cell.h b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/Cell.h index 4706977d08ba6..ad3b11d3c1eec 100644 --- a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/Cell.h +++ b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/Cell.h @@ -18,7 +18,7 @@ #include -#include "ITStracking/Constants.h" +#include "ITSMFTTracking/Constants.h" #include "ITStracking/LayerMask.h" #include "DataFormatsITS/TimeEstBC.h" #include "ReconstructionDataFormats/Track.h" diff --git a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/Cluster.h b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/Cluster.h index fb5f1a13ef3d2..7187c5f50cb4e 100644 --- a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/Cluster.h +++ b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/Cluster.h @@ -17,8 +17,8 @@ #define TRACKINGITSU_INCLUDE_CACLUSTER_H_ #include -#include "ITStracking/Constants.h" -#include "ITStracking/MathUtils.h" +#include "ITSMFTTracking/Constants.h" +#include "ITSMFTTracking/MathUtils.h" #include "GPUCommonRtypes.h" #include "GPUCommonDef.h" diff --git a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/ClusterLines.h b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/ClusterLines.h index bcb8a98a62cab..8110b2bdfb384 100644 --- a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/ClusterLines.h +++ b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/ClusterLines.h @@ -18,7 +18,7 @@ #include #include #include "ITStracking/Cluster.h" -#include "ITStracking/Constants.h" +#include "ITSMFTTracking/Constants.h" #include "ITStracking/Tracklet.h" #include "GPUCommonRtypes.h" diff --git a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/Configuration.h b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/Configuration.h index 8e1c68fc31c6c..20a497c3b8a58 100644 --- a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/Configuration.h +++ b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/Configuration.h @@ -25,7 +25,7 @@ #include "CommonUtils/EnumFlags.h" #include "DetectorsBase/Propagator.h" -#include "ITStracking/Constants.h" +#include "ITSMFTTracking/Constants.h" #include "ITStracking/LayerMask.h" namespace o2::its diff --git a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/FastMultEst.h b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/FastMultEst.h index f94c7c2034b46..d283d6eed2d45 100644 --- a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/FastMultEst.h +++ b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/FastMultEst.h @@ -22,7 +22,7 @@ #include "DataFormatsITSMFT/CompCluster.h" #include "DataFormatsITSMFT/PhysTrigger.h" #include "ITStracking/FastMultEstConfig.h" -#include "ITStracking/ROFLookupTables.h" +#include "ITSMFTTracking/ROFLookupTables.h" #include #include diff --git a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/IndexTableUtils.h b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/IndexTableUtils.h index a7b44e91d9093..427abd9b876a5 100644 --- a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/IndexTableUtils.h +++ b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/IndexTableUtils.h @@ -19,7 +19,7 @@ #include #include "ITStracking/Cluster.h" -#include "ITStracking/MathUtils.h" +#include "ITSMFTTracking/MathUtils.h" #include "CommonConstants/MathConstants.h" #include "GPUCommonMath.h" #include "GPUCommonDef.h" diff --git a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/LayerMask.h b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/LayerMask.h index 9fe9894b3b457..1a3854f4a6d2b 100644 --- a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/LayerMask.h +++ b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/LayerMask.h @@ -22,7 +22,7 @@ #include "GPUCommonDef.h" #include "GPUCommonMath.h" -#include "ITStracking/Constants.h" +#include "ITSMFTTracking/Constants.h" namespace o2::its { diff --git a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TimeFrame.h b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TimeFrame.h index ea11b6b5dd95a..416fa53fea1f6 100644 --- a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TimeFrame.h +++ b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TimeFrame.h @@ -32,7 +32,7 @@ #include "ITStracking/IndexTableUtils.h" #include "ITStracking/ExternalAllocator.h" #include "ITSMFTTracking/BoundedAllocator.h" -#include "ITStracking/ROFLookupTables.h" +#include "ITSMFTTracking/ROFLookupTables.h" #include "ITStracking/TrackingTopology.h" #include "SimulationDataFormat/MCCompLabel.h" #include "SimulationDataFormat/MCTruthContainer.h" diff --git a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TrackExtensionHypothesis.h b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TrackExtensionHypothesis.h index afa24c34afd4f..a3ebd47ff54aa 100644 --- a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TrackExtensionHypothesis.h +++ b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TrackExtensionHypothesis.h @@ -16,7 +16,7 @@ #include "GPUCommonDef.h" #include "DataFormatsITS/TimeEstBC.h" -#include "ITStracking/Constants.h" +#include "ITSMFTTracking/Constants.h" #include "ITStracking/TrackITSInternal.h" #include "ReconstructionDataFormats/Track.h" diff --git a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TrackFollower.h b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TrackFollower.h index 703b01e014dd7..cd3194807225f 100644 --- a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TrackFollower.h +++ b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TrackFollower.h @@ -22,10 +22,10 @@ #include "DetectorsBase/Propagator.h" #include "ITStracking/Cluster.h" -#include "ITStracking/Constants.h" +#include "ITSMFTTracking/Constants.h" #include "ITStracking/IndexTableUtils.h" -#include "ITStracking/MathUtils.h" -#include "ITStracking/ROFLookupTables.h" +#include "ITSMFTTracking/MathUtils.h" +#include "ITSMFTTracking/ROFLookupTables.h" #include "ITStracking/TrackExtensionHypothesis.h" #include "ITStracking/TrackHelpers.h" diff --git a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TrackHelpers.h b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TrackHelpers.h index b9924a8197d4c..1ed68342fc160 100644 --- a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TrackHelpers.h +++ b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TrackHelpers.h @@ -20,9 +20,9 @@ #include "DataFormatsITS/TrackITS.h" #include "ITStracking/Cell.h" #include "ITStracking/Cluster.h" -#include "ITStracking/Constants.h" +#include "ITSMFTTracking/Constants.h" #include "ITStracking/LayerMask.h" -#include "ITStracking/MathUtils.h" +#include "ITSMFTTracking/MathUtils.h" #include "ITStracking/TrackITSInternal.h" #include "DetectorsBase/Propagator.h" #include "ReconstructionDataFormats/Track.h" diff --git a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TrackITSInternal.h b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TrackITSInternal.h index 28a1117324550..2a0cbac70c870 100644 --- a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TrackITSInternal.h +++ b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TrackITSInternal.h @@ -17,7 +17,7 @@ #include "GPUCommonDef.h" #include "DataFormatsITS/TrackITS.h" #include "DataFormatsITS/TimeEstBC.h" -#include "ITStracking/Constants.h" +#include "ITSMFTTracking/Constants.h" #include "ReconstructionDataFormats/Track.h" namespace o2::its diff --git a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/Tracklet.h b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/Tracklet.h index be9d8840d88f5..ab55c77e373e7 100644 --- a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/Tracklet.h +++ b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/Tracklet.h @@ -16,7 +16,7 @@ #ifndef TRACKINGITS_INCLUDE_TRACKLET_H_ #define TRACKINGITS_INCLUDE_TRACKLET_H_ -#include "ITStracking/Constants.h" +#include "ITSMFTTracking/Constants.h" #include "DataFormatsITS/TimeEstBC.h" #include "ITStracking/Cluster.h" #include "MathUtils/Utils.h" diff --git a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/Vertexer.h b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/Vertexer.h index a932d0b7b35c5..5f3cd51608f2a 100644 --- a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/Vertexer.h +++ b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/Vertexer.h @@ -26,7 +26,7 @@ #include -#include "ITStracking/Constants.h" +#include "ITSMFTTracking/Constants.h" #include "ITStracking/Configuration.h" #include "ITStracking/TimeFrame.h" #include "ITStracking/VertexerTraits.h" diff --git a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/VertexerTraits.h b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/VertexerTraits.h index 23d3934a8764e..060d39da9a3f5 100644 --- a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/VertexerTraits.h +++ b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/VertexerTraits.h @@ -29,7 +29,7 @@ #include "ITStracking/IndexTableUtils.h" #include "ITStracking/TimeFrame.h" #include "ITStracking/Tracklet.h" -#include "ITStracking/MathUtils.h" +#include "ITSMFTTracking/MathUtils.h" #include "GPUCommonDef.h" #include "GPUCommonMath.h" diff --git a/Detectors/ITSMFT/ITS/tracking/src/Configuration.cxx b/Detectors/ITSMFT/ITS/tracking/src/Configuration.cxx index eb4d90ee9d15f..5e8e24664ed78 100644 --- a/Detectors/ITSMFT/ITS/tracking/src/Configuration.cxx +++ b/Detectors/ITSMFT/ITS/tracking/src/Configuration.cxx @@ -16,9 +16,9 @@ #include #include "Framework/Logger.h" -#include "ITStracking/Constants.h" +#include "ITSMFTTracking/Constants.h" #include "ITStracking/Configuration.h" -#include "ITStracking/TrackingConfigParam.h" +#include "ITSMFTTracking/ITSTrackingConfigParam.h" using namespace o2::its; diff --git a/Detectors/ITSMFT/ITS/tracking/src/IOUtils.cxx b/Detectors/ITSMFT/ITS/tracking/src/IOUtils.cxx index e2ce374ed1600..9b1f9836a1053 100644 --- a/Detectors/ITSMFT/ITS/tracking/src/IOUtils.cxx +++ b/Detectors/ITSMFT/ITS/tracking/src/IOUtils.cxx @@ -20,7 +20,7 @@ #include #include "ITSBase/GeometryTGeo.h" -#include "ITStracking/TrackingConfigParam.h" +#include "ITSMFTTracking/ITSTrackingConfigParam.h" #include "ITSMFTReconstruction/ChipMappingITS.h" namespace diff --git a/Detectors/ITSMFT/ITS/tracking/src/LineVertexerHelpers.cxx b/Detectors/ITSMFT/ITS/tracking/src/LineVertexerHelpers.cxx index de70d07f8c362..1ff9b7a9b73e6 100644 --- a/Detectors/ITSMFT/ITS/tracking/src/LineVertexerHelpers.cxx +++ b/Detectors/ITSMFT/ITS/tracking/src/LineVertexerHelpers.cxx @@ -23,8 +23,8 @@ #include #include -#include "ITStracking/Constants.h" -#include "ITStracking/MathUtils.h" +#include "ITSMFTTracking/Constants.h" +#include "ITSMFTTracking/MathUtils.h" #include "ITStracking/LineVertexerHelpers.h" namespace o2::its::line_vertexer diff --git a/Detectors/ITSMFT/ITS/tracking/src/TimeFrame.cxx b/Detectors/ITSMFT/ITS/tracking/src/TimeFrame.cxx index 2ce0b7dabb070..0ef8fe467f1b9 100644 --- a/Detectors/ITSMFT/ITS/tracking/src/TimeFrame.cxx +++ b/Detectors/ITSMFT/ITS/tracking/src/TimeFrame.cxx @@ -20,7 +20,7 @@ #include #include "ITStracking/TimeFrame.h" -#include "ITStracking/MathUtils.h" +#include "ITSMFTTracking/MathUtils.h" #include "DataFormatsITSMFT/CompCluster.h" #include "DataFormatsITSMFT/ROFRecord.h" #include "DataFormatsITSMFT/TopologyDictionary.h" diff --git a/Detectors/ITSMFT/ITS/tracking/src/Tracker.cxx b/Detectors/ITSMFT/ITS/tracking/src/Tracker.cxx index 0da93595297f4..8d6ff800afc2c 100644 --- a/Detectors/ITSMFT/ITS/tracking/src/Tracker.cxx +++ b/Detectors/ITSMFT/ITS/tracking/src/Tracker.cxx @@ -15,9 +15,9 @@ #include "ITStracking/Tracker.h" #include "ITSMFTTracking/BoundedAllocator.h" -#include "ITStracking/Constants.h" +#include "ITSMFTTracking/Constants.h" #include "ITStracking/TrackerTraits.h" -#include "ITStracking/TrackingConfigParam.h" +#include "ITSMFTTracking/ITSTrackingConfigParam.h" #include #include diff --git a/Detectors/ITSMFT/ITS/tracking/src/TrackerTraits.cxx b/Detectors/ITSMFT/ITS/tracking/src/TrackerTraits.cxx index 5351a80bc8337..a5211afa958df 100644 --- a/Detectors/ITSMFT/ITS/tracking/src/TrackerTraits.cxx +++ b/Detectors/ITSMFT/ITS/tracking/src/TrackerTraits.cxx @@ -32,10 +32,10 @@ #include "GPUCommonMath.h" #include "ITSMFTTracking/BoundedAllocator.h" #include "ITStracking/Cell.h" -#include "ITStracking/Constants.h" +#include "ITSMFTTracking/Constants.h" #include "ITStracking/IndexTableUtils.h" #include "ITStracking/LayerMask.h" -#include "ITStracking/ROFLookupTables.h" +#include "ITSMFTTracking/ROFLookupTables.h" #include "ITSMFTTracking/SlabBumpAllocator.h" #include "ITStracking/TrackerTraits.h" #include "ITStracking/TrackFollower.h" diff --git a/Detectors/ITSMFT/ITS/tracking/src/TrackingInterface.cxx b/Detectors/ITSMFT/ITS/tracking/src/TrackingInterface.cxx index 2ee30d69f6bbf..cf896efba731f 100644 --- a/Detectors/ITSMFT/ITS/tracking/src/TrackingInterface.cxx +++ b/Detectors/ITSMFT/ITS/tracking/src/TrackingInterface.cxx @@ -22,8 +22,8 @@ #include "ITStracking/FastMultEstConfig.h" #include "ITStracking/FastMultEst.h" -#include "ITStracking/ROFLookupTables.h" -#include "ITStracking/TrackingConfigParam.h" +#include "ITSMFTTracking/ROFLookupTables.h" +#include "ITSMFTTracking/ITSTrackingConfigParam.h" #include "ITStracking/TrackingInterface.h" #include "DataFormatsITSMFT/ROFRecord.h" diff --git a/Detectors/ITSMFT/ITS/tracking/src/TrackingLinkDef.h b/Detectors/ITSMFT/ITS/tracking/src/TrackingLinkDef.h index 46af692fe0c15..5ba4e36dda875 100644 --- a/Detectors/ITSMFT/ITS/tracking/src/TrackingLinkDef.h +++ b/Detectors/ITSMFT/ITS/tracking/src/TrackingLinkDef.h @@ -33,12 +33,6 @@ #pragma link C++ class o2::its::ClusterLines + ; #pragma link C++ class std::vector < o2::its::ClusterLines> + ; -#pragma link C++ class o2::its::VertexerParamConfig + ; -#pragma link C++ class o2::conf::ConfigurableParamHelper < o2::its::VertexerParamConfig> + ; - -#pragma link C++ class o2::its::TrackerParamConfig + ; -#pragma link C++ class o2::conf::ConfigurableParamHelper < o2::its::TrackerParamConfig> + ; - #pragma link C++ class o2::its::FastMultEstConfig + ; #pragma link C++ class o2::conf::ConfigurableParamHelper < o2::its::FastMultEstConfig> + ; diff --git a/Detectors/ITSMFT/ITS/tracking/src/Vertexer.cxx b/Detectors/ITSMFT/ITS/tracking/src/Vertexer.cxx index 25f97b37b8e21..4f6da24452258 100644 --- a/Detectors/ITSMFT/ITS/tracking/src/Vertexer.cxx +++ b/Detectors/ITSMFT/ITS/tracking/src/Vertexer.cxx @@ -16,7 +16,7 @@ #include "ITStracking/Vertexer.h" #include "ITSMFTTracking/BoundedAllocator.h" #include "ITStracking/VertexerTraits.h" -#include "ITStracking/TrackingConfigParam.h" +#include "ITSMFTTracking/ITSTrackingConfigParam.h" namespace o2::its { diff --git a/Detectors/ITSMFT/ITS/tracking/test/testROFLookupTables.cxx b/Detectors/ITSMFT/ITS/tracking/test/testROFLookupTables.cxx index 9626e42efd547..0a4323a619a5f 100644 --- a/Detectors/ITSMFT/ITS/tracking/test/testROFLookupTables.cxx +++ b/Detectors/ITSMFT/ITS/tracking/test/testROFLookupTables.cxx @@ -15,7 +15,7 @@ #define BOOST_TEST_DYN_LINK #include -#include "ITStracking/ROFLookupTables.h" +#include "ITSMFTTracking/ROFLookupTables.h" /// -------- Tests -------- // LayerTiming diff --git a/Detectors/ITSMFT/ITS/workflow/src/RecoWorkflow.cxx b/Detectors/ITSMFT/ITS/workflow/src/RecoWorkflow.cxx index 06b3f019a6be7..3a9c28b4935ec 100644 --- a/Detectors/ITSMFT/ITS/workflow/src/RecoWorkflow.cxx +++ b/Detectors/ITSMFT/ITS/workflow/src/RecoWorkflow.cxx @@ -16,7 +16,7 @@ #include "ITSMFTWorkflow/ClusterWriterSpec.h" #include "ITSWorkflow/TrackerSpec.h" #include "ITSWorkflow/TrackWriterSpec.h" -#include "ITStracking/TrackingConfigParam.h" +#include "ITSMFTTracking/ITSTrackingConfigParam.h" #include "ITSMFTWorkflow/DigitReaderSpec.h" #include "GlobalTrackingWorkflowWriters/IRFrameWriterSpec.h" #include "GPUWorkflow/GPUWorkflowSpec.h" diff --git a/Detectors/ITSMFT/ITS/workflow/src/TrackerSpec.cxx b/Detectors/ITSMFT/ITS/workflow/src/TrackerSpec.cxx index a198e638c5bac..cb53dae1ed905 100644 --- a/Detectors/ITSMFT/ITS/workflow/src/TrackerSpec.cxx +++ b/Detectors/ITSMFT/ITS/workflow/src/TrackerSpec.cxx @@ -19,7 +19,7 @@ #include "DataFormatsITSMFT/DPLAlpideParam.h" #include "ITSWorkflow/TrackerSpec.h" #include "ITStracking/Definitions.h" -#include "ITStracking/TrackingConfigParam.h" +#include "ITSMFTTracking/ITSTrackingConfigParam.h" namespace o2 { diff --git a/Detectors/ITSMFT/ITS/workflow/src/its-threshold-calib-workflow.cxx b/Detectors/ITSMFT/ITS/workflow/src/its-threshold-calib-workflow.cxx index a7d252d59b9f0..fe4176bea5269 100644 --- a/Detectors/ITSMFT/ITS/workflow/src/its-threshold-calib-workflow.cxx +++ b/Detectors/ITSMFT/ITS/workflow/src/its-threshold-calib-workflow.cxx @@ -11,7 +11,7 @@ #include "ITSWorkflow/ThresholdCalibratorSpec.h" #include "CommonUtils/ConfigurableParam.h" -#include "ITStracking/TrackingConfigParam.h" +#include "ITSMFTTracking/ITSTrackingConfigParam.h" #include "ITStracking/Configuration.h" #include "Framework/ConfigParamSpec.h" diff --git a/Detectors/ITSMFT/common/tracking/CMakeLists.txt b/Detectors/ITSMFT/common/tracking/CMakeLists.txt index a1ed4a37c83fc..af69c29a8583c 100644 --- a/Detectors/ITSMFT/common/tracking/CMakeLists.txt +++ b/Detectors/ITSMFT/common/tracking/CMakeLists.txt @@ -12,9 +12,20 @@ o2_add_library(ITSMFTTracking SOURCES src/BoundedAllocator.cxx src/CapacityEstimator.cxx + src/ITSTrackingConfigParam.cxx src/SlabBumpAllocator.cxx - PRIVATE_LINK_LIBRARIES O2::GPUCommon - O2::FrameworkLogger + PUBLIC_LINK_LIBRARIES O2::CommonConstants + O2::CommonDataFormat + O2::CommonUtils + O2::DataFormatsITS + O2::FrameworkLogger + O2::GPUCommon + O2::MathUtils + PRIVATE_LINK_LIBRARIES TBB::tbb) +o2_target_root_dictionary(ITSMFTTracking + HEADERS include/ITSMFTTracking/ITSTrackingConfigParam.h + LINKDEF src/ITSTrackingLinkDef.h) + add_subdirectory(test) diff --git a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/Constants.h b/Detectors/ITSMFT/common/tracking/include/ITSMFTTracking/Constants.h similarity index 100% rename from Detectors/ITSMFT/ITS/tracking/include/ITStracking/Constants.h rename to Detectors/ITSMFT/common/tracking/include/ITSMFTTracking/Constants.h diff --git a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TrackingConfigParam.h b/Detectors/ITSMFT/common/tracking/include/ITSMFTTracking/ITSTrackingConfigParam.h similarity index 99% rename from Detectors/ITSMFT/ITS/tracking/include/ITStracking/TrackingConfigParam.h rename to Detectors/ITSMFT/common/tracking/include/ITSMFTTracking/ITSTrackingConfigParam.h index ee67b7234450d..39aae6cb8330e 100644 --- a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TrackingConfigParam.h +++ b/Detectors/ITSMFT/common/tracking/include/ITSMFTTracking/ITSTrackingConfigParam.h @@ -15,7 +15,7 @@ #include #include "CommonUtils/ConfigurableParam.h" #include "CommonUtils/ConfigurableParamHelper.h" -#include "ITStracking/Constants.h" +#include "ITSMFTTracking/Constants.h" namespace o2::its { diff --git a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/MathUtils.h b/Detectors/ITSMFT/common/tracking/include/ITSMFTTracking/MathUtils.h similarity index 99% rename from Detectors/ITSMFT/ITS/tracking/include/ITStracking/MathUtils.h rename to Detectors/ITSMFT/common/tracking/include/ITSMFTTracking/MathUtils.h index fc903d32e5d5e..fd1595d425573 100644 --- a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/MathUtils.h +++ b/Detectors/ITSMFT/common/tracking/include/ITSMFTTracking/MathUtils.h @@ -17,7 +17,7 @@ #define O2_ITS_TRACKING_MATHUTILS_H_ #include "CommonConstants/MathConstants.h" -#include "ITStracking/Constants.h" +#include "ITSMFTTracking/Constants.h" #include "MathUtils/Utils.h" #include "GPUCommonMath.h" #include "GPUCommonDef.h" diff --git a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/ROFLookupTables.h b/Detectors/ITSMFT/common/tracking/include/ITSMFTTracking/ROFLookupTables.h similarity index 100% rename from Detectors/ITSMFT/ITS/tracking/include/ITStracking/ROFLookupTables.h rename to Detectors/ITSMFT/common/tracking/include/ITSMFTTracking/ROFLookupTables.h diff --git a/Detectors/ITSMFT/ITS/tracking/src/TrackingConfigParam.cxx b/Detectors/ITSMFT/common/tracking/src/ITSTrackingConfigParam.cxx similarity index 92% rename from Detectors/ITSMFT/ITS/tracking/src/TrackingConfigParam.cxx rename to Detectors/ITSMFT/common/tracking/src/ITSTrackingConfigParam.cxx index 47b5f8ffffdb1..063f51636894f 100644 --- a/Detectors/ITSMFT/ITS/tracking/src/TrackingConfigParam.cxx +++ b/Detectors/ITSMFT/common/tracking/src/ITSTrackingConfigParam.cxx @@ -9,6 +9,6 @@ // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. -#include "ITStracking/TrackingConfigParam.h" +#include "ITSMFTTracking/ITSTrackingConfigParam.h" O2ParamImpl(o2::its::VertexerParamConfig); O2ParamImpl(o2::its::TrackerParamConfig); diff --git a/Detectors/ITSMFT/common/tracking/src/ITSTrackingLinkDef.h b/Detectors/ITSMFT/common/tracking/src/ITSTrackingLinkDef.h new file mode 100644 index 0000000000000..630684d8bfda4 --- /dev/null +++ b/Detectors/ITSMFT/common/tracking/src/ITSTrackingLinkDef.h @@ -0,0 +1,24 @@ +// Copyright 2019-2026 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +#ifdef __CLING__ + +#pragma link off all globals; +#pragma link off all classes; +#pragma link off all functions; + +#pragma link C++ class o2::its::VertexerParamConfig + ; +#pragma link C++ class o2::conf::ConfigurableParamHelper < o2::its::VertexerParamConfig> + ; + +#pragma link C++ class o2::its::TrackerParamConfig + ; +#pragma link C++ class o2::conf::ConfigurableParamHelper < o2::its::TrackerParamConfig> + ; + +#endif diff --git a/Detectors/Upgrades/ALICE3/GlobalReconstruction/reconstruction/include/ALICE3GlobalReconstruction/TimeFrameMixin.h b/Detectors/Upgrades/ALICE3/GlobalReconstruction/reconstruction/include/ALICE3GlobalReconstruction/TimeFrameMixin.h index da1a80b77772b..4e08b460d999e 100644 --- a/Detectors/Upgrades/ALICE3/GlobalReconstruction/reconstruction/include/ALICE3GlobalReconstruction/TimeFrameMixin.h +++ b/Detectors/Upgrades/ALICE3/GlobalReconstruction/reconstruction/include/ALICE3GlobalReconstruction/TimeFrameMixin.h @@ -19,7 +19,7 @@ #include "CommonDataFormat/InteractionRecord.h" #include "DataFormatsTRKFT3/Cluster.h" #include "DataFormatsTRKFT3/ROFRecord.h" -#include "ITStracking/ROFLookupTables.h" +#include "ITSMFTTracking/ROFLookupTables.h" #include "ITStracking/TimeFrame.h" #include "SimulationDataFormat/MCCompLabel.h" #include "SimulationDataFormat/MCEventHeader.h" diff --git a/Detectors/Upgrades/ALICE3/GlobalReconstruction/workflow/src/TrackerSpec.cxx b/Detectors/Upgrades/ALICE3/GlobalReconstruction/workflow/src/TrackerSpec.cxx index 41779053bd36e..8e841000d483c 100644 --- a/Detectors/Upgrades/ALICE3/GlobalReconstruction/workflow/src/TrackerSpec.cxx +++ b/Detectors/Upgrades/ALICE3/GlobalReconstruction/workflow/src/TrackerSpec.cxx @@ -29,7 +29,7 @@ #include "Framework/ControlService.h" #include "Framework/ConfigParamRegistry.h" #include "Framework/CCDBParamSpec.h" -#include "ITStracking/TrackingConfigParam.h" +#include "ITSMFTTracking/ITSTrackingConfigParam.h" #include "SimulationDataFormat/MCEventHeader.h" #include "SimulationDataFormat/MCCompLabel.h" #include "SimulationDataFormat/MCTruthContainer.h" diff --git a/Detectors/Upgrades/ITS3/alignment/src/AlignmentSpec.cxx b/Detectors/Upgrades/ITS3/alignment/src/AlignmentSpec.cxx index d50eba24327ee..edba6718ab268 100644 --- a/Detectors/Upgrades/ITS3/alignment/src/AlignmentSpec.cxx +++ b/Detectors/Upgrades/ITS3/alignment/src/AlignmentSpec.cxx @@ -40,7 +40,7 @@ #include "ReconstructionDataFormats/VtxTrackRef.h" #include "ITS3Reconstruction/TopologyDictionary.h" #include "DataFormatsITSMFT/TopologyDictionary.h" -#include "ITStracking/MathUtils.h" +#include "ITSMFTTracking/MathUtils.h" #include "ITStracking/IOUtils.h" #include "ITS3Reconstruction/IOUtils.h" #include "ITS3Align/TrackFit.h" diff --git a/Detectors/Upgrades/ITS3/reconstruction/src/IOUtils.cxx b/Detectors/Upgrades/ITS3/reconstruction/src/IOUtils.cxx index 7f153945b458b..95f868bdaa615 100644 --- a/Detectors/Upgrades/ITS3/reconstruction/src/IOUtils.cxx +++ b/Detectors/Upgrades/ITS3/reconstruction/src/IOUtils.cxx @@ -16,7 +16,7 @@ #include "DataFormatsITSMFT/ROFRecord.h" #include "ITS3Reconstruction/TopologyDictionary.h" #include "ITSBase/GeometryTGeo.h" -#include "ITStracking/TrackingConfigParam.h" +#include "ITSMFTTracking/ITSTrackingConfigParam.h" namespace o2::its3::ioutils { diff --git a/Detectors/Upgrades/ITS3/workflow/src/TrackerSpec.cxx b/Detectors/Upgrades/ITS3/workflow/src/TrackerSpec.cxx index 94e711a05a2d6..7c1a87a778c8d 100644 --- a/Detectors/Upgrades/ITS3/workflow/src/TrackerSpec.cxx +++ b/Detectors/Upgrades/ITS3/workflow/src/TrackerSpec.cxx @@ -22,7 +22,7 @@ #include "DataFormatsITSMFT/ROFRecord.h" #include "DataFormatsITSMFT/PhysTrigger.h" -#include "ITStracking/TrackingConfigParam.h" +#include "ITSMFTTracking/ITSTrackingConfigParam.h" #include "DataFormatsITSMFT/DPLAlpideParam.h" #include "ITSBase/GeometryTGeo.h" diff --git a/Detectors/Upgrades/ITS3/workflow/src/its3-reco-workflow.cxx b/Detectors/Upgrades/ITS3/workflow/src/its3-reco-workflow.cxx index beb6815bc9bcd..ad76b60da5a06 100644 --- a/Detectors/Upgrades/ITS3/workflow/src/its3-reco-workflow.cxx +++ b/Detectors/Upgrades/ITS3/workflow/src/its3-reco-workflow.cxx @@ -11,7 +11,7 @@ #include "ITS3Workflow/RecoWorkflow.h" #include "CommonUtils/ConfigurableParam.h" -#include "ITStracking/TrackingConfigParam.h" +#include "ITSMFTTracking/ITSTrackingConfigParam.h" #include "ITStracking/Configuration.h" #include "DetectorsRaw/HBFUtilsInitializer.h" #include "Framework/CallbacksPolicy.h" diff --git a/GPU/Workflow/src/GPUWorkflowITS.cxx b/GPU/Workflow/src/GPUWorkflowITS.cxx index 2a0e36d65bb7a..794f7fc3fda22 100644 --- a/GPU/Workflow/src/GPUWorkflowITS.cxx +++ b/GPU/Workflow/src/GPUWorkflowITS.cxx @@ -22,7 +22,7 @@ #include "CommonUtils/ConfigurableParam.h" #include "CommonUtils/NameConf.h" #include "ITStracking/TrackingInterface.h" -#include "ITStracking/TrackingConfigParam.h" +#include "ITSMFTTracking/ITSTrackingConfigParam.h" #include #include diff --git a/GPU/Workflow/src/GPUWorkflowSpec.cxx b/GPU/Workflow/src/GPUWorkflowSpec.cxx index 5edf72eabd9b7..990873f053fe3 100644 --- a/GPU/Workflow/src/GPUWorkflowSpec.cxx +++ b/GPU/Workflow/src/GPUWorkflowSpec.cxx @@ -78,7 +78,7 @@ #include "GPUReconstructionConvert.h" #include "DetectorsRaw/RDHUtils.h" #include "ITStracking/TrackingInterface.h" -#include "ITStracking/TrackingConfigParam.h" +#include "ITSMFTTracking/ITSTrackingConfigParam.h" #include "GPUWorkflowInternal.h" #include "GPUDataTypesQA.h" // #include "Framework/ThreadPool.h" From 79f32e0390fc4eb30148c6586f61e720832284a8 Mon Sep 17 00:00:00 2001 From: Maximiliano Puccio Date: Fri, 28 Aug 2026 13:36:42 +0200 Subject: [PATCH 6/7] Address ITSMFT tracking review feedback --- .../GPU/ITStrackingGPU/TimeFrameGPU.h | 4 +- .../GPU/ITStrackingGPU/TrackingKernels.h | 11 +- .../ITS/tracking/GPU/cuda/TimeFrameGPU.cu | 4 +- .../tracking/GPU/cuda/TrackerTraitsGPU.cxx | 5 +- .../ITS/tracking/GPU/cuda/TrackingKernels.cu | 7 +- .../include/ITStracking/LineVertexerHelpers.h | 7 +- .../tracking/include/ITStracking/TimeFrame.h | 88 ++++----- .../tracking/include/ITStracking/Tracker.h | 4 +- .../include/ITStracking/TrackerTraits.h | 12 +- .../include/ITStracking/TrackingInterface.h | 2 +- .../tracking/include/ITStracking/Vertexer.h | 4 +- .../include/ITStracking/VertexerTraits.h | 6 +- .../ITS/tracking/src/LineVertexerHelpers.cxx | 100 +++++----- .../ITSMFT/ITS/tracking/src/TimeFrame.cxx | 177 +++++++++--------- Detectors/ITSMFT/ITS/tracking/src/Tracker.cxx | 8 +- .../ITSMFT/ITS/tracking/src/TrackerTraits.cxx | 72 +++---- .../ITS/tracking/src/TrackingInterface.cxx | 2 +- .../ITSMFT/ITS/tracking/src/Vertexer.cxx | 8 +- .../ITS/tracking/src/VertexerTraits.cxx | 20 +- .../ITSMFT/ITS/tracking/test/CMakeLists.txt | 6 - .../ITSMFTTracking/CapacityEstimator.h | 11 +- .../ITSMFTTracking/SlabBumpAllocator.h | 15 +- .../common/tracking/src/BoundedAllocator.cxx | 12 +- .../common/tracking/src/CapacityEstimator.cxx | 31 +-- .../common/tracking/test/CMakeLists.txt | 6 + .../tracking/test/testROFLookupTables.cxx | 2 +- .../tracking/test/testSlabBumpAllocator.cxx | 4 + 27 files changed, 326 insertions(+), 302 deletions(-) rename Detectors/ITSMFT/{ITS => common}/tracking/test/testROFLookupTables.cxx (99%) diff --git a/Detectors/ITSMFT/ITS/tracking/GPU/ITStrackingGPU/TimeFrameGPU.h b/Detectors/ITSMFT/ITS/tracking/GPU/ITStrackingGPU/TimeFrameGPU.h index ab27cc5b1804d..9d80edfaf2b92 100644 --- a/Detectors/ITSMFT/ITS/tracking/GPU/ITStrackingGPU/TimeFrameGPU.h +++ b/Detectors/ITSMFT/ITS/tracking/GPU/ITStrackingGPU/TimeFrameGPU.h @@ -250,8 +250,8 @@ class TimeFrameGPU : public TimeFrame std::bitset mPinnedTrackingFrameInfo{0}; // Temporary buffer for storing output tracks from GPU tracking - o2::itsmft::tracking::bounded_vector mTrackITSExt; - o2::itsmft::tracking::bounded_vector mTrackIndices; + bounded_vector mTrackITSExt; + bounded_vector mTrackIndices; }; template diff --git a/Detectors/ITSMFT/ITS/tracking/GPU/ITStrackingGPU/TrackingKernels.h b/Detectors/ITSMFT/ITS/tracking/GPU/ITStrackingGPU/TrackingKernels.h index ad83d9c9a06e9..3a7c5dbd6b510 100644 --- a/Detectors/ITSMFT/ITS/tracking/GPU/ITStrackingGPU/TrackingKernels.h +++ b/Detectors/ITSMFT/ITS/tracking/GPU/ITStrackingGPU/TrackingKernels.h @@ -26,6 +26,9 @@ namespace o2::its { +using o2::itsmft::tracking::bounded_vector; +using o2::itsmft::tracking::CapacityEstimator; + class CellSeed; struct CellNeighbour; template @@ -62,13 +65,13 @@ struct TrackingKernels { const bool selectUPCVertices, const float NSigmaCut, const typename TrackingTopology::View topology, - o2::itsmft::tracking::bounded_vector& linkPhiCuts, + bounded_vector& linkPhiCuts, const float resolutionPV, std::array& minR, std::array& maxR, - o2::itsmft::tracking::bounded_vector& resolutions, + bounded_vector& resolutions, std::vector& radii, - o2::itsmft::tracking::bounded_vector& linkMSAngles, + bounded_vector& linkMSAngles, o2::its::ExternalAllocator* alloc, gpu::Streams& streams); @@ -118,7 +121,7 @@ struct TrackingKernels { TrackSeed* seedsDevice, const int seedsCapacity, int& seedsCursor, - o2::itsmft::tracking::CapacityEstimator& estimator, + CapacityEstimator& estimator, const int iteration, const float bz, const float MaxChi2ClusterAttachment, diff --git a/Detectors/ITSMFT/ITS/tracking/GPU/cuda/TimeFrameGPU.cu b/Detectors/ITSMFT/ITS/tracking/GPU/cuda/TimeFrameGPU.cu index 25ec5b9c04928..c610ffe011171 100644 --- a/Detectors/ITSMFT/ITS/tracking/GPU/cuda/TimeFrameGPU.cu +++ b/Detectors/ITSMFT/ITS/tracking/GPU/cuda/TimeFrameGPU.cu @@ -491,8 +491,8 @@ void TimeFrameGPU::createTrackITSExtHost(const size_t nTracks) { GPUTimer timer("reserving host tracks"); mNTracks = nTracks; - mTrackITSExt = o2::itsmft::tracking::bounded_vector(nTracks, {}, this->getMemoryPool().get()); - mTrackIndices = o2::itsmft::tracking::bounded_vector(nTracks, 0, this->getMemoryPool().get()); + mTrackITSExt = bounded_vector(nTracks, {}, this->getMemoryPool().get()); + mTrackIndices = bounded_vector(nTracks, 0, this->getMemoryPool().get()); std::iota(mTrackIndices.begin(), mTrackIndices.end(), 0); } diff --git a/Detectors/ITSMFT/ITS/tracking/GPU/cuda/TrackerTraitsGPU.cxx b/Detectors/ITSMFT/ITS/tracking/GPU/cuda/TrackerTraitsGPU.cxx index a76fc258a06b5..767c02c646293 100644 --- a/Detectors/ITSMFT/ITS/tracking/GPU/cuda/TrackerTraitsGPU.cxx +++ b/Detectors/ITSMFT/ITS/tracking/GPU/cuda/TrackerTraitsGPU.cxx @@ -20,7 +20,6 @@ namespace o2::its { -using o2::itsmft::tracking::CapacityEstimator; using o2::itsmft::tracking::runOnSlab; using o2::itsmft::tracking::SlabSite; @@ -199,7 +198,7 @@ template void TrackerTraitsGPU::findCellsNeighbours(const int iteration) { const auto hostTopology = mTimeFrameGPU->getTrackingTopologyView(); - o2::itsmft::tracking::bounded_vector sourceTopologies(this->getMemoryPool().get()); + bounded_vector sourceTopologies(this->getMemoryPool().get()); sourceTopologies.reserve(hostTopology.nCells); for (int outerLayer{0}; outerLayer < NLayers; ++outerLayer) { for (int targetCellTopologyId{0}; targetCellTopologyId < hostTopology.nCells; ++targetCellTopologyId) { @@ -266,7 +265,7 @@ void TrackerTraitsGPU::findCellsNeighbours(const int iteration) template void TrackerTraitsGPU::findRoads(const int iteration) { - o2::itsmft::tracking::bounded_vector> firstClusters(this->mTrkParams[iteration].NLayers, o2::itsmft::tracking::bounded_vector(this->getMemoryPool().get()), this->getMemoryPool().get()); + bounded_vector> firstClusters(this->mTrkParams[iteration].NLayers, bounded_vector(this->getMemoryPool().get()), this->getMemoryPool().get()); firstClusters.resize(this->mTrkParams[iteration].NLayers); const auto hostTopology = mTimeFrameGPU->getTrackingTopologyView(); const bool extendTop = this->mTrkParams[iteration].PassFlags[IterationStep::TrackFollowerTop]; diff --git a/Detectors/ITSMFT/ITS/tracking/GPU/cuda/TrackingKernels.cu b/Detectors/ITSMFT/ITS/tracking/GPU/cuda/TrackingKernels.cu index 6fa466c6ac560..f2be0158ea343 100644 --- a/Detectors/ITSMFT/ITS/tracking/GPU/cuda/TrackingKernels.cu +++ b/Detectors/ITSMFT/ITS/tracking/GPU/cuda/TrackingKernels.cu @@ -53,7 +53,6 @@ using namespace o2::track; namespace o2::its { -using o2::itsmft::tracking::CapacityEstimator; using o2::itsmft::tracking::runOnSlab; using o2::itsmft::tracking::SlabSite; namespace gpu @@ -789,13 +788,13 @@ int TrackingKernels::computeTrackletsInROFsHandler(const IndexTableUtil const bool selectUPCVertices, const float NSigmaCut, const typename TrackingTopology::View topology, - o2::itsmft::tracking::bounded_vector& linkPhiCuts, + bounded_vector& linkPhiCuts, const float resolutionPV, std::array& minRs, std::array& maxRs, - o2::itsmft::tracking::bounded_vector& resolutions, + bounded_vector& resolutions, std::vector& radii, - o2::itsmft::tracking::bounded_vector& linkMSAngles, + bounded_vector& linkMSAngles, o2::its::ExternalAllocator* alloc, gpu::Streams& streams) { diff --git a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/LineVertexerHelpers.h b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/LineVertexerHelpers.h index 04b93a849718c..9e0f0d2eed02c 100644 --- a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/LineVertexerHelpers.h +++ b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/LineVertexerHelpers.h @@ -21,6 +21,9 @@ namespace o2::its::line_vertexer { +using o2::itsmft::tracking::bounded_vector; +using o2::itsmft::tracking::BoundedMemoryResource; + struct Settings { float beamX = 0.f; float beamY = 0.f; @@ -36,10 +39,10 @@ struct Settings { float maxZ = 0.f; int seedMemberRadiusTime = 1; int seedMemberRadiusZ = 2; - std::shared_ptr memoryPool; + std::shared_ptr memoryPool; }; -o2::itsmft::tracking::bounded_vector buildClusters(std::span lines, const Settings& settings); +bounded_vector buildClusters(std::span lines, const Settings& settings); } // namespace o2::its::line_vertexer diff --git a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TimeFrame.h b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TimeFrame.h index 416fa53fea1f6..e4bc7f045f64f 100644 --- a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TimeFrame.h +++ b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TimeFrame.h @@ -57,6 +57,10 @@ class ROFRecord; namespace its { +using o2::itsmft::tracking::bounded_vector; +using o2::itsmft::tracking::BoundedMemoryResource; +using o2::itsmft::tracking::CapacityEstimator; + namespace gpu { template @@ -179,7 +183,7 @@ struct TimeFrame { gsl::span getClusterLabels(int layerId, const int clId) const { return mClusterLabels[((mIsStaggered) ? layerId : 0)]->getLabels(mClusterExternalIndices[layerId][clId]); } int getClusterExternalIndex(int layerId, const int clId) const { return mClusterExternalIndices[layerId][clId]; } int getClusterSize(int layer, int clusterId) const { return mClusterSize[layer][clusterId]; } - void setClusterSize(int layer, o2::itsmft::tracking::bounded_vector& v) { mClusterSize[layer] = std::move(v); } + void setClusterSize(int layer, bounded_vector& v) { mClusterSize[layer] = std::move(v); } auto& getTrackletsLabel(int layer) { return mTrackletLabels[layer]; } auto& getCellsLabel(int layer) { return mCellLabels[layer]; } @@ -222,7 +226,7 @@ struct TimeFrame { size_t getNExtendedClusters() const { return mNExtendedClusters; } /// memory management - void setMemoryPool(std::shared_ptr pool); + void setMemoryPool(std::shared_ptr pool); auto& getMemoryPool() const noexcept { return mMemoryPool; } bool checkMemory(unsigned long max) { return getArtefactsMemory() < max; } unsigned long getArtefactsMemory() const; @@ -231,8 +235,8 @@ struct TimeFrame { /// staggering void setIsStaggered(bool b) noexcept { mIsStaggered = b; } - o2::itsmft::tracking::CapacityEstimator& getCapacityEstimator() noexcept { return mCapacityEstimator; } - const o2::itsmft::tracking::CapacityEstimator& getCapacityEstimator() const noexcept { return mCapacityEstimator; } + CapacityEstimator& getCapacityEstimator() noexcept { return mCapacityEstimator; } + const CapacityEstimator& getCapacityEstimator() const noexcept { return mCapacityEstimator; } // Vertexer void computeTrackletsPerROFScans(); @@ -258,7 +262,7 @@ struct TimeFrame { /// State if memory will be externally managed by the GPU framework ExternalAllocator* mExternalAllocator{nullptr}; - std::shared_ptr mExtMemoryPool; // host memory pool managed by the framework + std::shared_ptr mExtMemoryPool; // host memory pool managed by the framework auto getFrameworkAllocator() { return mExternalAllocator; }; void setFrameworkAllocator(ExternalAllocator* ext); bool hasFrameworkAllocator() const noexcept { return mExternalAllocator != nullptr; } @@ -274,28 +278,28 @@ struct TimeFrame { void addTrackingFrameInfoToLayer(int layer, T&&... args); void addClusterExternalIndexToLayer(int layer, const int idx) { mClusterExternalIndices[layer].push_back(idx); } - std::array, NLayers> mClusters; - std::array, NLayers> mTrackingFrameInfo; - std::array, NLayers> mClusterExternalIndices; - std::array, NLayers> mROFramesClusters; + std::array, NLayers> mClusters; + std::array, NLayers> mTrackingFrameInfo; + std::array, NLayers> mClusterExternalIndices; + std::array, NLayers> mROFramesClusters; std::array*, NLayers> mClusterLabels{nullptr}; - std::array, 2> mNTrackletsPerCluster; - std::array, 2> mNTrackletsPerClusterSum; - std::array, NLayers> mNClustersPerROF; - std::array, NLayers> mIndexTables; - std::vector> mTrackletsLookupTable; - std::array, NLayers> mUsedClusters; - - std::array, NLayers> mUnsortedClusters; - std::vector> mTracklets; - std::vector> mCells; - o2::itsmft::tracking::bounded_vector mTracks; - o2::itsmft::tracking::bounded_vector mTracksLabel; + std::array, 2> mNTrackletsPerCluster; + std::array, 2> mNTrackletsPerClusterSum; + std::array, NLayers> mNClustersPerROF; + std::array, NLayers> mIndexTables; + std::vector> mTrackletsLookupTable; + std::array, NLayers> mUsedClusters; + + std::array, NLayers> mUnsortedClusters; + std::vector> mTracklets; + std::vector> mCells; + bounded_vector mTracks; + bounded_vector mTracksLabel; size_t mNExtendedTracks = 0; size_t mNExtendedClusters = 0; - std::vector> mCellsNeighbours; - std::vector> mCellsNeighboursTopology; - std::vector> mCellsLookupTable; + std::vector> mCellsNeighbours; + std::vector> mCellsNeighboursTopology; + std::vector> mCellsLookupTable; const o2::base::PropagatorImpl* mPropagatorDevice = nullptr; // Needed only for GPU @@ -314,27 +318,27 @@ struct TimeFrame { bool isBeamPositionOverridden = false; std::array mMinR; std::array mMaxR; - o2::itsmft::tracking::bounded_vector mLinkPhiCuts; - o2::itsmft::tracking::bounded_vector mLinkMSAngles; - o2::itsmft::tracking::bounded_vector mPositionResolution; - std::array, NLayers> mClusterSize; + bounded_vector mLinkPhiCuts; + bounded_vector mLinkMSAngles; + bounded_vector mPositionResolution; + std::array, NLayers> mClusterSize; - o2::itsmft::tracking::bounded_vector> mPValphaX; /// PV x and alpha for track propagation - std::vector> mTrackletLabels; - std::vector> mCellLabels; - std::vector> mCellsNeighboursLUT; - o2::itsmft::tracking::bounded_vector mBogusClusters; /// keep track of clusters with wild coordinates + bounded_vector> mPValphaX; /// PV x and alpha for track propagation + std::vector> mTrackletLabels; + std::vector> mCellLabels; + std::vector> mCellsNeighboursLUT; + bounded_vector mBogusClusters; /// keep track of clusters with wild coordinates - o2::itsmft::tracking::CapacityEstimator mCapacityEstimator; + CapacityEstimator mCapacityEstimator; // Vertexer - o2::itsmft::tracking::bounded_vector mPrimaryVertices; - o2::itsmft::tracking::bounded_vector mPrimaryVerticesLabels; - std::vector> mNTrackletsPerROF; - std::vector> mLines; - std::vector> mTrackletClusters; - std::array, 2> mTrackletsIndexROF; - std::vector> mLinesLabels; + bounded_vector mPrimaryVertices; + bounded_vector mPrimaryVerticesLabels; + std::vector> mNTrackletsPerROF; + std::vector> mLines; + std::vector> mTrackletClusters; + std::array, 2> mTrackletsIndexROF; + std::vector> mLinesLabels; std::array mTotalTracklets = {0, 0}; uint32_t mTotalLines = 0; // \Vertexer @@ -356,7 +360,7 @@ struct TimeFrame { bool mIsStaggered{false}; - std::shared_ptr mMemoryPool; + std::shared_ptr mMemoryPool; }; template diff --git a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/Tracker.h b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/Tracker.h index 569f4e6e913a6..53c32be166f06 100644 --- a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/Tracker.h +++ b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/Tracker.h @@ -60,7 +60,7 @@ class Tracker const LogFunc& = [](const std::string& s) { std::cerr << s << '\n'; }); void setParameters(const std::vector& p) { mTrkParams = p; } - void setMemoryPool(std::shared_ptr pool) { mMemoryPool = pool; } + void setMemoryPool(std::shared_ptr pool) { mMemoryPool = pool; } std::vector& getParameters() { return mTrkParams; } void setBz(float bz) { mTraits->setBz(bz); } void setTimeSlice(size_t slice) noexcept { mTimeSlice = slice; } @@ -91,7 +91,7 @@ class Tracker unsigned int mNumberOfDroppedTFs{0}; unsigned int mTimeFrameCounter{0}; double mTotalTime{0}; - std::shared_ptr mMemoryPool; + std::shared_ptr mMemoryPool; enum Steps { TFInit = 0, diff --git a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TrackerTraits.h b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TrackerTraits.h index edfe908aaabff..bcf865e7d34b3 100644 --- a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TrackerTraits.h +++ b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TrackerTraits.h @@ -72,9 +72,9 @@ class TrackerTraits virtual void findRoads(const int iteration); template - void processNeighbours(int iteration, int defaultCellTopologyId, int iLevel, uint64_t capacityKey, const o2::itsmft::tracking::bounded_vector& currentSeeds, o2::itsmft::tracking::bounded_vector& updatedSeeds); + void processNeighbours(int iteration, int defaultCellTopologyId, int iLevel, uint64_t capacityKey, const bounded_vector& currentSeeds, bounded_vector& updatedSeeds); - void acceptTracks(int iteration, o2::itsmft::tracking::bounded_vector& tracks, const o2::itsmft::tracking::bounded_vector& trackIndices, o2::itsmft::tracking::bounded_vector>& firstClusters); + void acceptTracks(int iteration, bounded_vector& tracks, const bounded_vector& trackIndices, bounded_vector>& firstClusters); void markTracks(int iteration); void updateTrackingParameters(const std::vector& trkPars) @@ -86,7 +86,7 @@ class TrackerTraits float getBz() const { return mBz; } virtual const char* getName() const noexcept { return "CPU"; } virtual bool isGPU() const noexcept { return false; } - void setMemoryPool(std::shared_ptr pool) noexcept { mMemoryPool = pool; } + void setMemoryPool(std::shared_ptr pool) noexcept { mMemoryPool = pool; } auto getMemoryPool() const noexcept { return mMemoryPool; } // Others @@ -99,7 +99,7 @@ class TrackerTraits virtual int getTFNumberOfCells() const { return mTimeFrame->getNumberOfCells(); } private: - std::shared_ptr mMemoryPool; + std::shared_ptr mMemoryPool; protected: std::shared_ptr mTaskArena; @@ -110,8 +110,8 @@ class TrackerTraits { } - o2::itsmft::tracking::bounded_vector> activeHypotheses; - o2::itsmft::tracking::bounded_vector> nextHypotheses; + bounded_vector> activeHypotheses; + bounded_vector> nextHypotheses; }; bool finaliseTrackSeed(const TrackSeedN& seed, diff --git a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TrackingInterface.h b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TrackingInterface.h index 4ccc1259a33bf..f3fd4c26aeacd 100644 --- a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TrackingInterface.h +++ b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/TrackingInterface.h @@ -101,7 +101,7 @@ class ITSTrackingInterface std::unique_ptr mTracker = nullptr; std::unique_ptr mVertexer = nullptr; const o2::dataformats::MeanVertexObject* mMeanVertex{}; - std::shared_ptr mMemoryPool; + std::shared_ptr mMemoryPool; std::shared_ptr mTaskArena; }; diff --git a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/Vertexer.h b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/Vertexer.h index 5f3cd51608f2a..76647d923e2aa 100644 --- a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/Vertexer.h +++ b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/Vertexer.h @@ -52,7 +52,7 @@ class Vertexer auto& getVertParameters() const { return mTraits->getVertexingParameters(); } void setParameters(const std::vector& vertParams) { mVertParams = vertParams; } const auto& getParameters() const noexcept { return mVertParams; } - void setMemoryPool(std::shared_ptr pool) { mMemoryPool = pool; } + void setMemoryPool(std::shared_ptr pool) { mMemoryPool = pool; } float clustersToVertices(LogFunc = [](const std::string& s) { std::cout << s << '\n'; }); void filterMCTracklets(); @@ -105,7 +105,7 @@ class Vertexer TimeFrameN* mTimeFrame = nullptr; /// Observer pointer, not owned by this class std::vector mVertParams; - std::shared_ptr mMemoryPool; + std::shared_ptr mMemoryPool; enum Steps { Init = 0, diff --git a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/VertexerTraits.h b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/VertexerTraits.h index 060d39da9a3f5..0182e30cbf4ed 100644 --- a/Detectors/ITSMFT/ITS/tracking/include/ITStracking/VertexerTraits.h +++ b/Detectors/ITSMFT/ITS/tracking/include/ITStracking/VertexerTraits.h @@ -73,9 +73,9 @@ class VertexerTraits virtual bool isGPU() const noexcept { return false; } virtual const char* getName() const noexcept { return "CPU"; } virtual bool usesMemoryPool() const noexcept { return true; } - void setMemoryPool(std::shared_ptr pool) { mMemoryPool = pool; } + void setMemoryPool(std::shared_ptr pool) { mMemoryPool = pool; } - static VertexLabel computeMain(const o2::itsmft::tracking::bounded_vector& elements) + static VertexLabel computeMain(const bounded_vector& elements) { // we only care about the source&event of the tracks, not the trackId auto composeVtxLabel = [](const o2::MCCompLabel& lbl) -> o2::MCCompLabel { @@ -108,7 +108,7 @@ class VertexerTraits private: bool skipROF(int iteration, int rof) const; - std::shared_ptr mMemoryPool; + std::shared_ptr mMemoryPool; std::shared_ptr mTaskArena; }; diff --git a/Detectors/ITSMFT/ITS/tracking/src/LineVertexerHelpers.cxx b/Detectors/ITSMFT/ITS/tracking/src/LineVertexerHelpers.cxx index 1ff9b7a9b73e6..807863fc54895 100644 --- a/Detectors/ITSMFT/ITS/tracking/src/LineVertexerHelpers.cxx +++ b/Detectors/ITSMFT/ITS/tracking/src/LineVertexerHelpers.cxx @@ -80,13 +80,13 @@ struct LineRef { }; struct VertexSeed { - explicit VertexSeed(const std::shared_ptr& mr) : contributors(mr.get()), assigned(mr.get()) {} + explicit VertexSeed(const std::shared_ptr& mr) : contributors(mr.get()), assigned(mr.get()) {} std::array vertex = {}; TimeEstBC time; float scale2 = InitialScale2; - o2::itsmft::tracking::bounded_vector contributors; - o2::itsmft::tracking::bounded_vector assigned; + bounded_vector contributors; + bounded_vector assigned; bool valid = false; bool isUsableSeed() const noexcept { @@ -94,7 +94,7 @@ struct VertexSeed { } }; -void compactSeeds(o2::itsmft::tracking::bounded_vector& seeds) +void compactSeeds(bounded_vector& seeds) { seeds.erase(std::remove_if(seeds.begin(), seeds.end(), [](const VertexSeed& seed) { return !seed.isUsableSeed(); @@ -103,7 +103,7 @@ void compactSeeds(o2::itsmft::tracking::bounded_vector& seeds) } struct Histogram2D { - explicit Histogram2D(const std::shared_ptr& mr) : bins(mr.get()) {} + explicit Histogram2D(const std::shared_ptr& mr) : bins(mr.get()) {} int nTimeBins = 0; int nZBins = 0; @@ -111,7 +111,7 @@ struct Histogram2D { float zMin = 0.f; float timeBinSize = 1.f; float zBinSize = 1.f; - o2::itsmft::tracking::bounded_vector bins; + bounded_vector bins; int getIndex(const int tBin, const int zBin) const noexcept { @@ -313,9 +313,9 @@ class SeedHistogram return mHistogram.getNeighborhoodSum(peakIndex, mSeedMemberRadiusTime, mSeedMemberRadiusZ); } - o2::itsmft::tracking::bounded_vector collectLocalMembers(const int peakIndex, const int radiusTime, const int radiusZ) const + bounded_vector collectLocalMembers(const int peakIndex, const int radiusTime, const int radiusZ) const { - o2::itsmft::tracking::bounded_vector localMembers(mMemoryPool.get()); + bounded_vector localMembers(mMemoryPool.get()); localMembers.reserve(mMembers.size()); const auto [timeBin, zBin] = mHistogram.decodeIndex(peakIndex); for (const auto lineRefIdx : mMembers) { @@ -358,7 +358,7 @@ class SeedHistogram private: float medianTimeError(std::span lines) const { - o2::itsmft::tracking::bounded_vector errors(mMemoryPool.get()); + bounded_vector errors(mMemoryPool.get()); errors.reserve(mMembers.size()); for (const auto lineRefIdx : mMembers) { errors.push_back(static_cast(lines[mLineRefs[lineRefIdx].lineIndex].mTime.getTimeStampError())); @@ -371,17 +371,17 @@ class SeedHistogram std::span mLineRefs; int mSeedMemberRadiusTime = 1; int mSeedMemberRadiusZ = 2; - std::shared_ptr mMemoryPool; + std::shared_ptr mMemoryPool; Histogram2D mHistogram; }; -float updateScale2(const std::span chi2s, const std::shared_ptr& mr) noexcept +float updateScale2(const std::span chi2s, const std::shared_ptr& mr) noexcept { if (chi2s.empty()) { return MinScale2; } - o2::itsmft::tracking::bounded_vector sorted(chi2s.begin(), chi2s.end(), mr.get()); + bounded_vector sorted(chi2s.begin(), chi2s.end(), mr.get()); std::sort(sorted.begin(), sorted.end()); const auto median = sorted[sorted.size() / 2]; @@ -442,7 +442,7 @@ VertexSeed fitSeed(const VertexSeed& initialSeed, std::span members, std::span lineRefs, std::span lines, - const std::shared_ptr& mr, + const std::shared_ptr& mr, const float pairCut2) { VertexSeed seed{mr}; @@ -460,7 +460,7 @@ VertexSeed fitSeed(const VertexSeed& initialSeed, VertexFit vertexFit; TimeEstBC commonTime{}; bool hasCommonTime = false; - o2::itsmft::tracking::bounded_vector contributors{mr.get()}; + bounded_vector contributors{mr.get()}; const auto scale2 = std::max(seed.scale2, MinScale2); const auto tukeyFactor = 1.f / (scale2 * TukeyC2); @@ -511,7 +511,7 @@ VertexSeed fitSeed(const VertexSeed& initialSeed, seed.vertex = updatedVertex; seed.time = commonTime; - o2::itsmft::tracking::bounded_vector updatedChi2s{mr.get()}; + bounded_vector updatedChi2s{mr.get()}; updatedChi2s.reserve(contributors.size()); for (const auto lineRefIx : contributors) { updatedChi2s.push_back(Line::getDistance2FromPoint(lines[lineRefs[lineRefIx].lineIndex], seed.vertex) / pairCut2); @@ -547,14 +547,14 @@ size_t countSharedContributors(std::span lhs, std::span rh return shared; } -o2::itsmft::tracking::bounded_vector collectCompatibleContributors(const VertexSeed& seed, - std::span members, - std::span lineRefs, - std::span lines, - const std::shared_ptr& mr, - const float pairCut2) +bounded_vector collectCompatibleContributors(const VertexSeed& seed, + std::span members, + std::span lineRefs, + std::span lines, + const std::shared_ptr& mr, + const float pairCut2) { - o2::itsmft::tracking::bounded_vector contributors{mr.get()}; + bounded_vector contributors{mr.get()}; contributors.reserve(members.size()); for (const auto lineRefIdx : members) { const auto lineIdx = lineRefs[lineRefIdx].lineIndex; @@ -571,7 +571,7 @@ o2::itsmft::tracking::bounded_vector collectCompatibleContributors(const Ve return contributors; } -void deduplicateSeeds(o2::itsmft::tracking::bounded_vector& seeds, const Settings& settings) +void deduplicateSeeds(bounded_vector& seeds, const Settings& settings) { if (seeds.size() < 2) { return; @@ -622,7 +622,7 @@ void deduplicateSeeds(o2::itsmft::tracking::bounded_vector& seeds, c compactSeeds(seeds); } -void deduplicateRefittedSeeds(o2::itsmft::tracking::bounded_vector& seeds, const Settings& settings) +void deduplicateRefittedSeeds(bounded_vector& seeds, const Settings& settings) { if (seeds.size() < 2) { return; @@ -675,21 +675,21 @@ void deduplicateRefittedSeeds(o2::itsmft::tracking::bounded_vector& } struct OrderedComponent { - explicit OrderedComponent(const std::shared_ptr& mr) : members(mr.get()) {} + explicit OrderedComponent(const std::shared_ptr& mr) : members(mr.get()) {} float center = 0.f; - o2::itsmft::tracking::bounded_vector members; + bounded_vector members; }; -o2::itsmft::tracking::bounded_vector> buildCoarseClusters(std::span lineRefs, - std::span lines, - const Settings& settings) +bounded_vector> buildCoarseClusters(std::span lineRefs, + std::span lines, + const Settings& settings) { - o2::itsmft::tracking::bounded_vector> clusters(settings.memoryPool.get()); + bounded_vector> clusters(settings.memoryPool.get()); if (lineRefs.size() < 2) { return clusters; } - o2::itsmft::tracking::bounded_vector sortedByLower(lineRefs.size(), settings.memoryPool.get()); + bounded_vector sortedByLower(lineRefs.size(), settings.memoryPool.get()); std::iota(sortedByLower.begin(), sortedByLower.end(), 0); std::sort(sortedByLower.begin(), sortedByLower.end(), [&](const int lhs, const int rhs) { const auto lhsLower = lines[lineRefs[lhs].lineIndex].mTime.lower(); @@ -701,8 +701,8 @@ o2::itsmft::tracking::bounded_vector> }); const auto coarseZWindow = settings.coarseZWindow > 0.f ? settings.coarseZWindow : settings.clusterCut; - o2::itsmft::tracking::bounded_vector parent(lineRefs.size(), settings.memoryPool.get()); - o2::itsmft::tracking::bounded_vector componentSize(lineRefs.size(), 1, settings.memoryPool.get()); + bounded_vector parent(lineRefs.size(), settings.memoryPool.get()); + bounded_vector componentSize(lineRefs.size(), 1, settings.memoryPool.get()); std::iota(parent.begin(), parent.end(), 0); float minZ = std::numeric_limits::max(); float maxZ = std::numeric_limits::lowest(); @@ -742,10 +742,10 @@ o2::itsmft::tracking::bounded_vector> }; using ActiveEntry = std::pair; - o2::itsmft::tracking::bounded_vector activeEntries(settings.memoryPool.get()); - std::priority_queue, std::greater<>> activeByUpper(std::greater<>{}, std::move(activeEntries)); - o2::itsmft::tracking::bounded_vector activeMask(lineRefs.size(), 0, settings.memoryPool.get()); - o2::itsmft::tracking::bounded_vector> activeByZBin(settings.memoryPool.get()); + bounded_vector activeEntries(settings.memoryPool.get()); + std::priority_queue, std::greater<>> activeByUpper(std::greater<>{}, std::move(activeEntries)); + bounded_vector activeMask(lineRefs.size(), 0, settings.memoryPool.get()); + bounded_vector> activeByZBin(settings.memoryPool.get()); activeByZBin.reserve(nZBins); for (int iBin = 0; iBin < nZBins; ++iBin) { activeByZBin.emplace_back(); @@ -787,7 +787,7 @@ o2::itsmft::tracking::bounded_vector> activeByZBin[zBin].push_back(lineRefIdx); } - std::unordered_map> components; + std::unordered_map> components; components.reserve(lineRefs.size()); for (int lineRefIdx = 0; lineRefIdx < static_cast(lineRefs.size()); ++lineRefIdx) { const auto root = findRoot(lineRefIdx); @@ -796,7 +796,7 @@ o2::itsmft::tracking::bounded_vector> it->second.push_back(lineRefIdx); } - o2::itsmft::tracking::bounded_vector orderedComponents(settings.memoryPool.get()); + bounded_vector orderedComponents(settings.memoryPool.get()); orderedComponents.reserve(components.size()); for (auto& [root, members] : components) { (void)root; @@ -829,13 +829,13 @@ o2::itsmft::tracking::bounded_vector> return clusters; } -o2::itsmft::tracking::bounded_vector buildSeeds(std::span members, - std::span lineRefs, - std::span lines, - const Settings& settings) +bounded_vector buildSeeds(std::span members, + std::span lineRefs, + std::span lines, + const Settings& settings) { SeedHistogram histogram(members, lineRefs, lines, settings); - o2::itsmft::tracking::bounded_vector seeds(settings.memoryPool.get()); + bounded_vector seeds(settings.memoryPool.get()); seeds.reserve(MaxSeedsPerCluster); float leadingPeakSupport = 0.f; @@ -879,7 +879,7 @@ o2::itsmft::tracking::bounded_vector buildSeeds(std::span return seeds; } -void assignLinesToSeeds(o2::itsmft::tracking::bounded_vector& seeds, +void assignLinesToSeeds(bounded_vector& seeds, std::span members, std::span lineRefs, std::span lines, @@ -937,9 +937,9 @@ void assignLinesToSeeds(o2::itsmft::tracking::bounded_vector& seeds, ClusterLines materializeCluster(const VertexSeed& seed, std::span lineRefs, std::span lines, - const std::shared_ptr& mr) + const std::shared_ptr& mr) { - o2::itsmft::tracking::bounded_vector lineIndices{mr.get()}; + bounded_vector lineIndices{mr.get()}; lineIndices.reserve(seed.contributors.size()); for (const auto lineRefIdx : seed.contributors) { lineIndices.push_back(lineRefs[lineRefIdx].lineIndex); @@ -956,14 +956,14 @@ ClusterLines materializeCluster(const VertexSeed& seed, } // namespace -o2::itsmft::tracking::bounded_vector buildClusters(std::span lines, const Settings& settings) +bounded_vector buildClusters(std::span lines, const Settings& settings) { - o2::itsmft::tracking::bounded_vector clusters(settings.memoryPool.get()); + bounded_vector clusters(settings.memoryPool.get()); if (lines.size() < 2) { return clusters; } - o2::itsmft::tracking::bounded_vector refs(settings.memoryPool.get()); + bounded_vector refs(settings.memoryPool.get()); refs.reserve(lines.size()); for (int lineIdx = 0; lineIdx < static_cast(lines.size()); ++lineIdx) { LineRef ref(lines[lineIdx], lineIdx, settings.beamX, settings.beamY, settings.maxZ); diff --git a/Detectors/ITSMFT/ITS/tracking/src/TimeFrame.cxx b/Detectors/ITSMFT/ITS/tracking/src/TimeFrame.cxx index 0ef8fe467f1b9..9502f55bda9db 100644 --- a/Detectors/ITSMFT/ITS/tracking/src/TimeFrame.cxx +++ b/Detectors/ITSMFT/ITS/tracking/src/TimeFrame.cxx @@ -41,6 +41,9 @@ struct ClusterHelper { namespace o2::its { +using o2::itsmft::tracking::clearResizeBoundedVector; +using o2::itsmft::tracking::deepVectorClear; + constexpr float DefClusErrorRow = o2::itsmft::SegmentationAlpide::PitchRow * 0.5; constexpr float DefClusErrorCol = o2::itsmft::SegmentationAlpide::PitchCol * 0.5; constexpr float DefClusError2Row = DefClusErrorRow * DefClusErrorRow; @@ -150,16 +153,16 @@ template void TimeFrame::resetROFrameData(int layer) { if (layer >= 0) { - o2::itsmft::tracking::deepVectorClear(mUnsortedClusters[layer], getMaybeFrameworkHostResource()); - o2::itsmft::tracking::deepVectorClear(mTrackingFrameInfo[layer], getMaybeFrameworkHostResource()); - o2::itsmft::tracking::deepVectorClear(mClusterExternalIndices[layer], mMemoryPool.get()); - o2::itsmft::tracking::clearResizeBoundedVector(mROFramesClusters[layer], mROFOverlapTableView.getLayer(layer).mNROFsTF + 1, getMaybeFrameworkHostResource()); + deepVectorClear(mUnsortedClusters[layer], getMaybeFrameworkHostResource()); + deepVectorClear(mTrackingFrameInfo[layer], getMaybeFrameworkHostResource()); + deepVectorClear(mClusterExternalIndices[layer], mMemoryPool.get()); + clearResizeBoundedVector(mROFramesClusters[layer], mROFOverlapTableView.getLayer(layer).mNROFsTF + 1, getMaybeFrameworkHostResource()); } else { for (int iLayer{0}; iLayer < NLayers; ++iLayer) { - o2::itsmft::tracking::deepVectorClear(mUnsortedClusters[iLayer], getMaybeFrameworkHostResource()); - o2::itsmft::tracking::deepVectorClear(mTrackingFrameInfo[iLayer], getMaybeFrameworkHostResource()); - o2::itsmft::tracking::deepVectorClear(mClusterExternalIndices[iLayer], mMemoryPool.get()); - o2::itsmft::tracking::clearResizeBoundedVector(mROFramesClusters[iLayer], mROFOverlapTableView.getLayer(iLayer).mNROFsTF + 1, getMaybeFrameworkHostResource()); + deepVectorClear(mUnsortedClusters[iLayer], getMaybeFrameworkHostResource()); + deepVectorClear(mTrackingFrameInfo[iLayer], getMaybeFrameworkHostResource()); + deepVectorClear(mClusterExternalIndices[iLayer], mMemoryPool.get()); + clearResizeBoundedVector(mROFramesClusters[iLayer], mROFOverlapTableView.getLayer(iLayer).mNROFsTF + 1, getMaybeFrameworkHostResource()); } } } @@ -171,10 +174,10 @@ void TimeFrame::prepareROFrameData(gsl::span clusterCountPerLayer{0}; for (const auto& cls : clusters) { ++clusterCountPerLayer[geom->getLayer(cls.getChipID())]; @@ -195,9 +198,9 @@ void TimeFrame::prepareClusters(const TrackingParameters& trkParam, con const int stopLayer = std::min(trkParam.NLayers, maxLayers); tbb::parallel_for(0, stopLayer, [&](const int iLayer) { - o2::itsmft::tracking::bounded_vector cHelper(mMemoryPool.get()); - o2::itsmft::tracking::bounded_vector clsPerBin(numBins, 0, mMemoryPool.get()); - o2::itsmft::tracking::bounded_vector lutPerBin(numBins, 0, mMemoryPool.get()); + bounded_vector cHelper(mMemoryPool.get()); + bounded_vector clsPerBin(numBins, 0, mMemoryPool.get()); + bounded_vector lutPerBin(numBins, 0, mMemoryPool.get()); float minR{mMinR[iLayer]}; float maxR{mMaxR[iLayer]}; int bogus{0}; @@ -298,29 +301,29 @@ void TimeFrame::initialise(const TrackingParameters& trkParam, const in mTrackingTopologyView = iteration != constants::UnusedIndex ? mTrackerTopologies[iteration].getView() : (maxLayers == 3 ? mVertexingTopology.getView() : mDefaultTrackingTopology.getView()); if (trkParam.PassFlags[IterationStep::FirstPass]) { - o2::itsmft::tracking::deepVectorClear(mTracks); - o2::itsmft::tracking::deepVectorClear(mTracksLabel); - o2::itsmft::tracking::deepVectorClear(mLines); - o2::itsmft::tracking::deepVectorClear(mLinesLabels); + deepVectorClear(mTracks); + deepVectorClear(mTracksLabel); + deepVectorClear(mLines); + deepVectorClear(mLinesLabels); if (trkParam.PassFlags[IterationStep::ResetVertices]) { - o2::itsmft::tracking::deepVectorClear(mPrimaryVertices); - o2::itsmft::tracking::deepVectorClear(mPrimaryVerticesLabels); + deepVectorClear(mPrimaryVertices); + deepVectorClear(mPrimaryVerticesLabels); } - o2::itsmft::tracking::clearResizeBoundedVector(mLinesLabels, getNrof(1), mMemoryPool.get()); + clearResizeBoundedVector(mLinesLabels, getNrof(1), mMemoryPool.get()); mIndexTableUtils.setTrackingParameters(trkParam); - o2::itsmft::tracking::clearResizeBoundedVector(mPositionResolution, trkParam.NLayers, mMemoryPool.get()); - o2::itsmft::tracking::clearResizeBoundedVector(mBogusClusters, trkParam.NLayers, mMemoryPool.get()); - o2::itsmft::tracking::deepVectorClear(mTrackletClusters); + clearResizeBoundedVector(mPositionResolution, trkParam.NLayers, mMemoryPool.get()); + clearResizeBoundedVector(mBogusClusters, trkParam.NLayers, mMemoryPool.get()); + deepVectorClear(mTrackletClusters); for (unsigned int iLayer{0}; iLayer < std::min((int)mClusters.size(), maxLayers); ++iLayer) { - o2::itsmft::tracking::clearResizeBoundedVector(mClusters[iLayer], mUnsortedClusters[iLayer].size(), getMaybeFrameworkHostResource(maxLayers != NLayers)); - o2::itsmft::tracking::clearResizeBoundedVector(mUsedClusters[iLayer], mUnsortedClusters[iLayer].size(), getMaybeFrameworkHostResource(maxLayers != NLayers)); + clearResizeBoundedVector(mClusters[iLayer], mUnsortedClusters[iLayer].size(), getMaybeFrameworkHostResource(maxLayers != NLayers)); + clearResizeBoundedVector(mUsedClusters[iLayer], mUnsortedClusters[iLayer].size(), getMaybeFrameworkHostResource(maxLayers != NLayers)); mPositionResolution[iLayer] = o2::gpu::CAMath::Sqrt((0.5f * (trkParam.SystErrorZ2[iLayer] + trkParam.SystErrorY2[iLayer])) + (trkParam.LayerResolution[iLayer] * trkParam.LayerResolution[iLayer])); } - o2::itsmft::tracking::clearResizeBoundedVector(mLines, getNrof(1), mMemoryPool.get()); - o2::itsmft::tracking::clearResizeBoundedVector(mTrackletClusters, getNrof(1), mMemoryPool.get()); + clearResizeBoundedVector(mLines, getNrof(1), mMemoryPool.get()); + clearResizeBoundedVector(mTrackletClusters, getNrof(1), mMemoryPool.get()); for (int iLayer{0}; iLayer < NLayers; ++iLayer) { - o2::itsmft::tracking::clearResizeBoundedVector(mIndexTables[iLayer], getNrof(iLayer) * ((trkParam.ZBins * trkParam.PhiBins) + 1), getMaybeFrameworkHostResource()); + clearResizeBoundedVector(mIndexTables[iLayer], getNrof(iLayer) * ((trkParam.ZBins * trkParam.PhiBins) + 1), getMaybeFrameworkHostResource()); } for (int iLayer{0}; iLayer < trkParam.NLayers; ++iLayer) { if (trkParam.SystErrorY2[iLayer] > 0.f || trkParam.SystErrorZ2[iLayer] > 0.f) { @@ -335,20 +338,20 @@ void TimeFrame::initialise(const TrackingParameters& trkParam, const in mMinR.fill(std::numeric_limits::max()); mMaxR.fill(std::numeric_limits::min()); } - o2::itsmft::tracking::clearResizeBoundedVector(mCells, mTrackingTopologyView.nCells, mMemoryPool.get()); - o2::itsmft::tracking::clearResizeBoundedVector(mCellsLookupTable, mTrackingTopologyView.nCells, mMemoryPool.get()); - o2::itsmft::tracking::clearResizeBoundedVector(mCellsNeighbours, mTrackingTopologyView.nCells, mMemoryPool.get()); - o2::itsmft::tracking::clearResizeBoundedVector(mCellsNeighboursTopology, mTrackingTopologyView.nCells, mMemoryPool.get()); - o2::itsmft::tracking::clearResizeBoundedVector(mCellsNeighboursLUT, mTrackingTopologyView.nCells, mMemoryPool.get()); - o2::itsmft::tracking::clearResizeBoundedVector(mCellLabels, mTrackingTopologyView.nCells, mMemoryPool.get()); - o2::itsmft::tracking::clearResizeBoundedVector(mTracklets, mTrackingTopologyView.nLinks, mMemoryPool.get()); - o2::itsmft::tracking::clearResizeBoundedVector(mTrackletLabels, mTrackingTopologyView.nLinks, mMemoryPool.get()); - o2::itsmft::tracking::clearResizeBoundedVector(mTrackletsLookupTable, mTrackingTopologyView.nLinks, mMemoryPool.get()); - o2::itsmft::tracking::clearResizeBoundedVector(mLinkPhiCuts, mTrackingTopologyView.nLinks, mMemoryPool.get()); - o2::itsmft::tracking::clearResizeBoundedVector(mLinkMSAngles, mTrackingTopologyView.nLinks, mMemoryPool.get()); + clearResizeBoundedVector(mCells, mTrackingTopologyView.nCells, mMemoryPool.get()); + clearResizeBoundedVector(mCellsLookupTable, mTrackingTopologyView.nCells, mMemoryPool.get()); + clearResizeBoundedVector(mCellsNeighbours, mTrackingTopologyView.nCells, mMemoryPool.get()); + clearResizeBoundedVector(mCellsNeighboursTopology, mTrackingTopologyView.nCells, mMemoryPool.get()); + clearResizeBoundedVector(mCellsNeighboursLUT, mTrackingTopologyView.nCells, mMemoryPool.get()); + clearResizeBoundedVector(mCellLabels, mTrackingTopologyView.nCells, mMemoryPool.get()); + clearResizeBoundedVector(mTracklets, mTrackingTopologyView.nLinks, mMemoryPool.get()); + clearResizeBoundedVector(mTrackletLabels, mTrackingTopologyView.nLinks, mMemoryPool.get()); + clearResizeBoundedVector(mTrackletsLookupTable, mTrackingTopologyView.nLinks, mMemoryPool.get()); + clearResizeBoundedVector(mLinkPhiCuts, mTrackingTopologyView.nLinks, mMemoryPool.get()); + clearResizeBoundedVector(mLinkMSAngles, mTrackingTopologyView.nLinks, mMemoryPool.get()); mNTrackletsPerROF.resize(2); for (auto& v : mNTrackletsPerROF) { - v = o2::itsmft::tracking::bounded_vector(getNrof(1) + 1, 0, mMemoryPool.get()); + v = bounded_vector(getNrof(1) + 1, 0, mMemoryPool.get()); } if (trkParam.PassFlags[IterationStep::RebuildClusterLUT]) { prepareClusters(trkParam, maxLayers); @@ -356,8 +359,8 @@ void TimeFrame::initialise(const TrackingParameters& trkParam, const in mTotalTracklets = {0, 0}; if (maxLayers < trkParam.NLayers) { // Vertexer only, but in both iterations for (size_t iLayer{0}; iLayer < maxLayers; ++iLayer) { - o2::itsmft::tracking::deepVectorClear(mUsedClusters[iLayer]); - o2::itsmft::tracking::clearResizeBoundedVector(mUsedClusters[iLayer], mUnsortedClusters[iLayer].size(), mMemoryPool.get()); + deepVectorClear(mUsedClusters[iLayer]); + clearResizeBoundedVector(mUsedClusters[iLayer], mUnsortedClusters[iLayer].size(), mMemoryPool.get()); } } @@ -390,19 +393,19 @@ void TimeFrame::initialise(const TrackingParameters& trkParam, const in mLinkPhiCuts[linkId] = o2::gpu::CAMath::Min(o2::gpu::CAMath::ASin(0.5f * x * oneOverR) + 2.f * mLinkMSAngles[linkId] + delta, o2::constants::math::PI * 0.5f); // some cleanup - o2::itsmft::tracking::deepVectorClear(mTracklets[linkId]); - o2::itsmft::tracking::deepVectorClear(mTrackletLabels[linkId]); - o2::itsmft::tracking::deepVectorClear(mTrackletsLookupTable[linkId]); + deepVectorClear(mTracklets[linkId]); + deepVectorClear(mTrackletLabels[linkId]); + deepVectorClear(mTrackletsLookupTable[linkId]); mTrackletsLookupTable[linkId].resize(mClusters[link.fromLayer].size() + 1, 0); } for (int cellId{0}; cellId < (int)mCells.size(); ++cellId) { - o2::itsmft::tracking::deepVectorClear(mCells[cellId]); - o2::itsmft::tracking::deepVectorClear(mCellsLookupTable[cellId]); - o2::itsmft::tracking::deepVectorClear(mCellsNeighbours[cellId]); - o2::itsmft::tracking::deepVectorClear(mCellsNeighboursTopology[cellId]); - o2::itsmft::tracking::deepVectorClear(mCellsNeighboursLUT[cellId]); - o2::itsmft::tracking::deepVectorClear(mCellLabels[cellId]); + deepVectorClear(mCells[cellId]); + deepVectorClear(mCellsLookupTable[cellId]); + deepVectorClear(mCellsNeighbours[cellId]); + deepVectorClear(mCellsNeighboursTopology[cellId]); + deepVectorClear(mCellsNeighboursLUT[cellId]); + deepVectorClear(mCellLabels[cellId]); } } @@ -446,13 +449,13 @@ void TimeFrame::computeTrackletsPerROFScans() } template -void TimeFrame::setMemoryPool(std::shared_ptr pool) +void TimeFrame::setMemoryPool(std::shared_ptr pool) { mMemoryPool = pool; - auto initVector = [&](o2::itsmft::tracking::bounded_vector& vec, bool useExternal = false) { + auto initVector = [&](bounded_vector& vec, bool useExternal = false) { std::pmr::memory_resource* mr = (useExternal) ? mExtMemoryPool.get() : mMemoryPool.get(); - o2::itsmft::tracking::deepVectorClear(vec, mr); + deepVectorClear(vec, mr); }; auto initContainers = [&](Container& container, bool useExternal = false) { @@ -498,51 +501,51 @@ template void TimeFrame::setFrameworkAllocator(ExternalAllocator* ext) { mExternalAllocator = ext; - mExtMemoryPool = std::make_shared(std::make_unique(mExternalAllocator)); + mExtMemoryPool = std::make_shared(std::make_unique(mExternalAllocator)); } template void TimeFrame::wipe() { resetTrackExtensionCounters(); - o2::itsmft::tracking::deepVectorClear(mTracks); - o2::itsmft::tracking::deepVectorClear(mTracklets); - o2::itsmft::tracking::deepVectorClear(mCells); - o2::itsmft::tracking::deepVectorClear(mCellsNeighbours); - o2::itsmft::tracking::deepVectorClear(mCellsNeighboursTopology); - o2::itsmft::tracking::deepVectorClear(mCellsLookupTable); - o2::itsmft::tracking::deepVectorClear(mPrimaryVertices); - o2::itsmft::tracking::deepVectorClear(mTrackletsLookupTable); - o2::itsmft::tracking::deepVectorClear(mClusterExternalIndices); - o2::itsmft::tracking::deepVectorClear(mNTrackletsPerCluster); - o2::itsmft::tracking::deepVectorClear(mNTrackletsPerClusterSum); - o2::itsmft::tracking::deepVectorClear(mNClustersPerROF); - o2::itsmft::tracking::deepVectorClear(mLinkPhiCuts); - o2::itsmft::tracking::deepVectorClear(mLinkMSAngles); - o2::itsmft::tracking::deepVectorClear(mPositionResolution); - o2::itsmft::tracking::deepVectorClear(mClusterSize); - o2::itsmft::tracking::deepVectorClear(mPValphaX); - o2::itsmft::tracking::deepVectorClear(mBogusClusters); - o2::itsmft::tracking::deepVectorClear(mTrackletsIndexROF); - o2::itsmft::tracking::deepVectorClear(mTrackletClusters); - o2::itsmft::tracking::deepVectorClear(mLines); + deepVectorClear(mTracks); + deepVectorClear(mTracklets); + deepVectorClear(mCells); + deepVectorClear(mCellsNeighbours); + deepVectorClear(mCellsNeighboursTopology); + deepVectorClear(mCellsLookupTable); + deepVectorClear(mPrimaryVertices); + deepVectorClear(mTrackletsLookupTable); + deepVectorClear(mClusterExternalIndices); + deepVectorClear(mNTrackletsPerCluster); + deepVectorClear(mNTrackletsPerClusterSum); + deepVectorClear(mNClustersPerROF); + deepVectorClear(mLinkPhiCuts); + deepVectorClear(mLinkMSAngles); + deepVectorClear(mPositionResolution); + deepVectorClear(mClusterSize); + deepVectorClear(mPValphaX); + deepVectorClear(mBogusClusters); + deepVectorClear(mTrackletsIndexROF); + deepVectorClear(mTrackletClusters); + deepVectorClear(mLines); // if we use the external host allocator then the assumption is that we // don't clear the memory ourself if (!hasFrameworkAllocator()) { - o2::itsmft::tracking::deepVectorClear(mClusters); - o2::itsmft::tracking::deepVectorClear(mUsedClusters); - o2::itsmft::tracking::deepVectorClear(mUnsortedClusters); - o2::itsmft::tracking::deepVectorClear(mIndexTables); - o2::itsmft::tracking::deepVectorClear(mTrackingFrameInfo); - o2::itsmft::tracking::deepVectorClear(mROFramesClusters); + deepVectorClear(mClusters); + deepVectorClear(mUsedClusters); + deepVectorClear(mUnsortedClusters); + deepVectorClear(mIndexTables); + deepVectorClear(mTrackingFrameInfo); + deepVectorClear(mROFramesClusters); } // only needed to clear if we have MC info if (hasMCinformation()) { - o2::itsmft::tracking::deepVectorClear(mLinesLabels); - o2::itsmft::tracking::deepVectorClear(mPrimaryVerticesLabels); - o2::itsmft::tracking::deepVectorClear(mTrackletLabels); - o2::itsmft::tracking::deepVectorClear(mCellLabels); - o2::itsmft::tracking::deepVectorClear(mTracksLabel); + deepVectorClear(mLinesLabels); + deepVectorClear(mPrimaryVerticesLabels); + deepVectorClear(mTrackletLabels); + deepVectorClear(mCellLabels); + deepVectorClear(mTracksLabel); } } diff --git a/Detectors/ITSMFT/ITS/tracking/src/Tracker.cxx b/Detectors/ITSMFT/ITS/tracking/src/Tracker.cxx index 8d6ff800afc2c..08d61ce23bf26 100644 --- a/Detectors/ITSMFT/ITS/tracking/src/Tracker.cxx +++ b/Detectors/ITSMFT/ITS/tracking/src/Tracker.cxx @@ -105,7 +105,7 @@ float Tracker::clustersToTracks(const LogFunc& logger, const LogFunc& e } total += timeTracklets + timeCells + timeNeighbours + timeRoads; } - } catch (const o2::itsmft::tracking::BoundedMemoryResource::MemoryLimitExceeded& err) { + } catch (const BoundedMemoryResource::MemoryLimitExceeded& err) { handleException(err); return -1.f; } catch (const std::bad_alloc& err) { @@ -204,7 +204,7 @@ template void Tracker::sortTracks() { auto& trks = mTimeFrame->getTracks(); - o2::itsmft::tracking::bounded_vector indices(trks.size(), mMemoryPool.get()); + bounded_vector indices(trks.size(), mMemoryPool.get()); std::iota(indices.begin(), indices.end(), 0); std::sort(indices.begin(), indices.end(), [&trks](size_t i, size_t j) { // provide tracks sorted by lower-bound @@ -217,7 +217,7 @@ void Tracker::sortTracks() } return a.isBetter(b, 1e9); // then sort tracks in quality }); - o2::itsmft::tracking::bounded_vector sortedTrks(mMemoryPool.get()); + bounded_vector sortedTrks(mMemoryPool.get()); sortedTrks.reserve(trks.size()); for (size_t idx : indices) { sortedTrks.push_back(trks[idx]); @@ -225,7 +225,7 @@ void Tracker::sortTracks() trks.swap(sortedTrks); if (mTimeFrame->hasMCinformation()) { auto& trksLabels = mTimeFrame->getTracksLabel(); - o2::itsmft::tracking::bounded_vector sortedLabels(mMemoryPool.get()); + bounded_vector sortedLabels(mMemoryPool.get()); sortedLabels.reserve(trksLabels.size()); for (size_t idx : indices) { sortedLabels.push_back(trksLabels[idx]); diff --git a/Detectors/ITSMFT/ITS/tracking/src/TrackerTraits.cxx b/Detectors/ITSMFT/ITS/tracking/src/TrackerTraits.cxx index a5211afa958df..be1e6985f5ae6 100644 --- a/Detectors/ITSMFT/ITS/tracking/src/TrackerTraits.cxx +++ b/Detectors/ITSMFT/ITS/tracking/src/TrackerTraits.cxx @@ -45,7 +45,7 @@ namespace o2::its { -using o2::itsmft::tracking::CapacityEstimator; +using o2::itsmft::tracking::deepVectorClear; using o2::itsmft::tracking::GroupedSlabSink; using o2::itsmft::tracking::SlabSite; using o2::itsmft::tracking::UnorderedSlabSink; @@ -289,10 +289,10 @@ void TrackerTraits::computeLayerCells(const int iteration) mTaskArena->execute([&] { const int maxConcurrency = std::max(1, mTaskArena->max_concurrency()); auto clearTopology = [&](const int cellTopologyId) { - o2::itsmft::tracking::deepVectorClear(mTimeFrame->getCells()[cellTopologyId]); - o2::itsmft::tracking::deepVectorClear(mTimeFrame->getCellsLookupTable()[cellTopologyId]); + deepVectorClear(mTimeFrame->getCells()[cellTopologyId]); + deepVectorClear(mTimeFrame->getCellsLookupTable()[cellTopologyId]); if (createLabels) { - o2::itsmft::tracking::deepVectorClear(mTimeFrame->getCellsLabel(cellTopologyId)); + deepVectorClear(mTimeFrame->getCellsLabel(cellTopologyId)); } }; if (maxConcurrency > 1) { @@ -373,7 +373,7 @@ void TrackerTraits::computeLayerCells(const int iteration) } }; - o2::itsmft::tracking::bounded_vector activeTopologies(mMemoryPool.get()); + bounded_vector activeTopologies(mMemoryPool.get()); activeTopologies.reserve(topology.nCells); for (int cellTopologyId = 0; cellTopologyId < topology.nCells; ++cellTopologyId) { const auto& cellTopology = topology.getCell(cellTopologyId); @@ -438,8 +438,8 @@ void TrackerTraits::computeLayerCells(const int iteration) } auto clearTracklets = [&](const int linkId) { - o2::itsmft::tracking::deepVectorClear(mTimeFrame->getTracklets()[linkId]); - o2::itsmft::tracking::deepVectorClear(mTimeFrame->getTrackletsLabel(linkId)); + deepVectorClear(mTimeFrame->getTracklets()[linkId]); + deepVectorClear(mTimeFrame->getTrackletsLabel(linkId)); }; if (maxConcurrency > 1) { tbb::parallel_for(0, static_cast(topology.nLinks), clearTracklets); @@ -458,9 +458,9 @@ void TrackerTraits::findCellsNeighbours(const int iteration) mTaskArena->execute([&] { const int maxConcurrency = std::max(1, mTaskArena->max_concurrency()); auto clearNeighbours = [&](const int cellTopologyId) { - o2::itsmft::tracking::deepVectorClear(mTimeFrame->getCellsNeighbours()[cellTopologyId]); - o2::itsmft::tracking::deepVectorClear(mTimeFrame->getCellsNeighboursTopology()[cellTopologyId]); - o2::itsmft::tracking::deepVectorClear(mTimeFrame->getCellsNeighboursLUT()[cellTopologyId]); + deepVectorClear(mTimeFrame->getCellsNeighbours()[cellTopologyId]); + deepVectorClear(mTimeFrame->getCellsNeighboursTopology()[cellTopologyId]); + deepVectorClear(mTimeFrame->getCellsNeighboursLUT()[cellTopologyId]); }; if (maxConcurrency > 1) { tbb::parallel_for(0, static_cast(topology.nCells), clearNeighbours); @@ -476,7 +476,7 @@ void TrackerTraits::findCellsNeighbours(const int iteration) }; for (int outerLayer{0}; outerLayer < NLayers; ++outerLayer) { - o2::itsmft::tracking::bounded_vector activeTopologies(mMemoryPool.get()); + bounded_vector activeTopologies(mMemoryPool.get()); activeTopologies.reserve(topology.nCells); size_t sourceCellCount{0}; for (int cellTopologyId{0}; cellTopologyId < topology.nCells; ++cellTopologyId) { @@ -537,7 +537,7 @@ void TrackerTraits::findCellsNeighbours(const int iteration) } }; - o2::itsmft::tracking::bounded_vector waveNeighbours{mMemoryPool.get()}; + bounded_vector waveNeighbours{mMemoryPool.get()}; const auto key = CapacityEstimator::makeKey(SlabSite::Neighbours, iteration, 0, outerLayer); const auto scale = static_cast(sourceCellCount); if (maxConcurrency > 1) { @@ -572,7 +572,7 @@ void TrackerTraits::findCellsNeighbours(const int iteration) size_t begin; size_t end; }; - o2::itsmft::tracking::bounded_vector targetSpans{mMemoryPool.get()}; + bounded_vector targetSpans{mMemoryPool.get()}; targetSpans.reserve(topology.nCells); for (int targetTopologyId{0}; targetTopologyId < topology.nCells; ++targetTopologyId) { const auto first = std::lower_bound(waveNeighbours.begin(), waveNeighbours.end(), targetTopologyId, @@ -618,7 +618,7 @@ void TrackerTraits::findCellsNeighbours(const int iteration) // clean up LUTs auto clearCellLUT = [&](const int cellTopologyId) { - o2::itsmft::tracking::deepVectorClear(mTimeFrame->getCellsLookupTable()[cellTopologyId]); + deepVectorClear(mTimeFrame->getCellsLookupTable()[cellTopologyId]); }; if (maxConcurrency > 1) { tbb::parallel_for(0, static_cast(topology.nCells), clearCellLUT); @@ -632,7 +632,7 @@ void TrackerTraits::findCellsNeighbours(const int iteration) template template -void TrackerTraits::processNeighbours(int iteration, int defaultCellTopologyId, int iLevel, uint64_t capacityKey, const o2::itsmft::tracking::bounded_vector& currentSeeds, o2::itsmft::tracking::bounded_vector& updatedSeeds) +void TrackerTraits::processNeighbours(int iteration, int defaultCellTopologyId, int iLevel, uint64_t capacityKey, const bounded_vector& currentSeeds, bounded_vector& updatedSeeds) { constexpr bool IsInitial = std::is_same_v; static_assert(IsInitial || std::is_same_v); @@ -837,7 +837,7 @@ bool TrackerTraits::finaliseTrackSeed(const TrackSeedN& seed, template void TrackerTraits::findRoads(const int iteration) { - o2::itsmft::tracking::bounded_vector> firstClusters(mTrkParams[iteration].NLayers, o2::itsmft::tracking::bounded_vector(mMemoryPool.get()), mMemoryPool.get()); + bounded_vector> firstClusters(mTrkParams[iteration].NLayers, bounded_vector(mMemoryPool.get()), mMemoryPool.get()); firstClusters.resize(mTrkParams[iteration].NLayers); const auto propagator = o2::base::Propagator::Instance(); const TrackingFrameInfo* tfInfos[NLayers]{}; @@ -853,7 +853,7 @@ void TrackerTraits::findRoads(const int iteration) const track::TrackSeedSelector seedFilter{constants::MaxTrackSeedQ2Pt, mTrkParams[iteration].MaxChi2NDF, startLevel, mTrkParams[iteration].MaxHoles, mTrkParams[iteration].getMinSeedingClusters(), mTrkParams[iteration].HoleLayerMask, mTrkParams[iteration].getNonSeedingLayerMask()}; - o2::itsmft::tracking::bounded_vector trackSeeds(mMemoryPool.get()); + bounded_vector trackSeeds(mMemoryPool.get()); for (int startCellTopologyId{0}; startCellTopologyId < topology.nCells; ++startCellTopologyId) { const int startLayer = topology.getCell(startCellTopologyId).hitLayerMask.last(); if (!(mTrkParams[iteration].StartLayerMask.has(startLayer)) || @@ -862,7 +862,7 @@ void TrackerTraits::findRoads(const int iteration) continue; } - o2::itsmft::tracking::bounded_vector lastSeeds(mMemoryPool.get()), updatedSeeds(mMemoryPool.get()); + bounded_vector lastSeeds(mMemoryPool.get()), updatedSeeds(mMemoryPool.get()); auto roadKey = [&](int level) { return CapacityEstimator::makeKey(SlabSite::Roads, iteration, CapacityEstimator::makeVariant(startLevel, level), startCellTopologyId); @@ -873,11 +873,11 @@ void TrackerTraits::findRoads(const int iteration) int level = startLevel; while (level > 2 && !updatedSeeds.empty()) { lastSeeds.swap(updatedSeeds); - o2::itsmft::tracking::deepVectorClear(updatedSeeds); + deepVectorClear(updatedSeeds); --level; processNeighbours(iteration, constants::UnusedIndex, level, roadKey(level), lastSeeds, updatedSeeds); } - o2::itsmft::tracking::deepVectorClear(lastSeeds); + deepVectorClear(lastSeeds); if (!updatedSeeds.empty()) { trackSeeds.reserve(trackSeeds.size() + std::count_if(updatedSeeds.begin(), updatedSeeds.end(), [&](const auto& road) { return seedFilter(road.seed); })); @@ -912,7 +912,7 @@ void TrackerTraits::findRoads(const int iteration) std::max(1, mTrkParams[iteration].TrackFollowerMaxHypotheses), mTrkParams[iteration].TrackFollowerNSigmaCutPhi, mTrkParams[iteration].TrackFollowerNSigmaCutZ}; - o2::itsmft::tracking::bounded_vector tracks(mMemoryPool.get()); + bounded_vector tracks(mMemoryPool.get()); mTaskArena->execute([&] { const int nSeeds = static_cast(trackSeeds.size()); const int maxConcurrency = std::max(1, mTaskArena->max_concurrency()); @@ -920,7 +920,7 @@ void TrackerTraits::findRoads(const int iteration) // flush local track vector to global vector on reaching chunkSize std::mutex tracksMutex; - auto flushTracks = [&](o2::itsmft::tracking::bounded_vector& localTracks) { + auto flushTracks = [&](bounded_vector& localTracks) { if (localTracks.empty()) { return; } @@ -931,7 +931,7 @@ void TrackerTraits::findRoads(const int iteration) // each worker works on its own range tbb::parallel_for(tbb::blocked_range(0, nSeeds, chunkSize), [&](const auto& range) { - o2::itsmft::tracking::bounded_vector localTracks(mMemoryPool.get()); + bounded_vector localTracks(mMemoryPool.get()); localTracks.reserve(std::min(chunkSize, static_cast(range.size()))); auto& scratch = followerScratch.local(); for (int iSeed{range.begin()}; iSeed < range.end(); ++iSeed) { @@ -944,14 +944,14 @@ void TrackerTraits::findRoads(const int iteration) } } flushTracks(localTracks); // flush remaining - o2::itsmft::tracking::deepVectorClear(localTracks); + deepVectorClear(localTracks); }); - o2::itsmft::tracking::deepVectorClear(trackSeeds); + deepVectorClear(trackSeeds); }); // Sort tracks via indices to avoid moving TrackITSExt objects. - o2::itsmft::tracking::bounded_vector trackIndices(tracks.size(), mMemoryPool.get()); + bounded_vector trackIndices(tracks.size(), mMemoryPool.get()); std::iota(trackIndices.begin(), trackIndices.end(), 0); std::sort(trackIndices.begin(), trackIndices.end(), [&tracks](int a, int b) { return track::isBetter(tracks[a], tracks[b]); @@ -964,9 +964,9 @@ void TrackerTraits::findRoads(const int iteration) template void TrackerTraits::acceptTracks(int iteration, - o2::itsmft::tracking::bounded_vector& tracks, - const o2::itsmft::tracking::bounded_vector& trackIndices, - o2::itsmft::tracking::bounded_vector>& firstClusters) + bounded_vector& tracks, + const bounded_vector& trackIndices, + bounded_vector>& firstClusters) { auto& trks = mTimeFrame->getTracks(); trks.reserve(trks.size() + tracks.size()); @@ -1052,7 +1052,7 @@ void TrackerTraits::markTracks(int iteration) /// Now we have to set the shared cluster flag auto& tracks = mTimeFrame->getTracks(); - o2::itsmft::tracking::bounded_vector fclusSort(tracks.size(), mMemoryPool.get()); + bounded_vector fclusSort(tracks.size(), mMemoryPool.get()); std::iota(fclusSort.begin(), fclusSort.end(), 0); std::sort(fclusSort.begin(), fclusSort.end(), [&tracks](int a, int b) { return tracks[a].getFirstLayerClusterIndex() < tracks[b].getFirstLayerClusterIndex(); @@ -1114,16 +1114,16 @@ void TrackerTraits::setNThreads(int n, std::shared_ptr } template class TrackerTraits<7>; -template void TrackerTraits<7>::processNeighbours(int, int, int, uint64_t, const o2::itsmft::tracking::bounded_vector&, o2::itsmft::tracking::bounded_vector>&); -template void TrackerTraits<7>::processNeighbours>(int, int, int, uint64_t, const o2::itsmft::tracking::bounded_vector>&, o2::itsmft::tracking::bounded_vector>&); +template void TrackerTraits<7>::processNeighbours(int, int, int, uint64_t, const bounded_vector&, bounded_vector>&); +template void TrackerTraits<7>::processNeighbours>(int, int, int, uint64_t, const bounded_vector>&, bounded_vector>&); // ALICE3 upgrade #ifdef ENABLE_UPGRADES template class TrackerTraits<11>; -template void TrackerTraits<11>::processNeighbours(int, int, int, uint64_t, const o2::itsmft::tracking::bounded_vector&, o2::itsmft::tracking::bounded_vector>&); -template void TrackerTraits<11>::processNeighbours>(int, int, int, uint64_t, const o2::itsmft::tracking::bounded_vector>&, o2::itsmft::tracking::bounded_vector>&); +template void TrackerTraits<11>::processNeighbours(int, int, int, uint64_t, const bounded_vector&, bounded_vector>&); +template void TrackerTraits<11>::processNeighbours>(int, int, int, uint64_t, const bounded_vector>&, bounded_vector>&); template class TrackerTraits<13>; -template void TrackerTraits<13>::processNeighbours(int, int, int, uint64_t, const o2::itsmft::tracking::bounded_vector&, o2::itsmft::tracking::bounded_vector>&); -template void TrackerTraits<13>::processNeighbours>(int, int, int, uint64_t, const o2::itsmft::tracking::bounded_vector>&, o2::itsmft::tracking::bounded_vector>&); +template void TrackerTraits<13>::processNeighbours(int, int, int, uint64_t, const bounded_vector&, bounded_vector>&); +template void TrackerTraits<13>::processNeighbours>(int, int, int, uint64_t, const bounded_vector>&, bounded_vector>&); #endif } // namespace o2::its diff --git a/Detectors/ITSMFT/ITS/tracking/src/TrackingInterface.cxx b/Detectors/ITSMFT/ITS/tracking/src/TrackingInterface.cxx index cf896efba731f..a0e8d708cffa2 100644 --- a/Detectors/ITSMFT/ITS/tracking/src/TrackingInterface.cxx +++ b/Detectors/ITSMFT/ITS/tracking/src/TrackingInterface.cxx @@ -487,7 +487,7 @@ void ITSTrackingInterface::setTraitsFromProvider(VertexerTraitsN* vertexerTraits // set common memory resource if (!mMemoryPool) { - mMemoryPool = std::make_shared(); + mMemoryPool = std::make_shared(); } vertexerTraits->setMemoryPool(mMemoryPool); trackerTraits->setMemoryPool(mMemoryPool); diff --git a/Detectors/ITSMFT/ITS/tracking/src/Vertexer.cxx b/Detectors/ITSMFT/ITS/tracking/src/Vertexer.cxx index 4f6da24452258..356229bc201c5 100644 --- a/Detectors/ITSMFT/ITS/tracking/src/Vertexer.cxx +++ b/Detectors/ITSMFT/ITS/tracking/src/Vertexer.cxx @@ -86,7 +86,7 @@ float Vertexer::clustersToVertices(LogFunc logger) sortVertices(); } completed = true; - } catch (const o2::itsmft::tracking::BoundedMemoryResource::MemoryLimitExceeded& err) { + } catch (const BoundedMemoryResource::MemoryLimitExceeded& err) { handleException(err); } catch (const std::bad_alloc& err) { handleException(err); @@ -105,7 +105,7 @@ template void Vertexer::sortVertices() { auto& pvs = mTimeFrame->getPrimaryVertices(); - o2::itsmft::tracking::bounded_vector indices(pvs.size(), mMemoryPool.get()); + bounded_vector indices(pvs.size(), mMemoryPool.get()); std::iota(indices.begin(), indices.end(), 0); // provide vertices sorted by lower-bound std::sort(indices.begin(), indices.end(), [&pvs](size_t i, size_t j) { @@ -118,7 +118,7 @@ void Vertexer::sortVertices() } return pvs[i].getNContributors() > pvs[j].getNContributors(); }); - o2::itsmft::tracking::bounded_vector sortedVtx(mMemoryPool.get()); + bounded_vector sortedVtx(mMemoryPool.get()); sortedVtx.reserve(pvs.size()); for (const size_t idx : indices) { sortedVtx.push_back(pvs[idx]); @@ -126,7 +126,7 @@ void Vertexer::sortVertices() pvs.swap(sortedVtx); if (mTimeFrame->hasMCinformation()) { auto& mc = mTimeFrame->getPrimaryVerticesLabels(); - o2::itsmft::tracking::bounded_vector sortedMC(mMemoryPool.get()); + bounded_vector sortedMC(mMemoryPool.get()); for (const size_t idx : indices) { sortedMC.push_back(mc[idx]); } diff --git a/Detectors/ITSMFT/ITS/tracking/src/VertexerTraits.cxx b/Detectors/ITSMFT/ITS/tracking/src/VertexerTraits.cxx index 51cdce956f9d6..de39210cc74eb 100644 --- a/Detectors/ITSMFT/ITS/tracking/src/VertexerTraits.cxx +++ b/Detectors/ITSMFT/ITS/tracking/src/VertexerTraits.cxx @@ -34,6 +34,8 @@ namespace o2::its { +using o2::itsmft::tracking::deepVectorClear; + namespace { @@ -44,7 +46,7 @@ void trackleterKernelHost( const gsl::span& usedClustersNextLayer, // 0 2 const int* indexTableNext, const float phiCut, - o2::itsmft::tracking::bounded_vector& tracklets, + bounded_vector& tracklets, gsl::span foundTracklets, const IndexTableUtils& utils, const TimeEstBC& timErr, @@ -106,12 +108,12 @@ void trackletSelectionKernelHost( gsl::span usedClusters2, // global layer 2 used clusters const gsl::span& tracklets01, const gsl::span& tracklets12, - o2::itsmft::tracking::bounded_vector& usedTracklets, + bounded_vector& usedTracklets, const gsl::span foundTracklets01, const gsl::span foundTracklets12, - o2::itsmft::tracking::bounded_vector& lines, + bounded_vector& lines, const gsl::span& trackletLabels, - o2::itsmft::tracking::bounded_vector& linesLabels, + bounded_vector& linesLabels, const int nLayer1Clusters, const float tanLambdaCut, const float phiCut, @@ -306,7 +308,7 @@ void VertexerTraits::computeTrackletMatching(const int iteration) continue; } mTimeFrame->getLines(pivotRofId).reserve(std::min(mTimeFrame->getFoundTracklets(pivotRofId, 0).size(), mTimeFrame->getNTrackletsCluster(pivotRofId, 0).size() * constants::MaxSelectedTrackletsPerCluster)); - o2::itsmft::tracking::bounded_vector usedTracklets(mTimeFrame->getFoundTracklets(pivotRofId, 0).size(), 0, mMemoryPool.get()); + bounded_vector usedTracklets(mTimeFrame->getFoundTracklets(pivotRofId, 0).size(), 0, mMemoryPool.get()); trackletSelectionKernelHost( mTimeFrame->getClusters()[0].data(), mTimeFrame->getClusters()[1].data(), @@ -331,7 +333,7 @@ void VertexerTraits::computeTrackletMatching(const int iteration) }); // from here on we do not use tracklets anymore, so let's free them - o2::itsmft::tracking::deepVectorClear(mTimeFrame->getTracklets()); + deepVectorClear(mTimeFrame->getTracklets()); } template @@ -366,7 +368,7 @@ void VertexerTraits::computeVertices(const int iteration) } auto& lines = mTimeFrame->getLines(rofId); auto clusters = line_vertexer::buildClusters(std::span{lines.data(), lines.size()}, settings); - o2::itsmft::tracking::deepVectorClear(lines); // not needed after + deepVectorClear(lines); // not needed after auto clusterBeamDistance2 = [&](const ClusterLines& cluster) { return (mTimeFrame->getBeamX() - cluster.getVertex()[0]) * (mTimeFrame->getBeamX() - cluster.getVertex()[0]) + (mTimeFrame->getBeamY() - cluster.getVertex()[1]) * (mTimeFrame->getBeamY() - cluster.getVertex()[1]); @@ -392,7 +394,7 @@ void VertexerTraits::computeVertices(const int iteration) for (const auto& cluster : clusters) { minClusterZ = std::min(minClusterZ, cluster.getVertex()[2]); } - o2::itsmft::tracking::bounded_vector deduplicated(mMemoryPool.get()); + bounded_vector deduplicated(mMemoryPool.get()); deduplicated.reserve(clusters.size()); std::unordered_map> keptByZBin; for (auto& candidate : clusters) { @@ -541,7 +543,7 @@ void VertexerTraits::computeVertices(const int iteration) rofVertices[rofId].push_back(vertex); if (mTimeFrame->hasMCinformation()) { auto& lineLabels = mTimeFrame->getLinesLabel(rofId); - o2::itsmft::tracking::bounded_vector labels(mMemoryPool.get()); + bounded_vector labels(mMemoryPool.get()); for (auto& index : cluster.getLabels()) { labels.push_back(lineLabels[index]); } diff --git a/Detectors/ITSMFT/ITS/tracking/test/CMakeLists.txt b/Detectors/ITSMFT/ITS/tracking/test/CMakeLists.txt index 39d8d3c0fec63..b8858dda5ea92 100644 --- a/Detectors/ITSMFT/ITS/tracking/test/CMakeLists.txt +++ b/Detectors/ITSMFT/ITS/tracking/test/CMakeLists.txt @@ -9,12 +9,6 @@ # granted to it by virtue of its status as an Intergovernmental Organization # or submit itself to any jurisdiction. -o2_add_test(roflookuptables - SOURCES testROFLookupTables.cxx - COMPONENT_NAME its-tracking - LABELS "its;tracking" - PUBLIC_LINK_LIBRARIES O2::ITStracking) - o2_add_test(trackingtopology SOURCES testTrackingTopology.cxx COMPONENT_NAME its-tracking diff --git a/Detectors/ITSMFT/common/tracking/include/ITSMFTTracking/CapacityEstimator.h b/Detectors/ITSMFT/common/tracking/include/ITSMFTTracking/CapacityEstimator.h index d352613b397d1..c87162acbd3b3 100644 --- a/Detectors/ITSMFT/common/tracking/include/ITSMFTTracking/CapacityEstimator.h +++ b/Detectors/ITSMFT/common/tracking/include/ITSMFTTracking/CapacityEstimator.h @@ -38,6 +38,8 @@ enum SlabSite : uint8_t { }; constexpr const char* const SlabSiteNames[SlabSite::NSlabSite]{"Tracklets", "Cells", "Neighbours", "RoadCandidates", "Roads", "TrackSeeds", "TracksExtended", "Tracks"}; +struct SlabSinkStats; + class CapacityEstimator { public: @@ -68,8 +70,10 @@ class CapacityEstimator size_t granted{0}; size_t emitted{0}; size_t spilled{0}; + size_t maxEmitted{0}; uint32_t samples{0}; uint32_t overflowEvents{0}; + uint32_t nLowStreak{0}; }; static constexpr KeyType makeKey(SlabSite site, int iteration, int variant, int slot) noexcept @@ -121,12 +125,7 @@ class CapacityEstimator void update(uint64_t key, double scale, size_t emitted, size_t capacityUsed, bool overflowed, bool memoryLimited); void update(uint64_t key, double scale, size_t requested, size_t granted, size_t emitted, size_t spilled, bool overflowed, bool memoryLimited); - template - void update(uint64_t key, double scale, const Stats& stats) - { - update(key, scale, stats.requested, stats.capacity, stats.emitted, stats.spilled, - stats.overflowed, stats.memoryLimited); - } + void update(uint64_t key, double scale, const SlabSinkStats& stats); void print() const; private: diff --git a/Detectors/ITSMFT/common/tracking/include/ITSMFTTracking/SlabBumpAllocator.h b/Detectors/ITSMFT/common/tracking/include/ITSMFTTracking/SlabBumpAllocator.h index a7abd16c5115d..d55353938eee4 100644 --- a/Detectors/ITSMFT/common/tracking/include/ITSMFTTracking/SlabBumpAllocator.h +++ b/Detectors/ITSMFT/common/tracking/include/ITSMFTTracking/SlabBumpAllocator.h @@ -59,6 +59,13 @@ class ThreadLocalStorage using ParallelForBody = void (*)(void*, size_t, size_t); void parallelFor(size_t begin, size_t end, size_t grainSize, void* context, ParallelForBody body); +template +struct MoveContext { + T* staging; + int32_t* producerOf; + bounded_vector* destination; +}; + } // namespace detail class SlabBumpAllocator @@ -308,13 +315,9 @@ class SlabSink } deepVectorClear(cursor, mMR); - struct MoveContext { - T* staging; - int32_t* producerOf; - bounded_vector* destination; - } context{mStaging.data(), mProducerOf.data(), &dest}; + detail::MoveContext context{mStaging.data(), mProducerOf.data(), &dest}; detail::parallelFor(0, wm, 4096, &context, [](void* opaque, size_t begin, size_t end) { - auto& ctx = *static_cast(opaque); + auto& ctx = *static_cast*>(opaque); for (size_t s = begin; s != end; ++s) { const int d = ctx.producerOf[s]; if (d < 0) { diff --git a/Detectors/ITSMFT/common/tracking/src/BoundedAllocator.cxx b/Detectors/ITSMFT/common/tracking/src/BoundedAllocator.cxx index 83d196eb18573..152d3c8d7dd51 100644 --- a/Detectors/ITSMFT/common/tracking/src/BoundedAllocator.cxx +++ b/Detectors/ITSMFT/common/tracking/src/BoundedAllocator.cxx @@ -15,6 +15,7 @@ #include #include "GPUCommonLogger.h" +#include "ITSMFTTracking/Constants.h" namespace o2::itsmft::tracking { @@ -166,7 +167,6 @@ void BoundedMemoryResource::setMaxMemory(size_t max) std::string BoundedMemoryResource::asString() const { - constexpr double gigabyte = 1024. * 1024. * 1024.; const auto throwCount = mCountThrow.load(std::memory_order_relaxed); const auto used = static_cast(mUsedMemory.load(std::memory_order_relaxed)); const auto peak = static_cast(mPeakUsedMemory.load(std::memory_order_relaxed)); @@ -174,18 +174,18 @@ std::string BoundedMemoryResource::asString() const const auto maxMemory = mMaxMemory.load(std::memory_order_relaxed); std::string result; if (maxMemory == std::numeric_limits::max()) { - result += std::format("maxthrow={} maxmem=unbounded used={:.2f} GB stagepeak={:.2f} GB stagealloc={:.2f} GB", throwCount, used / gigabyte, peak / gigabyte, peakDelta / gigabyte); + result += std::format("maxthrow={} maxmem=unbounded used={:.2f} GB stagepeak={:.2f} GB stagealloc={:.2f} GB", throwCount, used / o2::its::constants::GB, peak / o2::its::constants::GB, peakDelta / o2::its::constants::GB); } else { - result += std::format("maxthrow={} maxmem={:.2f} GB used={:.2f} GB ({:.2f}%) stagepeak={:.2f} GB stagealloc={:.2f} GB", throwCount, static_cast(maxMemory) / gigabyte, used / gigabyte, 100.0 * used / static_cast(maxMemory), peak / gigabyte, peakDelta / gigabyte); + result += std::format("maxthrow={} maxmem={:.2f} GB used={:.2f} GB ({:.2f}%) stagepeak={:.2f} GB stagealloc={:.2f} GB", throwCount, static_cast(maxMemory) / o2::its::constants::GB, used / o2::its::constants::GB, 100.0 * used / static_cast(maxMemory), peak / o2::its::constants::GB, peakDelta / o2::its::constants::GB); } #ifdef BOUNDED_MR_STATS result += std::format(" peak={:.2f} GB live={} nAlloc={} nFree={} totalAlloc={:.2f} GB totalFreed={:.2f} GB maxAlign={} upstreamFail={}", - static_cast(mStats.peak.load(std::memory_order_relaxed)) / gigabyte, + static_cast(mStats.peak.load(std::memory_order_relaxed)) / o2::its::constants::GB, mStats.live.load(std::memory_order_relaxed), mStats.nAlloc.load(std::memory_order_relaxed), mStats.nFree.load(std::memory_order_relaxed), - static_cast(mStats.totalAlloc.load(std::memory_order_relaxed)) / gigabyte, - static_cast(mStats.totalFreed.load(std::memory_order_relaxed)) / gigabyte, + static_cast(mStats.totalAlloc.load(std::memory_order_relaxed)) / o2::its::constants::GB, + static_cast(mStats.totalFreed.load(std::memory_order_relaxed)) / o2::its::constants::GB, mStats.maxAlign.load(std::memory_order_relaxed), mStats.upstreamFailures.load(std::memory_order_relaxed)); #endif diff --git a/Detectors/ITSMFT/common/tracking/src/CapacityEstimator.cxx b/Detectors/ITSMFT/common/tracking/src/CapacityEstimator.cxx index ef3de7601f2f0..f6cdb03d075d0 100644 --- a/Detectors/ITSMFT/common/tracking/src/CapacityEstimator.cxx +++ b/Detectors/ITSMFT/common/tracking/src/CapacityEstimator.cxx @@ -22,6 +22,7 @@ #include #include "Framework/Logger.h" +#include "ITSMFTTracking/SlabBumpAllocator.h" namespace o2::itsmft::tracking { @@ -30,8 +31,6 @@ struct CapacityEstimator::Impl { struct Entry { float ratio{0.f}; float margin{0.f}; - size_t maxEmitted{0}; - uint32_t nLowStreak{0}; Statistics statistics{}; }; @@ -82,17 +81,17 @@ struct CapacityEstimator::Impl { } const auto sample = static_cast(double(emitted) / scale); e.ratio = firstSample ? sample : (cfg.alpha * sample) + ((1.f - cfg.alpha) * e.ratio); - e.maxEmitted = std::max(e.maxEmitted, emitted); + statistics.maxEmitted = std::max(statistics.maxEmitted, emitted); ++statistics.samples; if (memoryLimited) { - e.nLowStreak = 0; + statistics.nLowStreak = 0; e.margin = std::max(cfg.marginMin, e.margin * cfg.marginDown); return; } if (overflowed) { ++statistics.overflowEvents; - e.nLowStreak = 0; + statistics.nLowStreak = 0; if (!firstSample) { const float shortfall = granted ? static_cast(double(emitted) / double(granted)) : cfg.marginUp; e.margin = std::min(cfg.marginMax, e.margin * std::clamp(shortfall * cfg.marginOverflowSlack, 1.02f, cfg.marginUp)); @@ -101,12 +100,12 @@ struct CapacityEstimator::Impl { } const float util = granted ? float(double(emitted) / double(granted)) : 1.f; if (util < cfg.lowWatermark) { - if (++e.nLowStreak >= cfg.decayAfter) { + if (++statistics.nLowStreak >= cfg.decayAfter) { e.margin = std::max(cfg.marginMin, e.margin * cfg.marginDown); - e.nLowStreak = 0; + statistics.nLowStreak = 0; } - } else if (e.nLowStreak > 0) { - --e.nLowStreak; + } else if (statistics.nLowStreak > 0) { + --statistics.nLowStreak; } } }; @@ -180,7 +179,7 @@ size_t CapacityEstimator::capacity(uint64_t key, double scale) const // can be arbitrarily large, and applying it to a scale orders of magnitude bigger asks for a slab // nobody can allocate. Bound the request by what this site has ever actually emitted: overshooting // burns memory that a bump allocator cannot give back, undershooting only costs one retry. - const size_t ceiling = std::max(mImpl->cfg.floorSlots, static_cast(double(e.maxEmitted) * double(mImpl->cfg.marginMax))); + const size_t ceiling = std::max(mImpl->cfg.floorSlots, static_cast(double(e.statistics.maxEmitted) * double(mImpl->cfg.marginMax))); if (raw >= static_cast(ceiling)) { return ceiling; } @@ -191,11 +190,11 @@ size_t CapacityEstimator::peakCapacity(uint64_t key) const { std::lock_guard lock{mImpl->mutex}; const auto it = mImpl->entries.find(key); - if (it == mImpl->entries.end() || it->second.maxEmitted == 0) { + if (it == mImpl->entries.end() || it->second.statistics.maxEmitted == 0) { return mImpl->cfg.floorSlots; } const auto& e = it->second; - const double raw = double(e.maxEmitted) * double(e.margin); + const double raw = double(e.statistics.maxEmitted) * double(e.margin); if (!std::isfinite(raw) || raw >= static_cast(std::numeric_limits::max())) { return std::numeric_limits::max(); } @@ -247,6 +246,12 @@ void CapacityEstimator::update(uint64_t key, double scale, size_t requested, siz mImpl->observe(key, scale, requested, granted, emitted, spilled, overflowed, memoryLimited); } +void CapacityEstimator::update(uint64_t key, double scale, const SlabSinkStats& stats) +{ + update(key, scale, stats.requested, stats.capacity, stats.emitted, stats.spilled, + stats.overflowed, stats.memoryLimited); +} + void CapacityEstimator::print() const { std::lock_guard lock{mImpl->mutex}; @@ -269,7 +274,7 @@ void CapacityEstimator::print() const const auto& value = mImpl->entries.at(key); const auto& statistics = value.statistics; const auto decoded = decodeKey(key); - LOGP(info, "\tSite:{} | iter:{} | var:({},{}) | slot:{} | ratio:{} | margin:{} | maxEmitted:{} | samples:{} | low:{} | requested:{} | granted:{} | emitted:{} | spilled:{} | overflows:{}", SlabSiteNames[decoded.site], decoded.iteration, getVariantHigh(decoded.variant), getVariantLow(decoded.variant), decoded.slot, value.ratio, value.margin, value.maxEmitted, statistics.samples, value.nLowStreak, statistics.requested, statistics.granted, statistics.emitted, statistics.spilled, statistics.overflowEvents); + LOGP(info, "\tSite:{} | iter:{} | var:({},{}) | slot:{} | ratio:{} | margin:{} | maxEmitted:{} | samples:{} | low:{} | requested:{} | granted:{} | emitted:{} | spilled:{} | overflows:{}", SlabSiteNames[decoded.site], decoded.iteration, getVariantHigh(decoded.variant), getVariantLow(decoded.variant), decoded.slot, value.ratio, value.margin, statistics.maxEmitted, statistics.samples, statistics.nLowStreak, statistics.requested, statistics.granted, statistics.emitted, statistics.spilled, statistics.overflowEvents); } } diff --git a/Detectors/ITSMFT/common/tracking/test/CMakeLists.txt b/Detectors/ITSMFT/common/tracking/test/CMakeLists.txt index 102c00841e41f..e7f6d20e32773 100644 --- a/Detectors/ITSMFT/common/tracking/test/CMakeLists.txt +++ b/Detectors/ITSMFT/common/tracking/test/CMakeLists.txt @@ -9,6 +9,12 @@ # granted to it by virtue of its status as an Intergovernmental Organization # or submit itself to any jurisdiction. +o2_add_test(roflookuptables + SOURCES testROFLookupTables.cxx + COMPONENT_NAME itsmft-tracking + LABELS "itsmft;tracking" + PUBLIC_LINK_LIBRARIES O2::ITSMFTTracking) + o2_add_test(slabbumpallocator SOURCES testSlabBumpAllocator.cxx COMPONENT_NAME itsmft-tracking diff --git a/Detectors/ITSMFT/ITS/tracking/test/testROFLookupTables.cxx b/Detectors/ITSMFT/common/tracking/test/testROFLookupTables.cxx similarity index 99% rename from Detectors/ITSMFT/ITS/tracking/test/testROFLookupTables.cxx rename to Detectors/ITSMFT/common/tracking/test/testROFLookupTables.cxx index 0a4323a619a5f..486af25ee72cb 100644 --- a/Detectors/ITSMFT/ITS/tracking/test/testROFLookupTables.cxx +++ b/Detectors/ITSMFT/common/tracking/test/testROFLookupTables.cxx @@ -10,7 +10,7 @@ // or submit itself to any jurisdiction. #include -#define BOOST_TEST_MODULE ITS ROFLookupTables +#define BOOST_TEST_MODULE ITSMFT ROFLookupTables #define BOOST_TEST_MAIN #define BOOST_TEST_DYN_LINK diff --git a/Detectors/ITSMFT/common/tracking/test/testSlabBumpAllocator.cxx b/Detectors/ITSMFT/common/tracking/test/testSlabBumpAllocator.cxx index 1cbcd8feb427e..f12e1b3d2c1fd 100644 --- a/Detectors/ITSMFT/common/tracking/test/testSlabBumpAllocator.cxx +++ b/Detectors/ITSMFT/common/tracking/test/testSlabBumpAllocator.cxx @@ -129,8 +129,10 @@ void checkSnapshot(const EstimatorSnapshot& actual, const EstimatorSnapshot& exp BOOST_TEST(actual.statistics.granted == expected.statistics.granted); BOOST_TEST(actual.statistics.emitted == expected.statistics.emitted); BOOST_TEST(actual.statistics.spilled == expected.statistics.spilled); + BOOST_TEST(actual.statistics.maxEmitted == expected.statistics.maxEmitted); BOOST_TEST(actual.statistics.samples == expected.statistics.samples); BOOST_TEST(actual.statistics.overflowEvents == expected.statistics.overflowEvents); + BOOST_TEST(actual.statistics.nLowStreak == expected.statistics.nLowStreak); } void checkGrouped(int nProducers, size_t capacity, size_t slab, size_t maxMemory = std::numeric_limits::max()) @@ -620,8 +622,10 @@ BOOST_AUTO_TEST_CASE(estimator_updates_immediately_and_commit_retains_updates) BOOST_TEST(immediate.granted == 100u); BOOST_TEST(immediate.emitted == 95u); BOOST_TEST(immediate.spilled == 7u); + BOOST_TEST(immediate.maxEmitted == 95u); BOOST_TEST(immediate.samples == 1u); BOOST_TEST(immediate.overflowEvents == 1u); + BOOST_TEST(immediate.nLowStreak == 0u); est.beginTransaction(); est.update(key, 100., 80, 80, 70, 0, false, false); From 3df7d03cf7be6e5d060bb6a0af414fe7d0fdff63 Mon Sep 17 00:00:00 2001 From: Maximiliano Puccio Date: Fri, 28 Aug 2026 14:44:49 +0200 Subject: [PATCH 7/7] ITSMFT: decouple capacity estimator from slab sink statistics --- Detectors/ITSMFT/ITS/tracking/src/TrackerTraits.cxx | 12 ++++++++---- .../include/ITSMFTTracking/CapacityEstimator.h | 3 --- .../ITSMFT/common/tracking/src/CapacityEstimator.cxx | 7 ------- 3 files changed, 8 insertions(+), 14 deletions(-) diff --git a/Detectors/ITSMFT/ITS/tracking/src/TrackerTraits.cxx b/Detectors/ITSMFT/ITS/tracking/src/TrackerTraits.cxx index be1e6985f5ae6..f7caab856d8fd 100644 --- a/Detectors/ITSMFT/ITS/tracking/src/TrackerTraits.cxx +++ b/Detectors/ITSMFT/ITS/tracking/src/TrackerTraits.cxx @@ -205,7 +205,8 @@ void TrackerTraits::computeLayerTracklets(const int iteration, int iVer }); const auto st = sink.stats(); sink.finalizeUnordered(tracklets); - mTimeFrame->getCapacityEstimator().update(key, scale, st); + mTimeFrame->getCapacityEstimator().update(key, scale, st.requested, st.capacity, st.emitted, st.spilled, + st.overflowed, st.memoryLimited); }); } @@ -404,7 +405,8 @@ void TrackerTraits::computeLayerCells(const int iteration) }); const auto st = sink.stats(); sink.finalizeGrouped(size_t(currentLayerTrackletsNum), lut, layerCells); - mTimeFrame->getCapacityEstimator().update(key, scale, st); + mTimeFrame->getCapacityEstimator().update(key, scale, st.requested, st.capacity, st.emitted, st.spilled, + st.overflowed, st.memoryLimited); } else { lut.resize(currentLayerTrackletsNum + 1); for (int iTracklet{0}; iTracklet < currentLayerTrackletsNum; ++iTracklet) { @@ -554,7 +556,8 @@ void TrackerTraits::findCellsNeighbours(const int iteration) }); const auto st = sink.stats(); sink.finalizeUnordered(waveNeighbours); - mTimeFrame->getCapacityEstimator().update(key, scale, st); + mTimeFrame->getCapacityEstimator().update(key, scale, st.requested, st.capacity, st.emitted, st.spilled, + st.overflowed, st.memoryLimited); tbb::parallel_sort(waveNeighbours.begin(), waveNeighbours.end(), neighbourLess); } else { for (const int cellTopologyId : activeTopologies) { @@ -755,7 +758,8 @@ void TrackerTraits::processNeighbours(int iteration, int defaultCellTop }); const auto st = sink.stats(); sink.finalizeUnordered(updatedSeeds); - mTimeFrame->getCapacityEstimator().update(capacityKey, scale, st); + mTimeFrame->getCapacityEstimator().update(capacityKey, scale, st.requested, st.capacity, st.emitted, st.spilled, + st.overflowed, st.memoryLimited); } }); } diff --git a/Detectors/ITSMFT/common/tracking/include/ITSMFTTracking/CapacityEstimator.h b/Detectors/ITSMFT/common/tracking/include/ITSMFTTracking/CapacityEstimator.h index c87162acbd3b3..43b4e277fc290 100644 --- a/Detectors/ITSMFT/common/tracking/include/ITSMFTTracking/CapacityEstimator.h +++ b/Detectors/ITSMFT/common/tracking/include/ITSMFTTracking/CapacityEstimator.h @@ -38,8 +38,6 @@ enum SlabSite : uint8_t { }; constexpr const char* const SlabSiteNames[SlabSite::NSlabSite]{"Tracklets", "Cells", "Neighbours", "RoadCandidates", "Roads", "TrackSeeds", "TracksExtended", "Tracks"}; -struct SlabSinkStats; - class CapacityEstimator { public: @@ -125,7 +123,6 @@ class CapacityEstimator void update(uint64_t key, double scale, size_t emitted, size_t capacityUsed, bool overflowed, bool memoryLimited); void update(uint64_t key, double scale, size_t requested, size_t granted, size_t emitted, size_t spilled, bool overflowed, bool memoryLimited); - void update(uint64_t key, double scale, const SlabSinkStats& stats); void print() const; private: diff --git a/Detectors/ITSMFT/common/tracking/src/CapacityEstimator.cxx b/Detectors/ITSMFT/common/tracking/src/CapacityEstimator.cxx index f6cdb03d075d0..f54680ae9df71 100644 --- a/Detectors/ITSMFT/common/tracking/src/CapacityEstimator.cxx +++ b/Detectors/ITSMFT/common/tracking/src/CapacityEstimator.cxx @@ -22,7 +22,6 @@ #include #include "Framework/Logger.h" -#include "ITSMFTTracking/SlabBumpAllocator.h" namespace o2::itsmft::tracking { @@ -246,12 +245,6 @@ void CapacityEstimator::update(uint64_t key, double scale, size_t requested, siz mImpl->observe(key, scale, requested, granted, emitted, spilled, overflowed, memoryLimited); } -void CapacityEstimator::update(uint64_t key, double scale, const SlabSinkStats& stats) -{ - update(key, scale, stats.requested, stats.capacity, stats.emitted, stats.spilled, - stats.overflowed, stats.memoryLimited); -} - void CapacityEstimator::print() const { std::lock_guard lock{mImpl->mutex};