Skip to content

Commit 06b0b02

Browse files
cima22cima22
committed
ITS: new CPU + GPU seeding vertexer
Adds a seeding vertexer that runs as a prepended tracker pass (diamond trackleting -> cells -> lines -> parallel seeding), on both the CPU and GPU traits, replacing the per-ROF CPU vertexer for the seeding step.
1 parent a63c2a2 commit 06b0b02

22 files changed

Lines changed: 3025 additions & 73 deletions

‎Detectors/ITSMFT/ITS/tracking/GPU/ITStrackingGPU/ClusterLinesGPU.h‎

Lines changed: 194 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,194 @@
1+
// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
2+
// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
3+
// All rights not expressly granted are reserved.
4+
//
5+
// This software is distributed under the terms of the GNU General Public
6+
// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
7+
//
8+
// In applying this license CERN does not waive the privileges and immunities
9+
// granted to it by virtue of its status as an Intergovernmental Organization
10+
// or submit itself to any jurisdiction.
11+
12+
/// \file ClusterLinesGPU.h
13+
/// \brief device-side line + N-line vertex fit for the GPU seeding vertexer.
14+
15+
#ifndef O2_ITS_CLUSTERLINES_GPU_H
16+
#define O2_ITS_CLUSTERLINES_GPU_H
17+
18+
#include "DataFormatsITS/TimeEstBC.h"
19+
#include "GPUCommonDef.h"
20+
#include "GPUCommonMath.h"
21+
#include "ITStracking/LineProjection.h"
22+
23+
namespace o2::its::gpu
24+
{
25+
26+
using LineTime = o2::its::LineTime;
27+
using LineWindow = o2::its::LineWindow;
28+
29+
struct LineProjSoA {
30+
float* z{nullptr}; // projected z at the beamline; sort key and binary-search key, kept dense
31+
LineTime* t{nullptr}; // time centre + half-width
32+
int* idx{nullptr}; // sorted slot -> original line index
33+
int* rof{nullptr}; // ROF of the line
34+
};
35+
36+
struct VertexCand {
37+
float x, y, z;
38+
float rms2[6];
39+
float avgDist2;
40+
int nGood;
41+
float seed[3];
42+
o2::its::TimeEstBC time;
43+
int size;
44+
uint8_t ok; // 1 if the candidate passed the fit cuts
45+
uint8_t keep; // 1 if it survived duplicate suppression (subset of ok)
46+
uint8_t fine;
47+
};
48+
49+
// Device-side line: origin point + unit direction, with a time stamp
50+
struct GPULine {
51+
GPUhdDefault() GPULine() = default;
52+
53+
GPUhdi() GPULine(const float origin[3], const float direction[3], const o2::its::TimeEstBC& t) : mTime(t)
54+
{
55+
const float norm = o2::gpu::GPUCommonMath::Sqrt(direction[0] * direction[0] +
56+
direction[1] * direction[1] +
57+
direction[2] * direction[2]);
58+
const float inv = norm > 0.f ? 1.f / norm : 0.f;
59+
for (int i = 0; i < 3; ++i) {
60+
originPoint[i] = origin[i];
61+
cosinesDirector[i] = direction[i] * inv;
62+
}
63+
}
64+
65+
// Squared distance from a point to the (infinite) line: |delta - (delta.u) u|^2
66+
GPUhdi() static float getDistance2FromPoint(const GPULine& line, const float point[3])
67+
{
68+
float delta[3];
69+
float proj = 0.f;
70+
for (int i = 0; i < 3; ++i) {
71+
delta[i] = point[i] - line.originPoint[i];
72+
proj += delta[i] * line.cosinesDirector[i];
73+
}
74+
float d2 = 0.f;
75+
for (int i = 0; i < 3; ++i) {
76+
const float residual = delta[i] - proj * line.cosinesDirector[i];
77+
d2 += residual * residual;
78+
}
79+
return d2;
80+
}
81+
82+
GPUhdi() static void getDCAComponents(const GPULine& line, const float point[3], float out[6])
83+
{
84+
float delta[3];
85+
float proj = 0.f;
86+
for (int i = 0; i < 3; ++i) {
87+
delta[i] = line.originPoint[i] - point[i];
88+
proj += delta[i] * line.cosinesDirector[i];
89+
}
90+
float r[3];
91+
for (int i = 0; i < 3; ++i) {
92+
r[i] = delta[i] - proj * line.cosinesDirector[i];
93+
}
94+
out[0] = r[0]; // (0,0) XX
95+
out[1] = o2::gpu::GPUCommonMath::Hypot(r[0], r[1]); // (0,1) XY
96+
out[2] = r[1]; // (1,1) YY
97+
out[3] = o2::gpu::GPUCommonMath::Hypot(r[0], r[2]); // (0,2) XZ
98+
out[4] = o2::gpu::GPUCommonMath::Hypot(r[1], r[2]); // (1,2) YZ
99+
out[5] = r[2]; // (2,2) ZZ
100+
}
101+
102+
float originPoint[3] = {0.f, 0.f, 0.f};
103+
float cosinesDirector[3] = {0.f, 0.f, 0.f};
104+
o2::its::TimeEstBC mTime;
105+
};
106+
107+
class GPUClusterLinesFit
108+
{
109+
public:
110+
GPUhdDefault() GPUClusterLinesFit() = default;
111+
112+
// Add one line's contribution: A_ij += (delta_ij*|d|^2 - d_i*d_j)/|d|^2,
113+
// b_i += (d_i*(d.o) - |d|^2*o_i)/|d|^2. For a unit director |d|^2 == 1.
114+
GPUhdi() void add(const GPULine& line)
115+
{
116+
const double d0 = line.cosinesDirector[0], d1 = line.cosinesDirector[1], d2 = line.cosinesDirector[2];
117+
const double o0 = line.originPoint[0], o1 = line.originPoint[1], o2 = line.originPoint[2];
118+
const double det = d0 * d0 + d1 * d1 + d2 * d2; // == 1 for a normalised director
119+
if (det <= 0.) {
120+
return;
121+
}
122+
if (mNContributors <= 0) {
123+
mTime = line.mTime;
124+
} else {
125+
mTime += line.mTime;
126+
}
127+
mA[0] += (det - d0 * d0) / det;
128+
mA[1] += (-d0 * d1) / det;
129+
mA[2] += (-d0 * d2) / det;
130+
mA[3] += (det - d1 * d1) / det;
131+
mA[4] += (-d1 * d2) / det;
132+
mA[5] += (det - d2 * d2) / det;
133+
const double dDotO = d0 * o0 + d1 * o1 + d2 * o2;
134+
mB[0] += (d0 * dDotO - det * o0) / det;
135+
mB[1] += (d1 * dDotO - det * o1) / det;
136+
mB[2] += (d2 * dDotO - det * o2) / det;
137+
++mNContributors;
138+
}
139+
140+
// Solve the symmetric system and write the vertex (= -A^-1 B)
141+
GPUhdi() bool solve(float vertex[3]) const
142+
{
143+
const double a = mA[0], b = mA[1], c = mA[2], d = mA[3], e = mA[4], f = mA[5];
144+
const double c00 = d * f - e * e;
145+
const double c01 = c * e - b * f;
146+
const double c02 = b * e - c * d;
147+
const double c11 = a * f - c * c;
148+
const double c12 = b * c - a * e;
149+
const double c22 = a * d - b * b;
150+
const double det = a * c00 + b * c01 + c * c02;
151+
if (o2::gpu::GPUCommonMath::Abs(det) < 1.e-12) {
152+
return false;
153+
}
154+
const double invDet = 1. / det;
155+
const double x0 = (c00 * mB[0] + c01 * mB[1] + c02 * mB[2]) * invDet;
156+
const double x1 = (c01 * mB[0] + c11 * mB[1] + c12 * mB[2]) * invDet;
157+
const double x2 = (c02 * mB[0] + c12 * mB[1] + c22 * mB[2]) * invDet;
158+
vertex[0] = static_cast<float>(-x0);
159+
vertex[1] = static_cast<float>(-x1);
160+
vertex[2] = static_cast<float>(-x2);
161+
return true;
162+
}
163+
164+
GPUhdi() void addResidual(const GPULine& line, const float vertex[3])
165+
{
166+
float dca[6];
167+
GPULine::getDCAComponents(line, vertex, dca);
168+
const float d2 = GPULine::getDistance2FromPoint(line, vertex);
169+
++mResidualCount;
170+
const float inv = 1.f / static_cast<float>(mResidualCount);
171+
for (int i = 0; i < 6; ++i) {
172+
mRMS2[i] += (dca[i] - mRMS2[i]) * inv;
173+
}
174+
mAvgDistance2 += (d2 - mAvgDistance2) * inv;
175+
}
176+
177+
GPUhdi() int getNContributors() const { return mNContributors; }
178+
GPUhdi() const float* getRMS2() const { return mRMS2; } // Packed symmetric covariance in {XX, XY, YY, XZ, YZ, ZZ} order
179+
GPUhdi() float getAvgDistance2() const { return mAvgDistance2; }
180+
GPUhdi() const o2::its::TimeEstBC& getTimeStamp() const { return mTime; }
181+
182+
private:
183+
double mA[6] = {0., 0., 0., 0., 0., 0.};
184+
double mB[3] = {0., 0., 0.};
185+
int mNContributors = 0;
186+
float mRMS2[6] = {0.f, 0.f, 0.f, 0.f, 0.f, 0.f};
187+
float mAvgDistance2 = 0.f;
188+
int mResidualCount = 0;
189+
o2::its::TimeEstBC mTime;
190+
};
191+
192+
} // namespace o2::its::gpu
193+
194+
#endif /* O2_ITS_CLUSTERLINES_GPU_H */

‎Detectors/ITSMFT/ITS/tracking/GPU/ITStrackingGPU/TimeFrameGPU.h‎

Lines changed: 133 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
#include "ITStracking/Configuration.h"
2222
#include "ITStracking/TrackExtensionHypothesis.h"
2323
#include "ITStrackingGPU/Utils.h"
24+
#include "ITStrackingGPU/ClusterLinesGPU.h"
2425

2526
namespace o2::its::gpu
2627
{
@@ -54,10 +55,14 @@ class TimeFrameGPU : public TimeFrame<NLayers>
5455
void createTrackingFrameInfoDeviceArray(const int = NLayers);
5556
void loadUnsortedClustersDevice(const int);
5657
void createUnsortedClustersDeviceArray(const int = NLayers);
57-
void loadClustersDevice(const int);
5858
void createClustersDeviceArray(const int = NLayers);
5959
void loadClustersIndexTables(const int);
6060
void createClustersIndexTablesArray(const int = NLayers);
61+
void createClustersDevice(const int);
62+
void createClustersIndexTables(const int);
63+
void createClusterRadiiDevice();
64+
void uploadClusterRadii();
65+
void sortClustersDevice(const int layer, const TrackingParameters& trkParam);
6166
void createUsedClustersDevice(const int);
6267
void createUsedClustersDeviceArray(const int = NLayers);
6368
void loadUsedClustersDevice();
@@ -87,6 +92,35 @@ class TimeFrameGPU : public TimeFrame<NLayers>
8792
void createTrackExtensionScratchDevice(const int nThreads, const int maxHypotheses);
8893
void downloadTrackITSExtDevice();
8994

95+
// Seeding-vertexer
96+
void createClusterOwnersDeviceArray();
97+
void createClusterOwnersDevice();
98+
void resetClusterOwnersDevice();
99+
void createClusterSortScratchDevice(const int layer);
100+
101+
protected:
102+
void prepareClusters(const TrackingParameters& trkParam, const int maxLayers) override
103+
{
104+
if (maxLayers < NLayers) { // only if former seeding vertexer is run
105+
TimeFrame<NLayers>::prepareClusters(trkParam, maxLayers);
106+
}
107+
}
108+
void allocateClusterSortStorage(const TrackingParameters& trkParam, const int maxLayers) override
109+
{
110+
if (maxLayers < NLayers) { // only if former seeding vertexer is run
111+
TimeFrame<NLayers>::allocateClusterSortStorage(trkParam, maxLayers);
112+
}
113+
}
114+
115+
public:
116+
void createLinesDevice(const int nCells);
117+
void createDiamondDevice(const Vertex& diamond);
118+
unsigned int downloadLinesDevice();
119+
unsigned int getNLines();
120+
const auto& getHostLines() const { return mLinesHost; }
121+
const auto& getHostLineRof() const { return mLineRofHost; }
122+
const auto& getHostLineClusters() const { return mLineClustersHost; }
123+
90124
/// synchronization
91125
auto& getStream(const size_t stream) { return mGpuStreams[stream]; }
92126
auto& getStreams() { return mGpuStreams; }
@@ -111,6 +145,19 @@ class TimeFrameGPU : public TimeFrame<NLayers>
111145
auto& getTrackITSExt() { return mTrackITSExt; }
112146
auto& getTrackIndices() { return mTrackIndices; }
113147
Vertex* getDeviceVertices() { return mPrimaryVerticesDevice; }
148+
int* getDeviceROFramesClusters(const int layer) { return mROFramesClustersDevice[layer]; }
149+
int* getDeviceClusterSortKeys(const int layer) { return mClusterSortKeysDevice[layer]; }
150+
int* getDeviceClusterSortPerm(const int layer) { return mClusterSortPermDevice[layer]; }
151+
Cluster* getDeviceUnsortedClusters(const int layer) { return mUnsortedClustersDevice[layer]; }
152+
Cluster* getDeviceClusters(const int layer) { return mClustersDevice[layer]; }
153+
int* getDeviceClustersIndexTable(const int layer) { return mClustersIndexTablesDevice[layer]; }
154+
const float* getDeviceMinRs() const { return mClusterMinRDevice; }
155+
const float* getDeviceMaxRs() const { return mClusterMaxRDevice; }
156+
int* getDeviceROFramesPV() { return mROFramesPVDevice; }
157+
unsigned char* getDeviceUsedClusters(const int);
158+
const o2::base::Propagator* getChainPropagator();
159+
bool arePersistentTablesLoaded() { return mPersistentTablesLoaded; }
160+
void setPersistentTablesLoaded(bool setValue) { mPersistentTablesLoaded = setValue; }
114161

115162
// Hybrid
116163
TrackITSExt* getDeviceTrackITSExt() { return mTrackITSExtDevice; }
@@ -119,6 +166,45 @@ class TimeFrameGPU : public TimeFrame<NLayers>
119166
TrackExtensionHypothesis<NLayers>* getDeviceNextTrackExtensionHypotheses() { return mNextTrackExtensionHypothesesDevice; }
120167
int* getDeviceNeighboursLUT(const int layer) { return mNeighboursLUTDevice[layer]; }
121168
CellNeighbour** getDeviceArrayNeighbours() { return mNeighboursDeviceArray; }
169+
unsigned long long** getDeviceArrayClusterOwners() { return mClusterOwnersDeviceArray; }
170+
GPULine* getDeviceLines() { return mLinesDevice; }
171+
int* getDeviceLineSlots() { return mLineSlotsDevice; }
172+
int* getDeviceLineRof() { return mLineRofDevice; }
173+
int* getDeviceLineClusters() { return mLineClustersDevice; }
174+
float* getDeviceLineChi2() { return mLineChi2Device; }
175+
float* getDeviceLinePt() { return mLinePtDevice; }
176+
float* getDeviceLineZs() { return mLineZsDevice; }
177+
gpu::LineTime* getDeviceLineTimes() { return mLineTimesDevice; }
178+
int* getDeviceLineSortedIdx() { return mLinesSortedIdx; }
179+
LineProjSoA getLineProjSoA() { return {mLineZsDevice, mLineTimesDevice, mLinesSortedIdx, mLineRofDevice}; }
180+
LineProjSoA getLineProjSortedSoA() { return {mLineZsSortedDevice, mLineTimesSortedDevice, mLinesSortedIdx, mLineRofSortedDevice}; }
181+
int* getDeviceRofLineOffsets() { return mRofLineOffsetsDevice; }
182+
int* getDeviceLineDensity() { return mLineDensityDevice; }
183+
gpu::LineWindow* getDeviceLineWin() { return mLineWinDevice; }
184+
uint8_t* getDeviceLineIsPeak() { return mLineIsPeakDevice; }
185+
int* getDeviceLineDensityFine() { return mLineDensityFineDevice; }
186+
gpu::LineWindow* getDeviceLineWinFine() { return mLineWinFineDevice; }
187+
uint8_t* getDeviceLineIsPeakFine() { return mLineIsPeakFineDevice; }
188+
int* getDevicePeakScan() { return mPeakScanDevice; }
189+
int* getDevicePeakLineIdx() { return mPeakLineIdxDevice; }
190+
int* getDevicePeakOffsets() { return mPeakOffsetsDevice; }
191+
const int* getDeviceNPeaks() { return mPeakOffsetsDevice + this->getNrof(1); }
192+
VertexCand* getDeviceVertexCands() { return mVertexCandsDevice; }
193+
int* getDeviceMemberOffsets() { return mMemberOffsetsDevice; }
194+
int* getDeviceMemberLines() { return mMemberLinesDevice; }
195+
int downloadVertexCandsDevice();
196+
int getNMembers() const { return mNMembers; }
197+
void downloadMemberOffsetsDevice(); // (MC only)
198+
void createMemberLinesMCDevice(const int nMembers); // (MC only)
199+
void downloadMemberLinesDevice(); // (MC only)
200+
const auto& getHostVertexCands() const { return mVertexCandsHost; }
201+
const auto& getHostPeakOffsets() const { return mPeakOffsetsHost; }
202+
const auto& getHostMemberOffsets() const { return mMemberOffsetsHost; }
203+
const auto& getHostMemberLines() const { return mMemberLinesHost; }
204+
std::vector<o2::MCCompLabel>& getLineLabelFlat() { return mLineLabelFlatHost; }
205+
const std::vector<o2::MCCompLabel>& getLineLabelFlat() const { return mLineLabelFlatHost; }
206+
Vertex* getDeviceDiamond() { return mDiamondDevice; }
207+
std::array<CellNeighbour*, MaxCells>& getDeviceNeighboursAll() { return mNeighboursDevice; }
122208
CellNeighbour* getDeviceNeighbours(const int layer) { return mNeighboursDevice[layer]; }
123209
const TrackingFrameInfo** getDeviceArrayTrackingFrameInfo() const { return mTrackingFrameInfoDeviceArray; }
124210
const Cluster** getDeviceArrayClusters() const { return mClustersDeviceArray; }
@@ -215,6 +301,11 @@ class TimeFrameGPU : public TimeFrame<NLayers>
215301
const int** mClustersIndexTablesDeviceArray{nullptr};
216302
uint8_t** mUsedClustersDeviceArray{nullptr};
217303
const int** mROFramesClustersDeviceArray{nullptr};
304+
int* mROFramesPVDevice;
305+
std::array<int*, NLayers> mClusterSortKeysDevice{};
306+
std::array<int*, NLayers> mClusterSortPermDevice{};
307+
float* mClusterMinRDevice{nullptr};
308+
float* mClusterMaxRDevice{nullptr};
218309
std::array<Tracklet*, MaxLinks> mTrackletsDevice{};
219310
std::array<int*, MaxLinks> mTrackletsLUTDevice{};
220311
std::array<int*, MaxCells> mCellsLUTDevice{};
@@ -239,6 +330,47 @@ class TimeFrameGPU : public TimeFrame<NLayers>
239330
CellNeighbour** mNeighboursDeviceArray{nullptr};
240331
std::array<TrackingFrameInfo*, NLayers> mTrackingFrameInfoDevice{};
241332
const TrackingFrameInfo** mTrackingFrameInfoDeviceArray{nullptr};
333+
std::array<unsigned long long*, 3> mClusterOwnersDevice{};
334+
unsigned long long** mClusterOwnersDeviceArray{nullptr};
335+
int* mLineSlotsDevice{nullptr};
336+
GPULine* mLinesDevice{nullptr};
337+
int* mLineRofDevice{nullptr};
338+
int* mLineClustersDevice{nullptr};
339+
float* mLineChi2Device{nullptr};
340+
float* mLinePtDevice{nullptr};
341+
float* mLineZsDevice{nullptr};
342+
gpu::LineTime* mLineTimesDevice{nullptr};
343+
float* mLineZsSortedDevice{nullptr};
344+
gpu::LineTime* mLineTimesSortedDevice{nullptr};
345+
int* mLinesSortedIdx{nullptr};
346+
int* mLineRofSortedDevice{nullptr}; // per (sorted) line's ROF
347+
int* mRofLineOffsetsDevice{nullptr}; // CSR offsets into the (rof,z)-sorted lines, size nRofs+1
348+
int* mLineDensityDevice{nullptr}; // per (sorted) line: count of time-compatible neighbours in its z-window
349+
gpu::LineWindow* mLineWinDevice{nullptr}; // per (sorted) line: [lo,hi) bounds of its z-window (sorted coords)
350+
uint8_t* mLineIsPeakDevice{nullptr}; // per (sorted) line: 1 if it is a local density peak (vertex candidate)
351+
int* mLineDensityFineDevice{nullptr};
352+
gpu::LineWindow* mLineWinFineDevice{nullptr};
353+
uint8_t* mLineIsPeakFineDevice{nullptr};
354+
int* mPeakScanDevice{nullptr}; // per (sorted) line: number of peaks strictly before it
355+
int* mPeakLineIdxDevice{nullptr}; // per peak slot: the sorted line index it came from
356+
int* mPeakOffsetsDevice{nullptr}; // CSR offsets into the compacted peaks
357+
VertexCand* mVertexCandsDevice{nullptr};
358+
int* mMemberOffsetsDevice{nullptr};
359+
int* mMemberLinesDevice{nullptr};
360+
int mNLinesCapacity{0}; // = nCells the line buffers were sized for
361+
std::vector<GPULine> mLinesHost;
362+
std::vector<int> mLineRofHost;
363+
std::vector<int> mLineClustersHost;
364+
std::vector<VertexCand> mVertexCandsHost;
365+
std::vector<int> mPeakOffsetsHost;
366+
std::vector<int> mMemberOffsetsHost;
367+
std::vector<int> mMemberLinesHost;
368+
std::vector<o2::MCCompLabel> mLineLabelFlatHost;
369+
int mNMembers{0};
370+
Vertex* mDiamondDevice{nullptr};
371+
bool mPersistentTablesLoaded{false};
372+
std::bitset<NLayers> mUnsortedClustersUploaded{};
373+
std::bitset<NLayers> mTrackingFrameInfoUploaded{};
242374

243375
// State
244376
Streams mGpuStreams;

‎Detectors/ITSMFT/ITS/tracking/GPU/ITStrackingGPU/TrackerTraitsGPU.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,9 @@ class TrackerTraitsGPU final : public TrackerTraits<NLayers>
2929
void adoptTimeFrame(TimeFrame<NLayers>* tf) final;
3030
void initialiseTimeFrame(const int iteration) final;
3131

32+
void computeVertexCandidates(const int iteration) final;
33+
void computeVertices(const int iteration) final;
34+
3235
void computeLayerTracklets(const int iteration, int) final;
3336
void computeLayerCells(const int iteration) final;
3437
void findCellsNeighbours(const int iteration) final;

0 commit comments

Comments
 (0)