Skip to content

Bugfix/distributed hdbscan stability #18

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
graceli24 merged 4 commits into from
Sep 26, 2026
Merged

Bugfix/distributed hdbscan stability #18

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
f614995
Fix bug where lambda_min_edge was being incorrectly counted in sum_la…
graceli24 Sep 26, 2026
b9faecd
Finalize changing edge contraction and endpoint YGM maps to YGM array…
graceli24 Sep 26, 2026
c314c01
Fix name for edge_contraction_array
graceli24 Sep 26, 2026
6ed9207
Remove prescision printouts
graceli24 Sep 26, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 19 additions & 16 deletions src/clustering/distributed_clustering_build_hierarchy.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,13 +25,13 @@ namespace clams::clustering {
* contraction map so we can clear it and save space
*
* @param alpha_edge_map An empty YGM map of alpha edge id -> alpha edge info.
* @param edge_contraction_map YGM map of edge id -> edge contraction info.
* @param edge_contraction_array YGM array of edge id -> edge contraction info.
* @param root_chain_supernode Supernode name for the root chain.
*/
void fill_alpha_edge_map(
ygm::container::map<id_t, alpha_edge_info> &alpha_edge_map,
ygm::container::map<id_t, edge_contraction_info> &edge_contraction_map,
const supernode_t &root_chain_supernode) {
ygm::container::map<id_t, alpha_edge_info> &alpha_edge_map,
ygm::container::array<edge_contraction_info> &edge_contraction_array,
const supernode_t &root_chain_supernode) {
ygm::comm &comm = alpha_edge_map.comm();
auto set_alpha_edge_info = [&alpha_edge_map, &root_chain_supernode](
const id_t &edge_id,
Expand All @@ -51,15 +51,15 @@ void fill_alpha_edge_map(
chain_supernode);
}
};
edge_contraction_map.for_all(set_alpha_edge_info);
edge_contraction_array.for_all(set_alpha_edge_info);
comm.barrier();
}

/**
* @brief Goes through the edge contraction map and assigns all edges to chains,
* represented by their supernode.
*
* @param edge_contraction_map YGM map of edge id -> edge contraction info.
* @param edge_contraction_array YGM array of edge id -> edge contraction info.
* @param chain_map An empty YGM map of chain name (supernode) -> pair of
* (cluster map, chain info). The cluster map goes from edge id -> full cluster
* info, and the chain info stores the required chain info that doesn't go in
Expand All @@ -75,15 +75,15 @@ void fill_alpha_edge_map(
* tell which edges belong to the root chain.
*/
void assign_edges_to_chains(
ygm::container::map<id_t, edge_contraction_info> &edge_contraction_map,
ygm::container::array<edge_contraction_info> &edge_contraction_array,
ygm::container::map<supernode_t,
std::pair<std::map<id_t, full_cluster_info>,
full_chain_info>> &chain_map,
full_chain_info>> &chain_map,
std::vector<edge_id_with_dist_t> &local_root_chain_alpha_edges,
ygm::container::set<edge_id_with_dist_t> &root_chain_non_alpha_edge_set,
ygm::container::map<supernode_t, full_leaf_cluster_info> &leaf_cluster_map,
const uint32_t final_round) {
ygm::comm &comm = edge_contraction_map.comm();
ygm::comm &comm = edge_contraction_array.comm();

static auto add_alpha_edge_to_chain_lambda =
[]([[maybe_unused]] const supernode_t &chain_name,
Expand Down Expand Up @@ -162,7 +162,7 @@ void assign_edges_to_chains(
}
}
};
edge_contraction_map.for_all(assign_edge_to_chain_lambda);
edge_contraction_array.for_all(assign_edge_to_chain_lambda);
comm.barrier();
}

Expand Down Expand Up @@ -710,10 +710,11 @@ supernode_t fill_missing_root_chain_cluster_info(
full_root_chain_cluster_array_ptr) {
// Calculate num points added and sum lambda edges added for this
// cluster
id_t num_points_added = edges.size() - 1;
distance_t sum_lambda_edges_added = 0.0;
id_t num_points_added = edges.size() - 1;
distance_t sum_lambda_edges_added_including_min_edge = 0.0;
for (auto &edge : edges) {
sum_lambda_edges_added += lambda_from_dist(edge.second);
sum_lambda_edges_added_including_min_edge +=
lambda_from_dist(edge.second);
}

// Set the values in the cluster array
Expand All @@ -722,12 +723,14 @@ supernode_t fill_missing_root_chain_cluster_info(
[](const id_t &index,
std::pair<id_t, root_chain_cluster_info> &value,
const id_t &num_points_added,
const distance_t &sum_lambda_edges_added) {
const distance_t
&sum_lambda_edges_added_including_min_edge) {
value.second.num_points_added = num_points_added;
value.second.sum_lambda_edges_added =
sum_lambda_edges_added;
sum_lambda_edges_added_including_min_edge -
value.second.lambda_min_edge;
},
num_points_added, sum_lambda_edges_added);
num_points_added, sum_lambda_edges_added_including_min_edge);
};

root_chain_cluster_edges_map.async_visit(
Expand Down
70 changes: 36 additions & 34 deletions src/clustering/distributed_clustering_hdbscan.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1341,11 +1341,13 @@ calculate_root_chain_size_stability(
for (auto it = root_chain_array.local_begin();
it != root_chain_array.local_end(); ++it) {
root_chain_cluster_info cluster_info = it->value.second;
float stability = cluster_info.lambda_min_edge *
(cluster_info.size - cluster_info.num_points_added) +
cluster_info.sum_lambda_edges_added -
cluster_info.lambda_birth * cluster_info.size;
float stability =
(cluster_info.lambda_min_edge - cluster_info.lambda_birth) *
(cluster_info.size - cluster_info.num_points_added) +
cluster_info.sum_lambda_edges_added -
(cluster_info.lambda_birth * cluster_info.num_points_added);
it->value.second.stability = stability;

if (it->value.second.stability >=
it->value.second.stability_traversing_up) {
local_possible_clusters_for_selection.push_back(it->value);
Expand Down Expand Up @@ -1717,7 +1719,7 @@ std::vector<cluster_name_t> traverse_down_hierarchy_and_select_clusters(
* @param root_chain_cluster_edges_map YGM map of root chain cluster edge id ->
* vector of edges added to this cluster.
* @param point_to_cluster_id_map An empty YGM map of point id -> cluster label.
* @param edge_endpoints_map YGM map of edge id -> pair of point ids of its
* @param edge_endpoints_array YGM array of edge id -> pair of point ids of its
* endpoints.
* @param local_selected_clusters Vector of selected clusters to process on this
* rank.
Expand All @@ -1735,23 +1737,23 @@ void assign_points_cluster_ids(
ygm::container::map<supernode_t, full_leaf_cluster_info> &leaf_cluster_map,
ygm::container::map<id_t, root_chain_cluster_info> &root_chain_cluster_map,
ygm::container::map<id_t, std::vector<edge_id_with_dist_t>>
&root_chain_cluster_edges_map,
ygm::container::map<id_t, cluster_id_t> &point_to_cluster_id_map,
ygm::container::map<id_t, std::pair<id_t, id_t>> &edge_endpoints_map,
const std::vector<cluster_name_t> &local_selected_clusters,
&root_chain_cluster_edges_map,
ygm::container::map<id_t, cluster_id_t> &point_to_cluster_id_map,
ygm::container::array<std::pair<id_t, id_t>> &edge_endpoints_array,
const std::vector<cluster_name_t> &local_selected_clusters,
const cluster_id_t start_cluster_id, const supernode_t root_chain_supernode,
const supernode_t root_chain_second_child_name) {
ygm::comm &comm = chain_map.comm();
auto edge_endpoints_map_ptr = edge_endpoints_map.get_ygm_ptr();
auto point_to_cluster_id_map_ptr = point_to_cluster_id_map.get_ygm_ptr();
auto leaf_cluster_map_ptr = leaf_cluster_map.get_ygm_ptr();
auto chain_map_ptr = chain_map.get_ygm_ptr();
ygm::comm &comm = chain_map.comm();
auto edge_endpoints_array_ptr = edge_endpoints_array.get_ygm_ptr();
auto point_to_cluster_id_map_ptr = point_to_cluster_id_map.get_ygm_ptr();
auto leaf_cluster_map_ptr = leaf_cluster_map.get_ygm_ptr();
auto chain_map_ptr = chain_map.get_ygm_ptr();
auto root_chain_cluster_edges_map_ptr =
root_chain_cluster_edges_map.get_ygm_ptr();

// All points in the data set start out as noise points and when we
// pick clusters, this will be over written with the cluster id
edge_endpoints_map.for_all(
edge_endpoints_array.for_all(
[&point_to_cluster_id_map]([[maybe_unused]] const id_t &edge_id,
const std::pair<id_t, id_t> &edge_endpoints) {
point_to_cluster_id_map.async_insert(edge_endpoints.first,
Expand Down Expand Up @@ -1783,10 +1785,10 @@ void assign_points_cluster_ids(
static auto label_leaf_cluster_edge_endpoints_lambda =
[](const supernode_t &cluster_name,
const full_leaf_cluster_info &cluster_info,
const cluster_id_t &cluster_id, auto edge_endpoints_map_ptr,
const cluster_id_t &cluster_id, auto edge_endpoints_array_ptr,
auto point_to_cluster_id_map_ptr) {
for (const edge_id_with_dist_t &edge : cluster_info.edges) {
edge_endpoints_map_ptr->async_visit(
edge_endpoints_array_ptr->async_visit(
edge.first, label_edge_endpoints_lambda, cluster_id,
point_to_cluster_id_map_ptr);
}
Expand All @@ -1796,10 +1798,10 @@ void assign_points_cluster_ids(
// points with the provided cluster id
static auto label_root_chain_cluster_edge_endpoints_lambda =
[](const id_t &cluster_edge_id, std::vector<edge_id_with_dist_t> &edges,
const cluster_id_t &cluster_id, auto edge_endpoints_map_ptr,
const cluster_id_t &cluster_id, auto edge_endpoints_array_ptr,
auto point_to_cluster_id_map_ptr) {
for (const edge_id_with_dist_t &edge : edges) {
edge_endpoints_map_ptr->async_visit(
edge_endpoints_array_ptr->async_visit(
edge.first, label_edge_endpoints_lambda, cluster_id,
point_to_cluster_id_map_ptr);
}
Expand All @@ -1820,8 +1822,8 @@ void assign_points_cluster_ids(
const id_t &starting_cluster_edge_id, const cluster_id_t &cluster_id,
ygm::ygm_ptr<ygm::container::map<supernode_t, full_leaf_cluster_info>>
leaf_cluster_map_ptr,
ygm::ygm_ptr<ygm::container::map<id_t, std::pair<id_t, id_t>>>
edge_endpoints_map_ptr,
ygm::ygm_ptr<ygm::container::array<std::pair<id_t, id_t>>>
edge_endpoints_array_ptr,
ygm::ygm_ptr<ygm::container::map<id_t, cluster_id_t>>
point_to_cluster_id_map_ptr) {
// For all clusters in the chain below or equal to the provided starting
Expand All @@ -1838,7 +1840,7 @@ void assign_points_cluster_ids(

// Label each edge in the cluster with the provided label
for (const edge_id_with_dist_t &edge : cluster_info.edges) {
edge_endpoints_map_ptr->async_visit(
edge_endpoints_array_ptr->async_visit(
edge.first, label_edge_endpoints_lambda, cluster_id,
point_to_cluster_id_map_ptr);
}
Expand All @@ -1848,13 +1850,13 @@ void assign_points_cluster_ids(
if (cluster_info.child.second == 1) {
leaf_cluster_map_ptr->async_visit(
cluster_info.child, label_leaf_cluster_edge_endpoints_lambda,
cluster_id, edge_endpoints_map_ptr,
cluster_id, edge_endpoints_array_ptr,
point_to_cluster_id_map_ptr);
} else {
chain_map_ptr->async_visit(
cluster_info.child, label_points_functor(),
std::numeric_limits<id_t>::max(), cluster_id,
leaf_cluster_map_ptr, edge_endpoints_map_ptr,
leaf_cluster_map_ptr, edge_endpoints_array_ptr,
point_to_cluster_id_map_ptr);
}
}
Expand All @@ -1864,12 +1866,12 @@ void assign_points_cluster_ids(
if (chain.second.children[1].second == 1) {
leaf_cluster_map_ptr->async_visit(
chain.second.children[1], label_leaf_cluster_edge_endpoints_lambda,
cluster_id, edge_endpoints_map_ptr, point_to_cluster_id_map_ptr);
cluster_id, edge_endpoints_array_ptr, point_to_cluster_id_map_ptr);
} else {
chain_map_ptr->async_visit(
chain.second.children[1], label_points_functor(),
std::numeric_limits<id_t>::max(), cluster_id, leaf_cluster_map_ptr,
edge_endpoints_map_ptr, point_to_cluster_id_map_ptr);
edge_endpoints_array_ptr, point_to_cluster_id_map_ptr);
}
}
};
Expand All @@ -1887,7 +1889,7 @@ void assign_points_cluster_ids(
if (cluster_name.first.second == 1) {
leaf_cluster_map.async_visit(
cluster_name.first, label_leaf_cluster_edge_endpoints_lambda,
cluster_id, edge_endpoints_map_ptr, point_to_cluster_id_map_ptr);
cluster_id, edge_endpoints_array_ptr, point_to_cluster_id_map_ptr);
}
// If the selected cluster is in the root chain (there will be at most 1),
// then store its edge id and cluster id for processing separately
Expand All @@ -1900,7 +1902,7 @@ void assign_points_cluster_ids(
else {
chain_map.async_visit(cluster_name.first, label_points_functor(),
cluster_name.second, cluster_id,
leaf_cluster_map_ptr, edge_endpoints_map_ptr,
leaf_cluster_map_ptr, edge_endpoints_array_ptr,
point_to_cluster_id_map_ptr);
}
}
Expand All @@ -1916,7 +1918,7 @@ void assign_points_cluster_ids(
auto label_root_chain_cluster_lambda =
[selected_root_chain_cluster_edge_id, selected_root_chain_cluster_id,
chain_map_ptr, leaf_cluster_map_ptr, root_chain_cluster_edges_map_ptr,
edge_endpoints_map_ptr,
edge_endpoints_array_ptr,
point_to_cluster_id_map_ptr](const id_t &cluster_edge_id,
root_chain_cluster_info &cluster_info) {
// If this root chain cluster is the selected cluster or below it,
Expand All @@ -1928,15 +1930,15 @@ void assign_points_cluster_ids(
// Label this cluster's edges added
root_chain_cluster_edges_map_ptr->async_visit(
cluster_edge_id, label_root_chain_cluster_edge_endpoints_lambda,
selected_root_chain_cluster_id, edge_endpoints_map_ptr,
selected_root_chain_cluster_id, edge_endpoints_array_ptr,
point_to_cluster_id_map_ptr);

// Visit this cluster's non-root-chain child and label it
// If its a leaf cluster, just label it directly
if (cluster_info.child.second == 1) {
leaf_cluster_map_ptr->async_visit(
cluster_info.child, label_leaf_cluster_edge_endpoints_lambda,
selected_root_chain_cluster_id, edge_endpoints_map_ptr,
selected_root_chain_cluster_id, edge_endpoints_array_ptr,
point_to_cluster_id_map_ptr);
}
// If its a chain, go visit it with the label points functor
Expand All @@ -1945,7 +1947,7 @@ void assign_points_cluster_ids(
cluster_info.child, label_points_functor(),
std::numeric_limits<id_t>::max(),
selected_root_chain_cluster_id, leaf_cluster_map_ptr,
edge_endpoints_map_ptr, point_to_cluster_id_map_ptr);
edge_endpoints_array_ptr, point_to_cluster_id_map_ptr);
}
}
};
Expand All @@ -1956,13 +1958,13 @@ void assign_points_cluster_ids(
leaf_cluster_map.async_visit(root_chain_second_child_name,
label_leaf_cluster_edge_endpoints_lambda,
selected_root_chain_cluster_id,
edge_endpoints_map_ptr,
edge_endpoints_array_ptr,
point_to_cluster_id_map_ptr);
} else {
chain_map.async_visit(
root_chain_second_child_name, label_points_functor(),
std::numeric_limits<id_t>::max(), selected_root_chain_cluster_id,
leaf_cluster_map_ptr, edge_endpoints_map_ptr,
leaf_cluster_map_ptr, edge_endpoints_array_ptr,
point_to_cluster_id_map_ptr);
}
}
Expand Down
Loading
Loading