Skip to content

[HOLD] ibv: remove several 16-bit node number limitations #8

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
PHHargrove wants to merge 1 commit into
base: develop
Choose a base branch
from
Draft

[HOLD] ibv: remove several 16-bit node number limitations #8

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions ibv-conduit/gasnet_core.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2063,8 +2063,8 @@ static int gasnetc_init( gex_Client_t *client_p,
// Ensure work-arounds like MLX5_SCATTER_TO_CQE are propogated
gasneti_propagate_env("MLX5_", GASNETI_PROPAGATE_ENV_PREFIX);

/* bootstrapInit may set gasneti_nodes==0 if would overflow 16-bit field */
if (!gasneti_nodes || (gasneti_nodes > GASNET_MAXNODES)) {
// Node count from bootstrapInit might exceed GASNET_MAXNODES
if (gasneti_nodes > GASNET_MAXNODES) {
GASNETI_RETURN_ERRR(RESOURCE, "job size exceeds ibv-conduit capabilities");
}

Expand Down
17 changes: 7 additions & 10 deletions ibv-conduit/gasnet_core_connect.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -192,11 +192,7 @@ typedef struct gasnetc_xrc_snd_qp_s {
} gasnetc_xrc_snd_qp_t;

static gasnetc_xrc_snd_qp_t *gasnetc_xrc_snd_qp = NULL;
#if GASNET_MAXNODES <= 65535
static uint16_t *gasnetc_xrcd_map = NULL;
#else
static uint32_t *gasnetc_xrcd_map = NULL;
#endif
static int gasnetc_xrcd_simple;

static gasnetc_xrc_snd_qp_t *
Expand Down Expand Up @@ -379,7 +375,6 @@ static int _gasnetc_xrc_compare_keys(gex_Rank_t a_r, gex_Rank_t b_r) {
sizeof(uint16_t) * gasnetc_num_ports);
}
static int _gasnetc_xrc_compare_fn(const void *a_p, const void *b_p) {
gasneti_static_assert(GASNET_MAXNODES < INT_MAX);
gex_Rank_t a_r = *(gex_Rank_t *)a_p;
gex_Rank_t b_r = *(gex_Rank_t *)b_p;

Expand All @@ -388,7 +383,7 @@ static int _gasnetc_xrc_compare_fn(const void *a_p, const void *b_p) {
if (result) return result;

// tie-break using the rank itself
return (int)a_r - (int)b_r;
return (a_r == b_r) ? 0 : (a_r < b_r ? -1 : 1);
}

// Compute XRC domain mebership
Expand Down Expand Up @@ -1221,16 +1216,18 @@ typedef enum {
GASNETC_CONN_CMD_ACK
} gasnetc_conn_cmd_t;

#define GASNETC_CONN_CMD_MASK 0x0f
#define GASNETC_CONN_IS_ORIG 0x10
#define GASNETC_CONN_CMD_MASK 0x7
#define GASNETC_CONN_IS_ORIG 0x8

static gasnetc_ud_snd_desc_t *
conn_get_snd_desc(uint32_t flags)
{
gasnetc_ud_snd_desc_t *desc;
GASNETI_SPIN_UNTIL_TRACE((desc = gasneti_lifo_pop(&conn_snd_freelist)),
C, CONN_STALL_DESC, conn_snd_poll());
desc->wr.imm_data = flags | (gasneti_mynode << 16);
gasneti_assert_uint((flags >> 4) ,==, 0);
gasneti_assert_uint(gasneti_mynode ,<, (1 << 28));
desc->wr.imm_data = flags | (gasneti_mynode << 4);
return desc;
}

Expand Down Expand Up @@ -1934,7 +1931,7 @@ gasnetc_conn_rcv_wc(struct ibv_wc *comp)
gasnetc_ud_rcv_desc_t *desc = (gasnetc_ud_rcv_desc_t *)(1 ^ (uintptr_t)comp->wr_id);
gasnetc_conn_cmd_t cmd = (gasnetc_conn_cmd_t)(comp->imm_data & GASNETC_CONN_CMD_MASK);
uint32_t is_orig = comp->imm_data & GASNETC_CONN_IS_ORIG;
gex_Rank_t node = (comp->imm_data >> 16) & 0xffff;
gex_Rank_t node = (comp->imm_data >> 4) & 0x0fffffff;
gasneti_tick_t now = gasneti_ticks_now();

#if GASNET_DEBUG /* Drop 1 in N to aid debugging */
Expand Down
22 changes: 11 additions & 11 deletions ibv-conduit/gasnet_core_internal.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -529,17 +529,17 @@ typedef struct {
* gasnetc_epid_d is node and qp encoded together
* passing just a node (the default) means any qp to that node
*/
typedef uint32_t gasnetc_epid_t;
typedef uint64_t gasnetc_epid_t;

/* The 'epid' type holds 'node' in the low 16 bits.
* The upper 16 bits holds a qp index (qpi).
/* The 'epid' type holds 'node' in the low 32 bits.
* The upper 32 bits holds a qp index (qpi).
* A qpi of zero is a wildcard (an 'unbound' epid).
* Therefore, setting epid=node means "use any qp for that node".
* Non-zero qpi is 1 + the array index of the desired queue pair.
*/
#define gasnetc_epid2node(E) ((E)&0xffff)
#define gasnetc_epid2qpi(E) ((E)>>16)
#define gasnetc_epid(N,Q) ((N)|(((Q)+1)<<16))
#define gasnetc_epid2node(E) GASNETI_LOWORD(E)
#define gasnetc_epid2qpi(E) GASNETI_HIWORD(E)
#define gasnetc_epid(N,Q) GASNETI_MAKEWORD((Q)+1,(N))

/* Forward decl */
struct gasnetc_cep_t_;
Expand Down Expand Up @@ -638,7 +638,7 @@ struct gasnetc_cep_t_ {
// This is total size of R/W fields, with possible trailing padding prior to R/O fields,
// correct independent of GASNETI_THREADS
#define _GASNETC_CEP_RW_BYTES GASNETI_ALIGNUP_NOASSERT(_GASNETC_CEP_RW_EARLY+sizeof(gasnetc_sema_t),\
sizeof(void *))
sizeof(uint64_t))

//
// Read-only fields
Expand All @@ -647,8 +647,8 @@ struct gasnetc_cep_t_ {
// These are sorted by size to get dense packing

// 64-bit fields
// None currently
// Change _GASNETC_CEP_RW_BYTES's alignment to 8 if any are added
gasnetc_epid_t epid; // == uint64_t
#define _GASNETC_CEP_64_0 8

// Pointer-width fields
gasnetc_hca_t *hca;
Expand All @@ -673,8 +673,7 @@ struct gasnetc_cep_t_ {
#endif

// 32-bit fields
gasnetc_epid_t epid; // == uint32_t
#define _GASNETC_CEP_32_0 4
#define _GASNETC_CEP_32_0 0 // Placeholder for unconditional fields above
#if GASNETC_PIN_SEGMENT
uint32_t rkey; // Copy of hca->rkeys[gasnetc_epid2node(epid)]
#define _GASNETC_CEP_32_1 1*4
Expand Down Expand Up @@ -704,6 +703,7 @@ struct gasnetc_cep_t_ {

#define _GASNETC_CEP_TO_PAD (\
_GASNETC_CEP_RW_BYTES + \
_GASNETC_CEP_64_0 + \
_GASNETC_CEP_PTR_0+_GASNETC_CEP_PTR_1+_GASNETC_CEP_PTR_2 + \
_GASNETC_CEP_32_0+_GASNETC_CEP_32_1+_GASNETC_CEP_32_2+_GASNETC_CEP_32_3+_GASNETC_CEP_32_4)
#if GASNETI_THREADS
Expand Down