Skip to content

Commit

Permalink
prov/efa: Unit test additions and fixes for efa-direct
Browse files Browse the repository at this point in the history 
Remove temporary hacks added previously to test the efa-direct code path
and add new tests

Signed-off-by: Sai Sunku <[email protected]>
  • Loading branch information
@sunkuamzn
sunkuamzn committed Jan 31, 2025
1 parent 2b43f51 commit 37ac4f4
Show file tree
Hide file tree
Showing 5 changed files with 211 additions and 59 deletions.
62 changes: 26 additions & 36 deletions prov/efa/test/efa_unit_test_common.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,37 +52,41 @@ void efa_unit_test_construct_msg_rma(struct fi_msg_rma *msg, struct iovec *iov,
msg->data = data;
}

struct fi_info *efa_unit_test_alloc_hints(enum fi_ep_type ep_type, char *prov_name)
struct fi_info *efa_unit_test_alloc_hints(enum fi_ep_type ep_type, char *fabric_name)
{
struct fi_info *hints;

hints = fi_allocinfo();
if (!hints)
return NULL;

hints->fabric_attr->prov_name = strdup(prov_name);
if (fabric_name)
hints->fabric_attr->name = strdup(fabric_name);
hints->ep_attr->type = ep_type;

/* Use a minimal caps that efa / efa-direct should always support */
hints->domain_attr->mr_mode = MR_MODE_BITS;

/* EFA direct and dgram paths require FI_CONTEXT2 */
if (!fabric_name || !strcasecmp(fabric_name, EFA_DIRECT_FABRIC_NAME))
hints->mode |= FI_CONTEXT2;

if (ep_type == FI_EP_DGRAM) {
hints->mode |= FI_MSG_PREFIX;
hints->mode |= FI_MSG_PREFIX | FI_CONTEXT2;
}

return hints;
}

/* TODO: remove use_efa_direct after we have efa_direct implemented in fi_info */
void efa_unit_test_resource_construct_with_hints(struct efa_resource *resource,
enum fi_ep_type ep_type,
uint32_t fi_version, struct fi_info *hints,
bool enable_ep, bool open_cq, char* prov_name)
bool enable_ep, bool open_cq)
{
int ret = 0;
struct fi_av_attr av_attr = {0};
struct fi_cq_attr cq_attr = {0};
struct fi_eq_attr eq_attr = {0};
struct efa_domain *efa_domain;

ret = fi_getinfo(fi_version, NULL, NULL, 0ULL, hints, &resource->info);
if (ret)
Expand All @@ -96,17 +100,6 @@ void efa_unit_test_resource_construct_with_hints(struct efa_resource *resource,
if (ret)
goto err;

/*
* TODO: Remove this function pointer override when we have it assigned
* for efa-direct correctly.
*/
if (!strcmp(EFA_DIRECT_PROV_NAME, prov_name)) {
efa_domain = container_of(resource->domain, struct efa_domain, util_domain.domain_fid);

efa_domain->util_domain.domain_fid.ops->endpoint = efa_ep_open;
efa_domain->util_domain.domain_fid.ops->cq_open = efa_cq_open;
}

ret = fi_endpoint(resource->domain, resource->info, &resource->ep, NULL);
if (ret)
goto err;
Expand Down Expand Up @@ -146,19 +139,18 @@ void efa_unit_test_resource_construct_with_hints(struct efa_resource *resource,
assert_int_equal(ret, 0);
}

void efa_unit_test_resource_construct(struct efa_resource *resource, enum fi_ep_type ep_type, char *prov_name)
void efa_unit_test_resource_construct(struct efa_resource *resource, enum fi_ep_type ep_type, char *fabric_name)
{

/* TODO use prov_name here when efa-direct fi_info is implemented */
resource->hints = efa_unit_test_alloc_hints(ep_type, EFA_PROV_NAME);
resource->hints = efa_unit_test_alloc_hints(ep_type, fabric_name);
if (!resource->hints)
goto err;
if (!strcmp(EFA_DIRECT_PROV_NAME, prov_name))
if (!strcmp(EFA_DIRECT_FABRIC_NAME, fabric_name))
efa_unit_test_resource_construct_with_hints(resource, ep_type, FI_VERSION(2, 0),
resource->hints, true, true, prov_name);
resource->hints, true, true);
else
efa_unit_test_resource_construct_with_hints(resource, ep_type, FI_VERSION(1, 14),
resource->hints, true, true, prov_name);
resource->hints, true, true);
return;

err:
Expand All @@ -169,19 +161,18 @@ void efa_unit_test_resource_construct(struct efa_resource *resource, enum fi_ep_
}

void efa_unit_test_resource_construct_ep_not_enabled(struct efa_resource *resource,
enum fi_ep_type ep_type, char *prov_name)
enum fi_ep_type ep_type, char *fabric_name)
{
/* TODO use prov_name here when efa-direct fi_info is implemented */
resource->hints = efa_unit_test_alloc_hints(ep_type, EFA_PROV_NAME);
resource->hints = efa_unit_test_alloc_hints(ep_type, fabric_name);
if (!resource->hints)
goto err;

if (!strcmp(EFA_DIRECT_PROV_NAME, prov_name))
if (!strcmp(EFA_DIRECT_FABRIC_NAME, fabric_name))
efa_unit_test_resource_construct_with_hints(resource, ep_type, FI_VERSION(2, 0),
resource->hints, false, true, prov_name);
resource->hints, false, true);
else
efa_unit_test_resource_construct_with_hints(resource, ep_type, FI_VERSION(1, 14),
resource->hints, false, true, prov_name);
resource->hints, false, true);
return;

err:
Expand All @@ -192,19 +183,18 @@ void efa_unit_test_resource_construct_ep_not_enabled(struct efa_resource *resour
}

void efa_unit_test_resource_construct_no_cq_and_ep_not_enabled(struct efa_resource *resource,
enum fi_ep_type ep_type, char *prov_name)
enum fi_ep_type ep_type, char *fabric_name)
{
/* TODO use prov_name here when efa-direct fi_info is implemented */
resource->hints = efa_unit_test_alloc_hints(ep_type, EFA_PROV_NAME);
resource->hints = efa_unit_test_alloc_hints(ep_type, fabric_name);
if (!resource->hints)
goto err;

if (!strcmp(EFA_DIRECT_PROV_NAME, prov_name))
if (!strcmp(EFA_DIRECT_FABRIC_NAME, fabric_name))
efa_unit_test_resource_construct_with_hints(resource, ep_type, FI_VERSION(2, 0),
resource->hints, false, false, prov_name);
resource->hints, false, false);
else
efa_unit_test_resource_construct_with_hints(resource, ep_type, FI_VERSION(1, 14),
resource->hints, false, false, prov_name);
resource->hints, false, false);
return;

err:
Expand All @@ -227,7 +217,7 @@ void efa_unit_test_resource_construct_rdm_shm_disabled(struct efa_resource *reso
goto err;

efa_unit_test_resource_construct_with_hints(resource, FI_EP_RDM, FI_VERSION(1, 14),
resource->hints, false, true, EFA_PROV_NAME);
resource->hints, false, true);

ret = fi_setopt(&resource->ep->fid, FI_OPT_ENDPOINT,
FI_OPT_SHARED_MEMORY_PERMITTED, &shm_permitted,
Expand Down
12 changes: 6 additions & 6 deletions prov/efa/test/efa_unit_test_ep.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -597,7 +597,7 @@ void test_efa_rdm_ep_rma_queue_before_handshake(struct efa_resource **state, int
resource->hints->caps |= FI_MSG | FI_TAGGED | FI_RMA;
resource->hints->domain_attr->mr_mode |= MR_MODE_BITS;
efa_unit_test_resource_construct_with_hints(resource, FI_EP_RDM, FI_VERSION(1, 14),
resource->hints, true, true, EFA_PROV_NAME);
resource->hints, true, true);

/* ensure we don't have RMA capability. */
efa_rdm_ep = container_of(resource->ep, struct efa_rdm_ep, base_ep.util_ep.ep_fid);
Expand Down Expand Up @@ -685,7 +685,7 @@ void test_efa_rdm_ep_rma_inconsistent_unsolicited_write_recv(struct efa_resource
resource->hints->caps |= FI_MSG | FI_TAGGED | FI_RMA;
resource->hints->domain_attr->mr_mode |= MR_MODE_BITS;
efa_unit_test_resource_construct_with_hints(resource, FI_EP_RDM, FI_VERSION(1, 22),
resource->hints, true, true, EFA_PROV_NAME);
resource->hints, true, true);

efa_rdm_ep = container_of(resource->ep, struct efa_rdm_ep, base_ep.util_ep.ep_fid);

Expand Down Expand Up @@ -794,7 +794,7 @@ void test_efa_rdm_ep_rma_without_caps(struct efa_resource **state)
resource->hints->caps &= ~FI_RMA;
resource->hints->domain_attr->mr_mode |= MR_MODE_BITS;
efa_unit_test_resource_construct_with_hints(resource, FI_EP_RDM, FI_VERSION(1, 14),
resource->hints, true, true, EFA_PROV_NAME);
resource->hints, true, true);

/* ensure we don't have RMA capability. */
efa_rdm_ep = container_of(resource->ep, struct efa_rdm_ep, base_ep.util_ep.ep_fid);
Expand Down Expand Up @@ -845,7 +845,7 @@ void test_efa_rdm_ep_atomic_without_caps(struct efa_resource **state)
resource->hints->caps &= ~FI_ATOMIC;
resource->hints->domain_attr->mr_mode |= MR_MODE_BITS;
efa_unit_test_resource_construct_with_hints(resource, FI_EP_RDM, FI_VERSION(1, 14),
resource->hints, true, true, EFA_PROV_NAME);
resource->hints, true, true);

/* ensure we don't have ATOMIC capability. */
efa_rdm_ep = container_of(resource->ep, struct efa_rdm_ep, base_ep.util_ep.ep_fid);
Expand Down Expand Up @@ -1001,7 +1001,7 @@ static void test_efa_rdm_ep_use_zcpy_rx_impl(struct efa_resource *resource,
ofi_hmem_disable_p2p = cuda_p2p_disabled;

efa_unit_test_resource_construct_with_hints(resource, FI_EP_RDM, FI_VERSION(1, 14),
resource->hints, false, true, EFA_PROV_NAME);
resource->hints, false, true);

/* System memory P2P should always be enabled */
assert_true(g_efa_hmem_info[FI_HMEM_SYSTEM].initialized);
Expand Down Expand Up @@ -1314,7 +1314,7 @@ void test_efa_rdm_ep_rx_refill_impl(struct efa_resource **state, int threshold,
assert_non_null(resource->hints);
resource->hints->rx_attr->size = rx_size;
efa_unit_test_resource_construct_with_hints(resource, FI_EP_RDM, FI_VERSION(1, 14),
resource->hints, true, true, EFA_PROV_NAME);
resource->hints, true, true);

efa_rdm_ep = container_of(resource->ep, struct efa_rdm_ep, base_ep.util_ep.ep_fid);
assert_int_equal(efa_rdm_ep_get_rx_pool_size(efa_rdm_ep), rx_size);
Expand Down
Loading

0 comments on commit 37ac4f4

Please sign in to comment.