Skip to content

Commit

Permalink
prov/efa: Set msg_order based on hints
Browse files Browse the repository at this point in the history 
Efa device doesn't have ordering (EFA_MSG_ORDER == FI_ORDER_NONE)
If apps request an ordering that is relaxed than what provider supports, we should respect that.
This is specially true for FI_ORDER_NONE:
No ordering is specified. This value may be used as input in order to obtain
the default message order supported by the provider.

Also added unit tests.

Signed-off-by: Shi Jin <[email protected]>
  • Loading branch information
@shijin-aws
shijin-aws committed Apr 9, 2024
1 parent 0a201cf commit 25b55ae
Show file tree
Hide file tree
Showing 4 changed files with 172 additions and 0 deletions.
19 changes: 19 additions & 0 deletions prov/efa/src/efa_user_info.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -408,13 +408,32 @@ int efa_user_info_alter_rdm(int version, struct fi_info *info, const struct fi_i
*/
if (hints) {
if (hints->tx_attr) {
/* efa device doesn't have ordering (EFA_MSG_ORDER == FI_ORDER_NONE).
* if apps request an ordering that is relaxed than
* what provider supports, we should respect that.
* This is specially true for FI_ORDER_NONE:
* No ordering is specified. This value may be used as input in order to obtain
* the default message order supported by the provider.
*/
info->tx_attr->msg_order &= hints->tx_attr->msg_order;
atomic_ordering = FI_ORDER_ATOMIC_RAR | FI_ORDER_ATOMIC_RAW |
FI_ORDER_ATOMIC_WAR | FI_ORDER_ATOMIC_WAW;
if (!(hints->tx_attr->msg_order & atomic_ordering)) {
info->ep_attr->max_order_raw_size = 0;
}
}

if (hints->rx_attr) {
/* efa device doesn't have ordering (EFA_MSG_ORDER == FI_ORDER_NONE).
* if apps request an ordering that is relaxed than
* what provider supports, we should respect that.
* This is specially true for FI_ORDER_NONE:
* No ordering is specified. This value may be used as input in order to obtain
* the default message order supported by the provider.
*/
info->rx_attr->msg_order &= hints->rx_attr->msg_order;
}

/* We only support manual progress for RMA operations */
if (hints->caps & FI_RMA) {
info->domain_attr->control_progress = FI_PROGRESS_MANUAL;
Expand Down
141 changes: 141 additions & 0 deletions prov/efa/test/efa_unit_test_info.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -124,6 +124,147 @@ void test_info_ep_attr()
fi_freeinfo(info);
}

/**
* @brief Verify info->tx/rx_attr->msg_order is set according to hints.
*
*/
static void
test_info_tx_rx_msg_order_from_hints(struct fi_info *hints, int expected_ret)
{
struct fi_info *info;
int err;

err = fi_getinfo(FI_VERSION(FI_MAJOR_VERSION, FI_MINOR_VERSION), NULL, NULL, 0ULL, hints, &info);

assert_int_equal(err, expected_ret);

if (expected_ret == FI_SUCCESS) {
assert_true(hints->tx_attr->msg_order == info->tx_attr->msg_order);
assert_true(hints->rx_attr->msg_order == info->rx_attr->msg_order);
}

fi_freeinfo(info);
}

/**
* @brief Verify info->tx/rx_attr->op_flags is set according to hints.
*
*/
static void
test_info_tx_rx_op_flags_from_hints(struct fi_info *hints, int expected_ret)
{
struct fi_info *info;
int err;

err = fi_getinfo(FI_VERSION(FI_MAJOR_VERSION, FI_MINOR_VERSION), NULL, NULL, 0ULL, hints, &info);

assert_int_equal(err, expected_ret);

if (expected_ret == FI_SUCCESS) {
assert_true(hints->tx_attr->op_flags == info->tx_attr->op_flags);
assert_true(hints->rx_attr->op_flags == info->rx_attr->op_flags);
}

fi_freeinfo(info);
}

/**
* @brief Verify info->tx/rx_attr->size is set according to hints.
*
*/
static void test_info_tx_rx_size_from_hints(struct fi_info *hints, int expected_ret)
{
struct fi_info *info;
int err;

err = fi_getinfo(FI_VERSION(FI_MAJOR_VERSION, FI_MINOR_VERSION), NULL, NULL, 0ULL, hints, &info);

assert_int_equal(err, expected_ret);

if (expected_ret == FI_SUCCESS) {
assert_true(hints->tx_attr->size == info->tx_attr->size);
assert_true(hints->rx_attr->size == info->rx_attr->size);
}

fi_freeinfo(info);
}

void test_info_tx_rx_msg_order_rdm_order_none(struct efa_resource **state)
{
struct efa_resource *resource = *state;

resource->hints = efa_unit_test_alloc_hints(FI_EP_RDM);
assert_non_null(resource->hints);

resource->hints->tx_attr->msg_order = FI_ORDER_NONE;
resource->hints->rx_attr->msg_order = FI_ORDER_NONE;
test_info_tx_rx_msg_order_from_hints(resource->hints, 0);
}

void test_info_tx_rx_msg_order_rdm_order_sas(struct efa_resource **state)
{
struct efa_resource *resource = *state;

resource->hints = efa_unit_test_alloc_hints(FI_EP_RDM);
assert_non_null(resource->hints);

resource->hints->tx_attr->msg_order = FI_ORDER_SAS;
resource->hints->rx_attr->msg_order = FI_ORDER_SAS;
test_info_tx_rx_msg_order_from_hints(resource->hints, 0);
}

void test_info_tx_rx_msg_order_dgram_order_none(struct efa_resource **state)
{
struct efa_resource *resource = *state;

resource->hints = efa_unit_test_alloc_hints(FI_EP_DGRAM);
assert_non_null(resource->hints);

resource->hints->tx_attr->msg_order = FI_ORDER_NONE;
resource->hints->rx_attr->msg_order = FI_ORDER_NONE;
test_info_tx_rx_msg_order_from_hints(resource->hints, 0);
}

/**
* @brief dgram endpoint doesn't support any ordering, so fi_getinfo
* should return -FI_ENODATA if hints requests sas
*/
void test_info_tx_rx_msg_order_dgram_order_sas(struct efa_resource **state)
{
struct efa_resource *resource = *state;

resource->hints = efa_unit_test_alloc_hints(FI_EP_DGRAM);
assert_non_null(resource->hints);

resource->hints->tx_attr->msg_order = FI_ORDER_SAS;
resource->hints->rx_attr->msg_order = FI_ORDER_SAS;
test_info_tx_rx_msg_order_from_hints(resource->hints, -FI_ENODATA);
}

void test_info_tx_rx_op_flags_rdm(struct efa_resource **state)
{
struct efa_resource *resource = *state;

resource->hints = efa_unit_test_alloc_hints(FI_EP_RDM);
assert_non_null(resource->hints);

resource->hints->tx_attr->op_flags = FI_DELIVERY_COMPLETE;
resource->hints->rx_attr->op_flags = FI_COMPLETION;
test_info_tx_rx_op_flags_from_hints(resource->hints, 0);
}

void test_info_tx_rx_size_rdm(struct efa_resource **state)
{
struct efa_resource *resource = *state;

resource->hints = efa_unit_test_alloc_hints(FI_EP_RDM);
assert_non_null(resource->hints);

resource->hints->tx_attr->size = 16;
resource->hints->rx_attr->size = 16;
test_info_tx_rx_size_from_hints(resource->hints, 0);
}

static void test_info_check_shm_info_from_hints(struct fi_info *hints)
{
struct fi_info *info;
Expand Down
6 changes: 6 additions & 0 deletions prov/efa/test/efa_unit_tests.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,12 @@ int main(void)
cmocka_unit_test_setup_teardown(test_info_open_ep_with_wrong_info, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
cmocka_unit_test_setup_teardown(test_info_open_ep_with_api_1_1_info, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
cmocka_unit_test_setup_teardown(test_info_ep_attr, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
cmocka_unit_test_setup_teardown(test_info_tx_rx_msg_order_rdm_order_none, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
cmocka_unit_test_setup_teardown(test_info_tx_rx_msg_order_rdm_order_sas, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
cmocka_unit_test_setup_teardown(test_info_tx_rx_msg_order_dgram_order_none, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
cmocka_unit_test_setup_teardown(test_info_tx_rx_msg_order_dgram_order_sas, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
cmocka_unit_test_setup_teardown(test_info_tx_rx_op_flags_rdm, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
cmocka_unit_test_setup_teardown(test_info_tx_rx_size_rdm, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
cmocka_unit_test_setup_teardown(test_info_check_shm_info_hmem, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
cmocka_unit_test_setup_teardown(test_info_check_shm_info_op_flags, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
cmocka_unit_test_setup_teardown(test_info_check_shm_info_threading, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
Expand Down
6 changes: 6 additions & 0 deletions prov/efa/test/efa_unit_tests.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -124,6 +124,12 @@ void test_ibv_cq_ex_read_ignore_removed_peer();
void test_info_open_ep_with_wrong_info();
void test_info_open_ep_with_api_1_1_info();
void test_info_ep_attr();
void test_info_tx_rx_msg_order_rdm_order_none();
void test_info_tx_rx_msg_order_rdm_order_sas();
void test_info_tx_rx_msg_order_dgram_order_none();
void test_info_tx_rx_msg_order_dgram_order_sas();
void test_info_tx_rx_op_flags_rdm();
void test_info_tx_rx_size_rdm();
void test_info_check_shm_info_hmem();
void test_info_check_shm_info_op_flags();
void test_info_check_shm_info_threading();
Expand Down

0 comments on commit 25b55ae

Please sign in to comment.