Skip to content

Commit

Permalink
prov/efa: Move err msg and raw addr functions
Browse files Browse the repository at this point in the history 
Move the err_msg field from efa_rdm_ep to base_ep.
Move efa_rdm_ep_raw_addr_str, efa_rdm_ep_get_peer_raw_addr,
efa_rdm_ep_get_peer_raw_addr_str to base_ep.h so they can be used
by efa-raw.

Signed-off-by: Jessie Yang <[email protected]>
  • Loading branch information
@jiaxiyan
jiaxiyan committed Dec 30, 2024
1 parent f4e302b commit a6c0ce0
Show file tree
Hide file tree
Showing 8 changed files with 62 additions and 62 deletions.
38 changes: 38 additions & 0 deletions prov/efa/src/efa_base_ep.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -482,3 +482,41 @@ void efa_base_ep_write_eq_error(struct efa_base_ep *ep, ssize_t err, ssize_t pro
prov_errno, efa_strerror(prov_errno));
abort();
}

const char *efa_base_ep_raw_addr_str(struct efa_base_ep *base_ep, char *buf, size_t *buflen)
{
return ofi_straddr(buf, buflen, FI_ADDR_EFA, &base_ep->src_addr);
}

/**
* @brief return peer's raw address in #efa_ep_addr
*
* @param[in] ep end point
* @param[in] addr libfabric address
* @returns
* If peer exists, return peer's raw addrress as pointer to #efa_ep_addr;
* Otherwise, return NULL
*/
struct efa_ep_addr *efa_base_ep_get_peer_raw_addr(struct efa_base_ep *base_ep, fi_addr_t addr)
{
struct efa_av *efa_av;
struct efa_conn *efa_conn;

efa_av = base_ep->av;
efa_conn = efa_av_addr_to_conn(efa_av, addr);
return efa_conn ? efa_conn->ep_addr : NULL;
}

/**
* @brief return peer's raw address in a readable string
*
* @param[in] base_ep end point
* @param[in] addr libfabric address
* @param[out] buf a buffer to be used to store string
* @param[in,out] buflen length of `buf` as input. length of the string as output.
* @return a string with peer's raw address
*/
const char *efa_base_ep_get_peer_raw_addr_str(struct efa_base_ep *base_ep, fi_addr_t addr, char *buf, size_t *buflen)
{
return ofi_straddr(buf, buflen, FI_ADDR_EFA, efa_base_ep_get_peer_raw_addr(base_ep, addr));
}
11 changes: 11 additions & 0 deletions prov/efa/src/efa_base_ep.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@

#define EFA_QP_DEFAULT_SERVICE_LEVEL 0
#define EFA_QP_LOW_LATENCY_SERVICE_LEVEL 8
#define EFA_ERROR_MSG_BUFFER_LENGTH 1024

#define efa_rx_flags(efa_base_ep) ((efa_base_ep)->util_ep.rx_op_flags)
#define efa_tx_flags(efa_base_ep) ((efa_base_ep)->util_ep.tx_op_flags)
Expand Down Expand Up @@ -65,6 +66,7 @@ struct efa_base_ep {
/* Only used by RDM ep type */
struct efa_qp *user_recv_qp; /* Separate qp to receive pkts posted by users */
struct efa_recv_wr *user_recv_wr_vec;
char err_msg[EFA_ERROR_MSG_BUFFER_LENGTH]; /* A large enough buffer to store CQ/EQ error data used by e.g. fi_cq_readerr */
};

int efa_base_ep_bind_av(struct efa_base_ep *base_ep, struct efa_av *av);
Expand Down Expand Up @@ -99,4 +101,13 @@ void efa_base_ep_write_eq_error(struct efa_base_ep *ep,
ssize_t err,
ssize_t prov_errno);

const char *efa_base_ep_raw_addr_str(struct efa_base_ep *base_ep, char *buf,
size_t *buflen);

struct efa_ep_addr *efa_base_ep_get_peer_raw_addr(struct efa_base_ep *base_ep,
fi_addr_t addr);

const char *efa_base_ep_get_peer_raw_addr_str(struct efa_base_ep *base_ep,
fi_addr_t addr, char *buf,
size_t *buflen);
#endif
9 changes: 0 additions & 9 deletions prov/efa/src/rdm/efa_rdm_ep.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,6 @@
#include "efa_base_ep.h"
#include "efa_rdm_rxe_map.h"

#define EFA_RDM_ERROR_MSG_BUFFER_LENGTH 1024

/** @brief Information of a queued copy.
*
* This struct is used when receiving buffer is on device.
Expand Down Expand Up @@ -182,7 +180,6 @@ struct efa_rdm_ep {

bool sendrecv_in_order_aligned_128_bytes; /**< whether to support in order send/recv of each aligned 128 bytes memory region */
bool write_in_order_aligned_128_bytes; /**< whether to support in order write of each aligned 128 bytes memory region */
char err_msg[EFA_RDM_ERROR_MSG_BUFFER_LENGTH]; /* A large enough buffer to store CQ/EQ error data used by e.g. fi_cq_readerr */
struct efa_rdm_pke **pke_vec;
struct dlist_entry entry;
/* the count of opes queued before handshake is made with their peers */
Expand All @@ -196,12 +193,6 @@ int efa_rdm_ep_flush_queued_blocking_copy_to_hmem(struct efa_rdm_ep *ep);

struct efa_ep_addr *efa_rdm_ep_raw_addr(struct efa_rdm_ep *ep);

const char *efa_rdm_ep_raw_addr_str(struct efa_rdm_ep *ep, char *buf, size_t *buflen);

struct efa_ep_addr *efa_rdm_ep_get_peer_raw_addr(struct efa_rdm_ep *ep, fi_addr_t addr);

const char *efa_rdm_ep_get_peer_raw_addr_str(struct efa_rdm_ep *ep, fi_addr_t addr, char *buf, size_t *buflen);

struct efa_rdm_peer *efa_rdm_ep_get_peer(struct efa_rdm_ep *ep, fi_addr_t addr);

int32_t efa_rdm_ep_get_peer_ahn(struct efa_rdm_ep *ep, fi_addr_t addr);
Expand Down
2 changes: 1 addition & 1 deletion prov/efa/src/rdm/efa_rdm_ep_fiops.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1324,7 +1324,7 @@ static int efa_rdm_ep_ctrl(struct fid *fid, int command, void *arg)
efa_rdm_ep_set_extra_info(ep);

ep_addr_strlen = sizeof(ep_addr_str);
efa_rdm_ep_raw_addr_str(ep, ep_addr_str, &ep_addr_strlen);
efa_base_ep_raw_addr_str(&ep->base_ep, ep_addr_str, &ep_addr_strlen);
EFA_INFO(FI_LOG_EP_CTRL, "libfabric %s efa endpoint created! address: %s\n",
fi_tostr("1", FI_TYPE_VERSION), ep_addr_str);

Expand Down
40 changes: 0 additions & 40 deletions prov/efa/src/rdm/efa_rdm_ep_utils.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,31 +25,6 @@ struct efa_ep_addr *efa_rdm_ep_raw_addr(struct efa_rdm_ep *ep)
return &ep->base_ep.src_addr;
}

const char *efa_rdm_ep_raw_addr_str(struct efa_rdm_ep *ep, char *buf, size_t *buflen)
{
return ofi_straddr(buf, buflen, FI_ADDR_EFA, efa_rdm_ep_raw_addr(ep));
}

/**
* @brief return peer's raw address in #efa_ep_addr
*
* @param[in] ep end point
* @param[in] addr libfabric address
* @returns
* If peer exists, return peer's raw addrress as pointer to #efa_ep_addr;
* Otherwise, return NULL
* @relates efa_rdm_peer
*/
struct efa_ep_addr *efa_rdm_ep_get_peer_raw_addr(struct efa_rdm_ep *ep, fi_addr_t addr)
{
struct efa_av *efa_av;
struct efa_conn *efa_conn;

efa_av = ep->base_ep.av;
efa_conn = efa_av_addr_to_conn(efa_av, addr);
return efa_conn ? efa_conn->ep_addr : NULL;
}

/**
* @brief return peer's ahn
*
Expand All @@ -69,21 +44,6 @@ int32_t efa_rdm_ep_get_peer_ahn(struct efa_rdm_ep *ep, fi_addr_t addr)
return efa_conn ? efa_conn->ah->ahn : -1;
}

/**
* @brief return peer's raw address in a reable string
*
* @param[in] ep end point
* @param[in] addr libfabric address
* @param[out] buf a buffer tat to be used to store string
* @param[in,out] buflen length of `buf` as input. length of the string as output.
* @relates efa_rdm_peer
* @return a string with peer's raw address
*/
const char *efa_rdm_ep_get_peer_raw_addr_str(struct efa_rdm_ep *ep, fi_addr_t addr, char *buf, size_t *buflen)
{
return ofi_straddr(buf, buflen, FI_ADDR_EFA, efa_rdm_ep_get_peer_raw_addr(ep, addr));
}

/**
* @brief get pointer to efa_rdm_peer structure for a given libfabric address
*
Expand Down
8 changes: 4 additions & 4 deletions prov/efa/src/rdm/efa_rdm_pke_cmd.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -453,9 +453,9 @@ void efa_rdm_pke_handle_tx_error(struct efa_rdm_pke *pkt_entry, int prov_errno)
memset(&ep_addr_str, 0, sizeof(ep_addr_str));
memset(&peer_addr_str, 0, sizeof(peer_addr_str));
buflen = sizeof(ep_addr_str);
efa_rdm_ep_raw_addr_str(ep, ep_addr_str, &buflen);
efa_base_ep_raw_addr_str(&ep->base_ep, ep_addr_str, &buflen);
buflen = sizeof(peer_addr_str);
efa_rdm_ep_get_peer_raw_addr_str(ep, pkt_entry->addr, peer_addr_str, &buflen);
efa_base_ep_get_peer_raw_addr_str(&ep->base_ep, pkt_entry->addr, peer_addr_str, &buflen);
EFA_WARN(FI_LOG_CQ,
"While sending a handshake packet, an error occurred."
" Our address: %s, peer address: %s\n",
Expand Down Expand Up @@ -712,7 +712,7 @@ void efa_rdm_pke_handle_rx_error(struct efa_rdm_pke *pkt_entry, int prov_errno)

memset(&ep_addr_str, 0, sizeof(ep_addr_str));
buflen = sizeof(ep_addr_str);
efa_rdm_ep_raw_addr_str(ep, ep_addr_str, &buflen);
efa_base_ep_raw_addr_str(&ep->base_ep, ep_addr_str, &buflen);
EFA_WARN(FI_LOG_CQ,
"Packet receive error from non TX/RX packet. Our address: %s\n",
ep_addr_str);
Expand Down Expand Up @@ -751,7 +751,7 @@ fi_addr_t efa_rdm_pke_insert_addr(struct efa_rdm_pke *pkt_entry, void *raw_addr)
char self_raw_addr_str[OFI_ADDRSTRLEN];
size_t buflen = OFI_ADDRSTRLEN;

efa_rdm_ep_raw_addr_str(ep, self_raw_addr_str, &buflen);
efa_base_ep_raw_addr_str(&ep->base_ep, self_raw_addr_str, &buflen);
EFA_WARN(FI_LOG_CQ,
"Host %s received a packet with invalid protocol version %d.\n"
"This host can only support protocol version %d and above.\n",
Expand Down
14 changes: 7 additions & 7 deletions prov/efa/src/rdm/efa_rdm_util.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -119,9 +119,9 @@ int efa_rdm_write_error_msg(struct efa_rdm_ep *ep, fi_addr_t addr, int prov_errn
*buflen = 0;

len = sizeof(ep_addr_str);
efa_rdm_ep_raw_addr_str(ep, ep_addr_str, &len);
efa_base_ep_raw_addr_str(&ep->base_ep, ep_addr_str, &len);
len = sizeof(peer_addr_str);
efa_rdm_ep_get_peer_raw_addr_str(ep, addr, peer_addr_str, &len);
efa_base_ep_get_peer_raw_addr_str(&ep->base_ep, addr, peer_addr_str, &len);

if (!ep->host_id || EFA_HOST_ID_STRING_LENGTH != snprintf(local_host_id_str, EFA_HOST_ID_STRING_LENGTH + 1, "i-%017lx", ep->host_id)) {
strcpy(local_host_id_str, "N/A");
Expand All @@ -131,19 +131,19 @@ int efa_rdm_write_error_msg(struct efa_rdm_ep *ep, fi_addr_t addr, int prov_errn
strcpy(peer_host_id_str, "N/A");
}

int ret = snprintf(ep->err_msg, EFA_RDM_ERROR_MSG_BUFFER_LENGTH, "%s My EFA addr: %s My host id: %s Peer EFA addr: %s Peer host id: %s",
int ret = snprintf(ep->base_ep.err_msg, EFA_ERROR_MSG_BUFFER_LENGTH, "%s My EFA addr: %s My host id: %s Peer EFA addr: %s Peer host id: %s",
base_msg, ep_addr_str, local_host_id_str, peer_addr_str, peer_host_id_str);

if (ret < 0 || ret > EFA_RDM_ERROR_MSG_BUFFER_LENGTH - 1) {
if (ret < 0 || ret > EFA_ERROR_MSG_BUFFER_LENGTH - 1) {
return -FI_EINVAL;
}

if (strlen(ep->err_msg) >= EFA_RDM_ERROR_MSG_BUFFER_LENGTH) {
if (strlen(ep->base_ep.err_msg) >= EFA_ERROR_MSG_BUFFER_LENGTH) {
return -FI_ENOBUFS;
}

*buf = ep->err_msg;
*buflen = EFA_RDM_ERROR_MSG_BUFFER_LENGTH;
*buf = ep->base_ep.err_msg;
*buflen = EFA_ERROR_MSG_BUFFER_LENGTH;

return 0;
}
2 changes: 1 addition & 1 deletion prov/efa/test/efa_unit_test_cq.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -155,7 +155,7 @@ static void test_rdm_cq_read_bad_send_status(struct efa_resource *resource,
assert_int_equal(ret, -FI_EAVAIL);

/* Allocate memory to read CQ error */
cq_err_entry.err_data_size = EFA_RDM_ERROR_MSG_BUFFER_LENGTH;
cq_err_entry.err_data_size = EFA_ERROR_MSG_BUFFER_LENGTH;
cq_err_entry.err_data = malloc(cq_err_entry.err_data_size);
assert_non_null(cq_err_entry.err_data);

Expand Down

0 comments on commit a6c0ce0

Please sign in to comment.