diff options
| -rw-r--r-- | rpc/rpc-lib/src/Makefile.am | 3 | ||||
| -rw-r--r-- | rpc/rpc-lib/src/rpc-lib-messages.h | 83 | ||||
| -rw-r--r-- | rpc/rpc-transport/rdma/src/Makefile.am | 2 | ||||
| -rw-r--r-- | rpc/rpc-transport/rdma/src/name.c | 149 | ||||
| -rw-r--r-- | rpc/rpc-transport/rdma/src/rdma.c | 698 | ||||
| -rw-r--r-- | rpc/rpc-transport/rdma/src/rpc-trans-rdma-messages.h | 217 |
6 files changed, 780 insertions, 372 deletions
diff --git a/rpc/rpc-lib/src/Makefile.am b/rpc/rpc-lib/src/Makefile.am index 6a098c9a8ce..36eb245bc06 100644 --- a/rpc/rpc-lib/src/Makefile.am +++ b/rpc/rpc-lib/src/Makefile.am @@ -8,7 +8,8 @@ libgfrpc_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la libgfrpc_la_LDFLAGS = -version-info $(LIBGFRPC_LT_VERSION) $(GF_LDFLAGS) noinst_HEADERS = rpcsvc.h rpc-transport.h xdr-common.h xdr-rpc.h xdr-rpcclnt.h \ - rpc-clnt.h rpcsvc-common.h protocol-common.h rpc-drc.h rpc-clnt-ping.h + rpc-clnt.h rpcsvc-common.h protocol-common.h rpc-drc.h rpc-clnt-ping.h \ + rpc-lib-messages.h AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \ -I$(top_srcdir)/rpc/xdr/src \ diff --git a/rpc/rpc-lib/src/rpc-lib-messages.h b/rpc/rpc-lib/src/rpc-lib-messages.h new file mode 100644 index 00000000000..51351e82283 --- /dev/null +++ b/rpc/rpc-lib/src/rpc-lib-messages.h @@ -0,0 +1,83 @@ +/* + Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com> + This file is part of GlusterFS. + + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. +*/ + +#ifndef _RPC_LIB_MESSAGES_H_ +#define _RPC_LIB_MESSAGES_H_ + +#ifndef _CONFIG_H +#define _CONFIG_H +#include "config.h" +#endif + +#include "glfs-message-id.h" + +/* NOTE: Rules for message additions + * 1) Each instance of a message is _better_ left with a unique message ID, even + * if the message format is the same. Reasoning is that, if the message + * format needs to change in one instance, the other instances are not + * impacted or the new change does not change the ID of the instance being + * modified. + * 2) Addition of a message, + * - Should increment the GLFS_NUM_MESSAGES + * - Append to the list of messages defined, towards the end + * - Retain macro naming as glfs_msg_X (for redability across developers) + * NOTE: Rules for message format modifications + * 3) Check acorss the code if the message ID macro in question is reused + * anywhere. If reused then then the modifications should ensure correctness + * everywhere, or needs a new message ID as (1) above was not adhered to. If + * not used anywhere, proceed with the required modification. + * NOTE: Rules for message deletion + * 4) Check (3) and if used anywhere else, then cannot be deleted. If not used + * anywhere, then can be deleted, but will leave a hole by design, as + * addition rules specify modification to the end of the list and not filling + * holes. + */ + +#define GLFS_RPC_LIB_BASE GLFS_MSGID_COMP_RPC_LIB +#define GLFS_NUM_MESSAGES 12 +#define GLFS_RPC_LIB_MSGID_END (GLFS_RPC_LIB_BASE + GLFS_NUM_MESSAGES + 1) + +/* Messages with message IDs */ + +#define glfs_msg_start_x GLFS_RPC_LIB_BASE, "Invalid: Start of messages" + +/*------------*/ +/* First slot is allocated for common transport msg ids */ + +#define TRANS_MSG_ADDR_FAMILY_NOT_SPECIFIED (GLFS_RPC_LIB_BASE + 1) + +#define TRANS_MSG_UNKNOWN_ADDR_FAMILY (GLFS_RPC_LIB_BASE + 2) + +#define TRANS_MSG_REMOTE_HOST_ERROR (GLFS_RPC_LIB_BASE + 3) + +#define TRANS_MSG_DNS_RESOL_FAILED (GLFS_RPC_LIB_BASE + 4) + +#define TRANS_MSG_LISTEN_PATH_ERROR (GLFS_RPC_LIB_BASE + 5) + +#define TRANS_MSG_CONNECT_PATH_ERROR (GLFS_RPC_LIB_BASE + 6) + +#define TRANS_MSG_GET_ADDR_INFO_FAILED (GLFS_RPC_LIB_BASE + 7) + +#define TRANS_MSG_PORT_BIND_FAILED (GLFS_RPC_LIB_BASE + 8) + +#define TRANS_MSG_INET_ERROR (GLFS_RPC_LIB_BASE + 9) + +#define TRANS_MSG_GET_NAME_INFO_FAILED (GLFS_RPC_LIB_BASE + 10) + +#define TRANS_MSG_TRANSPORT_ERROR (GLFS_RPC_LIB_BASE + 11) + +#define TRANS_MSG_TIMEOUT_EXCEEDED (GLFS_RPC_LIB_BASE + 12) + +/*------------*/ + +#define glfs_msg_end_x GLFS_RPC_LIB_MSGID_END, "Invalid: End of messages" + +#endif /* !_RPC_LIB_MESSAGES_H_ */ + diff --git a/rpc/rpc-transport/rdma/src/Makefile.am b/rpc/rpc-transport/rdma/src/Makefile.am index fed139e7c8d..6526b6ce790 100644 --- a/rpc/rpc-transport/rdma/src/Makefile.am +++ b/rpc/rpc-transport/rdma/src/Makefile.am @@ -9,7 +9,7 @@ rdma_la_SOURCES = rdma.c name.c rdma_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la \ -libverbs -lrdmacm -noinst_HEADERS = rdma.h name.h +noinst_HEADERS = rdma.h name.h rpc-trans-rdma-messages.h -I$(top_srcdir)/libglusterfs/src -I$(top_srcdir)/rpc/rpc-lib/src/ \ -I$(top_srcdir)/xlators/protocol/lib/src/ $(GF_CFLAGS) diff --git a/rpc/rpc-transport/rdma/src/name.c b/rpc/rpc-transport/rdma/src/name.c index 52ce5bbce55..97b2e37121e 100644 --- a/rpc/rpc-transport/rdma/src/name.c +++ b/rpc/rpc-transport/rdma/src/name.c @@ -22,6 +22,8 @@ #include "rpc-transport.h" #include "rdma.h" #include "common-utils.h" +#include "rpc-lib-messages.h" +#include "rpc-trans-rdma-messages.h" int32_t @@ -38,7 +40,7 @@ af_inet_bind_to_port_lt_ceiling (struct rdma_cm_id *cm_id, { int32_t ret = -1; uint16_t port = ceiling - 1; - // by default assume none of the ports are blocked and all are available + /* by default assume none of the ports are blocked and all are available */ gf_boolean_t ports[1024] = {_gf_false,}; int i = 0; @@ -48,10 +50,8 @@ af_inet_bind_to_port_lt_ceiling (struct rdma_cm_id *cm_id, ports[i] = _gf_false; } - while (port) - { - switch (sockaddr->sa_family) - { + while (port) { + switch (sockaddr->sa_family) { case AF_INET6: ((struct sockaddr_in6 *)sockaddr)->sin6_port = htons (port); @@ -63,7 +63,7 @@ af_inet_bind_to_port_lt_ceiling (struct rdma_cm_id *cm_id, = htons (port); break; } - // ignore the reserved ports + /* ignore the reserved ports */ if (ports[port] == _gf_true) { port--; continue; @@ -134,7 +134,8 @@ client_fill_address_family (rpc_transport_t *this, struct sockaddr *sockaddr) if (!(remote_host_data || connect_path_data) || (remote_host_data && connect_path_data)) { - gf_log (this->name, GF_LOG_ERROR, + gf_msg (this->name, GF_LOG_ERROR, 0, + TRANS_MSG_ADDR_FAMILY_NOT_SPECIFIED, "address-family not specified and not able to " "determine the same from other options " "(remote-host:%s and connect-path:%s)", @@ -144,14 +145,13 @@ client_fill_address_family (rpc_transport_t *this, struct sockaddr *sockaddr) } if (remote_host_data) { - gf_log (this->name, GF_LOG_DEBUG, - "address-family not specified, guessing it " - "to be inet/inet6"); + gf_msg_debug (this->name, 0, "address-family not " + "specified, guessing it to be " + "inet/inet6"); sockaddr->sa_family = AF_UNSPEC; } else { - gf_log (this->name, GF_LOG_DEBUG, - "address-family not specified, guessing it " - "to be unix"); + gf_msg_debug (this->name, 0, "address-family not " + "specified, guessing it to be unix"); sockaddr->sa_family = AF_UNIX; } @@ -166,7 +166,8 @@ client_fill_address_family (rpc_transport_t *this, struct sockaddr *sockaddr) } else if (!strcasecmp (address_family, "inet-sdp")) { sockaddr->sa_family = AF_INET_SDP; } else { - gf_log (this->name, GF_LOG_ERROR, + gf_msg (this->name, GF_LOG_ERROR, 0, + TRANS_MSG_UNKNOWN_ADDR_FAMILY, "unknown address-family (%s) specified", address_family); sockaddr->sa_family = AF_UNSPEC; @@ -191,47 +192,40 @@ af_inet_client_get_remote_sockaddr (rpc_transport_t *this, int32_t ret = 0; remote_host_data = dict_get (options, "remote-host"); - if (remote_host_data == NULL) - { - gf_log (this->name, GF_LOG_ERROR, - "option remote-host missing in volume %s", - this->name); + if (remote_host_data == NULL) { + gf_msg (this->name, GF_LOG_ERROR, 0, + TRANS_MSG_REMOTE_HOST_ERROR, "option remote-host " + "missing in volume %s", this->name); ret = -1; goto err; } remote_host = data_to_str (remote_host_data); - if (remote_host == NULL) - { - gf_log (this->name, GF_LOG_ERROR, - "option remote-host has data NULL in volume %s", - this->name); + if (remote_host == NULL) { + gf_msg (this->name, GF_LOG_ERROR, 0, + TRANS_MSG_REMOTE_HOST_ERROR, "option remote-host " + "has data NULL in volume %s", this->name); ret = -1; goto err; } if (remote_port == 0) { remote_port_data = dict_get (options, "remote-port"); - if (remote_port_data == NULL) - { - gf_log (this->name, GF_LOG_DEBUG, - "option remote-port missing in volume %s. " - "Defaulting to %d", - this->name, GF_DEFAULT_RDMA_LISTEN_PORT); + if (remote_port_data == NULL) { + gf_msg_debug (this->name, 0, "option remote-port " + "missing in volume %s. Defaulting to %d", + this->name, GF_DEFAULT_RDMA_LISTEN_PORT); remote_port = GF_DEFAULT_RDMA_LISTEN_PORT; - } - else - { + } else { remote_port = data_to_uint16 (remote_port_data); } } - if (remote_port == -1) - { - gf_log (this->name, GF_LOG_ERROR, - "option remote-port has invalid port in volume %s", - this->name); + if (remote_port == -1) { + gf_msg (this->name, GF_LOG_ERROR, EINVAL, + RDMA_MSG_INVALID_ENTRY, "option remote-port has " + "invalid port in volume %s", this->name); ret = -1; goto err; } @@ -242,7 +236,7 @@ af_inet_client_get_remote_sockaddr (rpc_transport_t *this, sockaddr->sa_family, &this->dnscache, &addr_info); if (ret == -1) { - gf_log (this->name, GF_LOG_ERROR, + gf_msg (this->name, GF_LOG_ERROR, 0, TRANS_MSG_DNS_RESOL_FAILED, "DNS resolution failed on host %s", remote_host); goto err; } @@ -267,32 +261,32 @@ af_unix_client_get_remote_sockaddr (rpc_transport_t *this, connect_path_data = dict_get (this->options, "transport.rdma.connect-path"); if (!connect_path_data) { - gf_log (this->name, GF_LOG_ERROR, - "option transport.rdma.connect-path not " - "specified for address-family unix"); + gf_msg (this->name, GF_LOG_ERROR, 0, + TRANS_MSG_CONNECT_PATH_ERROR, "option " + "transport.rdma.connect-path not specified for " + "address-family unix"); ret = -1; goto err; } connect_path = data_to_str (connect_path_data); if (!connect_path) { - gf_log (this->name, GF_LOG_ERROR, - "connect-path is null-string"); + gf_msg (this->name, GF_LOG_ERROR, EINVAL, + RDMA_MSG_INVALID_ENTRY, "connect-path is null-string"); ret = -1; goto err; } if (strlen (connect_path) > UNIX_PATH_MAX) { - gf_log (this->name, GF_LOG_ERROR, + gf_msg (this->name, GF_LOG_ERROR, 0, + TRANS_MSG_CONNECT_PATH_ERROR, "connect-path value length %"GF_PRI_SIZET" > " "%d octets", strlen (connect_path), UNIX_PATH_MAX); ret = -1; goto err; } - gf_log (this->name, - GF_LOG_DEBUG, - "using connect-path %s", connect_path); + gf_msg_debug (this->name, 0, "using connect-path %s", connect_path); sockaddr_un = (struct sockaddr_un *)sockaddr; strcpy (sockaddr_un->sun_path, connect_path); *sockaddr_len = sizeof (struct sockaddr_un); @@ -315,7 +309,8 @@ af_unix_server_get_local_sockaddr (rpc_transport_t *this, listen_path_data = dict_get (this->options, "transport.rdma.listen-path"); if (!listen_path_data) { - gf_log (this->name, GF_LOG_ERROR, + gf_msg (this->name, GF_LOG_ERROR, 0, + TRANS_MSG_LISTEN_PATH_ERROR, "missing option listen-path"); ret = -1; goto err; @@ -328,8 +323,9 @@ af_unix_server_get_local_sockaddr (rpc_transport_t *this, #endif if (strlen (listen_path) > UNIX_PATH_MAX) { - gf_log (this->name, GF_LOG_ERROR, - "option listen-path has value length %"GF_PRI_SIZET" > %d", + gf_msg (this->name, GF_LOG_ERROR, 0, + TRANS_MSG_LISTEN_PATH_ERROR, "option listen-path has " + "value length %"GF_PRI_SIZET" > %d", strlen (listen_path), UNIX_PATH_MAX); ret = -1; goto err; @@ -399,7 +395,8 @@ af_inet_server_get_local_sockaddr (rpc_transport_t *this, ret = getaddrinfo(listen_host, service, &hints, &res); if (ret != 0) { - gf_log (this->name, GF_LOG_ERROR, + gf_msg (this->name, GF_LOG_ERROR, ret, + TRANS_MSG_GET_ADDR_INFO_FAILED, "getaddrinfo failed for host %s, service %s (%s)", listen_host, service, gai_strerror (ret)); ret = -1; @@ -422,8 +419,7 @@ gf_rdma_client_bind (rpc_transport_t *this, struct sockaddr *sockaddr, int ret = 0; *sockaddr_len = sizeof (struct sockaddr_in6); - switch (sockaddr->sa_family) - { + switch (sockaddr->sa_family) { case AF_INET_SDP: case AF_INET: *sockaddr_len = sizeof (struct sockaddr_in); @@ -433,7 +429,8 @@ gf_rdma_client_bind (rpc_transport_t *this, struct sockaddr *sockaddr, *sockaddr_len, GF_CLIENT_PORT_CEILING); if (ret == -1) { - gf_log (this->name, GF_LOG_WARNING, + gf_msg (this->name, GF_LOG_WARNING, errno, + RDMA_MSG_PORT_BIND_FAILED, "cannot bind rdma_cm_id to port " "less than %d (%s)", GF_CLIENT_PORT_CEILING, strerror (errno)); @@ -457,7 +454,8 @@ gf_rdma_client_bind (rpc_transport_t *this, struct sockaddr *sockaddr, break; default: - gf_log (this->name, GF_LOG_ERROR, + gf_msg (this->name, GF_LOG_ERROR, 0, + TRANS_MSG_UNKNOWN_ADDR_FAMILY, "unknown address family %d", sockaddr->sa_family); ret = -1; break; @@ -481,8 +479,7 @@ gf_rdma_client_get_remote_sockaddr (rpc_transport_t *this, goto err; } - switch (sockaddr->sa_family) - { + switch (sockaddr->sa_family) { case AF_INET_SDP: sockaddr->sa_family = AF_INET; is_inet_sdp = 1; @@ -508,7 +505,8 @@ gf_rdma_client_get_remote_sockaddr (rpc_transport_t *this, break; default: - gf_log (this->name, GF_LOG_ERROR, + gf_msg (this->name, GF_LOG_ERROR, 0, + TRANS_MSG_UNKNOWN_ADDR_FAMILY, "unknown address-family %d", sockaddr->sa_family); ret = -1; } @@ -541,22 +539,20 @@ gf_rdma_server_get_local_sockaddr (rpc_transport_t *this, } else if (!strcasecmp (address_family, "unix")) { addr->sa_family = AF_UNIX; } else { - gf_log (this->name, GF_LOG_ERROR, - "unknown address family (%s) specified", - address_family); + gf_msg (this->name, GF_LOG_ERROR, 0, + TRANS_MSG_UNKNOWN_ADDR_FAMILY, "unknown address" + " family (%s) specified", address_family); addr->sa_family = AF_UNSPEC; ret = -1; goto err; } } else { - gf_log (this->name, GF_LOG_DEBUG, - "option address-family not specified, defaulting " - "to inet"); + gf_msg_debug (this->name, 0, "option address-family not " + "specified, defaulting to inet"); addr->sa_family = AF_INET; } - switch (addr->sa_family) - { + switch (addr->sa_family) { case AF_INET_SDP: is_inet_sdp = 1; addr->sa_family = AF_INET; @@ -639,8 +635,8 @@ fill_inet6_inet_identifiers (rpc_transport_t *this, struct sockaddr_storage *add service, sizeof (service), NI_NUMERICHOST | NI_NUMERICSERV); if (ret != 0) { - gf_log (this->name, - GF_LOG_ERROR, + gf_msg (this->name, GF_LOG_ERROR, ret, + TRANS_MSG_GET_NAME_INFO_FAILED, "getnameinfo failed (%s)", gai_strerror (ret)); } @@ -655,21 +651,20 @@ gf_rdma_get_transport_identifiers (rpc_transport_t *this) int32_t ret = 0; char is_inet_sdp = 0; - switch (((struct sockaddr *) &this->myinfo.sockaddr)->sa_family) - { + switch (((struct sockaddr *) &this->myinfo.sockaddr)->sa_family) { case AF_INET_SDP: is_inet_sdp = 1; ((struct sockaddr *) &this->peerinfo.sockaddr)->sa_family = ((struct sockaddr *) &this->myinfo.sockaddr)->sa_family = AF_INET; case AF_INET: - case AF_INET6: - { + case AF_INET6: { ret = fill_inet6_inet_identifiers (this, &this->myinfo.sockaddr, this->myinfo.sockaddr_len, this->myinfo.identifier); if (ret == -1) { - gf_log (this->name, GF_LOG_ERROR, + gf_msg (this->name, GF_LOG_ERROR, 0, + TRANS_MSG_INET_ERROR, "can't fill inet/inet6 identifier for server"); goto err; } @@ -679,7 +674,8 @@ gf_rdma_get_transport_identifiers (rpc_transport_t *this) this->peerinfo.sockaddr_len, this->peerinfo.identifier); if (ret == -1) { - gf_log (this->name, GF_LOG_ERROR, + gf_msg (this->name, GF_LOG_ERROR, 0, + TRANS_MSG_INET_ERROR, "can't fill inet/inet6 identifier for client"); goto err; } @@ -703,7 +699,8 @@ gf_rdma_get_transport_identifiers (rpc_transport_t *this) break; default: - gf_log (this->name, GF_LOG_ERROR, + gf_msg (this->name, GF_LOG_ERROR, 0, + TRANS_MSG_UNKNOWN_ADDR_FAMILY, "unknown address family (%d)", ((struct sockaddr *) &this->myinfo.sockaddr)->sa_family); ret = -1; diff --git a/rpc/rpc-transport/rdma/src/rdma.c b/rpc/rpc-transport/rdma/src/rdma.c index f62f40116f5..89fb6cb17d7 100644 --- a/rpc/rpc-transport/rdma/src/rdma.c +++ b/rpc/rpc-transport/rdma/src/rdma.c @@ -22,6 +22,8 @@ #include "byte-order.h" #include "xlator.h" #include "xdr-rpc.h" +#include "rpc-lib-messages.h" +#include "rpc-trans-rdma-messages.h" #include <signal.h> #define GF_RDMA_LOG_NAME "rpc-transport/rdma" @@ -116,7 +118,8 @@ gf_rdma_new_post (rpc_transport_t *this, gf_rdma_device_t *device, int32_t len, post->buf_size, IBV_ACCESS_LOCAL_WRITE); if (!post->mr) { - gf_log (this->name, GF_LOG_WARNING, + gf_msg (this->name, GF_LOG_WARNING, errno, + RDMA_MSG_MR_ALOC_FAILED, "memory registration failed (%s)", strerror (errno)); goto out; @@ -324,12 +327,13 @@ __gf_rdma_create_posts (rpc_transport_t *this, int32_t count, int32_t size, priv = this->private; device = priv->device; - for (i=0 ; i<count ; i++) { + for (i = 0 ; i < count ; i++) { gf_rdma_post_t *post = NULL; post = gf_rdma_new_post (this, device, size + 2048, type); if (!post) { - gf_log (this->name, GF_LOG_ERROR, + gf_msg (this->name, GF_LOG_ERROR, 0, + RDMA_MSG_POST_CREATION_FAILED, "post creation failed"); ret = -1; break; @@ -399,9 +403,10 @@ gf_rdma_deregister_arena (struct list_head **mr_list, list_for_each_entry_safe (tmp, dummy, mr_list[i], list) { if (tmp->iobuf_arena == iobuf_arena) { if (ibv_dereg_mr(tmp->mr)) { - gf_log("rdma", GF_LOG_WARNING, - "deallocation of memory region " - "failed"); + gf_msg ("rdma", GF_LOG_WARNING, 0, + RDMA_MSG_DEREGISTER_ARENA_FAILED, + "deallocation of memory region " + "failed"); return -1; } list_del(&tmp->list); @@ -431,8 +436,10 @@ gf_rdma_register_arena (void **arg1, void *arg2) new = GF_CALLOC(1, sizeof(gf_rdma_arena_mr), gf_common_mt_rdma_arena_mr); if (new == NULL) { - gf_log ("rdma", GF_LOG_INFO, "Out of memory:" - " registering pre allocated buffer wth rdma device failed."); + gf_msg ("rdma", GF_LOG_INFO, ENOMEM, + RDMA_MSG_MR_ALOC_FAILED, "Out of " + "memory: registering pre allocated buffer " + "with rdma device failed."); return -1; } INIT_LIST_HEAD (&new->list); @@ -445,8 +452,9 @@ gf_rdma_register_arena (void **arg1, void *arg2) IBV_ACCESS_REMOTE_WRITE ); if (!mr) - gf_log("rdma", GF_LOG_WARNING, - "allocation of mr failed"); + gf_msg ("rdma", GF_LOG_WARNING, 0, + RDMA_MSG_MR_ALOC_FAILED, "allocation of mr " + "failed"); new->mr = mr; list_add (&new->list, &device[i]->all_mr); @@ -473,8 +481,10 @@ gf_rdma_register_iobuf_pool (gf_rdma_device_t *device, new = GF_CALLOC(1, sizeof(gf_rdma_arena_mr), gf_common_mt_rdma_arena_mr); if (new == NULL) { - gf_log ("rdma", GF_LOG_INFO, "Out of memory:" - " registering pre allocated buffer with rdma device failed."); + gf_msg ("rdma", GF_LOG_INFO, ENOMEM, + RDMA_MSG_MR_ALOC_FAILED, "Out of " + "memory: registering pre allocated " + "buffer with rdma device failed."); return; } INIT_LIST_HEAD (&new->list); @@ -486,8 +496,9 @@ gf_rdma_register_iobuf_pool (gf_rdma_device_t *device, IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE); if (!mr) { - gf_log ("rdma", GF_LOG_WARNING, "failed to pre" - " register buffers with rdma " + gf_msg ("rdma", GF_LOG_WARNING, 0, + RDMA_MSG_MR_ALOC_FAILED, "failed" + " to pre register buffers with rdma " "devices."); } @@ -557,7 +568,7 @@ gf_rdma_create_posts (rpc_transport_t *this) GF_RDMA_RECV_POST); if (!ret) { - for (i=0 ; i<options->recv_count ; i++) { + for (i = 0 ; i < options->recv_count ; i++) { post = gf_rdma_get_post (&device->recvq); if (gf_rdma_post_recv (device->srq, post) != 0) { ret = -1; @@ -614,14 +625,15 @@ gf_rdma_create_cq (rpc_transport_t *this) device->recv_chan, 0); if (!device->recv_cq) { - gf_log (this->name, GF_LOG_ERROR, - "creation of CQ for device %s failed", - device->device_name); + gf_msg (this->name, GF_LOG_ERROR, 0, + RDMA_MSG_CQ_CREATION_FAILED, "creation of CQ for " + "device %s failed", device->device_name); ret = -1; goto out; } else if (ibv_req_notify_cq (device->recv_cq, 0)) { - gf_log (this->name, GF_LOG_ERROR, - "ibv_req_notify_cq on recv CQ of device %s failed", + gf_msg (this->name, GF_LOG_ERROR, 0, + RDMA_MSG_REQ_NOTIFY_CQ_REVQ_FAILED, "ibv_req_notify_" + "cq on recv CQ of device %s failed", device->device_name); ret = -1; goto out; @@ -630,8 +642,9 @@ gf_rdma_create_cq (rpc_transport_t *this) do { ret = ibv_query_device (priv->device->context, &device_attr); if (ret != 0) { - gf_log (this->name, GF_LOG_ERROR, - "ibv_query_device on %s returned %d (%s)", + gf_msg (this->name, GF_LOG_ERROR, 0, + RDMA_MSG_QUERY_DEVICE_FAILED, "ibv_query_" + "device on %s returned %d (%s)", priv->device->device_name, ret, (ret > 0) ? strerror (ret) : ""); ret = -1; @@ -647,15 +660,17 @@ gf_rdma_create_cq (rpc_transport_t *this) send_cqe, device, device->send_chan, 0); if (!device->send_cq) { - gf_log (this->name, GF_LOG_ERROR, - "creation of send_cq for device %s failed", - device->device_name); + gf_msg (this->name, GF_LOG_ERROR, 0, + RDMA_MSG_CQ_CREATION_FAILED, + "creation of send_cq " + "for device %s failed", device->device_name); ret = -1; goto out; } if (ibv_req_notify_cq (device->send_cq, 0)) { - gf_log (this->name, GF_LOG_ERROR, + gf_msg (this->name, GF_LOG_ERROR, 0, + RDMA_MSG_REQ_NOTIFY_CQ_SENDQ_FAILED, "ibv_req_notify_cq on send_cq for device %s" " failed", device->device_name); ret = -1; @@ -734,16 +749,18 @@ gf_rdma_get_device (rpc_transport_t *this, struct ibv_context *ibctx, trav->send_chan = ibv_create_comp_channel (trav->context); if (!trav->send_chan) { - gf_log (this->name, GF_LOG_ERROR, - "could not create send completion channel for " + gf_msg (this->name, GF_LOG_ERROR, 0, + RDMA_MSG_SEND_COMP_CHAN_FAILED, "could not " + "create send completion channel for " "device (%s)", device_name); goto out; } trav->recv_chan = ibv_create_comp_channel (trav->context); if (!trav->recv_chan) { - gf_log (this->name, GF_LOG_ERROR, - "could not create recv completion channel for " + gf_msg (this->name, GF_LOG_ERROR, 0, + RDMA_MSG_RECV_COMP_CHAN_FAILED, "could not " + "create recv completion channel for " "device (%s)", device_name); /* TODO: cleanup current mess */ @@ -751,7 +768,8 @@ gf_rdma_get_device (rpc_transport_t *this, struct ibv_context *ibctx, } if (gf_rdma_create_cq (this) < 0) { - gf_log (this->name, GF_LOG_ERROR, + gf_msg (this->name, GF_LOG_ERROR, 0, + RDMA_MSG_CQ_CREATION_FAILED, "could not create CQ for device (%s)", device_name); goto out; @@ -761,9 +779,10 @@ gf_rdma_get_device (rpc_transport_t *this, struct ibv_context *ibctx, trav->pd = ibv_alloc_pd (trav->context); if (!trav->pd) { - gf_log (this->name, GF_LOG_ERROR, - "could not allocate protection domain for " - "device (%s)", device_name); + gf_msg (this->name, GF_LOG_ERROR, 0, + RDMA_MSG_ALOC_PROT_DOM_FAILED, "could not " + "allocate protection domain for device (%s)", + device_name); goto out; } @@ -777,8 +796,9 @@ gf_rdma_get_device (rpc_transport_t *this, struct ibv_context *ibctx, trav->srq = ibv_create_srq (trav->pd, &attr); if (!trav->srq) { - gf_log (this->name, GF_LOG_ERROR, - "could not create SRQ for device (%s)", + gf_msg (this->name, GF_LOG_ERROR, 0, + RDMA_MSG_CRE_SRQ_FAILED, "could not create SRQ" + " for device (%s)", device_name); goto out; } @@ -791,9 +811,9 @@ gf_rdma_get_device (rpc_transport_t *this, struct ibv_context *ibctx, gf_rdma_register_iobuf_pool(trav, iobuf_pool); if (gf_rdma_create_posts (this) < 0) { - gf_log (this->name, GF_LOG_ERROR, - "could not allocate posts for device (%s)", - device_name); + gf_msg (this->name, GF_LOG_ERROR, 0, + RDMA_MSG_ALOC_POST_FAILED, "could not allocate" + "posts for device (%s)", device_name); goto out; } @@ -802,7 +822,8 @@ gf_rdma_get_device (rpc_transport_t *this, struct ibv_context *ibctx, gf_rdma_send_completion_proc, trav->send_chan); if (ret) { - gf_log (this->name, GF_LOG_ERROR, + gf_msg (this->name, GF_LOG_ERROR, 0, + RDMA_MSG_SEND_COMP_THREAD_FAILED, "could not create send completion thread for " "device (%s)", device_name); goto out; @@ -812,7 +833,8 @@ gf_rdma_get_device (rpc_transport_t *this, struct ibv_context *ibctx, gf_rdma_recv_completion_proc, trav->recv_chan); if (ret) { - gf_log (this->name, GF_LOG_ERROR, + gf_msg (this->name, GF_LOG_ERROR, 0, + RDMA_MSG_RECV_COMP_THREAD_FAILED, "could not create recv completion thread " "for device (%s)", device_name); return NULL; @@ -822,14 +844,15 @@ gf_rdma_get_device (rpc_transport_t *this, struct ibv_context *ibctx, gf_rdma_async_event_thread, ibctx); if (ret) { - gf_log (this->name, GF_LOG_ERROR, + gf_msg (this->name, GF_LOG_ERROR, 0, + RDMA_MSG_ASYNC_EVENT_THEAD_FAILED, "could not create async_event_thread"); return NULL; } /* qpreg */ pthread_mutex_init (&trav->qpreg.lock, NULL); - for (i=0; i<42; i++) { + for (i = 0; i < 42; i++) { trav->qpreg.ents[i].next = &trav->qpreg.ents[i]; trav->qpreg.ents[i].prev = &trav->qpreg.ents[i]; } @@ -912,19 +935,20 @@ gf_rdma_transport_new (rpc_transport_t *listener, struct rdma_cm_id *cm_id) device_name = (char *)ibv_get_device_name (cm_id->verbs->device); if (device_name == NULL) { - gf_log (listener->name, GF_LOG_WARNING, - "cannot get device name (peer:%s me:%s)", - this->peerinfo.identifier, this->myinfo.identifier); + gf_msg (listener->name, GF_LOG_WARNING, 0, + RDMA_MSG_GET_DEVICE_NAME_FAILED, "cannot get device " + "name (peer:%s me:%s)", this->peerinfo.identifier, + this->myinfo.identifier); goto out; } priv->device = gf_rdma_get_device (this, cm_id->verbs, device_name); if (priv->device == NULL) { - gf_log (listener->name, GF_LOG_WARNING, - "cannot get infiniband device %s (peer:%s me:%s)", - device_name, this->peerinfo.identifier, - this->myinfo.identifier); + gf_msg (listener->name, GF_LOG_WARNING, 0, + RDMA_MSG_GET_IB_DEVICE_FAILED, "cannot get infiniband" + " device %s (peer:%s me:%s)", device_name, + this->peerinfo.identifier, this->myinfo.identifier); goto out; } @@ -978,23 +1002,25 @@ gf_rdma_cm_handle_connect_request (struct rdma_cm_event *event) this = gf_rdma_transport_new (listener, child_cm_id); if (this == NULL) { - gf_log (listener->name, GF_LOG_WARNING, - "could not create a transport for incoming connection" + gf_msg (listener->name, GF_LOG_WARNING, 0, + RDMA_MSG_CREAT_INC_TRANS_FAILED, "could not create " + "a transport for incoming connection" " (me.name:%s me.identifier:%s)", listener->name, listener->myinfo.identifier); rdma_destroy_id (child_cm_id); goto out; } - gf_log (listener->name, GF_LOG_TRACE, - "got a connect request (me:%s peer:%s)", - listener->myinfo.identifier, this->peerinfo.identifier); + gf_msg_trace (listener->name, 0, "got a connect request (me:%s peer:" + "%s)", listener->myinfo.identifier, + this->peerinfo.identifier); ret = gf_rdma_create_qp (this); if (ret < 0) { - gf_log (listener->name, GF_LOG_WARNING, - "could not create QP (peer:%s me:%s)", - this->peerinfo.identifier, this->myinfo.identifier); + gf_msg (listener->name, GF_LOG_WARNING, 0, + RDMA_MSG_CREAT_QP_FAILED, "could not create QP " + "(peer:%s me:%s)", this->peerinfo.identifier, + this->myinfo.identifier); gf_rdma_cm_handle_disconnect (this); goto out; } @@ -1006,8 +1032,9 @@ gf_rdma_cm_handle_connect_request (struct rdma_cm_event *event) ret = rdma_accept(child_cm_id, &conn_param); if (ret < 0) { - gf_log (listener->name, GF_LOG_WARNING, "rdma_accept failed " - "peer:%s me:%s (%s)", this->peerinfo.identifier, + gf_msg (listener->name, GF_LOG_WARNING, errno, + RDMA_MSG_ACCEPT_FAILED, "rdma_accept failed peer:%s " + "me:%s (%s)", this->peerinfo.identifier, this->myinfo.identifier, strerror (errno)); gf_rdma_cm_handle_disconnect (this); goto out; @@ -1042,9 +1069,10 @@ gf_rdma_cm_handle_route_resolved (struct rdma_cm_event *event) ret = gf_rdma_create_qp (this); if (ret != 0) { - gf_log (this->name, GF_LOG_WARNING, - "could not create QP (peer:%s me:%s)", - this->peerinfo.identifier, this->myinfo.identifier); + gf_msg (this->name, GF_LOG_WARNING, 0, + RDMA_MSG_CREAT_QP_FAILED, "could not create QP " + "(peer:%s me:%s)", this->peerinfo.identifier, + this->myinfo.identifier); gf_rdma_cm_handle_disconnect (this); goto out; } @@ -1057,14 +1085,15 @@ gf_rdma_cm_handle_route_resolved (struct rdma_cm_event *event) ret = rdma_connect(peer->cm_id, &conn_param); if (ret != 0) { - gf_log (this->name, GF_LOG_WARNING, + gf_msg (this->name, GF_LOG_WARNING, errno, + RDMA_MSG_CONNECT_FAILED, "rdma_connect failed (%s)", strerror (errno)); gf_rdma_cm_handle_disconnect (this); goto out; } - gf_log (this->name, GF_LOG_TRACE, "route resolved (me:%s peer:%s)", - this->myinfo.identifier, this->peerinfo.identifier); + gf_msg_trace (this->name, 0, "route resolved (me:%s peer:%s)", + this->myinfo.identifier, this->peerinfo.identifier); ret = 0; out: @@ -1099,15 +1128,16 @@ gf_rdma_cm_handle_addr_resolved (struct rdma_cm_event *event) ret = rdma_resolve_route(peer->cm_id, 2000); if (ret != 0) { - gf_log (this->name, GF_LOG_WARNING, - "rdma_resolve_route failed (me:%s peer:%s) (%s)", + gf_msg (this->name, GF_LOG_WARNING, errno, + RDMA_MSG_ROUTE_RESOLVE_FAILED, "rdma_resolve_route " + "failed (me:%s peer:%s) (%s)", this->myinfo.identifier, this->peerinfo.identifier, strerror (errno)); gf_rdma_cm_handle_disconnect (this); } - gf_log (this->name, GF_LOG_TRACE, "Address resolved (me:%s peer:%s)", - this->myinfo.identifier, this->peerinfo.identifier); + gf_msg_trace (this->name, 0, "Address resolved (me:%s peer:%s)", + this->myinfo.identifier, this->peerinfo.identifier); return ret; } @@ -1120,8 +1150,7 @@ gf_rdma_cm_handle_disconnect (rpc_transport_t *this) char need_unref = 0; priv = this->private; - gf_log (this->name, GF_LOG_DEBUG, - "peer disconnected, cleaning up"); + gf_msg_debug (this->name, 0, "peer disconnected, cleaning up"); pthread_mutex_lock (&priv->write_mutex); { @@ -1155,9 +1184,10 @@ gf_rdma_cm_handle_connect_init (struct rdma_cm_event *event) priv = this->private; if (priv->connected == 1) { - gf_log (this->name, GF_LOG_TRACE, - "received event RDMA_CM_EVENT_ESTABLISHED (me:%s peer:%s)", - this->myinfo.identifier, this->peerinfo.identifier); + gf_msg_trace (this->name, 0, "received event " + "RDMA_CM_EVENT_ESTABLISHED (me:%s peer:%s)", + this->myinfo.identifier, + this->peerinfo.identifier); return ret; } @@ -1171,9 +1201,10 @@ gf_rdma_cm_handle_connect_init (struct rdma_cm_event *event) pthread_mutex_unlock (&priv->write_mutex); if (priv->entity == GF_RDMA_CLIENT) { - gf_log (this->name, GF_LOG_TRACE, - "received event RDMA_CM_EVENT_ESTABLISHED (me:%s peer:%s)", - this->myinfo.identifier, this->peerinfo.identifier); + gf_msg_trace (this->name, 0, "received event " + "RDMA_CM_EVENT_ESTABLISHED (me:%s peer:%s)", + this->myinfo.identifier, + this->peerinfo.identifier); ret = rpc_transport_notify (this, RPC_TRANSPORT_CONNECT, this); } else if (priv->entity == GF_RDMA_SERVER) { @@ -1224,7 +1255,8 @@ gf_rdma_cm_event_handler (void *data) while (1) { ret = rdma_get_cm_event (event_channel, &event); if (ret != 0) { - gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING, + gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, errno, + RDMA_MSG_CM_EVENT_FAILED, "rdma_cm_get_event failed (%s)", strerror (errno)); break; @@ -1254,10 +1286,9 @@ gf_rdma_cm_event_handler (void *data) case RDMA_CM_EVENT_REJECTED: this = event->id->context; - gf_log (this->name, GF_LOG_WARNING, - "cma event %s, error %d (me:%s peer:%s)\n " - "Host Unreachable, Check your connection " - "with IPoIB", + gf_msg (this->name, GF_LOG_WARNING, 0, + RDMA_MSG_CM_EVENT_FAILED, "cma event %s, " + "error %d (me:%s peer:%s)\n", rdma_event_str(event->event), event->status, this->myinfo.identifier, this->peerinfo.identifier); @@ -1271,10 +1302,10 @@ gf_rdma_cm_event_handler (void *data) case RDMA_CM_EVENT_DISCONNECTED: this = event->id->context; - gf_log (this->name, GF_LOG_DEBUG, - "received disconnect (me:%s peer:%s)\n", - this->myinfo.identifier, - this->peerinfo.identifier); + gf_msg_debug (this->name, 0, "received disconnect " + "(me:%s peer:%s)\n", + this->myinfo.identifier, + this->peerinfo.identifier); rdma_ack_cm_event (event); event = NULL; @@ -1283,13 +1314,15 @@ gf_rdma_cm_event_handler (void *data) continue; case RDMA_CM_EVENT_DEVICE_REMOVAL: - gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING, - "device removed"); + gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0, + RDMA_MSG_CM_EVENT_FAILED, "device " + "removed"); gf_rdma_cm_handle_device_removal (event); break; default: - gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING, + gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0, + RDMA_MSG_CM_EVENT_FAILED, "unhandled event: %s, ignoring", rdma_event_str(event->event)); break; @@ -1367,8 +1400,8 @@ __gf_rdma_send_error (gf_rdma_peer_t *peer, gf_rdma_ioq_t *entry, len = __gf_rdma_encode_error (peer, reply_info, entry->rpchdr, (gf_rdma_header_t *)post->buf, err); if (len == -1) { - gf_log (GF_RDMA_LOG_NAME, GF_LOG_ERROR, - "encode error returned -1"); + gf_msg (GF_RDMA_LOG_NAME, GF_LOG_ERROR, 0, + RDMA_MSG_ENCODE_ERROR, "encode error returned -1"); goto out; } @@ -1378,7 +1411,8 @@ __gf_rdma_send_error (gf_rdma_peer_t *peer, gf_rdma_ioq_t *entry, if (!ret) { ret = len; } else { - gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING, + gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0, + RDMA_MSG_POST_SEND_FAILED, "gf_rdma_post_send (to %s) failed with ret = %d (%s)", peer->trans->peerinfo.identifier, ret, (ret > 0) ? strerror (ret) : ""); @@ -1428,7 +1462,8 @@ __gf_rdma_create_read_chunks_from_vector (gf_rdma_peer_t *peer, IBV_ACCESS_REMOTE_READ); } if (!mr) { - gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING, + gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, errno, + RDMA_MSG_MR_ALOC_FAILED, "memory registration failed (%s) (peer:%s)", strerror (errno), peer->trans->peerinfo.identifier); @@ -1480,7 +1515,8 @@ __gf_rdma_create_read_chunks (gf_rdma_peer_t *peer, gf_rdma_ioq_t *entry, entry->rpchdr_count, request_ctx); if (ret == -1) { - gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING, + gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0, + RDMA_MSG_READ_CHUNK_VECTOR_FAILED, "cannot create read chunks from vector " "entry->rpchdr"); goto out; @@ -1493,7 +1529,8 @@ __gf_rdma_create_read_chunks (gf_rdma_peer_t *peer, gf_rdma_ioq_t *entry, entry->proghdr_count, request_ctx); if (ret == -1) { - gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING, + gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0, + RDMA_MSG_READ_CHUNK_VECTOR_FAILED, "cannot create read chunks from vector " "entry->proghdr"); } @@ -1506,7 +1543,8 @@ __gf_rdma_create_read_chunks (gf_rdma_peer_t *peer, gf_rdma_ioq_t *entry, entry->prog_payload_count, request_ctx); if (ret == -1) { - gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING, + gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0, + RDMA_MSG_READ_CHUNK_VECTOR_FAILED, "cannot create read chunks from vector" " entry->prog_payload"); } @@ -1520,7 +1558,8 @@ __gf_rdma_create_read_chunks (gf_rdma_peer_t *peer, gf_rdma_ioq_t *entry, entry->prog_payload_count, request_ctx); if (ret == -1) { - gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING, + gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0, + RDMA_MSG_READ_CHUNK_VECTOR_FAILED, "cannot create read chunks from vector " "entry->prog_payload"); } @@ -1570,8 +1609,9 @@ __gf_rdma_create_write_chunks_from_vector (gf_rdma_peer_t *peer, } if (!mr) { - gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING, - "memory registration failed (%s) (peer:%s)", + gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, errno, + RDMA_MSG_MR_ALOC_FAILED, "memory " + "registration failed (%s) (peer:%s)", strerror (errno), peer->trans->peerinfo.identifier); goto out; @@ -1612,7 +1652,8 @@ __gf_rdma_create_write_chunks (gf_rdma_peer_t *peer, gf_rdma_ioq_t *entry, if ((chunk_type == gf_rdma_replych) && ((entry->msg.request.rsphdr_count != 1) || (entry->msg.request.rsphdr_vec[0].iov_base == NULL))) { - gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING, + gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0, + RDMA_MSG_BUFFER_ERROR, (entry->msg.request.rsphdr_count == 1) ? "chunktype specified as reply chunk but the vector " "specifying the buffer to be used for holding reply" @@ -1648,7 +1689,8 @@ __gf_rdma_create_write_chunks (gf_rdma_peer_t *peer, gf_rdma_ioq_t *entry, entry->msg.request.rsp_payload_count, request_ctx); if (ret == -1) { - gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING, + gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0, + RDMA_MSG_WRITE_CHUNK_VECTOR_FAILED, "cannot create write chunks from vector " "entry->rpc_payload"); goto out; @@ -1678,7 +1720,8 @@ __gf_rdma_create_write_chunks (gf_rdma_peer_t *peer, gf_rdma_ioq_t *entry, entry->msg.request.rsphdr_count, request_ctx); if (ret == -1) { - gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING, + gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0, + RDMA_MSG_WRITE_CHUNK_VECTOR_FAILED, "cannot create write chunks from vector " "entry->rpchdr"); goto out; @@ -1783,9 +1826,7 @@ __gf_rdma_request_context_destroy (gf_rdma_request_context_t *context) if (priv->connected) { ret = __gf_rdma_quota_put (peer); if (ret < 0) { - gf_log ("rdma", GF_LOG_DEBUG, - "failed to send " - "message"); + gf_msg_debug ("rdma", 0, "failed to send message"); mem_put (context); __gf_rdma_disconnect (peer->trans); goto out; @@ -1924,7 +1965,8 @@ __gf_rdma_ioq_churn_request (gf_rdma_peer_t *peer, gf_rdma_ioq_t *entry, if ((entry->msg.request.rsphdr_count != 0) && (entry->msg.request.rsp_payload_count != 0)) { ret = -1; - gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING, + gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0, + RDMA_MSG_WRITE_REPLY_CHUNCK_CONFLICT, "both write-chunklist and reply-chunk cannot be " "present"); goto out; @@ -1976,7 +2018,8 @@ __gf_rdma_ioq_churn_request (gf_rdma_peer_t *peer, gf_rdma_ioq_t *entry, if (chunk_count > GF_RDMA_MAX_SEGMENTS) { ret = -1; - gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING, + gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0, + RDMA_MSG_CHUNK_COUNT_GREAT_MAX_SEGMENTS, "chunk count(%d) exceeding maximum allowed RDMA " "segment count(%d)", chunk_count, GF_RDMA_MAX_SEGMENTS); goto out; @@ -2022,7 +2065,8 @@ __gf_rdma_ioq_churn_request (gf_rdma_peer_t *peer, gf_rdma_ioq_t *entry, &chunkptr, request_ctx); if (ret != 0) { - gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING, + gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0, + RDMA_MSG_CREATE_READ_CHUNK_FAILED, "creation of read chunks failed"); goto out; } @@ -2035,7 +2079,8 @@ __gf_rdma_ioq_churn_request (gf_rdma_peer_t *peer, gf_rdma_ioq_t *entry, &chunkptr, request_ctx); if (ret != 0) { - gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING, + gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0, + RDMA_MSG_CREATE_WRITE_REPLAY_FAILED, "creation of write/reply chunk failed"); goto out; } @@ -2069,7 +2114,8 @@ __gf_rdma_ioq_churn_request (gf_rdma_peer_t *peer, gf_rdma_ioq_t *entry, if (!ret) { ret = len; } else { - gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING, + gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0, + RDMA_MSG_POST_SEND_FAILED, "gf_rdma_post_send (to %s) failed with ret = %d (%s)", peer->trans->peerinfo.identifier, ret, (ret > 0) ? strerror (ret) : ""); @@ -2141,7 +2187,8 @@ __gf_rdma_send_reply_inline (gf_rdma_peer_t *peer, gf_rdma_ioq_t *entry, if (send_size > GLUSTERFS_RDMA_INLINE_THRESHOLD) { ret = __gf_rdma_send_error (peer, entry, post, reply_info, ERR_CHUNK); - gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING, + gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0, + RDMA_MSG_SEND_SIZE_GREAT_INLINE_THRESHOLD, "msg size (%d) is greater than maximum size " "of msg that can be sent inlined (%d)", send_size, GLUSTERFS_RDMA_INLINE_THRESHOLD); @@ -2178,8 +2225,9 @@ __gf_rdma_send_reply_inline (gf_rdma_peer_t *peer, gf_rdma_ioq_t *entry, if (!ret) { ret = send_size; } else { - gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING, - "posting send (to %s) failed with ret = %d (%s)", + gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0, + RDMA_MSG_POST_SEND_FAILED, "posting send (to %s) " + "failed with ret = %d (%s)", peer->trans->peerinfo.identifier, ret, (ret > 0) ? strerror (ret) : ""); gf_rdma_post_unref (post); @@ -2212,9 +2260,9 @@ __gf_rdma_reply_encode_write_chunks (gf_rdma_peer_t *peer, } if (chunk_size < payload_size) { - gf_log (GF_RDMA_LOG_NAME, GF_LOG_DEBUG, - "length of payload (%d) is exceeding the total " - "write chunk length (%d)", payload_size, chunk_size); + gf_msg_debug (GF_RDMA_LOG_NAME, 0, "length of payload (%d) is " + "exceeding the total write chunk length (%d)", + payload_size, chunk_size); goto out; } @@ -2280,9 +2328,10 @@ __gf_rdma_register_local_mr_for_rdma (gf_rdma_peer_t *peer, IBV_ACCESS_LOCAL_WRITE); } if (ctx->mr[ctx->mr_count] == NULL) { - gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING, - "registering memory for IBV_ACCESS_LOCAL_WRITE " - "failed (%s)", strerror (errno)); + gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, errno, + RDMA_MSG_MR_ALOC_FAILED, + "registering memory for IBV_ACCESS_LOCAL_WRITE" + " failed (%s)", strerror (errno)); goto out; } @@ -2330,9 +2379,9 @@ __gf_rdma_write (gf_rdma_peer_t *peer, gf_rdma_post_t *post, struct iovec *vec, for ((i = *idx), (num_sge = 0); (xfer_len != 0); i++, num_sge++) { size = min (xfer_len, vec[i].iov_len); - sg_list [num_sge].addr = (unsigned long)vec[i].iov_base; - sg_list [num_sge].length = size; - sg_list [num_sge].lkey = post->ctx.mr[i]->lkey; + sg_list[num_sge].addr = (unsigned long)vec[i].iov_base; + sg_list[num_sge].length = size; + sg_list[num_sge].lkey = post->ctx.mr[i]->lkey; xfer_len -= size; } @@ -2353,8 +2402,8 @@ __gf_rdma_write (gf_rdma_peer_t *peer, gf_rdma_post_t *post, struct iovec *vec, ret = ibv_post_send(peer->qp, &wr, &bad_wr); if (ret) { - gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING, - "rdma write to " + gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0, + RDMA_MSG_WRITE_CLIENT_ERROR, "rdma write to " "client (%s) failed with ret = %d (%s)", peer->trans->peerinfo.identifier, ret, (ret > 0) ? strerror (ret) : ""); @@ -2389,7 +2438,8 @@ __gf_rdma_do_gf_rdma_write (gf_rdma_peer_t *peer, gf_rdma_post_t *post, ret = __gf_rdma_register_local_mr_for_rdma (peer, vector, count, &post->ctx); if (ret == -1) { - gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING, + gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0, + RDMA_MSG_MR_ALOC_FAILED, "registering memory region for rdma failed"); goto out; } @@ -2406,8 +2456,9 @@ __gf_rdma_do_gf_rdma_write (gf_rdma_peer_t *peer, gf_rdma_post_t *post, &payload_idx, &reply_info->wc_array->wc_array[i]); if (ret == -1) { - gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING, - "rdma write to client (%s) failed", + gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0, + RDMA_MSG_WRITE_CLIENT_ERROR, "rdma write to " + "client (%s) failed", peer->trans->peerinfo.identifier); goto out; } @@ -2450,8 +2501,8 @@ __gf_rdma_send_reply_type_nomsg (gf_rdma_peer_t *peer, gf_rdma_ioq_t *entry, reply_info, (uint32_t **)&buf); if (ret == -1) { - gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING, - "encoding write chunks failed"); + gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0, + RDMA_MSG_ENCODE_ERROR, "encoding write chunks failed"); ret = __gf_rdma_send_error (peer, entry, post, reply_info, ERR_CHUNK); goto out; @@ -2470,18 +2521,19 @@ __gf_rdma_send_reply_type_nomsg (gf_rdma_peer_t *peer, gf_rdma_ioq_t *entry, ret = __gf_rdma_do_gf_rdma_write (peer, post, vector, count, entry->iobref, reply_info); if (ret == -1) { - gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING, - "rdma write to peer (%s) failed", - peer->trans->peerinfo.identifier); + gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0, + RDMA_MSG_WRITE_PEER_FAILED, "rdma write to peer " + "(%s) failed", peer->trans->peerinfo.identifier); gf_rdma_post_unref (post); goto out; } ret = gf_rdma_post_send (peer->qp, post, (buf - post->buf)); if (ret) { - gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING, - "posting a send request to client (%s) failed with " - "ret = %d (%s)", peer->trans->peerinfo.identifier, ret, + gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0, + RDMA_MSG_POST_SEND_FAILED, "posting a send request " + "to client (%s) failed with ret = %d (%s)", + peer->trans->peerinfo.identifier, ret, (ret > 0) ? strerror (ret) : ""); ret = -1; gf_rdma_post_unref (post); @@ -2509,7 +2561,8 @@ __gf_rdma_send_reply_type_msg (gf_rdma_peer_t *peer, gf_rdma_ioq_t *entry, + GLUSTERFS_RDMA_MAX_HEADER_SIZE; if (send_size > GLUSTERFS_RDMA_INLINE_THRESHOLD) { - gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING, + gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0, + RDMA_MSG_SEND_SIZE_GREAT_INLINE_THRESHOLD, "client has provided only write chunks, but the " "combined size of rpc and program header (%d) is " "exceeding the size of msg that can be sent using " @@ -2534,8 +2587,8 @@ __gf_rdma_send_reply_type_msg (gf_rdma_peer_t *peer, gf_rdma_ioq_t *entry, reply_info, (uint32_t **)&ptr); if (ret == -1) { - gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING, - "encoding write chunks failed"); + gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0, + RDMA_MSG_ENCODE_ERROR, "encoding write chunks failed"); ret = __gf_rdma_send_error (peer, entry, post, reply_info, ERR_CHUNK); goto out; @@ -2550,7 +2603,8 @@ __gf_rdma_send_reply_type_msg (gf_rdma_peer_t *peer, gf_rdma_ioq_t *entry, entry->prog_payload_count, entry->iobref, reply_info); if (ret == -1) { - gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING, "rdma write to peer " + gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0, + RDMA_MSG_WRITE_PEER_FAILED, "rdma write to peer " "(%s) failed", peer->trans->peerinfo.identifier); gf_rdma_post_unref (post); goto out; @@ -2564,7 +2618,8 @@ __gf_rdma_send_reply_type_msg (gf_rdma_peer_t *peer, gf_rdma_ioq_t *entry, ret = gf_rdma_post_send (peer->qp, post, (ptr - post->buf)); if (ret) { - gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING, + gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0, + RDMA_MSG_SEND_CLIENT_ERROR, "rdma send to client (%s) failed with ret = %d (%s)", peer->trans->peerinfo.identifier, ret, (ret > 0) ? strerror (ret) : ""); @@ -2640,7 +2695,8 @@ __gf_rdma_ioq_churn_reply (gf_rdma_peer_t *peer, gf_rdma_ioq_t *entry, ret = __gf_rdma_send_reply_inline (peer, entry, post, reply_info); if (ret < 0) { - gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING, + gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0, + RDMA_MSG_SEND_REPLY_FAILED, "failed to send reply to peer (%s) as an " "inlined rdma msg", peer->trans->peerinfo.identifier); @@ -2651,7 +2707,8 @@ __gf_rdma_ioq_churn_reply (gf_rdma_peer_t *peer, gf_rdma_ioq_t *entry, ret = __gf_rdma_send_reply_type_nomsg (peer, entry, post, reply_info); if (ret < 0) { - gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING, + gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0, + RDMA_MSG_SEND_REPLY_FAILED, "failed to send reply to peer (%s) as " "RDMA_NOMSG", peer->trans->peerinfo.identifier); } @@ -2661,7 +2718,8 @@ __gf_rdma_ioq_churn_reply (gf_rdma_peer_t *peer, gf_rdma_ioq_t *entry, ret = __gf_rdma_send_reply_type_msg (peer, entry, post, reply_info); if (ret < 0) { - gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING, + gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0, + RDMA_MSG_SEND_REPLY_FAILED, "failed to send reply with write chunks " "to peer (%s)", peer->trans->peerinfo.identifier); @@ -2669,7 +2727,8 @@ __gf_rdma_ioq_churn_reply (gf_rdma_peer_t *peer, gf_rdma_ioq_t *entry, break; default: - gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING, + gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0, + RDMA_MSG_INVALID_CHUNK_TYPE, "invalid chunktype (%d) specified for sending reply " " (peer:%s)", type, peer->trans->peerinfo.identifier); break; @@ -2715,7 +2774,8 @@ __gf_rdma_ioq_churn_entry (gf_rdma_peer_t *peer, gf_rdma_ioq_t *entry) if (entry->is_request) { ret = __gf_rdma_ioq_churn_request (peer, entry, post); if (ret < 0) { - gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING, + gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0, + RDMA_MSG_PROC_IOQ_ENTRY_FAILED, "failed to process request ioq entry " "to peer(%s)", peer->trans->peerinfo.identifier); @@ -2723,7 +2783,8 @@ __gf_rdma_ioq_churn_entry (gf_rdma_peer_t *peer, gf_rdma_ioq_t *entry) } else { ret = __gf_rdma_ioq_churn_reply (peer, entry, post); if (ret < 0) { - gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING, + gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0, + RDMA_MSG_PROC_IOQ_ENTRY_FAILED, "failed to process reply ioq entry " "to peer (%s)", peer->trans->peerinfo.identifier); @@ -2748,8 +2809,7 @@ __gf_rdma_ioq_churn (gf_rdma_peer_t *peer) gf_rdma_ioq_t *entry = NULL; int32_t ret = 0; - while (!list_empty (&peer->ioq)) - { + while (!list_empty (&peer->ioq)) { /* pick next entry */ entry = peer->ioq_next; @@ -2783,7 +2843,8 @@ gf_rdma_writev (rpc_transport_t *this, gf_rdma_ioq_t *entry) pthread_mutex_lock (&priv->write_mutex); { if (!priv->connected) { - gf_log (this->name, GF_LOG_WARNING, + gf_msg (this->name, GF_LOG_WARNING, 0, + RDMA_MSG_PEER_DISCONNECTED, "rdma is not connected to peer (%s)", this->peerinfo.identifier); ret = -1; @@ -2797,9 +2858,10 @@ gf_rdma_writev (rpc_transport_t *this, gf_rdma_ioq_t *entry) need_append = 0; if (ret < 0) { - gf_log (this->name, GF_LOG_WARNING, - "processing ioq entry destined " - "to (%s) failed", + gf_msg (this->name, GF_LOG_WARNING, 0, + RDMA_MSG_PROC_IOQ_ENTRY_FAILED, + "processing ioq entry destined" + " to (%s) failed", this->peerinfo.identifier); } } @@ -2942,7 +3004,8 @@ gf_rdma_submit_request (rpc_transport_t *this, rpc_transport_req_t *req) entry = gf_rdma_ioq_new (this, &data); if (entry == NULL) { - gf_log (this->name, GF_LOG_WARNING, + gf_msg (this->name, GF_LOG_WARNING, 0, + RDMA_MSG_NEW_IOQ_ENTRY_FAILED, "getting a new ioq entry failed (peer:%s)", this->peerinfo.identifier); goto out; @@ -2953,7 +3016,8 @@ gf_rdma_submit_request (rpc_transport_t *this, rpc_transport_req_t *req) if (ret > 0) { ret = 0; } else if (ret < 0) { - gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING, + gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0, + RDMA_MSG_WRITE_PEER_FAILED, "sending request to peer (%s) failed", this->peerinfo.identifier); rpc_transport_disconnect (this); @@ -2978,7 +3042,8 @@ gf_rdma_submit_reply (rpc_transport_t *this, rpc_transport_reply_t *reply) entry = gf_rdma_ioq_new (this, &data); if (entry == NULL) { - gf_log (this->name, GF_LOG_WARNING, + gf_msg (this->name, GF_LOG_WARNING, 0, + RDMA_MSG_NEW_IOQ_ENTRY_FAILED, "getting a new ioq entry failed (peer:%s)", this->peerinfo.identifier); goto out; @@ -2988,7 +3053,8 @@ gf_rdma_submit_reply (rpc_transport_t *this, rpc_transport_reply_t *reply) if (ret > 0) { ret = 0; } else if (ret < 0) { - gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING, + gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0, + RDMA_MSG_WRITE_PEER_FAILED, "sending request to peer (%s) failed", this->peerinfo.identifier); rpc_transport_disconnect (this); @@ -3129,7 +3195,9 @@ gf_rdma_create_qp (rpc_transport_t *this) device_name = (char *)ibv_get_device_name (peer->cm_id->verbs->device); if (device_name == NULL) { ret = -1; - gf_log (this->name, GF_LOG_WARNING, "cannot get device_name"); + gf_msg (this->name, GF_LOG_WARNING, 0, + RDMA_MSG_GET_DEVICE_NAME_FAILED, "cannot get " + "device_name"); goto out; } @@ -3137,7 +3205,8 @@ gf_rdma_create_qp (rpc_transport_t *this) device_name); if (device == NULL) { ret = -1; - gf_log (this->name, GF_LOG_WARNING, "cannot get device for " + gf_msg (this->name, GF_LOG_WARNING, 0, + RDMA_MSG_GET_DEVICE_FAILED, "cannot get device for " "device %s", device_name); goto out; } @@ -3161,9 +3230,9 @@ gf_rdma_create_qp (rpc_transport_t *this) ret = rdma_create_qp(peer->cm_id, device->pd, &init_attr); if (ret != 0) { - gf_log (peer->trans->name, GF_LOG_CRITICAL, - "%s: could not create QP (%s)", this->name, - strerror (errno)); + gf_msg (peer->trans->name, GF_LOG_CRITICAL, errno, + RDMA_MSG_CREAT_QP_FAILED, "%s: could not create QP " + "(%s)", this->name, strerror (errno)); ret = -1; goto out; } @@ -3358,8 +3427,9 @@ gf_rdma_decode_error_msg (gf_rdma_peer_t *peer, gf_rdma_post_t *post, ret = rpc_reply_to_xdr (&rpc_msg, iobuf_ptr (iobuf), iobuf_pagesize (iobuf), &post->ctx.vector[0]); if (ret == -1) { - gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING, - "Failed to create RPC reply"); + gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0, + RDMA_MSG_RPC_REPLY_CREATE_FAILED, "Failed to create " + "RPC reply"); goto out; } @@ -3400,8 +3470,9 @@ gf_rdma_decode_msg (gf_rdma_peer_t *peer, gf_rdma_post_t *post, ret = gf_rdma_get_read_chunklist (&ptr, readch); if (ret == -1) { - gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING, - "cannot get read chunklist from msg"); + gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0, + RDMA_MSG_GET_READ_CHUNK_FAILED, "cannot get read " + "chunklist from msg"); goto out; } @@ -3410,8 +3481,9 @@ gf_rdma_decode_msg (gf_rdma_peer_t *peer, gf_rdma_post_t *post, ret = gf_rdma_get_write_chunklist (&ptr, &write_ary); if (ret == -1) { - gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING, - "cannot get write chunklist from msg"); + gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0, + RDMA_MSG_GET_WRITE_CHUNK_FAILED, "cannot get write " + "chunklist from msg"); goto out; } @@ -3421,7 +3493,8 @@ gf_rdma_decode_msg (gf_rdma_peer_t *peer, gf_rdma_post_t *post, if (write_ary != NULL) { reply_info = gf_rdma_reply_info_alloc (peer); if (reply_info == NULL) { - gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING, + gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0, + RDMA_MSG_REPLY_INFO_ALLOC_FAILED, "reply_info_alloc failed"); ret = -1; goto out; @@ -3433,15 +3506,17 @@ gf_rdma_decode_msg (gf_rdma_peer_t *peer, gf_rdma_post_t *post, } else { ret = gf_rdma_get_write_chunklist (&ptr, &write_ary); if (ret == -1) { - gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING, - "cannot get reply chunklist from msg"); + gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0, + RDMA_MSG_CHUNKLIST_ERROR, "cannot get reply " + "chunklist from msg"); goto out; } if (write_ary != NULL) { reply_info = gf_rdma_reply_info_alloc (peer); if (reply_info == NULL) { - gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING, + gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0, + RDMA_MSG_REPLY_INFO_ALLOC_FAILED, "reply_info_alloc_failed"); ret = -1; goto out; @@ -3507,36 +3582,38 @@ gf_rdma_decode_header (gf_rdma_peer_t *peer, gf_rdma_post_t *post, case GF_RDMA_NOMSG: ret = gf_rdma_decode_msg (peer, post, readch, bytes_in_post); if (ret < 0) { - gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING, - "cannot decode msg of type (%d)", - header->rm_type); + gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0, + RDMA_MSG_ENCODE_ERROR, "cannot decode msg of " + "type (%d)", header->rm_type); } break; case GF_RDMA_MSGP: - gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING, - "rdma msg of msg-type GF_RDMA_MSGP should not have " - "been received"); + gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0, + RDMA_MSG_INVALID_ENTRY, "rdma msg of msg-type " + "GF_RDMA_MSGP should not have been received"); ret = -1; break; case GF_RDMA_DONE: - gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING, - "rdma msg of msg-type GF_RDMA_DONE should not have " - "been received"); + gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0, + RDMA_MSG_INVALID_ENTRY, "rdma msg of msg-type " + "GF_RDMA_DONE should not have been received"); ret = -1; break; case GF_RDMA_ERROR: - gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING, - "received a msg of type RDMA_ERROR"); + gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0, + RDMA_MSG_RDMA_ERROR_RECEIVED, "received a msg of type" + " RDMA_ERROR"); ret = gf_rdma_decode_error_msg (peer, post, bytes_in_post); break; default: - gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING, - "unknown rdma msg-type (%d)", header->rm_type); + gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0, + RDMA_MSG_INVALID_ENTRY, "unknown rdma msg-type (%d)", + header->rm_type); } return ret; @@ -3562,9 +3639,10 @@ gf_rdma_do_reads (gf_rdma_peer_t *peer, gf_rdma_post_t *post, } if (i == 0) { - gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING, - "message type specified as rdma-read but there are no " - "rdma read-chunks present"); + gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0, + RDMA_MSG_INVALID_CHUNK_TYPE, "message type specified " + "as rdma-read but there are no rdma read-chunks " + "present"); goto out; } @@ -3592,9 +3670,9 @@ gf_rdma_do_reads (gf_rdma_peer_t *peer, gf_rdma_post_t *post, pthread_mutex_lock (&priv->write_mutex); { if (!priv->connected) { - gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING, - "transport not connected to peer (%s), " - "not doing rdma reads", + gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0, + RDMA_MSG_PEER_DISCONNECTED, "transport not " + "connected to peer (%s), not doing rdma reads", peer->trans->peerinfo.identifier); goto unlock; } @@ -3623,7 +3701,8 @@ gf_rdma_do_reads (gf_rdma_peer_t *peer, gf_rdma_post_t *post, ret = __gf_rdma_register_local_mr_for_rdma (peer, &post->ctx.vector[count], 1, &post->ctx); if (ret == -1) { - gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING, + gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0, + RDMA_MSG_MR_ALOC_FAILED, "registering local memory" " for rdma read failed"); goto unlock; @@ -3652,9 +3731,9 @@ gf_rdma_do_reads (gf_rdma_peer_t *peer, gf_rdma_post_t *post, wr[i-1].next = NULL; ret = ibv_post_send (peer->qp, wr, &bad_wr); if (ret) { - gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING, - "rdma read from client " - "(%s) failed with ret = %d (%s)", + gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0, + RDMA_MSG_READ_CLIENT_ERROR, "rdma read from " + "client (%s) failed with ret = %d (%s)", peer->trans->peerinfo.identifier, ret, (ret > 0) ? strerror (ret) : ""); @@ -3746,16 +3825,15 @@ gf_rdma_pollin_notify (gf_rdma_peer_t *peer, gf_rdma_post_t *post) RPC_TRANSPORT_MAP_XID_REQUEST, &request_info); if (ret == -1) { - gf_log (GF_RDMA_LOG_NAME, GF_LOG_DEBUG, - "cannot get request information from rpc " - "layer"); + gf_msg_debug (GF_RDMA_LOG_NAME, 0, "cannot get request" + "information from rpc layer"); goto out; } rpc_req = request_info.rpc_req; if (rpc_req == NULL) { - gf_log (GF_RDMA_LOG_NAME, GF_LOG_DEBUG, - "rpc request structure not found"); + gf_msg_debug (GF_RDMA_LOG_NAME, 0, "rpc request " + "structure not found"); ret = -1; goto out; } @@ -3780,8 +3858,8 @@ gf_rdma_pollin_notify (gf_rdma_peer_t *peer, gf_rdma_post_t *post) ret = rpc_transport_notify (peer->trans, RPC_TRANSPORT_MSG_RECEIVED, pollin); if (ret < 0) { - gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING, - "transport_notify failed"); + gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0, + TRANS_MSG_TRANSPORT_ERROR, "transport_notify failed"); } out: @@ -3843,16 +3921,18 @@ gf_rdma_recv_reply (gf_rdma_peer_t *peer, gf_rdma_post_t *post) RPC_TRANSPORT_MAP_XID_REQUEST, &request_info); if (ret == -1) { - gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING, - "cannot get request information (peer:%s) from rpc " - "layer", peer->trans->peerinfo.identifier); + gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0, + TRANS_MSG_TRANSPORT_ERROR, "cannot get request " + "information (peer:%s) from rpc layer", + peer->trans->peerinfo.identifier); goto out; } rpc_req = request_info.rpc_req; if (rpc_req == NULL) { - gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING, - "rpc request structure not found"); + gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0, + RDMA_MSG_RPC_ST_ERROR, "rpc request structure not " + "found"); ret = -1; goto out; } @@ -3870,7 +3950,8 @@ out: if (ret == 0) { ret = gf_rdma_pollin_notify (peer, post); if (ret < 0) { - gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING, + gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0, + RDMA_MSG_POLL_IN_NOTIFY_FAILED, "pollin notify failed"); } } @@ -3888,14 +3969,16 @@ gf_rdma_recv_request (gf_rdma_peer_t *peer, gf_rdma_post_t *post, if (readch != NULL) { ret = gf_rdma_do_reads (peer, post, readch); if (ret < 0) { - gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING, + gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0, + RDMA_MSG_PEER_READ_FAILED, "rdma read from peer (%s) failed", peer->trans->peerinfo.identifier); } } else { ret = gf_rdma_pollin_notify (peer, post); if (ret == -1) { - gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING, + gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0, + RDMA_MSG_POLL_IN_NOTIFY_FAILED, "pollin notification failed"); } } @@ -3916,15 +3999,17 @@ gf_rdma_process_recv (gf_rdma_peer_t *peer, struct ibv_wc *wc) post = (gf_rdma_post_t *) (long) wc->wr_id; if (post == NULL) { - gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING, - "no post found in successful work completion element"); + gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0, + RDMA_MSG_POST_MISSING, "no post found in successful " + "work completion element"); goto out; } ret = gf_rdma_decode_header (peer, post, &readch, wc->byte_len); if (ret == -1) { - gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING, - "decoding of header failed"); + gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0, + RDMA_MSG_HEADER_DECODE_FAILED, "decoding of header " + "failed"); goto out; } @@ -3945,7 +4030,7 @@ gf_rdma_process_recv (gf_rdma_peer_t *peer, struct ibv_wc *wc) * instead of directly setting it to header->rm_credit. */ priv->peer.quota = header->rm_credit - - ( 1 - priv->peer.quota); + - (1 - priv->peer.quota); } } pthread_mutex_unlock (&priv->write_mutex); @@ -3966,7 +4051,8 @@ gf_rdma_process_recv (gf_rdma_peer_t *peer, struct ibv_wc *wc) case GF_RDMA_ERROR: if (header->rm_body.rm_error.rm_type == ERR_CHUNK) { - gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING, + gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0, + RDMA_MSG_RDMA_ERROR_RECEIVED, "peer (%s), couldn't encode or decode the msg " "properly or write chunks were not provided " "for replies that were bigger than " @@ -3975,36 +4061,40 @@ gf_rdma_process_recv (gf_rdma_peer_t *peer, struct ibv_wc *wc) GLUSTERFS_RDMA_INLINE_THRESHOLD); ret = gf_rdma_pollin_notify (peer, post); if (ret == -1) { - gf_log (GF_RDMA_LOG_NAME, GF_LOG_DEBUG, - "pollin notification failed"); + gf_msg_debug (GF_RDMA_LOG_NAME, 0, "pollin " + "notification failed"); } goto out; } else { - gf_log (GF_RDMA_LOG_NAME, GF_LOG_ERROR, - "an error has happened while transmission of " - "msg, disconnecting the transport"); + gf_msg (GF_RDMA_LOG_NAME, GF_LOG_ERROR, 0, + TRANS_MSG_TRANSPORT_ERROR, "an error has " + "happened while transmission of msg, " + "disconnecting the transport"); ret = -1; goto out; } default: - gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING, - "invalid rdma msg-type (%d)", header->rm_type); + gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0, + RDMA_MSG_INVALID_ENTRY, "invalid rdma msg-type (%d)", + header->rm_type); goto out; } if (msg_type == CALL) { ret = gf_rdma_recv_request (peer, post, readch); if (ret < 0) { - gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING, - "receiving a request from peer (%s) failed", + gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0, + RDMA_MSG_PEER_REQ_FAILED, "receiving a request" + " from peer (%s) failed", peer->trans->peerinfo.identifier); } } else { ret = gf_rdma_recv_reply (peer, post); if (ret < 0) { - gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING, - "receiving a reply from peer (%s) failed", + gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0, + RDMA_MSG_PEER_REP_FAILED, "receiving a reply " + "from peer (%s) failed", peer->trans->peerinfo.identifier); } } @@ -4029,21 +4119,22 @@ gf_rdma_async_event_thread (void *context) &event); if (ret && errno != EINTR) { - gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING, - "Error getting event (%s)", - strerror (errno)); + gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, errno, + RDMA_MSG_EVENT_ERROR, "Error getting " + "event (%s)", strerror (errno)); } - } while(ret && errno == EINTR); + } while (ret && errno == EINTR); switch (event.event_type) { case IBV_EVENT_SRQ_LIMIT_REACHED: - gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING, - "received srq_limit reached"); + gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0, + RDMA_MSG_EVENT_SRQ_LIMIT_REACHED, "received " + "srq_limit reached"); break; default: - gf_log (GF_RDMA_LOG_NAME, GF_LOG_DEBUG, - "event (%d) received", event.event_type); + gf_msg_debug (GF_RDMA_LOG_NAME, 0, "event (%d) " + "received", event.event_type); break; } @@ -4074,7 +4165,8 @@ gf_rdma_recv_completion_proc (void *data) failed = 0; ret = ibv_get_cq_event (chan, &event_cq, &event_ctx); if (ret) { - gf_log (GF_RDMA_LOG_NAME, GF_LOG_ERROR, + gf_msg (GF_RDMA_LOG_NAME, GF_LOG_ERROR, errno, + RDMA_MSG_IBV_GET_CQ_FAILED, "ibv_get_cq_event failed, terminating recv " "thread %d (%d)", ret, errno); continue; @@ -4084,7 +4176,8 @@ gf_rdma_recv_completion_proc (void *data) ret = ibv_req_notify_cq (event_cq, 0); if (ret) { - gf_log (GF_RDMA_LOG_NAME, GF_LOG_ERROR, + gf_msg (GF_RDMA_LOG_NAME, GF_LOG_ERROR, errno, + RDMA_MSG_IBV_REQ_NOTIFY_CQ_FAILED, "ibv_req_notify_cq on %s failed, terminating " "recv thread: %d (%d)", device->device_name, ret, errno); @@ -4119,9 +4212,10 @@ gf_rdma_recv_completion_proc (void *data) pthread_mutex_unlock (&device->qpreg.lock); if (wc[index].status != IBV_WC_SUCCESS) { - gf_log (GF_RDMA_LOG_NAME, - GF_LOG_ERROR, "recv work request " - "on `%s' returned error (%d)", + gf_msg (GF_RDMA_LOG_NAME, + GF_LOG_ERROR, 0, + RDMA_MSG_RECV_ERROR, "recv work " + "request on `%s' returned error (%d)", device->device_name, wc[index].status); failed = 1; @@ -4142,10 +4236,10 @@ gf_rdma_recv_completion_proc (void *data) &wc[index]); rpc_transport_unref (peer->trans); } else { - gf_log (GF_RDMA_LOG_NAME, - GF_LOG_DEBUG, - "could not lookup peer for qp_num: %d", - wc[index].qp_num); + gf_msg_debug (GF_RDMA_LOG_NAME, 0, + "could not lookup peer " + "for qp_num: %d", + wc[index].qp_num); } gf_rdma_post_unref (post); @@ -4153,8 +4247,8 @@ gf_rdma_recv_completion_proc (void *data) } if (ret < 0) { - gf_log (GF_RDMA_LOG_NAME, - GF_LOG_ERROR, + gf_msg (GF_RDMA_LOG_NAME, GF_LOG_ERROR, errno, + RDMA_MSG_IBV_POLL_CQ_ERROR, "ibv_poll_cq on `%s' returned error " "(ret = %d, errno = %d)", device->device_name, ret, errno); @@ -4185,7 +4279,8 @@ gf_rdma_handle_failed_send_completion (gf_rdma_peer_t *peer, struct ibv_wc *wc) post = (gf_rdma_post_t *) (long) wc->wr_id; - gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING, + gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0, + RDMA_MSG_RDMA_HANDLE_FAILED, "send work request on `%s' returned error " "wc.status = %d, wc.vendor_err = %d, post->buf = %p, " "wc.byte_len = %d, post->reused = %d", @@ -4193,12 +4288,13 @@ gf_rdma_handle_failed_send_completion (gf_rdma_peer_t *peer, struct ibv_wc *wc) wc->vendor_err, post->buf, wc->byte_len, post->reused); if (wc->status == IBV_WC_RETRY_EXC_ERR) { - gf_log ("rdma", GF_LOG_ERROR, "connection between client and" - " server not working. check by running " - "'ibv_srq_pingpong'. also make sure subnet manager" - " is running (eg: 'opensm'), or check if rdma port is " - "valid (or active) by running 'ibv_devinfo'. contact " - "Gluster Support Team if the problem persists."); + gf_msg ("rdma", GF_LOG_ERROR, 0, TRANS_MSG_TIMEOUT_EXCEEDED, + "connection between client and server not working. " + "check by running 'ibv_srq_pingpong'. also make sure " + "subnet manager is running (eg: 'opensm'), or check " + "if rdma port is valid (or active) by running " + "'ibv_devinfo'. contact Gluster Support Team if the " + "problem persists."); } if (peer) { @@ -4279,7 +4375,8 @@ gf_rdma_send_completion_proc (void *data) failed = 0; ret = ibv_get_cq_event (chan, &event_cq, &event_ctx); if (ret) { - gf_log (GF_RDMA_LOG_NAME, GF_LOG_ERROR, + gf_msg (GF_RDMA_LOG_NAME, GF_LOG_ERROR, errno, + RDMA_MSG_IBV_GET_CQ_FAILED, "ibv_get_cq_event on failed, terminating " "send thread: %d (%d)", ret, errno); continue; @@ -4289,7 +4386,8 @@ gf_rdma_send_completion_proc (void *data) ret = ibv_req_notify_cq (event_cq, 0); if (ret) { - gf_log (GF_RDMA_LOG_NAME, GF_LOG_ERROR, + gf_msg (GF_RDMA_LOG_NAME, GF_LOG_ERROR, errno, + RDMA_MSG_IBV_REQ_NOTIFY_CQ_FAILED, "ibv_req_notify_cq on %s failed, terminating " "send thread: %d (%d)", device->device_name, ret, errno); @@ -4351,10 +4449,9 @@ gf_rdma_send_completion_proc (void *data) quota_ret = gf_rdma_quota_put (peer); if (quota_ret < 0) { - gf_log ("rdma", - GF_LOG_DEBUG, - "failed to send " - "message"); + gf_msg_debug ("rdma", + 0, "failed to send " + "message"); } } } @@ -4362,7 +4459,7 @@ gf_rdma_send_completion_proc (void *data) if (peer) { rpc_transport_unref (peer->trans); } else { - gf_log (GF_RDMA_LOG_NAME, GF_LOG_DEBUG, + gf_msg_debug (GF_RDMA_LOG_NAME, 0, "could not lookup peer for qp_num: %d", wc[index].qp_num); @@ -4371,7 +4468,8 @@ gf_rdma_send_completion_proc (void *data) } if (ret < 0) { - gf_log (GF_RDMA_LOG_NAME, GF_LOG_ERROR, + gf_msg (GF_RDMA_LOG_NAME, GF_LOG_ERROR, errno, + RDMA_MSG_IBV_POLL_CQ_ERROR, "ibv_poll_cq on `%s' returned error (ret = %d," " errno = %d)", device->device_name, ret, errno); @@ -4442,26 +4540,30 @@ gf_rdma_options_init (rpc_transport_t *this) if (temp) mtu = data_to_int32 (temp); switch (mtu) { + case 256: options->mtu = IBV_MTU_256; break; + case 512: options->mtu = IBV_MTU_512; break; + case 1024: options->mtu = IBV_MTU_1024; break; + case 2048: options->mtu = IBV_MTU_2048; break; + case 4096: options->mtu = IBV_MTU_4096; break; default: if (temp) - gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING, - "%s: unrecognized MTU value '%s', defaulting " - "to '2048'", this->name, - data_to_str (temp)); + gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, 0, + RDMA_MSG_UNRECG_MTU_VALUE, "%s: unrecognized " + "MTU value '%s', defaulting to '2048'", + this->name, data_to_str (temp)); else - gf_log (GF_RDMA_LOG_NAME, GF_LOG_TRACE, - "%s: defaulting MTU to '2048'", - this->name); + gf_msg_trace (GF_RDMA_LOG_NAME, 0, "%s: defaulting " + "MTU to '2048'", this->name); options->mtu = IBV_MTU_2048; break; } @@ -4488,9 +4590,9 @@ __gf_rdma_ctx_create (void) pthread_mutex_init (&rdma_ctx->lock, NULL); rdma_ctx->rdma_cm_event_channel = rdma_create_event_channel (); if (rdma_ctx->rdma_cm_event_channel == NULL) { - gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING, - "rdma_cm event channel creation failed (%s)", - strerror (errno)); + gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, errno, + RDMA_MSG_CM_EVENT_FAILED, "rdma_cm event channel " + "creation failed (%s)", strerror (errno)); goto out; } @@ -4498,9 +4600,9 @@ __gf_rdma_ctx_create (void) gf_rdma_cm_event_handler, rdma_ctx->rdma_cm_event_channel); if (ret != 0) { - gf_log (GF_RDMA_LOG_NAME, GF_LOG_WARNING, - "creation of thread to handle rdma-cm events " - "failed (%s)", strerror (ret)); + gf_msg (GF_RDMA_LOG_NAME, GF_LOG_WARNING, ret, + RDMA_MSG_CM_EVENT_FAILED, "creation of thread to " + "handle rdma-cm events failed (%s)", strerror (ret)); goto out; } @@ -4525,7 +4627,7 @@ gf_rdma_init (rpc_transport_t *this) glusterfs_ctx_t *ctx = NULL; gf_rdma_options_t *options = NULL; - ctx= this->ctx; + ctx = this->ctx; priv = this->private; @@ -4602,8 +4704,8 @@ gf_rdma_connect (struct rpc_transport *this, int port) &sock_union.sa, &sockaddr_len, port); if (ret != 0) { - gf_log (this->name, GF_LOG_DEBUG, - "cannot get remote address to connect"); + gf_msg_debug (this->name, 0, "cannot get remote address to " + "connect"); goto out; } @@ -4623,9 +4725,9 @@ gf_rdma_connect (struct rpc_transport *this, int port) ret = rdma_create_id (rdma_ctx->rdma_cm_event_channel, &peer->cm_id, this, RDMA_PS_TCP); if (ret != 0) { - gf_log (this->name, GF_LOG_ERROR, - "creation of rdma_cm_id failed (%s)", - strerror (errno)); + gf_msg (this->name, GF_LOG_ERROR, errno, + RDMA_MSG_CM_EVENT_FAILED, "creation of " + "rdma_cm_id failed (%s)", strerror (errno)); ret = -errno; goto unlock; } @@ -4645,7 +4747,8 @@ gf_rdma_connect (struct rpc_transport *this, int port) &this->myinfo.sockaddr_len, peer->cm_id); if (ret != 0) { - gf_log (this->name, GF_LOG_WARNING, + gf_msg (this->name, GF_LOG_WARNING, errno, + RDMA_MSG_CLIENT_BIND_FAILED, "client bind failed: %s", strerror (errno)); goto unlock; } @@ -4653,7 +4756,8 @@ gf_rdma_connect (struct rpc_transport *this, int port) ret = rdma_resolve_addr (peer->cm_id, NULL, &sock_union.sa, 2000); if (ret != 0) { - gf_log (this->name, GF_LOG_WARNING, + gf_msg (this->name, GF_LOG_WARNING, errno, + RDMA_MSG_RDMA_RESOLVE_ADDR_FAILED, "rdma_resolve_addr failed (%s)", strerror (errno)); goto unlock; @@ -4699,7 +4803,8 @@ gf_rdma_listen (rpc_transport_t *this) ret = gf_rdma_server_get_local_sockaddr (this, &sock_union.sa, &sockaddr_len); if (ret != 0) { - gf_log (this->name, GF_LOG_WARNING, + gf_msg (this->name, GF_LOG_WARNING, 0, + RDMA_MSG_NW_ADDR_UNKNOWN, "cannot find network address of server to bind to"); goto err; } @@ -4707,9 +4812,9 @@ gf_rdma_listen (rpc_transport_t *this) ret = rdma_create_id (rdma_ctx->rdma_cm_event_channel, &peer->cm_id, this, RDMA_PS_TCP); if (ret != 0) { - gf_log (this->name, GF_LOG_WARNING, - "creation of rdma_cm_id failed (%s)", - strerror (errno)); + gf_msg (this->name, GF_LOG_WARNING, errno, + RDMA_MSG_CM_EVENT_FAILED, "creation of rdma_cm_id " + "failed (%s)", strerror (errno)); goto err; } @@ -4722,7 +4827,8 @@ gf_rdma_listen (rpc_transport_t *this) service, sizeof (service), NI_NUMERICHOST); if (ret != 0) { - gf_log (this->name, GF_LOG_ERROR, + gf_msg (this->name, GF_LOG_ERROR, ret, + TRANS_MSG_GET_NAME_INFO_FAILED, "getnameinfo failed (%s)", gai_strerror (ret)); goto err; } @@ -4733,21 +4839,24 @@ gf_rdma_listen (rpc_transport_t *this) RDMA_OPTION_ID_REUSEADDR, (void *)&optval, sizeof(optval)); if (ret != 0) { - gf_log (this->name, GF_LOG_WARNING, - "rdma option set failed (%s)", strerror (errno)); + gf_msg (this->name, GF_LOG_WARNING, errno, + RDMA_MSG_OPTION_SET_FAILED, "rdma option set failed " + "(%s)", strerror (errno)); goto err; } ret = rdma_bind_addr (peer->cm_id, &sock_union.sa); if (ret != 0) { - gf_log (this->name, GF_LOG_WARNING, + gf_msg (this->name, GF_LOG_WARNING, errno, + RDMA_MSG_RDMA_BIND_ADDR_FAILED, "rdma_bind_addr failed (%s)", strerror (errno)); goto err; } ret = rdma_listen (peer->cm_id, 10); if (ret != 0) { - gf_log (this->name, GF_LOG_WARNING, + gf_msg (this->name, GF_LOG_WARNING, errno, + RDMA_MSG_LISTEN_FAILED, "rdma_listen failed (%s)", strerror (errno)); goto err; } @@ -4789,7 +4898,8 @@ init (rpc_transport_t *this) this->private = priv; if (gf_rdma_init (this)) { - gf_log (this->name, GF_LOG_WARNING, + gf_msg (this->name, GF_LOG_WARNING, 0, + RDMA_MSG_INIT_IB_DEVICE_FAILED, "Failed to initialize IB Device"); this->private = NULL; GF_FREE (priv); @@ -4830,8 +4940,8 @@ fini (struct rpc_transport *this) pthread_mutex_destroy (&priv->recv_mutex); pthread_mutex_destroy (&priv->write_mutex); - gf_log (this->name, GF_LOG_TRACE, - "called fini on transport: %p", this); + gf_msg_trace (this->name, 0, + "called fini on transport: %p", this); GF_FREE (priv); } diff --git a/rpc/rpc-transport/rdma/src/rpc-trans-rdma-messages.h b/rpc/rpc-transport/rdma/src/rpc-trans-rdma-messages.h new file mode 100644 index 00000000000..0957de25bb1 --- /dev/null +++ b/rpc/rpc-transport/rdma/src/rpc-trans-rdma-messages.h @@ -0,0 +1,217 @@ +/* + Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com> + This file is part of GlusterFS. + + This file is licensed to you under your choice of the GNU Lesser + General Public License, version 3 or any later version (LGPLv3 or + later), or the GNU General Public License, version 2 (GPLv2), in all + cases as published by the Free Software Foundation. +*/ + +#ifndef _RPC_TRANS_RDMA_MESSAGES_H_ +#define _RPC_TRANS_RDMA_MESSAGES_H_ + +#ifndef _CONFIG_H +#define _CONFIG_H +#include "config.h" +#endif + +#include "glfs-message-id.h" + +/* NOTE: Rules for message additions + * 1) Each instance of a message is _better_ left with a unique message ID, even + * if the message format is the same. Reasoning is that, if the message + * format needs to change in one instance, the other instances are not + * impacted or the new change does not change the ID of the instance being + * modified. + * 2) Addition of a message, + * - Should increment the GLFS_NUM_MESSAGES_RDMA + * - Append to the list of messages defined, towards the end + * - Retain macro naming as glfs_msg_X (for redability across developers) + * NOTE: Rules for message format modifications + * 3) Check acorss the code if the message ID macro in question is reused + * anywhere. If reused then then the modifications should ensure correctness + * everywhere, or needs a new message ID as (1) above was not adhered to. If + * not used anywhere, proceed with the required modification. + * NOTE: Rules for message deletion + * 4) Check (3) and if used anywhere else, then cannot be deleted. If not used + * anywhere, then can be deleted, but will leave a hole by design, as + * addition rules specify modification to the end of the list and not filling + * holes. + */ + +#define GLFS_RPC_TRANS_RDMA_BASE GLFS_MSGID_COMP_RPC_TRANS_RDMA +#define GLFS_NUM_MESSAGES_RDMA 77 +#define GLFS_RPC_TRANS_RDMA_MSGID_END (GLFS_RPC_TRANS_RDMA_BASE + GLFS_NUM_MESSAGES_RDMA + 1) + +/* Messages with message IDs */ + +#define glfs_msg_start_rdma GLFS_RPC_TRANS_RDMA_BASE, "Invalid: Start of messages" + +/* This slot/segment is allocated for RDMA message IDs + * The allocation starts from BASE. + +*/ + +#define RDMA_MSG_PORT_BIND_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 1) + +#define RDMA_MSG_POST_CREATION_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 2) + +#define RDMA_MSG_DEREGISTER_ARENA_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 3) + +#define RDMA_MSG_MR_ALOC_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 4) + +#define RDMA_MSG_PREREG_BUFFER_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 5) + +#define RDMA_MSG_CQ_CREATION_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 6) + +#define RDMA_MSG_REQ_NOTIFY_CQ_REVQ_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 7) + +#define RDMA_MSG_QUERY_DEVICE_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 8) + +#define RDMA_MSG_REQ_NOTIFY_CQ_SENDQ_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 9) + +#define RDMA_MSG_SEND_COMP_CHAN_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 10) + +#define RDMA_MSG_RECV_COMP_CHAN_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 11) + +#define RDMA_MSG_ALOC_PROT_DOM_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 12) + +#define RDMA_MSG_CRE_SRQ_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 13) + +#define RDMA_MSG_ALOC_POST_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 14) + +#define RDMA_MSG_SEND_COMP_THREAD_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 15) + +#define RDMA_MSG_RECV_COMP_THREAD_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 16) + +#define RDMA_MSG_ASYNC_EVENT_THEAD_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 17) + +#define RDMA_MSG_GET_DEVICE_NAME_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 18) + +#define RDMA_MSG_GET_IB_DEVICE_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 19) + +#define RDMA_MSG_CREAT_INC_TRANS_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 20) + +#define RDMA_MSG_CREAT_QP_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 21) + +#define RDMA_MSG_ACCEPT_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 22) + +#define RDMA_MSG_CONNECT_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 23) + +#define RDMA_MSG_ROUTE_RESOLVE_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 24) + +#define RDMA_MSG_GET_DEVICE_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 25) + +#define RDMA_MSG_PEER_DISCONNECTED (GLFS_RPC_TRANS_RDMA_BASE + 26) + +#define RDMA_MSG_ENCODE_ERROR (GLFS_RPC_TRANS_RDMA_BASE + 27) + +#define RDMA_MSG_POST_SEND_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 28) + +#define RDMA_MSG_READ_CHUNK_VECTOR_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 29) + +#define RDMA_MSG_WRITE_CHUNK_VECTOR_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 30) + +#define RDMA_MSG_WRITE_REPLY_CHUNCK_CONFLICT (GLFS_RPC_TRANS_RDMA_BASE + 31) + +#define RDMA_MSG_CHUNK_COUNT_GREAT_MAX_SEGMENTS (GLFS_RPC_TRANS_RDMA_BASE + 32) + +#define RDMA_MSG_CREATE_READ_CHUNK_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 33) + +#define RDMA_MSG_CREATE_WRITE_REPLAY_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 34) + +#define RDMA_MSG_SEND_SIZE_GREAT_INLINE_THRESHOLD (GLFS_RPC_TRANS_RDMA_BASE + 35) + +#define RDMA_MSG_REG_ACCESS_LOCAL_WRITE_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 36) + +#define RDMA_MSG_WRITE_PEER_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 37) + +#define RDMA_MSG_SEND_REPLY_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 38) + +#define RDMA_MSG_INVALID_CHUNK_TYPE (GLFS_RPC_TRANS_RDMA_BASE + 39) + +#define RDMA_MSG_PROC_IOQ_ENTRY_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 40) + +#define RDMA_MSG_NEW_IOQ_ENTRY_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 41) + +#define RDMA_MSG_RPC_REPLY_CREATE_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 42) + +#define RDMA_MSG_GET_READ_CHUNK_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 43) + +#define RDMA_MSG_GET_WRITE_CHUNK_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 44) + +#define RDMA_MSG_REPLY_INFO_ALLOC_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 45) + +#define RDMA_MSG_RDMA_ERROR_RECEIVED (GLFS_RPC_TRANS_RDMA_BASE + 46) + +#define RDMA_MSG_GET_REQ_INFO_RPC_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 47) + +#define RDMA_MSG_POLL_IN_NOTIFY_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 48) + +#define RDMA_MSG_HEADER_DECODE_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 49) + +#define RDMA_MSG_EVENT_SRQ_LIMIT_REACHED (GLFS_RPC_TRANS_RDMA_BASE + 50) + +#define RDMA_MSG_UNRECG_MQ_VALUE (GLFS_RPC_TRANS_RDMA_BASE + 51) + +#define RDMA_MSG_BUFFER_ERROR (GLFS_RPC_TRANS_RDMA_BASE + 52) + +#define RDMA_MSG_OPTION_SET_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 53) + +#define RDMA_MSG_LISTEN_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 54) + +#define RDMA_MSG_INIT_IB_DEVICE_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 55) + +#define RDMA_MSG_WRITE_CLIENT_ERROR (GLFS_RPC_TRANS_RDMA_BASE + 56) + +#define RDMA_MSG_CHUNKLIST_ERROR (GLFS_RPC_TRANS_RDMA_BASE + 57) + +#define RDMA_MSG_INVALID_ENTRY (GLFS_RPC_TRANS_RDMA_BASE + 58) + +#define RDMA_MSG_READ_CLIENT_ERROR (GLFS_RPC_TRANS_RDMA_BASE + 59) + +#define RDMA_MSG_RPC_ST_ERROR (GLFS_RPC_TRANS_RDMA_BASE + 60) + +#define RDMA_MSG_PEER_READ_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 61) + +#define RDMA_MSG_POST_MISSING (GLFS_RPC_TRANS_RDMA_BASE + 62) + +#define RDMA_MSG_PEER_REQ_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 63) + +#define RDMA_MSG_PEER_REP_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 64) + +#define RDMA_MSG_EVENT_ERROR (GLFS_RPC_TRANS_RDMA_BASE + 65) + +#define RDMA_MSG_IBV_GET_CQ_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 66) + +#define RDMA_MSG_IBV_REQ_NOTIFY_CQ_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 67) + +#define RDMA_MSG_RECV_ERROR (GLFS_RPC_TRANS_RDMA_BASE + 68) + +#define RDMA_MSG_IBV_POLL_CQ_ERROR (GLFS_RPC_TRANS_RDMA_BASE + 69) + +#define RDMA_MSG_RDMA_HANDLE_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 70) + +#define RDMA_MSG_CM_EVENT_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 71) + +#define RDMA_MSG_CLIENT_BIND_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 72) + +#define RDMA_MSG_RDMA_RESOLVE_ADDR_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 73) + +#define RDMA_MSG_NW_ADDR_UNKNOWN (GLFS_RPC_TRANS_RDMA_BASE + 74) + +#define RDMA_MSG_RDMA_BIND_ADDR_FAILED (GLFS_RPC_TRANS_RDMA_BASE + 75) + +#define RDMA_MSG_SEND_CLIENT_ERROR (GLFS_RPC_TRANS_RDMA_BASE + 76) + +#define RDMA_MSG_UNRECG_MTU_VALUE (GLFS_RPC_TRANS_RDMA_BASE + 77) + + + +/*------------*/ + +#define glfs_msg_end_rdma GLFS_RPC_TRANS_RDMA_MSGID_END, "Invalid: End of messages" + +#endif /* !_RPC_TRANS_RDMA_MESSAGES_H_ */ + |