summaryrefslogtreecommitdiffstats
path: root/rpc/rpc-transport/rdma
diff options
context:
space:
mode:
authorRaghavendra G <raghavendra@gluster.com>2010-11-04 02:33:19 +0000
committerAnand V. Avati <avati@dev.gluster.com>2010-11-07 20:15:08 -0800
commitd19a72024c8cd5e40dc48df27c21fda701e76c54 (patch)
treeb6e8d167865295a98fe4bdba5aeb52964486983a /rpc/rpc-transport/rdma
parent56450b818756a762c925c76f8c6218ea1c5f7efe (diff)
rpc-transport/rdma: Fix 2KB as threshold size for msgs that can be transferred inline.
- Any fop/mop that can result a reply whose size is greater than this threshold allocates and submits a buffer along with the request to receive reply. Signed-off-by: Raghavendra G <raghavendra@gluster.com> Signed-off-by: Anand V. Avati <avati@dev.gluster.com> BUG: 513 (Introduce 0 copy rdma) URL: http://bugs.gluster.com/cgi-bin/bugzilla3/show_bug.cgi?id=513
Diffstat (limited to 'rpc/rpc-transport/rdma')
-rw-r--r--rpc/rpc-transport/rdma/src/rdma.c20
-rw-r--r--rpc/rpc-transport/rdma/src/rdma.h16
2 files changed, 23 insertions, 13 deletions
diff --git a/rpc/rpc-transport/rdma/src/rdma.c b/rpc/rpc-transport/rdma/src/rdma.c
index 4301f4119a0..b2258fbb99d 100644
--- a/rpc/rpc-transport/rdma/src/rdma.c
+++ b/rpc/rpc-transport/rdma/src/rdma.c
@@ -957,7 +957,7 @@ __rdma_ioq_churn_request (rdma_peer_t *peer, rdma_ioq_t *entry,
send_size = iov_length (entry->rpchdr, entry->rpchdr_count)
+ iov_length (entry->proghdr, entry->proghdr_count)
- + RDMA_MAX_HEADER_SIZE;
+ + GLUSTERFS_RDMA_MAX_HEADER_SIZE;
if (entry->prog_payload_count != 0) {
prog_payload_length
@@ -965,9 +965,10 @@ __rdma_ioq_churn_request (rdma_peer_t *peer, rdma_ioq_t *entry,
entry->prog_payload_count);
}
- if (send_size > RDMA_INLINE_THRESHOLD) {
+ if (send_size > GLUSTERFS_RDMA_INLINE_THRESHOLD) {
rtype = rdma_areadch;
- } else if ((send_size + prog_payload_length) < RDMA_INLINE_THRESHOLD) {
+ } else if ((send_size + prog_payload_length)
+ < GLUSTERFS_RDMA_INLINE_THRESHOLD) {
rtype = rdma_noch;
} else if (entry->prog_payload_count != 0) {
rtype = rdma_readch;
@@ -1151,7 +1152,7 @@ __rdma_send_reply_inline (rdma_peer_t *peer, rdma_ioq_t *entry,
* reply
*/
- if (send_size > RDMA_INLINE_THRESHOLD) {
+ if (send_size > GLUSTERFS_RDMA_INLINE_THRESHOLD) {
ret = __rdma_send_error (peer, entry, post, reply_info,
ERR_CHUNK);
goto out;
@@ -1492,14 +1493,15 @@ __rdma_send_reply_type_msg (rdma_peer_t *peer, rdma_ioq_t *entry,
send_size = iov_length (entry->rpchdr, entry->rpchdr_count)
+ iov_length (entry->proghdr, entry->proghdr_count)
- + RDMA_MAX_HEADER_SIZE;
+ + GLUSTERFS_RDMA_MAX_HEADER_SIZE;
- if (send_size > RDMA_INLINE_THRESHOLD) {
+ if (send_size > GLUSTERFS_RDMA_INLINE_THRESHOLD) {
gf_log (RDMA_LOG_NAME, GF_LOG_DEBUG,
"client has provided only write chunks, but the "
"combined size of rpc and program header (%d) is "
"exceeding the size of msg that can be sent using "
- "RDMA send (%d)", send_size, RDMA_INLINE_THRESHOLD);
+ "RDMA send (%d)", send_size,
+ GLUSTERFS_RDMA_INLINE_THRESHOLD);
ret = __rdma_send_error (peer, entry, post, reply_info,
ERR_CHUNK);
@@ -3550,8 +3552,8 @@ rdma_options_init (rpc_transport_t *this)
/* TODO: validate arguments from options below */
- options->send_size = this->ctx->page_size * 4; /* 512 KB */
- options->recv_size = this->ctx->page_size * 4; /* 512 KB */
+ options->send_size = GLUSTERFS_RDMA_INLINE_THRESHOLD;/*this->ctx->page_size * 4; 512 KB*/
+ options->recv_size = GLUSTERFS_RDMA_INLINE_THRESHOLD;/*this->ctx->page_size * 4; 512 KB*/
options->send_count = 32;
options->recv_count = 32;
diff --git a/rpc/rpc-transport/rdma/src/rdma.h b/rpc/rpc-transport/rdma/src/rdma.h
index 4bb5a3759e1..e24ce76ccf1 100644
--- a/rpc/rpc-transport/rdma/src/rdma.h
+++ b/rpc/rpc-transport/rdma/src/rdma.h
@@ -41,11 +41,12 @@
/* FIXME: give appropriate values to these macros */
#define GF_DEFAULT_RDMA_LISTEN_PORT (GF_DEFAULT_BASE_PORT + 1)
+
+/* If you are changing RDMA_MAX_SEGMENTS, please make sure to update
+ * GLUSTERFS_RDMA_MAX_HEADER_SIZE defined in glusterfs.h .
+ */
#define RDMA_MAX_SEGMENTS 8
-#define RDMA_MAX_HEADER_SIZE (sizeof (rdma_header_t) \
- + RDMA_MAX_SEGMENTS \
- * sizeof (rdma_read_chunk_t))
-#define RDMA_INLINE_THRESHOLD (1024 * 128)
+
#define RDMA_VERSION 1
#define RDMA_POOL_SIZE 512
@@ -76,6 +77,9 @@ typedef enum rdma_chunktype {
rdma_replych /* entire reply through rdma write */
}rdma_chunktype_t;
+/* If you are modifying __rdma_header, please make sure to change
+ * GLUSTERFS_RDMA_MAX_HEADER_SIZE defined in glusterfs.h to reflect your changes
+ */
struct __rdma_header {
uint32_t rm_xid; /* Mirrors the RPC header xid */
uint32_t rm_vers; /* Version of this protocol */
@@ -102,6 +106,10 @@ struct __rdma_header {
} __attribute__((packed));
typedef struct __rdma_header rdma_header_t;
+/* If you are modifying __rdma_segment or __rdma_read_chunk, please make sure
+ * to change GLUSTERFS_RDMA_MAX_HEADER_SIZE defined in glusterfs.h to reflect
+ * your changes.
+ */
struct __rdma_segment {
uint32_t rs_handle; /* Registered memory handle */
uint32_t rs_length; /* Length of the chunk in bytes */