From 6e952607f17dd5ea6e3748f2bbc26234f4b1e553 Mon Sep 17 00:00:00 2001 From: Raghavendra G Date: Fri, 24 Sep 2010 07:51:53 +0000 Subject: rpc/rpc-transport: fix frame-loss during rdma-reads. - total no of rdma reads in progress was tracked using refcount of post (refcount = total rdma reads + 1). However doing rdma_post_unref _after_ rdma_do_reads, can result in race condition between execution of rdma_post_unref and procedure handling rdma read completion. This makes it impossible to find out whether the current rdma read was the last rdma-read being done in procedure handling rdma read-completion. The fix to this should either do rdma_post_unref before posting an rdma read or use another variable to track the number of rdma-reads. This patch implements second method. Signed-off-by: Raghavendra G Signed-off-by: Vijay Bellur BUG: 1700 (write fops are bailing out on a distribute setup) URL: http://bugs.gluster.com/cgi-bin/bugzilla3/show_bug.cgi?id=1700 --- rpc/rpc-transport/rdma/src/rdma.c | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) (limited to 'rpc/rpc-transport/rdma/src/rdma.c') diff --git a/rpc/rpc-transport/rdma/src/rdma.c b/rpc/rpc-transport/rdma/src/rdma.c index 7790971f867..27a64d86ead 100644 --- a/rpc/rpc-transport/rdma/src/rdma.c +++ b/rpc/rpc-transport/rdma/src/rdma.c @@ -2921,6 +2921,15 @@ rdma_do_reads (rdma_peer_t *peer, rdma_post_t *post, rdma_read_chunk_t *readch) size += readch[i].rc_target.rs_length; } + if (i == 0) { + gf_log (RDMA_LOG_NAME, GF_LOG_DEBUG, + "message type specified as rdma-read but there are no " + "rdma read-chunks present"); + goto out; + } + + post->ctx.rdma_reads = i; + if (size > peer->trans->ctx->page_size) { gf_log (RDMA_LOG_NAME, GF_LOG_ERROR, "total size of rdma-read (%lu) is greater than " @@ -3388,7 +3397,7 @@ void rdma_handle_successful_send_completion (rdma_peer_t *peer, struct ibv_wc *wc) { rdma_post_t *post = NULL; - int refcount = 0, ret = 0; + int reads = 0, ret = 0; if (wc->opcode != IBV_WC_RDMA_READ) { goto out; @@ -3396,8 +3405,13 @@ rdma_handle_successful_send_completion (rdma_peer_t *peer, struct ibv_wc *wc) post = (rdma_post_t *)(long) wc->wr_id; - refcount = rdma_post_get_refcount (post); - if (refcount != 1) { + pthread_mutex_lock (&post->lock); + { + reads = --post->ctx.rdma_reads; + } + pthread_mutex_unlock (&post->lock); + + if (reads != 0) { /* if it is not the last rdma read, we've got nothing to do */ goto out; } -- cgit