summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorRaghavendra G <raghavendra@gluster.com>2010-09-24 07:51:53 +0000
committerVijay Bellur <vijay@dev.gluster.com>2010-09-24 07:10:27 -0700
commit6e952607f17dd5ea6e3748f2bbc26234f4b1e553 (patch)
treeb31bbfc39505c238058be5df1cb7baad7b826493
parentf2c5894f254d9b3d17b058adce7c7b4152da2f5e (diff)
rpc/rpc-transport: fix frame-loss during rdma-reads.v3.1.0qa31
- total no of rdma reads in progress was tracked using refcount of post (refcount = total rdma reads + 1). However doing rdma_post_unref _after_ rdma_do_reads, can result in race condition between execution of rdma_post_unref and procedure handling rdma read completion. This makes it impossible to find out whether the current rdma read was the last rdma-read being done in procedure handling rdma read-completion. The fix to this should either do rdma_post_unref before posting an rdma read or use another variable to track the number of rdma-reads. This patch implements second method. Signed-off-by: Raghavendra G <raghavendra@gluster.com> Signed-off-by: Vijay Bellur <vijay@dev.gluster.com> BUG: 1700 (write fops are bailing out on a distribute setup) URL: http://bugs.gluster.com/cgi-bin/bugzilla3/show_bug.cgi?id=1700
-rw-r--r--rpc/rpc-transport/rdma/src/rdma.c20
-rw-r--r--rpc/rpc-transport/rdma/src/rdma.h1
2 files changed, 18 insertions, 3 deletions
diff --git a/rpc/rpc-transport/rdma/src/rdma.c b/rpc/rpc-transport/rdma/src/rdma.c
index 7790971f867..27a64d86ead 100644
--- a/rpc/rpc-transport/rdma/src/rdma.c
+++ b/rpc/rpc-transport/rdma/src/rdma.c
@@ -2921,6 +2921,15 @@ rdma_do_reads (rdma_peer_t *peer, rdma_post_t *post, rdma_read_chunk_t *readch)
size += readch[i].rc_target.rs_length;
}
+ if (i == 0) {
+ gf_log (RDMA_LOG_NAME, GF_LOG_DEBUG,
+ "message type specified as rdma-read but there are no "
+ "rdma read-chunks present");
+ goto out;
+ }
+
+ post->ctx.rdma_reads = i;
+
if (size > peer->trans->ctx->page_size) {
gf_log (RDMA_LOG_NAME, GF_LOG_ERROR,
"total size of rdma-read (%lu) is greater than "
@@ -3388,7 +3397,7 @@ void
rdma_handle_successful_send_completion (rdma_peer_t *peer, struct ibv_wc *wc)
{
rdma_post_t *post = NULL;
- int refcount = 0, ret = 0;
+ int reads = 0, ret = 0;
if (wc->opcode != IBV_WC_RDMA_READ) {
goto out;
@@ -3396,8 +3405,13 @@ rdma_handle_successful_send_completion (rdma_peer_t *peer, struct ibv_wc *wc)
post = (rdma_post_t *)(long) wc->wr_id;
- refcount = rdma_post_get_refcount (post);
- if (refcount != 1) {
+ pthread_mutex_lock (&post->lock);
+ {
+ reads = --post->ctx.rdma_reads;
+ }
+ pthread_mutex_unlock (&post->lock);
+
+ if (reads != 0) {
/* if it is not the last rdma read, we've got nothing to do */
goto out;
}
diff --git a/rpc/rpc-transport/rdma/src/rdma.h b/rpc/rpc-transport/rdma/src/rdma.h
index 51242d6b4ad..04386a98faf 100644
--- a/rpc/rpc-transport/rdma/src/rdma.h
+++ b/rpc/rpc-transport/rdma/src/rdma.h
@@ -257,6 +257,7 @@ struct __rdma_post_context {
int count;
struct iobref *iobref;
char is_request;
+ int rdma_reads;
rdma_reply_info_t *reply_info;
};
typedef struct __rdma_post_context rdma_post_context_t;