summaryrefslogtreecommitdiffstats
path: root/libglusterfs/src/iobuf.c
diff options
context:
space:
mode:
authorMohammed Rafi KC <rkavunga@redhat.com>2015-02-17 20:17:58 +0530
committerRaghavendra Bhat <raghavendra@redhat.com>2015-03-27 04:34:08 -0700
commit7febb66a26f01c94f8e76bb90cf4edd7c6cc1421 (patch)
treefc455167a14965ac54490af2acbc08ad480a5d6c /libglusterfs/src/iobuf.c
parentd21990e093d99d8adbacae1ba2c56ff7606e2c37 (diff)
rdma: pre-register iobuf_pool with rdma devices.
Back port pf : http://review.gluster.org/9506 registring buffers with rdma device is a time consuming operation. So performing registration in code path will decrease the performance. Using a pre registered memory will give a bettor performance, ie, register iobuf_pool during rdma initialization. For dynamically created arena, we can register with all the device. Change-Id: Ic79183e2efd014c43faf5911fdb6d5cfbcee64ca BUG: 1202212 Signed-off-by: Mohammed Rafi KC <rkavunga@redhat.com> Reviewed-on: http://review.gluster.org/9506 Tested-by: Gluster Build System <jenkins@build.gluster.com> Reviewed-by: Raghavendra G <rgowdapp@redhat.com> Tested-by: Raghavendra G <rgowdapp@redhat.com> Reviewed-on: http://review.gluster.org/9889 Reviewed-by: Raghavendra Bhat <raghavendra@redhat.com>
Diffstat (limited to 'libglusterfs/src/iobuf.c')
-rw-r--r--libglusterfs/src/iobuf.c38
1 files changed, 32 insertions, 6 deletions
diff --git a/libglusterfs/src/iobuf.c b/libglusterfs/src/iobuf.c
index 82ffe2dd8fd..f8f1860889b 100644
--- a/libglusterfs/src/iobuf.c
+++ b/libglusterfs/src/iobuf.c
@@ -50,6 +50,7 @@ gf_iobuf_get_arena_index (size_t page_size)
return i;
}
+
size_t
gf_iobuf_get_pagesize (size_t page_size)
{
@@ -138,10 +139,15 @@ out:
void
-__iobuf_arena_destroy (struct iobuf_arena *iobuf_arena)
+__iobuf_arena_destroy (struct iobuf_pool *iobuf_pool,
+ struct iobuf_arena *iobuf_arena)
{
GF_VALIDATE_OR_GOTO ("iobuf", iobuf_arena, out);
+ if (iobuf_pool->rdma_deregistration)
+ iobuf_pool->rdma_deregistration (iobuf_pool->mr_list,
+ iobuf_arena);
+
__iobuf_arena_destroy_iobufs (iobuf_arena);
if (iobuf_arena->mem_base
@@ -169,6 +175,7 @@ __iobuf_arena_alloc (struct iobuf_pool *iobuf_pool, size_t page_size,
goto err;
INIT_LIST_HEAD (&iobuf_arena->list);
+ INIT_LIST_HEAD (&iobuf_arena->all_list);
INIT_LIST_HEAD (&iobuf_arena->active.list);
INIT_LIST_HEAD (&iobuf_arena->passive.list);
iobuf_arena->iobuf_pool = iobuf_pool;
@@ -188,6 +195,13 @@ __iobuf_arena_alloc (struct iobuf_pool *iobuf_pool, size_t page_size,
goto err;
}
+ if (iobuf_pool->rdma_registration) {
+ iobuf_pool->rdma_registration (iobuf_pool->device,
+ iobuf_arena);
+ }
+
+ list_add_tail (&iobuf_arena->all_list, &iobuf_pool->all_arenas);
+
__iobuf_arena_init_iobufs (iobuf_arena);
if (!iobuf_arena->iobufs) {
gf_log (THIS->name, GF_LOG_ERROR, "init failed");
@@ -199,7 +213,7 @@ __iobuf_arena_alloc (struct iobuf_pool *iobuf_pool, size_t page_size,
return iobuf_arena;
err:
- __iobuf_arena_destroy (iobuf_arena);
+ __iobuf_arena_destroy (iobuf_pool, iobuf_arena);
out:
return NULL;
@@ -258,8 +272,8 @@ __iobuf_pool_add_arena (struct iobuf_pool *iobuf_pool, size_t page_size,
gf_log (THIS->name, GF_LOG_WARNING, "arena not found");
return NULL;
}
+ list_add (&iobuf_arena->list, &iobuf_pool->arenas[index]);
- list_add_tail (&iobuf_arena->list, &iobuf_pool->arenas[index]);
return iobuf_arena;
}
@@ -299,7 +313,8 @@ iobuf_pool_destroy (struct iobuf_pool *iobuf_pool)
&iobuf_pool->arenas[i], list) {
list_del_init (&iobuf_arena->list);
iobuf_pool->arena_cnt--;
- __iobuf_arena_destroy (iobuf_arena);
+
+ __iobuf_arena_destroy (iobuf_pool, iobuf_arena);
}
}
@@ -347,7 +362,7 @@ iobuf_pool_new (void)
gf_common_mt_iobuf_pool);
if (!iobuf_pool)
goto out;
-
+ INIT_LIST_HEAD (&iobuf_pool->all_arenas);
pthread_mutex_init (&iobuf_pool->mutex, NULL);
for (i = 0; i <= IOBUF_ARENA_MAX_INDEX; i++) {
INIT_LIST_HEAD (&iobuf_pool->arenas[i]);
@@ -357,6 +372,16 @@ iobuf_pool_new (void)
iobuf_pool->default_page_size = 128 * GF_UNIT_KB;
+ iobuf_pool->rdma_registration = NULL;
+ iobuf_pool->rdma_deregistration = NULL;
+
+ for (i = 0; i < GF_RDMA_DEVICE_COUNT; i++) {
+
+ iobuf_pool->device[i] = NULL;
+ iobuf_pool->mr_list[i] = NULL;
+
+ }
+
arena_size = 0;
for (i = 0; i < IOBUF_ARENA_MAX_INDEX; i++) {
page_size = gf_iobuf_init_config[i].pagesize;
@@ -393,9 +418,10 @@ __iobuf_arena_prune (struct iobuf_pool *iobuf_pool,
/* All cases matched, destroy */
list_del_init (&iobuf_arena->list);
+ list_del_init (&iobuf_arena->all_list);
iobuf_pool->arena_cnt--;
- __iobuf_arena_destroy (iobuf_arena);
+ __iobuf_arena_destroy (iobuf_pool, iobuf_arena);
out:
return;