summaryrefslogtreecommitdiffstats
path: root/libglusterfs
diff options
context:
space:
mode:
authorMohammed Rafi KC <rkavunga@redhat.com>2015-02-17 20:17:58 +0530
committerRaghavendra G <rgowdapp@redhat.com>2015-02-17 20:09:54 -0800
commitabcb2017b07b0706aef0d9eeb729115ca605ad6d (patch)
tree416f3cad19987c108a583af9f9fd581def12052e /libglusterfs
parent7c6da2f7ceea2956197641b6cdb1e2f79cdb063e (diff)
rdma: pre-register iobuf_pool with rdma devices.
registring buffers with rdma device is a time consuming operation. So performing registration in code path will decrease the performance. Using a pre registered memory will give a bettor performance, ie, register iobuf_pool during rdma initialization. For dynamically created arena, we can register with all the device. Change-Id: Ic79183e2efd014c43faf5911fdb6d5cfbcee64ca BUG: 1187456 Signed-off-by: Mohammed Rafi KC <rkavunga@redhat.com> Reviewed-on: http://review.gluster.org/9506 Tested-by: Gluster Build System <jenkins@build.gluster.com> Reviewed-by: Raghavendra G <rgowdapp@redhat.com> Tested-by: Raghavendra G <rgowdapp@redhat.com>
Diffstat (limited to 'libglusterfs')
-rw-r--r--libglusterfs/src/iobuf.c38
-rw-r--r--libglusterfs/src/iobuf.h12
-rw-r--r--libglusterfs/src/mem-types.h1
3 files changed, 44 insertions, 7 deletions
diff --git a/libglusterfs/src/iobuf.c b/libglusterfs/src/iobuf.c
index 82ffe2dd8fd..f8f1860889b 100644
--- a/libglusterfs/src/iobuf.c
+++ b/libglusterfs/src/iobuf.c
@@ -50,6 +50,7 @@ gf_iobuf_get_arena_index (size_t page_size)
return i;
}
+
size_t
gf_iobuf_get_pagesize (size_t page_size)
{
@@ -138,10 +139,15 @@ out:
void
-__iobuf_arena_destroy (struct iobuf_arena *iobuf_arena)
+__iobuf_arena_destroy (struct iobuf_pool *iobuf_pool,
+ struct iobuf_arena *iobuf_arena)
{
GF_VALIDATE_OR_GOTO ("iobuf", iobuf_arena, out);
+ if (iobuf_pool->rdma_deregistration)
+ iobuf_pool->rdma_deregistration (iobuf_pool->mr_list,
+ iobuf_arena);
+
__iobuf_arena_destroy_iobufs (iobuf_arena);
if (iobuf_arena->mem_base
@@ -169,6 +175,7 @@ __iobuf_arena_alloc (struct iobuf_pool *iobuf_pool, size_t page_size,
goto err;
INIT_LIST_HEAD (&iobuf_arena->list);
+ INIT_LIST_HEAD (&iobuf_arena->all_list);
INIT_LIST_HEAD (&iobuf_arena->active.list);
INIT_LIST_HEAD (&iobuf_arena->passive.list);
iobuf_arena->iobuf_pool = iobuf_pool;
@@ -188,6 +195,13 @@ __iobuf_arena_alloc (struct iobuf_pool *iobuf_pool, size_t page_size,
goto err;
}
+ if (iobuf_pool->rdma_registration) {
+ iobuf_pool->rdma_registration (iobuf_pool->device,
+ iobuf_arena);
+ }
+
+ list_add_tail (&iobuf_arena->all_list, &iobuf_pool->all_arenas);
+
__iobuf_arena_init_iobufs (iobuf_arena);
if (!iobuf_arena->iobufs) {
gf_log (THIS->name, GF_LOG_ERROR, "init failed");
@@ -199,7 +213,7 @@ __iobuf_arena_alloc (struct iobuf_pool *iobuf_pool, size_t page_size,
return iobuf_arena;
err:
- __iobuf_arena_destroy (iobuf_arena);
+ __iobuf_arena_destroy (iobuf_pool, iobuf_arena);
out:
return NULL;
@@ -258,8 +272,8 @@ __iobuf_pool_add_arena (struct iobuf_pool *iobuf_pool, size_t page_size,
gf_log (THIS->name, GF_LOG_WARNING, "arena not found");
return NULL;
}
+ list_add (&iobuf_arena->list, &iobuf_pool->arenas[index]);
- list_add_tail (&iobuf_arena->list, &iobuf_pool->arenas[index]);
return iobuf_arena;
}
@@ -299,7 +313,8 @@ iobuf_pool_destroy (struct iobuf_pool *iobuf_pool)
&iobuf_pool->arenas[i], list) {
list_del_init (&iobuf_arena->list);
iobuf_pool->arena_cnt--;
- __iobuf_arena_destroy (iobuf_arena);
+
+ __iobuf_arena_destroy (iobuf_pool, iobuf_arena);
}
}
@@ -347,7 +362,7 @@ iobuf_pool_new (void)
gf_common_mt_iobuf_pool);
if (!iobuf_pool)
goto out;
-
+ INIT_LIST_HEAD (&iobuf_pool->all_arenas);
pthread_mutex_init (&iobuf_pool->mutex, NULL);
for (i = 0; i <= IOBUF_ARENA_MAX_INDEX; i++) {
INIT_LIST_HEAD (&iobuf_pool->arenas[i]);
@@ -357,6 +372,16 @@ iobuf_pool_new (void)
iobuf_pool->default_page_size = 128 * GF_UNIT_KB;
+ iobuf_pool->rdma_registration = NULL;
+ iobuf_pool->rdma_deregistration = NULL;
+
+ for (i = 0; i < GF_RDMA_DEVICE_COUNT; i++) {
+
+ iobuf_pool->device[i] = NULL;
+ iobuf_pool->mr_list[i] = NULL;
+
+ }
+
arena_size = 0;
for (i = 0; i < IOBUF_ARENA_MAX_INDEX; i++) {
page_size = gf_iobuf_init_config[i].pagesize;
@@ -393,9 +418,10 @@ __iobuf_arena_prune (struct iobuf_pool *iobuf_pool,
/* All cases matched, destroy */
list_del_init (&iobuf_arena->list);
+ list_del_init (&iobuf_arena->all_list);
iobuf_pool->arena_cnt--;
- __iobuf_arena_destroy (iobuf_arena);
+ __iobuf_arena_destroy (iobuf_pool, iobuf_arena);
out:
return;
diff --git a/libglusterfs/src/iobuf.h b/libglusterfs/src/iobuf.h
index 4e07910d722..7e5cfe37a28 100644
--- a/libglusterfs/src/iobuf.h
+++ b/libglusterfs/src/iobuf.h
@@ -19,6 +19,8 @@
#define GF_VARIABLE_IOBUF_COUNT 32
+#define GF_RDMA_DEVICE_COUNT 8
+
/* Lets try to define the new anonymous mapping
* flag, in case the system is still using the
* now deprecated MAP_ANON flag.
@@ -81,6 +83,7 @@ struct iobuf_arena {
};
};
+ struct list_head all_list;
size_t page_size; /* size of all iobufs in this arena */
size_t arena_size; /* this is equal to
(iobuf_pool->arena_size / page_size)
@@ -110,6 +113,7 @@ struct iobuf_pool {
size_t default_page_size; /* default size of iobuf */
int arena_cnt;
+ struct list_head all_arenas;
struct list_head arenas[GF_VARIABLE_IOBUF_COUNT];
/* array of arenas. Each element of the array is a list of arenas
holding iobufs of particular page_size */
@@ -121,7 +125,13 @@ struct iobuf_pool {
/* array of of arenas which can be purged */
uint64_t request_misses; /* mostly the requests for higher
- value of iobufs */
+ value of iobufs */
+ int rdma_device_count;
+ struct list_head *mr_list[GF_RDMA_DEVICE_COUNT];
+ void *device[GF_RDMA_DEVICE_COUNT];
+ int (*rdma_registration)(void **, void*);
+ int (*rdma_deregistration)(struct list_head**, struct iobuf_arena *);
+
};
diff --git a/libglusterfs/src/mem-types.h b/libglusterfs/src/mem-types.h
index dd24913278f..0db06f52c98 100644
--- a/libglusterfs/src/mem-types.h
+++ b/libglusterfs/src/mem-types.h
@@ -127,6 +127,7 @@ enum gf_common_mem_types_ {
gf_common_mt_regex_t = 111,
gf_common_mt_ereg = 112,
gf_common_mt_wr = 113,
+ gf_common_mt_rdma_arena_mr = 114,
gf_common_mt_end
};
#endif