changes to rpc

- use mem-pool for requests and saved_frames.
  - preserve the rpc_req structure till rpc invokes program's reply.
    This will enable us to store transport specific data that has to
    last till reply has come (eg., memory regions of chunk lists in
    case of rdma).
  - change signature of rpc_clnt_submit to accept rsphdr_vector and
    rsppayload_vector. The buffers pointed by these vectors will be
    from iobufs and these iobufs are added to an iobref which should
    also be passed as an arguement to rpc_clnt_submit.

Signed-off-by: Raghavendra G <raghavendra@gluster.com>
Signed-off-by: Anand V. Avati <avati@dev.gluster.com>

BUG: 875 (Implement a new protocol to provide proper backward/forward compatibility)
URL: http://bugs.gluster.com/cgi-bin/bugzilla3/show_bug.cgi?id=875

5 files changed, 16 insertions, 23 deletions

diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c
index 1d4a8d2..338ebc2 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.c
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.c
@@ -310,7 +310,7 @@ glusterd_submit_request (glusterd_peerinfo_t *peerinfo, void *req,
         /* Send the msg */
         ret = rpc_clnt_submit (peerinfo->rpc, prog, procnum, cbkfn,
                                &iov, count,
-                               NULL, 0, iobref, frame);
+                               NULL, 0, iobref, frame, NULL, 0, NULL, 0, NULL);
 
         if (ret == 0) {
                 pthread_mutex_lock (&peerinfo->rpc->conn.lock);
diff --git a/xlators/protocol/client/src/client.c b/xlators/protocol/client/src/client.c
index 383f52b..c8bbd46 100644
--- a/xlators/protocol/client/src/client.c
+++ b/xlators/protocol/client/src/client.c
@@ -96,7 +96,7 @@ client_submit_request (xlator_t *this, void *req, call_frame_t *frame,
         }
         /* Send the msg */
         ret = rpc_clnt_submit (conf->rpc, prog, procnum, cbk, &iov, count, NULL, 0,
-                               iobref, frame);
+                               iobref, frame, NULL, 0, NULL, 0, NULL);
 
         if (ret == 0) {
                 pthread_mutex_lock (&conf->rpc->conn.lock);
diff --git a/xlators/protocol/client/src/client3_1-fops.c b/xlators/protocol/client/src/client3_1-fops.c
index 136f84b..52d5c09 100644
--- a/xlators/protocol/client/src/client3_1-fops.c
+++ b/xlators/protocol/client/src/client3_1-fops.c
@@ -80,7 +80,8 @@ client_submit_vec_request (xlator_t  *this, void *req, call_frame_t  *frame,
         }
         /* Send the msg */
         ret = rpc_clnt_submit (conf->rpc, prog, procnum, cbk, &iov, count,
-                               payload, payloadcnt, iobref, frame);
+                               payload, payloadcnt, iobref, frame, NULL, 0,
+                               NULL, 0, NULL);
 
         if (ret == 0) {
                 pthread_mutex_lock (&conf->rpc->conn.lock);
@@ -2001,13 +2002,12 @@ client3_1_readv_cbk (struct rpc_req *req, struct iovec *iov, int count,
         }
 
         if (rsp.op_ret != -1) {
-                iobref = iobref_new ();
+                iobref = req->rsp_iobref;
                 gf_stat_to_iatt (&rsp.stat, &stat);
                 vector.iov_len  = rsp.op_ret;
 
                 if (rsp.op_ret > 0) {
-                        vector.iov_base = req->rsp_procpayload->ptr;
-                        iobref_add (iobref, req->rsp_procpayload);
+                        vector.iov_base = req->rsp[1].iov_base;
                 }
         }
 out:
@@ -2015,10 +2015,6 @@ out:
                              gf_error_to_errno (rsp.op_errno), &vector, 1,
                              &stat, iobref);
 
-        if (iobref) {
-                iobref_unref (iobref);
-        }
-
         return 0;
 }
 
diff --git a/xlators/protocol/server/src/server.h b/xlators/protocol/server/src/server.h
index 81ac7ed..2085387 100644
--- a/xlators/protocol/server/src/server.h
+++ b/xlators/protocol/server/src/server.h
@@ -162,6 +162,8 @@ struct _server_state {
 	fd_t             *fd;
 	int               flags;
         int               wbflags;
+        struct iovec      payload_vector;
+        int               payload_count;
         struct iobuf     *iobuf;
         struct iobref    *iobref;
 
diff --git a/xlators/protocol/server/src/server3_1-fops.c b/xlators/protocol/server/src/server3_1-fops.c
index 18ace87..fb69cf8 100644
--- a/xlators/protocol/server/src/server3_1-fops.c
+++ b/xlators/protocol/server/src/server3_1-fops.c
@@ -2445,10 +2445,8 @@ server_writev_resume (call_frame_t *frame, xlator_t *bound_xl)
         if (state->resolve.op_ret != 0)
                 goto err;
 
-        iov.iov_len  = state->size;
-
-        if (state->iobuf) {
-                iov.iov_base = state->iobuf->ptr;
+        if (state->payload_count) {
+                iov = state->payload_vector;
         }
 
         STACK_WIND (frame, server_writev_cbk,
@@ -2987,10 +2985,10 @@ server_writev (rpcsvc_request_t *req)
 
 
 int
-server_writev_vec (rpcsvc_request_t *req, struct iobuf *iobuf)
+server_writev_vec (rpcsvc_request_t *req, struct iovec *payload,
+                   int payload_count, struct iobref *iobref)
 {
         server_state_t      *state  = NULL;
-        struct iobref       *iobref = NULL;
         call_frame_t        *frame  = NULL;
         gfs3_write_req       args   = {0,};
 
@@ -3022,14 +3020,11 @@ server_writev_vec (rpcsvc_request_t *req, struct iobuf *iobuf)
         state->resolve.fd_no = args.fd;
         state->offset        = args.offset;
 
-        if (iobuf) {
-                iobref = iobref_new ();
-                iobref_add (iobref, iobuf);
-
-                state->iobref = iobref;
-                state->iobuf  = iobuf_ref (iobuf);
-
+        if (payload_count != 0) {
+                state->iobref = iobref_ref (iobref);
                 state->size = req->msg[1].iov_len;
+                state->payload_count = payload_count;
+                state->payload_vector = *payload;
         }
 
         resolve_and_resume (frame, server_writev_resume);