syncop: accomodate non-syncenv calls

Use mutex/cond and support syncop_XXXXXX() calls in non-syncenv
environments. syncenv environments continue to use swapcontext
based soft context switches. In non-syncenv environments this
blocks the caller thread on the mutex. The intended use case is
in libgfapi where it is expected to block the caller thread while
performing synchronous calls.

Change-Id: Id6470c99bdc2fe4b7610372139f7fa99b2da400b
BUG: 839950
Signed-off-by: Anand Avati <avati@redhat.com>
Reviewed-on: http://review.gluster.com/3662
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Amar Tumballi <amarts@redhat.com>

2 files changed, 63 insertions, 9 deletions

diff --git a/libglusterfs/src/syncop.c b/libglusterfs/src/syncop.c
index 491f6ae1..7b34c631 100644
--- a/libglusterfs/src/syncop.c
+++ b/libglusterfs/src/syncop.c
@@ -1015,14 +1015,15 @@ syncop_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 }
 
 int
-syncop_writev (xlator_t *subvol, fd_t *fd, struct iovec *vector,
+syncop_writev (xlator_t *subvol, fd_t *fd, const struct iovec *vector,
                int32_t count, off_t offset, struct iobref *iobref,
                uint32_t flags)
 {
         struct syncargs args = {0, };
 
         SYNCOP (subvol, (&args), syncop_writev_cbk, subvol->fops->writev,
-                fd, vector, count, offset, flags, iobref, NULL);
+                fd, (struct iovec *) vector, count, offset, flags, iobref,
+		NULL);
 
         errno = args.op_errno;
         return args.op_ret;
diff --git a/libglusterfs/src/syncop.h b/libglusterfs/src/syncop.h
index 2db2fb7e..9f87673d 100644
--- a/libglusterfs/src/syncop.h
+++ b/libglusterfs/src/syncop.h
@@ -112,22 +112,75 @@ struct syncargs {
 
         /* do not touch */
         struct synctask    *task;
+	pthread_mutex_t     mutex;
+	pthread_cond_t      cond;
+	int                 done;
 };
 
-#define __wake(args) synctask_wake(args->task)
+
+#define __yawn(args) do {					\
+	if (!args->task) {					\
+		pthread_mutex_init (&args->mutex, NULL);	\
+		pthread_cond_init (&args->cond, NULL);		\
+		args->done = 0;					\
+	}							\
+	} while (0)
+
+
+#define __wake(args) do {					\
+	if (args->task) {					\
+		synctask_wake (args->task);			\
+	} else {						\
+		pthread_mutex_lock (&args->mutex);		\
+		{						\
+			args->done = 1;				\
+			pthread_cond_signal (&args->cond);	\
+		}						\
+		pthread_mutex_unlock (&args->mutex);		\
+	}							\
+	} while (0)
+
+
+#define __yield(args) do {						\
+	if (args->task) {						\
+		synctask_yield (args->task);				\
+	} else {							\
+		pthread_mutex_lock (&args->mutex);			\
+		{							\
+			while (!args->done)				\
+				pthread_cond_wait (&args->cond,		\
+						   &args->mutex);	\
+		}							\
+		pthread_mutex_unlock (&args->mutex);			\
+		pthread_mutex_destroy (&args->mutex);			\
+		pthread_cond_destroy (&args->cond);			\
+	}								\
+	} while (0)
 
 
 #define SYNCOP(subvol, stb, cbk, op, params ...) do {                   \
                 struct  synctask        *task = NULL;                   \
+		call_frame_t            *frame = NULL;			\
                                                                         \
                 task = synctask_get ();                                 \
                 stb->task = task;                                       \
+		if (task)						\
+			frame = task->opframe;				\
+		else							\
+			frame = create_frame (THIS, THIS->ctx->pool);	\
+									\
+		__yawn (stb);						\
                                                                         \
-                STACK_WIND_COOKIE (task->opframe, cbk, (void *)stb,     \
-                                   subvol, op, params);                 \
-                task->state = SYNCTASK_SUSPEND;                         \
-                synctask_yield (stb->task);                             \
-                STACK_RESET (task->opframe->root);                      \
+                STACK_WIND_COOKIE (frame, cbk, (void *)stb, subvol,	\
+				   op, params);				\
+		if (task)						\
+			task->state = SYNCTASK_SUSPEND;			\
+									\
+                __yield (stb);						\
+		if (task)						\
+			STACK_RESET (frame->root);			\
+		else							\
+			STACK_DESTROY (frame->root);			\
         } while (0)
 
 
@@ -180,7 +233,7 @@ int syncop_close (fd_t *fd);
 
 int syncop_write (xlator_t *subvol, fd_t *fd, const char *buf, int size,
                   off_t offset, struct iobref *iobref, uint32_t flags);
-int syncop_writev (xlator_t *subvol, fd_t *fd, struct iovec *vector,
+int syncop_writev (xlator_t *subvol, fd_t *fd, const struct iovec *vector,
                    int32_t count, off_t offset, struct iobref *iobref,
                    uint32_t flags);
 int syncop_readv (xlator_t *subvol, fd_t *fd, size_t size, off_t off,