storage/posix: implement batched fsync in a single thread

Because of the extra fsync()s issued by AFR transaction, they
could potentially "clog" all the io-threads denying unrelated
operations from making progress.

This patch assigns a dedicated thread to issues fsyncs, as
an experimental feature to understand performance characteristics
with the approach.

As a basis, incoming individual fsync requests are grouped into
batches, falling in the same @batch-fsync-delay-usec window of
time. These windows can extend in practice, as processing of
the previous batch can take longer than @batch-fsync-delay-usec
while new requests are getting batched.

The feature support three modes (similar to the -S modes of fs_mark)

- syncfs: In this mode one syncfs() is issued per batch, instead
  of N fsync()s (one per file.)

- syncfs-single-fsync: In this mode one syncfs() is issued per
  batch (which, on Linux, guarantees the completion of write-out
  of dirty pages in the filesystem up to that point) and one single
  fsync() to synchronize or flush the controller/drive cache. This
  corresponds to -S 2 of fsmark.

- syncfs-reverse-fsync: In this mode, one syncfs() is issued per
  batch, and all the open files in that batch are fsync()'ed in
  the reverse order of the queue. This corresponds to -S 4 of
  fsmark.

- reverse-fsync: In this mode, no syncfs() is issued and all the
  files in the batch are fsync()'ed in the reverse order. This
  corresponds to -S 3 of fsmark.

Change-Id: Ia1e170a810c780c8d80e02cf910accc4170c4cd4
BUG: 927146
Signed-off-by: Anand Avati <avati@redhat.com>
Reviewed-on: http://review.gluster.org/4746
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Vijay Bellur <vbellur@redhat.com>

1 files changed, 140 insertions, 0 deletions

diff --git a/xlators/storage/posix/src/posix-helpers.c b/xlators/storage/posix/src/posix-helpers.c
index a13e02078..0c0fdbabc 100644
--- a/xlators/storage/posix/src/posix-helpers.c
+++ b/xlators/storage/posix/src/posix-helpers.c
@@ -1177,3 +1177,143 @@ posix_spawn_health_check_thread (xlator_t *xl)
 unlock:
         UNLOCK (&priv->lock);
 }
+
+int
+posix_fsyncer_pick (xlator_t *this, struct list_head *head)
+{
+	struct posix_private *priv = NULL;
+	int count = 0;
+
+	priv = this->private;
+	pthread_mutex_lock (&priv->fsync_mutex);
+	{
+		while (list_empty (&priv->fsyncs))
+			pthread_cond_wait (&priv->fsync_cond,
+					   &priv->fsync_mutex);
+
+		count = priv->fsync_queue_count;
+		priv->fsync_queue_count = 0;
+		list_splice_init (&priv->fsyncs, head);
+	}
+	pthread_mutex_unlock (&priv->fsync_mutex);
+
+	return count;
+}
+
+
+void
+posix_fsyncer_process (xlator_t *this, call_stub_t *stub, gf_boolean_t do_fsync)
+{
+	struct posix_fd *pfd = NULL;
+	int ret = -1;
+	struct posix_private *priv = NULL;
+
+	priv = this->private;
+
+	ret = posix_fd_ctx_get (stub->args.fd, this, &pfd);
+	if (ret < 0) {
+		gf_log (this->name, GF_LOG_ERROR,
+			"could not get fdctx for fd(%s)",
+			uuid_utoa (stub->args.fd->inode->gfid));
+		call_unwind_error (stub, -1, EINVAL);
+		return;
+	}
+
+	if (do_fsync) {
+#ifdef HAVE_FDATASYNC
+		if (stub->args.datasync)
+			ret = fdatasync (pfd->fd);
+		else
+#endif
+			ret = fsync (pfd->fd);
+	} else {
+		ret = 0;
+	}
+
+	if (ret) {
+		gf_log (this->name, GF_LOG_ERROR,
+			"could not fstat fd(%s)",
+			uuid_utoa (stub->args.fd->inode->gfid));
+		call_unwind_error (stub, -1, errno);
+		return;
+	}
+
+	call_unwind_error (stub, 0, 0);
+}
+
+
+static void
+posix_fsyncer_syncfs (xlator_t *this, struct list_head *head)
+{
+	call_stub_t *stub = NULL;
+	struct posix_fd *pfd = NULL;
+	int ret = -1;
+
+	stub = list_entry (head->prev, call_stub_t, list);
+	ret = posix_fd_ctx_get (stub->args.fd, this, &pfd);
+	if (ret)
+		return;
+
+#ifdef GF_LINUX_HOST_OS
+	/* syncfs() is not "declared" in RHEL's glibc even though
+	   the kernel has support.
+	*/
+#include <sys/syscall.h>
+#include <unistd.h>
+	syscall (SYS_syncfs, pfd->fd);
+#else
+	sync();
+#endif
+
+}
+
+
+void *
+posix_fsyncer (void *d)
+{
+	xlator_t *this = d;
+	struct posix_private *priv = NULL;
+	call_stub_t *stub = NULL;
+	call_stub_t *tmp = NULL;
+	struct list_head list;
+	int count = 0;
+	gf_boolean_t do_fsync = _gf_true;
+
+	priv = this->private;
+
+	for (;;) {
+		INIT_LIST_HEAD (&list);
+
+		count = posix_fsyncer_pick (this, &list);
+
+		usleep (priv->batch_fsync_delay_usec);
+
+		gf_log (this->name, GF_LOG_DEBUG,
+			"picked %d fsyncs", count);
+
+		switch (priv->batch_fsync_mode) {
+		case BATCH_NONE:
+		case BATCH_REVERSE_FSYNC:
+			break;
+		case BATCH_SYNCFS:
+		case BATCH_SYNCFS_SINGLE_FSYNC:
+		case BATCH_SYNCFS_REVERSE_FSYNC:
+			posix_fsyncer_syncfs (this, &list);
+			break;
+		}
+
+		if (priv->batch_fsync_mode == BATCH_SYNCFS)
+			do_fsync = _gf_false;
+		else
+			do_fsync = _gf_true;
+
+		list_for_each_entry_safe_reverse (stub, tmp, &list, list) {
+			list_del_init (&stub->list);
+
+			posix_fsyncer_process (this, stub, do_fsync);
+
+			if (priv->batch_fsync_mode == BATCH_SYNCFS_SINGLE_FSYNC)
+				do_fsync = _gf_false;
+		}
+	}
+}