quota/glusterd: enhance quota enable and disable process

Previously quota crawl was done from the single mount point,
this is very slow process if there are huge number of files exists
in the volume

This RFE will now spawn crawl process for each brick in the
volume, and  files are looked in parallel independently for each
brick. This improves the speed of crawling process for
entire files-system

This patch also fixes below problem
* Previously, mountdir was created under '/tmp'.
  If someone tries to cleanup '/tmp'/ directory
  then it is very dangerous that we loose volume data
  So create a mount point under /var/run/gluster/tmp
  instead

* Previously, file-system crawl is performed from all the nodes,
  which is a redundant operation and performance will degrade
  The problem is fixed with this patch

Change-Id: Icabedeb44182139ace9c8106793803122388cab8
BUG: 1290766
Signed-off-by: vmallika <vmallika@redhat.com>
Reviewed-on: http://review.gluster.org/12952
Smoke: Gluster Build System <jenkins@build.gluster.com>
NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>
CentOS-regression: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Manikandan Selvaganesh <mselvaga@redhat.com>
Reviewed-by: Atin Mukherjee <amukherj@redhat.com>

9 files changed, 399 insertions, 43 deletions

diff --git a/libglusterfs/src/logging.h b/libglusterfs/src/logging.h
index 3c7346f8cc7..f3a694ceefd 100644
--- a/libglusterfs/src/logging.h
+++ b/libglusterfs/src/logging.h
@@ -89,8 +89,9 @@ typedef enum {
         /* NOTE: In the future journald, lumberjack, next new thing here */
 } gf_log_logger_t;
 
-#define DEFAULT_LOG_FILE_DIRECTORY            DATADIR "/log/glusterfs"
-#define DEFAULT_LOG_LEVEL                     GF_LOG_INFO
+#define DEFAULT_LOG_FILE_DIRECTORY          DATADIR "/log/glusterfs"
+#define DEFAULT_QUOTA_CRAWL_LOG_DIRECTORY   DATADIR "/log/glusterfs/quota_crawl"
+#define DEFAULT_LOG_LEVEL                   GF_LOG_INFO
 
 typedef struct gf_log_handle_ {
         pthread_mutex_t   logfile_mutex;
diff --git a/tests/bugs/quota/bug-1293601.t b/tests/bugs/quota/bug-1293601.t
index 04c350e688d..52b03bcc059 100644
--- a/tests/bugs/quota/bug-1293601.t
+++ b/tests/bugs/quota/bug-1293601.t
@@ -4,19 +4,22 @@
 . $(dirname $0)/../../volume.rc
 
 cleanup;
-TESTS_EXPECTED_IN_LOOP=1024
 
 TEST glusterd
 
 TEST $CLI volume create $V0 replica 2 $H0:$B0/${V0}{1,2,3,4}
 TEST $CLI volume start $V0
 TEST $CLI volume quota $V0 enable
-TEST $CLI volume quota $V0 limit-usage / 2MB
 
 TEST glusterfs --volfile-server=$H0 --volfile-id=$V0 $M0;
 
-for i in {1..1024}; do
-    TEST_IN_LOOP dd if=/dev/zero of=$M0/f$i bs=1k count=1
+for i in {1..512}; do
+    dd if=/dev/zero of=$M0/f$i bs=1k count=1
+done
+
+mkdir $M0/dir1
+for i in {513..1024}; do
+    dd if=/dev/zero of=$M0/dir1/f$i bs=1k count=1
 done
 
 EXPECT_WITHIN $MARKER_UPDATE_TIMEOUT "1.0MB" quotausage "/"
@@ -24,7 +27,6 @@ EXPECT_WITHIN $MARKER_UPDATE_TIMEOUT "1.0MB" quotausage "/"
 TEST $CLI volume quota $V0 disable
 TEST $CLI volume quota $V0 enable
 
-TEST $CLI volume quota $V0 limit-usage / 2MB
 EXPECT_WITHIN $MARKER_UPDATE_TIMEOUT "1.0MB" quotausage "/"
 
 cleanup;
diff --git a/xlators/mgmt/glusterd/src/glusterd-handshake.c b/xlators/mgmt/glusterd/src/glusterd-handshake.c
index 59e6c19f8df..0ea66a027bf 100644
--- a/xlators/mgmt/glusterd/src/glusterd-handshake.c
+++ b/xlators/mgmt/glusterd/src/glusterd-handshake.c
@@ -156,6 +156,27 @@ out:
         return ret;
 }
 
+int32_t
+glusterd_get_client_per_brick_volfile (glusterd_volinfo_t *volinfo,
+                                       char *filename, char *path, int path_len)
+{
+        char                    workdir[PATH_MAX]      = {0,};
+        glusterd_conf_t        *priv                   = NULL;
+        int32_t                 ret                    = -1;
+
+        GF_VALIDATE_OR_GOTO ("glusterd", THIS, out);
+        priv = THIS->private;
+        GF_VALIDATE_OR_GOTO (THIS->name, priv, out);
+
+        GLUSTERD_GET_VOLUME_DIR (workdir, volinfo, priv);
+
+        snprintf (path, path_len, "%s/%s", workdir, filename);
+
+        ret = 0;
+out:
+        return ret;
+}
+
 static size_t
 build_volfile_path (char *volume_id, char *path,
                     size_t path_len, char *trusted_str)
@@ -264,6 +285,49 @@ build_volfile_path (char *volume_id, char *path,
                 goto out;
         }
 
+        volid_ptr = strstr (volume_id, "client_per_brick/");
+        if (volid_ptr) {
+                volid_ptr = strchr (volid_ptr, '/');
+                if (!volid_ptr) {
+                        ret = -1;
+                        goto out;
+                }
+                volid_ptr++;
+
+                dup_volname = gf_strdup (volid_ptr);
+                if (!dup_volname) {
+                        gf_msg (this->name, GF_LOG_ERROR, ENOMEM,
+                                GD_MSG_NO_MEMORY,
+                                "strdup failed");
+                        ret = -1;
+                        goto out;
+                }
+
+                /* Split the volume name */
+                vol = strtok_r (dup_volname, ".", &save_ptr);
+                if (!vol) {
+                        ret = -1;
+                        goto out;
+                }
+                ret = glusterd_volinfo_find (vol, &volinfo);
+                if (ret == -1) {
+                        gf_msg (this->name, GF_LOG_ERROR, 0,
+                                GD_MSG_VOLINFO_GET_FAIL,
+                                "Couldn't find volinfo");
+                        goto out;
+                }
+                ret = glusterd_get_client_per_brick_volfile (volinfo, volid_ptr,
+                                                             path, path_len);
+                if (ret < 0) {
+                        gf_msg (this->name, GF_LOG_ERROR, 0,
+                                GD_MSG_NO_MEMORY, "failed to get volinfo path");
+                        goto out;
+                }
+
+                ret = sys_access (path, F_OK);
+                goto out;
+        }
+
         if (volume_id[0] == '/') {
                 /* Normal behavior */
                 volid_ptr = volume_id;
diff --git a/xlators/mgmt/glusterd/src/glusterd-quota.c b/xlators/mgmt/glusterd/src/glusterd-quota.c
index f6277a382d8..4390eaab3e6 100644
--- a/xlators/mgmt/glusterd/src/glusterd-quota.c
+++ b/xlators/mgmt/glusterd/src/glusterd-quota.c
@@ -94,12 +94,17 @@ glusterd_is_quota_supported (int32_t type, char **op_errstr)
 
         /* Quota xattr version implemented in 3.7.6
          * quota-version is incremented when quota is enabled
-         * so don't allow enabling quota in heterogeneous
+         * Quota enable and disable performance enhancement has been done
+         * in version 3.7.12.
+         * so don't allow enabling/disabling quota in heterogeneous
          * cluster during upgrade
          */
-        if (conf->op_version < GD_OP_VERSION_3_7_6 &&
-            type == GF_QUOTA_OPTION_TYPE_ENABLE)
-                goto out;
+        if (type == GF_QUOTA_OPTION_TYPE_ENABLE ||
+            type == GF_QUOTA_OPTION_TYPE_ENABLE_OBJECTS ||
+            type == GF_QUOTA_OPTION_TYPE_DISABLE) {
+                if (conf->op_version < GD_OP_VERSION_3_7_12)
+                        goto out;
+        }
 
         supported = _gf_true;
 
@@ -229,37 +234,73 @@ out:
 }
 
 int32_t
-glusterd_quota_initiate_fs_crawl (glusterd_conf_t *priv, char *volname,
-                                  int type)
+_glusterd_quota_initiate_fs_crawl (glusterd_conf_t *priv,
+                                  glusterd_volinfo_t *volinfo,
+                                  glusterd_brickinfo_t *brick, int type,
+                                  char *pid_dir)
 {
         pid_t                      pid;
-        int32_t                    ret               = 0;
-        int                        status            = 0;
-        char                       mountdir[]        = "/tmp/mntXXXXXX";
-        char                       logfile[PATH_MAX] = {0,};
-        runner_t                   runner            = {0};
-        char                       *volfileserver    = NULL;
+        int32_t                    ret                 = -1;
+        int                        status              = 0;
+        char                       mountdir[PATH_MAX]  = {0,};
+        char                       logfile[PATH_MAX]   = {0,};
+        char                       brickpath[PATH_MAX] = {0,};
+        char                       vol_id[PATH_MAX]    = {0,};
+        char                       pidfile[PATH_MAX]   = {0,};
+        runner_t                   runner              = {0};
+        char                      *volfileserver       = NULL;
+        FILE                      *pidfp               = NULL;
+
+        GF_VALIDATE_OR_GOTO ("glusterd", THIS, out);
+
+        GLUSTERD_GET_TMP_PATH (mountdir, "/");
+        ret = sys_mkdir (mountdir, 0777);
+        if (ret && errno != EEXIST) {
+                gf_msg (THIS->name, GF_LOG_WARNING, errno,
+                        GD_MSG_MOUNT_REQ_FAIL, "failed to create temporary "
+                        "directory %s", mountdir);
+                ret = -1;
+                goto out;
+        }
 
+        strcat (mountdir, "mntXXXXXX");
         if (mkdtemp (mountdir) == NULL) {
-                gf_msg_debug ("glusterd", 0,
-                        "failed to create a temporary mount directory");
+                gf_msg (THIS->name, GF_LOG_WARNING, errno,
+                        GD_MSG_MOUNT_REQ_FAIL, "failed to create a temporary "
+                        "mount directory: %s", mountdir);
                 ret = -1;
                 goto out;
         }
+
+        GLUSTERD_REMOVE_SLASH_FROM_PATH (brick->path, brickpath);
         snprintf (logfile, sizeof (logfile),
-                  DEFAULT_LOG_FILE_DIRECTORY"/%s-quota-crawl.log", volname);
+                  DEFAULT_QUOTA_CRAWL_LOG_DIRECTORY"/%s.log",
+                  brickpath);
 
         if (dict_get_str (THIS->options, "transport.socket.bind-address",
                           &volfileserver) != 0)
                 volfileserver = "localhost";
 
+        snprintf (vol_id, sizeof (vol_id), "client_per_brick/%s.%s.%s.%s.vol",
+                  volinfo->volname, "client", brick->hostname, brickpath);
+
         runinit (&runner);
-        runner_add_args (&runner, SBIN_DIR"/glusterfs",
-                         "-s", volfileserver,
-                         "--volfile-id", volname,
-			 "--use-readdirp=no",
-                         "--client-pid", QUOTA_CRAWL_PID,
-                         "-l", logfile, mountdir, NULL);
+
+        if (type == GF_QUOTA_OPTION_TYPE_ENABLE ||
+            type == GF_QUOTA_OPTION_TYPE_ENABLE_OBJECTS)
+                runner_add_args (&runner, SBIN_DIR"/glusterfs",
+                                 "-s", volfileserver,
+                                 "--volfile-id", vol_id,
+                                 "--use-readdirp=yes",
+                                 "--client-pid", QUOTA_CRAWL_PID,
+                                 "-l", logfile, mountdir, NULL);
+        else
+                runner_add_args (&runner, SBIN_DIR"/glusterfs",
+                                 "-s", volfileserver,
+                                 "--volfile-id", vol_id,
+                                 "--use-readdirp=no",
+                                 "--client-pid", QUOTA_CRAWL_PID,
+                                 "-l", logfile, mountdir, NULL);
 
         synclock_unlock (&priv->big_lock);
         ret = runner_run_reuse (&runner);
@@ -272,7 +313,7 @@ glusterd_quota_initiate_fs_crawl (glusterd_conf_t *priv, char *volname,
         runner_end (&runner);
 
         if ((pid = fork ()) < 0) {
-                gf_msg ("glusterd", GF_LOG_WARNING, 0,
+                gf_msg (THIS->name, GF_LOG_WARNING, 0,
                         GD_MSG_FORK_FAIL, "fork from parent failed");
                 ret = -1;
                 goto out;
@@ -286,7 +327,7 @@ glusterd_quota_initiate_fs_crawl (glusterd_conf_t *priv, char *volname,
 
                 ret = chdir (mountdir);
                 if (ret == -1) {
-                        gf_msg ("glusterd", GF_LOG_WARNING, errno,
+                        gf_msg (THIS->name, GF_LOG_WARNING, errno,
                                 GD_MSG_DIR_OP_FAILED, "chdir %s failed",
                                 mountdir);
                         exit (EXIT_FAILURE);
@@ -295,9 +336,7 @@ glusterd_quota_initiate_fs_crawl (glusterd_conf_t *priv, char *volname,
 
                 if (type == GF_QUOTA_OPTION_TYPE_ENABLE ||
                     type == GF_QUOTA_OPTION_TYPE_ENABLE_OBJECTS)
-                        runner_add_args (&runner, "/usr/bin/find", ".",
-                                         "-exec", "/usr/bin/stat",
-                                         "{}", "\\", ";", NULL);
+                        runner_add_args (&runner, "/usr/bin/find", ".", NULL);
 
                 else if (type == GF_QUOTA_OPTION_TYPE_DISABLE) {
 
@@ -321,8 +360,19 @@ glusterd_quota_initiate_fs_crawl (glusterd_conf_t *priv, char *volname,
 
                 }
 
-                if (runner_start (&runner) == -1)
+                if (runner_start (&runner) == -1) {
+                        gf_umount_lazy ("glusterd", mountdir, 1);
                         _exit (EXIT_FAILURE);
+                }
+
+                snprintf (pidfile, sizeof (pidfile), "%s/%s.pid", pid_dir,
+                          brickpath);
+                pidfp = fopen (pidfile, "w");
+                if (pidfp) {
+                        fprintf (pidfp, "%d\n", runner.chpid);
+                        fflush (pidfp);
+                        fclose (pidfp);
+                }
 
 #ifndef GF_LINUX_HOST_OS
                 runner_end (&runner); /* blocks in waitpid */
@@ -338,6 +388,100 @@ out:
         return ret;
 }
 
+void
+glusterd_stop_all_quota_crawl_service (glusterd_conf_t *priv,
+                                       glusterd_volinfo_t *volinfo, int type)
+{
+        char                       pid_dir[PATH_MAX]  = {0, };
+        char                       pidfile[PATH_MAX]  = {0,};
+        struct dirent             *entry              = NULL;
+        DIR                       *dir                = NULL;
+
+        GLUSTERD_GET_QUOTA_CRAWL_PIDDIR (pid_dir, volinfo, type);
+
+        dir = sys_opendir (pid_dir);
+        if (dir == NULL)
+                return;
+
+        GF_FOR_EACH_ENTRY_IN_DIR (entry, dir);
+        while (entry) {
+                snprintf (pidfile, sizeof (pidfile), "%s/%s",
+                          pid_dir, entry->d_name);
+
+                glusterd_service_stop_nolock ("quota_crawl", pidfile, SIGKILL,
+                                              _gf_true);
+                sys_unlink (pidfile);
+
+                GF_FOR_EACH_ENTRY_IN_DIR (entry, dir);
+        }
+        sys_closedir (dir);
+}
+
+int32_t
+glusterd_quota_initiate_fs_crawl (glusterd_conf_t *priv,
+                                  glusterd_volinfo_t *volinfo, int type)
+{
+        int32_t                    ret                = -1;
+        glusterd_brickinfo_t      *brick              = NULL;
+        char                       pid_dir[PATH_MAX]  = {0, };
+
+        GF_VALIDATE_OR_GOTO ("glusterd", THIS, out);
+
+        ret = glusterd_generate_client_per_brick_volfile (volinfo);
+        if (ret) {
+                gf_msg (THIS->name, GF_LOG_ERROR, 0,
+                        GD_MSG_GLUSTERD_OP_FAILED,
+                        "failed to generate client volume file");
+                goto out;
+        }
+
+        ret = mkdir_p (DEFAULT_QUOTA_CRAWL_LOG_DIRECTORY, 0777, _gf_true);
+        if (ret) {
+                gf_msg (THIS->name, GF_LOG_ERROR, errno,
+                        GD_MSG_GLUSTERD_OP_FAILED,
+                        "failed to create dir %s: %s",
+                        DEFAULT_QUOTA_CRAWL_LOG_DIRECTORY, strerror (errno));
+                goto out;
+        }
+
+        GLUSTERD_GET_QUOTA_CRAWL_PIDDIR (pid_dir, volinfo, type);
+        ret = mkdir_p (pid_dir, 0777, _gf_true);
+        if (ret) {
+                gf_msg (THIS->name, GF_LOG_ERROR, errno,
+                        GD_MSG_GLUSTERD_OP_FAILED,
+                        "failed to create dir %s: %s",
+                        pid_dir, strerror (errno));
+                goto out;
+        }
+
+        /* When quota enable is performed, stop alreday running enable crawl
+         * process and start fresh crawl process. let disable process continue
+         * if running to cleanup the older xattrs
+         * When quota disable is performed, stop both enable/disable crawl
+         * process and start fresh crawl process to cleanup the xattrs
+         */
+        glusterd_stop_all_quota_crawl_service (priv, volinfo,
+                                               GF_QUOTA_OPTION_TYPE_ENABLE);
+        if (type == GF_QUOTA_OPTION_TYPE_DISABLE)
+                glusterd_stop_all_quota_crawl_service (priv, volinfo,
+                                               GF_QUOTA_OPTION_TYPE_DISABLE);
+
+        cds_list_for_each_entry (brick, &volinfo->bricks, brick_list) {
+                if (gf_uuid_compare (brick->uuid, MY_UUID))
+                        continue;
+
+                ret = _glusterd_quota_initiate_fs_crawl (priv, volinfo, brick,
+                                                         type, pid_dir);
+
+                if (ret)
+                        goto out;
+        }
+
+        ret = 0;
+out:
+        return ret;
+}
+
 int32_t
 glusterd_quota_get_default_soft_limit (glusterd_volinfo_t *volinfo,
                                        dict_t *rsp_dict)
@@ -1539,7 +1683,7 @@ glusterd_op_quota (dict_t *dict, char **op_errstr, dict_t *rsp_dict)
         }
 
         if (rsp_dict && start_crawl == _gf_true)
-                glusterd_quota_initiate_fs_crawl (priv, volname, type);
+                glusterd_quota_initiate_fs_crawl (priv, volinfo, type);
 
         ret = 0;
 out:
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.c b/xlators/mgmt/glusterd/src/glusterd-utils.c
index 807b4ecf5e6..216089db85a 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.c
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.c
@@ -1574,6 +1574,74 @@ out:
         return ret;
 }
 
+int32_t
+glusterd_service_stop_nolock (const char *service, char *pidfile, int sig,
+                              gf_boolean_t force_kill)
+{
+        int32_t    ret                = -1;
+        pid_t      pid                = -1;
+        xlator_t  *this               = NULL;
+        FILE      *file               = NULL;
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        file = fopen (pidfile, "r+");
+        if (file) {
+                ret = fscanf (file, "%d", &pid);
+                if (ret <= 0) {
+                        gf_msg_debug (this->name, 0,
+                                      "Unable to read pidfile: %s", pidfile);
+                        goto out;
+                }
+        }
+
+        if (kill (pid, 0) < 0) {
+                ret = 0;
+                gf_msg_debug (this->name, 0, "%s process not running: (%d) %s",
+                              service, pid, strerror (errno));
+                goto out;
+        }
+        gf_msg_debug (this->name, 0, "Stopping gluster %s service running with "
+                      "pid: %d", service, pid);
+
+        ret = kill (pid, sig);
+        if (ret) {
+                switch (errno) {
+                case ESRCH:
+                        gf_msg_debug (this->name, 0, "%s is already stopped",
+                                service);
+                        ret = 0;
+                        goto out;
+                default:
+                        gf_msg (this->name, GF_LOG_ERROR, errno,
+                                GD_MSG_SVC_KILL_FAIL, "Unable to kill %s "
+                                "service, reason:%s", service,
+                                strerror (errno));
+                }
+        }
+        if (!force_kill)
+                goto out;
+
+        sleep (1);
+        if (kill(pid, 0) == 0) {
+                ret = kill (pid, SIGKILL);
+                if (ret) {
+                        gf_msg (this->name, GF_LOG_ERROR, errno,
+                                GD_MSG_PID_KILL_FAIL, "Unable to kill pid:%d, "
+                                "reason:%s", pid, strerror(errno));
+                        goto out;
+                }
+        }
+
+        ret = 0;
+
+out:
+        if (file)
+                fclose (file);
+
+        return ret;
+}
 void
 glusterd_set_socket_filepath (char *sock_filepath, char *sockpath, size_t len)
 {
diff --git a/xlators/mgmt/glusterd/src/glusterd-utils.h b/xlators/mgmt/glusterd/src/glusterd-utils.h
index 730bc657199..9f3c4a7098a 100644
--- a/xlators/mgmt/glusterd/src/glusterd-utils.h
+++ b/xlators/mgmt/glusterd/src/glusterd-utils.h
@@ -127,6 +127,10 @@ int32_t
 glusterd_service_stop(const char *service, char *pidfile, int sig,
                       gf_boolean_t force_kill);
 
+int32_t
+glusterd_service_stop_nolock (const char *service, char *pidfile, int sig,
+                              gf_boolean_t force_kill);
+
 int
 glusterd_get_next_available_brickid (glusterd_volinfo_t *volinfo);
 
diff --git a/xlators/mgmt/glusterd/src/glusterd-volgen.c b/xlators/mgmt/glusterd/src/glusterd-volgen.c
index 24e99d2be9f..23890238933 100644
--- a/xlators/mgmt/glusterd/src/glusterd-volgen.c
+++ b/xlators/mgmt/glusterd/src/glusterd-volgen.c
@@ -5081,7 +5081,7 @@ build_nfs_graph (volgen_graph_t *graph, dict_t *mod_dict)
 
 static void
 get_brick_filepath (char *filename, glusterd_volinfo_t *volinfo,
-                    glusterd_brickinfo_t *brickinfo)
+                    glusterd_brickinfo_t *brickinfo, char *prefix)
 {
         char  path[PATH_MAX]   = {0,};
         char  brick[PATH_MAX]  = {0,};
@@ -5092,10 +5092,14 @@ get_brick_filepath (char *filename, glusterd_volinfo_t *volinfo,
         GLUSTERD_REMOVE_SLASH_FROM_PATH (brickinfo->path, brick);
         GLUSTERD_GET_VOLUME_DIR (path, volinfo, priv);
 
-        snprintf (filename, PATH_MAX, "%s/%s.%s.%s.vol",
-                  path, volinfo->volname,
-                  brickinfo->hostname,
-                  brick);
+        if (prefix)
+                snprintf (filename, PATH_MAX, "%s/%s.%s.%s.%s.vol",
+                          path, volinfo->volname, prefix,
+                          brickinfo->hostname, brick);
+        else
+                snprintf (filename, PATH_MAX, "%s/%s.%s.%s.vol",
+                          path, volinfo->volname,
+                          brickinfo->hostname, brick);
 }
 
 gf_boolean_t
@@ -5128,7 +5132,7 @@ glusterd_is_valid_volfpath (char *volname, char *brick)
                 goto out;
         }
         strncpy (volinfo->volname, volname, sizeof (volinfo->volname));
-        get_brick_filepath (volfpath, volinfo, brickinfo);
+        get_brick_filepath (volfpath, volinfo, brickinfo, NULL);
 
         ret = ((strlen(volfpath) < PATH_MAX) &&
                 strlen (strrchr(volfpath, '/')) < _POSIX_PATH_MAX);
@@ -5153,7 +5157,7 @@ glusterd_generate_brick_volfile (glusterd_volinfo_t *volinfo,
         GF_ASSERT (volinfo);
         GF_ASSERT (brickinfo);
 
-        get_brick_filepath (filename, volinfo, brickinfo);
+        get_brick_filepath (filename, volinfo, brickinfo, NULL);
 
         ret = build_server_graph (&graph, volinfo, mod_dict, brickinfo);
         if (!ret)
@@ -5417,6 +5421,55 @@ generate_single_transport_client_volfile (glusterd_volinfo_t *volinfo,
         return ret;
 }
 
+int
+glusterd_generate_client_per_brick_volfile (glusterd_volinfo_t *volinfo)
+{
+        char                   filepath[PATH_MAX]   = {0, };
+        glusterd_brickinfo_t  *brick                = NULL;
+        volgen_graph_t         graph                = {0, };
+        dict_t                *dict                 = NULL;
+        xlator_t              *xl                   = NULL;
+        int                    ret                  = -1;
+
+        dict = dict_new ();
+        if (!dict)
+                goto out;
+
+        ret = dict_set_uint32 (dict, "trusted-client", GF_CLIENT_TRUSTED);
+        if (ret)
+                goto out;
+
+        cds_list_for_each_entry (brick, &volinfo->bricks, brick_list) {
+                xl = volgen_graph_build_client (&graph, volinfo,
+                                                brick->hostname, brick->path,
+                                                brick->brick_id,
+                                                "tcp", dict);
+                if (!xl) {
+                        ret = -1;
+                        goto out;
+                }
+
+                get_brick_filepath (filepath, volinfo, brick, "client");
+                ret = volgen_write_volfile (&graph, filepath);
+                if (ret < 0)
+                        goto out;
+
+                volgen_graph_free (&graph);
+                memset (&graph, 0, sizeof (graph));
+        }
+
+
+        ret = 0;
+out:
+        if (ret)
+                volgen_graph_free (&graph);
+
+        if (dict)
+                dict_unref (dict);
+
+        return ret;
+}
+
 static void
 enumerate_transport_reqs (gf_transport_type type, char **types)
 {
@@ -6089,7 +6142,7 @@ glusterd_delete_volfile (glusterd_volinfo_t *volinfo,
         GF_ASSERT (volinfo);
         GF_ASSERT (brickinfo);
 
-        get_brick_filepath (filename, volinfo, brickinfo);
+        get_brick_filepath (filename, volinfo, brickinfo, NULL);
         ret = sys_unlink (filename);
         if (ret)
                 gf_msg ("glusterd", GF_LOG_ERROR, errno,
diff --git a/xlators/mgmt/glusterd/src/glusterd-volgen.h b/xlators/mgmt/glusterd/src/glusterd-volgen.h
index 9d206f692c1..df6d9e5da7e 100644
--- a/xlators/mgmt/glusterd/src/glusterd-volgen.h
+++ b/xlators/mgmt/glusterd/src/glusterd-volgen.h
@@ -176,6 +176,9 @@ glusterd_create_volfiles (glusterd_volinfo_t *volinfo);
 int
 glusterd_create_volfiles_and_notify_services (glusterd_volinfo_t *volinfo);
 
+int
+glusterd_generate_client_per_brick_volfile (glusterd_volinfo_t *volinfo);
+
 void
 glusterd_get_nfs_filepath (char *filename);
 
diff --git a/xlators/mgmt/glusterd/src/glusterd.h b/xlators/mgmt/glusterd/src/glusterd.h
index 587c2e82774..65ab156e41b 100644
--- a/xlators/mgmt/glusterd/src/glusterd.h
+++ b/xlators/mgmt/glusterd/src/glusterd.h
@@ -595,6 +595,11 @@ typedef ssize_t (*gd_serialize_t) (struct iovec outmsg, void *args);
         snprintf (abspath, sizeof (abspath)-1,                          \
                   DEFAULT_VAR_RUN_DIRECTORY"/%s%s", volname, path);
 
+#define GLUSTERD_GET_TMP_PATH(abspath, path) do {                       \
+        snprintf (abspath, sizeof (abspath)-1,                          \
+                  DEFAULT_VAR_RUN_DIRECTORY"/tmp%s", path);             \
+        } while (0)
+
 #define GLUSTERD_REMOVE_SLASH_FROM_PATH(path,string) do {                  \
                 int i = 0;                                                 \
                 for (i = 1; i < strlen (path); i++) {                      \
@@ -623,6 +628,18 @@ typedef ssize_t (*gd_serialize_t) (struct iovec outmsg, void *args);
                           quotadpath);                                     \
         }
 
+#define GLUSTERD_GET_QUOTA_CRAWL_PIDDIR(piddir, volinfo, type) do {           \
+                char _volpath[PATH_MAX]  = {0,};                              \
+                GLUSTERD_GET_VOLUME_DIR (_volpath, volinfo, priv);            \
+                if (type == GF_QUOTA_OPTION_TYPE_ENABLE ||                    \
+                    type == GF_QUOTA_OPTION_TYPE_ENABLE_OBJECTS)              \
+                        snprintf (piddir, PATH_MAX, "%s/run/quota/enable",    \
+                                  _volpath);                                  \
+                else                                                          \
+                        snprintf (piddir, PATH_MAX, "%s/run/quota/disable",   \
+                                  _volpath);                                  \
+        } while (0)
+
 #define GLUSTERD_STACK_DESTROY(frame) do {\
                 frame->local = NULL;                                    \
                 STACK_DESTROY (frame->root);                            \