summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorAravinda VK <avishwan@redhat.com>2013-12-18 00:02:58 +0530
committerVijay Bellur <vbellur@redhat.com>2014-02-02 20:25:27 -0800
commit540de26e2ef7c45eb0ef6c049844a864542bd38a (patch)
tree0b51f4ffd3068250879cd93fdd2d12c1f5fa0205
parent0f018e33ec8510ef0afc7daa9f12b7e21c8c03dc (diff)
geo-rep: optimizing update stime after directory synchronization
Since xsync crawl generates new changelog when number of entries reaches 8K or when directory is reached. If a directory has number of files less than 8K then respective changelog file will have less entries. Since xsync generated changelog files processed one after the other, so syncjobs are underutilized. hence low bandwidth utilization. With this patch, changelog will be generated for 8K entries only, but stime will be accumulated. Multiple dirs stime will be updated together since the generated changelog will have entries accross the dirs. BUG: 1036539 Change-Id: Ic4b9da6d8c1f11dd9ef7c13bcc7df80efe2c5caf Signed-off-by: Aravinda VK <avishwan@redhat.com> Reviewed-on: http://review.gluster.org/6744 Reviewed-on: http://review.gluster.org/6847 Tested-by: Gluster Build System <jenkins@build.gluster.com> Reviewed-by: Venky Shankar <vshankar@redhat.com> Reviewed-by: Vijay Bellur <vbellur@redhat.com>
-rw-r--r--geo-replication/syncdaemon/master.py20
1 files changed, 14 insertions, 6 deletions
diff --git a/geo-replication/syncdaemon/master.py b/geo-replication/syncdaemon/master.py
index 721fe18bd..662d2ba74 100644
--- a/geo-replication/syncdaemon/master.py
+++ b/geo-replication/syncdaemon/master.py
@@ -1091,6 +1091,7 @@ class GMasterXsyncMixin(GMasterChangelogMixin):
def register(self):
self.counter = 0
self.comlist = []
+ self.stimes = []
self.sleep_interval = 60
self.tempdir = self.setup_working_dir()[0]
self.tempdir = os.path.join(self.tempdir, 'xsync')
@@ -1169,10 +1170,15 @@ class GMasterXsyncMixin(GMasterChangelogMixin):
if last:
self.put('finale', None)
- def sync_done(self, stime=None, last=False):
+ def sync_done(self, stime=[], last=False):
self.sync_xsync(last)
if stime:
- self.sync_stime(stime, last)
+ # Send last as True only for last stime entry
+ for st in stime[:-1]:
+ self.sync_stime(st, False)
+
+ if stime and stime[-1]:
+ self.sync_stime(stime[-1], last)
def Xcrawl(self, path='.', xtr_root=None):
"""
@@ -1204,7 +1210,7 @@ class GMasterXsyncMixin(GMasterChangelogMixin):
xtr = max(xtr, xtr_root)
if not self.need_sync(path, xtl, xtr):
if path == '.':
- self.sync_done((path, xtl), True)
+ self.sync_done([(path, xtl)], True)
return
self.xtime_reversion_hook(path, xtl, xtr)
logging.debug("entering " + path)
@@ -1232,11 +1238,12 @@ class GMasterXsyncMixin(GMasterChangelogMixin):
mo = st.st_mode
self.counter += 1
if self.counter == self.XSYNC_MAX_ENTRIES:
- self.sync_done()
+ self.sync_done(self.stimes, False)
+ self.stimes = []
if stat.S_ISDIR(mo):
self.write_entry_change("E", [gfid, 'MKDIR', str(mo), str(st.st_uid), str(st.st_gid), escape(os.path.join(pargfid, bname))])
self.Xcrawl(e, xtr_root)
- self.sync_done((e, xte), False)
+ self.stimes.append((e, xte))
elif stat.S_ISLNK(mo):
self.write_entry_change("E", [gfid, 'SYMLINK', escape(os.path.join(pargfid, bname))])
elif stat.S_ISREG(mo):
@@ -1250,7 +1257,8 @@ class GMasterXsyncMixin(GMasterChangelogMixin):
self.write_entry_change("E", [gfid, 'LINK', escape(os.path.join(pargfid, bname))])
self.write_entry_change("D", [gfid])
if path == '.':
- self.sync_done((path, xtl), True)
+ self.stimes.append((path, xtl))
+ self.sync_done(self.stimes, True)
class BoxClosedErr(Exception):
pass