summaryrefslogtreecommitdiffstats
path: root/xlators
diff options
context:
space:
mode:
authorsarvotham s pai <spai@redhat.com>2013-04-01 00:44:34 +0530
committerAnand Avati <avati@redhat.com>2013-04-04 01:04:19 -0700
commit6956c710d9bffcc0d99526a9d6c051c4cebc0bdb (patch)
treef6ed40e7cd65e70e71fda89e9d91c8cf581088ed /xlators
parentfff8008c6cfa3b224859bc4ac7cf8a3d35c22204 (diff)
gsync: Display additional information in status command
Added code to display extra information when status command is executed. Information shown now are 1 Number of files synced 2 crawl time 3 total sync time 4 bytes synced bytes synced is taken from rsync output . --stats option of rsync gives extra infor mation about the sync.In stats output there is a field called Total transferred file size which states the ammount of bytes synced . This information is parsed from stdout output using regular expressions.Bytes synced information can be used to calculate throughput. Change-Id: Id9bba9fff45ee7049bb8257c6fd918e5237e05b1 BUG: 947774 Signed-off-by: sarvotham s pai <spai@redhat.com> Reviewed-on: http://review.gluster.org/4749 Tested-by: Gluster Build System <jenkins@build.gluster.com> Reviewed-by: Anand Avati <avati@redhat.com>
Diffstat (limited to 'xlators')
-rw-r--r--xlators/features/marker/utils/syncdaemon/master.py50
-rw-r--r--xlators/features/marker/utils/syncdaemon/resource.py6
2 files changed, 52 insertions, 4 deletions
diff --git a/xlators/features/marker/utils/syncdaemon/master.py b/xlators/features/marker/utils/syncdaemon/master.py
index 19cb09e7be5..f903f30595d 100644
--- a/xlators/features/marker/utils/syncdaemon/master.py
+++ b/xlators/features/marker/utils/syncdaemon/master.py
@@ -7,6 +7,7 @@ import signal
import logging
import socket
import errno
+import re
from errno import ENOENT, ENODATA, EPIPE
from threading import currentThread, Condition, Lock
from datetime import datetime
@@ -394,6 +395,12 @@ class GMasterBase(object):
self.lastreport = {'crawls': 0, 'turns': 0}
self.start = None
self.change_seen = None
+ self.syncTime=0
+ self.lastSyncTime=0
+ self.crawlStartTime=0
+ self.crawlTime=0
+ self.filesSynced=0
+ self.bytesSynced=0
# the authoritative (foreign, native) volinfo pair
# which lets us deduce what to do when we refetch
# the volinfos from system
@@ -435,6 +442,34 @@ class GMasterBase(object):
ts += '.' + str(tpair[1])
return ts
+ def get_extra_info(self):
+ str_info="\nFile synced : %d" %(self.filesSynced)
+ str_info+="\nBytes Synced : %d KB" %(self.syncer.bytesSynced)
+ str_info+="\nSync Time : %f seconds" %(self.syncTime)
+ self.crawlTime=datetime.now()-self.crawlStartTime
+ years , days =divmod(self.crawlTime.days,365.25)
+ years=int(years)
+ days=int(days)
+
+ date=""
+ m, s = divmod(self.crawlTime.seconds, 60)
+ h, m = divmod(m, 60)
+
+ if years!=0 :
+ date+=str(years)+" year "
+ if days!=0 :
+ date+=str(days)+" day "
+ if h!=0 :
+ date+=str(h)+" H : "
+ if m!=0 or h!=0 :
+ date+=str(m)+" M : "
+
+ date+=str(s)+" S"
+ self.crawlTime=date
+ str_info+="\nCrawl Time : %s" %(str(self.crawlTime))
+ str_info+="\n\0"
+ return str_info
+
def checkpt_service(self, chan, chkpt, tgt):
"""checkpoint service loop
@@ -446,7 +481,7 @@ class GMasterBase(object):
while True:
select([chan], [], [])
conn, _ = chan.accept()
- conn.send('\0')
+ conn.send(self.get_extra_info())
conn.close()
completed = self._checkpt_param(chkpt, 'completed', xtimish=False)
if completed:
@@ -482,7 +517,7 @@ class GMasterBase(object):
try:
conn, _ = chan.accept()
try:
- conn.send(" | checkpoint %s %s\0" % (chkpt, status))
+ conn.send(" | checkpoint %s %s %s" % (chkpt, status,self.get_extra_info()))
except:
exc = sys.exc_info()[1]
if (isinstance(exc, OSError) or isinstance(exc, IOError)) and \
@@ -536,6 +571,7 @@ class GMasterBase(object):
t = Thread(target=keep_alive)
t.start()
self.lastreport['time'] = time.time()
+ self.crawlStartTime=datetime.now()
while not self.terminate:
self.crawl()
@@ -779,10 +815,16 @@ class GMasterBase(object):
elif stat.S_ISREG(mo):
logging.debug("syncing %s ..." % e)
pb = self.syncer.add(e)
+ timeA=datetime.now()
def regjob(e, xte, pb):
if pb.wait():
logging.debug("synced " + e)
self.sendmark_regular(e, xte)
+
+ timeB=datetime.now()
+ self.lastSyncTime=timeB-timeA
+ self.syncTime=(self.syncTime+self.lastSyncTime.microseconds)/(10.0**6)
+ self.filesSynced=self.filesSynced+1
return True
else:
logging.warn("failed to sync " + e)
@@ -878,6 +920,7 @@ class Syncer(object):
self.slave = slave
self.lock = Lock()
self.pb = PostBox()
+ self.bytesSynced=0
for i in range(int(gconf.sync_jobs)):
t = Thread(target=self.syncjob)
t.start()
@@ -897,6 +940,9 @@ class Syncer(object):
pb.close()
po = self.slave.rsync(pb)
if po.returncode == 0:
+ regEx=re.search('\ *total\ *transferred\ *file\ *size:\ *(\d+)\ *bytes\ *',po.stdout.read(),re.IGNORECASE)
+ if regEx:
+ self.bytesSynced+=(int(regEx.group(1)))/1024
ret = True
elif po.returncode in (23, 24):
# partial transfer (cf. rsync(1)), that's normal
diff --git a/xlators/features/marker/utils/syncdaemon/resource.py b/xlators/features/marker/utils/syncdaemon/resource.py
index adcf5585964..73102fbcb44 100644
--- a/xlators/features/marker/utils/syncdaemon/resource.py
+++ b/xlators/features/marker/utils/syncdaemon/resource.py
@@ -537,16 +537,18 @@ class SlaveRemote(object):
raise GsyncdError("no files to sync")
logging.debug("files: " + ", ".join(files))
argv = gconf.rsync_command.split() + \
- ['-aR0', '--files-from=-', '--super', '--numeric-ids', '--no-implied-dirs'] + \
+ ['-aR0', '--files-from=-', '--super','--stats', '--numeric-ids', '--no-implied-dirs'] + \
gconf.rsync_options.split() + (boolify(gconf.use_rsync_xattrs) and ['--xattrs'] or []) + \
['.'] + list(args)
- po = Popen(argv, stdin=subprocess.PIPE, stderr=subprocess.PIPE)
+ po = Popen(argv, stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
for f in files:
po.stdin.write(f)
po.stdin.write('\0')
+
po.stdin.close()
po.wait()
po.terminate_geterr(fail_on_err = False)
+
return po