syncdaemon: yet another try to exit properly

The final cleanup sequence + call to _exit, which was just done in the main
thread, now is called for in each thread when the thread crashes. Seems we
aren't left there hanging this way.

Signed-off-by: Csaba Henk <csaba@gluster.com>
Signed-off-by: Anand Avati <avati@gluster.com>

BUG: 2736 (gsyncd hangs if crash occurs in the non-main thread)
URL: http://bugs.gluster.com/cgi-bin/bugzilla3/show_bug.cgi?id=2736

1 files changed, 85 insertions, 3 deletions

diff --git a/xlators/features/marker/utils/syncdaemon/syncdutils.py b/xlators/features/marker/utils/syncdaemon/syncdutils.py
index 48694d238..c8f751d33 100644
--- a/xlators/features/marker/utils/syncdaemon/syncdutils.py
+++ b/xlators/features/marker/utils/syncdaemon/syncdutils.py
@@ -1,7 +1,15 @@
 import os
+import sys
+import time
 import fcntl
+import shutil
+import logging
 from threading import Thread as baseThread
-from signal import SIGTERM
+from errno import EACCES, EAGAIN
+from signal import SIGTERM, SIGKILL
+from time import sleep
+
+from gconf import gconf
 
 try:
     # py 3
@@ -49,6 +57,78 @@ def update_file(path, updater, merger = lambda f: True):
             if fx:
                 fx.close()
 
+def grabfile(fname, content=None):
+    # damn those messy open() mode codes
+    fd = os.open(fname, os.O_CREAT|os.O_RDWR)
+    f = os.fdopen(fd, 'r+b', 0)
+    try:
+        fcntl.lockf(f, fcntl.LOCK_EX|fcntl.LOCK_NB)
+    except:
+        ex = sys.exc_info()[1]
+        f.close()
+        if isinstance(ex, IOError) and ex.errno in (EACCES, EAGAIN):
+            # cannot grab, it's taken
+            return
+        raise
+    if content:
+        try:
+            f.truncate()
+            f.write(content)
+        except:
+            f.close()
+            raise
+    gconf.permanent_handles.append(f)
+    return f
+
+def grabpidfile(fname=None, setpid=True):
+    if not fname:
+        fname = gconf.pid_file
+    content = None
+    if setpid:
+        content = str(os.getpid()) + '\n'
+    return grabfile(fname, content=content)
+
+def finalize(*a):
+    if getattr(gconf, 'pid_file', None):
+        rm_pidf = gconf.pid_file_owned
+        if gconf.cpid:
+            # exit path from parent branch of daemonization
+            rm_pidf = False
+            while True:
+                f = grabpidfile(setpid=False)
+                if not f:
+                    # child has already taken over pidfile
+                    break
+                if os.waitpid(gconf.cpid, os.WNOHANG)[0] == gconf.cpid:
+                    # child has terminated
+                    rm_pidf = True
+                    break;
+                time.sleep(0.1)
+        if rm_pidf:
+            try:
+                os.unlink(gconf.pid_file)
+            except:
+                ex = sys.exc_info()[1]
+                if ex.errno == ENOENT:
+                    pass
+                else:
+                    raise
+    if gconf.ssh_ctl_dir and not gconf.cpid:
+        shutil.rmtree(gconf.ssh_ctl_dir)
+    sys.stdout.flush()
+    sys.stderr.flush()
+
+def log_raise_exception(excont):
+    exc = sys.exc_info()[1]
+    if isinstance(exc, SystemExit):
+        excont.exval = exc.code or 0
+        raise
+    else:
+        logging.exception("FAIL: ")
+        sys.stderr.write("failed with %s.\n" % type(exc).__name__)
+        excont.exval = 1
+        sys.exit(excont.exval)
+
 
 class FreeObject(object):
     """wildcard class for which any attribute can be set"""
@@ -63,13 +143,15 @@ class Thread(baseThread):
         tf = kw.get('target')
         if tf:
             def twrap(*aa):
+                excont = FreeObject(exval = 0)
                 try:
                     tf(*aa)
                 except:
                     try:
-                        raise
+                        log_raise_exception(excont)
                     finally:
-                        os.kill(os.getpid(), SIGTERM)
+                        finalize()
+                        os._exit(excont.exval)
             kw['target'] = twrap
         baseThread.__init__(self, *a, **kw)
         self.setDaemon(True)