#!/usr/bin/python3 # # Copyright (c) 2015 Red Hat, Inc. # This file is part of GlusterFS. # # This file is licensed to you under your choice of the GNU Lesser # General Public License, version 3 or any later version (LGPLv3 or # later), or the GNU General Public License, version 2 (GPLv2), in all # cases as published by the Free Software Foundation. from __future__ import print_function import subprocess import os import os.path import sys import time import logging import logging.handlers import fcntl GCRON_TASKS = "/run/gluster/shared_storage/snaps/glusterfs_snap_cron_tasks" GCRON_CROND_TASK = "/etc/cron.d/glusterfs_snap_cron_tasks" GCRON_RELOAD_FLAG = "/var/run/gluster/crond_task_reload_flag" LOCK_FILE_DIR = "/run/gluster/shared_storage/snaps/lock_files/" log = logging.getLogger("gcron-logger") start_time = 0.0 def initLogger(script_name): log.setLevel(logging.DEBUG) logFormat = "[%(asctime)s %(filename)s:%(lineno)s %(funcName)s] "\ "%(levelname)s %(message)s" formatter = logging.Formatter(logFormat) sh = logging.handlers.SysLogHandler() sh.setLevel(logging.ERROR) sh.setFormatter(formatter) process = subprocess.Popen(["gluster", "--print-logdir"], stdout=subprocess.PIPE, universal_newlines=True) out, err = process.communicate() if process.returncode == 0: logfile = os.path.join(out.strip(), script_name[:-3]+".log") fh = logging.FileHandler(logfile) fh.setLevel(logging.DEBUG) fh.setFormatter(formatter) log.addHandler(sh) log.addHandler(fh) def takeSnap(volname="", snapname=""): success = True if volname == "": log.debug("No volname given") return False if snapname == "": log.debug("No snapname given") return False cli = ["gluster", "snapshot", "create", snapname, volname] log.debug("Running command '%s'", " ".join(cli)) p = subprocess.Popen(cli, stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, err = p.communicate() rv = p.returncode log.debug("Command '%s' returned '%d'", " ".join(cli), rv) if rv: log.error("Snapshot of %s failed", volname) log.error("Command output:") log.error(err) success = False else: log.info("Snapshot of %s successful", volname) return success def doJob(name, lockFile, jobFunc, volname): success = True try: f = os.open(lockFile, os.O_CREAT | os.O_RDWR | os.O_NONBLOCK) try: fcntl.flock(f, fcntl.LOCK_EX | fcntl.LOCK_NB) mtime = os.path.getmtime(lockFile) global start_time log.debug("%s last modified at %s", lockFile, time.ctime(mtime)) if mtime < start_time: log.debug("Processing job %s", name) if jobFunc(volname, name): log.info("Job %s succeeded", name) else: log.error("Job %s failed", name) success = False os.utime(lockFile, None) else: log.info("Job %s has been processed already", name) fcntl.flock(f, fcntl.LOCK_UN) except (OSError, IOError): log.info("Job %s is being processed by another agent", name) os.close(f) except (OSError, IOError) as e: log.debug("Failed to open lock file %s : %s", lockFile, e) log.error("Failed to process job %s", name) success = False return success def main(): script_name = os.path.basename(__file__) initLogger(script_name) global start_time if sys.argv[1] == "--update": if not os.path.exists(GCRON_TASKS): # Create a flag in /var/run/gluster which indicates that this # node doesn't have access to GCRON_TASKS right now, so that # when the mount is available and GCRON_TASKS is available # the flag will tell this routine to reload GCRON_CROND_TASK try: f = os.open(GCRON_RELOAD_FLAG, os.O_CREAT | os.O_NONBLOCK, 0o644) os.close(f) except OSError as e: if errno != EEXIST: log.error("Failed to create %s : %s", GCRON_RELOAD_FLAG, e) output("Failed to create %s. Error: %s" % (GCRON_RELOAD_FLAG, e)) return if not os.path.exists(GCRON_CROND_TASK): return # As GCRON_TASKS exists now, we should check if GCRON_RELOAD_FLAG # also exists. If so we should touch GCRON_CROND_TASK and remove # the GCRON_RELOAD_FLAG if os.path.exists(GCRON_RELOAD_FLAG): try: os.remove(GCRON_RELOAD_FLAG); process = subprocess.Popen(["touch", "-h", GCRON_CROND_TASK], stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, err = process.communicate() if process.returncode != 0: log.error("Failed to touch %s. Error: %s.", GCRON_CROND_TASK, err) except (IOError, OSError) as e: log.error("Failed to touch %s. Error: %s.", GCRON_CROND_TASK, e) return if os.lstat(GCRON_TASKS).st_mtime > \ os.lstat(GCRON_CROND_TASK).st_mtime: try: process = subprocess.Popen(["touch", "-h", GCRON_CROND_TASK], stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, err = process.communicate() if process.returncode != 0: log.error("Failed to touch %s. Error: %s.", GCRON_CROND_TASK, err) except IOError as e: log.error("Failed to touch %s. Error: %s.", GCRON_CROND_TASK, e) return volname = sys.argv[1] jobname = sys.argv[2] locking_file = os.path.join(LOCK_FILE_DIR, jobname) log.debug("locking_file = %s", locking_file) log.debug("volname = %s", volname) log.debug("jobname = %s", jobname) start_time = int(time.time()) doJob("Scheduled-" + jobname + "-" + volname, locking_file, takeSnap, volname) if __name__ == "__main__": main()