# This script was developed by Vijaykumar Koppad (vkoppad@redhat.com) # The latest version of this script can found at # http://github.com/vijaykumar-koppad/crefi from __future__ import with_statement import os import re import sys import time import errno import xattr import string import random import logging import tarfile import argparse datsiz = 0 timr = 0 def get_ascii_upper_alpha_digits(): if sys.version_info > (3,0): return string.ascii_uppercase+string.digits else: return string.uppercase+string.digits def setLogger(filename): global logger logger = logging.getLogger(filename) logger.setLevel(logging.DEBUG) return def setupLogger(filename): logger = logging.getLogger(filename) logger.setLevel(logging.DEBUG) formatter = logging.Formatter('%(asctime)s - %(message)s') ch = logging.StreamHandler() ch.setLevel(logging.INFO) ch.setFormatter(formatter) logger.addHandler(ch) return logger def os_rd(src, size): global datsiz fd = os.open(src, os.O_RDONLY) data = os.read(fd, size) os.close(fd) datsiz = datsiz + size return data def os_wr(dest, data): global timr st = time.time() fd = os.open(dest, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o644) os.write(fd, data) os.close(fd) ed = time.time() timr = timr+(ed-st) return def create_sparse_file(fil, size, mins, maxs, rand): if rand: size = random.randint(mins, maxs) else: size = size data = os_rd("/dev/zero", size) os_wr(fil, data) return def create_binary_file(fil, size, mins, maxs, rand): if rand: size = random.randint(mins, maxs) else: size = size data = os_rd("/dev/urandom", size) os_wr(fil, data) return def create_txt_file(fil, size, mins, maxs, rand): if rand: size = random.randint(mins, maxs) if size < 500*1024: data = os_rd("/etc/services", size) os_wr(fil, data) else: data = os_rd("/etc/services", 512*1024) file_size = 0 fd = os.open(fil, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o644) while file_size < size: os.write(fd, data) file_size += 500*1024 os.close(fd) return def create_tar_file(fil, size, mins, maxs, rand): if rand: size = random.randint(mins, maxs) else: size = size data = os_rd("/dev/urandom", size) os_wr(fil, data) tar = tarfile.open(fil+".tar.gz", "w:gz") tar.add(fil) tar.close() os.unlink(fil) return def get_filename(flen): size = flen char = get_ascii_upper_alpha_digits() st = ''.join(random.choice(char) for i in range(size)) ti = str((hex(int(str(time.time()).split('.')[0])))[2:]) return ti+"%%"+st def text_files(files, file_count, inter, size, mins, maxs, rand, flen, randname, dir_path): global datsiz, timr for k in range(files): if not file_count % inter: logger.info("Total files created -- "+str(file_count)) if not randname: fil = dir_path+"/"+"file"+str(k) else: fil = dir_path+"/"+get_filename(flen) create_txt_file(fil, size, mins, maxs, rand) file_count += 1 return file_count def sparse_files(files, file_count, inter, size, mins, maxs, rand, flen, randname, dir_path): for k in range(files): if not file_count % inter: logger.info("Total files created -- "+str(file_count)) if not randname: fil = dir_path+"/"+"file"+str(k) else: fil = dir_path+"/"+get_filename(flen) create_sparse_file(fil, size, mins, maxs, rand) file_count += 1 return file_count def binary_files(files, file_count, inter, size, mins, maxs, rand, flen, randname, dir_path): for k in range(files): if not file_count % inter: logger.info("Total files created -- "+str(file_count)) if not randname: fil = dir_path+"/"+"file"+str(k) else: fil = dir_path+"/"+get_filename(flen) create_binary_file(fil, size, mins, maxs, rand) file_count += 1 return file_count def tar_files(files, file_count, inter, size, mins, maxs, rand, flen, randname, dir_path): for k in range(files): if not file_count % inter: logger.info("Total files created -- "+str(file_count)) if not randname: fil = dir_path+"/"+"file"+str(k) else: fil = dir_path+"/"+get_filename(flen) create_tar_file(fil, size, mins, maxs, rand) file_count += 1 return file_count def setxattr_files(files, randname, dir_path): char = get_ascii_upper_alpha_digits() if not randname: for k in range(files): v = ''.join(random.choice(char) for i in range(10)) n = "user."+v xattr.setxattr(dir_path+"/"+"file"+str(k), n, v) else: dirs = os.listdir(dir_path+"/") for fil in dirs: v = ''.join(random.choice(char) for i in range(10)) n = "user."+v xattr.setxattr(dir_path+"/"+fil, n, v) return def rename_files(files, flen, randname, dir_path): if not randname: for k in range(files): os.rename(dir_path + "/" + "file" + str(k), dir_path + "/" + "file" + str(files+k)) else: dirs = os.listdir(dir_path) for fil in dirs: if not os.path.isdir(fil): newfil = get_filename(flen) os.rename(dir_path + "/" + fil, dir_path + "/" + newfil) return def truncate_files(files, mins, maxs, randname, dir_path): if not randname: for k in range(files): byts = random.randint(mins, maxs) fd = os.open(dir_path + "/" + "file" + str(k), os.O_WRONLY) os.ftruncate(fd, byts) os.close(fd) else: dirs = os.listdir(dir_path) for fil in dirs: if not os.path.isdir(dir_path+"/"+fil): byts = random.randint(mins, maxs) fd = os.open(dir_path+"/"+fil, os.O_WRONLY) os.ftruncate(fd, byts) os.close(fd) return def chmod_files(files, flen, randname, dir_path): if not randname: for k in range(files): mod = random.randint(0, 511) os.chmod(dir_path+"/"+"file"+str(k), mod) else: dirs = os.listdir(dir_path) for fil in dirs: mod = random.randint(0, 511) os.chmod(dir_path+"/"+fil, mod) return def random_og(path): u = random.randint(1025, 65536) g = -1 os.chown(path, u, g) def chown_files(files, flen, randname, dir_path): if not randname: for k in range(files): random_og(dir_path+"/"+"file"+str(k)) else: dirs = os.listdir(dir_path) for fil in dirs: random_og(dir_path+"/"+fil) return def chgrp_files(files, flen, randname, dir_path): if not randname: for k in range(files): random_og(dir_path+"/"+"file"+str(k)) else: dirs = os.listdir(dir_path) for fil in dirs: random_og(dir_path+"/"+fil) return def symlink_files(files, flen, randname, dir_path): try: os.makedirs(dir_path+"/"+"symlink_to_files") except OSError as ex: if ex.errno is not errno.EEXIST: raise if not randname: for k in range(files): src_file = "file"+str(k) os.symlink(dir_path+"/"+src_file, dir_path+"/"+"symlink_to_files/file"+str(k)+"_sym") else: dirs = os.listdir(dir_path) for fil in dirs: newfil = get_filename(flen) os.symlink(dir_path+"/"+fil, dir_path+"/"+"symlink_to_files/"+newfil) return def hardlink_files(files, flen, randname, dir_path): try: os.makedirs(dir_path+"/"+"hardlink_to_files") except OSError as ex: if ex.errno is not errno.EEXIST: raise if not randname: for k in range(files): src_file = "file"+str(k) os.link(dir_path+"/"+src_file, dir_path+"/"+"hardlink_to_files/file"+str(k)+"_hard") else: dirs = os.listdir(dir_path) for fil in dirs: if not os.path.isdir(dir_path+"/"+fil): newfil = get_filename(flen) os.link(dir_path+"/"+fil, dir_path+"/"+"hardlink_to_files/"+newfil) return def human2bytes(size): size_short = { 1024: ['K', 'KB', 'KiB', 'k', 'kB', 'kiB'], 1024*1024: ['M', 'MB', 'MiB'], 1024*1024*1024: ['G', 'GB', 'GiB'] } num = re.search('(\d+)', size).group() ext = size[len(num):] num = int(num) if ext == '': return num for value, keys in size_short.items(): if ext in keys: size = num*value return size def bytes2human(byts): abbr = { 1 << 30: "GB", 1 << 20: "MB", 1 << 10: "KB", 1: "bytes" } if byts == 1: return '1 bytes' for factor, suffix in abbr.items(): if byts >= factor: break return "%.3f %s" % (byts / factor, suffix) def multipledir(mnt_pnt, brdth, depth, files, fop, file_type="text", inter="1000", size="100K", mins="10K", maxs="500K", rand=False, l=10, randname=False): files_count = 1 size = human2bytes(size) maxs = human2bytes(maxs) mins = human2bytes(mins) for i in range(brdth): dir_path = mnt_pnt for j in range(depth): dir_path = dir_path+"/"+"level"+str(j)+str(i) try: os.makedirs(dir_path) except OSError as ex: if ex.errno is not errno.EEXIST: raise if fop == "create": logger.info("Entering the directory level"+str(j)+str(i)) if file_type == "text": files_count = text_files(files, files_count, inter, size, mins, maxs, rand, l, randname, dir_path) elif file_type == "sparse": files_count = sparse_files(files, files_count, inter, size, mins, maxs, rand, l, randname, dir_path) elif file_type == "binary": files_count = binary_files(files, files_count, inter, size, mins, maxs, rand, l, randname, dir_path) elif file_type == "tar": files_count = tar_files(files, files_count, inter, size, mins, maxs, rand, l, randname, dir_path) else: logger.error("Not a valid file type") sys.exit(1) elif fop == "rename": logger.info("Started renaming files for the files 0 to " + str(files)+" in the directory level"+str(j) + str(i)+" ...") rename_files(files, l, randname, dir_path) logger.info("Finished renaming files for the files 0 to " + str(files)+" in the directory level"+str(j)+str(i)) elif fop == "chmod": logger.info("Started changing permission of files for the " + "files 0 to "+str(files)+" in the directory level" + str(j)+str(i)+" ...") chmod_files(files, l, randname, dir_path) logger.info("Finished changing permission of files for " + "the files 0 to "+str(files) + " in the directory level"+str(j)+str(i)) elif fop == "chown": logger.info("Started changing ownership of files for the " + "files 0 to " + str(files) + " in the directory level"+str(j)+str(i)+" ...") chown_files(files, l, randname, dir_path) logger.info("Finished changing ownership of files for " + "the files 0 to "+str(files) + " in the directory level"+str(j)+str(i)) elif fop == "chgrp": logger.info("Started changing group ownership of files for " + "the files 0 to " + str(files) + " in the directory level"+str(j)+str(i)+" ...") chgrp_files(files, l, randname, dir_path) logger.info("Finished changing group ownership of files for " + "the files 0 to "+str(files) + " in the directory level"+str(j)+str(i)) elif fop == "symlink": logger.info("Started creating symlink to the files 0 to " + str(files)+" in the directory level" + str(j)+str(i)+"...") symlink_files(files, l, randname, dir_path) logger.info("Finished creating symlink to the files 0 to " + str(files) + " in the directory level" + str(j)+str(i)) elif fop == "hardlink": logger.info("Started creating hardlink to the files 0 to " + str(files)+" in the directory level" + str(j)+str(i)+"...") hardlink_files(files, l, randname, dir_path) logger.info("Finished creating hardlink to the files 0 to " + str(files) + " in the directory level" + str(j)+str(i)) elif fop == "truncate": logger.info("Started truncating the files 0 to " + str(files)+" in the directory level" + str(j)+str(i)+"...") truncate_files(files, mins, maxs, randname, dir_path) logger.info("Finished truncating the files 0 to " + str(files)+" in the directory level" + str(j)+str(i)) elif fop == "setxattr": logger.info("Started setxattr to the files 0 to " + str(files)+" in the directory level" + str(j)+str(i)+"...") setxattr_files(files, randname, dir_path) logger.info("Finished setxattr to the files 0 to " + str(files)+" in the directory level" + str(j)+str(i)) if fop == "create": thrpt = datsiz / timr logger.info("finished creating files with throughput ---- " + bytes2human(thrpt)+"ps") def singledir(mnt_pnt, files, fop, file_type="text", inter="1000", size="100K", mins="10K", maxs="500K", rand=False, l=10, randname=False): files_count = 1 size = human2bytes(size) maxs = human2bytes(maxs) mins = human2bytes(mins) if fop == "create": if file_type == "text": files_count = text_files(files, files_count, inter, size, mins, maxs, rand, l, randname, mnt_pnt) elif file_type == "sparse": files_count = sparse_files(files, files_count, inter, size, mins, maxs, rand, l, randname, mnt_pnt) elif file_type == "binary": files_count = binary_files(files, files_count, inter, size, mins, maxs, rand, l, randname, mnt_pnt) elif file_type == "tar": files_count = tar_files(files, files_count, inter, size, mins, maxs, rand, l, randname, mnt_pnt) else: logger.info("Not a valid file type") sys.exit(1) thrpt = datsiz / timr logger.info("finished creating files with avg throughput ---- " + bytes2human(thrpt)+"ps") elif fop == "rename": logger.info("Started renaming files for the files 0 to " + str(files) + "...") rename_files(files, l, randname, mnt_pnt) logger.info("Finished renaming files for the files 0 to "+str(files)) elif fop == "chmod": logger.info("Started changing permission for the files 0 to " + str(files)+" ...") chmod_files(files, l, randname, mnt_pnt) logger.info("Finished changing permission files for the files 0 to " + str(files)) elif fop == "chown": logger.info("Started changing ownership for the files 0 to " + str(files)+"...") chown_files(files, l, randname, mnt_pnt) logger.info("Finished changing ownership for the files 0 to " + str(files)) elif fop == "chgrp": logger.info("Started changing group ownership for the files 0 to " + str(files)+"...") chgrp_files(files, l, randname, mnt_pnt) logger.info("Finished changing group ownership for the files 0 to " + str(files)) elif fop == "symlink": logger.info("Started creating symlink to the files 0 to " + str(files)+"...") symlink_files(files, l, randname, mnt_pnt) logger.info("Finished creating symlink to the files 0 to " + str(files)) elif fop == "hardlink": logger.info("Started creating hardlink to the files 0 to " + str(files)+"...") hardlink_files(files, l, randname, mnt_pnt) logger.info("Finished creating hardlink to the files 0 to " + str(files)) elif fop == "truncate": logger.info("Started truncating the files 0 to " + str(files)+"...") truncate_files(files, mins, maxs, randname, mnt_pnt) logger.info("Finished truncating the files 0 to " + str(files)) elif fop == "setxattr": logger.info("Started setxattr to the files 0 to " + str(files)+"...") setxattr_files(files, randname, mnt_pnt) logger.info("Finished setxattr to the files 0 to " + str(files)) if __name__ == '__main__': usage = "usage: %prog [option] " parser = argparse.ArgumentParser(formatter_class=argparse. ArgumentDefaultsHelpFormatter) parser.add_argument("-n", dest="files", type=int, default=100, help="number of files in each level ") parser.add_argument("--size", action="store", default="100k", help="size of the files to be used ") parser.add_argument("--random", action="store_true", default=False, help="random size of the file between --min and --max") parser.add_argument("--max", action="store", default="500K", help="maximum size of the files, if random is True") parser.add_argument("--min", action="store", default="10K", help="minimum size of the files, if random is True") parser.add_argument("--single", action="store_true", dest="dir", default=True, help="create files in single directory") parser.add_argument("--multi", action="store_false", dest="dir", help="create files in multiple directories") parser.add_argument("-b", dest="brdth", type=int, default=5, help="number of directories in one level(works " + "with --multi) ") parser.add_argument("-d", dest="depth", type=int, default=5, help="number of levels of directories (works " + "with --multi) ") parser.add_argument("-l", dest="flen", type=int, default=10, help="number of bytes for filename ( Used only when " + "randname is enabled) ") parser.add_argument("-t", action="store", dest="file_type", default="text", choices=["text", "sparse", "binary", "tar"], help="type of the file to be created ()") parser.add_argument("-I", dest="inter", type=int, default=100, help="print number files created of interval") parser.add_argument("--fop", action="store", dest="fop", default="create", choices=["create", "rename", "chmod", "chown", "chgrp", "symlink", "hardlink", "truncate", "setxattr"], help="fop to be performed on the files") parser.add_argument("-R", dest="randname", action="store_false", default=True, help="To disable random file name " + "(default: Enabled)") parser.add_argument("mntpnt", help="Mount point") args = parser.parse_args() logger = setupLogger("testlost") args.mntpnt = os.path.abspath(args.mntpnt) if args.dir: singledir(args.mntpnt, args.files, args.fop, args.file_type, args.inter, args.size, args.min, args.max, args.random, args.flen, args.randname) else: multipledir(args.mntpnt, args.brdth, args.depth, args.files, args.fop, args.file_type, args.inter, args.size, args.min, args.max, args.random, args.flen, args.randname)