diff options
Diffstat (limited to 'glustolibs-io/shared_files/scripts/file_dir_ops.py')
| -rwxr-xr-x | glustolibs-io/shared_files/scripts/file_dir_ops.py | 473 |
1 files changed, 336 insertions, 137 deletions
diff --git a/glustolibs-io/shared_files/scripts/file_dir_ops.py b/glustolibs-io/shared_files/scripts/file_dir_ops.py index 96e53262d..908a48c8e 100755 --- a/glustolibs-io/shared_files/scripts/file_dir_ops.py +++ b/glustolibs-io/shared_files/scripts/file_dir_ops.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# Copyright (C) 2015-2018 Red Hat, Inc. <http://www.redhat.com> +# Copyright (C) 2015-2019 Red Hat, Inc. <http://www.redhat.com> # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -20,18 +20,22 @@ """ from __future__ import print_function -import os import argparse -import sys -import random -import string +import contextlib import datetime from multiprocessing import Process -import subprocess -from docx import Document -import contextlib +from multiprocessing.pool import ThreadPool +import os import platform +import random import shutil +import string +import subprocess +import sys + +from docx import Document +import numpy as np +from sh import rsync as sh_rsync if platform.system() == "Windows": path_sep = "\\" @@ -49,9 +53,9 @@ def is_root(path): True if path is '/' , False otherwise """ if os.path.realpath(os.path.abspath(path)) == '/': - print ("Directory '%s' is the root of filesystem. " - "Not performing any operations on the root of filesystem" % - os.path.abspath(path)) + print("Directory '%s' is the root of filesystem. " + "Not performing any operations on the root of filesystem" % ( + os.path.abspath(path))) return True else: return False @@ -106,7 +110,7 @@ def create_dir(dir_path): try: os.makedirs(dir_abs_path) except (OSError, IOError): - print ("Unable to create dir: %s" % dir_abs_path) + print("Unable to create dir: %s" % dir_abs_path) return 1 return 0 @@ -138,16 +142,16 @@ def create_dirs(dir_path, depth, num_of_dirs, num_of_files=0, base_file_name, file_types) except (OSError, IOError) as e: if 'File exists' not in e.strerror: - print ("Unable to create dir '%s' : %s" - % (dir_path, e.strerror)) + print("Unable to create dir '%s' : %s" % ( + dir_path, e.strerror)) with open("/tmp/file_dir_ops_create_dirs_rc", "w") as fd: try: fd.write("1") fd.flush() fd.close() - except IOError as e: - print ("Unable to write the rc to the " - "/tmp/file_dir_ops_create_dirs_rc file") + except IOError: + print("Unable to write the rc to the " + "/tmp/file_dir_ops_create_dirs_rc file") if depth == 0: return 0 for i in range(num_of_dirs): @@ -183,9 +187,10 @@ def create_deep_dirs(args): for i in range(dirname_start_num, (dirname_start_num + dir_length)): num_of_dirs = random.choice(range(1, max_num_of_dirs + 1)) process_dir_path = os.path.join(dir_path, "user%d" % i) - process_list.append(Process(target=create_dirs, - args=(process_dir_path, dir_depth, - num_of_dirs))) + process_list.append(Process( + target=create_dirs, + args=(process_dir_path, dir_depth, num_of_dirs) + )) for each_process in process_list: each_process.start() @@ -237,11 +242,11 @@ def create_deep_dirs_with_files(args): for i in range(dirname_start_num, (dirname_start_num + dir_length)): num_of_dirs = random.choice(range(1, max_num_of_dirs + 1)) process_dir_path = os.path.join(dir_path, "user%d" % i) - process_list.append(Process(target=create_dirs, - args=(process_dir_path, dir_depth, - num_of_dirs, num_of_files, - fixed_file_size, base_file_name, - file_types))) + process_list.append(Process( + target=create_dirs, + args=(process_dir_path, dir_depth, num_of_dirs, num_of_files, + fixed_file_size, base_file_name, file_types) + )) for each_process in process_list: each_process.start() @@ -256,6 +261,48 @@ def create_deep_dirs_with_files(args): return int(rc) +def _create_file(file_abs_path, file_type, file_size): + rc = 0 + + if file_type == 'txt': + file_abs_path += ".txt" + + with open(file_abs_path, "w+") as new_file: + try: + new_file.write(''.join( + np.random.choice(list(string.printable), file_size))) + new_file.flush() + new_file.close() + except IOError as err: + print("Unable to write to file '%s' : %s" % ( + file_abs_path, err.strerror)) + rc = 1 + + elif file_type == 'docx': + file_abs_path += ".docx" + try: + document = Document() + str_to_write = list(string.ascii_letters + string.digits) + file_str = ''.join(np.random.choice(str_to_write, file_size)) + document.add_paragraph(file_str) + document.save(file_abs_path) + except Exception as err: + print("Unable to write to file '%s' : %s" % ( + file_abs_path, err.strerror)) + rc = 1 + + elif file_type == 'empty_file': + try: + with open(file_abs_path, "w+") as new_file: + new_file.close() + except IOError as err: + print("Unable to write to file '%s' : %s" % ( + file_abs_path, err.strerror)) + rc = 1 + + return rc + + def _create_files(dir_path, num_of_files, fixed_file_size=None, base_file_name='testfile', file_types='txt'): rc = 0 @@ -264,62 +311,38 @@ def _create_files(dir_path, num_of_files, fixed_file_size=None, '1k': 1024, '10k': 10240, '512k': 524288, - '1M': 1048576 - } + '1M': 1048576, + } # Create dir_path rc = create_dir(dir_path) if rc != 0: return rc - for count in range(num_of_files): - fname = base_file_name + str(count) - fname_abs_path = os.path.join(dir_path, fname) - if fixed_file_size is None: - file_size = ( - file_sizes_dict[random.choice(list(file_sizes_dict.keys()))]) - else: - try: - file_size = file_sizes_dict[fixed_file_size] - except KeyError as e: - print ("File sizes can be [1k, 10k, 512k, 1M]") - return 1 + fname_abs_path = os.path.join(dir_path, base_file_name) + if fixed_file_size is None: + # this generator yields file tuples: (file name, file type, file size) + files = ((fname_abs_path + str(num), + random.choice(file_types_list), + random.choice(list(file_sizes_dict.values()))) + for num in range(num_of_files)) + else: + try: + files = ((fname_abs_path + str(num), + random.choice(file_types_list), + file_sizes_dict[fixed_file_size]) + for num in range(num_of_files)) + except KeyError: + print("File sizes can be [1k, 10k, 512k, 1M]") + return 1 - type = random.choice(file_types_list) - if type == 'txt': - fname_abs_path = fname_abs_path + ".txt" + # Thread per filetype (for now) + pool = ThreadPool(len(file_types_list)) + ret = pool.map(lambda file_tuple: _create_file(*file_tuple), files) + pool.close() + pool.join() + rc = 1 if any(ret) else 0 - with open(fname_abs_path, "w+") as fd: - try: - fd.write(''.join(random.choice(string.printable) for x in - range(file_size))) - fd.flush() - fd.close() - except IOError as e: - print ("Unable to write to file '%s' : %s" % - (fname_abs_path, e.strerror)) - rc = 1 - elif type == 'docx': - fname_abs_path = fname_abs_path + ".docx" - try: - document = Document() - str_to_write = string.ascii_letters + string.digits - file_str = (''.join(random.choice(str_to_write) - for x in range(file_size))) - document.add_paragraph(file_str) - document.save(fname_abs_path) - except Exception as e: - print ("Unable to write to file '%s' : %s" % - (fname_abs_path, e.strerror)) - rc = 1 - elif type == 'empty_file': - try: - with open(fname_abs_path, "w+") as fd: - fd.close() - except IOError as e: - print ("Unable to write to file '%s' : %s" % - (fname_abs_path, e.strerror)) - rc = 1 return rc @@ -367,7 +390,7 @@ def rename(args): # Check if dir_path exists if not path_exists(dir_path): - print ("Directory '%s' does not exist" % dir_path) + print("Directory '%s' does not exist" % dir_path) return 1 rc = 0 @@ -381,7 +404,7 @@ def rename(args): os.rename(old, new) except OSError: rc = 1 - print ("Unable to rename %s -> %s" % (old, new)) + print("Unable to rename %s -> %s" % (old, new)) # rename dirs if dirName != dir_path: @@ -391,19 +414,18 @@ def rename(args): os.rename(old, new) except OSError: rc = 1 - print ("Unable to rename %s -> %s" % (old, new)) + print("Unable to rename %s -> %s" % (old, new)) return rc def ls(args): - """Recursively list all the files/dirs under 'dir' - """ + """Recursively list all the files/dirs under 'dir'.""" dir_path = os.path.abspath(args.dir) log_file_name = args.log_file_name # Check if dir_path exists if not path_exists(dir_path): - print ("Directory '%s' does not exist" % dir_path) + print("Directory '%s' does not exist" % dir_path) return 1 with open_file_to_write(log_file_name) as file_handle: @@ -423,12 +445,10 @@ def ls(args): def _get_path_stats(path): - """Get the stat of a specified path. - """ + """Get the stat of a specified path.""" rc = 0 path = os.path.abspath(args.path) file_stats = {} - file_stats = {} if platform.system() == "Linux": cmd = "stat -c " + "'%A %U %G' " + path @@ -455,8 +475,8 @@ def _get_path_stats(path): 'mtime': stat.st_mtime, 'ctime': stat.st_ctime, 'inode': stat.st_ino, - 'stat': stat - }) + 'stat': stat, + }) except Exception: rc = 1 err = "Unable to get the stat of path %s" % path @@ -465,41 +485,39 @@ def _get_path_stats(path): def get_path_stats(args): - """Get file/dir Stat - """ + """Get file/dir Stat.""" path = os.path.abspath(args.path) recursive = args.recursive log_file_name = args.log_file_name # Check if dir_path exists if not path_exists(path): - print ("PATH '%s' does not exist" % path) + print("PATH '%s' does not exist" % path) return 1 file_stats = {} if os.path.isfile(path): - file_stats[path] = (_get_path_stats(path)) + file_stats[path] = _get_path_stats(path) if os.path.isdir(path): if recursive: for dirName, subdirList, fileList in os.walk(path, topdown=False): - file_stats[dirName] = (_get_path_stats(dirName)) + file_stats[dirName] = _get_path_stats(dirName) for fname in fileList: fname_abs_path = os.path.join(dirName, fname) - file_stats[fname_abs_path] = (_get_path_stats( - fname_abs_path)) + file_stats[fname_abs_path] = _get_path_stats( + fname_abs_path) else: - file_stats[path] = (_get_path_stats(path)) + file_stats[path] = _get_path_stats(path) rc = 0 with open_file_to_write(log_file_name) as file_handle: if log_file_name: time_str = _get_current_time() - file_handle.write("Starting 'stat %s' : %s" % ( - path, time_str)) + file_handle.write("Starting 'stat %s' : %s" % (path, time_str)) for key in file_stats.keys(): file_handle.write("\nFile: %s" % key) ret, file_stat, err = file_stats[key] @@ -510,8 +528,7 @@ def get_path_stats(args): file_handle.write("\t%s\n" % file_stat) if log_file_name: time_str = _get_current_time() - file_handle.write("Ending 'stat %s' : %s" % ( - path, time_str)) + file_handle.write("Ending 'stat %s' : %s" % (path, time_str)) file_handle.write("\n") return rc @@ -531,7 +548,7 @@ def compress(args): # Check if dir_path exists if not path_exists(dir_path): - print ("Directory '%s' does not exist" % dir_path) + print("Directory '%s' does not exist" % dir_path) return 1 # Create dir_path @@ -546,16 +563,16 @@ def compress(args): proc_list = [] for each_dir in dirs: if compress_type == '7z': - file_name = (dest_dir + path_sep + - os.path.basename(each_dir) + "_7z.7z") + file_name = dest_dir + path_sep + os.path.basename( + each_dir) + "_7z.7z" cmd = "7z a -t7z " + file_name + " " + each_dir elif compress_type == 'gzip': - tmp_file_name = (dir_path + path_sep + - os.path.basename(each_dir) + "_tar.tar") - file_name = (dest_dir + path_sep + - os.path.basename(each_dir) + "_tgz.tgz") - cmd = ("7z a -ttar -so " + tmp_file_name + " " + - each_dir + " | 7z a -si " + file_name) + tmp_file_name = dir_path + path_sep + os.path.basename( + each_dir) + "_tar.tar" + file_name = dest_dir + path_sep + os.path.basename( + each_dir) + "_tgz.tgz" + cmd = ("7z a -ttar -so " + tmp_file_name + " " + + each_dir + " | 7z a -si " + file_name) proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) proc_list.append(proc) @@ -570,12 +587,12 @@ def compress(args): file_name = dest_dir + path_sep + os.path.basename(dir_path) + "_7z.7z" cmd = "7z a -t7z " + file_name + " " + dir_path elif compress_type == 'gzip': - tmp_file_name = (dest_dir + path_sep + os.path.basename(dir_path) + - "_tar.tar") - file_name = (dest_dir + path_sep + os.path.basename(dir_path) + - "_tgz.tgz") - cmd = ("7z a -ttar -so " + tmp_file_name + " " + dir_path + - " | 7z a -si " + file_name) + tmp_file_name = (dest_dir + path_sep + os.path.basename(dir_path) + + "_tar.tar") + file_name = dest_dir + path_sep + os.path.basename( + dir_path) + "_tgz.tgz" + cmd = ("7z a -ttar -so " + tmp_file_name + " " + dir_path + + " | 7z a -si " + file_name) proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) proc.communicate() @@ -587,13 +604,12 @@ def compress(args): def uncompress(args): - """UnCompress the given compressed file - """ + """UnCompress the given compressed file.""" compressed_file = os.path.abspath(args.compressed_file) dest_dir = args.dest_dir date_time = datetime.datetime.now().strftime("%I_%M%p_%B_%d_%Y") - cmd = ("7z x " + compressed_file + " -o" + dest_dir + path_sep + - "uncompress_" + date_time + " -y") + cmd = ("7z x " + compressed_file + " -o" + dest_dir + path_sep + + "uncompress_" + date_time + " -y") proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) proc.communicate() @@ -605,13 +621,12 @@ def uncompress(args): def uncompress_dir(args): - """UnCompress all compressed files in destination directory - """ + """UnCompress all compressed files in destination directory.""" dir_path = os.path.abspath(args.dir) dest_dir = args.dest_dir date_time = datetime.datetime.now().strftime("%I_%M%p_%B_%d_%Y") - cmd = ("7z x " + dir_path + " -o" + dest_dir + path_sep + - "uncompress_" + date_time + " -y") + cmd = ("7z x " + dir_path + " -o" + dest_dir + path_sep + + "uncompress_" + date_time + " -y") proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) proc.communicate() @@ -623,7 +638,7 @@ def uncompress_dir(args): def create_hard_links(args): - """Creates hard link""" + """Create hard link.""" src_dir = os.path.abspath(args.src_dir) dest_dir = args.dest_dir @@ -633,7 +648,7 @@ def create_hard_links(args): # Check if src_dir exists if not path_exists(src_dir): - print ("Directory '%s' does not exist" % src_dir) + print("Directory '%s' does not exist" % src_dir) return 1 # Create dir_path @@ -650,8 +665,8 @@ def create_hard_links(args): rc = create_dir(dest_dir + path_sep + tmp_dir) if rc != 0: rc = 1 - link_file = (dest_dir + path_sep + tmp_dir + path_sep + - new_fname + "_h") + link_file = (dest_dir + path_sep + tmp_dir + path_sep + + new_fname + "_h") target_file = os.path.join(dir_name, fname) if platform.system() == "Windows": cmd = "mklink /H " + link_file + " " + target_file @@ -702,9 +717,7 @@ def read(args): def copy(args): - """ - Copies files/dirs under 'dir' to destination directory - """ + """Copy files/dirs under 'dir' to destination directory.""" src_dir = os.path.abspath(args.src_dir) dest_dir = args.dest_dir @@ -714,7 +727,7 @@ def copy(args): # Check if src_dir exists if not path_exists(src_dir): - print ("Directory '%s' does not exist" % src_dir) + print("Directory '%s' does not exist" % src_dir) return 1 # Create dest_dir @@ -735,8 +748,8 @@ def copy(args): if dir_name != src_dir: try: src = dir_name - dst = (dest_dir + path_sep + - os.path.basename(os.path.normpath(src))) + dst = (dest_dir + path_sep + + os.path.basename(os.path.normpath(src))) shutil.copytree(src, dst) except OSError: rc = 1 @@ -744,9 +757,7 @@ def copy(args): def delete(args): - """ - Deletes files/dirs under 'dir' - """ + """Delete files/dirs under 'dir'.""" dir_path = os.path.abspath(args.dir) # Check if dir_path is '/' @@ -755,7 +766,7 @@ def delete(args): # Check if dir_path exists if not path_exists(dir_path): - print ("Directory '%s' does not exist" % dir_path) + print("Directory '%s' does not exist" % dir_path) return 1 rc = 0 @@ -774,8 +785,137 @@ def delete(args): return rc +sizes_dict = { + '1k': 1024, + '10k': 10240, + '512k': 524288, + '1M': 1048576, + '0.5k': 513 +} + + +def append(args): + """ + Appends all files under 'dir' with randomly sized data. + """ + dir_path = os.path.abspath(args.dir) + if not path_exists(args.dir): + return 1 + rc = 0 + + for dir_name, subdir_list, file_list in os.walk(dir_path, topdown=False): + for fname in file_list: + append_size = sizes_dict[ + random.choice(list(sizes_dict.keys()))] + try: + file = os.path.join(dir_name, fname) + with open(file, "a") as fd: + try: + fd.write(''.join(random.choice(string.printable) + for x in range(append_size))) + fd.flush() + except IOError as e: + print("Unable to append to file '%s' : %s" % + (file, e.strerror)) + rc = 1 + + except OSError: + rc = 1 + + return rc + + +def overwrite(args): + """ + Truncates everything present and overwrites the file with new data. + """ + dir_path = os.path.abspath(args.dir) + if not path_exists(args.dir): + return 1 + rc = 0 + + for dir_name, subdir_list, file_list in os.walk(dir_path, topdown=False): + for fname in file_list: + new_size = sizes_dict[ + random.choice(list(sizes_dict.keys()))] + try: + file = os.path.join(dir_name, fname) + with open(file, "w+") as fd: + try: + fd.write(''.join(random.choice(string.printable) + for x in range(new_size))) + fd.flush() + except IOError as e: + print("Unable to write to file '%s' : %s" % + (file, e.strerror)) + rc = 1 + except OSError: + rc = 1 + return rc + + +def truncate(args): + """ + Truncates files to a certain size calculated randomly. + """ + dir_path = os.path.abspath(args.dir) + if not path_exists(args.dir): + return 1 + rc = 0 + + for dir_name, subdir_list, file_list in os.walk(dir_path, topdown=False): + for fname in file_list: + try: + file = os.path.join(dir_name, fname) + with open(file, "a+") as fd: + try: + fsize = os.path.getsize(file) + new_size = random.randrange( + 0, fsize//random.choice([2, 3, 4, 5])) + fd.truncate(new_size) + + except IOError as e: + print("Unable to truncate file '%s' : %s" % + (file, e.strerror)) + rc = 1 + except OSError: + rc = 1 + return rc + + +def rsync(args): + """ + rsync files from source to destination. + """ + src_dir = os.path.abspath(args.src_dir) + remote_dir = args.remote_dir + + if platform.system() == "Windows": + print("rsync not supported on Windows,Exiting!") + return 1 + + # Check if src_dir exists + if not path_exists(src_dir): + print("Directory '%s' does not exist" % src_dir) + return 1 + + # Create dest_dir + rc = create_dir(remote_dir) + if rc != 0: + return rc + rc = 0 + + try: + sh_rsync("-r", remote_dir, src_dir) + + except Exception as e: + print("Can't rsync! : %s" % e.strerror) + rc = 1 + return rc + + if __name__ == "__main__": - print ("Starting File/Dir Ops: %s" % _get_current_time()) + print("Starting File/Dir Ops: %s" % _get_current_time()) test_start_time = datetime.datetime.now().replace(microsecond=0) parser = argparse.ArgumentParser( @@ -1019,7 +1159,66 @@ if __name__ == "__main__": help="Directory on which operations has to be performed") read_parser.set_defaults(func=read) - # copy all files/directories under dir + # Appends files under dir + append_parser = subparsers.add_parser( + 'append', + help=("Appends data to already created files. "), + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + append_parser.add_argument( + '--log-file', help="Output log filename to log the " + "contents of file", + metavar=('log_file'), dest='log_file', + type=str, default=default_log_file) + append_parser.add_argument( + 'dir', metavar='DIR', type=str, + help="Directory on which operations has to be performed") + append_parser.set_defaults(func=append) + + # Overwrites files under dir + overwrite_parser = subparsers.add_parser( + 'overwrite', + help=("Overwrites existing files with new data "), + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + overwrite_parser.add_argument( + '--log-file', help="Output log filename to log the " + "contents of file", + metavar=('log_file'), dest='log_file', + type=str, default=default_log_file) + overwrite_parser.add_argument( + 'dir', metavar='DIR', type=str, + help="Directory on which operations has to be performed") + overwrite_parser.set_defaults(func=overwrite) + + # rsync dir to a remote directory + rsyncs_parser = subparsers.add_parser( + 'rsync', + help=("Rsync all dirs in a remote location to 'dir'. "), + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + rsyncs_parser.add_argument( + '--remote-dir', help="Remote location to rsync from)", + metavar=('remote_dir'), dest='remote_dir', + type=str) + rsyncs_parser.add_argument( + 'src_dir', metavar='src_dir', type=str, + help="Directory on which operations has to be performed") + rsyncs_parser.set_defaults(func=rsync) + + # Truncates files under dir + truncate_parser = subparsers.add_parser( + 'truncate', + help=("Truncates existing files "), + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + truncate_parser.add_argument( + '--log-file', help="Output log filename to log the " + "contents of file", + metavar=('log_file'), dest='log_file', + type=str, default=default_log_file) + truncate_parser.add_argument( + 'dir', metavar='DIR', type=str, + help="Directory on which operations has to be performed") + truncate_parser.set_defaults(func=truncate) + + # Copy all files/directories under dir copy_parser = subparsers.add_parser( 'copy', help=("Copy all files/directories under 'dir'. "), @@ -1047,6 +1246,6 @@ if __name__ == "__main__": rc = args.func(args) test_end_time = datetime.datetime.now().replace(microsecond=0) - print ("Execution time: %s" % (test_end_time - test_start_time)) - print ("Ending File/Dir Ops %s" % _get_current_time()) + print("Execution time: %s" % (test_end_time - test_start_time)) + print("Ending File/Dir Ops %s" % _get_current_time()) sys.exit(rc) |