"""testcases for replicate/self-heal """ import re import os import sys import time import afrutils import atfutils import clientutils import glusterutils import hostutils import serverutils import validate from atfglobals import GlobalObj from atfutils import commands urandom = "/dev/urandom" write_op = ">" append_op = ">>" mounts = ['mount1'] bricks = ['brick1', 'brick2'] brick_reboot_time = 15 def reset_testenv(): """ Description: * Unmount all mounts specified in the testenv file. * Stop the Volume (Active Volume) * Delete the Volume * Stop Glusterd on all servers * Remove the glusterd dir (/etc/glusterd) * Remove all glusterd logs from all servers """ output = clientutils.umountall() assert_success_status = atfutils.assert_success(output["exitstatus"]) if assert_success_status is not 0: return assert_success_status glusterutils.volume_stop("server1", force=True) glusterutils.volume_delete("server1") glusterutils.glusterd_stop_allservers() glusterutils.glusterd_remove_dir_allservers() glusterutils.glusterd_remove_logs_allservers() return 0 def setup_testenv(): """ Description: * Start glusterd on all servers * Peer Probe all servers in the storage pool * Create Bricks on the servers * Create Volume * Start Volume * Mount to the Volume from all mounts specified in testenv """ output = glusterutils.glusterd_start_allservers(force=True) assert_success_status = atfutils.assert_success(output["exitstatus"]) if assert_success_status is not 0: return assert_success_status output = glusterutils.create_brick_allservers() assert_success_status = atfutils.assert_success(output["exitstatus"]) if assert_success_status is not 0: return assert_success_status output = glusterutils.peer_probe("server1") assert_success_status = atfutils.assert_success(output["exitstatus"]) if assert_success_status is not 0: return assert_success_status output = glusterutils.volume_create("server1") assert_success_status = atfutils.assert_success(output["exitstatus"]) if assert_success_status is not 0: return assert_success_status output = glusterutils.volume_set("server1", key="self-heal-daemon", value="off") assert_success_status = atfutils.assert_success(output["exitstatus"]) if assert_success_status is not 0: return assert_success_status output = glusterutils.volume_set("server1", key="diagnostics.client-log-level", value="DEBUG") output = glusterutils.volume_set("server1", key="diagnostics.brick-log-level", value="DEBUG") output = glusterutils.volume_start("server1") assert_success_status = atfutils.assert_success(output["exitstatus"]) if assert_success_status is not 0: return assert_success_status output = clientutils.mountall() assert_success_status = atfutils.assert_success(output["exitstatus"]) if assert_success_status is not 0: return assert_success_status return 0 def test001(): """ Description: Testing if the source is selected based on entry transaction for directory """ return_status = reset_testenv() if return_status is not 0: return return_status return_status = setup_testenv() if return_status is not 0: return return_status dirname = "test1/d1" command = ' '.join([commands['mkdir'], dirname]) expected_output = "" output = clientutils.execute_on_mount("mount1", command) validation_status = atfutils.validate_output(output, 0, expected_output) if validation_status is not 0: return validation_status output = afrutils.set_read_subvolume("server1", "client-0") assert_success_status = atfutils.assert_success(output["exitstatus"]) if assert_success_status is not 0: return assert_success_status output = afrutils.disable_self_heal("server1", 'data', 'entry', 'metadata') assert_success_status = atfutils.assert_success(output["exitstatus"]) if assert_success_status is not 0: return assert_success_status output = glusterutils.volume_stop_brick("brick2") assert_success_status = atfutils.assert_success(output["exitstatus"]) if assert_success_status is not 0: return assert_success_status filename = "f1" abs_path = os.path.join(dirname, filename) command = ' '.join([commands['touch'], abs_path]) output = clientutils.execute_on_mount("mount1", command) expected_output = "" validation_status = atfutils.validate_output(output, 0, expected_output) if validation_status is not 0: return validation_status output = glusterutils.volume_start("server1", force=True) assert_success_status = atfutils.assert_success(output["exitstatus"]) if assert_success_status is not 0: return assert_success_status command = ' '.join([commands['ls'], abs_path]) output = clientutils.execute_on_mount("mount1", command) expected_output = abs_path validation_status = atfutils.validate_output(output, 0, expected_output) if validation_status is not 0: return validation_status return 0 def test002(): """ Description: Test if the source is selected based on data transaction for reg file """ filename = "file1" input_file = urandom output_file = filename return_status = reset_testenv() if return_status is not 0: return return_status return_status = setup_testenv() if return_status is not 0: return return_status output = afrutils.set_read_subvolume("server1", "client-0") assert_success_status = atfutils.assert_success(output["exitstatus"]) if assert_success_status is not 0: return assert_success_status output = afrutils.disable_self_heal("server1", 'data', 'entry', 'metadata') assert_success_status = atfutils.assert_success(output["exitstatus"]) if assert_success_status is not 0: return assert_success_status command = ' '.join([commands['dd'], "if="+input_file, "of="+output_file, "bs=1M", "count=1"]) output = clientutils.execute_on_mount("mount1", command) expected_output = "1048576" validation_status = atfutils.validate_output(output, 0, expected_output, stream="stderr") if validation_status is not 0: return validation_status output = glusterutils.volume_stop_brick("brick2") assert_success_status = atfutils.assert_success(output["exitstatus"]) if assert_success_status is not 0: return assert_success_status command = ' '.join([commands['dd'], "if="+input_file, "of="+output_file, "bs=1M", "count=10"]) output = clientutils.execute_on_mount("mount1", command) expected_output = "10485760" validation_status = atfutils.validate_output(output, 0, expected_output, stream="stderr") if validation_status is not 0: return validation_status output = glusterutils.volume_start("server1", force=True) assert_success_status = atfutils.assert_success(output["exitstatus"]) if assert_success_status is not 0: return assert_success_status command = ' '.join([commands['ls'], "-l", filename ]) output = clientutils.execute_on_mount("mount1", command) expected_output = "10485760" validation_status = atfutils.validate_output(output, 0, expected_output) if validation_status is not 0: return validation_status return 0 def test003(): """ Description: Test if the source is selected based on metadata transaction for linkfile """ filename1 = "testsymlink.txt" filename2 = "symlink_to_testsymlink.txt" input_file = urandom return_status = reset_testenv() if return_status is not 0: return return_status return_status = setup_testenv() if return_status is not 0: return return_status output = afrutils.set_read_subvolume("server1", "client-0") assert_success_status = atfutils.assert_success(output["exitstatus"]) if assert_success_status is not 0: return assert_success_status output = afrutils.disable_self_heal("server1", 'data', 'entry', 'metadata') assert_success_status = atfutils.assert_success(output["exitstatus"]) if assert_success_status is not 0: return assert_success_status """ Create a file """ command = ' '.join([commands['touch'], filename1]) output = clientutils.execute_on_mount("mount1", command) expected_output = "" validation_status = atfutils.validate_output(output, 0, expected_output) if validation_status is not 0: return validation_status """ Create a symlink to the file created above """ command = ' '.join([commands['symlink'], filename1, filename2]) output = clientutils.execute_on_mount("mount1", command) expected_output = "" validation_status = atfutils.validate_output(output, 0, expected_output) if validation_status is not 0: return validation_status """ Stop brick2 """ output = glusterutils.volume_stop_brick("brick2") assert_success_status = atfutils.assert_success(output["exitstatus"]) if assert_success_status is not 0: return assert_success_status """ execute dd on the symlink provide rwx permissions to all """ output_file = filename2 command = ' '.join([commands['dd'], "if="+input_file, "of="+output_file, "bs=1M", "count=10"]) output = clientutils.execute_on_mount("mount1", command) expected_output = "10485760" validation_status = atfutils.validate_output(output, 0, expected_output, stream="stderr") if validation_status is not 0: return validation_status command = ' '.join([commands['chmod'], "777", filename2]) output = clientutils.execute_on_mount("mount1", command) expected_output = "" validation_status = atfutils.validate_output(output, 0, expected_output) if validation_status is not 0: return validation_status """ Restart the brick2 """ output = glusterutils.volume_start("server1", force=True) assert_success_status = atfutils.assert_success(output["exitstatus"]) if assert_success_status is not 0: return assert_success_status command = ' '.join([commands['ls'], "-l", filename1]) output = clientutils.execute_on_mount("mount1", command) expected_output = "10485760" validation_status = atfutils.validate_output(output, 0, expected_output) if validation_status is not 0: return validation_status command = ' '.join([commands['ls'], "-l", filename1]) output = clientutils.execute_on_mount("mount1", command) expected_output = "-rwxrwxrwx" validation_status = atfutils.validate_output(output, 0, expected_output) if validation_status is not 0: return validation_status return 0 def test004(): """ Description: Self-Heal: Enoent + Success """ input_file = urandom dir1 = "d1" dir2 = "d2" file1 = "f1" file2 = "f2" testdir1 = "testdir1" testdir2 = "testdir2" t1_d1 = os.path.join(testdir1, dir1) t1_d2 = os.path.join(testdir1, dir2) t2_d1 = os.path.join(testdir2, dir1) t2_d2 = os.path.join(testdir2, dir2) return_status = reset_testenv() if return_status is not 0: return return_status return_status = setup_testenv() if return_status is not 0: return return_status command = ' '.join([commands['mkdir'], t1_d1, t1_d2]) output = clientutils.execute_on_mount("mount1", command) expected_output = "" validation_status = atfutils.validate_output(output, 0, expected_output) if validation_status is not 0: return validation_status t1_f1 = os.path.join(testdir1, file1) output_file = t1_f1 command = ' '.join([commands['dd'], "if="+input_file, "of="+output_file, "bs=1M", "count=1"]) output = clientutils.execute_on_mount("mount1", command) expected_output = "1048576" validation_status = atfutils.validate_output(output, 0, expected_output, stream="stderr") if validation_status is not 0: return validation_status t1_d1_f1= os.path.join(t1_d1, file1) output_file = t1_d1_f1 command = ' '.join([commands['dd'], "if="+input_file, "of="+output_file, "bs=1M", "count=1"]) output = clientutils.execute_on_mount("mount1", command) expected_output = "1048576" validation_status = atfutils.validate_output(output, 0, expected_output, stream="stderr") if validation_status is not 0: return validation_status t1_d1_f2 = os.path.join(t1_d1, file2) output_file = t1_d1_f2 command = ' '.join([commands['dd'], "if="+input_file, "of="+output_file, "bs=1M", "count=2"]) output = clientutils.execute_on_mount("mount1", command) expected_output = "2097152" validation_status = atfutils.validate_output(output, 0, expected_output, stream="stderr") if validation_status is not 0: return validation_status t1_d2_f1 = os.path.join(t1_d2, file1) output_file = t1_d2_f1 command = ' '.join([commands['dd'], "if="+input_file, "of="+output_file, "bs=1M", "count=1"]) output = clientutils.execute_on_mount("mount1", command) expected_output = "1048576" validation_status = atfutils.validate_output(output, 0, expected_output, stream="stderr") if validation_status is not 0: return validation_status t1_d2_f2 = os.path.join(t1_d2, file2) output_file = t1_d2_f2 command = ' '.join([commands['dd'], "if="+input_file, "of="+output_file, "bs=1M", "count=2"]) output = clientutils.execute_on_mount("mount1", command) expected_output = "2097152" validation_status = atfutils.validate_output(output, 0, expected_output, stream="stderr") if validation_status is not 0: return validation_status """ Stop brick2 """ output = glusterutils.volume_stop_brick("brick2") assert_success_status = atfutils.assert_success(output["exitstatus"]) if assert_success_status is not 0: return assert_success_status command = ' '.join([commands['unlink_dir_force'], t1_d2]) output = clientutils.execute_on_mount("mount1", command) expected_output = "" validation_status = atfutils.validate_output(output, 0, expected_output) if validation_status is not 0: return validation_status command = ' '.join([commands['truncate'], "-s 0", t1_d1_f1]) output = clientutils.execute_on_mount("mount1", command) expected_output = "" validation_status = atfutils.validate_output(output, 0, expected_output) if validation_status is not 0: return validation_status command = ' '.join([commands['chmod'], "777", t1_d1_f2]) output = clientutils.execute_on_mount("mount1", command) expected_output = "" validation_status = atfutils.validate_output(output, 0, expected_output) if validation_status is not 0: return validation_status command = ' '.join([commands['mkdir'], t2_d1, t2_d2]) output = clientutils.execute_on_mount("mount1", command) expected_output = "" validation_status = atfutils.validate_output(output, 0, expected_output) if validation_status is not 0: return validation_status t2_f1 = os.path.join(testdir2, file1) output_file = t2_f1 command = ' '.join([commands['dd'], "if="+input_file, "of="+output_file, "bs=1M", "count=1"]) output = clientutils.execute_on_mount("mount1", command) expected_output = "1048576" validation_status = atfutils.validate_output(output, 0, expected_output, stream="stderr") if validation_status is not 0: return validation_status t2_d1_f1= os.path.join(t2_d1, file1) output_file = t2_d1_f1 command = ' '.join([commands['dd'], "if="+input_file, "of="+output_file, "bs=1M", "count=1"]) output = clientutils.execute_on_mount("mount1", command) expected_output = "1048576" validation_status = atfutils.validate_output(output, 0, expected_output, stream="stderr") if validation_status is not 0: return validation_status t2_d1_f2 = os.path.join(t2_d1, file2) output_file = t2_d1_f2 command = ' '.join([commands['dd'], "if="+input_file, "of="+output_file, "bs=1M", "count=2"]) output = clientutils.execute_on_mount("mount1", command) expected_output = "2097152" validation_status = atfutils.validate_output(output, 0, expected_output, stream="stderr") if validation_status is not 0: return validation_status t2_d2_f1 = os.path.join(t2_d2, file1) output_file = t2_d2_f1 command = ' '.join([commands['dd'], "if="+input_file, "of="+output_file, "bs=1M", "count=1"]) output = clientutils.execute_on_mount("mount1", command) expected_output = "1048576" validation_status = atfutils.validate_output(output, 0, expected_output, stream="stderr") if validation_status is not 0: return validation_status t2_d2_f2 = os.path.join(t2_d2, file2) output_file = t2_d2_f2 command = ' '.join([commands['dd'], "if="+input_file, "of="+output_file, "bs=1M", "count=2"]) output = clientutils.execute_on_mount("mount1", command) expected_output = "2097152" validation_status = atfutils.validate_output(output, 0, expected_output, stream="stderr") if validation_status is not 0: return validation_status command = ' '.join([commands['chmod'], "777", t2_d1_f2]) output = clientutils.execute_on_mount("mount1", command) expected_output = "" validation_status = atfutils.validate_output(output, 0, expected_output, stream="stderr") if validation_status is not 0: return validation_status """ Restart the brick2 """ output = glusterutils.volume_start("server1", force=True) assert_success_status = atfutils.assert_success(output["exitstatus"]) if assert_success_status is not 0: return assert_success_status brick_reboot_status = afrutils.wait_till_brick_reboot(brick_reboot_time) if brick_reboot_status is not 0: return brick_reboot_status command = "find . | xargs stat" output = clientutils.execute_on_mount("mount1", command) expected_output = "" validation_status = atfutils.validate_output(output, 0, expected_output) if validation_status is not 0: return validation_status selfheal_completion_status = afrutils.wait_till_selfheal_completes(30) if selfheal_completion_status is not 0: return selfheal_completion_status validation_status = validate.validate_md5sums(mounts, bricks) if validation_status is not 0: return validation_status return 0 def test005(): """ Description: Self-heal: Success + File type differs """ data = "Hello World. TestCase : Self-heal - Success + File type differs " filename = dirname = "test" return_status = reset_testenv() if return_status is not 0: return return_status return_status = setup_testenv() if return_status is not 0: return return_status command = ' '.join([commands['echo'], data, write_op, filename]) output = clientutils.execute_on_mount("mount1", command) expected_output = "" validation_status = atfutils.validate_output(output, 0, expected_output) if validation_status is not 0: return validation_status command = ' '.join([commands['stat'], filename]) output = clientutils.execute_on_mount("mount1", command) expected_output = "regular file" validation_status = atfutils.validate_output(output, 0, expected_output) if validation_status is not 0: return validation_status validation_status = validate.validate_on_bricks(bricks, command, 0, expected_output) if validation_status is not 0: return validation_status """ Stop Brick2 """ output = glusterutils.volume_stop_brick("brick2") assert_success_status = atfutils.assert_success(output["exitstatus"]) if assert_success_status is not 0: return assert_success_status command = ' '.join([commands['unlink_force'], filename]) output = clientutils.execute_on_mount("mount1", command) expected_output = "" validation_status = atfutils.validate_output(output, 0, expected_output) if validation_status is not 0: return validation_status command = ' '.join([commands['mkdir'], dirname]) output = clientutils.execute_on_mount("mount1", command) expected_output = "" validation_status = atfutils.validate_output(output, 0, expected_output) if validation_status is not 0: return validation_status """ Force Start the Volume. This will restart the brick2 """ output = glusterutils.volume_start("server1", force=True) assert_success_status = atfutils.assert_success(output["exitstatus"]) if assert_success_status is not 0: return assert_success_status brick_reboot_status = afrutils.wait_till_brick_reboot(brick_reboot_time) if brick_reboot_status is not 0: return brick_reboot_status command = ' '.join([commands['stat'], dirname]) output = clientutils.execute_on_mount("mount1", command) expected_output = "directory" validation_status = atfutils.validate_output(output, 0, expected_output) if validation_status is not 0: return validation_status validation_status = validate.validate_on_bricks(bricks, command, 0, expected_output) if validation_status is not 0: return validation_status validation_status = validate.validate_md5sums(mounts, bricks) if validation_status is not 0: return validation_status return 0 def test006(): """ Description: Self-heal: Success + Permission differs """ data = "Hello World. TestCase: Self-heal - Success + Permission differs" filename = "test" return_status = reset_testenv() if return_status is not 0: return return_status return_status = setup_testenv() if return_status is not 0: return return_status command = ' '.join([commands['echo'], data, write_op, filename]) output = clientutils.execute_on_mount("mount1", command) expected_output = "" validation_status = atfutils.validate_output(output, 0, expected_output) if validation_status is not 0: return validation_status command = ' '.join([commands['chmod'], "666", filename]) output = clientutils.execute_on_mount("mount1", command) expected_output = "" validation_status = atfutils.validate_output(output, 0, expected_output) if validation_status is not 0: return validation_status output = glusterutils.volume_stop_brick("brick2") assert_success_status = atfutils.assert_success(output["exitstatus"]) if assert_success_status is not 0: return assert_success_status command = ' '.join([commands['chmod'], "777", filename]) output = clientutils.execute_on_mount("mount1", command) expected_output = "" validation_status = atfutils.validate_output(output, 0, expected_output) if validation_status is not 0: return validation_status output = glusterutils.volume_start("server1", force=True) assert_success_status = atfutils.assert_success(output["exitstatus"]) if assert_success_status is not 0: return assert_success_status brick_reboot_status = afrutils.wait_till_brick_reboot(brick_reboot_time) if brick_reboot_status is not 0: return brick_reboot_status command = ' '.join([commands['ls'], "-l", filename]) output = clientutils.execute_on_mount("mount1", command) expected_output = "-rwxrwxrwx" validation_status = atfutils.validate_output(output, 0, expected_output) if validation_status is not 0: return validation_status validation_status = validate.validate_on_bricks(bricks, command, 0, expected_output) if validation_status is not 0: return validation_status return 0 def test007(): """ Description: Self-heal: Success + Ownership differs """ user = "qa" data = "Hello World. testcase: Self-heal - Success Ownership differs" filename = "test" return_status = reset_testenv() if return_status is not 0: return return_status return_status = setup_testenv() if return_status is not 0: return return_status command = ' '.join([commands['echo'], data, write_op, filename]) output = clientutils.execute_on_mount("mount1", command) expected_output = "" validation_status = atfutils.validate_output(output, 0, expected_output) if validation_status is not 0: return validation_status output = glusterutils.volume_stop_brick("brick2") assert_success_status = atfutils.assert_success(output["exitstatus"]) if assert_success_status is not 0: return assert_success_status user_group = ':'.join([user, user]) command = ' '.join([commands['chown'], user_group, filename]) output = clientutils.execute_on_mount("mount1", command) expected_output = "" validation_status = atfutils.validate_output(output, 0, expected_output) if validation_status is not 0: return validation_status output = glusterutils.volume_start("server1", force=True) assert_success_status = atfutils.assert_success(output["exitstatus"]) if assert_success_status is not 0: return assert_success_status brick_reboot_status = afrutils.wait_till_brick_reboot(brick_reboot_time) if brick_reboot_status is not 0: return brick_reboot_status command = ' '.join([commands['ls'], "-l", filename]) output = clientutils.execute_on_mount("mount1", command) expected_output = ' '.join([user, user]) validation_status = atfutils.validate_output(output, 0, expected_output) if validation_status is not 0: return validation_status validation_status = validate.validate_on_bricks(bricks, command, 0, expected_output) if validation_status is not 0: return validation_status return 0 def test008(): """ Description: Self-Heal: Success + Size Differs """ data1 = "Hello World. TestCase: Self-Heal - Success + Size Differs" data2 = "Appending data to file." filename = "test" return_status = reset_testenv() if return_status is not 0: return return_status return_status = setup_testenv() if return_status is not 0: return return_status command = ' '.join([commands['echo'], data1, write_op, filename]) output = clientutils.execute_on_mount("mount1", command) expected_output = "" validation_status = atfutils.validate_output(output, 0, expected_output) if validation_status is not 0: return validation_status output = glusterutils.volume_stop_brick("brick2") assert_success_status = atfutils.assert_success(output["exitstatus"]) if assert_success_status is not 0: return assert_success_status command = ' '.join([commands['echo'], data2, append_op, filename]) output = clientutils.execute_on_mount("mount1", command) expected_output = "" validation_status = atfutils.validate_output(output, 0, expected_output) if validation_status is not 0: return validation_status output = glusterutils.volume_start("server1", force=True) assert_success_status = atfutils.assert_success(output["exitstatus"]) if assert_success_status is not 0: return assert_success_status brick_reboot_status = afrutils.wait_till_brick_reboot(brick_reboot_time) if brick_reboot_status is not 0: return brick_reboot_status command = "find . | xargs stat" output = clientutils.execute_on_mount("mount1", command) expected_output = "" validation_status = atfutils.validate_output(output, 0, expected_output) if validation_status is not 0: return validation_status validation_status = validate.validate_md5sums(mounts, bricks) if validation_status is not 0: return validation_status output = glusterutils.volume_stop_brick("brick2") assert_success_status = atfutils.assert_success(output["exitstatus"]) if assert_success_status is not 0: return assert_success_status command = ' '.join([commands['truncate'], "-s 0", filename]) output = clientutils.execute_on_mount("mount1", command) expected_output = "" validation_status = atfutils.validate_output(output, 0, expected_output) if validation_status is not 0: return validation_status output = glusterutils.volume_start("server1", force=True) assert_success_status = atfutils.assert_success(output["exitstatus"]) if assert_success_status is not 0: return assert_success_status brick_reboot_status = afrutils.wait_till_brick_reboot(brick_reboot_time) if brick_reboot_status is not 0: return brick_reboot_status command = ' '.join([commands['stat'], filename]) output = clientutils.execute_on_mount("mount1", command) expected_output = "regular empty file" validation_status = atfutils.validate_output(output, 0, expected_output) if validation_status is not 0: return validation_status validation_status = validate.validate_on_bricks(bricks, command, 0, expected_output) if validation_status is not 0: return validation_status validation_status = validate.validate_md5sums(mounts, bricks) if validation_status is not 0: return validation_status return 0 def test009(): """ Description: Self-Heal: Success + gfid differs """ data = "Hello World. TestCase : Self-heal - Success + GFID differs" filename = "test" file_gfids = {} return_status = reset_testenv() if return_status is not 0: return return_status return_status = setup_testenv() if return_status is not 0: return return_status command = ' '.join([commands['echo'], data, write_op, filename]) output = clientutils.execute_on_mount("mount1", command) expected_output = "" validation_status = atfutils.validate_output(output, 0, expected_output) if validation_status is not 0: return validation_status output = glusterutils.volume_stop_brick("brick2") assert_success_status = atfutils.assert_success(output["exitstatus"]) if assert_success_status is not 0: return assert_success_status command = ' '.join([commands['unlink_force'], filename]) output = clientutils.execute_on_mount("mount1", command) expected_output = "" validation_status = atfutils.validate_output(output, 0, expected_output) if validation_status is not 0: return validation_status command = ' '.join([commands['echo'], data, write_op, filename]) output = clientutils.execute_on_mount("mount1", command) expected_output = "" validation_status = atfutils.validate_output(output, 0, expected_output) if validation_status is not 0: return validation_status output = glusterutils.volume_start("server1", force=True) assert_success_status = atfutils.assert_success(output["exitstatus"]) if assert_success_status is not 0: return assert_success_status brick_reboot_status = afrutils.wait_till_brick_reboot(brick_reboot_time) if brick_reboot_status is not 0: return brick_reboot_status command = ' '.join([commands['cat'], filename]) output = clientutils.execute_on_mount("mount1", command) expected_output = "" validation_status = atfutils.validate_output(output, 0, expected_output) if validation_status is not 0: return validation_status validation_status = validate.validate_gfids(bricks, filename) if validation_status is not 0: return validation_status validation_status = validate.validate_md5sums(mounts, bricks) if validation_status is not 0: return validation_status return 0 def test010a(): """ Description: Self-Heal: xattr data pending """ data1 = "Hello World. TestCase: Self-Heal - Success + Size Differs" data2 = data1 * 50 filename = "test" return_status = reset_testenv() if return_status is not 0: return return_status return_status = setup_testenv() if return_status is not 0: return return_status command = ' '.join([commands['echo'], data1, write_op, filename]) output = clientutils.execute_on_mount("mount1", command) expected_output = "" validation_status = atfutils.validate_output(output, 0, expected_output) if validation_status is not 0: return validation_status output = glusterutils.volume_stop_brick("brick2") assert_success_status = atfutils.assert_success(output["exitstatus"]) if assert_success_status is not 0: return assert_success_status command = ' '.join([commands['echo'], data2, append_op, filename]) output = clientutils.execute_on_mount("mount1", command) expected_output = "" validation_status = atfutils.validate_output(output, 0, expected_output) if validation_status is not 0: return validation_status output = glusterutils.volume_start("server1", force=True) assert_success_status = atfutils.assert_success(output["exitstatus"]) if assert_success_status is not 0: return assert_success_status brick_reboot_status = afrutils.wait_till_brick_reboot(brick_reboot_time) if brick_reboot_status is not 0: return brick_reboot_status command = "find . | xargs stat" output = clientutils.execute_on_mount("mount1", command) expected_output = "" validation_status = atfutils.validate_output(output, 0, expected_output) if validation_status is not 0: return validation_status validation_status = validate.validate_md5sums(mounts, bricks) if validation_status is not 0: return validation_status return 0 def test010b(): """ Description: Self-heal: xattr metadata pending """ user1 = "qa" user2 = "root" data = "Hello World. testcase: Self-heal - Success Ownership differs" filename = "test" return_status = reset_testenv() if return_status is not 0: return return_status return_status = setup_testenv() if return_status is not 0: return return_status command = ' '.join([commands['echo'], data, write_op, filename]) output = clientutils.execute_on_mount("mount1", command) expected_output = "" validation_status = atfutils.validate_output(output, 0, expected_output) if validation_status is not 0: return validation_status command = ' '.join([commands['chown'], user1, filename]) output = clientutils.execute_on_mount("mount1", command) expected_output = "" validation_status = atfutils.validate_output(output, 0, expected_output) if validation_status is not 0: return validation_status output = glusterutils.volume_stop_brick("brick2") assert_success_status = atfutils.assert_success(output["exitstatus"]) if assert_success_status is not 0: return assert_success_status command = ' '.join([commands['chown'], user2, filename]) output = clientutils.execute_on_mount("mount1", command) expected_output = "" validation_status = atfutils.validate_output(output, 0, expected_output) if validation_status is not 0: return validation_status command = ' '.join([commands['chmod'], "777", filename]) output = clientutils.execute_on_mount("mount1", command) expected_output = "" validation_status = atfutils.validate_output(output, 0, expected_output) if validation_status is not 0: return validation_status output = glusterutils.volume_start("server1", force=True) assert_success_status = atfutils.assert_success(output["exitstatus"]) if assert_success_status is not 0: return assert_success_status brick_reboot_status = afrutils.wait_till_brick_reboot(brick_reboot_time) if brick_reboot_status is not 0: return brick_reboot_status command = ' '.join([commands['ls'], "-l", filename]) output = clientutils.execute_on_mount("mount1", command) expected_output = ' '.join(["-rwxrwxrwx.", "1", user2]) validation_status = atfutils.validate_output(output, 0, expected_output) if validation_status is not 0: return validation_status validation_status = validate.validate_on_bricks(bricks, command, 0, expected_output) if validation_status is not 0: return validation_status return 0 def test010c(): """ Description: Self-heal: xattr entry pending """ input_file = urandom dir1 = "d1" dir2 = "d2" file1 = "f1" file2 = "f2" testdir1 = "testdir1" testdir2 = "testdir2" t1_d1 = os.path.join(testdir1, dir1) t1_d2 = os.path.join(testdir1, dir2) t2_d1 = os.path.join(testdir2, dir1) t2_d2 = os.path.join(testdir2, dir2) return_status = reset_testenv() if return_status is not 0: return return_status return_status = setup_testenv() if return_status is not 0: return return_status command = ' '.join([commands['mkdir'], t1_d1, t1_d2]) output = clientutils.execute_on_mount("mount1", command) expected_output = "" validation_status = atfutils.validate_output(output, 0, expected_output) if validation_status is not 0: return validation_status t1_d1_f1= os.path.join(t1_d1, file1) output_file = t1_d1_f1 command = ' '.join([commands['dd'], "if="+input_file, "of="+output_file, "bs=1M", "count=1"]) output = clientutils.execute_on_mount("mount1", command) expected_output = "1048576" validation_status = atfutils.validate_output(output, 0, expected_output, stream="stderr") if validation_status is not 0: return validation_status t1_d2_f1 = os.path.join(t1_d2, file1) output_file = t1_d2_f1 command = ' '.join([commands['dd'], "if="+input_file, "of="+output_file, "bs=1M", "count=1"]) output = clientutils.execute_on_mount("mount1", command) expected_output = "1048576" validation_status = atfutils.validate_output(output, 0, expected_output, stream="stderr") if validation_status is not 0: return validation_status t1_d2_f2 = os.path.join(t1_d2, file2) output_file = t1_d2_f2 command = ' '.join([commands['dd'], "if="+input_file, "of="+output_file, "bs=1M", "count=2"]) output = clientutils.execute_on_mount("mount1", command) expected_output = "2097152" validation_status = atfutils.validate_output(output, 0, expected_output, stream="stderr") if validation_status is not 0: return validation_status """ Stop brick2 """ output = glusterutils.volume_stop_brick("brick2") assert_success_status = atfutils.assert_success(output["exitstatus"]) if assert_success_status is not 0: return assert_success_status command = ' '.join([commands['unlink_dir_force'], t1_d2]) output = clientutils.execute_on_mount("mount1", command) expected_output = "" validation_status = atfutils.validate_output(output, 0, expected_output) if validation_status is not 0: return validation_status command = ' '.join([commands['mkdir'], t2_d1, t2_d2]) output = clientutils.execute_on_mount("mount1", command) expected_output = "" validation_status = atfutils.validate_output(output, 0, expected_output) if validation_status is not 0: return validation_status t2_f1 = os.path.join(testdir2, file1) output_file = t2_f1 command = ' '.join([commands['dd'], "if="+input_file, "of="+output_file, "bs=1M", "count=1"]) output = clientutils.execute_on_mount("mount1", command) expected_output = "1048576" validation_status = atfutils.validate_output(output, 0, expected_output, stream="stderr") if validation_status is not 0: return validation_status t2_d1_f1= os.path.join(t2_d1, file1) output_file = t2_d1_f1 command = ' '.join([commands['dd'], "if="+input_file, "of="+output_file, "bs=1M", "count=1"]) output = clientutils.execute_on_mount("mount1", command) expected_output = "1048576" validation_status = atfutils.validate_output(output, 0, expected_output, stream="stderr") if validation_status is not 0: return validation_status t2_d2_f2 = os.path.join(t2_d2, file2) output_file = t2_d2_f2 command = ' '.join([commands['dd'], "if="+input_file, "of="+output_file, "bs=1M", "count=2"]) output = clientutils.execute_on_mount("mount1", command) expected_output = "2097152" validation_status = atfutils.validate_output(output, 0, expected_output, stream="stderr") if validation_status is not 0: return validation_status """ Restart the brick2 """ output = glusterutils.volume_start("server1", force=True) assert_success_status = atfutils.assert_success(output["exitstatus"]) if assert_success_status is not 0: return assert_success_status brick_reboot_status = afrutils.wait_till_brick_reboot(brick_reboot_time) if brick_reboot_status is not 0: return brick_reboot_status command = "find . | xargs stat" output = clientutils.execute_on_mount("mount1", command) expected_output = "" validation_status = atfutils.validate_output(output, 0, expected_output) if validation_status is not 0: return validation_status selfheal_completion_status = afrutils.wait_till_selfheal_completes(30) if selfheal_completion_status is not 0: return selfheal_completion_status validation_status = validate.validate_md5sums(mounts, bricks) if validation_status is not 0: return validation_status return 0 def test011(): """ Description: Mark source with the lowest uid """ user1 = "qa" user2 = "root" data = "Hello World. testcase: Self-heal - Source with the Lowest UID" filename = "test" return_status = reset_testenv() if return_status is not 0: return return_status return_status = setup_testenv() if return_status is not 0: return return_status command = ' '.join([commands['echo'], data, write_op, filename]) output = clientutils.execute_on_mount("mount1", command) expected_output = "" validation_status = atfutils.validate_output(output, 0, expected_output) if validation_status is not 0: return validation_status user_group = ':'.join([user1, user1]) command = ' '.join([commands['chown'], user_group, filename]) output = clientutils.execute_on_mount("mount1", command) expected_output = "" validation_status = atfutils.validate_output(output, 0, expected_output) if validation_status is not 0: return validation_status # Change the user, group ownership to root on one of the brick from backend user_group = ':'.join([user2, user2]) command = ' '.join([commands['chown'], user_group, filename]) output = serverutils.execute_on_brick("brick1", command) expected_output = "" validation_status = atfutils.validate_output(output, 0, expected_output) if validation_status is not 0: return validation_status command = ' '.join([commands['ls'], "-l", filename]) output = clientutils.execute_on_mount("mount1", command) expected_output = ' '.join([user2, user2]) validation_status = atfutils.validate_output(output, 0, expected_output) if validation_status is not 0: return validation_status validation_status = validate.validate_on_bricks(bricks, command, 0, expected_output) if validation_status is not 0: return validation_status return 0 def test012(): """ Description: Forced merge of directory test """ input_file = urandom file1 = "f1" file2 = "f2" return_status = reset_testenv() if return_status is not 0: return return_status return_status = afrutils.setup_testenv_gfid_heal() if return_status is not 0: return return_status command = ' '.join([commands['dd'], "if="+input_file, "of="+file1, "bs=1M", "count=1"]) output = serverutils.execute_on_brick("brick1", command) expected_output = "1048576" validation_status = atfutils.validate_output(output, 0, expected_output, stream="stderr") if validation_status is not 0: return validation_status name = "trusted.gfid" value = "0s+wqm/34LQnm8Ec8tCmoHEg==" command = ' '.join([commands['setattr'],"-n", name, "-v", value, file1]) output = serverutils.execute_on_brick("brick1", command) expected_output = "" validation_status = atfutils.validate_output(output, 0, expected_output) if validation_status is not 0: return validation_status command = ' '.join([commands['dd'], "if="+input_file, "of="+file2, "bs=1M", "count=1"]) output = serverutils.execute_on_brick("brick2", command) expected_output = "1048576" validation_status = atfutils.validate_output(output, 0, expected_output, stream="stderr") if validation_status is not 0: return validation_status name = "trusted.gfid" value = "0sQTi1LjUkQwOhek4Oqx+daA==" command = ' '.join([commands['setattr'],"-n", name, "-v", value, file2]) output = serverutils.execute_on_brick("brick2", command) expected_output = "" validation_status = atfutils.validate_output(output, 0, expected_output) if validation_status is not 0: return validation_status return_status = afrutils.create_volume_gfid_heal("server1") if return_status is not 0: return return_status output = glusterutils.volume_start("server1", force=True) assert_success_status = atfutils.assert_success(output["exitstatus"]) if assert_success_status is not 0: return assert_success_status output = clientutils.mount("mount1") assert_success_status = atfutils.assert_success(output["exitstatus"]) if assert_success_status is not 0: return assert_success_status command = ' '.join([commands['ls'], "-l"]) output = clientutils.execute_on_mount("mount1", command) expected_output = "" validation_status = atfutils.validate_output(output, 0, expected_output) if validation_status is not 0: return validation_status base_command = "getfattr -d -m . -e hex" for _file in (file1, file2): command = ' '.join([base_command, _file]) outputs = serverutils.execute_on_bricks(bricks, command) assert_success_status = atfutils.assert_success_of_outputs(outputs) if assert_success_status is not 0: return assert_success_status selfheal_completion_status = afrutils.wait_till_selfheal_completes(15) if selfheal_completion_status is not 0: return selfheal_completion_status for _file in (file1, file2): command = ' '.join([commands['ls'], "-l", _file]) expected_output = "1048576" validation_status = validate.validate_on_bricks(bricks, command, 0, expected_output) if validation_status is not 0: return validation_status validation_status = validate.validate_md5sums(mounts, bricks) if validation_status is not 0: return validation_status return 0 def test013(): """ Description: Do not selfheal files without gfid's, when path to the files to be self-heal is not specified Ex:- Execution of find . | xargs stat on mount point will not self heal files if the files doesn't have gfid """ input_file = urandom file1 = "f1" return_status = reset_testenv() if return_status is not 0: return return_status return_status = afrutils.setup_testenv_gfid_heal() if return_status is not 0: return return_status output_file = file1 command = ' '.join([commands['dd'], "if="+input_file, "of="+output_file, "bs=1M", "count=1"]) output = serverutils.execute_on_brick("brick1", command) expected_output = "1048576" validation_status = atfutils.validate_output(output, 0, expected_output, stream="stderr") if validation_status is not 0: return validation_status command = ' '.join([commands['ls'], output_file]) output = serverutils.execute_on_brick("brick1", command) expected_output = output_file validation_status = atfutils.validate_output(output, 0, expected_output) if validation_status is not 0: return validation_status return_status = afrutils.create_volume_gfid_heal("server1") if return_status is not 0: return return_status output = glusterutils.volume_start("server1", force=True) assert_success_status = atfutils.assert_success(output["exitstatus"]) if assert_success_status is not 0: return assert_success_status output = clientutils.mount("mount1") assert_success_status = atfutils.assert_success(output["exitstatus"]) if assert_success_status is not 0: return assert_success_status command = "find . | xargs stat" output = clientutils.execute_on_mount("mount1", command) expected_output = "" validation_status = atfutils.validate_output(output, 0, expected_output) if validation_status is not 0: return validation_status command = ' '.join([commands['ls'], output_file]) output = serverutils.execute_on_brick("brick2", command) expected_output = "No such file or directory" validation_status = atfutils.validate_output(output, 1, expected_output, stream="stderr") if validation_status is not 0: return validation_status return 0 def test014(): """ Description: Selfheal files without gfid's when path to the files to be self-heal is specified Ex:- Execution of ls on mount point will self heal files even if the files doesn't have gfid """ input_file = urandom file1 = "f1" return_status = reset_testenv() if return_status is not 0: return return_status return_status = afrutils.setup_testenv_gfid_heal() if return_status is not 0: return return_status output_file = file1 command = ' '.join([commands['dd'], "if="+input_file, "of="+output_file, "bs=1M", "count=1"]) output = serverutils.execute_on_brick("brick1", command) expected_output = "1048576" validation_status = atfutils.validate_output(output, 0, expected_output, stream="stderr") if validation_status is not 0: return validation_status command = ' '.join([commands['ls'], output_file]) output = serverutils.execute_on_brick("brick1", command) expected_output = output_file validation_status = atfutils.validate_output(output, 0, expected_output) if validation_status is not 0: return validation_status command = ' '.join([commands['ls'], output_file]) output = serverutils.execute_on_brick("brick2", command) expected_output = "No such file or directory" validation_status = atfutils.validate_output(output, 1, expected_output, stream="stderr") if validation_status is not 0: return validation_status return_status = afrutils.create_volume_gfid_heal("server1") if return_status is not 0: return return_status output = glusterutils.volume_start("server1", force=True) assert_success_status = atfutils.assert_success(output["exitstatus"]) if assert_success_status is not 0: return assert_success_status output = clientutils.mount("mount1") assert_success_status = atfutils.assert_success(output["exitstatus"]) if assert_success_status is not 0: return assert_success_status command = ' '.join([commands['ls'], "-l", output_file]) output = clientutils.execute_on_mount("mount1", command) expected_output = "1048576" validation_status = atfutils.validate_output(output, 0, expected_output) if validation_status is not 0: return validation_status validation_status = validate.validate_on_bricks(bricks, command, 0, expected_output) if validation_status is not 0: return validation_status validation_status = validate.validate_md5sums(mounts, bricks) if validation_status is not 0: return validation_status return 0 def test015(): """ Description: * Birck1 has data without GFID * Brick2 doesn't have data * Create Volume * Start volume * execute 'ls -lR ' * Triggers Self-Heal. """ input_file = urandom dir1 = "d1" dir2 = "d2" file1 = "f1" file2 = "f2" testdir1 = "testdir1" t1_d1 = os.path.join(testdir1, dir1) t1_d2 = os.path.join(testdir1, dir2) return_status = reset_testenv() if return_status is not 0: return return_status return_status = afrutils.setup_testenv_gfid_heal() if return_status is not 0: return return_status command = ' '.join([commands['mkdir'], t1_d1, t1_d2]) output = serverutils.execute_on_brick("brick1", command) expected_output = "" validation_status = atfutils.validate_output(output, 0, expected_output) if validation_status is not 0: return validation_status t1_f1 = os.path.join(testdir1, file1) output_file = t1_f1 command = ' '.join([commands['dd'], "if="+input_file, "of="+output_file, "bs=1M", "count=1"]) output = serverutils.execute_on_brick("brick1", command) expected_output = "1048576" validation_status = atfutils.validate_output(output, 0, expected_output, stream="stderr") if validation_status is not 0: return validation_status t1_d1_f1= os.path.join(t1_d1, file1) output_file = t1_d1_f1 command = ' '.join([commands['dd'], "if="+input_file, "of="+output_file, "bs=1M", "count=1"]) output = serverutils.execute_on_brick("brick1", command) expected_output = "1048576" validation_status = atfutils.validate_output(output, 0, expected_output, stream="stderr") if validation_status is not 0: return validation_status t1_d1_f2 = os.path.join(t1_d1, file2) output_file = t1_d1_f2 command = ' '.join([commands['dd'], "if="+input_file, "of="+output_file, "bs=1M", "count=2"]) output = serverutils.execute_on_brick("brick1", command) expected_output = "2097152" validation_status = atfutils.validate_output(output, 0, expected_output, stream="stderr") if validation_status is not 0: return validation_status t1_d2_f1 = os.path.join(t1_d2, file1) output_file = t1_d2_f1 command = ' '.join([commands['dd'], "if="+input_file, "of="+output_file, "bs=1M", "count=1"]) output = serverutils.execute_on_brick("brick1", command) expected_output = "1048576" validation_status = atfutils.validate_output(output, 0, expected_output, stream="stderr") if validation_status is not 0: return validation_status t1_d2_f2 = os.path.join(t1_d2, file2) output_file = t1_d2_f2 command = ' '.join([commands['dd'], "if="+input_file, "of="+output_file, "bs=1M", "count=2"]) output = serverutils.execute_on_brick("brick1", command) expected_output = "2097152" validation_status = atfutils.validate_output(output, 0, expected_output, stream="stderr") if validation_status is not 0: return validation_status return_status = afrutils.create_volume_gfid_heal("server1") if return_status is not 0: return return_status output = glusterutils.volume_start("server1", force=True) assert_success_status = atfutils.assert_success(output["exitstatus"]) if assert_success_status is not 0: return assert_success_status output = clientutils.mount("mount1") assert_success_status = atfutils.assert_success(output["exitstatus"]) if assert_success_status is not 0: return assert_success_status command = ' '.join([commands['ls'], "-Rl", testdir1]) output = clientutils.execute_on_mount("mount1", command) expected_output = "" validation_status = atfutils.validate_output(output, 0, expected_output) if validation_status is not 0: return validation_status selfheal_completion_status = afrutils.wait_till_selfheal_completes(30) if selfheal_completion_status is not 0: return selfheal_completion_status validation_status = validate.validate_md5sums(mounts, bricks) if validation_status is not 0: return validation_status return 0 def test016(): """ Description: Self-Heal of files without GFID should return I/O Error when some childern are down """ input_file = urandom file1 = "f1" return_status = reset_testenv() if return_status is not 0: return return_status return_status = afrutils.setup_testenv_gfid_heal() if return_status is not 0: return return_status output_file = file1 command = ' '.join([commands['dd'], "if="+input_file, "of="+output_file, "bs=1M", "count=1"]) output = serverutils.execute_on_brick("brick1", command) expected_output = "1048576" validation_status = atfutils.validate_output(output, 0, expected_output, stream="stderr") if validation_status is not 0: return validation_status command = ' '.join([commands['ls'], output_file]) output = serverutils.execute_on_brick("brick1", command) expected_output = output_file validation_status = atfutils.validate_output(output, 0, expected_output) if validation_status is not 0: return validation_status command = ' '.join([commands['ls'], output_file]) output = serverutils.execute_on_brick("brick2", command) expected_output = "No such file or directory" validation_status = atfutils.validate_output(output, 1, expected_output, stream="stderr") if validation_status is not 0: return validation_status return_status = afrutils.create_volume_gfid_heal("server1") if return_status is not 0: return return_status output = glusterutils.volume_start("server1", force=True) assert_success_status = atfutils.assert_success(output["exitstatus"]) if assert_success_status is not 0: return assert_success_status output = glusterutils.volume_stop_brick("brick2") assert_success_status = atfutils.assert_success(output["exitstatus"]) if assert_success_status is not 0: return assert_success_status output = clientutils.mount("mount1") assert_success_status = atfutils.assert_success(output["exitstatus"]) if assert_success_status is not 0: return assert_success_status command = "find . | xargs stat" output = clientutils.execute_on_mount("mount1", command) expected_output = "Input/output error" validation_status = atfutils.validate_output(output, 1, expected_output, stream="stderr") if validation_status is not 0: return validation_status command = ' '.join([commands['ls'], "-Rl"]) output = clientutils.execute_on_mount("mount1", command) expected_output = "Input/output error" validation_status = atfutils.validate_output(output, 1, expected_output, stream="stderr") if validation_status is not 0: return validation_status command = ' '.join([commands['ls'], "-Rl", output_file]) output = clientutils.execute_on_mount("mount1", command) expected_output = "Input/output error" validation_status = atfutils.validate_output(output, 1, expected_output, stream="stderr") if validation_status is not 0: return validation_status command = ' '.join([commands['ls'], output_file]) output = serverutils.execute_on_brick("brick2", command) expected_output = "No such file or directory" validation_status = atfutils.validate_output(output, 1, expected_output, stream="stderr") if validation_status is not 0: return validation_status return 0 def test017(): """ Description: * create a on brick1 from the backend * create a replicate volume with brick1, brick2 * brick down brick1 * from the mount, dd * bring back brick1 * ls * The file on brick1 has to be replaced by new file """ input_file = urandom file1 = "f1" return_status = reset_testenv() if return_status is not 0: return return_status return_status = afrutils.setup_testenv_gfid_heal() if return_status is not 0: return return_status output_file = file1 command = ' '.join([commands['dd'], "if="+input_file, "of="+output_file, "bs=1M", "count=1"]) output = serverutils.execute_on_brick("brick1", command) expected_output = "1048576" validation_status = atfutils.validate_output(output, 0, expected_output, stream="stderr") if validation_status is not 0: return validation_status command = ' '.join([commands['ls'], output_file]) output = serverutils.execute_on_brick("brick1", command) expected_output = output_file validation_status = atfutils.validate_output(output, 0, expected_output) if validation_status is not 0: return validation_status command = ' '.join([commands['ls'], output_file]) output = serverutils.execute_on_brick("brick2", command) expected_output = "No such file or directory" validation_status = atfutils.validate_output(output, 1, expected_output, stream="stderr") if validation_status is not 0: return validation_status return_status = afrutils.create_volume_gfid_heal("server1") if return_status is not 0: return return_status output = glusterutils.volume_start("server1", force=True) assert_success_status = atfutils.assert_success(output["exitstatus"]) if assert_success_status is not 0: return assert_success_status output = glusterutils.volume_stop_brick("brick1") assert_success_status = atfutils.assert_success(output["exitstatus"]) if assert_success_status is not 0: return assert_success_status output = clientutils.mount("mount1") assert_success_status = atfutils.assert_success(output["exitstatus"]) if assert_success_status is not 0: return assert_success_status command = ' '.join([commands['ls'], "|", commands['wc'], "-l"]) output = clientutils.execute_on_mount("mount1", command) expected_output = "0" validation_status = atfutils.validate_output(output, 0, expected_output) if validation_status is not 0: return validation_status output_file = file1 command = ' '.join([commands['dd'], "if="+input_file, "of="+output_file, "bs=1M", "count=10"]) output = clientutils.execute_on_mount("mount1", command) expected_output = "10485760" validation_status = atfutils.validate_output(output, 0, expected_output, stream="stderr") if validation_status is not 0: return validation_status output = glusterutils.volume_start("server1", force=True) assert_success_status = atfutils.assert_success(output["exitstatus"]) if assert_success_status is not 0: return assert_success_status brick_reboot_status = afrutils.wait_till_brick_reboot(brick_reboot_time) if brick_reboot_status is not 0: return brick_reboot_status command = ' '.join([commands['ls'], "-l", output_file]) output = clientutils.execute_on_mount("mount1", command) expected_output = "10485760" validation_status = atfutils.validate_output(output, 0, expected_output) if validation_status is not 0: return validation_status selfheal_completion_status = afrutils.wait_till_selfheal_completes(15) if selfheal_completion_status is not 0: return selfheal_completion_status validation_status = validate.validate_on_bricks(bricks, command, 0, expected_output) if validation_status is not 0: return validation_status validation_status = validate.validate_md5sums(mounts, bricks) if validation_status is not 0: return validation_status return 0 def test018(): """ Testing split-brain """ input_file = urandom file1 = "f1" file2 = "f2" return_status = reset_testenv() if return_status is not 0: return return_status return_status = reset_testenv() if return_status is not 0: return return_status return_status = afrutils.setup_testenv_gfid_heal() if return_status is not 0: return return_status return_status = afrutils.create_volume_gfid_heal("server1") if return_status is not 0: return return_status output = glusterutils.volume_start("server1") assert_success_status = atfutils.assert_success(output["exitstatus"]) if assert_success_status is not 0: return assert_success_status output = glusterutils.volume_stop_brick("brick2") assert_success_status = atfutils.assert_success(output["exitstatus"]) if assert_success_status is not 0: return assert_success_status output = clientutils.mount("mount1") assert_success_status = atfutils.assert_success(output["exitstatus"]) if assert_success_status is not 0: return assert_success_status output_file = file1 command = ' '.join([commands['dd'], "if="+input_file, "of="+output_file, "bs=1M", "count=10"]) expected_output = "10485760" output = clientutils.execute_on_mount("mount1", command) validation_status = atfutils.validate_output(output, 0, expected_output, stream="stderr") if validation_status is not 0: return validation_status command = ' '.join([commands['ls'], "-l", output_file]) expected_output = "10485760" output = clientutils.execute_on_mount("mount1", command) validation_status = atfutils.validate_output(output, 0, expected_output) if validation_status is not 0: return validation_status output = serverutils.execute_on_brick("brick1", command) validation_status = atfutils.validate_output(output, 0, expected_output) if validation_status is not 0: return validation_status output = glusterutils.volume_start("server1", force=True) assert_success_status = atfutils.assert_success(output["exitstatus"]) if assert_success_status is not 0: return assert_success_status brick_reboot_status = afrutils.wait_till_brick_reboot(brick_reboot_time) if brick_reboot_status is not 0: return brick_reboot_status output = glusterutils.volume_stop_brick("brick1") assert_success_status = atfutils.assert_success(output["exitstatus"]) if assert_success_status is not 0: return assert_success_status output_file = file2 command = ' '.join([commands['dd'], "if="+input_file, "of="+output_file, "bs=1M", "count=10"]) expected_output = "10485760" output = clientutils.execute_on_mount("mount1", command) validation_status = atfutils.validate_output(output, 0, expected_output, stream="stderr") if validation_status is not 0: return validation_status command = ' '.join([commands['ls'], "-l", output_file]) expected_output = "10485760" output = clientutils.execute_on_mount("mount1", command) validation_status = atfutils.validate_output(output, 0, expected_output) if validation_status is not 0: return validation_status output = serverutils.execute_on_brick("brick2", command) validation_status = atfutils.validate_output(output, 0, expected_output) if validation_status is not 0: return validation_status output = glusterutils.volume_start("server1", force=True) assert_success_status = atfutils.assert_success(output["exitstatus"]) if assert_success_status is not 0: return assert_success_status brick_reboot_status = afrutils.wait_till_brick_reboot(brick_reboot_time) if brick_reboot_status is not 0: return brick_reboot_status command = "" expected_output = "Input/output error" output = clientutils.execute_on_mount("mount1", command) validation_status = atfutils.validate_output(output, 1, expected_output, stream="stderr") if validation_status is not 0: return validation_status return 0 def bug2840(): """ Description: Device Files not getting self-healed """ file1 = "file1" device_file = "devicefile" input_file = urandom return_status = reset_testenv() if return_status is not 0: return return_status return_status = setup_testenv() if return_status is not 0: return return_status output_file = file1 command = ' '.join([commands['dd'], "if="+input_file, "of="+output_file, "bs=1M", "count=1"]) expected_output = "1048576" output = clientutils.execute_on_mount("mount1", command) validation_status = atfutils.validate_output(output, 0, expected_output, stream="stderr") if validation_status is not 0: return validation_status command = ' '.join([commands['ls'], "-l", output_file]) expected_output = "1048576" output = clientutils.execute_on_mount("mount1", command) validation_status = atfutils.validate_output(output, 0, expected_output) if validation_status is not 0: return validation_status validation_status = validate.validate_on_bricks(bricks, command, 0, expected_output) if validation_status is not 0: return validation_status output = glusterutils.volume_stop_brick("brick2") assert_success_status = atfutils.assert_success(output["exitstatus"]) if assert_success_status is not 0: return assert_success_status command = "mknod -m 0600 %s b 2 0" % device_file expected_output = "" output = clientutils.execute_on_mount("mount1", command) validation_status = atfutils.validate_output(output, 0, expected_output) if validation_status is not 0: return validation_status command = ' '.join([commands['ls'], "-l", device_file]) expected_output = "brw-------." output = clientutils.execute_on_mount("mount1", command) validation_status = atfutils.validate_output(output, 0, expected_output) if validation_status is not 0: return validation_status output = serverutils.execute_on_brick("brick1", command) validation_status = atfutils.validate_output(output, 0, expected_output) if validation_status is not 0: return validation_status expected_output = "No such file or directory" output = serverutils.execute_on_brick("brick2", command) validation_status = atfutils.validate_output(output, 1, expected_output, stream="stderr") if validation_status is not 0: return validation_status output = glusterutils.volume_start("server1", force=True) assert_success_status = atfutils.assert_success(output["exitstatus"]) if assert_success_status is not 0: return assert_success_status brick_reboot_status = afrutils.wait_till_brick_reboot(brick_reboot_time) if brick_reboot_status is not 0: return brick_reboot_status command = "find . | xargs stat" output = clientutils.execute_on_mount("mount1", command) expected_output = "" validation_status = atfutils.validate_output(output, 0, expected_output) if validation_status is not 0: return validation_status command = ' '.join([commands['ls'], "-l", device_file]) expected_output = device_file output = clientutils.execute_on_mount("mount1", command) validation_status = atfutils.validate_output(output, 0, expected_output) if validation_status is not 0: return validation_status validation_status = validate.validate_on_bricks(bricks, command, 0, expected_output) if validation_status is not 0: return validation_status validation_status = validate.validate_md5sums(mounts, bricks) if validation_status is not 0: return validation_status return 0 def bug3503(): """ Description: open should trigger data self-heal instead of lookup """ file1 = "file1" input_file = urandom return_status = reset_testenv() if return_status is not 0: return return_status return_status = setup_testenv() if return_status is not 0: return return_status output = glusterutils.volume_set("server1", key="data-self-heal", value="open") assert_success_status = atfutils.assert_success(output["exitstatus"]) if assert_success_status is not 0: return assert_success_status output_file = file1 command = ' '.join([commands['dd'], "if="+input_file, "of="+output_file, "bs=1M", "count=1"]) output = clientutils.execute_on_mount("mount1", command) expected_output = "1048576" validation_status = atfutils.validate_output(output, 0, expected_output, stream="stderr") if validation_status is not 0: return validation_status command = ' '.join([commands['ls'], "-l", output_file]) expected_output = "1048576" output = clientutils.execute_on_mount("mount1", command) validation_status = atfutils.validate_output(output, 0, expected_output) if validation_status is not 0: return validation_status validation_status = validate.validate_on_bricks(bricks, command, 0, expected_output) if validation_status is not 0: return validation_status output = glusterutils.volume_stop_brick("brick2") assert_success_status = atfutils.assert_success(output["exitstatus"]) if assert_success_status is not 0: return assert_success_status command = ' '.join([commands['truncate'], "-s 0", output_file]) output = clientutils.execute_on_mount("mount1", command) expected_output = "" validation_status = atfutils.validate_output(output, 0, expected_output) if validation_status is not 0: return validation_status command = ' '.join([commands['ls'], "-l", output_file]) expected_output = "0" output = clientutils.execute_on_mount("mount1", command) validation_status = atfutils.validate_output(output, 0, expected_output) if validation_status is not 0: return validation_status output = serverutils.execute_on_brick("brick1", command) validation_status = atfutils.validate_output(output, 0, expected_output) if validation_status is not 0: return validation_status output = serverutils.execute_on_brick("brick2", command) expected_output = "1048576" validation_status = atfutils.validate_output(output, 0, expected_output) if validation_status is not 0: return validation_status output = glusterutils.volume_start("server1", force=True) assert_success_status = atfutils.assert_success(output["exitstatus"]) if assert_success_status is not 0: return assert_success_status brick_reboot_status = afrutils.wait_till_brick_reboot(brick_reboot_time) if brick_reboot_status is not 0: return brick_reboot_status # Do not data self heal when ls is executed command = ' '.join([commands['ls'], "-l", output_file]) expected_output = "0" output = clientutils.execute_on_mount("mount1", command) validation_status = atfutils.validate_output(output, 0, expected_output) if validation_status is not 0: return validation_status output = serverutils.execute_on_brick("brick1", command) validation_status = atfutils.validate_output(output, 0, expected_output) if validation_status is not 0: return validation_status output = serverutils.execute_on_brick("brick2", command) expected_output = "1048576" validation_status = atfutils.validate_output(output, 0, expected_output) if validation_status is not 0: return validation_status # Self heal with open or fd based fop command = ' '.join([commands['cat'], output_file]) output = clientutils.execute_on_mount("mount1", command) expected_output = "" validation_status = atfutils.validate_output(output, 0, expected_output) if validation_status is not 0: return validation_status command = ' '.join([commands['ls'], "-l", output_file]) expected_output = "0" output = clientutils.execute_on_mount("mount1", command) validation_status = atfutils.validate_output(output, 0, expected_output) if validation_status is not 0: return validation_status validation_status = validate.validate_on_bricks(bricks, command, 0, expected_output) if validation_status is not 0: return validation_status return 0 def test019(): """ Self-Heal of Special Files """ return_status = reset_testenv() if return_status is not 0: return return_status return_status = setup_testenv() if return_status is not 0: return return_status fifo_file = "fifo" block_file = "block" char_file = "char" user = "qa" root = "root" # Create a FIFO file command = "mkfifo -m 0600 %s" % fifo_file output = clientutils.execute_on_mount("mount1", command) assert_success_status = atfutils.assert_success(output["exitstatus"]) if assert_success_status is not 0: return assert_success_status command = ' '.join([commands['ls'], "-l", fifo_file]) expected_output = "prw-------." output = clientutils.execute_on_mount("mount1", command) validation_status = atfutils.validate_output(output, 0, expected_output) if validation_status is not 0: return validation_status validation_status = validate.validate_on_bricks(bricks, command, 0, expected_output) if validation_status is not 0: return validation_status # Create a Block File command = "mknod -m 0600 %s b 2 0" % block_file output = clientutils.execute_on_mount("mount1", command) assert_success_status = atfutils.assert_success(output["exitstatus"]) if assert_success_status is not 0: return assert_success_status command = ' '.join([commands['ls'], "-l", block_file]) expected_output = "brw-------." output = clientutils.execute_on_mount("mount1", command) validation_status = atfutils.validate_output(output, 0, expected_output) if validation_status is not 0: return validation_status validation_status = validate.validate_on_bricks(bricks, command, 0, expected_output) if validation_status is not 0: return validation_status # Create a Char File command = "mknod -m 0600 %s c 2 0" % char_file output = clientutils.execute_on_mount("mount1", command) assert_success_status = atfutils.assert_success(output["exitstatus"]) if assert_success_status is not 0: return assert_success_status command = ' '.join([commands['ls'], "-l", char_file]) expected_output = "crw-------." output = clientutils.execute_on_mount("mount1", command) validation_status = atfutils.validate_output(output, 0, expected_output) if validation_status is not 0: return validation_status validation_status = validate.validate_on_bricks(bricks, command, 0, expected_output) if validation_status is not 0: return validation_status output = glusterutils.volume_stop_brick("brick2") assert_success_status = atfutils.assert_success(output["exitstatus"]) if assert_success_status is not 0: return assert_success_status # Change the User and Group of FIFO, Block, Char Files user_group = ':'.join([user, user]) for _file in (fifo_file, block_file, char_file): command = ' '.join([commands['chown'], user_group, _file]) output = clientutils.execute_on_mount("mount1", command) assert_success_status = atfutils.assert_success(output["exitstatus"]) if assert_success_status is not 0: return assert_success_status # Validate the changes made to FIFO, Block, Char Files for _file in (fifo_file, block_file, char_file): command = ' '.join([commands['ls'], "-l", _file]) expected_output = ' '.join([user, user]) output = clientutils.execute_on_mount("mount1", command) validation_status = atfutils.validate_output(output, 0, expected_output) if validation_status is not 0: return validation_status output = serverutils.execute_on_brick("brick1", command) validation_status = atfutils.validate_output(output, 0, expected_output) if validation_status is not 0: return validation_status expected_output = ' '.join([root, root]) output = serverutils.execute_on_brick("brick2", command) validation_status = atfutils.validate_output(output, 0, expected_output) if validation_status is not 0: return validation_status # Change the Permissions for FIFO, Block, Char Files for _file in (fifo_file, block_file, char_file): command = ' '.join([commands['chmod'], "0644", _file]) output = clientutils.execute_on_mount("mount1", command) assert_success_status = atfutils.assert_success(output["exitstatus"]) if assert_success_status is not 0: return assert_success_status # Validate the Permission changes performed on FIFO File command = ' '.join([commands['ls'], "-l", fifo_file]) expected_output = "prw-r--r--." output = clientutils.execute_on_mount("mount1", command) validation_status = atfutils.validate_output(output, 0, expected_output) if validation_status is not 0: return validation_status output = serverutils.execute_on_brick("brick1", command) validation_status = atfutils.validate_output(output, 0, expected_output) if validation_status is not 0: return validation_status expected_output = "prw-------." output = serverutils.execute_on_brick("brick2", command) validation_status = atfutils.validate_output(output, 0, expected_output) if validation_status is not 0: return validation_status # Validate the Permission changes performed on Block File command = ' '.join([commands['ls'], "-l", block_file]) expected_output = "brw-r--r--." output = clientutils.execute_on_mount("mount1", command) validation_status = atfutils.validate_output(output, 0, expected_output) if validation_status is not 0: return validation_status output = serverutils.execute_on_brick("brick1", command) validation_status = atfutils.validate_output(output, 0, expected_output) if validation_status is not 0: return validation_status expected_output = "brw-------." output = serverutils.execute_on_brick("brick2", command) validation_status = atfutils.validate_output(output, 0, expected_output) if validation_status is not 0: return validation_status # Validate the Permission changes performed on Char File command = ' '.join([commands['ls'], "-l", char_file]) expected_output = "crw-r--r--." output = clientutils.execute_on_mount("mount1", command) validation_status = atfutils.validate_output(output, 0, expected_output) if validation_status is not 0: return validation_status output = serverutils.execute_on_brick("brick1", command) validation_status = atfutils.validate_output(output, 0, expected_output) if validation_status is not 0: return validation_status expected_output = "crw-------." output = serverutils.execute_on_brick("brick2", command) validation_status = atfutils.validate_output(output, 0, expected_output) if validation_status is not 0: return validation_status output = glusterutils.volume_start("server1", force=True) assert_success_status = atfutils.assert_success(output["exitstatus"]) if assert_success_status is not 0: return assert_success_status brick_reboot_status = afrutils.wait_till_brick_reboot(brick_reboot_time) if brick_reboot_status is not 0: return brick_reboot_status command = "find . | xargs stat" output = clientutils.execute_on_mount("mount1", command) assert_success_status = atfutils.assert_success(output["exitstatus"]) if assert_success_status is not 0: return assert_success_status # Validate the changes made to FIFO, Block, Char Files for _file in (fifo_file, block_file, char_file): command = ' '.join([commands['ls'], "-l", _file]) expected_output = ' '.join([user, user]) output = clientutils.execute_on_mount("mount1", command) validation_status = atfutils.validate_output(output, 0, expected_output) if validation_status is not 0: return validation_status validation_status = validate.validate_on_bricks(bricks, command, 0, expected_output) if validation_status is not 0: return validation_status # Validate the Permission changes performed on FIFO File command = ' '.join([commands['ls'], "-l", fifo_file]) expected_output = "prw-r--r--." output = clientutils.execute_on_mount("mount1", command) validation_status = atfutils.validate_output(output, 0, expected_output) if validation_status is not 0: return validation_status validation_status = validate.validate_on_bricks(bricks, command, 0, expected_output) if validation_status is not 0: return validation_status # Validate the Permission changes performed on Block File command = ' '.join([commands['ls'], "-l", block_file]) expected_output = "brw-r--r--." output = clientutils.execute_on_mount("mount1", command) validation_status = atfutils.validate_output(output, 0, expected_output) if validation_status is not 0: return validation_status validation_status = validate.validate_on_bricks(bricks, command, 0, expected_output) if validation_status is not 0: return validation_status # Validate the Permission changes performed on Char File command = ' '.join([commands['ls'], "-l", char_file]) expected_output = "crw-r--r--." output = clientutils.execute_on_mount("mount1", command) validation_status = atfutils.validate_output(output, 0, expected_output) if validation_status is not 0: return validation_status validation_status = validate.validate_on_bricks(bricks, command, 0, expected_output) if validation_status is not 0: return validation_status return 0