1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
|
# Copyright (C) 2020 Red Hat, Inc. <http://www.redhat.com>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
from random import choice
from time import sleep
from glusto.core import Glusto as g
from glustolibs.gluster.exceptions import ExecutionError
from glustolibs.gluster.gluster_base_class import GlusterBaseClass, runs_on
from glustolibs.gluster.brick_libs import get_all_bricks, bring_bricks_offline
from glustolibs.gluster.volume_ops import volume_start
from glustolibs.gluster.volume_libs import get_volume_type
from glustolibs.gluster.glusterfile import create_link_file
@runs_on([['replicated', 'arbiter',
'distributed', 'distributed-arbiter',
'distributed-replicated'],
['glusterfs']])
class TestRenameFilesBrickDown(GlusterBaseClass):
# pylint: disable=too-many-statements
def setUp(self):
"""
Setup and mount volume or raise ExecutionError
"""
self.get_super_method(self, 'setUp')()
# Setup Volume
ret = self.setup_volume_and_mount_volume(self.mounts)
if not ret:
g.log.error("Failed to Setup and Mount Volume")
raise ExecutionError("Failed to Setup and Mount Volume")
def tearDown(self):
# Unmount and cleanup original volume
ret = self.unmount_volume_and_cleanup_volume(mounts=self.mounts)
if not ret:
raise ExecutionError("Failed to umount the vol & cleanup Volume")
g.log.info("Successful in umounting the volume and Cleanup")
# Calling GlusterBaseClass tearDown
self.get_super_method(self, 'tearDown')()
def test_rename_files_with_brick_down(self):
"""
Description: Tests to check that there is no data loss when rename is
performed with a brick of volume down.
Steps :
1) Create a volume.
2) Mount the volume using FUSE.
3) Create 1000 files on the mount point.
4) Create the soft-link for file{1..100}
5) Create the hard-link for file{101..200}
6) Check for the file count on the mount point.
7) Begin renaming the files, in multiple iterations.
8) Let few iterations of the rename complete successfully.
9) Then while rename is still in progress, kill a brick part of the
volume.
10) Let the brick be down for sometime, such that the a couple
of rename iterations are completed.
11) Bring the brick back online.
12) Wait for the IO to complete.
13) Check if there is any data loss.
14) Check if all the files are renamed properly.
"""
# Creating 1000 files on volume root
m_point = self.mounts[0].mountpoint
command = 'touch ' + m_point + '/file{1..1000}_0'
ret, _, _ = g.run(self.clients[0], command)
self.assertEqual(ret, 0, "File creation failed on %s"
% m_point)
g.log.info("Files successfully created on the mount point")
# Create soft links for a few files
for i in range(1, 100):
ret = create_link_file(self.clients[0],
'{}/file{}_0'.format(m_point, i),
'{}/soft_link_file{}_0'.format(m_point, i),
soft=True)
self.assertTrue(ret, "Failed to create soft links for files")
g.log.info("Created soft links for files successfully")
# Create hard links for a few files
for i in range(101, 200):
ret = create_link_file(self.clients[0],
'{}/file{}_0'.format(m_point, i),
'{}/hard_link_file{}_0'.format(m_point, i),
soft=False)
self.assertTrue(ret, "Failed to create hard links for files")
g.log.info("Created hard links for files successfully")
# Calculate file count for the mount-point
cmd = ("ls -lR %s/ | wc -l" % m_point)
ret, count_before, _ = g.run(self.clients[0], cmd)
self.assertEqual(ret, 0, "Failed to get file count")
g.log.info("File count before rename is:%s", count_before)
# Start renaming the files in multiple iterations
g.log.info("Starting to rename the files")
all_mounts_procs = []
cmd = ('for i in `seq 1 1000`; do for j in `seq 0 5`;do mv -f '
'%s/file$i\\_$j %s/file$i\\_$(expr $j + 1); done; done'
% (m_point, m_point))
proc = g.run_async(self.mounts[0].client_system, cmd,
user=self.mounts[0].user)
all_mounts_procs.append(proc)
# Waiting for some time for a iteration of rename to complete
g.log.info("Waiting for few rename iterations to complete")
sleep(120)
# Get the information about the bricks part of the volume
brick_list = get_all_bricks(self.mnode, self.volname)
# Kill a brick part of the volume
ret = bring_bricks_offline(self.volname, choice(brick_list))
self.assertTrue(ret, "Failed to bring brick offline")
g.log.info("Successfully brought brick offline")
# Let the brick be down for some time
g.log.info("Keeping brick down for few minutes")
sleep(60)
# Bring the brick online using gluster v start force
ret, _, _ = volume_start(self.mnode, self.volname, force=True)
self.assertEqual(ret, 0, "Volume start with force failed")
g.log.info("Volume start with force successful")
# Close connection and check if rename has completed
ret, _, _ = proc.async_communicate()
self.assertEqual(ret, 0, "Rename is not completed")
g.log.info("Rename is completed")
# Do lookup on the files
# Calculate file count from mount
cmd = ("ls -lR %s/ | wc -l" % m_point)
ret, count_after, _ = g.run(self.clients[0], cmd)
self.assertEqual(ret, 0, "Failed to do lookup and"
"get file count")
g.log.info("Lookup successful. File count after"
" rename is:%s", count_after)
# Check if there is any data loss
self.assertEqual(int(count_before), int(count_after),
"The file count before and after"
" rename is not same. There is data loss.")
g.log.info("The file count before and after rename is same."
" No data loss occurred.")
# Checking if all files were renamed Successfully
ret = get_volume_type(brick_list[0] + "/")
if ret in ("Replicate", "Disperse", "Arbiter", "Distributed-Replicate",
"Distribute-Disperse", "Distribute-Arbiter"):
cmd = ("ls -lR %s/file*_6 | wc -l" % m_point)
ret, out, _ = g.run(self.clients[0], cmd)
self.assertEqual(int(out), 1000, "Rename failed on some files")
g.log.info("All the files are renamed successfully")
|
