summaryrefslogtreecommitdiffstats
path: root/tests/functional/glusterd/test_bricks_online_after_node_reboot.py
blob: 87f6301a3730df205e98b908841eb9747f203c0b (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
#  Copyright (C) 2019-2020  Red Hat, Inc. <http://www.redhat.com>
#
#  This program is free software; you can redistribute it and/or modify
#  it under the terms of the GNU General Public License as published by
#  the Free Software Foundation; either version 2 of the License, or
#  any later version.
#
#  This program is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU General Public License for more details.
#
#  You should have received a copy of the GNU General Public License along
#  with this program; if not, write to the Free Software Foundation, Inc.,
#  51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.

"""
Test Cases in this module related to gluster bricks are online
after node reboot or not
"""
from random import choice
from time import sleep
from glusto.core import Glusto as g
from glustolibs.gluster.exceptions import ExecutionError
from glustolibs.gluster.gluster_base_class import GlusterBaseClass, runs_on
from glustolibs.gluster.volume_ops import (volume_start, volume_stop,
                                           volume_create, set_volume_options,
                                           get_volume_list)
from glustolibs.gluster.gluster_init import is_glusterd_running
from glustolibs.gluster.volume_libs import cleanup_volume
from glustolibs.gluster.brick_libs import wait_for_bricks_to_be_online
from glustolibs.gluster.lib_utils import form_bricks_list
from glustolibs.misc.misc_libs import reboot_nodes_and_wait_to_come_online


@runs_on([['distributed-dispersed'], ['glusterfs']])
class BricksOnlineAfterNodeReboot(GlusterBaseClass):
    def setUp(self):
        """
        setUp method for every test
        """
        # calling GlusterBaseClass setUp
        self.get_super_method(self, 'setUp')()

        # Creating Volume
        g.log.info("Started creating volume")
        ret = self.setup_volume()
        if ret:
            g.log.info("Volme created successfully : %s", self.volname)
        else:
            raise ExecutionError("Volume creation failed: %s" % self.volname)

    def tearDown(self):
        """
        tearDown for every test
        """
        # Cleaning up the volume
        volume_list = get_volume_list(choice(self.servers))
        for volume in volume_list:
            ret = cleanup_volume(self.mnode, volume)
            if not ret:
                raise ExecutionError("Failed Cleanup the Volume %s" % volume)
        g.log.info("Successfully cleaned up all the volumes")

        # Calling GlusterBaseClass tearDown
        self.get_super_method(self, 'tearDown')()

    def check_bricks_online(self, all_volumes):
        for volume in all_volumes:
            self.assertTrue(wait_for_bricks_to_be_online(
                self.mnode, volume), "Unexpected: Few bricks are offline")
            g.log.info("All bricks are online in the volume %s ", volume)

    def check_node_after_reboot(self, server):
        count = 0
        while count < 80:
            ret = is_glusterd_running(server)
            if not ret:
                ret = self.validate_peers_are_connected()
                if ret:
                    g.log.info("glusterd is running and all peers are in"
                               "connected state")
                    break
            count += 1
            sleep(10)
        self.assertNotEqual(count, 60, "Either glusterd is not runnig or peers"
                                       " are not in connected state")

    def test_bricks_online_after_node_reboot(self):
        '''
        Create all types of volumes
        Start the volume and check the bricks are online
        Reboot a node at random
        After the node is up check the bricks are online
        Set brick-mux to on
        stop and start the volume to get the brick-mux into effect
        Check all bricks are online
        Now perform node reboot
        After node reboot all bricks should be online
        '''

        # Creating all types of volumes disperse, replicate, arbiter
        all_volumes = ['disperse', 'replicate', 'arbiter']
        for volume in all_volumes:
            bricks_list = form_bricks_list(self.mnode, volume,
                                           6 if volume == "disperse" else 3,
                                           self.servers,
                                           self.all_servers_info)
            if volume == "disperse":
                ret, _, _ = volume_create(self.mnode, volume, bricks_list,
                                          disperse_count=6,
                                          redundancy_count=2)
            elif volume == "replicate":
                ret, _, _ = volume_create(self.mnode, volume, bricks_list,
                                          replica_count=3)
            else:
                ret, _, _ = volume_create(self.mnode, volume, bricks_list,
                                          replica_count=3, arbiter_count=1)
            self.assertEqual(ret, 0, "Unexpected: Volume create '%s' failed"
                             % volume)
            g.log.info("volume create %s succeeded", volume)
        # All volumes start
        for volume in all_volumes:
            ret, _, _ = volume_start(self.mnode, volume)
            self.assertEqual(ret, 0, "Unexpected: Volume start succeded %s"
                             % volume)
            g.log.info("Volume started succesfully %s", volume)

        # Adding self.volname to the all_volumes list
        all_volumes.append(self.volname)

        # Validate whether all volume bricks are online or not
        self.check_bricks_online(all_volumes)
        # Perform node reboot
        random_server = choice(self.servers)
        ret, _ = reboot_nodes_and_wait_to_come_online(random_server)
        self.assertTrue(ret, "Reboot Failed on node %s" % random_server)
        g.log.info("Node: %s rebooted successfully", random_server)

        # Wait till glusterd is started on the node rebooted
        self.check_node_after_reboot(random_server)

        # After reboot check bricks are online
        self.check_bricks_online(all_volumes)

        # Enable brick-mux on and stop and start the volumes
        ret = set_volume_options(self.mnode, 'all',
                                 {"cluster.brick-multiplex": "enable"})
        self.assertTrue(ret, "Unable to set the volume option")
        g.log.info("Brick-mux option enabled successfully")
        self.addCleanup(set_volume_options, self.mnode, 'all',
                        {"cluster.brick-multiplex": "disable"})

        # Stop all the volumes in the cluster
        for vol in all_volumes:
            ret, _, _ = volume_stop(self.mnode, vol)
            self.assertEqual(ret, 0, "volume stop failed on %s" % vol)
            g.log.info("volume: %s stopped successfully", vol)

        # Starting the volume to get brick-mux into effect
        for vol in all_volumes:
            ret, _, _ = volume_start(self.mnode, vol)
            self.assertEqual(ret, 0, "volume start failed on %s" % vol)
            g.log.info("volume: %s started successfully", vol)

        # Checking all bricks are online or not
        self.check_bricks_online(all_volumes)

        # Perform node reboot
        ret, _ = reboot_nodes_and_wait_to_come_online(random_server)
        self.assertTrue(ret, "Reboot Failed on node %s" % random_server)
        g.log.info("Node: %s rebooted successfully", random_server)

        # Wait till glusterd is started on the node rebooted
        self.check_node_after_reboot(random_server)

        # Validating bricks are online after node reboot
        self.check_bricks_online(all_volumes)