summaryrefslogtreecommitdiffstats
path: root/tests/functional/gluster_stability/test_restart_gluster_services.py
blob: ea188f36db1a97a3e1c4a72f59e3bda159e9ba17 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
from datetime import datetime
import time
from unittest import skip

import ddt
from glusto.core import Glusto as g
import pytest

from openshiftstoragelibs.baseclass import GlusterBlockBaseClass
from openshiftstoragelibs.gluster_ops import (
    get_gluster_vol_hosting_nodes,
    match_heketi_and_gluster_block_volumes_by_prefix,
    restart_file_volume,
    restart_gluster_vol_brick_processes,
    wait_to_heal_complete,
)
from openshiftstoragelibs.heketi_ops import (
    heketi_blockvolume_list_by_name_prefix,
    heketi_server_operation_cleanup,
    heketi_server_operations_list,
)
from openshiftstoragelibs.openshift_ops import (
    match_pv_and_heketi_block_volumes,
    match_pvc_and_pv,
    oc_create_pvc,
    oc_delete,
    oc_rsh,
    restart_service_on_gluster_pod_or_node,
    verify_pvc_status_is_bound,
    wait_for_pod_be_ready,
    wait_for_resource_absence,
    wait_for_service_status_on_gluster_pod_or_node,
)
from openshiftstoragelibs.openshift_storage_version import (
    get_openshift_storage_version
)
from openshiftstoragelibs import utils


HEKETI_BLOCK_VOLUME_REGEX = "^Id:(.*).Cluster:(.*).Name:%s_(.*)$"
SERVICE_TARGET = "gluster-block-target"
SERVICE_BLOCKD = "gluster-blockd"
SERVICE_TCMU = "tcmu-runner"


@ddt.ddt
class GlusterStabilityTestSetup(GlusterBlockBaseClass):
    """class for gluster stability (restarts different servces) testcases
    """

    def setUp(self):
        """Deploys, Verifies and adds resources required for testcases
           in cleanup method
        """
        super(GlusterStabilityTestSetup, self).setUp()
        self.oc_node = self.ocp_master_node[0]
        self.prefix = "autotest-%s" % utils.get_random_str()

    def deploy_and_verify_resouces(self):
        """Deploys and verifies required resources storage class, PVC
           and user app with continous I/O runnig.
        """
        # using pvc size count as 1 by default
        self.pvcsize = 1

        # using pvc count as 10 by default
        self.pvccount = 10

        self.sc_name = self.create_storage_class(
            vol_name_prefix=self.prefix)
        self.pvc_name = self.create_and_wait_for_pvc(
            pvc_name_prefix=self.prefix, sc_name=self.sc_name)
        self.dc_name, self.pod_name = self.create_dc_with_pvc(self.pvc_name)

        # create pvc's to test
        self.pvc_list = []
        for pvc in range(self.pvccount):
            test_pvc_name = oc_create_pvc(
                self.oc_node, self.sc_name,
                pvc_name_prefix=self.prefix, pvc_size=self.pvcsize
            )
            self.pvc_list.append(test_pvc_name)
            self.addCleanup(
                wait_for_resource_absence, self.oc_node, "pvc", test_pvc_name,
                timeout=600, interval=10
            )

        for pvc_name in self.pvc_list:
            self.addCleanup(oc_delete, self.oc_node, "pvc", pvc_name)

    def validate_volumes_and_blocks(self):
        """Validates PVC and block volumes generated through heketi and OCS
        """
        heketi_operations = heketi_server_operations_list(
            self.heketi_client_node, self.heketi_server_url,
            secret=self.heketi_cli_key, user=self.heketi_cli_user)

        for heketi_operation in heketi_operations:
            if heketi_operation["status"] == "failed":
                heketi_server_operation_cleanup(
                    self.heketi_client_node, self.heketi_server_url,
                    heketi_operation["id"], secret=self.heketi_cli_key,
                    user=self.heketi_cli_user
                )

        # verify pvc status is in "Bound" for all the pvc
        for pvc in self.pvc_list:
            verify_pvc_status_is_bound(
                self.oc_node, pvc, timeout=300, wait_step=10
            )

        # validate pvcs and pvs created on OCS
        match_pvc_and_pv(self.oc_node, self.prefix)

        # get list of block volumes using heketi
        h_blockvol_list = heketi_blockvolume_list_by_name_prefix(
            self.heketi_client_node, self.heketi_server_url, self.prefix)

        # validate block volumes listed by heketi and pvs
        heketi_blockvolume_ids = sorted([bv[0] for bv in h_blockvol_list])
        match_pv_and_heketi_block_volumes(
            self.oc_node, heketi_blockvolume_ids, self.prefix)

        # validate block volumes listed by heketi and gluster
        heketi_blockvolume_names = sorted([
            bv[1].replace("%s_" % self.prefix, "") for bv in h_blockvol_list])
        match_heketi_and_gluster_block_volumes_by_prefix(
            heketi_blockvolume_names, "%s_" % self.prefix)

    def get_io_time(self):
        """Gets last io time of io pod by listing log file directory
           /mnt on pod
        """
        ret, stdout, stderr = oc_rsh(
            self.oc_node, self.pod_name, "ls -l /mnt/ | awk '{print $8}'"
        )
        if ret != 0:
            err_msg = "failed to get io time for pod %s" % self.pod_name
            g.log.error(err_msg)
            raise AssertionError(err_msg)

        get_time = None
        try:
            get_time = datetime.strptime(stdout.strip(), "%H:%M")
        except Exception:
            g.log.error("invalid time format ret %s, stout: %s, "
                        "stderr: %s" % (ret, stdout, stderr))
            raise

        return get_time

    def restart_block_hosting_volume_wait_for_heal(self, block_hosting_vol):
        """restarts block hosting volume and wait for heal to complete

        Args:
            block_hosting_vol (str): block hosting volume which need to
                                     restart
        """
        start_io_time = self.get_io_time()

        restart_file_volume(block_hosting_vol)

        # Explicit wait to start ios on pvc after volume start
        time.sleep(5)
        resume_io_time = self.get_io_time()

        self.assertGreater(resume_io_time, start_io_time, "IO has not stopped")

        wait_to_heal_complete()

    @skip("Blocked by BZ-1634745, BZ-1635736, BZ-1636477")
    @pytest.mark.tier4
    @ddt.data(SERVICE_BLOCKD, SERVICE_TCMU, SERVICE_TARGET)
    def test_restart_services_provision_volume_and_run_io(self, service):
        """Restart gluster service then validate volumes"""
        skip_msg = (
            "Skipping this test case due to bugs "
            "BZ-1634745, BZ-1635736, BZ-1636477, BZ-1641668")

        # TODO(vamahaja): Add check for CRS version
        if not self.is_containerized_gluster():
            self.skipTest(skip_msg + " and not implemented CRS version check")

        if get_openshift_storage_version() < "3.11.2":
            self.skipTest(skip_msg)

        self.deploy_and_verify_resouces()

        block_hosting_vol = self.get_block_hosting_volume_by_pvc_name(
            self.pvc_name)
        g_nodes = get_gluster_vol_hosting_nodes(block_hosting_vol)
        self.assertGreater(len(g_nodes), 2)

        # restarts glusterfs service
        restart_service_on_gluster_pod_or_node(
            self.oc_node, service, g_nodes[0])

        # wait for deployed user pod to be in Running state after restarting
        # service
        wait_for_pod_be_ready(
            self.oc_node, self.pod_name, timeout=60, wait_step=5)

        # checks if all glusterfs services are in running state
        for g_node in g_nodes:
            for service in (SERVICE_BLOCKD, SERVICE_TCMU, SERVICE_TARGET):
                state = "exited" if service == SERVICE_TARGET else "running"
                self.assertTrue(wait_for_service_status_on_gluster_pod_or_node(
                    self.oc_node, service, 'active', state, g_node))

        # validates pvc, pv, heketi block and gluster block count after
        # service restarts
        self.validate_volumes_and_blocks()

    @skip("Blocked by BZ-1634745, BZ-1635736, BZ-1636477")
    @pytest.mark.tier4
    def test_target_side_failures_brick_failure_on_block_hosting_volume(self):
        """Target side failures - Brick failure on block hosting volume"""
        skip_msg = (
            "Skipping this test case due to bugs "
            "BZ-1634745, BZ-1635736, BZ-1636477, BZ-1641668")

        # TODO(vamahaja): Add check for CRS version
        if not self.is_containerized_gluster():
            self.skipTest(skip_msg + " and not impleted CRS version check")

        if get_openshift_storage_version() < "3.11.2":
            self.skipTest(skip_msg)

        self.deploy_and_verify_resouces()

        # get block hosting volume from pvc name
        block_hosting_vol = self.get_block_hosting_volume_by_pvc_name(
            self.pvc_name)

        # restarts 2 brick processes of block hosting volume
        g_nodes = get_gluster_vol_hosting_nodes(block_hosting_vol)
        self.assertGreater(len(g_nodes), 2)
        restart_gluster_vol_brick_processes(
            self.oc_node, block_hosting_vol, g_nodes[:2])

        # checks if all glusterfs services are in running state
        for g_node in g_nodes:
            for service in (SERVICE_BLOCKD, SERVICE_TCMU, SERVICE_TARGET):
                state = "exited" if service == SERVICE_TARGET else "running"
                self.assertTrue(wait_for_service_status_on_gluster_pod_or_node(
                    self.oc_node, service, 'active', state, g_node))

        # validates pvc, pv, heketi block and gluster block count after
        # service restarts
        self.validate_volumes_and_blocks()

    @skip("Blocked by BZ-1634745, BZ-1635736, BZ-1636477")
    @pytest.mark.tier4
    def test_start_stop_block_volume_service(self):
        """Validate block hosting volume by start/stop operation

           Perform stop/start operation on block hosting volume when
           IO's and provisioning are going on
        """
        skip_msg = (
            "Skipping this test case due to bugs "
            "BZ-1634745, BZ-1635736, BZ-1636477, BZ-1641668")

        # TODO(vamahaja): Add check for CRS version
        if not self.is_containerized_gluster():
            self.skipTest(skip_msg + " and not impleted CRS version check")

        if get_openshift_storage_version() < "3.11.2":
            self.skipTest(skip_msg)

        self.deploy_and_verify_resouces()

        # get block hosting volume from pvc name
        block_hosting_vol = self.get_block_hosting_volume_by_pvc_name(
            self.pvc_name
        )

        # restarts one of the block hosting volume and checks heal
        self.restart_block_hosting_volume_wait_for_heal(block_hosting_vol)

        # validates pvc, pv, heketi block and gluster block count after
        # service restarts
        self.validate_volumes_and_blocks()