1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
|
import pytest
from glustolibs.gluster import volume_ops
import six
from openshiftstoragelibs.baseclass import BaseClass
from openshiftstoragelibs import exceptions
from openshiftstoragelibs import heketi_ops
from openshiftstoragelibs import heketi_version
from openshiftstoragelibs import node_ops
from openshiftstoragelibs import openshift_ops
from openshiftstoragelibs import podcmd
from openshiftstoragelibs import waiter
class TestHeketiBrickEvict(BaseClass):
"""Test Heketi brick evict functionality."""
def setUp(self):
super(TestHeketiBrickEvict, self).setUp()
version = heketi_version.get_heketi_version(self.heketi_client_node)
if version < '9.0.0-14':
self.skipTest(
"heketi-client package {} does not support brick evict".format(
version.v_str))
self.ocp_client = self.ocp_master_node[0]
node_list = heketi_ops.heketi_node_list(
self.heketi_client_node, self.heketi_server_url)
if len(node_list) > 3:
return
for node_id in node_list:
node_info = heketi_ops.heketi_node_info(
self.heketi_client_node, self.heketi_server_url, node_id,
json=True)
if len(node_info["devices"]) < 2:
self.skipTest("does not have extra device/node to evict brick")
@podcmd.GlustoPod()
def _get_gluster_vol_info(self, file_vol):
"""Get Gluster vol info.
Args:
ocp_client (str): Node to execute OCP commands.
file_vol (str): file volume name.
Returns:
dict: Info of the given gluster vol.
"""
g_vol_info = volume_ops.get_volume_info(
"auto_get_gluster_endpoint", file_vol)
if not g_vol_info:
raise AssertionError("Failed to get volume info for gluster "
"volume {}".format(file_vol))
if file_vol in g_vol_info:
g_vol_info = g_vol_info.get(file_vol)
return g_vol_info
@pytest.mark.tier1
def test_heketi_brick_evict(self):
"""Test brick evict basic functionality and verify it replace a brick
properly
"""
h_node, h_server = self.heketi_client_node, self.heketi_server_url
size = 1
vol_info_old = heketi_ops.heketi_volume_create(
h_node, h_server, size, json=True)
self.addCleanup(
heketi_ops.heketi_volume_delete, h_node, h_server,
vol_info_old['id'])
heketi_ops.heketi_brick_evict(
h_node, h_server, vol_info_old["bricks"][0]['id'])
vol_info_new = heketi_ops.heketi_volume_info(
h_node, h_server, vol_info_old['id'], json=True)
bricks_old = set({brick['path'] for brick in vol_info_old["bricks"]})
bricks_new = set({brick['path'] for brick in vol_info_new["bricks"]})
self.assertEqual(
len(bricks_new - bricks_old), 1,
"Brick was not replaced with brick evict for vol \n {}".format(
vol_info_new))
gvol_info = self._get_gluster_vol_info(vol_info_new['name'])
gbricks = set(
{brick['name'].split(":")[1]
for brick in gvol_info["bricks"]["brick"]})
self.assertEqual(
bricks_new, gbricks, "gluster vol info and heketi vol info "
"mismatched after brick evict {} \n {}".format(
gvol_info, vol_info_new))
def _wait_for_gluster_pod_after_node_reboot(self, node_hostname):
"""Wait for glusterfs pod to be ready after node reboot"""
openshift_ops.wait_for_ocp_node_be_ready(
self.ocp_client, node_hostname)
gluster_pod = openshift_ops.get_gluster_pod_name_for_specific_node(
self.ocp_client, node_hostname)
openshift_ops.wait_for_pod_be_ready(self.ocp_client, gluster_pod)
services = (
("glusterd", "running"), ("gluster-blockd", "running"),
("tcmu-runner", "running"), ("gluster-block-target", "exited"))
for service, state in services:
openshift_ops.check_service_status_on_pod(
self.ocp_client, gluster_pod, service, "active", state)
@pytest.mark.tier4
def test_brick_evict_with_node_down(self):
"""Test brick evict basic functionality and verify brick evict
after node down"""
h_node, h_server = self.heketi_client_node, self.heketi_server_url
# Disable node if more than 3
node_list = heketi_ops.heketi_node_list(h_node, h_server)
if len(node_list) > 3:
for node_id in node_list[3:]:
heketi_ops.heketi_node_disable(h_node, h_server, node_id)
self.addCleanup(
heketi_ops.heketi_node_enable, h_node, h_server, node_id)
# Create heketi volume
vol_info = heketi_ops.heketi_volume_create(
h_node, h_server, 1, json=True)
self.addCleanup(
heketi_ops.heketi_volume_delete,
h_node, h_server, vol_info.get('id'))
# Get node on which heketi pod is scheduled
heketi_pod = openshift_ops.get_pod_name_from_dc(
self.ocp_client, self.heketi_dc_name)
heketi_node = openshift_ops.oc_get_custom_resource(
self.ocp_client, 'pod', '.:spec.nodeName', heketi_pod)[0]
# Get list of hostname from node id
host_list = []
for node_id in node_list[3:]:
node_info = heketi_ops.heketi_node_info(
h_node, h_server, node_id, json=True)
host_list.append(node_info.get('hostnames').get('manage')[0])
# Get brick id and glusterfs node which is not heketi node
for node in vol_info.get('bricks', {}):
node_info = heketi_ops.heketi_node_info(
h_node, h_server, node.get('node'), json=True)
hostname = node_info.get('hostnames').get('manage')[0]
if (hostname != heketi_node) and (hostname not in host_list):
brick_id = node.get('id')
break
# Bring down the glusterfs node
vm_name = node_ops.find_vm_name_by_ip_or_hostname(hostname)
self.addCleanup(
self._wait_for_gluster_pod_after_node_reboot, hostname)
self.addCleanup(node_ops.power_on_vm_by_name, vm_name)
node_ops.power_off_vm_by_name(vm_name)
# Wait glusterfs node to become NotReady
custom = r'":.status.conditions[?(@.type==\"Ready\")]".status'
for w in waiter.Waiter(300, 20):
status = openshift_ops.oc_get_custom_resource(
self.ocp_client, 'node', custom, hostname)
if status[0] in ['False', 'Unknown']:
break
if w.expired:
raise exceptions.ExecutionError(
"Failed to bring down node {}".format(hostname))
# Perform brick evict operation
try:
heketi_ops.heketi_brick_evict(h_node, h_server, brick_id)
except AssertionError as e:
if ('No Replacement was found' not in six.text_type(e)):
raise
|
