From f253c71feecfe0968ef96cd41855920468ea08ed Mon Sep 17 00:00:00 2001 From: Shwetha Panduranga Date: Mon, 20 Mar 2017 19:30:22 +0530 Subject: Adding sanity heal tests when IO in progress. 1) test heal with replace-brick when io in progress 2) test heal when bricks goes offline and comes back online when io in progress. Change-Id: Id9002c465aec8617217a12fa36846cdc1f61d7a4 Signed-off-by: Shwetha Panduranga Signed-off-by: ShwethaHP --- .../glustolibs/gluster/brick_libs.py | 386 +++++++++++++++++++++ .../glustolibs/gluster/volume_libs.py | 289 ++++++++++++++- 2 files changed, 674 insertions(+), 1 deletion(-) (limited to 'glustolibs-gluster/glustolibs') diff --git a/glustolibs-gluster/glustolibs/gluster/brick_libs.py b/glustolibs-gluster/glustolibs/gluster/brick_libs.py index a67ef1d7e..15110300f 100644 --- a/glustolibs-gluster/glustolibs/gluster/brick_libs.py +++ b/glustolibs-gluster/glustolibs/gluster/brick_libs.py @@ -17,8 +17,12 @@ """ Description: Module for gluster brick related helper functions. """ import random +from math import ceil from glusto.core import Glusto as g from glustolibs.gluster.volume_ops import (get_volume_info, get_volume_status) +from glustolibs.gluster.volume_libs import (get_subvols, is_tiered_volume, + get_client_quorum_info, + get_volume_type_info) def get_all_bricks(mnode, volname): @@ -178,6 +182,9 @@ def bring_bricks_offline(volname, bricks_list, elif isinstance(bring_bricks_offline_methods, str): bring_bricks_offline_methods = [bring_bricks_offline_methods] + if isinstance(bricks_list, str): + bricks_list = [bricks_list] + _rc = True failed_to_bring_offline_list = [] for brick in bricks_list: @@ -419,3 +426,382 @@ def delete_bricks(bricks_list): brick_path, brick_node) _rc = False return _rc + + +def select_bricks_to_bring_offline(mnode, volname): + """Randomly selects bricks to bring offline without affecting the cluster + + Args: + mnode (str): Node on which commands will be executed. + volname (str): Name of the volume. + + Returns: + dict: On success returns dict. Value of each key is list of bricks to + bring offline. + If volume doesn't exist returns dict with value of each item + being empty list. + Example: + brick_to_bring_offline = { + 'is_tier': False, + 'hot_tier_bricks': [], + 'cold_tier_bricks': [], + 'volume_bricks': [] + } + """ + # Defaulting the values to empty list + bricks_to_bring_offline = { + 'is_tier': False, + 'hot_tier_bricks': [], + 'cold_tier_bricks': [], + 'volume_bricks': [] + } + + volinfo = get_volume_info(mnode, volname) + if volinfo is None: + g.log.error("Unable to get the volume info for volume %s", volname) + return bricks_to_bring_offline + + if is_tiered_volume(mnode, volname): + bricks_to_bring_offline['is_tier'] = True + # Select bricks from tiered volume. + bricks_to_bring_offline = ( + select_tier_volume_bricks_to_bring_offline(mnode, volname)) + else: + # Select bricks from non-tiered volume. + volume_bricks = select_volume_bricks_to_bring_offline(mnode, volname) + bricks_to_bring_offline['volume_bricks'] = volume_bricks + + return bricks_to_bring_offline + + +def select_volume_bricks_to_bring_offline(mnode, volname): + """Randomly selects bricks to bring offline without affecting the cluster + from a non-tiered volume. + + Args: + mnode (str): Node on which commands will be executed. + volname (str): Name of the volume. + + Returns: + list: On success returns list of bricks that can be brough offline. + If volume doesn't exist or is a tiered volume returns empty list + """ + volume_bricks_to_bring_offline = [] + + # Check if volume is tiered + if is_tiered_volume(mnode, volname): + return volume_bricks_to_bring_offline + + # get volume type + volume_type_info = get_volume_type_info(mnode, volname) + volume_type = volume_type_info['volume_type_info']['typeStr'] + + # get subvols + subvols_dict = get_subvols(mnode, volname) + volume_subvols = subvols_dict['volume_subvols'] + + # select bricks from distribute volume + if volume_type == 'Distribute': + volume_bricks_to_bring_offline = [] + + # select bricks from replicated, distributed-replicated volume + elif (volume_type == 'Replicate' or + volume_type == 'Distributed-Replicate'): + # Get replica count + volume_replica_count = (volume_type_info['volume_type_info'] + ['replicaCount']) + + # Get quorum info + quorum_info = get_client_quorum_info(mnode, volname) + volume_quorum_info = quorum_info['volume_quorum_info'] + + # Get list of bricks to bring offline + volume_bricks_to_bring_offline = ( + get_bricks_to_bring_offline_from_replicated_volume( + volume_subvols, volume_replica_count, volume_quorum_info)) + + # select bricks from Disperse, Distribured-Disperse volume + elif (volume_type == 'Disperse' or + volume_type == 'Distributed-Disperse'): + + # Get redundancy count + volume_redundancy_count = (volume_type_info['volume_type_info'] + ['redundancyCount']) + + # Get list of bricks to bring offline + volume_bricks_to_bring_offline = ( + get_bricks_to_bring_offline_from_disperse_volume( + volume_subvols, volume_redundancy_count)) + + return volume_bricks_to_bring_offline + + +def select_tier_volume_bricks_to_bring_offline(mnode, volname): + """Randomly selects bricks to bring offline without affecting the cluster + from a tiered volume. + + Args: + mnode (str): Node on which commands will be executed. + volname (str): Name of the volume. + + Returns: + dict: On success returns dict. Value of each key is list of bricks to + bring offline. + If volume doesn't exist or is not a tiered volume returns dict + with value of each item being empty list. + Example: + brick_to_bring_offline = { + 'hot_tier_bricks': [], + 'cold_tier_bricks': [], + } + """ + # Defaulting the values to empty list + bricks_to_bring_offline = { + 'hot_tier_bricks': [], + 'cold_tier_bricks': [], + } + + volinfo = get_volume_info(mnode, volname) + if volinfo is None: + g.log.error("Unable to get the volume info for volume %s", volname) + return bricks_to_bring_offline + + if is_tiered_volume(mnode, volname): + # Select bricks from both hot tier and cold tier. + hot_tier_bricks = (select_hot_tier_bricks_to_bring_offline + (mnode, volname)) + cold_tier_bricks = (select_cold_tier_bricks_to_bring_offline + (mnode, volname)) + bricks_to_bring_offline['hot_tier_bricks'] = hot_tier_bricks + bricks_to_bring_offline['cold_tier_bricks'] = cold_tier_bricks + return bricks_to_bring_offline + else: + return bricks_to_bring_offline + + +def select_hot_tier_bricks_to_bring_offline(mnode, volname): + """Randomly selects bricks to bring offline without affecting the cluster + from a hot tier. + + Args: + mnode (str): Node on which commands will be executed. + volname (str): Name of the volume. + + Returns: + list: On success returns list of bricks that can be brough offline + from hot tier. If volume doesn't exist or is a non tiered volume + returns empty list. + """ + hot_tier_bricks_to_bring_offline = [] + + # Check if volume is tiered + if not is_tiered_volume(mnode, volname): + return hot_tier_bricks_to_bring_offline + + # get volume type + volume_type_info = get_volume_type_info(mnode, volname) + hot_tier_type = volume_type_info['hot_tier_type_info']['hotBrickType'] + + # get subvols + subvols_dict = get_subvols(mnode, volname) + hot_tier_subvols = subvols_dict['hot_tier_subvols'] + + # select bricks from distribute volume + if hot_tier_type == 'Distribute': + hot_tier_bricks_to_bring_offline = [] + + # select bricks from replicated, distributed-replicated volume + if (hot_tier_type == 'Replicate' or + hot_tier_type == 'Distributed-Replicate'): + # Get replica count + hot_tier_replica_count = (volume_type_info + ['hot_tier_type_info']['hotreplicaCount']) + + # Get quorum info + quorum_info = get_client_quorum_info(mnode, volname) + hot_tier_quorum_info = quorum_info['hot_tier_quorum_info'] + + # Get list of bricks to bring offline + hot_tier_bricks_to_bring_offline = ( + get_bricks_to_bring_offline_from_replicated_volume( + hot_tier_subvols, hot_tier_replica_count, + hot_tier_quorum_info)) + + return hot_tier_bricks_to_bring_offline + + +def select_cold_tier_bricks_to_bring_offline(mnode, volname): + """Randomly selects bricks to bring offline without affecting the cluster + from a cold tier. + + Args: + mnode (str): Node on which commands will be executed. + volname (str): Name of the volume. + + Returns: + list: On success returns list of bricks that can be brough offline + from cold tier. If volume doesn't exist or is a non tiered volume + returns empty list. + """ + cold_tier_bricks_to_bring_offline = [] + + # Check if volume is tiered + if not is_tiered_volume(mnode, volname): + return cold_tier_bricks_to_bring_offline + + # get volume type + volume_type_info = get_volume_type_info(mnode, volname) + cold_tier_type = volume_type_info['cold_tier_type_info']['coldBrickType'] + + # get subvols + subvols_dict = get_subvols(mnode, volname) + cold_tier_subvols = subvols_dict['cold_tier_subvols'] + + # select bricks from distribute volume + if cold_tier_type == 'Distribute': + cold_tier_bricks_to_bring_offline = [] + + # select bricks from replicated, distributed-replicated volume + elif (cold_tier_type == 'Replicate' or + cold_tier_type == 'Distributed-Replicate'): + # Get replica count + cold_tier_replica_count = (volume_type_info['cold_tier_type_info'] + ['coldreplicaCount']) + + # Get quorum info + quorum_info = get_client_quorum_info(mnode, volname) + cold_tier_quorum_info = quorum_info['cold_tier_quorum_info'] + + # Get list of bricks to bring offline + cold_tier_bricks_to_bring_offline = ( + get_bricks_to_bring_offline_from_replicated_volume( + cold_tier_subvols, cold_tier_replica_count, + cold_tier_quorum_info)) + + # select bricks from Disperse, Distribured-Disperse volume + elif (cold_tier_type == 'Disperse' or + cold_tier_type == 'Distributed-Disperse'): + + # Get redundancy count + cold_tier_redundancy_count = (volume_type_info['cold_tier_type_info'] + ['coldredundancyCount']) + + # Get list of bricks to bring offline + cold_tier_bricks_to_bring_offline = ( + get_bricks_to_bring_offline_from_disperse_volume( + cold_tier_subvols, cold_tier_redundancy_count)) + + return cold_tier_bricks_to_bring_offline + + +def get_bricks_to_bring_offline_from_replicated_volume(subvols_list, + replica_count, + quorum_info): + """Randomly selects bricks to bring offline without affecting the cluster + for a replicated volume. + + Args: + subvols_list: list of subvols. It can be volume_subvols, + hot_tier_subvols or cold_tier_subvols. + For example: + subvols = volume_libs.get_subvols(mnode, volname) + volume_subvols = subvols_dict['volume_subvols'] + hot_tier_subvols = subvols_dict['hot_tier_subvols'] + cold_tier_subvols = subvols_dict['cold_tier_subvols'] + replica_count: Replica count of a Replicate or Distributed-Replicate + volume. + quorum_info: dict containing quorum info of the volume. The dict should + have the following info: + - is_quorum_applicable, quorum_type, quorum_count + For example: + quorum_dict = get_client_quorum_info(mnode, volname) + volume_quorum_info = quorum_info['volume_quorum_info'] + hot_tier_quorum_info = quorum_info['hot_tier_quorum_info'] + cold_tier_quorum_info = quorum_info['cold_tier_quorum_info'] + + Returns: + list: List of bricks that can be brought offline without affecting the + cluster. On any failure return empty list. + """ + list_of_bricks_to_bring_offline = [] + try: + is_quorum_applicable = quorum_info['is_quorum_applicable'] + quorum_type = quorum_info['quorum_type'] + quorum_count = quorum_info['quorum_count'] + except KeyError: + g.log.error("Unable to get the proper quorum data from quorum info: " + "%s", quorum_info) + return list_of_bricks_to_bring_offline + + """ + offline_bricks_limit: Maximum Number of bricks that can be offline without + affecting the cluster + """ + if is_quorum_applicable: + if 'fixed' in quorum_type: + if quorum_count is None: + g.log.error("Quorum type is 'fixed' for the volume. But " + "Quorum count not specified. Invalid Quorum") + return list_of_bricks_to_bring_offline + else: + offline_bricks_limit = int(replica_count) - int(quorum_count) + + elif 'auto' in quorum_type: + offline_bricks_limit = ceil(int(replica_count) / 2) + + elif quorum_type is None: + offline_bricks_limit = int(replica_count) - 1 + + else: + g.log.error("Invalid Quorum Type : %s", quorum_type) + return list_of_bricks_to_bring_offline + + for subvol in subvols_list: + random.shuffle(subvol) + + # select a random count. + random_count = random.randint(1, offline_bricks_limit) + + # select random bricks. + bricks_to_bring_offline = random.sample(subvol, random_count) + + # Append the list with selected bricks to bring offline. + list_of_bricks_to_bring_offline.extend(bricks_to_bring_offline) + + return list_of_bricks_to_bring_offline + + +def get_bricks_to_bring_offline_from_disperse_volume(subvols_list, + redundancy_count): + """Randomly selects bricks to bring offline without affecting the cluster + for a disperse volume. + + Args: + subvols_list: list of subvols. It can be volume_subvols, + hot_tier_subvols or cold_tier_subvols. + For example: + subvols = volume_libs.get_subvols(mnode, volname) + volume_subvols = subvols_dict['volume_subvols'] + hot_tier_subvols = subvols_dict['hot_tier_subvols'] + cold_tier_subvols = subvols_dict['cold_tier_subvols'] + redundancy_count: Redundancy count of a Disperse or + Distributed-Disperse volume. + + Returns: + list: List of bricks that can be brought offline without affecting the + cluster.On any failure return empty list. + """ + list_of_bricks_to_bring_offline = [] + for subvol in subvols_list: + random.shuffle(subvol) + + # select a random value from 1 to redundancy_count. + random_count = random.randint(1, int(redundancy_count)) + + # select random bricks. + bricks_to_bring_offline = random.sample(subvol, random_count) + + # Append the list with selected bricks to bring offline. + list_of_bricks_to_bring_offline.extend(bricks_to_bring_offline) + + return list_of_bricks_to_bring_offline diff --git a/glustolibs-gluster/glustolibs/gluster/volume_libs.py b/glustolibs-gluster/glustolibs/gluster/volume_libs.py index 7b9dd5647..f7cdbd9ac 100644 --- a/glustolibs-gluster/glustolibs/gluster/volume_libs.py +++ b/glustolibs-gluster/glustolibs/gluster/volume_libs.py @@ -39,7 +39,7 @@ from glustolibs.gluster.uss_ops import enable_uss, is_uss_enabled from glustolibs.gluster.snap_ops import snap_delete_by_volumename from glustolibs.gluster.brick_libs import are_bricks_online, get_all_bricks from glustolibs.gluster.heal_libs import are_all_self_heal_daemons_are_online -from glustolibs.gluster.brick_ops import add_brick, remove_brick +from glustolibs.gluster.brick_ops import add_brick, remove_brick, replace_brick def volume_exists(mnode, volname): @@ -1545,3 +1545,290 @@ def shrink_volume(mnode, volname, subvol_num=None, replica_num=None, _, _, _ = g.run(brick_node, "rm -rf %s" % brick_path) return True + + +def replace_brick_from_volume(mnode, volname, servers, all_servers_info, + src_brick=None, dst_brick=None, + delete_brick=True, + replace_brick_from_hot_tier=False): + """Replace faulty brick from the volume. + + Args: + mnode (str): Node on which commands has to be executed + volname (str): volume name + servers (list): List of servers in the storage pool. + all_servers_info (dict): Information about all servers. + example : + all_servers_info = { + 'abc.lab.eng.xyz.com': { + 'host': 'abc.lab.eng.xyz.com', + 'brick_root': '/bricks', + 'devices': ['/dev/vdb', '/dev/vdc', '/dev/vdd', '/dev/vde'] + }, + 'def.lab.eng.xyz.com':{ + 'host': 'def.lab.eng.xyz.com', + 'brick_root': '/bricks', + 'devices': ['/dev/vdb', '/dev/vdc', '/dev/vdd', '/dev/vde'] + } + } + + Kwargs: + src_brick (str): Faulty brick which needs to be replaced + + dst_brick (str): New brick to replace the faulty brick + + delete_bricks (bool): After remove-brick delete the removed bricks. + + replace_brick_from_hot_tier (bool): True If brick are to be + replaced from hot_tier. False otherwise. Defaults to False. + + Returns: + bool: True if replacing brick from the volume is successful. + False otherwise. + """ + # Check if volume exists + if not volume_exists(mnode, volname): + g.log.error("Volume %s doesn't exists.", volname) + return False + + # Get Subvols + subvols_info = get_subvols(mnode, volname) + + if not dst_brick: + dst_brick = form_bricks_list(mnode=mnode, volname=volname, + number_of_bricks=1, + servers=servers, + servers_info=all_servers_info) + if not dst_brick: + g.log.error("Failed to get a new brick to replace the faulty " + "brick") + return False + dst_brick = dst_brick[0] + + if not src_brick: + # Randomly pick up a brick to bring the brick down and replace. + if subvols_info['is_tier']: + if replace_brick_from_hot_tier: + subvols_list = subvols_info['hot_tier_subvols'] + else: + subvols_list = subvols_info['cold_tier_subvols'] + else: + subvols_list = subvols_info['volume_subvols'] + + src_brick = (random.choice(random.choice(subvols_list))) + + # Brick the source brick offline + from glustolibs.gluster.brick_libs import bring_bricks_offline + g.log.info("Bringing brick %s offline of the volume %s", src_brick, + volname) + ret = bring_bricks_offline(volname, src_brick) + if not ret: + g.log.error("Unable to bring brick %s offline for replace-brick " + "operation on volume %s", src_brick, volname) + return False + g.log.info("Successfully brought the brick %s offline for replace-brick " + "operation on volume %s", src_brick, volname) + + # adding delay before performing replace-brick + time.sleep(15) + + # Log volume status before replace-brick + g.log.info("Logging volume status before performing replace-brick") + ret, _, _ = volume_status(mnode, volname) + if ret != 0: + g.log.error("Failed to get volume status before performing " + "replace-brick") + return False + + # Replace brick + g.log.info("Start replace-brick commit force of brick %s -> %s " + "on the volume %s", src_brick, dst_brick, volname) + ret, _, _ = replace_brick(mnode, volname, src_brick, dst_brick) + if ret != 0: + g.log.error("Failed to replace-brick commit force of brick %s -> %s " + "on the volume %s", src_brick, dst_brick, volname) + g.log.info("Start replace-brick commit force of brick %s -> %s " + "on the volume %s", src_brick, dst_brick, volname) + + # Delete the replaced brick + if delete_brick: + g.log.info("Deleting the replaced brick") + brick_node, brick_path = src_brick.split(":") + _, _, _ = g.run(brick_node, "rm -rf %s" % brick_path) + + return True + + +def get_client_quorum_info(mnode, volname): + """Get the client quorum information. i.e the quorum type, + quorum count. + Args: + mnode (str): Node on which commands are executed. + volname (str): Name of the volume. + + Returns: + dict: client quorum information for the volume. + client_quorum_dict = { + 'is_tier': False, + 'hot_tier_quorum_info':{ + 'is_quorum_applicable': False, + 'quorum_type': None, + 'quorum_count': None + }, + 'cold_tier_quorum_info':{ + 'is_quorum_applicable': False, + 'quorum_type': None, + 'quorum_count': None + }, + 'volume_quorum_info':{ + 'is_quorum_applicable': False, + 'quorum_type': None, + 'quorum_count': None + } + } + NoneType: None if volume doesnot exist. + """ + client_quorum_dict = { + 'is_tier': False, + 'hot_tier_quorum_info': { + 'is_quorum_applicable': False, + 'quorum_type': None, + 'quorum_count': None + }, + 'cold_tier_quorum_info': { + 'is_quorum_applicable': False, + 'quorum_type': None, + 'quorum_count': None + }, + 'volume_quorum_info': { + 'is_quorum_applicable': False, + 'quorum_type': None, + 'quorum_count': None + } + } + + # Get quorum-type + volume_option = get_volume_options(mnode, volname, 'cluster.quorum-type') + if volume_option is None: + g.log.error("Unable to get the volume option 'cluster.quorum-type' " + "for volume %s", volname) + return client_quorum_dict + quorum_type = volume_option['cluster.quorum-type'] + + # Get quorum-count + volume_option = get_volume_options(mnode, volname, 'cluster.quorum-count') + if volume_option is None: + g.log.error("Unable to get the volume option 'cluster.quorum-count' " + "for volume %s", volname) + return client_quorum_dict + quorum_count = volume_option['cluster.quorum-count'] + + # Set the quorum info + volume_type_info = get_volume_type_info(mnode, volname) + if volume_type_info['is_tier'] is True: + client_quorum_dict['is_tier'] = True + + # Hot Tier quorum info + hot_tier_type = volume_type_info['hot_tier_type_info']['hotBrickType'] + if (hot_tier_type == 'Replicate' or + hot_tier_type == 'Distributed-Replicate'): + + (client_quorum_dict['hot_tier_quorum_info'] + ['is_quorum_applicable']) = True + replica_count = (volume_type_info['hot_tier_type_info'] + ['hotreplicaCount']) + + # Case1: Replica 2 + if int(replica_count) == 2: + if 'none' not in quorum_type: + (client_quorum_dict['hot_tier_quorum_info'] + ['quorum_type']) = quorum_type + + if quorum_type == 'fixed': + if not quorum_count == '(null)': + (client_quorum_dict['hot_tier_quorum_info'] + ['quorum_count']) = quorum_count + + # Case2: Replica > 2 + if int(replica_count) > 2: + if quorum_type == 'none': + (client_quorum_dict['hot_tier_quorum_info'] + ['quorum_type']) = 'auto' + quorum_type == 'auto' + else: + (client_quorum_dict['hot_tier_quorum_info'] + ['quorum_type']) = quorum_type + if quorum_type == 'fixed': + if not quorum_count == '(null)': + (client_quorum_dict['hot_tier_quorum_info'] + ['quorum_count']) = quorum_count + + # Cold Tier quorum info + cold_tier_type = (volume_type_info['cold_tier_type_info'] + ['coldBrickType']) + if (cold_tier_type == 'Replicate' or + cold_tier_type == 'Distributed-Replicate'): + (client_quorum_dict['cold_tier_quorum_info'] + ['is_quorum_applicable']) = True + replica_count = (volume_type_info['cold_tier_type_info'] + ['coldreplicaCount']) + + # Case1: Replica 2 + if int(replica_count) == 2: + if 'none' not in quorum_type: + (client_quorum_dict['cold_tier_quorum_info'] + ['quorum_type']) = quorum_type + + if quorum_type == 'fixed': + if not quorum_count == '(null)': + (client_quorum_dict['cold_tier_quorum_info'] + ['quorum_count']) = quorum_count + + # Case2: Replica > 2 + if int(replica_count) > 2: + if quorum_type == 'none': + (client_quorum_dict['cold_tier_quorum_info'] + ['quorum_type']) = 'auto' + quorum_type == 'auto' + else: + (client_quorum_dict['cold_tier_quorum_info'] + ['quorum_type']) = quorum_type + if quorum_type == 'fixed': + if not quorum_count == '(null)': + (client_quorum_dict['cold_tier_quorum_info'] + ['quorum_count']) = quorum_count + else: + volume_type = (volume_type_info['volume_type_info']['typeStr']) + if (volume_type == 'Replicate' or + volume_type == 'Distributed-Replicate'): + (client_quorum_dict['volume_quorum_info'] + ['is_quorum_applicable']) = True + replica_count = (volume_type_info['volume_type_info'] + ['replicaCount']) + + # Case1: Replica 2 + if int(replica_count) == 2: + if 'none' not in quorum_type: + (client_quorum_dict['volume_quorum_info'] + ['quorum_type']) = quorum_type + + if quorum_type == 'fixed': + if not quorum_count == '(null)': + (client_quorum_dict['volume_quorum_info'] + ['quorum_count']) = quorum_count + + # Case2: Replica > 2 + if int(replica_count) > 2: + if quorum_type == 'none': + (client_quorum_dict['volume_quorum_info'] + ['quorum_type']) = 'auto' + quorum_type == 'auto' + else: + (client_quorum_dict['volume_quorum_info'] + ['quorum_type']) = quorum_type + if quorum_type == 'fixed': + if not quorum_count == '(null)': + (client_quorum_dict['volume_quorum_info'] + ['quorum_count']) = quorum_count + + return client_quorum_dict -- cgit