#!/bin/bash function geo_rep_checkpoint_status() { echo "Verifying the sync status using geo-rep checkpoint" >> $LOG_FILE local timeout=300 local temp_status="NOTOK" echo "setting the checkpoint" >> $LOG_FILE $CLI volume geo-rep $GMV0 $H0::$GSV0 config checkpoint now >> $LOG_FILE 2>&1 # There is a bug, where in after checkpoint set, geo-rep status still # shows the old data for the first execution of geo-rep status. Running #geo-rep status to clear that. $CLI volume geo-replication $GMV0 $H0::$GSV0 status >> $LOG_FILE 2>&1 while [ $timeout -gt 0 ] && [ $temp_status == "NOTOK" ]; do $CLI volume geo-replication $GMV0 $H0::$GSV0 status | \ egrep -i "not reached yet" 2>&1 >/dev/null test $? -ne 0 && temp_status="completed" echo "Waiting for the files to sync ..." >> $LOG_FILE sleep 20 timeout=`expr $timeout - 20` echo "temp_status is $temp_status" >> $LOG_FILE echo "geo-rep status output:" >> $LOG_FILE $CLI volume geo-replication $GMV0 $H0::$GSV0 status detail >> \ $LOG_FILE 2>&1 done echo "resetting the geo-rep checkpoint" >> $LOG_FILE $CLI volume geo-rep $GMV0 $H0::$GSV0 config \!checkpoint >> $LOG_FILE 2>&1 if test $temp_status = "completed" ; then echo "geo-rep checkpoint has completed" >> $LOG_FILE return 0 elif test $temp_status = "NOTOK" ; then echo "geo-rep checkpoint has failed to complete within 300 seconds" >> \ $LOG_FILE return 1 fi } function geo_rep_arequal_status() { echo "Verifying the sync status using arequal" >> $LOG_FILE local timeout=300 local temp_status="NOTOK" local comp_arequal="$(dirname $0)/compare-arequal.py" while [ $timeout -gt 0 ] && [ $temp_status == "NOTOK" ]; do echo "Waiting for the files to sync ..." >> $LOG_FILE sleep 20 timeout=`expr $timeout - 20` echo "calculating and comparing arequal checksum between $GMV0 and \ $GSV0 " >> $LOG_FILE $PYTHON $comp_arequal $H0::$GMV0 $H0::$GSV0 >> $LOG_FILE 2>&1 local ret=$? # There is a bug, where sometimes metadata checksum of directories # and regular files don't match. This is to avoid that for now. if [[ $ret -eq 0 || $ret -eq 67 || $ret -eq 68 ]] ;then temp_status="completed" fi done if test $temp_status = "completed" ; then echo "checksum between master and slave match " >> $LOG_FILE return 0 elif test $temp_status = "NOTOK" ; then echo "checksum between master and slave doesn't match" >> $LOG_FILE return 1 fi } function geo_rep_filecount_status() { echo "Verifying the sync status through files count" >> $LOG_FILE local timeout=300 local temp_status="NOTOK" local comp_arequal="$(dirname $0)/compare-arequal.py" while [ $timeout -gt 0 ] && [ $temp_status == "NOTOK" ]; do echo "Waiting for the files to sync ..." >> $LOG_FILE sleep 20 timeout=`expr $timeout - 20` echo "calculating and comparing files count between $GMV0 and \ $GSV0 " >> $LOG_FILE $PYTHON $comp_arequal -c "find" $H0::$GMV0 $H0::$GSV0 >> $LOG_FILE 2>&1 if [ $? -eq 0 ];then temp_status="completed" fi done if test $temp_status = "completed" ; then echo "files count between master and slave match " >> $LOG_FILE return 0 elif test $temp_status = "NOTOK" ; then echo "files count between master and slave doesn't match" >> $LOG_FILE return 1 fi } function check_status_arequal() { # checkpoint is failing to reach even though all the files got synced in the latest build. # Hence not using checkpoint to check for sync status. # geo_rep_checkpoint_status local comp_arequal="$(dirname $0)/compare-arequal.py" local comp_gfid="$(dirname $0)/compare-gfid.py" geo_rep_filecount_status geo_rep_arequal_status echo "calculating and comparing gfids between $GMV0 and $GSV0 " \ >> $LOG_FILE $PYTHON $comp_gfid $H0::$GMV0 $H0::$GSV0 >> $LOG_FILE 2>&1 if [ $? != 0 ]; then return 1 else echo "gfids between master and slave match" >> $LOG_FILE fi echo "calculating and comparing arequal checksum between $GMV0 and $GSV0 " \ >> $LOG_FILE $PYTHON $comp_arequal $H0::$GMV0 $H0::$GSV0 >> $LOG_FILE 2>&1 local rett=$? if [[ $rett -eq 0 || $rett -eq 67 || $rett -eq 68 ]] ;then reta=0 else reta=1 fi return $reta } function create_data() { fop=$1 MNT_PNT=$2 create_data="$(dirname $0)/../utils/create-files.py" if [ $DIR_STR == "MULTI" ];then $PYTHON $create_data -n $nf --multi -b 10 -d 10 --random --max=2K \ --min=1K -t $FILE_TYPE --fop=$fop $MNT_PNT >> $LOG_FILE 2>&1 elif [ $DIR_STR == "SINGLE" ];then $PYTHON $create_data -n $ns --random --max=2K --min=1K -t $FILE_TYPE \ --fop=$fop $MNT_PNT >> $LOG_FILE 2>&1 else echo "Wrong option for the create-files" >> $LOG_FILE fi } function result() { local ret=$1 local test=$2 if [ $ret -ne 0 ]; then echo -e "\n[ FAIL ] : $test has failed" >> $LOG_FILE exit 1 else echo -e "\n[ PASS ] : $test has passed" >> $LOG_FILE fi } ## hybrid crawl test-cases function hybrid_mode_test() { local FOP=$1 local MNT_PNT=$2 echo -e "\n:::::::::::::::::::::::" >> $LOG_FILE echo "Start of hybrid-mode-$DIR_STR-$FILE_TYPE-$FOP-test with \ $MNT_PNT client" >> $LOG_FILE echo -e ":::::::::::::::::::::::\n" >> $LOG_FILE local ret=0 echo "stopping geo-rep session before creating data" >> $LOG_FILE $CLI volume geo-rep $GMV0 $H0::$GSV0 stop force >> $LOG_FILE 2>&1 if [ $? -ne 0 ]; then echo "stopping geo-rep session has failed" >> $LOG_FILE return 1 fi create_data $FOP $MNT_PNT $CLI volume geo-rep $GMV0 $H0::$GSV0 start >> $LOG_FILE 2>&1 if [ $? -ne 0 ]; then echo "starting geo-rep session has failed" >> $LOG_FILE return 1 fi check_status_arequal if [ $? -ne 0 ]; then ret=1 fi result $ret "hybrid-mode-$DIR_STR-$FILE_TYPE-$FOP-test with $CLIENT client" return $ret } #### Changelog based test-cases function changelog_mode_test() { local FOP=$1 local MNT_PNT=$2 echo -e "\n:::::::::::::::::::::::" >> $LOG_FILE echo "Start of changelog-mode-$DIR_STR-$FILE_TYPE-$FOP-test with \ $MNT_PNT client" >> $LOG_FILE echo -e ":::::::::::::::::::::::\n" >> $LOG_FILE local ret=0 create_data $FOP $MNT_PNT check_status_arequal if [ $? -ne 0 ]; then ret=1 fi result $ret "basic-changelog-$DIR_STR-$FILE_TYPE-$FOP-test with $CLIENT \ client" return $ret } function changelog_mode_remove_test() { MNT_PNT=$1 echo -e "\n:::::::::::::::::::::::" >> $LOG_FILE echo "Start of changelog-mode-$DIR_STR-$FILE_TYPE-remove-test with \ $MNT_PNT client" >> $LOG_FILE echo -e ":::::::::::::::::::::::\n" >> $LOG_FILE local ret=0 if [ ! -z $MNT_PNT ]; then rm -rvf $MNT_PNT >> $LOG_FILE else echo "Value of MNT_PNT is NULL" >> $LOG_FILE fi check_status_arequal if [ $? -ne 0 ]; then ret=1 fi result $ret "chnagelog-mode-$DIR_STR-$FILE_TYPE-remove-test with \ $MNT_PNT client" return $ret }