fb-remote-test: Put 50 odds tests back into the pool

Summary:
- Identify 50 odd tests that can pass with higher timeout and 3 attempts
- sync fb-remote-test.sh with 3.6.
- correspnding fix to .arcconfig

Test Plan: Run --all manually

Differential Revision: https://phabricator.intern.facebook.com/D6062906

Change-Id: I01157248eb6543d8a4c8dbccfb9961f46de43ebb

3 files changed, 139 insertions, 169 deletions

diff --git a/.arcconfig b/.arcconfig
index fb594f62782..624665a59d9 100644
--- a/.arcconfig
+++ b/.arcconfig
@@ -1,7 +1,7 @@
 {
     "aliases" : {
-        "smoke" : [ "!./fb-remote-test.sh" ],
-        "test" : [ "!./fb-remote-test.sh --tests=all" ],
-        "asan-noleaks" : [ "!./fb-remote-test.sh --tests=all --asan-noleaks" ]
+        "smoke" : [ "!./fb-remote-test.sh --smoke" ],
+        "test" : [ "!./fb-remote-test.sh --all" ],
+        "asan-noleaks" : [ "!./fb-remote-test.sh --all --asan-noleaks" ]
     }
 }
diff --git a/fb-remote-test.sh b/fb-remote-test.sh
index 38648d84fde..13b923e037d 100755
--- a/fb-remote-test.sh
+++ b/fb-remote-test.sh
@@ -5,8 +5,10 @@ source ./test_env
 FBCODE="$HOME/fbsource/fbcode"
 N=0
 HOSTS=$(smcc ls-hosts -s gluster.build.ash gluster.build.prn | xargs)
-TESTS=$DESIRED_TESTS
+TESTS='all'
 FLAKY=$KNOWN_FLAKY_TESTS
+BROKEN=$BROKEN_TESTS
+TEST_TIMEOUT_S=900
 
 FLAGS=""
 
@@ -16,7 +18,8 @@ function print_env {
     echo "N=$N"
     echo -e "-------\nHOSTS\n$HOSTS\n-------"
     echo -e "TESTS\n$TESTS\n-------"
-    echo -e "FLAKY\n$FLAKY\n-------"
+    echo -e "SKIP\n$FLAKY $BROKEN\n-------"
+    echo -e "TEST_TIMEOUT_S=$TEST_TIMEOUT_S s\n"
 }
 
 function cleanup {
@@ -26,9 +29,11 @@ function cleanup {
 function usage {
     echo "Usage: $0 [-h or --help] [-v or --verbose]
              [--fbcode <fbcode root>]
+             [--all] [--flaky] [--smoke] [--broken]
              [--valgrind] [--asan] [--asan-noleaks]
              [--hosts <hosts>] [--smc-tier <tier name>] [-n <parallelism>]
-             [--tests <tests>] [--flaky <tests>]
+             [--tests <tests>]
+             [--id-rsa <ssh private key>]
     "
 }
 
@@ -40,10 +45,11 @@ function tiers_to_hosts {
     echo $hosts
 }
 
+
 function parse_args () {
     args=`getopt \
             -o hvn: \
-            --long help,verbose,valgrind,asan,asan-noleaks,fbcode:,hosts:,smc-tier:,tests:,flaky: \
+            --long help,verbose,valgrind,asan,asan-noleaks,all,smoke,flaky,broken,fbcode:,hosts:,smc-tier:,tests:,id-rsa:,test-timeout: \
             -n 'fb-remote-test.sh' --  "$@"`
 
     if [ $? != 0 ]; then
@@ -63,8 +69,13 @@ function parse_args () {
             --asan) FLAGS="$FLAGS --asan" ; shift ;;
             --hosts) HOSTS=$2; shift 2 ;;
             --smc-tier) HOSTS=$(tiers_to_hosts $2) ; shift 2 ;;
-            --tests) TESTS=$2; shift 2 ;;
-            --flaky) FLAKY=$2; shift 2 ;;
+            --tests) TESTS=$2; FLAKY= ; BROKEN= ; shift 2 ;;
+            --test-timeout) TEST_TIMEOUT_S=$2; shift 2 ;;
+            --all) TESTS='all' ; shift 1 ;;
+            --flaky) TESTS=$FLAKY; FLAKY= ; shift 1 ;;
+            --smoke) TESTS=$SMOKE_TESTS; shift 1 ;;
+            --broken) TESTS=$BROKEN_TESTS; FLAKY= ; BROKEN= ; shift 1 ;;
+            --id-rsa) FLAGS="$FLAGS --id-rsa $2" ; shift 2 ;;
             -n) N=$2; shift 2 ;;
             *) break ;;
             esac
@@ -85,7 +96,7 @@ function main {
     cleanup
 
     "$FBCODE/storage/gluster/gluster-build/fb-gluster-test.py" $FLAGS --tester \
-        --n "$N" --hosts "$HOSTS" --tests "$TESTS" --flaky_tests "$FLAKY"
+        --n "$N" --hosts "$HOSTS" --tests "$TESTS" --flaky_tests "$FLAKY $BROKEN" --test-timeout "$TEST_TIMEOUT_S"
 
     exit $?
 }
diff --git a/test_env b/test_env
index b868a6afb06..dcd9b5f57bd 100644
--- a/test_env
+++ b/test_env
@@ -1,167 +1,126 @@
 #!/bin/bash
 
-DESIRED_TESTS="\
-  tests/basic/*.t\
-  tests/basic/afr/*.t\
-  tests/basic/distribute/*.t\
-  tests/bugs/fb*.t\
-  tests/features/brick-min-free-space.t\
+SMOKE_TESTS="\
+    tests/basic/*.t\
+    tests/basic/afr/*.t\
+    tests/basic/distribute/*.t\
+    tests/bugs/fb*.t\
+    tests/features/brick-min-free-space.t\
 "
 
 KNOWN_FLAKY_TESTS="\
-   tests/bugs/glusterd/bug-1173414-mgmt-v3-remote-lock-failure.t\
-   tests/bugs/glusterd/bug-1420637-volume-sync-fix.t\
-   tests/bugs/glusterd/bug-1104642.t\
-   tests/bugs/glusterd/bug-1022055.t\
-   tests/bugs/glusterd/bug-1293414-import-brickinfo-uuid.t\
-   tests/bugs/transport/bug-873367.t\
-   tests/bugs/ec/bug-1161621.t\
-   tests/bugs/quota/bug-1287996.t\
-   tests/bugs/fb8149516.t\
-   tests/bugs/posix/bug-990028.t\
-   tests/bugs/glusterd/bug-1345727-bricks-stop-on-no-quorum-validation.t\
-   tests/bugs/core/bug-1402841.t-mt-dir-scan-race.t\
-   tests/bugs/write-behind/bug-1279730.t\
-   tests/bugs/cli/bug-1320388.t\
-   tests/bugs/snapshot/bug-1112613.t\
-   tests/bugs/snapshot/bug-1087203.t\
-   tests/bugs/snapshot/bug-1202436-calculate-quota-cksum-during-snap-restore.t\
-   tests/bugs/snapshot/bug-1205592.t\
-   tests/bugs/snapshot/bug-1140162-file-snapshot-features-encrypt-opts-validation.t\
-   tests/bugs/glusterd/bug-1231437-rebalance-test-in-cluster.t\
-   tests/bugs/snapshot/bug-1049834.t\
-   tests/bugs/shard/zero-flag.t\
-   tests/bugs/bitrot/1207029-bitrot-daemon-should-start-on-valid-node.t\
-   tests/bugs/bitrot/1209752-volume-status-should-show-bitrot-scrub-info.t\
-   tests/bugs/snapshot/bug-1399598-uss-with-ssl.t\
-   tests/bugs/tier/bug-1279376-rename-demoted-file.t\
-   tests/bugs/tier/bug-1286974.t\
-   tests/bugs/tier/bug-1205545-CTR-and-trash-integration.t\
-   tests/features/ipc.t\
-   tests/features/ssl-authz.t\
-   tests/bugs/glusterd/bug-948686.t\
-   tests/bugs/core/bug-986429.t\
-   tests/bugs/fb4482137.t\
-   tests/bugs/glusterd/bug-913555.t\
-   tests/basic/rpm.t\
-   tests/basic/accept-v6v4.t\
-   tests/basic/afr/granular-esh/granular-esh.t\
-   tests/basic/afr/granular-esh/cli.t\
-   tests/basic/afr/granular-esh/granular-indices-but-non-granular-heal.t\
-   tests/basic/afr/granular-esh/conservative-merge.t\
-   tests/basic/afr/granular-esh/add-brick.t\
-   tests/basic/afr/granular-esh/replace-brick.t\
-   tests/basic/bd.t tests/basic/uss.t\
-   tests/basic/glusterd/arbiter-volume-probe.t\
-   tests/basic/meta.t\
-   tests/basic/gfapi/bug1291259.t\
-   tests/basic/gfapi/gfapi-ssl-test.t\
-   tests/basic/tier/bug-1214222-directories_missing_after_attach_tier.t\
-   tests/basic/tier/record-metadata-heat.t\
-   tests/basic/tier/readdir-during-migration.t\
-   tests/basic/fops-sanity-gfproxy.t\
-   tests/basic/tier/ctr-rename-overwrite.t\
-   tests/basic/tier/frequency-counters.t\
-   tests/basic/mgmt_v3-locks.t\
-   tests/basic/tier/file_with_spaces.t\
-   tests/basic/tier/tier_lookup_heal.t\
-   tests/basic/glusterd/volfile_server_switch.t\
-   tests/basic/tier/tier-file-create.t\
-   tests/basic/tier/locked_file_migration.t\
-   tests/basic/tier/tier-snapshot.t\
-   tests/basic/volume-snapshot.t\
-   tests/basic/tier/new-tier-cmds.t\
-   tests/basic/quota-nfs.t\
-   tests/geo-rep/georep-basic-dr-rsync.t\
-   tests/basic/tier/unlink-during-migration.t\
-   tests/basic/tier/fops-during-migration-pause.t\
-   tests/basic/volume-snapshot-clone.t\
-   tests/bugs/nfs/bug-1166862.t\
-   tests/basic/tier/legacy-many.t\
-   tests/bugs/nfs/bug-1116503.t\
-   tests/bugs/nfs/bug-904065.t\
-   tests/bugs/rpc/bug-921072.t\
-   tests/bugs/rpc/bug-847624.t\
-   tests/bugs/glusterfs-server/bug-904300.t\
-   tests/bugs/replicate/886998/strict-readdir.t\
-   tests/basic/tier/fops-during-migration.t\
-   tests/basic/tier/tierd_check.t\
-   tests/basic/tier/tier.t\
-   tests/bugs/replicate/bug-1250170-fsync.t\
-   tests/basic/cache.t\
-   tests/geo-rep/georep-basic-dr-tarssh.t\
-   tests/bugs/replicate/bug-1190069-afr-stale-index-entries.t\
-   tests/bitrot/bug-1294786.t\
-   tests/bugs/quick-read/bug-846240.t\
-   tests/bugs/quota/afr-quota-xattr-mdata-heal.t\
-   tests/bugs/quota/bug-1288474.t\
-   tests/bugs/glusterd/bug-1344407-volume-delete-on-node-down.t\
-   tests/bugs/glusterd/859927/repl.t\
-   tests/bugs/glusterd/bug-1238706-daemons-stop-on-peer-cleanup.t\
-   tests/bugs/replicate/bug-1290965-detect-bitrotten-objects.t\
-   tests/bugs/glusterd/bug-1303028-Rebalance-glusterd-rpc-connection-issue.t\
-   tests/bugs/replicate/bug-859581.t\
-   tests/bugs/glusterd/bug-1047955.t\
-   tests/bugs/glusterd/bug-1213295-snapd-svc-uninitialized.t\
-   tests/bugs/glusterd/bug-1260185-donot-allow-detach-commit-unnecessarily.t\
-   tests/bugs/glusterd/bug-1230121-replica_subvol_count_correct_cal.t\
-   tests/bugs/glusterd/bug-1245045-remove-brick-validation.t\
-   tests/bugs/glusterd/bug-948729/bug-948729.t\
-   tests/bugs/glusterd/bug-948729/bug-948729-force.t\
-   tests/bugs/glusterd/bug-948729/bug-948729-mode-script.t\
-   tests/bugs/glusterd/bug-964059.t\
-   tests/bugs/glusterd/bug-888752.t\
-   tests/bugs/glusterd/bug-1177132-quorum-validation.t\
-   tests/bugs/glusterd/bug-889630.t\
-   tests/bugs/glusterd/bug-857330/xml.t\
-   tests/bugs/glusterd/bug-857330/normal.t
-   tests/bugs/glusterd/bug-1367478-volume-start-validation-after-glusterd-restart.t\
-   tests/bugs/glusterd/bug-1223213-peerid-fix.t\
-   tests/bugs/glusterd/bug-1245142-rebalance_test.t\
-   tests/bugs/glusterd/bug-1091935-brick-order-check-from-cli-to-glusterd.t\
-   tests/bugs/glusterd/bug-1323287-real_path-handshake-test.t\
-   tests/bugs/glusterd/bug-1266818-shared-storage-disable.t\
-   tests/bugs/replicate/bug-802417.t\
-   tests/bugs/glusterd/bug-1173414-mgmt-v3-remote-lock-failure.t\
-   tests/bugs/glusterd/bug-1420637-volume-sync-fix.t\
-   tests/bugs/glusterd/bug-1104642.t\
-   tests/bugs/glusterd/bug-1293414-import-brickinfo-uuid.t\
-   tests/bugs/glusterd/bug-1022055.t\
-   tests/bugs/transport/bug-873367.t\
-   tests/bugs/quota/bug-1287996.t\
-   tests/bugs/fb8149516.t\
-   tests/bugs/posix/bug-990028.t\
-   tests/bugs/glusterd/bug-1345727-bricks-stop-on-no-quorum-validation.t\
-   tests/bugs/core/bug-1402841.t-mt-dir-scan-race.t\
-   tests/bugs/write-behind/bug-1279730.t\
-   tests/bugs/cli/bug-1320388.t\
-   tests/bugs/ec/bug-1304988.t\
-   tests/bugs/glusterd/bug-948686.t\
-   tests/bugs/snapshot/bug-1112613.t\
-   tests/bugs/snapshot/bug-1087203.t\
-   tests/bugs/glusterd/bug-913555.t\
-   tests/bugs/snapshot/bug-1202436-calculate-quota-cksum-during-snap-restore.t\
-   tests/bugs/snapshot/bug-1140162-file-snapshot-features-encrypt-opts-validation.t\
-   tests/bugs/snapshot/bug-1205592.t tests/bugs/glusterd/bug-1231437-rebalance-test-in-cluster.t\
-   tests/bugs/snapshot/bug-1227646.t tests/bugs/shard/zero-flag.t\
-   tests/bugs/snapshot/bug-1399598-uss-with-ssl.t\
-   tests/bugs/snapshot/bug-1049834.t\
-   tests/bugs/core/bug-986429.t\
-   tests/bugs/bitrot/1207029-bitrot-daemon-should-start-on-valid-node.t\
-   tests/bugs/bitrot/1209752-volume-status-should-show-bitrot-scrub-info.t\
-   tests/bugs/fb4482137.t\
-   tests/bugs/tier/bug-1205545-CTR-and-trash-integration.t\
-   tests/features/ipc.t\
-   tests/bugs/tier/bug-1286974.t\
-   tests/bugs/tier/bug-1279376-rename-demoted-file.t\
-   tests/features/ssl-authz.t\
-   tests/bugs/glusterd/bug-857330/normal.t\
-   tests/bugs/distribute/bug-862967.t\
-   tests/basic/quota-anon-fd-nfs.t\
-   tests/basic/rpc-coverage.t\
-   tests/basic/afr/gfid-mismatch.t\
-   tests/basic/read-size.t\
+    tests/bugs/ec/bug-1161621.t\
+    tests/bugs/glusterd/bug-916549.t\
+    tests/bugs/posix/bug-1034716.t\
+    tests/bugs/core/bug-924075.t\
+    tests/bugs/replicate/bug-853680.t\
+    tests/basic/geo-replication/marker-xattrs.t\
+    tests/features/nuke.t\
+    tests/bugs/distribute-bug-1247563.t\
+    tests/basic/gfproxy.t\
+    tests/bugs/distribute/bug-1066798.t\
+    tests/bugs/bug-1110262.t\
 "
 
-DESIRED_TESTS=$(echo $DESIRED_TESTS | tr -s ' ' ' ')
+BROKEN_TESTS="\
+    tests/basic/read-size.t\
+    tests/bugs/glusterd/bug-857330/normal.t\
+    tests/bugs/tier/bug-1279376-rename-demoted-file.t\
+    tests/bugs/tier/bug-1286974.t\
+    tests/features/ipc.t\
+    tests/bugs/tier/bug-1205545-CTR-and-trash-integration.t\
+    tests/features/ssl-authz.t\
+    tests/bugs/shard/zero-flag.t\
+    tests/bugs/snapshot/bug-1140162-file-snapshot-features-encrypt-opts-validation.t\
+    tests/bugs/snapshot/bug-1227646.t\
+    tests/bugs/bitrot/1207029-bitrot-daemon-should-start-on-valid-node.t\
+    tests/bugs/glusterd/bug-948729/bug-948729-mode-script.t\
+    tests/bugs/transport/bug-873367.t\
+    tests/bugs/write-behind/bug-1279730.t\
+    tests/bugs/posix/bug-990028.t\
+    tests/bugs/glusterd/bug-857330/xml.t\
+    tests/bugs/glusterd/bug-948729/bug-948729.t\
+    tests/bugs/bitrot/1209752-volume-status-should-show-bitrot-scrub-info.t\
+    tests/bugs/snapshot/bug-1399598-uss-with-ssl.t\
+    tests/bugs/glusterd/bug-857330/normal.t\
+    tests/bugs/replicate/bug-859581.t\
+    tests/bugs/glusterd/bug-1260185-donot-allow-detach-commit-unnecessarily.t\
+    tests/bugs/glusterd/bug-1245045-remove-brick-validation.t\
+    tests/bugs/glusterd/bug-948729/bug-948729-force.t\
+    tests/bugs/glusterd/859927/repl.t\
+    tests/bugs/glusterd/bug-1238706-daemons-stop-on-peer-cleanup.t\
+    tests/bugs/quota/afr-quota-xattr-mdata-heal.t\
+    tests/bugs/cli/bug-1320388.t\
+    tests/bugs/replicate/886998/strict-readdir.t\
+    tests/basic/tier/tierd_check.t tests/bugs/glusterfs-server/bug-904300.t\
+    tests/basic/cache.t tests/bugs/quota/bug-1288474.t\
+    tests/bugs/replicate/bug-1290965-detect-bitrotten-objects.t\
+    tests/geo-rep/georep-basic-dr-tarssh.t\
+    tests/bugs/nfs/bug-904065.t\
+    tests/bugs/nfs/bug-1116503.t\
+    tests/bugs/rpc/bug-847624.t\
+    tests/basic/tier/fops-during-migration.t\
+    tests/bugs/nfs/bug-1166862.t\
+    tests/bugs/glusterd/bug-1303028-Rebalance-glusterd-rpc-connection-issue.t\
+    tests/basic/tier/tier.t\
+    tests/bugs/glusterd/bug-948686.t\
+    tests/bugs/glusterd/bug-913555.t\
+    tests/bugs/quota/bug-1287996.t\
+    tests/geo-rep/georep-basic-dr-rsync.t\
+    tests/bitrot/bug-1294786.t\
+    tests/basic/tier/tier-snapshot.t\
+    tests/bugs/fb4482137.t\
+    tests/basic/tier/unlink-during-migration.t\
+    tests/basic/tier/fops-during-migration-pause.t\
+    tests/basic/tier/tier-file-create.t\
+    tests/bugs/glusterd/bug-1231437-rebalance-test-in-cluster.t\
+    tests/basic/tier/legacy-many.t\
+    tests/basic/tier/new-tier-cmds.t\
+    tests/basic/tier/file_with_spaces.t\
+    tests/basic/tier/locked_file_migration.t\
+    tests/basic/tier/frequency-counters.t\
+    tests/basic/tier/tier_lookup_heal.t\
+    tests/basic/tier/bug-1214222-directories_missing_after_attach_tier.t\
+    tests/basic/tier/record-metadata-heat.t\
+    tests/basic/tier/readdir-during-migration.t\
+    tests/basic/gfapi/gfapi-ssl-test.t\
+    tests/basic/tier/ctr-rename-overwrite.t\
+    tests/basic/volume-snapshot-clone.t\
+    tests/basic/gfapi/bug1291259.t\
+    tests/bugs/bitrot/1207029-bitrot-daemon-should-start-on-valid-node.t\
+    tests/basic/afr/granular-esh/granular-esh.t\
+    tests/basic/afr/granular-esh/granular-indices-but-non-granular-heal.t\
+    tests/basic/bd.t\
+    tests/basic/afr/granular-esh/replace-brick.t\
+    tests/basic/afr/granular-esh/conservative-merge.t\
+    tests/basic/meta.t\
+    tests/bugs/tier/bug-1279376-rename-demoted-file.t\
+    tests/bugs/tier/bug-1205545-CTR-and-trash-integration.t\
+    tests/bugs/bitrot/1209752-volume-status-should-show-bitrot-scrub-info.t\
+    tests/basic/rpm.t\
+    tests/features/ipc.t\
+    tests/features/ssl-authz.t\
+    tests/basic/afr/granular-esh/add-brick.t\
+    tests/basic/accept-v6v4.t\
+    tests/bugs/shard/zero-flag.t\
+    tests/basic/afr/granular-esh/cli.t\
+    tests/basic/uss.t\
+    tests/bugs/tier/bug-1286974.t\
+    tests/bugs/snapshot/bug-1399598-uss-with-ssl.t\
+    tests/bugs/snapshot/bug-1140162-file-snapshot-features-encrypt-opts-validation.t\
+    tests/bugs/write-behind/bug-1279730.t\
+    tests/basic/volume-snapshot.t\
+    tests/bugs/glusterd/bug-913555.t\
+    tests/bugs/cli/bug-1320388.t\
+    tests/bugs/posix/bug-990028.t\
+    tests/bugs/transport/bug-873367.t\
+    tests/basic/glusterd/volfile_server_switch.t\
+    tests/bugs/glusterd/bug-948686.t\
+    tests/bugs/fb4482137.t\
+    tests/bugs/glusterd/bug-1231437-rebalance-test-in-cluster.t \
+    tests/bugs/quota/bug-1287996.t\
+"
+
+SMOKE_TESTS=$(echo $SMOKE_TESTS | tr -s ' ' ' ')
 KNOWN_FLAKY_TESTS=$(echo $KNOWN_FLAKY_TESTS | tr -s ' ' ' ')
+BROKEN_TESTS=$(echo $BROKEN_TESTS | tr -s ' ' ' ')