scripts: Find hung frames given a directory with statedumps

Given a directory with statedumps captured at different times if
there are any stacks that appear in multiple statedumps, it prints
them.

Sample output:
glusterdump.25425.dump  repeats=5  stack=0x7f53642cb968  pid=0      unique=0  lk-owner=
glusterdump.25427.dump  repeats=5  stack=0x7f85002cb968  pid=0      unique=0  lk-owner=
glusterdump.25428.dump  repeats=5  stack=0x7f962c2cb968  pid=0      unique=0  lk-owner=
glusterdump.25428.dump  repeats=2  stack=0x7f962c329f18  pid=60830  unique=0  lk-owner=88f50620967f0000
glusterdump.25429.dump  repeats=5  stack=0x7f20782cb968  pid=0      unique=0  lk-owner=
glusterdump.25472.dump  repeats=5  stack=0x7f27ac2cb968  pid=0      unique=0  lk-owner=
glusterdump.25473.dump  repeats=5  stack=0x7f4fbc2cb9d8  pid=0      unique=0  lk-owner=
NOTE: stacks with lk-owner=""/lk-owner=0000000000000000/unique=0 may not be hung frames and need further inspection

fixes bz#1714415
Change-Id: Ib64a3fca63f49df2fafedcd4baa57e9b25411b08
Signed-off-by: Pranith Kumar K <pkarampu@redhat.com>

1 files changed, 53 insertions, 0 deletions

diff --git a/extras/identify-hangs.sh b/extras/identify-hangs.sh
new file mode 100755
index 00000000000..ebc6bf144aa
--- /dev/null
+++ b/extras/identify-hangs.sh
@@ -0,0 +1,53 @@
+#!/bin/bash
+function get_statedump_fnames_without_timestamps
+{
+    ls | grep -E "[.]dump[.][0-9][0-9]*" | cut -f1-3 -d'.' | sort -u
+}
+
+function get_non_uniq_fields
+{
+    local statedump_fname_prefix=$1
+    print_stack_lkowner_unique_in_one_line "$statedump_fname_prefix" | sort | uniq -c | grep -vE "^\s*1 " | awk '{$1="repeats="$1; print $0}'
+}
+
+function print_stack_lkowner_unique_in_one_line
+{
+    local statedump_fname_prefix=$1
+    sed -e '/./{H;$!d;}' -e 'x;/unique=/!d;/stack=/!d;/lk-owner=/!d;/pid=/!d;' "${statedump_fname_prefix}"* | grep -E "(stack|lk-owner|unique|pid)=" | paste -d " " - - - -
+}
+
+function get_stacks_that_appear_in_multiple_statedumps
+{
+    #If a stack with same 'unique/lk-owner/stack' appears in multiple statedumps
+    #print the stack
+    local statedump_fname_prefix=$1
+    while read -r non_uniq_stack;
+    do
+        if [ -z "$printed" ];
+        then
+            printed="1"
+        fi
+        echo "$statedump_fname_prefix" "$non_uniq_stack"
+    done < <(get_non_uniq_fields "$statedump_fname_prefix")
+}
+
+statedumpdir=${1}
+if [ -z "$statedumpdir" ];
+then
+    echo "Usage: $0 <statedump-dir>"
+    exit 1
+fi
+
+if [ ! -d "$statedumpdir" ];
+then
+    echo "$statedumpdir: Is not a directory"
+    echo "Usage: $0 <statedump-dir>"
+    exit 1
+fi
+
+cd "$statedumpdir" || exit 1
+for statedump_fname_prefix in $(get_statedump_fnames_without_timestamps);
+do
+    get_stacks_that_appear_in_multiple_statedumps "$statedump_fname_prefix"
+done | column -t
+echo "NOTE: stacks with lk-owner=\"\"/lk-owner=0000000000000000/unique=0 may not be hung frames and need further inspection" >&2