diff options
author | Pranith Kumar K <pkarampu@redhat.com> | 2019-05-27 19:00:20 +0530 |
---|---|---|
committer | Amar Tumballi <amarts@redhat.com> | 2019-05-30 15:55:36 +0000 |
commit | 2855c6247f7c5a2fafe1c403a112c38eca80d020 (patch) | |
tree | 5de4baf02c72bd6721131da17f69b8abc5038f92 | |
parent | 14d9c6179715205fb6ac816d4621dd087f5afbb2 (diff) |
scripts: Find hung frames given a directory with statedumps
Given a directory with statedumps captured at different times if
there are any stacks that appear in multiple statedumps, it prints
them.
Sample output:
glusterdump.25425.dump repeats=5 stack=0x7f53642cb968 pid=0 unique=0 lk-owner=
glusterdump.25427.dump repeats=5 stack=0x7f85002cb968 pid=0 unique=0 lk-owner=
glusterdump.25428.dump repeats=5 stack=0x7f962c2cb968 pid=0 unique=0 lk-owner=
glusterdump.25428.dump repeats=2 stack=0x7f962c329f18 pid=60830 unique=0 lk-owner=88f50620967f0000
glusterdump.25429.dump repeats=5 stack=0x7f20782cb968 pid=0 unique=0 lk-owner=
glusterdump.25472.dump repeats=5 stack=0x7f27ac2cb968 pid=0 unique=0 lk-owner=
glusterdump.25473.dump repeats=5 stack=0x7f4fbc2cb9d8 pid=0 unique=0 lk-owner=
NOTE: stacks with lk-owner=""/lk-owner=0000000000000000/unique=0 may not be hung frames and need further inspection
fixes bz#1714415
Change-Id: Ib64a3fca63f49df2fafedcd4baa57e9b25411b08
Signed-off-by: Pranith Kumar K <pkarampu@redhat.com>
-rwxr-xr-x | extras/identify-hangs.sh | 53 |
1 files changed, 53 insertions, 0 deletions
diff --git a/extras/identify-hangs.sh b/extras/identify-hangs.sh new file mode 100755 index 00000000000..ebc6bf144aa --- /dev/null +++ b/extras/identify-hangs.sh @@ -0,0 +1,53 @@ +#!/bin/bash +function get_statedump_fnames_without_timestamps +{ + ls | grep -E "[.]dump[.][0-9][0-9]*" | cut -f1-3 -d'.' | sort -u +} + +function get_non_uniq_fields +{ + local statedump_fname_prefix=$1 + print_stack_lkowner_unique_in_one_line "$statedump_fname_prefix" | sort | uniq -c | grep -vE "^\s*1 " | awk '{$1="repeats="$1; print $0}' +} + +function print_stack_lkowner_unique_in_one_line +{ + local statedump_fname_prefix=$1 + sed -e '/./{H;$!d;}' -e 'x;/unique=/!d;/stack=/!d;/lk-owner=/!d;/pid=/!d;' "${statedump_fname_prefix}"* | grep -E "(stack|lk-owner|unique|pid)=" | paste -d " " - - - - +} + +function get_stacks_that_appear_in_multiple_statedumps +{ + #If a stack with same 'unique/lk-owner/stack' appears in multiple statedumps + #print the stack + local statedump_fname_prefix=$1 + while read -r non_uniq_stack; + do + if [ -z "$printed" ]; + then + printed="1" + fi + echo "$statedump_fname_prefix" "$non_uniq_stack" + done < <(get_non_uniq_fields "$statedump_fname_prefix") +} + +statedumpdir=${1} +if [ -z "$statedumpdir" ]; +then + echo "Usage: $0 <statedump-dir>" + exit 1 +fi + +if [ ! -d "$statedumpdir" ]; +then + echo "$statedumpdir: Is not a directory" + echo "Usage: $0 <statedump-dir>" + exit 1 +fi + +cd "$statedumpdir" || exit 1 +for statedump_fname_prefix in $(get_statedump_fnames_without_timestamps); +do + get_stacks_that_appear_in_multiple_statedumps "$statedump_fname_prefix" +done | column -t +echo "NOTE: stacks with lk-owner=\"\"/lk-owner=0000000000000000/unique=0 may not be hung frames and need further inspection" >&2 |