diff options
| -rwxr-xr-x | extras/profiler/glusterfs-profiler | 641 |
1 files changed, 599 insertions, 42 deletions
diff --git a/extras/profiler/glusterfs-profiler b/extras/profiler/glusterfs-profiler index f843ae69a..7a5ca7405 100755 --- a/extras/profiler/glusterfs-profiler +++ b/extras/profiler/glusterfs-profiler @@ -1,5 +1,425 @@ #!/usr/bin/env python +# texttable - module for creating simple ASCII tables +# Copyright (C) 2003-2009 Gerome Fournier <jefke(at)free.fr> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +# Incorporated from texttable.py downloaded from +# http://jefke.free.fr/stuff/python/texttable/texttable-0.7.0.tar.gz + +import sys +import string + +try: + if sys.version >= '2.3': + import textwrap + elif sys.version >= '2.2': + from optparse import textwrap + else: + from optik import textwrap +except ImportError: + sys.stderr.write("Can't import textwrap module!\n") + raise + +try: + True, False +except NameError: + (True, False) = (1, 0) + +def len(iterable): + """Redefining len here so it will be able to work with non-ASCII characters + """ + if not isinstance(iterable, str): + return iterable.__len__() + + try: + return len(unicode(iterable, 'utf')) + except: + return iterable.__len__() + +class ArraySizeError(Exception): + """Exception raised when specified rows don't fit the required size + """ + + def __init__(self, msg): + self.msg = msg + Exception.__init__(self, msg, '') + + def __str__(self): + return self.msg + +class Texttable: + + BORDER = 1 + HEADER = 1 << 1 + HLINES = 1 << 2 + VLINES = 1 << 3 + + def __init__(self, max_width=80): + """Constructor + + - max_width is an integer, specifying the maximum width of the table + - if set to 0, size is unlimited, therefore cells won't be wrapped + """ + + if max_width <= 0: + max_width = False + self._max_width = max_width + self._deco = Texttable.VLINES | Texttable.HLINES | Texttable.BORDER | \ + Texttable.HEADER + self.set_chars(['-', '|', '+', '=']) + self.reset() + + def reset(self): + """Reset the instance + + - reset rows and header + """ + + self._hline_string = None + self._row_size = None + self._header = [] + self._rows = [] + + def header(self, array): + """Specify the header of the table + """ + + self._check_row_size(array) + self._header = map(str, array) + + def add_row(self, array): + """Add a row in the rows stack + + - cells can contain newlines and tabs + """ + + self._check_row_size(array) + self._rows.append(map(str, array)) + + def add_rows(self, rows, header=True): + """Add several rows in the rows stack + + - The 'rows' argument can be either an iterator returning arrays, + or a by-dimensional array + - 'header' specifies if the first row should be used as the header + of the table + """ + + # nb: don't use 'iter' on by-dimensional arrays, to get a + # usable code for python 2.1 + if header: + if hasattr(rows, '__iter__') and hasattr(rows, 'next'): + self.header(rows.next()) + else: + self.header(rows[0]) + rows = rows[1:] + for row in rows: + self.add_row(row) + + def set_chars(self, array): + """Set the characters used to draw lines between rows and columns + + - the array should contain 4 fields: + + [horizontal, vertical, corner, header] + + - default is set to: + + ['-', '|', '+', '='] + """ + + if len(array) != 4: + raise ArraySizeError, "array should contain 4 characters" + array = [ x[:1] for x in [ str(s) for s in array ] ] + (self._char_horiz, self._char_vert, + self._char_corner, self._char_header) = array + + def set_deco(self, deco): + """Set the table decoration + + - 'deco' can be a combinaison of: + + Texttable.BORDER: Border around the table + Texttable.HEADER: Horizontal line below the header + Texttable.HLINES: Horizontal lines between rows + Texttable.VLINES: Vertical lines between columns + + All of them are enabled by default + + - example: + + Texttable.BORDER | Texttable.HEADER + """ + + self._deco = deco + + def set_cols_align(self, array): + """Set the desired columns alignment + + - the elements of the array should be either "l", "c" or "r": + + * "l": column flushed left + * "c": column centered + * "r": column flushed right + """ + + self._check_row_size(array) + self._align = array + + def set_cols_valign(self, array): + """Set the desired columns vertical alignment + + - the elements of the array should be either "t", "m" or "b": + + * "t": column aligned on the top of the cell + * "m": column aligned on the middle of the cell + * "b": column aligned on the bottom of the cell + """ + + self._check_row_size(array) + self._valign = array + + def set_cols_width(self, array): + """Set the desired columns width + + - the elements of the array should be integers, specifying the + width of each column. For example: + + [10, 20, 5] + """ + + self._check_row_size(array) + try: + array = map(int, array) + if reduce(min, array) <= 0: + raise ValueError + except ValueError: + sys.stderr.write("Wrong argument in column width specification\n") + raise + self._width = array + + def draw(self): + """Draw the table + + - the table is returned as a whole string + """ + + if not self._header and not self._rows: + return + self._compute_cols_width() + self._check_align() + out = "" + if self._has_border(): + out += self._hline() + if self._header: + out += self._draw_line(self._header, isheader=True) + if self._has_header(): + out += self._hline_header() + length = 0 + for row in self._rows: + length += 1 + out += self._draw_line(row) + if self._has_hlines() and length < len(self._rows): + out += self._hline() + if self._has_border(): + out += self._hline() + return out[:-1] + + def _check_row_size(self, array): + """Check that the specified array fits the previous rows size + """ + + if not self._row_size: + self._row_size = len(array) + elif self._row_size != len(array): + raise ArraySizeError, "array should contain %d elements" \ + % self._row_size + + def _has_vlines(self): + """Return a boolean, if vlines are required or not + """ + + return self._deco & Texttable.VLINES > 0 + + def _has_hlines(self): + """Return a boolean, if hlines are required or not + """ + + return self._deco & Texttable.HLINES > 0 + + def _has_border(self): + """Return a boolean, if border is required or not + """ + + return self._deco & Texttable.BORDER > 0 + + def _has_header(self): + """Return a boolean, if header line is required or not + """ + + return self._deco & Texttable.HEADER > 0 + + def _hline_header(self): + """Print header's horizontal line + """ + + return self._build_hline(True) + + def _hline(self): + """Print an horizontal line + """ + + if not self._hline_string: + self._hline_string = self._build_hline() + return self._hline_string + + def _build_hline(self, is_header=False): + """Return a string used to separated rows or separate header from + rows + """ + horiz = self._char_horiz + if (is_header): + horiz = self._char_header + # compute cell separator + s = "%s%s%s" % (horiz, [horiz, self._char_corner][self._has_vlines()], + horiz) + # build the line + l = string.join([horiz*n for n in self._width], s) + # add border if needed + if self._has_border(): + l = "%s%s%s%s%s\n" % (self._char_corner, horiz, l, horiz, + self._char_corner) + else: + l += "\n" + return l + + def _len_cell(self, cell): + """Return the width of the cell + + Special characters are taken into account to return the width of the + cell, such like newlines and tabs + """ + + cell_lines = cell.split('\n') + maxi = 0 + for line in cell_lines: + length = 0 + parts = line.split('\t') + for part, i in zip(parts, range(1, len(parts) + 1)): + length = length + len(part) + if i < len(parts): + length = (length/8 + 1)*8 + maxi = max(maxi, length) + return maxi + + def _compute_cols_width(self): + """Return an array with the width of each column + + If a specific width has been specified, exit. If the total of the + columns width exceed the table desired width, another width will be + computed to fit, and cells will be wrapped. + """ + + if hasattr(self, "_width"): + return + maxi = [] + if self._header: + maxi = [ self._len_cell(x) for x in self._header ] + for row in self._rows: + for cell,i in zip(row, range(len(row))): + try: + maxi[i] = max(maxi[i], self._len_cell(cell)) + except (TypeError, IndexError): + maxi.append(self._len_cell(cell)) + items = len(maxi) + length = reduce(lambda x,y: x+y, maxi) + if self._max_width and length + items*3 + 1 > self._max_width: + maxi = [(self._max_width - items*3 -1) / items \ + for n in range(items)] + self._width = maxi + + def _check_align(self): + """Check if alignment has been specified, set default one if not + """ + + if not hasattr(self, "_align"): + self._align = ["l"]*self._row_size + if not hasattr(self, "_valign"): + self._valign = ["t"]*self._row_size + + def _draw_line(self, line, isheader=False): + """Draw a line + + Loop over a single cell length, over all the cells + """ + + line = self._splitit(line, isheader) + space = " " + out = "" + for i in range(len(line[0])): + if self._has_border(): + out += "%s " % self._char_vert + length = 0 + for cell, width, align in zip(line, self._width, self._align): + length += 1 + cell_line = cell[i] + fill = width - len(cell_line) + if isheader: + align = "c" + if align == "r": + out += "%s " % (fill * space + cell_line) + elif align == "c": + out += "%s " % (fill/2 * space + cell_line \ + + (fill/2 + fill%2) * space) + else: + out += "%s " % (cell_line + fill * space) + if length < len(line): + out += "%s " % [space, self._char_vert][self._has_vlines()] + out += "%s\n" % ['', self._char_vert][self._has_border()] + return out + + def _splitit(self, line, isheader): + """Split each element of line to fit the column width + + Each element is turned into a list, result of the wrapping of the + string to the desired width + """ + + line_wrapped = [] + for cell, width in zip(line, self._width): + array = [] + for c in cell.split('\n'): + array.extend(textwrap.wrap(unicode(c, 'utf'), width)) + line_wrapped.append(array) + max_cell_lines = reduce(max, map(len, line_wrapped)) + for cell, valign in zip(line_wrapped, self._valign): + if isheader: + valign = "t" + if valign == "m": + missing = max_cell_lines - len(cell) + cell[:0] = [""] * (missing / 2) + cell.extend([""] * (missing / 2 + missing % 2)) + elif valign == "b": + cell[:0] = [""] * (max_cell_lines - len(cell)) + else: + cell.extend([""] * (max_cell_lines - len(cell))) + return line_wrapped + + # Copyright (c) 2010 Gluster, Inc. <http://www.gluster.com> # This file is part of GlusterFS. @@ -17,8 +437,14 @@ # along with this program. If not, see # <http://www.gnu.org/licenses/>. -import numpy as np -import matplotlib.pyplot as plt +graph_available = True + +try: + import numpy as np + import matplotlib.pyplot as plt +except ImportError: + graph_available = False + import re import sys @@ -87,7 +513,7 @@ def calc_latency_heights (xlator_order): # have sufficient number of colors colors = ["violet", "blue", "green", "yellow", "orange", "red"] -def latency_profile (title, xlator_order): +def latency_profile (title, xlator_order, mode): heights = calc_latency_heights (xlator_order) N = len (latencies[xlator_order[0]].keys()) @@ -100,30 +526,75 @@ def latency_profile (title, xlator_order): bottoms[Nxl-1] = map (lambda x: 0, latencies[xlator_order[0]].keys()) + k = latencies[xlator_order[0]].keys() + k.sort() + for i in range (Nxl-1): xl = xlator_order[i+1] - k = latencies[xl].keys() - k.sort() - bottoms[i] = [float(latencies[xl][key]) for key in k] - for i in range(Nxl): - pieces[i] = plt.bar (ind, heights[i], width, color=colors[i], - bottom=bottoms[i]) + if mode == 'text': + print "\n%sLatency profile for %s\n" % (' '*20, title) + print "Average latency (microseconds):\n" - plt.ylabel ("Average Latency (microseconds)") - plt.title ("Latency Profile for '%s'" % title) - k = latencies[xlator_order[0]].keys() - k.sort () - plt.xticks (ind+width/2., k) + table = Texttable() + + table.set_cols_align(["l", "r"] + ["r"] * len(xlator_order)) + rows = [] + + header = ['OP', 'OP Average (us)'] + xlator_order + rows = [] + + for op in k: + sum = reduce (lambda x, y: x + y, [heights[xlator_order.index(xl)][k.index(op)] for xl in xlator_order], + 0) + + row = [op] + row += ["%5.2f" % sum] + + for xl in xlator_order: + op_index = k.index(op) + row += ["%5.2f" % (heights[xlator_order.index(xl)][op_index])] + + rows.append(row) + + def row_sort(r1, r2): + v1 = float(r1[1]) + v2 = float(r2[1]) - m = round (max(map (float, latencies[xlator_order[0]].values())), -2) - plt.yticks (np.arange(0, m + m*0.1, m/10)) - plt.legend (map (lambda p: p[0], pieces), xlator_order) + if v1 < v2: + return -1 + elif v1 == v2: + return 0 + else: + return 1 - plt.show () + rows.sort(row_sort, reverse=True) + table.add_rows([header] + rows) + print table.draw() -def fop_distribution (title, xlator_order): + elif mode == 'graph': + for i in range(Nxl): + pieces[i] = plt.bar (ind, heights[i], width, color=colors[i], + bottom=bottoms[i]) + + plt.ylabel ("Average Latency (microseconds)") + plt.title ("Latency Profile for '%s'" % title) + k = latencies[xlator_order[0]].keys() + k.sort () + plt.xticks (ind+width/2., k) + + m = round (max(map (float, latencies[xlator_order[0]].values())), -2) + plt.yticks (np.arange(0, m + m*0.1, m/10)) + plt.legend (map (lambda p: p[0], pieces), xlator_order) + + plt.show () + else: + print "Unknown mode specified!" + sys.exit(1) + + +def fop_distribution (title, xlator_order, mode): plt.ylabel ("Percentage of calls") plt.title ("FOP distribution for '%s'" % title) k = counts[xlator_order[0]].keys() @@ -143,17 +614,52 @@ def fop_distribution (title, xlator_order): for op in k: heights.append (float(counts[top_xl][op])/total * 100) - bars = plt.bar (ind, heights, width, color="red") + if mode == 'text': + print "\n%sFOP distribution for %s\n" % (' '*20, title) + print "Total number of calls: %d\n" % total + + table = Texttable() + + table.set_cols_align(["l", "r", "r"]) + + rows = [] + header = ["OP", "% of Calls", "Count"] - for bar in bars: - height = bar.get_height() - plt.text (bar.get_x()+bar.get_width()/2., 1.05*height, - "%d%%" % int(height)) + for op in k: + row = [op, "%5.2f" % (float(counts[top_xl][op])/total * 100), counts[top_xl][op]] + rows.append(row) - plt.xticks(ind+width/2., k) - plt.yticks(np.arange (0, 110, 10)) + def row_sort(r1, r2): + v1 = float(r1[1]) + v2 = float(r2[1]) + + if v1 < v2: + return -1 + elif v1 == v2: + return 0 + else: + return 1 + + rows.sort(row_sort, reverse=True) + table.add_rows([header] + rows) + print table.draw() + + elif mode == 'graph': + bars = plt.bar (ind, heights, width, color="red") + + for bar in bars: + height = bar.get_height() + plt.text (bar.get_x()+bar.get_width()/2., 1.05*height, + "%d%%" % int(height)) + + plt.xticks(ind+width/2., k) + plt.yticks(np.arange (0, 110, 10)) + + plt.show() + else: + print "mode not specified!" + sys.exit(1) - plt.show() def calc_workload_heights (xlator_order, scaling): workload_heights = map (lambda x: [], xlator_order) @@ -181,7 +687,7 @@ def calc_workload_heights (xlator_order, scaling): return workload_heights -def workload_profile(title, xlator_order): +def workload_profile(title, xlator_order, mode): plt.ylabel ("Percentage of Total Time") plt.title ("Workload Profile for '%s'" % title) k = totals[xlator_order[0]].keys() @@ -216,20 +722,63 @@ def workload_profile(title, xlator_order): bottoms[i] = [float(totals[xl][key]) / float(totals[top_xl][key]) * p_heights[k.index(key)] for key in k] - for i in range(Nxl): - pieces[i] = plt.bar (ind, heights[i], width, color=colors[i], - bottom=bottoms[i]) + if mode == 'text': + print "\n%sWorkload profile for %s\n" % (' '*20, title) + print "Total Time: %d microseconds = %.1f seconds = %.1f minutes\n" % (total, total / 1000000.0, total / 6000000.0) + + table = Texttable() + table.set_cols_align(["l", "r"] + ["r"] * len(xlator_order)) + rows = [] + + header = ['OP', 'OP Total (%)'] + xlator_order + rows = [] + + for op in k: + sum = reduce (lambda x, y: x + y, [heights[xlator_order.index(xl)][k.index(op)] for xl in xlator_order], + 0) + row = [op] + row += ["%5.2f" % sum] - for key in k: - bar = pieces[Nxl-1][k.index(key)] - plt.text (bar.get_x() + bar.get_width()/2., 1.05*p_heights[k.index(key)], - "%d%%" % int(p_heights[k.index(key)])) + for xl in xlator_order: + op_index = k.index(op) + row += ["%5.2f" % heights[xlator_order.index(xl)][op_index]] - plt.xticks(ind+width/2., k) - plt.yticks(np.arange (0, 110, 10)) - plt.legend (map (lambda p: p[0], pieces), xlator_order) + rows.append(row) + + def row_sort(r1, r2): + v1 = float(r1[1]) + v2 = float(r2[1]) + + if v1 < v2: + return -1 + elif v1 == v2: + return 0 + else: + return 1 + + rows.sort(row_sort, reverse=True) + table.add_rows([header] + rows) + print table.draw() + + elif mode == 'graph': + for i in range(Nxl): + pieces[i] = plt.bar (ind, heights[i], width, color=colors[i], + bottom=bottoms[i]) + + for key in k: + bar = pieces[Nxl-1][k.index(key)] + plt.text (bar.get_x() + bar.get_width()/2., 1.05*p_heights[k.index(key)], + "%d%%" % int(p_heights[k.index(key)])) + + plt.xticks(ind+width/2., k) + plt.yticks(np.arange (0, 110, 10)) + plt.legend (map (lambda p: p[0], pieces), xlator_order) + + plt.show() + else: + print "Unknown mode specified!" + sys.exit(1) - plt.show() def main (): parser = OptionParser(usage="usage: %prog [-l | -d | -w] -x <xlator order> <state dump file>") @@ -241,6 +790,7 @@ def main (): help="Produce workload profile") parser.add_option("-t", "--title", dest="title", help="Set the title of the graph") parser.add_option("-x", "--xlator-order", dest="xlator_order", help="Specify the order of xlators") + parser.add_option("-m", "--mode", dest="mode", help="Output format, can be text[default] or graph") (options, args) = parser.parse_args() @@ -255,13 +805,20 @@ def main (): collect_data(file (args[0], 'r')) + mode = 'text' + if (options.mode): + mode = options.mode + if options.mode == 'graph' and graph_available == False: + print "matplotlib not available, falling back to text mode" + mode = 'text' + if (options.latency): - latency_profile (options.title, xlator_order) + latency_profile (options.title, xlator_order, mode) if (options.distribution): - fop_distribution(options.title, xlator_order) + fop_distribution(options.title, xlator_order, mode) if (options.workload): - workload_profile(options.title, xlator_order) + workload_profile(options.title, xlator_order, mode) main () |