extras: Ease creation of new xlators

Add a python generator that creates a template for new xlator.

Co-Authored by Jeff Darcy.

Change-Id: I10820c0483794dcd450656684cf954f7b8f159e0
Signed-off-by: Poornima G <pgurusid@redhat.com>
Reviewed-on: http://review.gluster.org/13061
Smoke: Gluster Build System <jenkins@build.gluster.com>
NetBSD-regression: NetBSD Build System <jenkins@build.gluster.org>
CentOS-regression: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Jeff Darcy <jdarcy@redhat.com>

4 files changed, 430 insertions, 0 deletions

diff --git a/extras/create_new_xlator/README.md b/extras/create_new_xlator/README.md
new file mode 100644
index 00000000000..fdc1ba00812
--- /dev/null
+++ b/extras/create_new_xlator/README.md
@@ -0,0 +1,24 @@
+####This document explains how to create a template for your new xlator.
+
+`$ python ./generate_xlator.py <XLATOR_DIRECTORY> <XLATOR_NAME> <FOP_PREFIX>`
+        * XLATOR_DIRECTORY: Directory path where the new xlator folder will reside
+        * XLATOR_NAME: Name of the xlator you wish to create
+        * FOP_PREFIX: This is the fop prefix that you wish to prefix every fop definition in your xlator, fop prefix is generally different than xlator name, if the xlator name is too long.
+
+Eg: `python ./generate_xlator.py /home/u1/glusterfs/xlators/features compression cmpr`
+This command will create the following files with some initial contents like copyright, fops definition etc.
+Note that there shouldn't be a "/" specified at the end of the <XLATOR_DIRECTORY>
+        `* /home/u1/glusterfs/xlators/features/compression/Makefile.am
+        * /home/u1/glusterfs/xlators/features/compression/src/Makefile.am
+        * /home/u1/glusterfs/xlators/features/compression/src/compression.c
+        * /home/u1/glusterfs/xlators/features/compression/src/compression.h
+        * /home/u1/glusterfs/xlators/features/compression/src/compression-mem-types.h
+        * /home/u1/glusterfs/xlators/features/compression/src/compression-messages.h`
+
+By default all the fops and functions are generated, if you wish to not implement certain fops and functions, comment those lines (by adding '#' at the start of the line) in libglusterfs/src/generate_xlator.py
+
+Few other manual steps required to get the new xlator completely functional:
+* Change configure.ac
+* Change `<XLATOR_DIRECTORY>/Makefile.am` to include the new xlator directory.
+  Eg:  `/home/u1/glusterfs/xlators/features/Makefile.am`
+* Change vol file or glusterd volgen to include the new xlator in volfile
diff --git a/extras/create_new_xlator/generate_xlator.py b/extras/create_new_xlator/generate_xlator.py
new file mode 100755
index 00000000000..30bcdd81c92
--- /dev/null
+++ b/extras/create_new_xlator/generate_xlator.py
@@ -0,0 +1,205 @@
+#!/usr/bin/python
+import os
+import re
+import sys
+import string
+import time
+path = os.path.abspath(os.path.dirname(__file__)) + '/../../libglusterfs/src'
+sys.path.append(path)
+from generator import ops, xlator_cbks, xlator_dumpops
+from config import fops, xl_cbk, dumpops
+
+MAKEFILE_FMT = """
+xlator_LTLIBRARIES = @XL_NAME@.la
+xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/@XL_TYPE@
+@XL_NAME_NO_HYPHEN@_la_LDFLAGS = -module -avoid-version
+@XL_NAME_NO_HYPHEN@_la_SOURCES = @XL_NAME@.c
+@XL_NAME_NO_HYPHEN@_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
+noinst_HEADERS = @XL_NAME@.h @XL_NAME@-mem-types.h @XL_NAME@-messages.h
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+AM_CFLAGS = -Wall -fno-strict-aliasing $(GF_CFLAGS)
+CLEANFILES =
+"""
+
+fop_subs = {}
+cbk_subs = {}
+fn_subs = {}
+
+
+def get_error_arg(type_str):
+        if type_str.find(" *") != -1:
+                return "NULL"
+        return "-1"
+
+
+def get_param(names, types):
+        # Convert two separate tuples to one of (name, type) sub-tuples.
+        as_tuples = zip(types, names)
+        # Convert each sub-tuple into a "type name" string.
+        as_strings = map(string.join, as_tuples)
+        # Join all of those into one big string.
+        return string.join(as_strings, ",\n\t")
+
+
+def generate(tmpl, name, table):
+        w_arg_names = [a[1] for a in table[name] if a[0] == 'fop-arg']
+        w_arg_types = [a[2] for a in table[name] if a[0] == 'fop-arg']
+        u_arg_names = [a[1] for a in table[name] if a[0] == 'cbk-arg']
+        u_arg_types = [a[2] for a in table[name] if a[0] == 'cbk-arg']
+        fn_arg_names = [a[1] for a in table[name] if a[0] == 'fn-arg']
+        fn_arg_types = [a[2] for a in table[name] if a[0] == 'fn-arg']
+        ret_type = [a[1] for a in table[name] if a[0] == 'ret-val']
+        ret_var = [a[2] for a in table[name] if a[0] == 'ret-val']
+
+        sdict = {}
+        #Parameters are (t1, var1), (t2, var2)...
+        #Args are (var1, var2,...)
+        sdict["@WIND_ARGS@"] = string.join(w_arg_names, ", ")
+        sdict["@UNWIND_ARGS@"] = string.join(u_arg_names, ", ")
+        sdict["@ERROR_ARGS@"] = string.join(map(get_error_arg, u_arg_types), ", ")
+        sdict["@WIND_PARAMS@"] = get_param(w_arg_names, w_arg_types)
+        sdict["@UNWIND_PARAMS@"] = get_param(u_arg_names, u_arg_types)
+        sdict["@FUNC_PARAMS@"] = get_param(fn_arg_names, fn_arg_types)
+        sdict["@NAME@"] = name
+        sdict["@FOP_PREFIX@"] = fop_prefix
+        sdict["@RET_TYPE@"] = string.join(ret_type, "")
+        sdict["@RET_VAR@"] = string.join(ret_var, "")
+
+        for old, new in sdict.iteritems():
+                tmpl = tmpl.replace(old, new)
+        # TBD: reindent/reformat the result for maximum readability.
+        return tmpl
+
+
+def gen_xlator():
+        xl = open(src_dir_path+"/"+xl_name+".c", 'w+')
+
+        print >> xl, COPYRIGHT
+        print >> xl, fragments["INCLUDE_IN_SRC_FILE"].replace("@XL_NAME@",
+                                                              xl_name)
+
+        #Generate cbks and fops
+        for fop in fops:
+                print >> xl, generate(fragments["CBK_TEMPLATE"], fop, ops)
+                print >> xl, generate(fragments["FOP_TEMPLATE"], fop, ops)
+
+        for cbk in xl_cbk:
+                print >> xl, generate(fragments["FUNC_TEMPLATE"], cbk,
+                                      xlator_cbks)
+
+        for dops in dumpops:
+                print >> xl, generate(fragments["FUNC_TEMPLATE"], dops,
+                                      xlator_dumpops)
+
+        print >> xl, fragments["XLATOR_METHODS"]
+
+        #Generate fop table
+        print >> xl, "struct xlator_fops fops = {"
+        for fop in fops:
+                print >> xl, "        .{0:20} = {1}_{2},".format(fop, fop_prefix, fop)
+        print >> xl, "};"
+
+        #Generate xlator_cbks table
+        print >> xl, "struct xlator_cbks cbks = {"
+        for cbk in xl_cbk:
+                print >> xl, "        .{0:20} = {1}_{2},".format(cbk, fop_prefix, cbk)
+        print >> xl, "};"
+
+        #Generate xlator_dumpops table
+        print >> xl, "struct xlator_dumpops dumpops = {"
+        for dops in dumpops:
+                print >> xl, "        .{0:20} = {1}_{2},".format(dops, fop_prefix, dops)
+        print >> xl, "};"
+
+        xl.close()
+
+
+def create_dir_struct():
+        if not os.path.exists(dir_path+"/src"):
+                os.makedirs(dir_path+"/src")
+
+
+def gen_header_files():
+        upname = xl_name_no_hyphen.upper()
+        h = open(src_dir_path+"/"+xl_name+".h", 'w+')
+        print >> h, COPYRIGHT
+        txt = fragments["HEADER_FMT"].replace("@HFL_NAME@", upname)
+        txt2 = fragments["INCLUDE_IN_HEADER_FILE"].replace("@XL_NAME@", xl_name)
+        txt = txt.replace("@INCLUDE_SECT@",txt2)
+        print >> h, txt
+        h.close()
+
+        h = open(src_dir_path+"/"+xl_name+"-mem-types.h", 'w+')
+        print >> h, COPYRIGHT
+        txt = fragments["HEADER_FMT"].replace("@HFL_NAME@", upname+"_MEM_TYPES")
+        txt = txt.replace("@INCLUDE_SECT@", '#include "mem-types.h"')
+        print >> h, txt
+        h.close()
+
+        h = open(src_dir_path+"/"+xl_name+"-messages.h", 'w+')
+        print >> h, COPYRIGHT
+        txt = fragments["HEADER_FMT"].replace("@HFL_NAME@", upname+"_MESSAGES")
+        txt = txt.replace("@INCLUDE_SECT@", '')
+        print >> h, txt
+        h.close()
+
+
+def gen_makefiles():
+        m = open(dir_path+"/Makefile.am", 'w+')
+        print >> m, "SUBDIRS = src\n\nCLEANFILES ="
+        m.close()
+
+        m = open(src_dir_path+"/Makefile.am", 'w+')
+        txt = MAKEFILE_FMT.replace("@XL_NAME@", xl_name)
+        txt = txt.replace("@XL_NAME_NO_HYPHEN@", xl_name_no_hyphen)
+        txt = txt.replace("@XL_TYPE@",xlator_type)
+        print >> m, txt
+        m.close()
+
+def get_copyright ():
+        return fragments["CP"].replace("@CURRENT_YEAR@",
+                                       time.strftime("%Y"))
+
+def load_fragments ():
+        pragma_re = re.compile('pragma fragment (.*)')
+        cur_symbol = None
+        cur_value = ""
+        result = {}
+        basepath = os.path.abspath(os.path.dirname(__file__))
+        fragpath = basepath + "/new-xlator-tmpl.c"
+        for line in open(fragpath,"r").readlines():
+                m = pragma_re.search(line)
+                if m:
+                        if cur_symbol:
+                                result[cur_symbol] = cur_value
+                        cur_symbol = m.group(1)
+                        cur_value = ""
+                else:
+                        cur_value += line
+        if cur_symbol:
+                result[cur_symbol] = cur_value
+        return result
+
+if __name__ == '__main__':
+
+        if len(sys.argv) < 3:
+                print "USAGE: ./gen_xlator <XLATOR_DIR> <XLATOR_NAME> <FOP_PREFIX>"
+                sys.exit(0)
+
+        xl_name = sys.argv[2]
+        xl_name_no_hyphen = xl_name.replace("-", "_")
+        if sys.argv[1].endswith('/'):
+                dir_path = sys.argv[1] + xl_name
+        else:
+                dir_path = sys.argv[1] + "/" + xl_name
+        xlator_type = os.path.basename(sys.argv[1])
+        fop_prefix = sys.argv[3]
+        src_dir_path = dir_path + "/src"
+
+        fragments = load_fragments()
+
+        COPYRIGHT = get_copyright()
+        create_dir_struct()
+        gen_xlator()
+        gen_header_files()
+        gen_makefiles()
diff --git a/extras/create_new_xlator/new-xlator-tmpl.c b/extras/create_new_xlator/new-xlator-tmpl.c
new file mode 100644
index 00000000000..ac08f3732a7
--- /dev/null
+++ b/extras/create_new_xlator/new-xlator-tmpl.c
@@ -0,0 +1,89 @@
+#pragma fragment CBK_TEMPLATE
+int32_t
+@FOP_PREFIX@_@NAME@_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+        int32_t op_ret, int32_t op_errno, @UNWIND_PARAMS@)
+{
+        STACK_UNWIND_STRICT (@NAME@, frame, op_ret, op_errno,
+                             @UNWIND_ARGS@);
+        return 0;
+}
+
+#pragma fragment COMMENT
+If you are generating the leaf xlators, remove the STACK_WIND
+and replace the @ERROR_ARGS@ to @UNWIND_ARGS@ if necessary
+
+#pragma fragment FOP_TEMPLATE
+int32_t
+@FOP_PREFIX@_@NAME@ (call_frame_t *frame, xlator_t *this,
+        @WIND_PARAMS@)
+{
+        STACK_WIND (frame, @FOP_PREFIX@_@NAME@_cbk,
+                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->@NAME@,
+                    @WIND_ARGS@);
+        return 0;
+err:
+        STACK_UNWIND_STRICT (@NAME@, frame, -1, errno,
+                             @ERROR_ARGS@);
+        return 0;
+}
+
+#pragma fragment FUNC_TEMPLATE
+@RET_TYPE@
+@FOP_PREFIX@_@NAME@ (@FUNC_PARAMS@)
+{
+        return @RET_VAR@;
+}
+
+#pragma fragment CP
+/*
+ *   Copyright (c) @CURRENT_YEAR@ Red Hat, Inc. <http://www.redhat.com>
+ *   This file is part of GlusterFS.
+ *
+ *   This file is licensed to you under your choice of the GNU Lesser
+ *   General Public License, version 3 or any later version (LGPLv3 or
+ *   later), or the GNU General Public License, version 2 (GPLv2), in all
+ *   cases as published by the Free Software Foundation.
+ */
+
+#pragma fragment INCLUDE_IN_SRC_FILE
+#include "@XL_NAME@.h"
+
+#pragma fragment INCLUDE_IN_HEADER_FILE
+#include "@XL_NAME@-mem-types.h"
+#include "@XL_NAME@-messages.h"
+#include "glusterfs.h"
+#include "xlator.h"
+#include "defaults.h"
+
+#pragma fragment XLATOR_METHODS
+int32_t
+init (xlator_t *this)
+{
+        return 0;
+}
+
+void
+fini (xlator_t *this)
+{
+        return;
+}
+
+int32_t
+reconfigure (xlator_t *this, dict_t *dict)
+{
+        return 0;
+}
+
+int
+notify (xlator_t *this, int event, void *data, ...)
+{
+        return default_notify (this, event, data);
+}
+
+#pragma fragment HEADER_FMT
+#ifndef __@HFL_NAME@_H__
+#define __@HFL_NAME@_H__
+
+@INCLUDE_SECT@
+
+#endif /* __@HFL_NAME@_H__ */
diff --git a/libglusterfs/src/generator.py b/libglusterfs/src/generator.py
index 8be68337baa..8b854b2e7f7 100755
--- a/libglusterfs/src/generator.py
+++ b/libglusterfs/src/generator.py
@@ -62,6 +62,8 @@ import string
 # proper inode_link call(s).
 
 ops = {}
+xlator_cbks = {}
+xlator_dumpops = {}
 
 ops['fgetxattr'] = (
 	('fop-arg', 'fd',			'fd_t *'),
@@ -532,6 +534,116 @@ ops['getspec'] = (
 	('cbk-arg',	'spec_data',	'char *'),
 )
 
+#####################################################################
+xlator_cbks['forget'] = (
+        ('fn-arg',      'this',        'xlator_t *'),
+        ('fn-arg',      'inode',       'inode_t *'),
+	('ret-val',     'int32_t',     '0'),
+)
+
+xlator_cbks['release'] = (
+        ('fn-arg',      'this',        'xlator_t *'),
+        ('fn-arg',      'fd',          'fd_t *'),
+        ('ret-val',     'int32_t',     '0'),
+)
+
+xlator_cbks['releasedir'] = (
+        ('fn-arg',      'this',        'xlator_t *'),
+        ('fn-arg',      'fd',          'fd_t *'),
+        ('ret-val',     'int32_t',     '0'),
+)
+
+xlator_cbks['invalidate'] = (
+        ('fn-arg',      'this',        'xlator_t *'),
+        ('fn-arg',      'inode',       'inode_t *'),
+        ('ret-val',     'int32_t',     '0'),
+)
+
+xlator_cbks['client_destroy'] = (
+        ('fn-arg',      'this',        'xlator_t *'),
+        ('fn-arg',      'client',      'client_t *'),
+        ('ret-val',     'int32_t',     '0'),
+)
+
+xlator_cbks['client_disconnect'] = (
+        ('fn-arg',      'this',        'xlator_t *'),
+        ('fn-arg',      'client',      'client_t *'),
+        ('ret-val',     'int32_t',     '0'),
+)
+
+xlator_cbks['ictxmerge'] = (
+        ('fn-arg',      'this',        'xlator_t *'),
+        ('fn-arg',      'fd',          'fd_t *'),
+        ('fn-arg',      'inode',       'inode_t *'),
+        ('fn-arg',      'linked_inode', 'inode_t *'),
+        ('ret-val',     'void',        ''),
+)
+
+#####################################################################
+xlator_dumpops['priv'] = (
+        ('fn-arg',      'this',        'xlator_t *'),
+        ('ret-val',     'int32_t',     '0'),
+)
+
+xlator_dumpops['inode'] = (
+        ('fn-arg',      'this',        'xlator_t *'),
+        ('ret-val',     'int32_t',     '0'),
+)
+
+xlator_dumpops['fd'] = (
+        ('fn-arg',      'this',        'xlator_t *'),
+        ('ret-val',     'int32_t',     '0'),
+)
+
+xlator_dumpops['inodectx'] = (
+        ('fn-arg',      'this',        'xlator_t *'),
+        ('fn-arg',      'ino',         'inode_t *'),
+        ('ret-val',     'int32_t',     '0'),
+)
+
+xlator_dumpops['fdctx'] = (
+        ('fn-arg',      'this',        'xlator_t *'),
+        ('fn-arg',      'fd',          'fd_t *'),
+        ('ret-val',     'int32_t',     '0'),
+)
+
+xlator_dumpops['priv_to_dict'] = (
+        ('fn-arg',      'this',        'xlator_t *'),
+        ('fn-arg',      'dict',        'dict_t *'),
+        ('ret-val',     'int32_t',     '0'),
+)
+
+xlator_dumpops['inode_to_dict'] = (
+        ('fn-arg',      'this',        'xlator_t *'),
+        ('fn-arg',      'dict',        'dict_t *'),
+        ('ret-val',     'int32_t',     '0'),
+)
+
+xlator_dumpops['fd_to_dict'] = (
+        ('fn-arg',      'this',        'xlator_t *'),
+        ('fn-arg',      'dict',        'dict_t *'),
+        ('ret-val',     'int32_t',     '0'),
+)
+
+xlator_dumpops['inodectx_to_dict'] = (
+        ('fn-arg',      'this',        'xlator_t *'),
+        ('fn-arg',      'ino',         'inode_t *'),
+        ('fn-arg',      'dict',        'dict_t *'),
+        ('ret-val',     'int32_t',     '0'),
+)
+
+xlator_dumpops['fdctx_to_dict'] = (
+        ('fn-arg',      'this',        'xlator_t *'),
+        ('fn-arg',      'fd',          'fd_t *'),
+        ('fn-arg',      'dict',        'dict_t *'),
+        ('ret-val',     'int32_t',     '0'),
+)
+
+xlator_dumpops['history'] = (
+        ('fn-arg',      'this',        'xlator_t *'),
+        ('ret-val',     'int32_t',     '0'),
+)
+
 def get_error_arg (type_str):
 	if type_str.find(" *") != -1:
 		return "NULL"