diff options
Diffstat (limited to 'extras/volgen')
| -rw-r--r-- | extras/volgen/CreateBooster.py | 123 | ||||
| -rw-r--r-- | extras/volgen/CreateVolfile.py | 304 | ||||
| -rw-r--r-- | extras/volgen/Makefile.am | 8 | ||||
| -rwxr-xr-x | extras/volgen/glusterfs-volgen.in | 198 |
4 files changed, 0 insertions, 633 deletions
diff --git a/extras/volgen/CreateBooster.py b/extras/volgen/CreateBooster.py deleted file mode 100644 index 488426436de..00000000000 --- a/extras/volgen/CreateBooster.py +++ /dev/null @@ -1,123 +0,0 @@ -GLUSTERFS_BOOSTER_FSTAB = "booster.fstab" -GLUSTERFS_UNFS3_EXPORTS = "boosterexports" -GLUSTERFS_CIFS_CONFIG = "boostersmb.conf" -LOGDIR = "/var/log/glusterfs" -fstype = "glusterfs" - -class CreateBooster: - - def __init__ (self, options, transports): - - self.volume_name = options.volume_name - self.need_nfs = options.need_nfs - self.need_cifs = options.need_cifs - self.username = options.cifs_username - self.enable_guest = options.enable_guest - self.conf_dir = options.conf_dir - self.transports = transports - - def configure_booster_fstab (self): - - _fstab = "" - _options = "" - _options_log = "" - _options_ext = "" - - if self.conf_dir: - booster_fstab_fd = file ("%s/%s" % (str(self.conf_dir), - GLUSTERFS_BOOSTER_FSTAB), "a") - else: - booster_fstab_fd = file (GLUSTERFS_BOOSTER_FSTAB, "a") - - if self.need_nfs: - for transport in self.transports: - if self.conf_dir: - _fstab = "%s/%s-%s.vol %s" % (str(self.conf_dir), - self.volume_name, - transport, - str("/nfs/" + - self.volume_name)) - else: - _fstab = "%s-%s.vol %s" % (self.volume_name, - transport, - str("/nfs/" + - self.volume_name)) - - _options = "%s" % fstype - _options_log = "logfile=%s/%s-nfs.log" % (LOGDIR, self.volume_name) - _options_ext = "loglevel=ERROR,attr_timeout=0" - booster_fstab_fd.write ("%s %s %s,%s\n" % - (_fstab, - _options, - _options_log, - _options_ext)) - - if self.need_cifs: - for transport in self.transports: - if self.conf_dir: - _fstab = "%s/%s-%s.vol %s" % (self.conf_dir, - self.volume_name, - transport, - str("/cifs/" + - self.volume_name)) - else: - _fstab = "%s-%s.vol %s" % (self.volume_name, - transport, - str("/cifs/" + - self.volume_name)) - - _options = "%s" % fstype - _options_log = "logfile=%s/%s-cifs.log" % (LOGDIR, self.volume_name) - _options_ext = "loglevel=ERROR,attr_timeout=0" - booster_fstab_fd.write ("%s %s %s,%s\n" % - (_fstab, - _options, - _options_log, - _options_ext)) - - return - - def configure_nfs_booster (self): - - if self.conf_dir: - nfs_exports_fd = file ("%s/%s" % (str(self.conf_dir), - GLUSTERFS_UNFS3_EXPORTS), "a") - else: - nfs_exports_fd = file (GLUSTERFS_UNFS3_EXPORTS, "a") - - nfs_exports_fd.write ("%s 0.0.0.0/0(rw,no_root_squash)\n" % - str("/nfs/" + self.volume_name)) - return - - def configure_cifs_booster (self): - - if self.conf_dir: - cifs_config_fd = file ("%s/%s" % (str(self.conf_dir), - GLUSTERFS_CIFS_CONFIG), "a") - else: - cifs_config_fd = file (GLUSTERFS_CIFS_CONFIG, "a") - cifs_config_fd.write ("[%s]\n" % self.volume_name) - cifs_config_fd.write ("comment = %s volume served by Gluster\n" % - self.volume_name) - cifs_config_fd.write ("path = %s\n" % str("/cifs/" + self.volume_name)) - - if self.enable_guest: - cifs_config_fd.write ("guest ok = yes\n") - - cifs_config_fd.write ("public = yes\n") - cifs_config_fd.write ("writable = yes\n") - cifs_config_fd.write ("users = %s\n" % self.username) - cifs_config_fd.close() - return - - def configure_booster (self): - - self.configure_booster_fstab() - if self.need_nfs: - self.configure_nfs_booster() - print "Generating booster configuration for NFS reexport" - if self.need_cifs: - self.configure_cifs_booster() - print "Generating booster configuration for CIFS reexport" - - return diff --git a/extras/volgen/CreateVolfile.py b/extras/volgen/CreateVolfile.py deleted file mode 100644 index 035d2a9251e..00000000000 --- a/extras/volgen/CreateVolfile.py +++ /dev/null @@ -1,304 +0,0 @@ -import os, sys, string -import subprocess - -num_replica = 2 -num_stripe = 4 -#Cachesize calculator -cache_size = "`echo $(( $(grep 'MemTotal' /proc/meminfo | sed 's/[^0-9]//g') / 5120 ))`" - -class CreateVolfile: - - def __init__ (self, server_dict, server, transport, - transports, options, server_array): - - self.host_dict = server_dict - self.host = server - self.volume_name = options.volume_name - self.transport = transport - self.transports = transports - self.gfs_port = options.port - self.gfs_ib_port = options.port + 1 - self.auth_parameters = options.auth_param - self.raid_type = options.raid_type - self.ib_devport = options.ib_dev - self.num_servers = len (self.host_dict.keys()) - self.conf_dir = options.conf_dir - self.host_array = server_array - self.unused = options.unused - self.debug = options.debug - self.volume_size_server = options.size_server - self.volume_size_client = options.size_client - - def create_mount_volfile (self): - - raid_type = self.raid_type - - if self.conf_dir: - mount_fd = file ("%s/%s-%s.vol" % (self.conf_dir, - str(self.volume_name), - str(self.transport)), "w") - else: - mount_fd = file ("%s-%s.vol" % (str(self.volume_name), - str(self.transport)), "w") - - print "Generating client volfiles.. for transport '%s'" % (self.transport) - - - cmdline = string.join (sys.argv, ' ') - - mount_fd.write ("## file auto generated by %s (mount.vol)\n" % - sys.argv[0]) - mount_fd.write ("# Cmd line:\n") - mount_fd.write ("# $ %s\n\n" % cmdline) - - if raid_type is not None: - # Used for later usage - mount_fd.write ("# RAID %d\n" % raid_type) - - mount_fd.write ("# TRANSPORT-TYPE %s\n" % self.transport) - subvolumes = [] - for host in self.host_dict.keys(): - i = 1 - for exports in self.host_dict[host]: - mount_fd.write ("volume %s-%s\n" % (host,i)) - mount_fd.write (" type protocol/client\n") - mount_fd.write (" option transport-type %s\n" % - self.transport) - - mount_fd.write (" option remote-host %s\n" % host) - if self.transport == 'ib-verbs': - mount_fd.write (" option transport.ib-verbs.port %d\n" % - self.ib_devport) - mount_fd.write (" option transport.remote-port %d\n" % - self.gfs_ib_port) - if self.transport == 'tcp': - mount_fd.write (" option transport.socket.nodelay on\n") - mount_fd.write (" option transport.remote-port %d\n" % - self.gfs_port) - - mount_fd.write (" option remote-subvolume brick%s\n" % - i) - mount_fd.write ("end-volume\n\n") - i += 1 - - exportlist = {} - for entry in self.host_array: - node = entry.split(':')[0] - if not exportlist.has_key(node): - exportlist[node] = 1 - else: - exportlist[node] += 1 - subvolumes.append(str(node) + '-' + str(exportlist[node])) - - # Stripe section.. if given - if raid_type is 0: - max_stripe_idx = len (subvolumes) / num_stripe - stripe_idx = 0 - index = 0 - while index < max_stripe_idx: - mount_fd.write ("volume stripe-%d\n" % index) - mount_fd.write (" type cluster/stripe\n") - if self.unused: - mount_fd.write ("# option block-size 128k\n") - mount_fd.write ("# option use-xattr no\n") - - mount_fd.write (" subvolumes %s %s %s %s\n" % - (subvolumes[stripe_idx], - subvolumes[stripe_idx+1], - subvolumes[stripe_idx+2], - subvolumes[stripe_idx+3])) - mount_fd.write ("end-volume\n\n") - stripe_idx += 4 - index +=1 - - # Replicate section - if raid_type is 1: - max_mirror_idx = len (subvolumes) / num_replica - mirror_idx = 0 - index = 0 - while index < max_mirror_idx: - mount_fd.write ("volume mirror-%d\n" % index) - mount_fd.write (" type cluster/replicate\n") - mount_fd.write (" subvolumes %s %s\n" % - (subvolumes[mirror_idx], - subvolumes[mirror_idx+1])) - mount_fd.write ("end-volume\n\n") - mirror_idx += 2 - index += 1 - - # Distribute section - if raid_type is 0: - subvolumes = [] - flag = 0 - while flag < index: - subvolumes.append ("stripe-%d" % flag) - flag += 1 - if raid_type is 1: - subvolumes = [] - flag = 0 - while flag < index: - subvolumes.append ("mirror-%d" % flag) - flag += 1 - - if len (subvolumes) > 1: - mount_fd.write ("volume distribute\n") - mount_fd.write (" type cluster/distribute\n") - if self.unused: - mount_fd.write("# option unhashed-sticky-bit yes # Used for migrating data while adding new nodes\n") - mount_fd.write("# option min-free-disk 5% # Minimum free disk available on the volume\n") - - - mount_fd.write (" subvolumes %s\n" % - string.join (subvolumes,' ')) - mount_fd.write ("end-volume\n\n") - subvolumes[0] = "distribute" - - if self.volume_size_client: - mount_fd.write ("volume quota\n") - mount_fd.write (" type features/quota\n") - mount_fd.write (" option disk-usage-limit %s\n" % self.volume_size_client) - if self.unused: - mount_fd.write ("# option minimum-free-disk-limit 10GB " - "# minimum free disk value (default) 0\n") - mount_fd.write ("# option refresh-interval 10\n") - mount_fd.write (" subvolumes %s\n" % subvolumes[0]) - mount_fd.write ("end-volume\n\n") - - mount_fd.write ("volume readahead\n") - mount_fd.write (" type performance/read-ahead\n") - mount_fd.write (" option page-count 4\n") - - if self.unused: - mount_fd.write ("# option force-atime-update yes # force updating atimes, default off\n") - if self.volume_size_client: - mount_fd.write (" subvolumes quota\n") - else: - mount_fd.write (" subvolumes %s\n" % subvolumes[0]) - mount_fd.write ("end-volume\n\n") - - mount_fd.write ("volume iocache\n") - mount_fd.write (" type performance/io-cache\n") - mount_fd.write (" option cache-size %sMB\n" % cache_size) - mount_fd.write (" option cache-timeout 1\n") - if self.unused: - mount_fd.write ("# option priority *.html:1,abc*:2 # Priority list for iocaching files\n") - mount_fd.write (" subvolumes readahead\n") - mount_fd.write ("end-volume\n\n") - - mount_fd.write ("volume quickread\n") - mount_fd.write (" type performance/quick-read\n") - mount_fd.write (" option cache-timeout 1\n") - mount_fd.write (" option max-file-size 64kB\n") - mount_fd.write (" subvolumes iocache\n") - mount_fd.write ("end-volume\n\n") - - mount_fd.write ("volume writebehind\n") - mount_fd.write (" type performance/write-behind\n") - mount_fd.write (" option cache-size 4MB\n") - if self.unused: - mount_fd.write ("# option enable-trickling-writes yes # Flush final write calls when network is free\n") - mount_fd.write ("# option enable-O_SYNC yes # Enable O_SYNC for write-behind\n") - mount_fd.write ("# option disable-for-first-nbytes 1 # Disable first nbytes with very small initial writes\n") - - mount_fd.write (" subvolumes quickread\n") - mount_fd.write ("end-volume\n\n") - - mount_fd.write ("volume statprefetch\n") - mount_fd.write (" type performance/stat-prefetch\n") - mount_fd.write (" subvolumes writebehind\n") - mount_fd.write ("end-volume\n\n") - - - - return - - def create_export_volfile (self): - - cmdline = string.join (sys.argv, ' ') - - if self.conf_dir: - exp_fd = file ("%s/%s-export.vol" % - (self.conf_dir, - str(self.host + '-' + self.volume_name)),"w") - else: - exp_fd = file ("%s-export.vol" % - (str(self.host + '-' + self.volume_name)),"w") - - print "Generating server volfiles.. for server '%s'" % (self.host) - - exp_fd.write ("## file auto generated by %s (export.vol)\n" % - sys.argv[0]) - exp_fd.write ("# Cmd line:\n") - exp_fd.write ("# $ %s\n\n" % cmdline) - total_bricks = [] - i=1 - for export in self.host_dict[self.host]: - exp_fd.write ("volume posix%d\n" % i) - exp_fd.write (" type storage/posix\n") - if self.unused: - exp_fd.write("# option o-direct enable # (default: disable) boolean type only\n") - exp_fd.write("# option export-statfs-size no # (default: yes) boolean type only\n") - exp_fd.write("# option mandate-attribute off # (default: on) boolean type only\n") - exp_fd.write("# option span-devices 8 # (default: 0) integer value\n") - exp_fd.write("# option background-unlink yes # (default: no) boolean type\n") - - exp_fd.write (" option directory %s\n" % export) - exp_fd.write ("end-volume\n\n") - - if self.volume_size_server: - exp_fd.write ("volume quota%d\n" % i) - exp_fd.write (" type features/quota\n") - exp_fd.write (" option disk-usage-limit %s\n" % self.volume_size_server) - if self.unused: - exp_fd.write ("# option minimum-free-disk-limit 10GB " - "# minimum free disk value (default) 0\n") - exp_fd.write ("# option refresh-interval 10\n") - exp_fd.write (" subvolumes posix%d\n" % i) - exp_fd.write ("end-volume\n\n") - - - exp_fd.write ("volume locks%d\n" % i) - exp_fd.write (" type features/locks\n") - if self.unused: - exp_fd.write ("# option mandatory on # Default off, used in specific applications\n") - if self.volume_size_server: - exp_fd.write (" subvolumes quota%d\n" % i) - else: - exp_fd.write (" subvolumes posix%d\n" % i) - exp_fd.write ("end-volume\n\n") - - exp_fd.write ("volume brick%d\n" % i) - exp_fd.write (" type performance/io-threads\n") - exp_fd.write (" option thread-count 8\n") - if self.unused: - exp_fd.write ("# option autoscaling yes # Heuristic for autoscaling threads on demand\n") - exp_fd.write ("# option min-threads 2 # min count for thread pool\n") - exp_fd.write ("# option max-threads 64 # max count for thread pool\n") - - exp_fd.write (" subvolumes locks%d\n" % i) - exp_fd.write ("end-volume\n\n") - - total_bricks.append("brick%s" % i) - i += 1 - - for transport in self.transports: - exp_fd.write ("volume server-%s\n" % transport) - exp_fd.write (" type protocol/server\n") - exp_fd.write (" option transport-type %s\n" % transport) - for brick in total_bricks: - exp_fd.write (" option auth.addr.%s.allow %s\n" % - (brick, self.auth_parameters)) - - if transport == 'ib-verbs': - exp_fd.write (" option transport.ib-verbs.listen-port %d\n" % self.gfs_ib_port) - exp_fd.write (" option transport.ib-verbs.port %d\n" % - self.ib_devport) - if transport == 'tcp': - exp_fd.write (" option transport.socket.listen-port %d\n" % self.gfs_port) - exp_fd.write (" option transport.socket.nodelay on\n") - - exp_fd.write (" subvolumes %s\n" % - string.join(total_bricks, ' ')) - exp_fd.write ("end-volume\n\n") - - return diff --git a/extras/volgen/Makefile.am b/extras/volgen/Makefile.am deleted file mode 100644 index 73b62356ed3..00000000000 --- a/extras/volgen/Makefile.am +++ /dev/null @@ -1,8 +0,0 @@ - -volgendir = $(datadir)/glusterfs -dist_volgen_DATA = CreateVolfile.py CreateBooster.py - -dist_bin_SCRIPTS = glusterfs-volgen - -CLEANFILES = - diff --git a/extras/volgen/glusterfs-volgen.in b/extras/volgen/glusterfs-volgen.in deleted file mode 100755 index 7ba0718246e..00000000000 --- a/extras/volgen/glusterfs-volgen.in +++ /dev/null @@ -1,198 +0,0 @@ -#!/usr/bin/python - -import getopt, sys, os, string -from optparse import OptionParser,OptionGroup,make_option - -prefix = "@prefix@" - -if not (prefix + "/share/glusterfs") in sys.path: - sys.path.append(prefix + "/share/glusterfs") - -from CreateVolfile import * -from CreateBooster import * - -def check_duplicate_entry(args): - """Check duplicate entries in incoming arguments""" - _tmp = [] - for server in args: - if server not in _tmp: - _tmp.append (server) - else: - print "Duplicate arguments detected (%s)" % server - raise ValueError - - return - -def args2dict(args): - - keyvalue = {} - for arg in args: - if int(arg.find(':')) == -1: - continue - first = arg.split(':')[0] - keyvalue[first] = [] - - for arg in args: - if int(arg.find(':')) == -1: - continue - first = arg.split(':')[0] - if arg.split(':')[1] not in keyvalue[first]: - if arg.split(':')[1][0] != '/': - print "Absolute export path required for %s" % arg - raise ValueError - keyvalue[first].append (arg.split(':')[1]) - - return keyvalue - -def args2array(args): - - array = [] - - for arg in args: - if int(arg.find(':')) == -1: - continue - array.append(arg) - - return array - -def generate_volume_files (): - - num_stripe = 4 - num_replica = 2 - - usage_str = "%prog: -n <VOLUMENAME> -t <TRANSPORT> -p <NUMBER> -a <AUTH> -r <TYPE>" - version_str = "%prog 3.0" - desc_str = "A tool to generate volume files for GlusterFS." - - parse = OptionParser(usage=usage_str, version=version_str, description=desc_str) - - # Basic option list - group = OptionGroup(parse, "Basic Options") - group.add_option("-n", "--name", dest="volume_name", - help="<volume-name>") - group.add_option("-t", "--transport", dest="transport_type", - default="tcp", help="tcp,ib-verbs default: tcp") - group.add_option("-p", "--port", type="int", - dest="port", default=6996, - help="<port> number") - group.add_option("-a", "--auth", dest="auth_param", default="*", - help="comma seperated ip range") - group.add_option("-r", "--raid", type="int", dest="raid_type", - help="0|1") - group.add_option("--nfs", action="store_true", dest="need_nfs", - default=False, help="booster nfs reexport") - group.add_option("--cifs", action="store_true", dest="need_cifs", - default=False, help="booster cifs reexport") - parse.add_option_group(group) - - # CIFS option list - group = OptionGroup(parse, "CIFS Options") - group.add_option("--username", dest="cifs_username", - default="gluster", help="<cifs_username>") - group.add_option("--guest", action="store_true", - dest="enable_guest", default=False, - help="enable guest access") - parse.add_option_group(group) - # NFS option list - - # Advanced option list - group = OptionGroup(parse, "Advanced Options") - group.add_option("--ibdev", type="int", dest="ib_dev", - default=1, help="Infiniband device number <N>") - group.add_option("-c", "--conf-dir", dest="conf_dir", - help="output directory for volume files") - group.add_option("--volume-size-server", dest="size_server", - help="volume size for each server") - group.add_option("--volume-size-client", dest="size_client", - help="volume size for each client") - - parse.add_option_group(group) - - group = OptionGroup(parse, "Extra Options") - group.add_option("--unused", action="store_true", - dest="unused", default=False, - help="enable unused options") - group.add_option("--debug", action="store_true", - dest="debug", default=False, - help="add all debug modules to volumes") - - parse.add_option_group(group) - - (options, args) = parse.parse_args() - - if options.volume_name is None: - print "Error: volume name is mandatory, please provide volume name" - raise ValueError - - if options.transport_type: - transports = options.transport_type.split(',') - for transport in transports: - if (transport != "tcp" and transport != "ib-verbs"): - print "Error: --transport: option " + transport + \ - " is not valid transport type" - raise ValueError - - if options.raid_type: - if (options.raid_type != 1 and options.raid_type != 0): - print "Error: --raid: option " + str(options.raid_type) + " is not valid raid type" - raise ValueError - - - check_duplicate_entry(args) - - server_dict = {} - - server_dict = args2dict(args) - - server_array = args2array(args) - - if len (server_dict.keys()) == 0: - print "Error: no servers provided, please provide atleast one server" - raise ValueError - - if options.raid_type == 1: - if (len(server_array) % num_replica) != 0: - print "raid type (%d) and number of volumes (%d) invalid" % (options.raid_type, - len(server_array)) - raise ValueError - - if options.raid_type == 0: - if (len(server_array) % num_stripe) != 0: - print "raid type (%d) and number of volumes (%d) invalid" % (options.raid_type, - len(server_array)) - raise ValueError - - for server in server_dict.keys(): - create_exp = CreateVolfile (server_dict, server, - None, transports, - options, None) - try: - create_exp.create_export_volfile () - except IOError, (errno, strerror): - print "Got %s creating server volfiles for %s" % (strerror, server) - - for transport in transports: - create_mnt = CreateVolfile (server_dict, None, - transport, transports, - options, server_array) - try: - create_mnt.create_mount_volfile () - except IOError, (errno, strerror): - print "Got %s creating client volfiles for transport '%s'" % (strerror, transport) - - - create_booster = CreateBooster (options, transports) - try: - create_booster.configure_booster () - except IOError, (errno, strerror): - print "Got %s creating booster configuration" % (strerror) - -def main (): - - try: - generate_volume_files() - except ValueError: - sys.exit(1) - - -main() |