diff options
Diffstat (limited to 'doc')
154 files changed, 7391 insertions, 21081 deletions
diff --git a/doc/Makefile.am b/doc/Makefile.am index 5eea6618220..de68c20b4d7 100644 --- a/doc/Makefile.am +++ b/doc/Makefile.am @@ -1,9 +1,9 @@ -EXTRA_DIST = glusterfs.vol.sample glusterfsd.vol.sample glusterfs.8 mount.glusterfs.8\ - glusterd.vol gluster.8 glusterd.8 glusterfsd.8 +EXTRA_DIST = glusterfs.8 mount.glusterfs.8 gluster.8 \ + glusterd.8 glusterfsd.8 -voldir = $(sysconfdir)/glusterfs -vol_DATA = glusterd.vol - -man8_MANS = glusterfs.8 mount.glusterfs.8 gluster.8 glusterd.8 glusterfsd.8 +man8_MANS = glusterfs.8 mount.glusterfs.8 gluster.8 +if WITH_SERVER +man8_MANS += glusterd.8 glusterfsd.8 +endif CLEANFILES = diff --git a/doc/README.md b/doc/README.md new file mode 100644 index 00000000000..6aa28642ef4 --- /dev/null +++ b/doc/README.md @@ -0,0 +1,26 @@ +## Developer Guide + +Gluster's contributors can check about the internals by visiting [Developer Guide Section](developer-guide). While it is not 'comprehensive', it can help you to get started. + +Also while coding, keep [Coding Standard](developer-guide/coding-standard.md) in mind. + +When you are ready to commit the changes, make sure you meet our [Commit message standard](developer-guide/commit-guidelines.md). + +## Admin Guide ## + +The gluster administration guide is maintained at [github](https://github.com/gluster/glusterdocs). The browsable admin guide can be found [here](http://docs.gluster.org/en/latest/Administrator%20Guide/). + +The doc patch has to be sent against the above mentioned repository. + +## Features/Spec ## + +The Gluster features which are 'in progress' or implemented can be found at [github](https://github.com/gluster/glusterfs-specs). + +## Upgrade Guide ## + +The gluster upgrade guide is maintained at [github](https://github.com/gluster/glusterdocs). The browsable upgrade guide can be found [here](http://docs.gluster.org/en/latest/Upgrade-Guide) + +The doc patch has to be sent against the above mentioned repository. + + +For more details about the docuemntation workflow please refer [this discussion](https://www.mail-archive.com/gluster-users@gluster.org/msg21168.html) diff --git a/doc/admin-guide/en-US/Administration_Guide.ent b/doc/admin-guide/en-US/Administration_Guide.ent deleted file mode 100644 index 3381b2bfec1..00000000000 --- a/doc/admin-guide/en-US/Administration_Guide.ent +++ /dev/null @@ -1,4 +0,0 @@ -<!ENTITY PRODUCT "Documentation"> -<!ENTITY BOOKID "Administration_Guide"> -<!ENTITY YEAR "2012"> -<!ENTITY HOLDER "Red Hat Inc"> diff --git a/doc/admin-guide/en-US/Administration_Guide.xml b/doc/admin-guide/en-US/Administration_Guide.xml deleted file mode 100644 index 483855b1a02..00000000000 --- a/doc/admin-guide/en-US/Administration_Guide.xml +++ /dev/null @@ -1,27 +0,0 @@ -<?xml version='1.0' encoding='utf-8' ?> -<!DOCTYPE book PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN" "http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd" [ -<!ENTITY % BOOK_ENTITIES SYSTEM "Administration_Guide.ent"> -%BOOK_ENTITIES; -]> -<book> - <xi:include href="Book_Info.xml" xmlns:xi="http://www.w3.org/2001/XInclude" /> - <xi:include href="Preface.xml" xmlns:xi="http://www.w3.org/2001/XInclude" /> - <xi:include href="gfs_introduction.xml" xmlns:xi="http://www.w3.org/2001/XInclude" /> - <xi:include href="admin_start_stop_daemon.xml" xmlns:xi="http://www.w3.org/2001/XInclude" /> - <xi:include href="admin_console.xml" xmlns:xi="http://www.w3.org/2001/XInclude" /> - <xi:include href="admin_storage_pools.xml" xmlns:xi="http://www.w3.org/2001/XInclude" /> - <xi:include href="admin_setting_volumes.xml" xmlns:xi="http://www.w3.org/2001/XInclude" /> - <xi:include href="admin_settingup_clients.xml" xmlns:xi="http://www.w3.org/2001/XInclude" /> - <xi:include href="admin_managing_volumes.xml" xmlns:xi="http://www.w3.org/2001/XInclude" /> - <xi:include href="admin_geo-replication.xml" xmlns:xi="http://www.w3.org/2001/XInclude" /> - <xi:include href="admin_directory_Quota.xml" xmlns:xi="http://www.w3.org/2001/XInclude" /> - <xi:include href="admin_monitoring_workload.xml" xmlns:xi="http://www.w3.org/2001/XInclude" /> - <xi:include href="admin_ACLs.xml" xmlns:xi="http://www.w3.org/2001/XInclude" /> - <xi:include href="admin_UFO.xml" xmlns:xi="http://www.w3.org/2001/XInclude" /> - <xi:include href="admin_Hadoop.xml" xmlns:xi="http://www.w3.org/2001/XInclude" /> - <xi:include href="admin_troubleshooting.xml" xmlns:xi="http://www.w3.org/2001/XInclude" /> - <xi:include href="admin_commandref.xml" xmlns:xi="http://www.w3.org/2001/XInclude" /> - <xi:include href="glossary.xml" xmlns:xi="http://www.w3.org/2001/XInclude" /> - <xi:include href="Revision_History.xml" xmlns:xi="http://www.w3.org/2001/XInclude" /> -</book> - diff --git a/doc/admin-guide/en-US/Author_Group.xml b/doc/admin-guide/en-US/Author_Group.xml deleted file mode 100644 index f3fa3174037..00000000000 --- a/doc/admin-guide/en-US/Author_Group.xml +++ /dev/null @@ -1,17 +0,0 @@ -<?xml version='1.0' encoding='utf-8' ?> -<!DOCTYPE authorgroup PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN" "http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd" [ -<!ENTITY % BOOK_ENTITIES SYSTEM "Administration_Guide.ent"> -%BOOK_ENTITIES; -]> -<authorgroup> - <author> - <firstname>Divya</firstname> - <surname>Muntimadugu</surname> - <affiliation> - <orgname>Red Hat</orgname> - <orgdiv>Engineering Content Services</orgdiv> - </affiliation> - <email>divya@redhat.com</email> - </author> -</authorgroup> - diff --git a/doc/admin-guide/en-US/Book_Info.xml b/doc/admin-guide/en-US/Book_Info.xml deleted file mode 100644 index 6be6a7816ca..00000000000 --- a/doc/admin-guide/en-US/Book_Info.xml +++ /dev/null @@ -1,28 +0,0 @@ -<?xml version='1.0' encoding='utf-8' ?> -<!DOCTYPE bookinfo PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN" "http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd" [ -<!ENTITY % BOOK_ENTITIES SYSTEM "Administration_Guide.ent"> -%BOOK_ENTITIES; -]> -<bookinfo id="book-Administration_Guide-Administration_Guide"> - <title>Administration Guide</title> - <subtitle>Using Gluster File System <remark> Beta 3</remark> </subtitle> - <productname>Gluster File System</productname> - <productnumber>3.3</productnumber> - <edition>1</edition> - <pubsnumber>1</pubsnumber> - <abstract> - <para> - This guide describes Gluster File System (GlusterFS) and provides information on how to configure, operate, and manage GlusterFS. - </para> - </abstract> - <corpauthor> - <inlinemediaobject> - <imageobject> - <imagedata fileref="Common_Content/images/title_logo.svg" format="SVG" /> - </imageobject> - </inlinemediaobject> - </corpauthor> - <xi:include href="Common_Content/Legal_Notice.xml" xmlns:xi="http://www.w3.org/2001/XInclude" /> - <xi:include href="Author_Group.xml" xmlns:xi="http://www.w3.org/2001/XInclude" /> -</bookinfo> - diff --git a/doc/admin-guide/en-US/Chapter.xml b/doc/admin-guide/en-US/Chapter.xml deleted file mode 100644 index 4a1cef872c8..00000000000 --- a/doc/admin-guide/en-US/Chapter.xml +++ /dev/null @@ -1,33 +0,0 @@ -<?xml version='1.0' encoding='utf-8' ?> -<!DOCTYPE chapter PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN" "http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd" [ -<!ENTITY % BOOK_ENTITIES SYSTEM "Administration_Guide.ent"> -%BOOK_ENTITIES; -]> -<chapter id="chap-Administration_Guide-Test_Chapter"> - <title>Test Chapter</title> - <para> - This is a test paragraph - </para> - <section id="sect-Administration_Guide-Test_Chapter-Test_Section_1"> - <title>Test Section 1</title> - <para> - This is a test paragraph in a section - </para> - </section> - - <section id="sect-Administration_Guide-Test_Chapter-Test_Section_2"> - <title>Test Section 2</title> - <para> - This is a test paragraph in Section 2 - <orderedlist> - <listitem> - <para> - listitem text - </para> - </listitem> - </orderedlist> - </para> - </section> - -</chapter> - diff --git a/doc/admin-guide/en-US/Preface.xml b/doc/admin-guide/en-US/Preface.xml deleted file mode 100644 index 320311906d0..00000000000 --- a/doc/admin-guide/en-US/Preface.xml +++ /dev/null @@ -1,24 +0,0 @@ -<?xml version='1.0' encoding='UTF-8'?> -<!-- This document was created with Syntext Serna Free. --> -<!DOCTYPE preface PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN" "http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd" [ -<!ENTITY % BOOK_ENTITIES SYSTEM "Administration_Guide.ent"> -%BOOK_ENTITIES; -]> -<preface id="pref-Administration_Guide-Preface"> - <title>Preface</title> - <para>This guide describes how to configure, operate, and manage Gluster File System (GlusterFS).</para> - <section> - <title>Audience</title> - <para>This guide is intended for Systems Administrators interested in configuring and managing GlusterFS.</para> - <para>This guide assumes that you are familiar with the Linux operating system, concepts of File System, GlusterFS concepts, and GlusterFS Installation</para> - </section> - <section> - <title>License</title> - <para>The License information is available at <ulink url="http://www.redhat.com/licenses/rhel_rha_eula.html"/>.</para> - </section> - <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="Common_Content/Conventions.xml"/> - <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="Feedback.xml"> - <xi:fallback xmlns:xi="http://www.w3.org/2001/XInclude"> <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="Common_Content/Feedback.xml"/> - </xi:fallback> - </xi:include> -</preface> diff --git a/doc/admin-guide/en-US/Revision_History.xml b/doc/admin-guide/en-US/Revision_History.xml deleted file mode 100644 index 09320821fb0..00000000000 --- a/doc/admin-guide/en-US/Revision_History.xml +++ /dev/null @@ -1,27 +0,0 @@ -<?xml version='1.0' encoding='utf-8' ?> -<!DOCTYPE appendix PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN" "http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd" [ -<!ENTITY % BOOK_ENTITIES SYSTEM "Administration_Guide.ent"> -%BOOK_ENTITIES; -]> -<appendix id="appe-Administration_Guide-Revision_History"> - <title>Revision History</title> - <simpara> - <revhistory> - <revision> - <revnumber>1-0</revnumber> - <date>Thu Apr 5 2012</date> - <author> - <firstname>Divya</firstname> - <surname>Muntimadugu</surname> - <email>divya@redhat.com</email> - </author> - <revdescription> - <simplelist> - <member>Draft </member> - </simplelist> - </revdescription> - </revision> - </revhistory> - </simpara> -</appendix> - diff --git a/doc/admin-guide/en-US/admin_ACLs.xml b/doc/admin-guide/en-US/admin_ACLs.xml deleted file mode 100644 index 156e52c17f2..00000000000 --- a/doc/admin-guide/en-US/admin_ACLs.xml +++ /dev/null @@ -1,206 +0,0 @@ -<?xml version='1.0' encoding='UTF-8'?> -<!-- This document was created with Syntext Serna Free. --><!DOCTYPE chapter PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN" "docbookV4.5/docbookx.dtd" []> -<chapter id="chap-Administration_Guide-ACLs"> - <title>POSIX Access Control Lists </title> - <para>POSIX Access Control Lists (ACLs) allows you to assign different permissions for different users or -groups even though they do not correspond to the original owner or the owning group. - </para> - <para>For example: User john creates a file but does not want to allow anyone to do anything with this -file, except another user, antony (even though there are other users that belong to the group john). -</para> - <para>This means, in addition to the file owner, the file group, and others, additional users and groups can -be granted or denied access by using POSIX ACLs. -</para> - <section id="sect-Administration_Guide-ACLs-Activating_ACLs"> - <title>Activating POSIX ACLs Support </title> - <para>To use POSIX ACLs for a file or directory, the partition of the file or directory must be mounted with -POSIX ACLs support. -</para> - <section id="sect-Administration_Guide-ACLs-Activating_ACLs-Server"> - <title>Activating POSIX ACLs Support on Sever </title> - <para>To mount the backend export directories for POSIX ACLs support, use the following command: -</para> - <para><command># mount -o acl <replaceable>device-name</replaceable><replaceable>partition</replaceable></command> -</para> - <para>For example: -</para> - <para><command># mount -o acl /dev/sda1 /export1 </command></para> - <para>Alternatively, if the partition is listed in the /etc/fstab file, add the following entry for the partition -to include the POSIX ACLs option: -</para> - <para><command>LABEL=/work /export1 ext3 rw, acl 14 </command></para> - </section> - <section> - <title>Activating POSIX ACLs Support on Client </title> - <para>To mount the glusterfs volumes for POSIX ACLs support, use the following command: -</para> - <para><command># mount –t glusterfs -o acl <replaceable>severname:volume-id</replaceable><replaceable>mount point</replaceable></command> -</para> - <para>For example: -</para> - <para><command># mount -t glusterfs -o acl 198.192.198.234:glustervolume /mnt/gluster</command> -</para> - </section> - </section> - <section> - <title>Setting POSIX ACLs </title> - <para>You can set two types of POSIX ACLs, that is, access ACLs and default ACLs. You can use -access ACLs to grant permission for a specific file or directory. You can use default ACLs only -on a directory but if a file inside that directory does not have an ACLs, it inherits the permissions of -the default ACLs of the directory. -</para> - <para>You can set ACLs for per user, per group, for users not in the user group for the file, and via the -effective right mask. -</para> - <section> - <title>Setting Access ACLs </title> - <para>You can apply access ACLs to grant permission for both files and directories. -</para> - <para><emphasis role="bold">To set or modify Access ACLs</emphasis> -</para> - <para>You can set or modify access ACLs use the following command: -</para> - <para><command># setfacl –m <replaceable>entry type</replaceable> file </command></para> - <para>The ACL entry types are the POSIX ACLs representations of owner, group, and other. -</para> - <para>Permissions must be a combination of the characters <command>r</command> (read), <command>w</command> (write), and <command>x</command> (execute). You must -specify the ACL entry in the following format and can specify multiple entry types separated by -commas. -</para> - <informaltable frame="all"> - <tgroup cols="2"> - <colspec colname="c1"/> - <colspec colname="c2"/> - <thead> - <row> - <entry>ACL Entry</entry> - <entry>Description</entry> - </row> - </thead> - <tbody> - <row> - <entry>u:uid:<permission> </entry> - <entry>Sets the access ACLs for a user. You can specify user name or UID </entry> - </row> - <row> - <entry>g:gid:<permission> </entry> - <entry>Sets the access ACLs for a group. You can specify group name or GID. </entry> - </row> - <row> - <entry>m:<permission> </entry> - <entry>Sets the effective rights mask. The mask is the combination of all access permissions of the owning group and all of the user and group entries. </entry> - </row> - <row> - <entry>o:<permission> </entry> - <entry>Sets the access ACLs for users other than the ones in the group for the file. </entry> - </row> - </tbody> - </tgroup> - </informaltable> - <para>If a file or directory already has an POSIX ACLs, and the setfacl command is used, the additional -permissions are added to the existing POSIX ACLs or the existing rule is modified. -</para> - <para>For example, to give read and write permissions to user antony: -</para> - <para><command># setfacl -m u:antony:rw /mnt/gluster/data/testfile </command></para> - </section> - <section> - <title>Setting Default ACLs </title> - <para>You can apply default ACLs only to directories. They determine the permissions of a file system -objects that inherits from its parent directory when it is created. -</para> - <para>To set default ACLs -</para> - <para>You can set default ACLs for files and directories using the following command: -</para> - <para><command># setfacl –m –-set <replaceable>entry type directory</replaceable></command> -</para> - <para>For example, to set the default ACLs for the /data directory to read for users not in the user group: -</para> - <para><command># setfacl –m --set o::r /mnt/gluster/data </command></para> - <para><note> - <para>An access ACLs set for an individual file can override the default ACLs permissions. -</para> - </note></para> - <para><emphasis role="bold">Effects of a Default ACLs </emphasis></para> - <para>The following are the ways in which the permissions of a directory's default ACLs are passed to the -files and subdirectories in it: -</para> - <itemizedlist> - <listitem> - <para>A subdirectory inherits the default ACLs of the parent directory both as its default ACLs and as an -access ACLs. -</para> - </listitem> - <listitem> - <para>A file inherits the default ACLs as its access ACLs. -</para> - </listitem> - </itemizedlist> - </section> - </section> - <section> - <title>Retrieving POSIX ACLs </title> - <para>You can view the existing POSIX ACLs for a file or directory. -</para> - <para><emphasis role="bold">To view existing POSIX ACLs </emphasis></para> - <itemizedlist> - <listitem> - <para>View the existing access ACLs of a file using the following command: -</para> - <para><command># getfacl <replaceable>path/filename</replaceable></command> -</para> - <para>For example, to view the existing POSIX ACLs for sample.jpg -</para> - <programlisting># getfacl /mnt/gluster/data/test/sample.jpg -# owner: antony -# group: antony -user::rw- -group::rw- -other::r--</programlisting> - </listitem> - <listitem> - <para>View the default ACLs of a directory using the following command: -</para> - <para><command># getfacl <replaceable>directory name</replaceable></command></para> - <para>For example, to view the existing ACLs for /data/doc -</para> - <programlisting># getfacl /mnt/gluster/data/doc -# owner: antony -# group: antony -user::rw- -user:john:r-- -group::r-- -mask::r-- -other::r-- -default:user::rwx -default:user:antony:rwx -default:group::r-x -default:mask::rwx -default:other::r-x</programlisting> - </listitem> - </itemizedlist> - </section> - <section> - <title>Removing POSIX ACLs </title> - <para>To remove all the permissions for a user, groups, or others, use the following command: -</para> - <para><command># setfacl -x <replaceable>ACL entry type file</replaceable></command></para> - <para>For example, to remove all permissions from the user antony: -</para> - <para><command># setfacl -x u:antony /mnt/gluster/data/test-file</command></para> - </section> - <section> - <title>Samba and ACLs </title> - <para>If you are using Samba to access GlusterFS FUSE mount, then POSIX ACLs are enabled by default. -Samba has been compiled with the <command>--with-acl-support</command> option, so no special flags are required -when accessing or mounting a Samba share. -</para> - </section> - <section> - <title>NFS and ACLs </title> - <para>Currently we do not support ACLs configuration through NFS, i.e. setfacl and getfacl commands do -not work. However, ACLs permissions set using Gluster Native Client applies on NFS mounts. -</para> - </section> -</chapter> diff --git a/doc/admin-guide/en-US/admin_Hadoop.xml b/doc/admin-guide/en-US/admin_Hadoop.xml deleted file mode 100644 index 08bac89615b..00000000000 --- a/doc/admin-guide/en-US/admin_Hadoop.xml +++ /dev/null @@ -1,244 +0,0 @@ -<?xml version='1.0' encoding='UTF-8'?> -<!-- This document was created with Syntext Serna Free. --><!DOCTYPE chapter PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN" "http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd" [ -<!ENTITY % BOOK_ENTITIES SYSTEM "Administration_Guide.ent"> -%BOOK_ENTITIES; -]> -<chapter id="chap-Administration_Guide-Hadoop"> - <title>Managing Hadoop Compatible Storage </title> - <para>GlusterFS provides compatibility for Apache Hadoop and it uses the standard file system -APIs available in Hadoop to provide a new storage option for Hadoop deployments. Existing -MapReduce based applications can use GlusterFS seamlessly. This new functionality opens up data -within Hadoop deployments to any file-based or object-based application. - - </para> - <section id="sect-Administration_Guide-Hadoop-Introduction-Architecture_Overview"> - <title>Architecture Overview </title> - <para>The following diagram illustrates Hadoop integration with GlusterFS: -<mediaobject> - <imageobject> - <imagedata fileref="images/Hadoop_Architecture.png"/> - </imageobject> - </mediaobject> - </para> - </section> - <section id="sect-Administration_Guide-Hadoop-Introduction-Advantages"> - <title>Advantages </title> - <para> -The following are the advantages of Hadoop Compatible Storage with GlusterFS: - - - </para> - <itemizedlist> - <listitem> - <para>Provides simultaneous file-based and object-based access within Hadoop. -</para> - </listitem> - <listitem> - <para>Eliminates the centralized metadata server. -</para> - </listitem> - <listitem> - <para>Provides compatibility with MapReduce applications and rewrite is not required. -</para> - </listitem> - <listitem> - <para>Provides a fault tolerant file system. -</para> - </listitem> - </itemizedlist> - </section> - <section> - <title>Preparing to Install Hadoop Compatible Storage</title> - <para>This section provides information on pre-requisites and list of dependencies that will be installed -during installation of Hadoop compatible storage. - -</para> - <section id="sect-Administration_Guide-Hadoop-Preparation"> - <title>Pre-requisites </title> - <para>The following are the pre-requisites to install Hadoop Compatible -Storage : - - </para> - <itemizedlist> - <listitem> - <para>Hadoop 0.20.2 is installed, configured, and is running on all the machines in the cluster. -</para> - </listitem> - <listitem> - <para>Java Runtime Environment -</para> - </listitem> - <listitem> - <para>Maven (mandatory only if you are building the plugin from the source) -</para> - </listitem> - <listitem> - <para>JDK (mandatory only if you are building the plugin from the source) -</para> - </listitem> - <listitem> - <para>getfattr -- command line utility</para> - </listitem> - </itemizedlist> - </section> - </section> - <section> - <title>Installing, and Configuring Hadoop Compatible Storage</title> - <para>This section describes how to install and configure Hadoop Compatible Storage in your storage -environment and verify that it is functioning correctly. - -</para> - <orderedlist> - <para>To install and configure Hadoop compatible storage:</para> - <listitem> - <para>Download <filename>glusterfs-hadoop-0.20.2-0.1.x86_64.rpm</filename> file to each server on your cluster. You can download the file from <ulink url="http://download.gluster.com/pub/gluster/glusterfs/qa-releases/3.3-beta-2/glusterfs-hadoop-0.20.2-0.1.x86_64.rpm"/>. - -</para> - </listitem> - <listitem> - <para>To install Hadoop Compatible Storage on all servers in your cluster, run the following command: -</para> - <para><command># rpm –ivh --nodeps glusterfs-hadoop-0.20.2-0.1.x86_64.rpm</command> -</para> - <para>The following files will be extracted: - </para> - <itemizedlist> - <listitem> - <para>/usr/local/lib/glusterfs-<replaceable>Hadoop-version-gluster_plugin_version</replaceable>.jar </para> - </listitem> - <listitem> - <para> /usr/local/lib/conf/core-site.xml</para> - </listitem> - </itemizedlist> - </listitem> - <listitem> - <para>(Optional) To install Hadoop Compatible Storage in a different location, run the following -command: -</para> - <para><command># rpm –ivh --nodeps –prefix /usr/local/glusterfs/hadoop glusterfs-hadoop- 0.20.2-0.1.x86_64.rpm</command> -</para> - </listitem> - <listitem> - <para>Edit the <filename>conf/core-site.xml</filename> file. The following is the sample <filename>conf/core-site.xml</filename> file: -</para> - <para><programlisting><configuration> - <property> - <name>fs.glusterfs.impl</name> - <value>org.apache.hadoop.fs.glusterfs.Gluster FileSystem</value> -</property> - -<property> - <name>fs.default.name</name> - <value>glusterfs://fedora1:9000</value> -</property> - -<property> - <name>fs.glusterfs.volname</name> - <value>hadoopvol</value> -</property> - -<property> - <name>fs.glusterfs.mount</name> - <value>/mnt/glusterfs</value> -</property> - -<property> - <name>fs.glusterfs.server</name> - <value>fedora2</value> -</property> - -<property> - <name>quick.slave.io</name> - <value>Off</value> -</property> -</configuration> -</programlisting></para> - <para>The following are the configurable fields: -</para> - <para><informaltable frame="none"> - <tgroup cols="3"> - <colspec colnum="1" colname="c0" colsep="0"/> - <colspec colnum="2" colname="c1" colsep="0"/> - <colspec colnum="3" colname="c2" colsep="0"/> - <thead> - <row> - <entry>Property Name </entry> - <entry>Default Value </entry> - <entry>Description </entry> - </row> - </thead> - <tbody> - <row> - <entry>fs.default.name </entry> - <entry>glusterfs://fedora1:9000</entry> - <entry>Any hostname in the cluster as the server and any port number. </entry> - </row> - <row> - <entry>fs.glusterfs.volname </entry> - <entry>hadoopvol </entry> - <entry>GlusterFS volume to mount. </entry> - </row> - <row> - <entry>fs.glusterfs.mount </entry> - <entry>/mnt/glusterfs</entry> - <entry>The directory used to fuse mount the volume.</entry> - </row> - <row> - <entry>fs.glusterfs.server </entry> - <entry>fedora2</entry> - <entry>Any hostname or IP address on the cluster except the client/master. </entry> - </row> - <row> - <entry>quick.slave.io </entry> - <entry>Off </entry> - <entry>Performance tunable option. If this option is set to On, the plugin will try to perform I/O directly from the disk file system (like ext3 or ext4) the file resides on. Hence read performance will improve and job would run faster. <note> - <para>This option is not tested widely</para> - </note></entry> - </row> - </tbody> - </tgroup> - </informaltable></para> - </listitem> - <listitem> - <para>Create a soft link in Hadoop’s library and configuration directory for the downloaded files (in -Step 3) using the following commands: -</para> - <para><command># ln -s <replaceable><target location> <source location</replaceable>></command> -</para> - <para>For example, -</para> - <para><command># ln –s /usr/local/lib/glusterfs-0.20.2-0.1.jar <replaceable>$HADOOP_HOME</replaceable>/lib/glusterfs-0.20.2-0.1.jar</command> -</para> - <para><command># ln –s /usr/local/lib/conf/core-site.xml <replaceable>$HADOOP_HOME</replaceable>/conf/core-site.xml </command></para> - </listitem> - <listitem> - <para> (Optional) You can run the following command on Hadoop master to build the plugin and deploy -it along with core-site.xml file, instead of repeating the above steps: -</para> - <para><command># build-deploy-jar.py -d <replaceable>$HADOOP_HOME</replaceable> -c </command></para> - </listitem> - </orderedlist> - </section> - <section> - <title>Starting and Stopping the Hadoop MapReduce Daemon</title> - <para>To start and stop MapReduce daemon</para> - <itemizedlist> - <listitem> - <para>To start MapReduce daemon manually, enter the following command: -</para> - <para><command># <replaceable>$HADOOP_HOME</replaceable>/bin/start-mapred.sh</command> -</para> - </listitem> - <listitem> - <para>To stop MapReduce daemon manually, enter the following command: -</para> - <para><command># <replaceable>$HADOOP_HOME</replaceable>/bin/stop-mapred.sh </command></para> - </listitem> - </itemizedlist> - <para><note> - <para>You must start Hadoop MapReduce daemon on all servers. -</para> - </note></para> - </section> -</chapter> diff --git a/doc/admin-guide/en-US/admin_UFO.xml b/doc/admin-guide/en-US/admin_UFO.xml deleted file mode 100644 index 03be14dc9dc..00000000000 --- a/doc/admin-guide/en-US/admin_UFO.xml +++ /dev/null @@ -1,1588 +0,0 @@ -<?xml version='1.0' encoding='UTF-8'?> -<!-- This document was created with Syntext Serna Free. --><!DOCTYPE chapter PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN" "http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd" [ -<!ENTITY % BOOK_ENTITIES SYSTEM "Administration_Guide.ent"> -%BOOK_ENTITIES; -]> -<chapter id="chap-Administration_Guide-UFO"> - <title>Managing Unified File and Object Storage</title> - <para>Unified File and Object Storage (UFO) unifies NAS and object storage technology. It -provides a system for data storage that enables users to access the same data, both as an object and as a -file, thus simplifying management and controlling storage costs. - -</para> - <para>Unified File and Object Storage is built upon Openstack's Object Storage Swift. Open Stack Object Storage allows users to store and retrieve files and content through a simple Web Service (REST: Representational State Transfer) interface as objects and GlusterFS, allows users to store and retrieve files using Native Fuse and NFS mounts. It uses GlusterFS as a backend file system for Open Stack Swift. It also leverages on Open Stack Swift's web interface for storing and retrieving files over the web combined with GlusterFS features like scalability and high availability, replication, elastic volume management for data management at disk level.</para> - <para>Unified File and Object Storage technology enables enterprises to adopt and deploy -cloud storage solutions. It allows users to access and modify data as objects from a -REST interface along with the ability to access and modify files from NAS interfaces including NFS -and CIFS. In addition to decreasing cost and making it faster and easier to access object data, -it also delivers massive scalability, high availability and replication of object storage. -Infrastructure as a Service (IaaS) providers can utilize GlusterFS Unified File and Object Storage technology to enable their own cloud -storage service. Enterprises can use this technology to accelerate the process of preparing file-based -applications for the cloud and simplify new application development for cloud computing -environments. - -</para> - <para>OpenStack Object Storage is scalable object storage system and it is not a traditional file system. You will not be able to mount this system like traditional SAN or NAS -volumes and perform POSIX compliant operations. </para> - <para><figure> - <title>Unified File and Object Storage Architecture</title> - <mediaobject> - <imageobject> - <imagedata fileref="images/UFO_Architecture.png"/> - </imageobject> - </mediaobject> - </figure></para> - <section> - <title>Components of Object Storage</title> - <para>The major components of Object Storage are: - </para> - <para><emphasis role="bold">Proxy Server</emphasis> - -</para> - <para>All REST requests to the UFO are routed through the Proxy Server. - - -</para> - <para><emphasis role="bold">Objects and Containers </emphasis></para> - <para>An object is the basic storage entity and any optional metadata that represents the data -you store. When you upload data, the data is stored as-is (with no compression or encryption). - -</para> - <para>A container is a storage compartment for your data and provides a way for you to organize -your data. Containers can be visualized as directories in a Linux system. Data must be stored in a container and hence objects are created within a container. - -</para> - <para>It implements objects as files and directories under the container. The object name is a '/' separated path and UFO maps it to directories until the last name in the path, which is marked as a file. With this approach, objects can be accessed as files and directories from native GlusterFS (FUSE) or NFS mounts by providing the '/' separated path.</para> - <para><emphasis role="bold">Accounts and Account Servers</emphasis></para> - <para>The OpenStack Object Storage system is designed to be used by many different storage -consumers. Each user is associated with one or more accounts and must identify themselves using an authentication system. While authenticating, users must provide the name of the account for which the authentication is requested. - -</para> - <para>UFO implements accounts as GlusterFS volumes. So, when a user is granted read/write permission on an account, it means that that user has access to all the data available on that GlusterFS volume. - - - - -</para> - <para><emphasis role="bold">Authentication and Access Permissions</emphasis> - -</para> - <para>You must authenticate against an authentication service to receive OpenStack Object -Storage connection parameters and an authentication token. The token must be passed -in for all subsequent container or object operations. One authentication service that you -can use as a middleware example is called <literal>tempauth</literal>.</para> - <para>By default, each user has their own storage account and has full access to that -account. Users must authenticate with their credentials as described above, but once -authenticated they can manage containers and objects within that account. If a user wants to access the content from another account, they must have API access key or a session token provided by their authentication system.</para> - </section> - <section> - <title>Advantages of using GlusterFS Unified File and Object Storage</title> - <para>The following are the advantages of using GlusterFS UFO:</para> - <itemizedlist> - <listitem> - <para>No limit on upload and download files sizes as compared to Open Stack Swift which limits the object size to 5GB.</para> - </listitem> - <listitem> - <para>A unified view of data across NAS and Object Storage technologies.</para> - </listitem> - <listitem> - <para>Using GlusterFS's UFO has other advantages like the following: </para> - <para><itemizedlist> - <listitem> - <para>High availability</para> - </listitem> - <listitem> - <para>Scalability</para> - </listitem> - <listitem> - <para>Replication</para> - </listitem> - <listitem> - <para>Elastic Volume management</para> - </listitem> - </itemizedlist></para> - </listitem> - </itemizedlist> - </section> - <section> - <title>Preparing to Deploy Unified File and Object Storage</title> - <para>This section provides information on pre-requisites and list of dependencies that will be installed -during the installation of Unified File and Object Storage. -</para> - <section> - <title>Pre-requisites </title> - <para>GlusterFS's Unified File and Object Storage needs <literal>user_xattr</literal> support from the underlying disk file system. -Use the following command to enable <literal>user_xattr</literal> for GlusterFS bricks backend: -</para> - <para><command># mount –o remount,user_xattr <replaceable>device name</replaceable></command></para> - <para>For example, -</para> - <para><command># mount –o remount,user_xattr /dev/hda1 </command> -</para> - </section> - <section> - <title>Dependencies </title> - <para>The following packages are installed on GlusterFS when you install Unified File and Object -Storage: - -</para> - <itemizedlist> - <listitem> - <para>curl - - - - - - - - - - - - - -</para> - </listitem> - <listitem> - <para>memcached</para> - </listitem> - <listitem> - <para>openssl</para> - </listitem> - <listitem> - <para>xfsprogs</para> - </listitem> - <listitem> - <para>python2.6</para> - </listitem> - <listitem> - <para>pyxattr</para> - </listitem> - <listitem> - <para>python-configobj -</para> - </listitem> - <listitem> - <para>python-setuptools - -</para> - </listitem> - <listitem> - <para>python-simplejson - -</para> - </listitem> - <listitem> - <para>python-webob - -</para> - </listitem> - <listitem> - <para>python-eventlet - -</para> - </listitem> - <listitem> - <para>python-greenlet - -</para> - </listitem> - <listitem> - <para>python-pastedeploy - -</para> - </listitem> - <listitem> - <para>python-netifaces -</para> - </listitem> - </itemizedlist> - </section> - </section> - <section> - <title>Installing and Configuring Unified File and Object Storage</title> - <para>This section provides instructions on how to install and configure Unified File and Object Storage in your storage -environment.</para> - <section id="chap-ation_Guide-Dir_Quota-Enable"> - <title>Installing Unified File and Object Storage</title> - <para>To install Unified File and Object Storage:</para> - <orderedlist> - <listitem> - <para>Download <filename>rhel_install.sh</filename> install script from <ulink url="http://download.gluster.com/pub/gluster/glusterfs/3.2/UFO/"/> . -</para> - </listitem> - <listitem> - <para>Run - <filename>rhel_install.sh</filename> script using the following command: -</para> - <para><command># sh rhel_install.sh</command></para> - </listitem> - <listitem> - <para>Download <filename>swift-1.4.5-1.noarch.rpm</filename> and <filename>swift-plugin-1.0.-1.el6.noarch.rpm</filename> files from <ulink url="http://download.gluster.com/pub/gluster/glusterfs/3.2/UFO/"/>.</para> - </listitem> - <listitem> - <para>Install <filename>swift-1.4.5-1.noarch.rpm</filename> and <filename>swift-plugin-1.0.-1.el6.noarch.rpm</filename> using the following commands:</para> - <para><command># rpm -ivh swift-1.4.5-1.noarch.rpm</command></para> - <para><command># rpm -ivh swift-plugin-1.0.-1.el6.noarch.rpm</command></para> - <para><note> - <para>You must repeat the above steps on all the machines on which you want to install Unified File and Object Storage. If you install the Unified File and Object Storage on multiple servers, you can use a load balancer like pound, nginx, and so on to distribute the request across the machines.</para> - </note></para> - </listitem> - </orderedlist> - </section> - <section> - <title>Adding Users</title> - <para>The authentication system allows the administrator to grant different levels of access to different users based on the requirement. The following are the types of user permissions: - </para> - <itemizedlist> - <listitem> - <para>admin user - </para> - </listitem> - <listitem> - <para>normal user</para> - </listitem> - </itemizedlist> - <para>Admin user has read and write permissions on the account. By default, a normal user has no read or write permissions. A normal user can only authenticate itself to get a Auth-Token. Read or write permission are provided through ACLs by the admin users.</para> - <para>Add a new user by adding the following entry in <filename>/etc/swift/proxy-server.conf</filename> file:</para> - <para><command>user_<account-name>_<user-name> = <password> [.admin]</command></para> - <para>For example, </para> - <para><command>user_test_tester = testing .admin</command> -</para> - <note> - <para>During installation, the installation script adds few sample users to the <filename>proxy-server.conf</filename> file. It is highly recommended that you remove all the default sample user entries from the configuration file. -</para> - </note> - <para>For more information on setting ACLs, see <xref linkend="chap-Administration_Guide-Working_UFO-Setting_ACLs"/>.</para> - </section> - <section> - <title>Configuring Proxy Server</title> - <para>The Proxy Server is responsible for connecting to the rest of the OpenStack Object Storage architecture. For each request, it looks up the location of the account, container, or object in the ring and route the request accordingly. The public API is also exposed through the proxy server. When objects are streamed to or from an object server, they are streamed directly through the proxy server to or from the user – the proxy server does not spool them. -</para> - <para>The configurable options pertaining to proxy server are stored in <filename>/etc/swift/proxy-server.conf</filename>. The following is the sample <filename>proxy-server.conf</filename> file:</para> - <para><programlisting>[app:proxy-server] -use = egg:swift#proxy -allow_account_management=true -account_autocreate=true - -[filter:tempauth] -use = egg:swift#tempauth user_admin_admin=admin.admin.reseller_admin -user_test_tester=testing.admin -user_test2_tester2=testing2.admin -user_test_tester3=testing3 - -[filter:healthcheck] -use = egg:swift#healthcheck - -[filter:cache] -use = egg:swift#memcache</programlisting></para> - <para>By default, GlusterFS's Unified File and Object Storage is configured to support HTTP protocol and uses temporary authentication to authenticate the HTTP requests.</para> - </section> - <section> - <title>Configuring Authentication System</title> - <para>Proxy server must be configured to authenticate using <literal> - <literal>tempauth</literal> - </literal>. </para> - </section> - <section> - <title>Configuring Proxy Server for HTTPS</title> - <para>By default, proxy server only handles HTTP request. To configure the proxy server to process HTTPS requests, perform the following steps:</para> - <orderedlist> - <listitem> - <para>Create self-signed cert for SSL using the following commands:</para> - <para><programlisting>cd /etc/swift -openssl req -new -x509 -nodes -out cert.crt -keyout cert.key</programlisting></para> - </listitem> - <listitem> - <para>Add the following lines to <filename>/etc/swift/proxy-server.conf </filename>under <replaceable>[DEFAULT]</replaceable></para> - <para><programlisting>bind_port = 443 - cert_file = /etc/swift/cert.crt - key_file = /etc/swift/cert.key</programlisting></para> - </listitem> - <listitem> - <para>Restart the servers using the following commands:</para> - <para><programlisting>swift-init main stop -swift-init main start</programlisting></para> - </listitem> - </orderedlist> - <para>The following are the configurable options: -</para> - <table frame="all"> - <title>proxy-server.conf Default Options in the [DEFAULT] section </title> - <tgroup cols="3"> - <colspec colname="c1"/> - <colspec colname="c2"/> - <colspec colname="c3"/> - <thead> - <row> - <entry>Option </entry> - <entry>Default </entry> - <entry>Description </entry> - </row> - </thead> - <tbody> - <row> - <entry>bind_ip </entry> - <entry>0.0.0.0 </entry> - <entry>IP Address for server to bind</entry> - </row> - <row> - <entry>bind_port </entry> - <entry>80 </entry> - <entry>Port for server to bind </entry> - </row> - <row> - <entry>swift_dir </entry> - <entry>/etc/swift </entry> - <entry>Swift configuration directory </entry> - </row> - <row> - <entry>workers </entry> - <entry>1</entry> - <entry>Number of workers to fork </entry> - </row> - <row> - <entry>user </entry> - <entry>swift </entry> - <entry>swift user</entry> - </row> - <row> - <entry>cert_file </entry> - <entry/> - <entry>Path to the ssl .crt </entry> - </row> - <row> - <entry>key_file </entry> - <entry/> - <entry>Path to the ssl .key </entry> - </row> - </tbody> - </tgroup> - </table> - <table frame="all"> - <title>proxy-server.conf Server Options in the [proxy-server] section </title> - <tgroup cols="3"> - <colspec colname="c1"/> - <colspec colname="c2"/> - <colspec colname="c3"/> - <thead> - <row> - <entry>Option </entry> - <entry>Default </entry> - <entry>Description </entry> - </row> - </thead> - <tbody> - <row> - <entry>use </entry> - <entry/> - <entry>paste.deploy entry point for the container server. For most cases, this should be <literal>egg:swift#container</literal>. </entry> - </row> - <row> - <entry>log_name </entry> - <entry>proxy-server </entry> - <entry>Label used when logging </entry> - </row> - <row> - <entry>log_facility </entry> - <entry>LOG_LOCAL0 </entry> - <entry>Syslog log facility </entry> - </row> - <row> - <entry>log_level </entry> - <entry>INFO </entry> - <entry>Log level </entry> - </row> - <row> - <entry>log_headers </entry> - <entry>True </entry> - <entry>If True, log headers in each request </entry> - </row> - <row> - <entry>recheck_account_existence </entry> - <entry>60 </entry> - <entry>Cache timeout in seconds to send memcached for account existence </entry> - </row> - <row> - <entry>recheck_container_existence </entry> - <entry>60 </entry> - <entry>Cache timeout in seconds to send memcached for container existence </entry> - </row> - <row> - <entry>object_chunk_size </entry> - <entry>65536 </entry> - <entry>Chunk size to read from object servers </entry> - </row> - <row> - <entry>client_chunk_size </entry> - <entry>65536 </entry> - <entry>Chunk size to read from clients </entry> - </row> - <row> - <entry>memcache_servers </entry> - <entry>127.0.0.1:11211 </entry> - <entry>Comma separated list of memcached servers ip:port </entry> - </row> - <row> - <entry>node_timeout </entry> - <entry>10 </entry> - <entry>Request timeout to external services </entry> - </row> - <row> - <entry>client_timeout </entry> - <entry>60 </entry> - <entry>Timeout to read one chunk from a client </entry> - </row> - <row> - <entry>conn_timeout </entry> - <entry>0.5 </entry> - <entry>Connection timeout to external services </entry> - </row> - <row> - <entry>error_suppression_interval </entry> - <entry>60 </entry> - <entry>Time in seconds that must elapse since the last error for a node to be considered no longer error limited </entry> - </row> - <row> - <entry>error_suppression_limit </entry> - <entry>10 </entry> - <entry>Error count to consider a node error limited </entry> - </row> - <row> - <entry>allow_account_management </entry> - <entry>false </entry> - <entry>Whether account <literal>PUT</literal>s and <literal>DELETE</literal>s are even callable </entry> - </row> - </tbody> - </tgroup> - </table> - </section> - <section> - <title>Configuring Object Server</title> - <para>The Object Server is a very simple blob storage server that can store, retrieve, and delete objects stored on local devices. Objects are stored as binary files on the file system with metadata stored in the file’s extended attributes (xattrs). This requires that the underlying file system choice for object servers support xattrs on files. - -</para> - <para>The configurable options pertaining Object Server are stored in the file <filename>/etc/swift/object-server/1.conf</filename>. The following is the sample <filename>object-server/1.conf</filename> file:</para> - <para><programlisting>[DEFAULT] -devices = /srv/1/node -mount_check = false -bind_port = 6010 -user = root -log_facility = LOG_LOCAL2 - -[pipeline:main] -pipeline = gluster object-server - -[app:object-server] -use = egg:swift#object - -[filter:gluster] -use = egg:swift#gluster - -[object-replicator] -vm_test_mode = yes - -[object-updater] -[object-auditor]</programlisting></para> - <para>The following are the configurable options: -</para> - <table frame="all"> - <title>object-server.conf Default Options in the [DEFAULT] section </title> - <tgroup cols="3"> - <colspec colname="c1"/> - <colspec colname="c2"/> - <colspec colname="c3"/> - <thead> - <row> - <entry>Option </entry> - <entry>Default </entry> - <entry>Description </entry> - </row> - </thead> - <tbody> - <row> - <entry>swift_dir </entry> - <entry>/etc/swift </entry> - <entry>Swift configuration directory </entry> - </row> - <row> - <entry>devices </entry> - <entry>/srv/node </entry> - <entry>Mount parent directory where devices are mounted </entry> - </row> - <row> - <entry>mount_check </entry> - <entry>true </entry> - <entry>Whether or not check if the devices are mounted to prevent accidentally writing to the root device </entry> - </row> - <row> - <entry>bind_ip </entry> - <entry>0.0.0.0 </entry> - <entry>IP Address for server to bind</entry> - </row> - <row> - <entry>bind_port </entry> - <entry>6000 </entry> - <entry>Port for server to bind</entry> - </row> - <row> - <entry>workers </entry> - <entry>1 </entry> - <entry>Number of workers to fork </entry> - </row> - </tbody> - </tgroup> - </table> - <table frame="all"> - <title>object-server.conf Server Options in the [object-server] section </title> - <tgroup cols="3"> - <colspec colname="c1"/> - <colspec colname="c2"/> - <colspec colname="c3"/> - <thead> - <row> - <entry>Option </entry> - <entry>Default </entry> - <entry>Description </entry> - </row> - </thead> - <tbody> - <row> - <entry>use </entry> - <entry/> - <entry>paste.deploy entry point for the object server. For most cases, this should be <literal>egg:swift#object</literal>. </entry> - </row> - <row> - <entry>log_name </entry> - <entry>object-server </entry> - <entry>log name used when logging </entry> - </row> - <row> - <entry>log_facility </entry> - <entry>LOG_LOCAL0 </entry> - <entry>Syslog log facility </entry> - </row> - <row> - <entry>log_level </entry> - <entry>INFO </entry> - <entry>Logging level </entry> - </row> - <row> - <entry>log_requests </entry> - <entry>True </entry> - <entry>Whether or not to log each request </entry> - </row> - <row> - <entry>user </entry> - <entry>swift </entry> - <entry>swift user</entry> - </row> - <row> - <entry>node_timeout </entry> - <entry>3</entry> - <entry>Request timeout to external services </entry> - </row> - <row> - <entry>conn_timeout </entry> - <entry>0.5</entry> - <entry>Connection timeout to external services </entry> - </row> - <row> - <entry>network_chunk_size </entry> - <entry>65536 </entry> - <entry>Size of chunks to read or write over the network </entry> - </row> - <row> - <entry>disk_chunk_size </entry> - <entry>65536 </entry> - <entry>Size of chunks to read or write to disk </entry> - </row> - <row> - <entry>max_upload_time </entry> - <entry>65536 </entry> - <entry>Maximum time allowed to upload an object </entry> - </row> - <row> - <entry>slow </entry> - <entry>0</entry> - <entry>If > 0, Minimum time in seconds for a <literal>PUT</literal> or <literal>DELETE</literal> request to complete </entry> - </row> - </tbody> - </tgroup> - </table> - </section> - <section> - <title>Configuring Container Server</title> - <para>The Container Server’s primary job is to handle listings of objects. The listing is done by querying the GlusterFS mount point with path. This query returns a list of all files and directories present under that container. -</para> - <para>The configurable options pertaining to container server are stored in <filename>/etc/swift/container-server/1.conf</filename> file. The following is the sample <filename>container-server/1.conf</filename> file:</para> - <para><programlisting>[DEFAULT] -devices = /srv/1/node -mount_check = false -bind_port = 6011 -user = root -log_facility = LOG_LOCAL2 - -[pipeline:main] -pipeline = gluster container-server - -[app:container-server] -use = egg:swift#container - -[filter:gluster] -use = egg:swift#gluster - -[container-replicator] -[container-updater] -[container-auditor]</programlisting></para> - <para>The following are the configurable options:</para> - <table frame="all"> - <title>container-server.conf Default Options in the [DEFAULT] section </title> - <tgroup cols="3"> - <colspec colname="c1"/> - <colspec colname="c2"/> - <colspec colname="c3"/> - <thead> - <row> - <entry>Option </entry> - <entry>Default </entry> - <entry>Description </entry> - </row> - </thead> - <tbody> - <row> - <entry>swift_dir </entry> - <entry>/etc/swift </entry> - <entry>Swift configuration directory </entry> - </row> - <row> - <entry>devices </entry> - <entry>/srv/node </entry> - <entry>Mount parent directory where devices are mounted </entry> - </row> - <row> - <entry>mount_check </entry> - <entry>true </entry> - <entry>Whether or not check if the devices are mounted to prevent accidentally writing to the root device </entry> - </row> - <row> - <entry>bind_ip </entry> - <entry>0.0.0.0 </entry> - <entry>IP Address for server to bind</entry> - </row> - <row> - <entry>bind_port </entry> - <entry>6001 </entry> - <entry>Port for server to bind</entry> - </row> - <row> - <entry>workers </entry> - <entry>1 </entry> - <entry>Number of workers to fork </entry> - </row> - <row> - <entry>user </entry> - <entry>swift </entry> - <entry>Swift user</entry> - </row> - </tbody> - </tgroup> - </table> - <table frame="all"> - <title>container-server.conf Server Options in the [container-server] section </title> - <tgroup cols="3"> - <colspec colname="c1"/> - <colspec colname="c2"/> - <colspec colname="c3"/> - <thead> - <row> - <entry>Option </entry> - <entry>Default </entry> - <entry>Description </entry> - </row> - </thead> - <tbody> - <row> - <entry>use </entry> - <entry/> - <entry>paste.deploy entry point for the container server. For most cases, this should be <literal>egg:swift#container</literal>. </entry> - </row> - <row> - <entry>log_name </entry> - <entry>container-server </entry> - <entry>Label used when logging </entry> - </row> - <row> - <entry>log_facility </entry> - <entry>LOG_LOCAL0 </entry> - <entry>Syslog log facility </entry> - </row> - <row> - <entry>log_level </entry> - <entry>INFO </entry> - <entry>Logging level </entry> - </row> - <row> - <entry>node_timeout </entry> - <entry>3 </entry> - <entry>Request timeout to external services </entry> - </row> - <row> - <entry>conn_timeout </entry> - <entry>0.5 </entry> - <entry>Connection timeout to external services </entry> - </row> - </tbody> - </tgroup> - </table> - </section> - <section> - <title>Configuring Account Server</title> - <para>The Account Server is very similar to the Container Server, except that it is responsible for listing of containers rather than objects. In UFO, each gluster volume is an account. -</para> - <para>The configurable options pertaining to account server are stored in <filename>/etc/swift/account-server/1.conf</filename> file. The following is the sample <filename>account-server/1.conf</filename> file: </para> - <para><programlisting>[DEFAULT] -devices = /srv/1/node -mount_check = false -bind_port = 6012 -user = root -log_facility = LOG_LOCAL2 - -[pipeline:main] -pipeline = gluster account-server - -[app:account-server] -use = egg:swift#account - -[filter:gluster] -use = egg:swift#gluster - -[account-replicator] -vm_test_mode = yes - -[account-auditor] -[account-reaper]</programlisting></para> - <para>The following are the configurable options:</para> - <table frame="all"> - <title>account-server.conf Default Options in the [DEFAULT] section </title> - <tgroup cols="3"> - <colspec colname="c1"/> - <colspec colname="c2"/> - <colspec colname="c3"/> - <thead> - <row> - <entry>Option </entry> - <entry>Default </entry> - <entry>Description </entry> - </row> - </thead> - <tbody> - <row> - <entry>swift_dir </entry> - <entry>/etc/swift </entry> - <entry>Swift configuration directory </entry> - </row> - <row> - <entry>devices </entry> - <entry>/srv/node </entry> - <entry>mount parent directory where devices are mounted </entry> - </row> - <row> - <entry>mount_check </entry> - <entry>true </entry> - <entry>Whether or not check if the devices are mounted to prevent accidentally writing to the root device </entry> - </row> - <row> - <entry>bind_ip </entry> - <entry>0.0.0.0 </entry> - <entry>IP Address for server to bind</entry> - </row> - <row> - <entry>bind_port </entry> - <entry>6002 </entry> - <entry>Port for server to bind</entry> - </row> - <row> - <entry>workers </entry> - <entry>1 </entry> - <entry>Number of workers to fork </entry> - </row> - <row> - <entry>user </entry> - <entry>swift </entry> - <entry>Swift user</entry> - </row> - </tbody> - </tgroup> - </table> - <table frame="all"> - <title>account-server.conf Server Options in the [account-server] section </title> - <tgroup cols="3"> - <colspec colname="c1"/> - <colspec colname="c2"/> - <colspec colname="c3"/> - <thead> - <row> - <entry>Option </entry> - <entry>Default </entry> - <entry>Description </entry> - </row> - </thead> - <tbody> - <row> - <entry>use </entry> - <entry/> - <entry>paste.deploy entry point for the container server. For most cases, this should be <literal>egg:swift#container</literal>. </entry> - </row> - <row> - <entry>log_name </entry> - <entry>account-server </entry> - <entry>Label used when logging </entry> - </row> - <row> - <entry>log_facility </entry> - <entry>LOG_LOCAL0 </entry> - <entry>Syslog log facility </entry> - </row> - <row> - <entry>log_level </entry> - <entry>INFO </entry> - <entry>Logging level </entry> - </row> - </tbody> - </tgroup> - </table> - </section> - <section> - <title>Starting and Stopping Server</title> - <para>You must start the server manually when system reboots and whenever you update/modify the configuration files.</para> - <itemizedlist> - <listitem> - <para>To start the server, enter the following command:</para> - <para><command># swift_init main start</command></para> - </listitem> - <listitem> - <para>To stop the server, enter the following command:</para> - <para><command># swift_init main stop</command></para> - </listitem> - </itemizedlist> - </section> - </section> - <section> - <title>Working with Unified File and Object Storage</title> - <para>This section describes the REST API for administering and managing Object Storage. All requests will -be directed to the host and URL described in the <filename>X-Storage-URL HTTP</filename> header obtained during -successful authentication. -</para> - <section> - <title>Configuring Authenticated Access </title> - <para>Authentication is the process of proving identity to the system. To use the REST interface, you must -obtain an authorization token using GET method and supply it with v1.0 as the path. -</para> - <para>Each REST request against the Object Storage system requires the addition of a specific authorization -token HTTP x-header, defined as X-Auth-Token. The storage URL and authentication token are -returned in the headers of the response. -</para> - <itemizedlist> - <listitem> - <para>To authenticate, run the following command: -</para> - <programlisting>GET auth/v1.0 HTTP/1.1 -Host: <auth URL> -X-Auth-User: <account name>:<user name> -X-Auth-Key: <user-Password></programlisting> - <para>For example, -</para> - <programlisting>GET auth/v1.0 HTTP/1.1 -Host: auth.example.com -X-Auth-User: test:tester -X-Auth-Key: testing - -HTTP/1.1 200 OK -X-Storage-Url: https:/example.storage.com:443/v1/AUTH_test -X-Storage-Token: AUTH_tkde3ad38b087b49bbbac0494f7600a554 -X-Auth-Token: AUTH_tkde3ad38b087b49bbbac0494f7600a554 -Content-Length: 0 -Date: Wed, 10 jul 2011 06:11:51 GMT</programlisting> - <para>To authenticate access using cURL (for the above example), run the following -command: -</para> - <programlisting>curl -v -H 'X-Storage-User: test:tester' -H 'X-Storage-Pass:testing' -k -https://auth.example.com:443/auth/v1.0</programlisting> - <para>The X-Auth-Url has to be parsed and used in the connection and request line of all subsequent -requests to the server. In the example output, users connecting to server will send most -container/object requests with a host header of example.storage.com and the request line's version -and account as v1/AUTH_test. - -</para> - </listitem> - </itemizedlist> - <note> - <para>The authentication tokens are valid for a 24 hour period. -</para> - </note> - </section> - <section> - <title>Working with Accounts </title> - <para>This section describes the list of operations you can perform at the account level of the URL. -</para> - <section> - <title>Displaying Container Information </title> - <para>You can list the objects of a specific container, or all containers, as needed using GET command. You -can use the following optional parameters with GET request to refine the results: -</para> - <para><informaltable frame="none"> - <tgroup cols="2"> - <colspec colnum="1" colname="c0" colsep="0"/> - <colspec colnum="2" colname="c1" colsep="0"/> - <thead> - <row> - <entry>Parameter </entry> - <entry>Description </entry> - </row> - </thead> - <tbody> - <row> - <entry>limit </entry> - <entry>Limits the number of results to at most <emphasis role="italic">n</emphasis> value. </entry> - </row> - <row> - <entry>marker </entry> - <entry>Returns object names greater in value than the specified marker. </entry> - </row> - <row> - <entry>format </entry> - <entry>Specify either json or xml to return the respective serialized response. </entry> - </row> - </tbody> - </tgroup> - </informaltable></para> - <para><emphasis role="bold">To display container information </emphasis></para> - <itemizedlist> - <listitem> - <para>List all the containers of an account using the following command: -</para> - <para><programlisting>GET /<apiversion>/<account> HTTP/1.1 -Host: <storage URL> -X-Auth-Token: <authentication-token-key></programlisting></para> - <para>For example, -</para> - <programlisting>GET /v1/AUTH_test HTTP/1.1 -Host: example.storage.com -X-Auth-Token: AUTH_tkd3ad38b087b49bbbac0494f7600a554 - -HTTP/1.1 200 Ok -Date: Wed, 13 Jul 2011 16:32:21 GMT -Server: Apache -Content-Type: text/plain; charset=UTF-8 -Content-Length: 39 - -songs -movies -documents -reports</programlisting> - </listitem> - </itemizedlist> - <para>To display container information using cURL (for the above example), run the following -command: -</para> - <para><programlisting>curl -v -X GET -H 'X-Auth-Token: AUTH_tkde3ad38b087b49bbbac0494f7600a554' -https://example.storage.com:443/v1/AUTH_test -k</programlisting></para> - </section> - <section> - <title>Displaying Account Metadata Information </title> - <para>You can issue HEAD command to the storage service to view the number of containers and the total -bytes stored in the account. -</para> - <itemizedlist> - <listitem> - <para>To display containers and storage used, run the following command: -</para> - <programlisting>HEAD /<apiversion>/<account> HTTP/1.1 -Host: <storage URL> -X-Auth-Token: <authentication-token-key></programlisting> - <para>For example, -</para> - <programlisting>HEAD /v1/AUTH_test HTTP/1.1 -Host: example.storage.com -X-Auth-Token: AUTH_tkd3ad38b087b49bbbac0494f7600a554 - -HTTP/1.1 204 No Content -Date: Wed, 13 Jul 2011 16:52:21 GMT -Server: Apache -X-Account-Container-Count: 4 -X-Account-Total-Bytes-Used: 394792</programlisting> - <para>To display account metadata information using cURL (for the above example), run the following -command: -</para> - <programlisting>curl -v -X HEAD -H 'X-Auth-Token: -AUTH_tkde3ad38b087b49bbbac0494f7600a554' -https://example.storage.com:443/v1/AUTH_test -k</programlisting> - </listitem> - </itemizedlist> - </section> - </section> - <section> - <title>Working with Containers </title> - <para>This section describes the list of operations you can perform at the container level of the URL. -</para> - <section> - <title> Creating Containers </title> - <para>You can use PUT command to create containers. Containers are the storage folders for your data. -The URL encoded name must be less than 256 bytes and cannot contain a forward slash '/' character. -</para> - <itemizedlist> - <listitem> - <para>To create a container, run the following command: -</para> - <programlisting>PUT /<apiversion>/<account>/<container>/ HTTP/1.1 -Host: <storage URL> -X-Auth-Token: <authentication-token-key></programlisting> - <para>For example, -</para> - <programlisting>PUT /v1/AUTH_test/pictures/ HTTP/1.1 -Host: example.storage.com -X-Auth-Token: AUTH_tkd3ad38b087b49bbbac0494f7600a554 -HTTP/1.1 201 Created - -Date: Wed, 13 Jul 2011 17:32:21 GMT -Server: Apache -Content-Type: text/plain; charset=UTF-8</programlisting> - <para>To create container using cURL (for the above example), run the following command: -</para> - <programlisting>curl -v -X PUT -H 'X-Auth-Token: -AUTH_tkde3ad38b087b49bbbac0494f7600a554' -https://example.storage.com:443/v1/AUTH_test/pictures -k</programlisting> - <para>The status code of 201 (Created) indicates that you have successfully created the container. If a -container with same is already existed, the status code of 202 is displayed. -</para> - </listitem> - </itemizedlist> - </section> - <section> - <title>Displaying Objects of a Container </title> - <para>You can list the objects of a container using GET command. You can use the following optional -parameters with GET request to refine the results: -</para> - <para><informaltable frame="none"> - <tgroup cols="2"> - <colspec colnum="1" colname="c0" colsep="0"/> - <colspec colnum="2" colname="c1" colsep="0"/> - <thead> - <row> - <entry>Parameter </entry> - <entry>Description </entry> - </row> - </thead> - <tbody> - <row> - <entry>limit </entry> - <entry>Limits the number of results to at most <emphasis role="italic">n</emphasis> value. </entry> - </row> - <row> - <entry>marker </entry> - <entry>Returns object names greater in value than the specified marker. </entry> - </row> - <row> - <entry>prefix </entry> - <entry>Displays the results limited to object names beginning with the substring x. beginning with the substring x. </entry> - </row> - <row> - <entry>path </entry> - <entry>Returns the object names nested in the pseudo path. </entry> - </row> - <row> - <entry>format </entry> - <entry>Specify either json or xml to return the respective serialized response. </entry> - </row> - <row> - <entry>delimiter </entry> - <entry>Returns all the object names nested in the container. </entry> - </row> - </tbody> - </tgroup> - </informaltable></para> - <para>To display objects of a container -</para> - <itemizedlist> - <listitem> - <para>List objects of a specific container using the following command: -</para> - </listitem> - </itemizedlist> - <programlisting>GET /<apiversion>/<account>/<container>[parm=value] HTTP/1.1 -Host: <storage URL> -X-Auth-Token: <authentication-token-key></programlisting> - <para>For example, -</para> - <programlisting>GET /v1/AUTH_test/images HTTP/1.1 -Host: example.storage.com -X-Auth-Token: AUTH_tkd3ad38b087b49bbbac0494f7600a554 - -HTTP/1.1 200 Ok -Date: Wed, 13 Jul 2011 15:42:21 GMT -Server: Apache -Content-Type: text/plain; charset=UTF-8 -Content-Length: 139 - -sample file.jpg -test-file.pdf -You and Me.pdf -Puddle of Mudd.mp3 -Test Reports.doc</programlisting> - <para>To display objects of a container using cURL (for the above example), run the following -command: -</para> - <programlisting>curl -v -X GET-H 'X-Auth-Token: AUTH_tkde3ad38b087b49bbbac0494f7600a554' -https://example.storage.com:443/v1/AUTH_test/images -k</programlisting> - </section> - <section> - <title>Displaying Container Metadata Information </title> - <para>You can issue HEAD command to the storage service to view the number of objects in a container and -the total bytes of all the objects stored in the container. -</para> - <itemizedlist> - <listitem> - <para>To display list of objects and storage used, run the following command: -</para> - <programlisting>HEAD /<apiversion>/<account>/<container> HTTP/1.1 -Host: <storage URL> -X-Auth-Token: <authentication-token-key></programlisting> - <para>For example,</para> - <programlisting>HEAD /v1/AUTH_test/images HTTP/1.1 -Host: example.storage.com -X-Auth-Token: AUTH_tkd3ad38b087b49bbbac0494f7600a554 - -HTTP/1.1 204 No Content -Date: Wed, 13 Jul 2011 19:52:21 GMT -Server: Apache -X-Account-Object-Count: 8 -X-Container-Bytes-Used: 472</programlisting> - <para>To display list of objects and storage used in a container using cURL (for the above example), run -the following command: -</para> - <programlisting>curl -v -X HEAD -H 'X-Auth-Token: -AUTH_tkde3ad38b087b49bbbac0494f7600a554' -https://example.storage.com:443/v1/AUTH_test/images -k</programlisting> - </listitem> - </itemizedlist> - </section> - <section> - <title>Deleting Container </title> - <para>You can use DELETE command to permanently delete containers. The container must be empty -before it can be deleted. -</para> - <para>You can issue HEAD command to determine if it contains any objects. -</para> - <itemizedlist> - <listitem> - <para>To delete a container, run the following command: -</para> - <programlisting>DELETE /<apiversion>/<account>/<container>/ HTTP/1.1 -Host: <storage URL> -X-Auth-Token: <authentication-token-key></programlisting> - <para>For example,</para> - <programlisting>DELETE /v1/AUTH_test/pictures HTTP/1.1 -Host: example.storage.com -X-Auth-Token: AUTH_tkd3ad38b087b49bbbac0494f7600a554 - -HTTP/1.1 204 No Content -Date: Wed, 13 Jul 2011 17:52:21 GMT -Server: Apache -Content-Length: 0 -Content-Type: text/plain; charset=UTF-8</programlisting> - <para>To delete a container using cURL (for the above example), run the following command: -</para> - <programlisting>curl -v -X DELETE -H 'X-Auth-Token: -AUTH_tkde3ad38b087b49bbbac0494f7600a554' -https://example.storage.com:443/v1/AUTH_test/pictures -k</programlisting> - <para>The status code of 204 (No Content) indicates that you have successfully deleted the container. If -that container does not exist, the status code 404 (Not Found) is displayed, and if the container is -not empty, the status code 409 (Conflict) is displayed. -</para> - </listitem> - </itemizedlist> - </section> - <section> - <title>Updating Container Metadata </title> - <para>You can update the metadata of container using POST operation, metadata keys should be prefixed -with 'x-container-meta'. -</para> - <itemizedlist> - <listitem> - <para>To update the metadata of the object, run the following command: -</para> - <programlisting>POST /<apiversion>/<account>/<container> HTTP/1.1 -Host: <storage URL> -X-Auth-Token: <Authentication-token-key> -X-Container-Meta-<key>: <new value> -X-Container-Meta-<key>: <new value></programlisting> - <para>For example, -</para> - <para><programlisting>POST /v1/AUTH_test/images HTTP/1.1 -Host: example.storage.com -X-Auth-Token: AUTH_tkd3ad38b087b49bbbac0494f7600a554 -X-Container-Meta-Zoo: Lion -X-Container-Meta-Home: Dog - -HTTP/1.1 204 No Content -Date: Wed, 13 Jul 2011 20:52:21 GMT -Server: Apache -Content-Type: text/plain; charset=UTF-8</programlisting></para> - <para>To update the metadata of the object using cURL (for the above example), run the following -command: -</para> - <para><programlisting>curl -v -X POST -H 'X-Auth-Token: -AUTH_tkde3ad38b087b49bbbac0494f7600a554' -https://example.storage.com:443/v1/AUTH_test/images -H ' X-Container-Meta-Zoo: Lion' -H 'X-Container-Meta-Home: Dog' -k</programlisting></para> - <para>The status code of 204 (No Content) indicates the container's metadata is updated successfully. If -that object does not exist, the status code 404 (Not Found) is displayed. -</para> - </listitem> - </itemizedlist> - </section> - <section id="chap-Administration_Guide-Working_UFO-Setting_ACLs"> - <title> Setting ACLs on Container </title> - <para>You can set the container access control list by using POST command on container with <command>x- container-read</command> and<command> x-container-write</command> keys. -</para> - <para>The ACL format is <command>[item[,item...]]</command>. Each item can be a group name to give access to or a -referrer designation to grant or deny based on the HTTP Referer header. -</para> - <para>The referrer designation format is:<command> .r:[-]value</command>. -</para> - <para>The .r can also be <command>.ref, .referer, </command>or .<command>referrer</command>; though it will be shortened to.r for -decreased character count usage. The value can be <command>*</command> to specify any referrer host is allowed access. The leading minus sign (-) -indicates referrer hosts that should be denied access. -</para> - <para>Examples of valid ACLs: -</para> - <para><programlisting>.r:* -.r:*,bobs_account,sues_account:sue -bobs_account,sues_account:sue</programlisting></para> - <para>Examples of invalid ACLs:</para> - <para><programlisting>.r: -.r:-</programlisting></para> - <para>By default, allowing read access via <command><command>.</command>r </command>will not allow listing objects in the container but allows -retrieving objects from the container. To turn on listings, use the .<command>rlistings</command> directive. Also, <command>.r</command> -designations are not allowed in headers whose names include the word write. -</para> - <para>For example, to set all the objects access rights to "public‟ inside the container using cURL (for the -above example), run the following command: -</para> - <para><programlisting>curl -v -X POST -H 'X-Auth-Token: -AUTH_tkde3ad38b087b49bbbac0494f7600a554' -https://example.storage.com:443/v1/AUTH_test/images --H 'X-Container-Read: .r:*' -k</programlisting></para> - </section> - </section> - <section> - <title> Working with Objects </title> - <para>An object represents the data and any metadata for the files stored in the system. Through the REST -interface, metadata for an object can be included by adding custom HTTP headers to the request -and the data payload as the request body. Objects name should not exceed 1024 bytes after URL -encoding. -</para> - <para>This section describes the list of operations you can perform at the object level of the URL. -</para> - <section> - <title>Creating or Updating Object </title> - <para>You can use PUT command to write or update an object's content and metadata. -</para> - <para>You can verify the data integrity by including an MD5checksum for the object's data in the ETag -header. ETag header is optional and can be used to ensure that the object's contents are stored -successfully in the storage system. -</para> - <para>You can assign custom metadata to objects by including additional HTTP headers on the PUT request. -The objects created with custom metadata via HTTP headers are identified with the<command>X-Object- Meta</command>- prefix. -</para> - <itemizedlist> - <listitem> - <para>To create or update an object, run the following command: -</para> - <para><programlisting>PUT /<apiversion>/<account>/<container>/<object> HTTP/1.1 -Host: <storage URL> -X-Auth-Token: <authentication-token-key> -ETag: da1e100dc9e7becc810986e37875ae38 -Content-Length: 342909 -X-Object-Meta-PIN: 2343</programlisting></para> - <para>For example,</para> - <para><programlisting>PUT /v1/AUTH_test/pictures/dog HTTP/1.1 -Host: example.storage.com -X-Auth-Token: AUTH_tkd3ad38b087b49bbbac0494f7600a554 -ETag: da1e100dc9e7becc810986e37875ae38 - -HTTP/1.1 201 Created -Date: Wed, 13 Jul 2011 18:32:21 GMT -Server: Apache -ETag: da1e100dc9e7becc810986e37875ae38 -Content-Length: 0 -Content-Type: text/plain; charset=UTF-8</programlisting></para> - <para>To create or update an object using cURL (for the above example), run the following command: -</para> - <para><programlisting>curl -v -X PUT -H 'X-Auth-Token: -AUTH_tkde3ad38b087b49bbbac0494f7600a554' -https://example.storage.com:443/v1/AUTH_test/pictures/dog -H 'Content- -Length: 0' -k</programlisting></para> - <para>The status code of 201 (Created) indicates that you have successfully created or updated the object. -If there is a missing content-Length or Content-Type header in the request, the status code of 412 -(Length Required) is displayed. (Optionally) If the MD5 checksum of the data written to the storage -system does not match the ETag value, the status code of 422 (Unprocessable Entity) is displayed. -</para> - </listitem> - </itemizedlist> - <section> - <title>Chunked Transfer Encoding </title> - <para>You can upload data without knowing the size of the data to be uploaded. You can do this by -specifying an HTTP header of Transfer-Encoding: chunked and without using a Content-Length -header. -</para> - <para>You can use this feature while doing a DB dump, piping the output through gzip, and then piping the -data directly into Object Storage without having to buffer the data to disk to compute the file size. -</para> - <itemizedlist> - <listitem> - <para>To create or update an object, run the following command: - </para> - <para><programlisting>PUT /<apiversion>/<account>/<container>/<object> HTTP/1.1 -Host: <storage URL> -X-Auth-Token: <authentication-token-key> -Transfer-Encoding: chunked -X-Object-Meta-PIN: 2343</programlisting></para> - <para>For example, -</para> - <para><programlisting>PUT /v1/AUTH_test/pictures/cat HTTP/1.1 -Host: example.storage.com -X-Auth-Token: AUTH_tkd3ad38b087b49bbbac0494f7600a554 -Transfer-Encoding: chunked -X-Object-Meta-PIN: 2343 -19 -A bunch of data broken up -D -into chunks. -0</programlisting> - -</para> - </listitem> - </itemizedlist> - </section> - </section> - <section> - <title>Copying Object </title> - <para>You can copy object from one container to another or add a new object and then add reference to -designate the source of the data from another container. -</para> - <para><emphasis role="bold">To copy object from one container to another </emphasis></para> - <itemizedlist> - <listitem> - <para>To add a new object and designate the source of the data from another container, run the -following command: -</para> - <para><programlisting>COPY /<apiversion>/<account>/<container>/<sourceobject> HTTP/1.1 -Host: <storage URL> -X-Auth-Token: < authentication-token-key> -Destination: /<container>/<destinationobject></programlisting></para> - <para>For example, -</para> - <para><programlisting>COPY /v1/AUTH_test/images/dogs HTTP/1.1 -Host: example.storage.com -X-Auth-Token: AUTH_tkd3ad38b087b49bbbac0494f7600a554 -Destination: /photos/cats - -HTTP/1.1 201 Created -Date: Wed, 13 Jul 2011 18:32:21 GMT -Server: Apache -Content-Length: 0 -Content-Type: text/plain; charset=UTF-8</programlisting></para> - <para>To copy an object using cURL (for the above example), run the following command: -</para> - <para><programlisting>curl -v -X COPY -H 'X-Auth-Token: -AUTH_tkde3ad38b087b49bbbac0494f7600a554' -H 'Destination: /photos/cats' -k https://example.storage.com:443/v1/AUTH_test/images/dogs</programlisting></para> - <para>The status code of 201 (Created) indicates that you have successfully copied the object. If there is a -missing content-Length or Content-Type header in the request, the status code of 412 (Length -Required) is displayed. -</para> - <para>You can also use PUT command to copy object by using additional header <command>X-Copy-From: container/obj</command>. -</para> - </listitem> - <listitem> - <para>To use PUT command to copy an object, run the following command: -</para> - <para><programlisting>PUT /v1/AUTH_test/photos/cats HTTP/1.1 -Host: example.storage.com -X-Auth-Token: AUTH_tkd3ad38b087b49bbbac0494f7600a554 -X-Copy-From: /images/dogs - -HTTP/1.1 201 Created -Date: Wed, 13 Jul 2011 18:32:21 GMT -Server: Apache -Content-Type: text/plain; charset=UTF-8</programlisting></para> - <para>To copy an object using cURL (for the above example), run the following command: -</para> - <para><programlisting>curl -v -X PUT -H 'X-Auth-Token: AUTH_tkde3ad38b087b49bbbac0494f7600a554' --H 'X-Copy-From: /images/dogs' –k -https://example.storage.com:443/v1/AUTH_test/images/cats</programlisting></para> - <para>The status code of 201 (Created) indicates that you have successfully copied the object. -</para> - </listitem> - </itemizedlist> - </section> - <section> - <title>Displaying Object Information </title> - <para>You can issue GET command on an object to view the object data of the object. -</para> - <itemizedlist> - <listitem> - <para>To display the content of an object run the following command:</para> - <para><programlisting>GET /<apiversion>/<account>/<container>/<object> HTTP/1.1 -Host: <storage URL> -X-Auth-Token: <Authentication-token-key></programlisting></para> - <para>For example, -</para> - <para><programlisting>GET /v1/AUTH_test/images/cat HTTP/1.1 -Host: example.storage.com -X-Auth-Token: AUTH_tkd3ad38b087b49bbbac0494f7600a554 - -HTTP/1.1 200 Ok -Date: Wed, 13 Jul 2011 23:52:21 GMT -Server: Apache -Last-Modified: Thu, 14 Jul 2011 13:40:18 GMT -ETag: 8a964ee2a5e88be344f36c22562a6486 -Content-Length: 534210 -[.........]</programlisting></para> - <para>To display the content of an object using cURL (for the above example), run the following -command: -</para> - <para><programlisting>curl -v -X GET -H 'X-Auth-Token: -AUTH_tkde3ad38b087b49bbbac0494f7600a554' -https://example.storage.com:443/v1/AUTH_test/images/cat -k</programlisting></para> - <para>The status code of 200 (Ok) indicates the object‟s data is displayed successfully. If that object does -not exist, the status code 404 (Not Found) is displayed. -</para> - </listitem> - </itemizedlist> - </section> - <section> - <title>Displaying Object Metadata </title> - <para>You can issue HEAD command on an object to view the object metadata and other standard HTTP -headers. You must send only authorization token as header. -</para> - <itemizedlist> - <listitem> - <para>To display the metadata of the object, run the following command: -</para> - </listitem> - </itemizedlist> - <para><programlisting>HEAD /<apiversion>/<account>/<container>/<object> HTTP/1.1 -Host: <storage URL> -X-Auth-Token: <Authentication-token-key></programlisting></para> - <para>For example, -</para> - <para><programlisting>HEAD /v1/AUTH_test/images/cat HTTP/1.1 -Host: example.storage.com -X-Auth-Token: AUTH_tkd3ad38b087b49bbbac0494f7600a554 - -HTTP/1.1 204 No Content -Date: Wed, 13 Jul 2011 21:52:21 GMT -Server: Apache -Last-Modified: Thu, 14 Jul 2011 13:40:18 GMT -ETag: 8a964ee2a5e88be344f36c22562a6486 -Content-Length: 512000 -Content-Type: text/plain; charset=UTF-8 -X-Object-Meta-House: Cat -X-Object-Meta-Zoo: Cat -X-Object-Meta-Home: Cat -X-Object-Meta-Park: Cat</programlisting></para> - <para>To display the metadata of the object using cURL (for the above example), run the following -command: -</para> - <para><programlisting>curl -v -X HEAD -H 'X-Auth-Token: -AUTH_tkde3ad38b087b49bbbac0494f7600a554' -https://example.storage.com:443/v1/AUTH_test/images/cat -k</programlisting></para> - <para>The status code of 204 (No Content) indicates the object‟s metadata is displayed successfully. If that -object does not exist, the status code 404 (Not Found) is displayed. -</para> - </section> - <section> - <title>Updating Object Metadata </title> - <para>You can issue POST command on an object name only to set or overwrite arbitrary key metadata. You -cannot change the object‟s other headers such as Content-Type, ETag and others using POST -operation. The POST command will delete all the existing metadata and replace it with the new -arbitrary key metadata. -</para> - <para>You must prefix <emphasis role="bold">X-Object-Meta-</emphasis> to the key names. -</para> - <itemizedlist> - <listitem> - <para>To update the metadata of an object, run the following command:</para> - <para><programlisting>POST /<apiversion>/<account>/<container>/<object> HTTP/1.1 -Host: <storage URL> -X-Auth-Token: <Authentication-token-key> -X-Object-Meta-<key>: <new value> -X-Object-Meta-<key>: <new value></programlisting> -</para> - <para>For example, -</para> - <para><programlisting>POST /v1/AUTH_test/images/cat HTTP/1.1 -Host: example.storage.com -X-Auth-Token: AUTH_tkd3ad38b087b49bbbac0494f7600a554 -X-Object-Meta-Zoo: Lion -X-Object-Meta-Home: Dog - -HTTP/1.1 202 Accepted -Date: Wed, 13 Jul 2011 22:52:21 GMT -Server: Apache -Content-Length: 0 -Content-Type: text/plain; charset=UTF-8</programlisting></para> - <para>To update the metadata of an object using cURL (for the above example), run the following -command: -</para> - <para><programlisting>curl -v -X POST -H 'X-Auth-Token: -AUTH_tkde3ad38b087b49bbbac0494f7600a554' -https://example.storage.com:443/v1/AUTH_test/images/cat -H ' X-Object- -Meta-Zoo: Lion' -H 'X-Object-Meta-Home: Dog' -k</programlisting></para> - <para>The status code of 202 (Accepted) indicates that you have successfully updated the object‟s -metadata. If that object does not exist, the status code 404 (Not Found) is displayed. - -</para> - </listitem> - </itemizedlist> - </section> - <section> - <title>Deleting Object </title> - <para>You can use DELETE command to permanently delete the object. -</para> - <para>The DELETE command on an object will be processed immediately and any subsequent operations -like GET, HEAD, POST, or DELETE on the object will display 404 (Not Found) error. -</para> - <itemizedlist> - <listitem> - <para>To delete an object, run the following command: -</para> - <para><programlisting>DELETE /<apiversion>/<account>/<container>/<object> HTTP/1.1 -Host: <storage URL> -X-Auth-Token: <Authentication-token-key></programlisting></para> - <para>For example, -</para> - <para><programlisting>DELETE /v1/AUTH_test/pictures/cat HTTP/1.1 -Host: example.storage.com -X-Auth-Token: AUTH_tkd3ad38b087b49bbbac0494f7600a554 - -HTTP/1.1 204 No Content -Date: Wed, 13 Jul 2011 20:52:21 GMT -Server: Apache -Content-Type: text/plain; charset=UTF-8</programlisting></para> - <para>To delete an object using cURL (for the above example), run the following command: -</para> - <para><programlisting>curl -v -X DELETE -H 'X-Auth-Token: -AUTH_tkde3ad38b087b49bbbac0494f7600a554' -https://example.storage.com:443/v1/AUTH_test/pictures/cat -k</programlisting></para> - <para>The status code of 204 (No Content) indicates that you have successfully deleted the object. If that -object does not exist, the status code 404 (Not Found) is displayed. -</para> - </listitem> - </itemizedlist> - </section> - </section> - </section> -</chapter> diff --git a/doc/admin-guide/en-US/admin_commandref.xml b/doc/admin-guide/en-US/admin_commandref.xml deleted file mode 100644 index df4c78f4869..00000000000 --- a/doc/admin-guide/en-US/admin_commandref.xml +++ /dev/null @@ -1,334 +0,0 @@ -<?xml version='1.0' encoding='UTF-8'?> -<!-- This document was created with Syntext Serna Free. --><!DOCTYPE chapter PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN" "docbookV4.5/docbookx.dtd" []> -<chapter id="chap-Administration_Guide-Com_Ref"> - <title>Command Reference </title> - <para>This section describes the available commands and includes the -following section: -</para> - <itemizedlist> - <listitem> - <para>gluster Command -</para> - <para>Gluster Console Manager (command line interpreter) -</para> - </listitem> - <listitem> - <para>glusterd Daemon -</para> - <para>Gluster elastic volume management daemon -</para> - </listitem> - </itemizedlist> - <section> - <title>gluster Command </title> - <para><emphasis role="bold">NAME</emphasis> -</para> - <para>gluster - Gluster Console Manager (command line interpreter) -</para> - <para><emphasis role="bold">SYNOPSIS</emphasis> -</para> - <para>To run the program and display the gluster prompt: -</para> - <para><emphasis role="bold">gluster</emphasis> -</para> - <para>To specify a command directly: -gluster [COMMANDS] [OPTIONS] -</para> - <para><emphasis role="bold">DESCRIPTION</emphasis> -</para> - <para>The Gluster Console Manager is a command line utility for elastic volume management. You can run -the gluster command on any export server. The command enables administrators to perform cloud -operations such as creating, expanding, shrinking, rebalancing, and migrating volumes without -needing to schedule server downtime. -</para> - <para><emphasis role="bold">COMMANDS</emphasis> -</para> - <para><informaltable frame="none"> - <tgroup cols="3"> - <colspec colnum="1" colname="c0" colsep="0"/> - <colspec colnum="2" colname="cgen1" colsep="0"/> - <colspec colnum="3" colname="c1" colsep="0"/> - <thead> - <row> - <entry>Command</entry> - <entry namest="cgen1" nameend="c1">Description</entry> - </row> - </thead> - <tbody> - <row> - <entry namest="c0" nameend="c1" align="left"> - <emphasis role="bold">Volume</emphasis> - </entry> - </row> - <row> - <entry>volume info [all | VOLNAME]</entry> - <entry namest="cgen1" nameend="c1">Displays information about all volumes, or the specified volume.</entry> - </row> - <row> - <entry>volume create NEW-VOLNAME [stripe COUNT] [replica COUNT] [transport tcp | rdma | tcp,rdma] NEW-BRICK ...</entry> - <entry namest="cgen1" nameend="c1">Creates a new volume of the specified type using the specified bricks and transport type (the default transport type is tcp).</entry> - </row> - <row> - <entry>volume delete VOLNAME</entry> - <entry namest="cgen1" nameend="c1">Deletes the specified volume.</entry> - </row> - <row> - <entry>volume start VOLNAME </entry> - <entry namest="cgen1" nameend="c1">Starts the specified volume.</entry> - </row> - <row> - <entry>volume stop VOLNAME [force] </entry> - <entry namest="cgen1" nameend="c1">Stops the specified volume. </entry> - </row> - <row> - <entry>volume rename VOLNAME NEW-VOLNAME </entry> - <entry namest="cgen1" nameend="c1">Renames the specified volume.</entry> - </row> - <row> - <entry>volume help </entry> - <entry namest="cgen1" nameend="c1">Displays help for the volume command.</entry> - </row> - <row> - <entry namest="c0" nameend="c1" align="left"> - <emphasis role="bold">Brick</emphasis> - </entry> - </row> - <row> - <entry>volume add-brick VOLNAME NEW-BRICK ... </entry> - <entry namest="cgen1" nameend="c1">Adds the specified brick to the specified volume.</entry> - </row> - <row> - <entry>volume replace-brick VOLNAME (BRICK NEW-BRICK) start | pause | abort | status </entry> - <entry namest="cgen1" nameend="c1">Replaces the specified brick.</entry> - </row> - <row> - <entry>volume remove-brick VOLNAME [(replica COUNT)|(stripe COUNT)] BRICK ... </entry> - <entry namest="cgen1" nameend="c1">Removes the specified brick from the specified volume.</entry> - </row> - <row> - <entry namest="c0" nameend="c1" align="left"> - <emphasis role="bold">Rebalance</emphasis> - </entry> - </row> - <row> - <entry>volume rebalance VOLNAME start</entry> - <entry namest="cgen1" nameend="c1">Starts rebalancing the specified volume.</entry> - </row> - <row> - <entry>volume rebalance VOLNAME stop </entry> - <entry namest="cgen1" nameend="c1">Stops rebalancing the specified volume. </entry> - </row> - <row> - <entry>volume rebalance VOLNAME status </entry> - <entry namest="cgen1" nameend="c1">Displays the rebalance status of the specified volume.</entry> - </row> - <row> - <entry namest="c0" nameend="c1" align="left"> - <emphasis role="bold">Log</emphasis> - </entry> - </row> - <row> - <entry>volume log filename VOLNAME [BRICK] DIRECTORY </entry> - <entry namest="cgen1" nameend="c1">Sets the log directory for the corresponding volume/brick. </entry> - </row> - <row> - <entry>volume log rotate VOLNAME [BRICK] </entry> - <entry namest="cgen1" nameend="c1">Rotates the log file for corresponding volume/brick.</entry> - </row> - <row> - <entry>volume log locate VOLNAME [BRICK] </entry> - <entry namest="cgen1" nameend="c1">Locates the log file for corresponding volume/brick. </entry> - </row> - <row> - <entry namest="c0" nameend="c1" align="left"> - <emphasis role="bold">Peer</emphasis> - </entry> - </row> - <row> - <entry>peer probe HOSTNAME </entry> - <entry namest="cgen1" nameend="c1">Probes the specified peer. </entry> - </row> - <row> - <entry>peer detach HOSTNAME </entry> - <entry namest="cgen1" nameend="c1">Detaches the specified peer. </entry> - </row> - <row> - <entry>peer status </entry> - <entry namest="cgen1" nameend="c1">Displays the status of peers. </entry> - </row> - <row> - <entry>peer help </entry> - <entry namest="cgen1" nameend="c1">Displays help for the peer command.</entry> - </row> - <row> - <entry namest="c0" nameend="c1" align="left"> - <emphasis role="bold">Geo-replication</emphasis> - </entry> - </row> - <row> - <entry>volume geo-replication MASTER SLAVE start</entry> - <entry namest="cgen1" nameend="c1"> - <para>Start geo-replication between the hosts specified by MASTER and SLAVE. You can specify a local master volume as :VOLNAME.</para> - <para>You can specify a local slave volume as :VOLUME and a local slave directory as /DIRECTORY/SUB-DIRECTORY. You can specify a remote slave volume as DOMAIN::VOLNAME and a remote slave directory as DOMAIN:/DIRECTORY/SUB-DIRECTORY.</para> - </entry> - </row> - <row> - <entry>volume geo-replication MASTER SLAVE stop</entry> - <entry namest="cgen1" nameend="c1"> - <para>Stop geo-replication between the hosts specified by MASTER and SLAVE. You can specify a local master volume as :VOLNAME and a local master directory as /DIRECTORY/SUB-DIRECTORY.</para> - <para>You can specify a local slave volume as :VOLNAME and a local slave directory as /DIRECTORY/SUB-DIRECTORY. You can specify a remote slave volume as DOMAIN::VOLNAME and a remote slave directory as DOMAIN:/DIRECTORY/SUB-DIRECTORY. -</para> - </entry> - </row> - <row> - <entry morerows="10">volume geo-replication MASTER SLAVE config [options]</entry> - <entry/> - <entry>Configure geo-replication options between the hosts specified by MASTER and SLAVE. </entry> - </row> - <row> - <entry>gluster-command COMMAND</entry> - <entry>The path where the gluster command is installed.</entry> - </row> - <row> - <entry>gluster-log-level LOGFILELEVEL</entry> - <entry>The log level for gluster processes.</entry> - </row> - <row> - <entry>log-file LOGFILE</entry> - <entry>The path to the geo-replication log file.</entry> - </row> - <row> - <entry>log-level LOGFILELEVEL</entry> - <entry>The log level for geo-replication.</entry> - </row> - <row> - <entry>remote-gsyncd COMMAND</entry> - <entry>The path where the gsyncd binary is installed on the remote machine.</entry> - </row> - <row> - <entry>ssh-command COMMAND</entry> - <entry>The ssh command to use to connect to the remote machine (the default is ssh).</entry> - </row> - <row> - <entry>rsync-command COMMAND</entry> - <entry>The rsync command to use for synchronizing the files (the default is rsync).</entry> - </row> - <row> - <entry>volume_id= UID</entry> - <entry>The command to delete the existing master UID for the intermediate/slave node.</entry> - </row> - <row> - <entry>timeout SECONDS</entry> - <entry>The timeout period.</entry> - </row> - <row> - <entry>sync-jobs N</entry> - <entry>The number of simultaneous files/directories that can be synchronized.</entry> - </row> - <row> - <entry/> - <entry>ignore-deletes</entry> - <entry>If this option is set to 1, a file deleted on master will not trigger a delete operation on the slave. Hence, the slave will remain as a superset of the master and can be used to recover the master in case of crash and/or accidental delete.</entry> - </row> - <row> - <entry namest="c0" nameend="c1" align="left"> - <emphasis role="bold">Other</emphasis> - </entry> - </row> - <row> - <entry>help</entry> - <entry/> - <entry>Display the command options.</entry> - </row> - <row> - <entry>quit</entry> - <entry/> - <entry>Exit the gluster command line interface.</entry> - </row> - </tbody> - </tgroup> - </informaltable></para> - <para><emphasis role="bold">FILES</emphasis> - -</para> - <para>/etc/glusterd/* -</para> - <para><emphasis role="bold">SEE ALSO </emphasis></para> - <para>fusermount(1), mount.glusterfs(8), glusterfs-volgen(8), glusterfs(8), glusterd(8)</para> - </section> - <section> - <title>glusterd Daemon </title> - <para><emphasis role="bold">NAME</emphasis> -</para> - <para>glusterd - Gluster elastic volume management daemon</para> - <para><emphasis role="bold">SYNOPSIS</emphasis> -</para> - <para>glusterd [OPTION...] -</para> - <para><emphasis role="bold">DESCRIPTION</emphasis> -</para> - <para>The glusterd daemon is used for elastic volume management. The daemon must be run on all export servers. -</para> - <para><emphasis role="bold">OPTIONS</emphasis> -</para> - <para><informaltable frame="none"> - <tgroup cols="2"> - <colspec colnum="1" colname="c0" colsep="0"/> - <colspec colnum="2" colname="c1" colsep="0"/> - <thead> - <row> - <entry>Option</entry> - <entry>Description</entry> - </row> - </thead> - <tbody> - <row> - <entry namest="c0" nameend="c1" align="left"> - <emphasis role="bold">Basic</emphasis> - </entry> - </row> - <row> - <entry>-l=LOGFILE, --log-file=LOGFILE</entry> - <entry>Files to use for logging (the default is /usr/local/var/log/glusterfs/glusterfs.log).</entry> - </row> - <row> - <entry>-L=LOGLEVEL, --log-level=LOGLEVEL</entry> - <entry>Logging severity. Valid options are TRACE, DEBUG, INFO, WARNING, ERROR and CRITICAL (the default is INFO). </entry> - </row> - <row> - <entry>--debug</entry> - <entry>Runs the program in debug mode. This option sets --no-daemon, --log-level to DEBUG, and --log-file to console.</entry> - </row> - <row> - <entry>-N, --no-daemon</entry> - <entry>Runs the program in the foreground.</entry> - </row> - <row> - <entry namest="c0" nameend="c1" align="left"> - <emphasis role="bold">Miscellaneous</emphasis> - </entry> - </row> - <row> - <entry>-?, --help</entry> - <entry>Displays this help.</entry> - </row> - <row> - <entry>--usage</entry> - <entry>Displays a short usage message.</entry> - </row> - <row> - <entry>-V, --version</entry> - <entry>Prints the program version.</entry> - </row> - </tbody> - </tgroup> - </informaltable></para> - <para><emphasis role="bold">FILES</emphasis> - -</para> - <para>/etc/glusterd/* -</para> - <para><emphasis role="bold">SEE ALSO </emphasis></para> - <para>fusermount(1), mount.glusterfs(8), glusterfs-volgen(8), glusterfs(8), gluster(8)</para> - </section> -</chapter> diff --git a/doc/admin-guide/en-US/admin_console.xml b/doc/admin-guide/en-US/admin_console.xml deleted file mode 100644 index ebf273935ca..00000000000 --- a/doc/admin-guide/en-US/admin_console.xml +++ /dev/null @@ -1,28 +0,0 @@ -<?xml version='1.0' encoding='UTF-8'?> -<!-- This document was created with Syntext Serna Free. --><!DOCTYPE chapter PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN" "docbookV4.5/docbookx.dtd" []> -<chapter> - <title>Using the Gluster Console Manager – Command Line Utility</title> - <para>The Gluster Console Manager is a single command line utility that simplifies configuration and management of your storage environment. The Gluster Console Manager is similar to the LVM (Logical Volume Manager) CLI or ZFS Command Line Interface, but across multiple storage servers. You can use the Gluster Console Manager online, while volumes are mounted and active. Gluster automatically synchronizes volume configuration information across all Gluster servers.</para> - <para>Using the Gluster Console Manager, you can create new volumes, start volumes, and stop volumes, as required. You can also add bricks to volumes, remove bricks from existing volumes, as well as change translator settings, among other operations.</para> - <para>You can also use the commands to create scripts for automation, as well as use the commands as an API to allow integration with third-party applications. </para> - <para><emphasis role="bold">Running the Gluster Console Manager</emphasis></para> - <para>You can run the Gluster Console Manager on any GlusterFS server either by invoking the commands or by running the Gluster CLI in interactive mode. You can also use the gluster command remotely using SSH. </para> - <itemizedlist> - <listitem> - <para>To run commands directly: </para> - <para><command> # gluster peer <replaceable>command</replaceable></command></para> - <para>For example:</para> - <para><command> # gluster peer status </command></para> - </listitem> - <listitem> - <para>To run the Gluster Console Manager in interactive mode </para> - <para><command># gluster</command></para> - <para>You can execute gluster commands from the Console Manager prompt:</para> - <para><command> gluster> <replaceable>command</replaceable></command> </para> - <para>For example, to view the status of the peer server:</para> - <para># <command>gluster </command></para> - <para><command>gluster > peer status </command></para> - <para>Display the status of the peer.</para> - </listitem> - </itemizedlist> -</chapter> diff --git a/doc/admin-guide/en-US/admin_directory_Quota.xml b/doc/admin-guide/en-US/admin_directory_Quota.xml deleted file mode 100644 index 8a1012a6ac2..00000000000 --- a/doc/admin-guide/en-US/admin_directory_Quota.xml +++ /dev/null @@ -1,179 +0,0 @@ -<?xml version='1.0' encoding='UTF-8'?> -<!-- This document was created with Syntext Serna Free. --><!DOCTYPE chapter PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN" "docbookV4.5/docbookx.dtd" []> -<chapter id="chap-Administration_Guide-Dir_Quota"> - <title>Managing Directory Quota </title> - <para>Directory quotas in GlusterFS allow you to set limits on usage of disk space by directories or volumes. -The storage administrators can control the disk space utilization at the directory and/or volume -levels in GlusterFS by setting limits to allocatable disk space at any level in the volume and directory -hierarchy. This is particularly useful in cloud deployments to facilitate utility billing model. - </para> - <para> <note> - <para>For now, only Hard limit is supported. Here, the limit cannot be exceeded and attempts to use -more disk space or inodes beyond the set limit will be denied. -</para> - </note></para> - <para>System administrators can also monitor the resource utilization to limit the storage for the users -depending on their role in the organization. -</para> - <para>You can set the quota at the following levels: - </para> - <itemizedlist> - <listitem> - <para>Directory level – limits the usage at the directory level - </para> - </listitem> - <listitem> - <para>Volume level – limits the usage at the volume level - </para> - </listitem> - </itemizedlist> - <note> - <para>You can set the disk limit on the directory even if it is not created. The disk limit is enforced -immediately after creating that directory. For more information on setting disk limit, see <xref linkend="chap-Administration_Guide-Dir_Quota-Set_Replace"/>. -</para> - </note> - <section id="chap-Administration_Guide-Dir_Quota-Enable"> - <title>Enabling Quota </title> - <para>You must enable Quota to set disk limits. -</para> - <para><emphasis role="bold">To enable quota</emphasis> -</para> - <itemizedlist> - <listitem> - <para>Enable the quota using the following command: -</para> - <para><command># gluster volume quota <replaceable>VOLNAME</replaceable> enable </command></para> - <para>For example, to enable quota on test-volume: -</para> - <programlisting># gluster volume quota test-volume enable -Quota is enabled on /test-volume</programlisting> - </listitem> - </itemizedlist> - </section> - <section id="chap-Administration_Guide-Dir_Quota-Disable"> - <title>Disabling Quota </title> - <para>You can disable Quota, if needed. -</para> - <para><emphasis role="bold">To disable quota:</emphasis> -</para> - <itemizedlist> - <listitem> - <para>Disable the quota using the following command: -</para> - <para><command># gluster volume quota <replaceable>VOLNAME</replaceable> disable </command></para> - <para>For example, to disable quota translator on test-volume: -</para> - <programlisting># gluster volume quota test-volume disable -Quota translator is disabled on /test-volume</programlisting> - </listitem> - </itemizedlist> - </section> - <section id="chap-Administration_Guide-Dir_Quota-Set_Replace"> - <title>Setting or Replacing Disk Limit </title> - <para>You can create new directories in your storage environment and set the disk limit or set disk limit for -the existing directories. The directory name should be relative to the volume with the export -directory/mount being treated as "/". -</para> - <para><emphasis role="bold">To set or replace disk limit</emphasis> -</para> - <itemizedlist> - <listitem> - <para>Set the disk limit using the following command: -</para> - <para><command># gluster volume quota <replaceable>VOLNAME</replaceable> limit-usage /<replaceable>directory</replaceable><replaceable>limit-value</replaceable></command></para> - <para>For example, to set limit on data directory on test-volume where data is a directory under the -export directory: -</para> - <programlisting># gluster volume quota test-volume limit-usage /data 10GB -Usage limit has been set on /data</programlisting> - <para><note> - <para>In a multi-level directory hierarchy, the strictest disk limit will be considered for enforcement. -</para> - </note></para> - </listitem> - </itemizedlist> - </section> - <section id="chap-Administration_Guide-Dir_Quota-Display"> - <title>Displaying Disk Limit Information </title> - <para>You can display disk limit information on all the directories on which the limit is set. -</para> - <para><emphasis role="bold">To display disk limit information</emphasis> -</para> - <itemizedlist> - <listitem> - <para>Display disk limit information of all the directories on which limit is set, using the following -command: -</para> - <para><command># gluster volume quota <replaceable>VOLNAME</replaceable> list</command> -</para> - <para>For example, to see the set disks limit on test-volume: -</para> - <programlisting># gluster volume quota test-volume list - -<emphasis role="underline"> - <emphasis role="underline"><emphasis role="underline">Path</emphasis>__________Limit______Set Size</emphasis> - </emphasis> -/Test/data 10 GB 6 GB -/Test/data1 10 GB 4 GB</programlisting> - </listitem> - <listitem> - <para>Display disk limit information on a particular directory on which limit is set, using the following -command: -</para> - <para><command># gluster volume quota <replaceable>VOLNAME</replaceable> list <replaceable>/directory name</replaceable></command> -</para> - <para>For example, to see the set limit on /data directory of test-volume:</para> - <programlisting># gluster volume quota test-volume list /data - -<emphasis role="underline"><emphasis role="underline">Path</emphasis>__________Limit______Set Size</emphasis> -/Test/data 10 GB 6 GB</programlisting> - </listitem> - </itemizedlist> - </section> - <section id="chap-Administration_Guide-Dir_Quota-Update"> - <title> Updating Memory Cache Size </title> - <para>For performance reasons, quota caches the directory sizes on client. You can set timeout indicating -the maximum valid duration of directory sizes in cache, from the time they are populated. -</para> - <para>For example: If there are multiple clients writing to a single directory, there are chances that some -other client might write till the quota limit is exceeded. However, this new file-size may not get -reflected in the client till size entry in cache has become stale because of timeout. If writes happen -on this client during this duration, they are allowed even though they would lead to exceeding of -quota-limits, since size in cache is not in sync with the actual size. When timeout happens, the size -in cache is updated from servers and will be in sync and no further writes will be allowed. A timeout -of zero will force fetching of directory sizes from server for every operation that modifies file data -and will effectively disables directory size caching on client side. -</para> - <para><emphasis role="bold">To update the memory cache size</emphasis> -</para> - <itemizedlist> - <listitem> - <para>Update the memory cache size using the following command: -</para> - <para><command># gluster volume set <replaceable>VOLNAME</replaceable> features.quota-timeout<replaceable> value</replaceable></command></para> - <para>For example, to update the memory cache size for every 5 seconds on test-volume: -</para> - <programlisting># gluster volume set test-volume features.quota-timeout 5 -Set volume successful</programlisting> - </listitem> - </itemizedlist> - </section> - <section id="chap-Administration_Guide-Dir_Quota-Remove"> - <title>Removing Disk Limit </title> - <para>You can remove set disk limit, if you do not want quota anymore. -</para> - <para><emphasis role="bold">To remove disk limit </emphasis></para> - <itemizedlist> - <listitem> - <para>Remove disk limit set on a particular directory using the following command: -</para> - <para><command># gluster volume quota <replaceable>VOLNAME</replaceable> remove <replaceable>/directory name</replaceable></command> -</para> - <para>For example, to remove the disk limit on /data directory of test-volume: -</para> - <programlisting># gluster volume quota test-volume remove /data -Usage limit set on /data is removed</programlisting> - </listitem> - </itemizedlist> - </section> -</chapter> diff --git a/doc/admin-guide/en-US/admin_geo-replication.xml b/doc/admin-guide/en-US/admin_geo-replication.xml deleted file mode 100644 index b546bb8da8c..00000000000 --- a/doc/admin-guide/en-US/admin_geo-replication.xml +++ /dev/null @@ -1,732 +0,0 @@ -<?xml version='1.0' encoding='UTF-8'?> -<!-- This document was created with Syntext Serna Free. --><!DOCTYPE chapter PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN" "docbookV4.5/docbookx.dtd" []> -<chapter id="chap-Administration_Guide-Geo_Rep"> - <title>Managing Geo-replication</title> - <para>Geo-replication provides a continuous, asynchronous, and incremental replication service from one site to another over Local Area Networks (LANs), Wide Area Network (WANs), and across the Internet. </para> - <para>Geo-replication uses a master–slave model, whereby replication and mirroring occurs between the following partners:</para> - <itemizedlist> - <listitem> - <para>Master – a GlusterFS volume </para> - </listitem> - <listitem> - <para>Slave – a slave which can be of the following types: </para> - <itemizedlist> - <listitem> - <para>A local directory which can be represented as file URL like <filename>file:///path/to/dir</filename>. You can use shortened form, for example, <filename> /path/to/dir</filename>.</para> - </listitem> - <listitem> - <para>A GlusterFS Volume - Slave volume can be either a local volume like <filename>gluster://localhost:volname</filename> (shortened form - <filename>:volname</filename>) or a volume served by different host like <filename>gluster://host:volname</filename> (shortened form - <filename>host:volname</filename>).</para> - </listitem> - </itemizedlist> - <note> - <para> Both of the above types can be accessed remotely using SSH tunnel. To use SSH, add an SSH prefix to either a file URL or gluster type URL. For example, <literal> ssh://root@remote-host:/path/to/dir</literal> (shortened form - <literal>root@remote-host:/path/to/dir</literal>) or <literal>ssh://root@remote-host:gluster://localhost:volname</literal> (shortened from - <literal>root@remote-host::volname</literal>). </para> - </note> - </listitem> - </itemizedlist> - <para>This section introduces Geo-replication, illustrates the various deployment scenarios, and explains how to configure the system to provide replication and mirroring in your environment. </para> - <section id="chap-Administration_Guide-Geo_Rep-Replicated_volumes"> - <title>Replicated Volumes vs Geo-replication</title> - <para>The following table lists the difference between replicated volumes and geo-replication:</para> - <informaltable frame="all"> - <tgroup cols="2"> - <colspec colname="c1"/> - <colspec colname="c2"/> - <thead> - <row> - <entry>Replicated Volumes</entry> - <entry>Geo-replication</entry> - </row> - </thead> - <tbody> - <row> - <entry>Mirrors data across clusters</entry> - <entry>Mirrors data across geographically distributed clusters </entry> - </row> - <row> - <entry>Provides high-availability</entry> - <entry>Ensures backing up of data for disaster recovery</entry> - </row> - <row> - <entry>Synchronous replication (each and every file operation is sent across all the bricks)</entry> - <entry>Asynchronous replication (checks for the changes in files periodically and syncs them on detecting differences) </entry> - </row> - </tbody> - </tgroup> - </informaltable> - </section> - <section id="chap-Administration_Guide-Geo_Rep-Preparation"> - <title>Preparing to Deploy Geo-replication</title> - <para>This section provides an overview of the Geo-replication deployment scenarios, describes how you can check the minimum system requirements, and explores common deployment scenarios.</para> - <itemizedlist> - <listitem> - <para><xref linkend="chap-Administration_Guide-Geo_Rep-Preparation-Deployment_options"/></para> - </listitem> - <listitem> - <para><xref linkend="chap-Administration_Guide-Geo_Rep-Preparation-Deployment_Overview"/></para> - </listitem> - <listitem> - <para><xref linkend="chap-Administration_Guide-Geo_Rep-Preparation-Minimum_Reqs"/></para> - </listitem> - <listitem> - <para><xref linkend="chap-Administration_Guide-Geo_Rep-Preparation-Settingup_Environment"/></para> - </listitem> - <listitem> - <para><xref linkend="chap-Administration_Guide-Geo_Rep-Preparation-Settingup_Slave"/></para> - </listitem> - </itemizedlist> - <section id="chap-Administration_Guide-Geo_Rep-Preparation-Deployment_options"> - <title>Exploring Geo-replication Deployment Scenarios</title> - <para>Geo-replication provides an incremental replication service over Local Area Networks (LANs), Wide Area Network (WANs), and across the Internet. This section illustrates the most common deployment scenarios for Geo-replication, including the following: </para> - <itemizedlist> - <listitem> - <para>Geo-replication over LAN -</para> - </listitem> - <listitem> - <para>Geo-replication over WAN -</para> - </listitem> - <listitem> - <para>Geo-replication over the Internet</para> - </listitem> - <listitem> - <para>Multi-site cascading Geo-replication</para> - </listitem> - </itemizedlist> - <para><emphasis role="bold">Geo-replication over LAN</emphasis></para> - <para>You can configure Geo-replication to mirror data over a Local Area Network. </para> - <mediaobject> - <textobject> - <phrase>Geo-replication over LAN</phrase> - </textobject> - <imageobject> - <imagedata fileref="images/Geo-Rep_LAN.png"/> - </imageobject> - </mediaobject> - <para><emphasis role="bold">Geo-replication over WAN</emphasis></para> - <para>You can configure Geo-replication to replicate data over a Wide Area Network.</para> - <mediaobject> - <textobject> - <phrase> - <phrase>Geo-replication over WAN</phrase> - </phrase> - </textobject> - <imageobject> - <imagedata fileref="images/Geo-Rep_WAN.png"/> - </imageobject> - </mediaobject> - <para><emphasis role="bold">Geo-replication over Internet</emphasis></para> - <para>You can configure Geo-replication to mirror data over the Internet.</para> - <mediaobject> - <textobject> - <phrase> - <phrase>Geo-replication over Internet</phrase> - </phrase> - </textobject> - <imageobject> - <imagedata fileref="images/Geo-Rep03_Internet.png"/> - </imageobject> - </mediaobject> - <para><emphasis role="bold">Multi-site cascading Geo-replication</emphasis> </para> - <para>You can configure Geo-replication to mirror data in a cascading fashion across multiple sites. </para> - <mediaobject> - <textobject> - <phrase> - <phrase>Multi-site cascading Geo-replication </phrase> - </phrase> - </textobject> - <imageobject> - <imagedata fileref="images/Geo-Rep04_Cascading.png"/> - </imageobject> - </mediaobject> - </section> - <section id="chap-Administration_Guide-Geo_Rep-Preparation-Deployment_Overview"> - <title>Geo-replication Deployment Overview</title> - <para>Deploying Geo-replication involves the following steps:</para> - <orderedlist> - <listitem> - <para>Verify that your environment matches the minimum system requirement. For more information, see <xref linkend="chap-Administration_Guide-Geo_Rep-Preparation-Minimum_Reqs"/>.</para> - </listitem> - <listitem> - <para>Determine the appropriate deployment scenario. For more information, see <xref linkend="chap-Administration_Guide-Geo_Rep-Preparation-Deployment_options"/>.</para> - </listitem> - <listitem> - <para>Start Geo-replication on master and slave systems, as required. For more information, see <xref linkend="chap-Administration_Guide-Geo_Rep-Starting"/>.</para> - </listitem> - </orderedlist> - </section> - <section id="chap-Administration_Guide-Geo_Rep-Preparation-Minimum_Reqs"> - <title>Checking Geo-replication Minimum Requirements</title> - <para condition="gfs">Before deploying GlusterFS Geo-replication, verify that your systems match the minimum requirements. </para> - <para condition="gfs">The following table outlines the minimum requirements for both master and slave nodes within your environment:</para> - <informaltable frame="all" condition="gfs"> - <tgroup cols="3"> - <colspec colname="c1"/> - <colspec colname="c2"/> - <colspec colname="c3"/> - <thead> - <row> - <entry>Component</entry> - <entry>Master</entry> - <entry>Slave</entry> - </row> - </thead> - <tbody> - <row> - <entry>Operating System </entry> - <entry>GNU/Linux</entry> - <entry>GNU/Linux</entry> - </row> - <row> - <entry>Filesystem</entry> - <entry>GlusterFS 3.2 or higher</entry> - <entry>GlusterFS 3.2 or higher (GlusterFS needs to be installed, but does not need to be running), ext3, ext4, or XFS (any other POSIX compliant file system would work, but has not been tested extensively) </entry> - </row> - <row> - <entry>Python </entry> - <entry>Python 2.4 (with ctypes external module), or Python 2.5 (or higher)</entry> - <entry>Python 2.4 (with ctypes external module), or Python 2.5 (or higher)</entry> - </row> - <row> - <entry>Secure shell </entry> - <entry>OpenSSH version 4.0 (or higher)</entry> - <entry>SSH2-compliant daemon </entry> - </row> - <row> - <entry>Remote synchronization</entry> - <entry>rsync 3.0.7 or higher </entry> - <entry>rsync 3.0.7 or higher </entry> - </row> - <row> - <entry>FUSE </entry> - <entry>GlusterFS supported versions </entry> - <entry>GlusterFS supported versions </entry> - </row> - </tbody> - </tgroup> - </informaltable> - </section> - <section id="chap-Administration_Guide-Geo_Rep-Preparation-Settingup_Environment"> - <title>Setting Up the Environment for Geo-replication</title> - <para><emphasis role="bold">Time Synchronization</emphasis> </para> - <itemizedlist> - <listitem> - <para>On bricks of a geo-replication master volume, all the servers' time must be uniform. You are recommended to set up NTP (Network Time Protocol) service to keep the bricks sync in time and avoid out-of-time sync effect.</para> - <para>For example: In a Replicated volume where brick1 of the master is at 12.20 hrs and brick 2 of the master is at 12.10 hrs with 10 minutes time lag, all the changes in brick2 between this period may go unnoticed during synchronization of files with Slave.</para> - <para>For more information on setting up NTP, see <ulink url="http://docs.redhat.com/docs/en-US/Red_Hat_Enterprise_Linux/6/html/Migration_Planning_Guide/ch04s07.html"/>.</para> - </listitem> - </itemizedlist> - <para><emphasis role="bold">To setup Geo-replication for SSH </emphasis></para> - <para>Password-less login has to be set up between the host machine (where geo-replication Start command will be issued) and the remote machine (where slave process should be launched through SSH).</para> - <orderedlist> - <listitem> - <para>On the node where geo-replication sessions are to be set up, run the following command:</para> - <para><command># ssh-keygen -f /etc/glusterd/geo-replication/secret.pem</command> -</para> - <para>Press Enter twice to avoid passphrase. -</para> - </listitem> - <listitem> - <para>Run the following command on master for all the slave hosts: </para> - <para><command># ssh-copy-id -i /etc/glusterd/geo-replication/secret.pem.pub <varname>user</varname>@<varname>slavehost</varname></command></para> - </listitem> - </orderedlist> - </section> - <section id="chap-Administration_Guide-Geo_Rep-Preparation-Settingup_Slave"> - <title>Setting Up the Environment for a Secure Geo-replication Slave</title> - <para>You can configure a secure slave using SSH so that master is granted a -restricted access. With GlusterFS, you need not specify -configuration parameters regarding the slave on the master-side -configuration. For example, the master does not require the location of -the rsync program on slave but the slave must ensure that rsync is in -the PATH of the user which the master connects using SSH. The only -information that master and slave have to negotiate are the slave-side -user account, slave's resources that master uses as slave resources, and -the master's public key. Secure access to the slave can be established -using the following options:</para> - <itemizedlist> - <listitem> - <para>Restricting Remote Command Execution</para> - </listitem> - <listitem> - <para>Using <filename>Mountbroker</filename> for Slaves</para> - </listitem> - <listitem> - <para>Using IP based Access Control</para> - </listitem> - </itemizedlist> - <para><emphasis role="bold">Backward Compatibility</emphasis> </para> - <para>Your existing Ge-replication environment will work with GlusterFS, -except for the following:</para> - <itemizedlist> - <listitem> - <para>The process of secure reconfiguration affects only the glusterfs -instance on slave. The changes are transparent to master with the -exception that you may have to change the SSH target to an unprivileged - account on slave.</para> - </listitem> - <listitem> - <para>The following are the some exceptions where this might not work:</para> - <para><itemizedlist> - <listitem> - <para>Geo-replication URLs which specify the slave resource when configuring master will include the following special characters: space, *, ?, [;</para> - </listitem> - <listitem> - <para>Slave must have a running instance of glusterd, even if there is no -gluster volume among the mounted slave resources (that is, file tree -slaves are used exclusively) .</para> - </listitem> - </itemizedlist></para> - </listitem> - </itemizedlist> - <section> - <title>Restricting Remote Command Execution</title> - <para>If you restrict remote command execution, then the Slave audits commands -coming from the master and the commands related to the given -geo-replication session is allowed. The Slave also provides access only -to the files within the slave resource which can be read or manipulated -by the Master.</para> - <para>To restrict remote command execution:</para> - <orderedlist> - <listitem> - <para>Identify the location of the gsyncd helper utility on Slave. This utility is installed in <filename>PREFIX/libexec/glusterfs/gsyncd</filename>, where PREFIX is a compile-time parameter of glusterfs. For example, <filename>--prefix=PREFIX</filename> to the configure script with the following common values<filename> /usr, /usr/local, and /opt/glusterfs/glusterfs_version</filename>.</para> - </listitem> - <listitem> - <para>Ensure that command invoked from master to slave passed through the slave's gsyncd utility. </para> - <para>You can use either of the following two options:</para> - <itemizedlist> - <listitem> - <para>Set gsyncd with an absolute path as the shell for the account -which the master connects through SSH. If you need to use a privileged -account, then set it up by creating a new user with UID 0. </para> - </listitem> - <listitem> - <para>Setup key authentication with command enforcement to gsyncd. You must prefix the copy of master's public key in the Slave account's <filename>authorized_keys</filename> file with the following command:</para> - <para><filename>command=<path to gsyncd></filename>. </para> - <para>For example, <command>command="PREFIX/glusterfs/gsyncd" ssh-rsa AAAAB3Nza....</command></para> - </listitem> - </itemizedlist> - </listitem> - </orderedlist> - </section> - <section> - <title>Using Mountbroker for Slaves </title> - <para><filename>mountbroker</filename> is a new service of glusterd. This service allows an -unprivileged process to own a GlusterFS mount by registering a label -(and DSL (Domain-specific language) options ) with glusterd through a -glusterd volfile. Using CLI, you can send a mount request to glusterd to -receive an alias (symlink) of the mounted volume.</para> - <para>A request from the agent , the unprivileged slave agents use the -mountbroker service of glusterd to set up an auxiliary gluster mount for -the agent in a special environment which ensures that the agent is only -allowed to access with special parameters that provide administrative -level access to the particular volume.</para> - <para><emphasis role="bold">To setup an auxiliary gluster mount for the agent</emphasis>:</para> - <orderedlist> - <listitem> - <para>Create a new group. For example, <filename>geogroup</filename>.</para> - </listitem> - <listitem> - <para>Create a unprivileged account. For example, <filename> geoaccount</filename>. Make it a member of <filename> geogroup</filename>.</para> - </listitem> - <listitem> - <para>Create a new directory owned by root and with permissions <emphasis role="italic">0711.</emphasis> For example, create a create mountbroker-root directory <filename>/var/mountbroker-root</filename>.</para> - </listitem> - <listitem> - <para>Add the following options to the glusterd volfile, assuming the name of the slave gluster volume as <filename>slavevol</filename>:</para> - <para><command>option mountbroker-root /var/mountbroker-root </command></para> - <para><command>option mountbroker-geo-replication.geoaccount slavevol</command></para> - <para><command>option geo-replication-log-group geogroup</command></para> - <para>If you are unable to locate the glusterd volfile at <filename>/etc/glusterfs/glusterd.vol</filename>, you can create a volfile containing both the default configuration and the above options and place it at <filename>/etc/glusterfs/</filename>. </para> - <para>A sample glusterd volfile along with default options:</para> - <para><screen>volume management - type mgmt/glusterd - option working-directory /etc/glusterd - option transport-type socket,rdma - option transport.socket.keepalive-time 10 - option transport.socket.keepalive-interval 2 - option transport.socket.read-fail-log off - - option mountbroker-root /var/mountbroker-root - option mountbroker-geo-replication.geoaccount slavevol - option geo-replication-log-group geogroup -end-volume</screen></para> - <para>If you host multiple slave volumes on Slave, you can repeat step 2. for each of them and add the following options to the <filename>volfile</filename>:</para> - <para><screen>option mountbroker-geo-replication.geoaccount2 slavevol2 -option mountbroker-geo-replication.geoaccount3 slavevol3</screen></para> - </listitem> - <listitem> - <para>Setup Master to access Slave as <filename>geoaccount@Slave</filename>.</para> - <para>You can add multiple slave volumes within the same account (geoaccount) by providing comma-separated list (without spaces) as the argument of <command>mountbroker-geo-replication.geogroup</command>. You can also have multiple options of the form <command>mountbroker-geo-replication.*</command>. It is recommended to use one service account per Master machine. For example, if there are multiple slave volumes on Slave for the master machines Master1, Master2, and Master3, then create a dedicated service user on Slave for them by repeating Step 2. for each (like geogroup1, geogroup2, and geogroup3), and then add the following corresponding options to the volfile: -</para> - <para><command>option mountbroker-geo-replication.geoaccount1 slavevol11,slavevol12,slavevol13</command></para> - <para><command>option mountbroker-geo-replication.geoaccount2 slavevol21,slavevol22</command></para> - <para><command>option mountbroker-geo-replication.geoaccount3 slavevol31</command></para> - <para> -Now set up Master1 to ssh to geoaccount1@Slave, etc. -</para> - <para>You must restart glusterd after making changes in the configuration to effect the updates. </para> - </listitem> - </orderedlist> - </section> - <section> - <title>Using IP based Access Control</title> - <para>You can use IP based access control method to provide access control for -the slave resources using IP address. You can use method for both Slave -and file tree slaves, but in the section, we are focusing on file tree -slaves using this method.</para> - <para>To set access control based on IP address for file tree slaves:</para> - <orderedlist> - <listitem> - <para>Set a general restriction for accessibility of file tree resources: -</para> - <para><command># gluster volume geo-replication '/*' config allow-network ::1,127.0.0.1 </command></para> - <para>This will refuse all requests for spawning slave agents except for -requests initiated locally.</para> - </listitem> - <listitem> - <para>If you want the to lease file tree at <filename>/data/slave-tree</filename> to Master, enter the following command:</para> - <para><command># gluster volume geo-replication<varname> /data/slave-tree </varname>config allow-network <varname>MasterIP</varname></command></para> - <para><varname>MasterIP</varname> is the IP address of Master. The slave agent spawn request from -master will be accepted if it is executed at <filename>/data/slave-tree</filename>.</para> - </listitem> - </orderedlist> - <para>If the Master side network configuration does not enable the Slave to -recognize the exact IP address of Master, you can use CIDR notation to -specify a subnet instead of a single IP address as MasterIP or even -comma-separated lists of CIDR subnets.</para> - <para>If you want to extend IP based access control to gluster slaves, use the following command:</para> - <para><command># gluster volume geo-replication '*' config allow-network ::1,127.0.0.1</command></para> - </section> - </section> - </section> - <section id="chap-Administration_Guide-Geo_Rep-Starting"> - <title>Starting Geo-replication</title> - <para>This section describes how to configure and start Gluster Geo-replication in your storage environment, and verify that it is functioning correctly. </para> - <itemizedlist> - <listitem> - <para><xref linkend="chap-Administration_Guide-Geo_Rep-Starting-Start"/></para> - </listitem> - <listitem> - <para><xref linkend="chap-Administration_Guide-Geo_Rep-Starting-Verify"/></para> - </listitem> - <listitem> - <para><xref linkend="chap-Administration_Guide-Geo_Rep-Starting-Display"/></para> - </listitem> - <listitem> - <para><xref linkend="chap-Administration_Guide-Geo_Rep-Starting-Configure"/></para> - </listitem> - <listitem> - <para><xref linkend="chap-Administration_Guide-Geo_Rep-Starting-Stop"/></para> - </listitem> - </itemizedlist> - <section id="chap-Administration_Guide-Geo_Rep-Starting-Start"> - <title>Starting Geo-replication</title> - <para>To start Gluster Geo-replication </para> - <itemizedlist> - <listitem> - <para>Start geo-replication between the hosts using the following command: - </para> - <para><command># gluster volume geo-replication <replaceable>MASTER SLAVE</replaceable> start</command> -</para> - <para>For example: -</para> - <para><programlisting># gluster volume geo-replication Volume1 example.com:/data/remote_dir start -Starting geo-replication session between Volume1 -example.com:/data/remote_dir has been successful</programlisting></para> - <para><note> - <para>You may need to configure the service before starting Gluster Geo-replication. For more information, see <xref linkend="chap-Administration_Guide-Geo_Rep-Starting-Configure"/>.</para> - </note></para> - </listitem> - </itemizedlist> - </section> - <section id="chap-Administration_Guide-Geo_Rep-Starting-Verify"> - <title>Verifying Successful Deployment</title> - <para>You can use the gluster command to verify the status of Gluster Geo-replication in your environment.</para> - <para><emphasis role="bold">To verify the status Gluster Geo-replication</emphasis></para> - <itemizedlist> - <listitem> - <para>Verify the status by issuing the following command on host:</para> - <para><command># gluster volume geo-replication <replaceable>MASTER SLAVE</replaceable> status</command> -</para> - <para>For example: -</para> - <para><command># gluster volume geo-replication Volume1 example.com:/data/remote_dir status</command> -</para> - <para><programlisting># gluster volume geo-replication Volume1 example.com:/data/remote_dir status - -MASTER SLAVE STATUS -______ ______________________________ ____________ -Volume1 root@example.com:/data/remote_dir Starting....</programlisting> -</para> - </listitem> - </itemizedlist> - </section> - <section id="chap-Administration_Guide-Geo_Rep-Starting-Display"> - <title>Displaying Geo-replication Status Information</title> - <para>You can display status information about a specific geo-replication master session, or a particular master-slave session, or all geo-replication sessions, as needed.</para> - <para><emphasis role="bold">To display geo-replication status information</emphasis></para> - <itemizedlist> - <listitem> - <para>Display information of all geo-replication sessions using the following command:</para> - <para><programlisting># gluster volume geo-replication Volume1 example.com:/data/remote_dir status - -MASTER SLAVE STATUS -______ ______________________________ ____________ -Volume1 root@example.com:/data/remote_dir Starting....</programlisting></para> - </listitem> - </itemizedlist> - <itemizedlist> - <listitem> - <para>Display information of a particular master slave session using the following command: -</para> - <para><command># gluster volume geo-replication <replaceable>MASTER SLAVE</replaceable> status</command> -</para> - <para>For example, to display information of Volume1 and example.com:/data/remote_dir -</para> - <para><command># gluster volume geo-replication Volume1 example.com:/data/remote_dir status</command> -</para> - <para>The status of the geo-replication between Volume1 and example.com:/data/remote_dir is displayed.</para> - </listitem> - <listitem> - <para>Display information of all geo-replication sessions belonging to a master</para> - <para><command># gluster volume geo-replication MASTER status</command> -</para> - <para>For example, to display information of Volume1</para> - <para><programlisting># gluster volume geo-replication Volume1 example.com:/data/remote_dir status - -MASTER SLAVE STATUS -______ ______________________________ ____________ -Volume1 ssh://example.com:gluster://127.0.0.1:remove_volume OK - -Volume1 ssh://example.com:file:///data/remote_dir OK</programlisting></para> - <para>The status of a session could be one of the following four:</para> - </listitem> - <listitem> - <para><emphasis role="bold">Starting</emphasis>: This is the initial phase of the Geo-replication session; it remains in this state for a minute, to make sure no abnormalities are present.</para> - </listitem> - <listitem> - <para><emphasis role="bold">OK</emphasis>: The geo-replication session is in a stable state.</para> - </listitem> - <listitem> - <para><emphasis role="bold">Faulty</emphasis>: The geo-replication session has witnessed some abnormality and the situation has to be investigated further. For further information, see <xref linkend="chap-Administration_Guide-Troubleshooting"/> section.</para> - </listitem> - <listitem> - <para><emphasis role="bold">Corrupt</emphasis>: The monitor thread which is monitoring the geo-replication session has died. This situation should not occur normally, if it persists contact Red Hat Support<ulink url="www.redhat.com/support/"/>.</para> - </listitem> - </itemizedlist> - </section> - <section id="chap-Administration_Guide-Geo_Rep-Starting-Configure"> - <title>Configuring Geo-replication</title> - <para>To configure Gluster Geo-replication </para> - <itemizedlist> - <listitem> - <para>Use the following command at the Gluster command line: -</para> - <para><command># gluster volume geo-replication <replaceable>MASTER SLAVE</replaceable> config [options]</command> -</para> - <para>For more information about the options, see <xref linkend="chap-Administration_Guide-Com_Ref"/>. -</para> - <para>For example: -</para> - <para>To view list of all option/value pair, use the following command: -</para> - <para><command># gluster volume geo-replication Volume1 example.com:/data/remote_dir config</command> -</para> - </listitem> - </itemizedlist> - </section> - <section id="chap-Administration_Guide-Geo_Rep-Starting-Stop"> - <title>Stopping Geo-replication</title> - <para>You can use the gluster command to stop Gluster Geo-replication (syncing of data from Master to Slave) in your environment. </para> - <para><emphasis role="bold">To stop Gluster Geo-replication</emphasis> </para> - <itemizedlist> - <listitem> - <para>Stop geo-replication between the hosts using the following command: -</para> - <para><command># gluster volume geo-replication <replaceable>MASTER SLAVE</replaceable> stop </command></para> - <para>For example: -</para> - <para><programlisting># gluster volume geo-replication Volume1 example.com:/data/remote_dir stop -Stopping geo-replication session between Volume1 and -example.com:/data/remote_dir has been successful</programlisting></para> - <para>See <xref linkend="chap-Administration_Guide-Com_Ref"/> for more information about the gluster command. -</para> - </listitem> - </itemizedlist> - </section> - </section> - <section id="chap-Administration_Guide-Geo_Rep-Restoring_Data"> - <title>Restoring Data from the Slave</title> - <para>You can restore data from the slave to the master volume, whenever the master volume becomes faulty for reasons like hardware failure. -</para> - <para>The example in this section assumes that you are using the Master Volume (Volume1) with the following configuration: -</para> - <para><programlisting>machine1# gluster volume info -Type: Distribute -Status: Started -Number of Bricks: 2 -Transport-type: tcp -Bricks: -Brick1: machine1:/export/dir16 -Brick2: machine2:/export/dir16 -Options Reconfigured: -geo-replication.indexing: on</programlisting></para> - <para>The data is syncing from master volume (Volume1) to slave directory (example.com:/data/remote_dir). To view the status of this geo-replication session run the following command on Master: </para> - <programlisting># gluster volume geo-replication Volume1 root@example.com:/data/remote_dir status - -MASTER SLAVE STATUS -______ ______________________________ ____________ -Volume1 root@example.com:/data/remote_dir OK</programlisting> - <para><emphasis role="bold">Before Failure</emphasis> -</para> - <para>Assume that the Master volume had 100 files and was mounted at /mnt/gluster on one of the client machines (client). Run the following command on Client machine to view the list of files: -</para> - <para><programlisting>client# ls /mnt/gluster | wc –l -100</programlisting></para> - <para>The slave directory (example.com) will have same data as in the master volume and same can be viewed by running the following command on slave: -</para> - <para><programlisting>example.com# ls /data/remote_dir/ | wc –l -100</programlisting></para> - <para><emphasis role="bold">After Failure</emphasis> -</para> - <para>If one of the bricks (machine2) fails, then the status of Geo-replication session is changed from "OK" to "Faulty". To view the status of this geo-replication session run the following command on Master: -</para> - <programlisting># gluster volume geo-replication Volume1 root@example.com:/data/remote_dir status - -MASTER SLAVE STATUS -______ ______________________________ ____________ -Volume1 root@example.com:/data/remote_dir Faulty</programlisting> - <para>Machine2 is failed and now you can see discrepancy in number of files between master and slave. Few files will be missing from the master volume but they will be available only on slave as shown below. -</para> - <para>Run the following command on Client: - </para> - <para><programlisting>client # ls /mnt/gluster | wc –l -52</programlisting></para> - <para>Run the following command on slave (example.com): -</para> - <para><programlisting>Example.com# # ls /data/remote_dir/ | wc –l -100</programlisting></para> - <para><emphasis role="bold">To restore data from the slave machine</emphasis></para> - <orderedlist> - <listitem> - <para>Stop all Master's geo-replication sessions using the following command: -</para> - <para><command># gluster volume geo-replication <replaceable>MASTER SLAVE</replaceable> stop</command> -</para> - <para>For example: -</para> - <para><programlisting>machine1# gluster volume geo-replication Volume1 -example.com:/data/remote_dir stop - -Stopping geo-replication session between Volume1 & -example.com:/data/remote_dir has been successful</programlisting></para> - <para><note> - <para>Repeat <command># gluster volume geo-replication <replaceable>MASTER SLAVE</replaceable> stop </command>command on all active geo-replication sessions of master volume.</para> - </note></para> - </listitem> - <listitem> - <para>Replace the faulty brick in the master by using the following command: -</para> - <para><command># gluster volume replace-brick <replaceable>VOLNAME BRICK NEW-BRICK</replaceable> start</command> -</para> - <para>For example: -</para> - <para><programlisting>machine1# gluster volume replace-brick Volume1 machine2:/export/dir16 machine3:/export/dir16 start -Replace-brick started successfully</programlisting></para> - </listitem> - <listitem> - <para>Commit the migration of data using the following command: -</para> - <para><command># gluster volume replace-brick <replaceable>VOLNAME BRICK NEW-BRICK</replaceable> commit force </command></para> - <para>For example: -</para> - <para><programlisting>machine1# gluster volume replace-brick Volume1 machine2:/export/dir16 machine3:/export/dir16 commit force -Replace-brick commit successful</programlisting></para> - </listitem> - <listitem> - <para>Verify the migration of brick by viewing the volume info using the following command: -</para> - <para><command># gluster volume info <replaceable>VOLNAME</replaceable></command></para> - <para>For example: -</para> - <para><programlisting>machine1# gluster volume info -Volume Name: Volume1 -Type: Distribute -Status: Started -Number of Bricks: 2 -Transport-type: tcp -Bricks: -Brick1: machine1:/export/dir16 -Brick2: machine3:/export/dir16 -Options Reconfigured: -geo-replication.indexing: on</programlisting></para> - </listitem> - <listitem> - <para>Run rsync command manually to sync data from slave to master volume's client (mount point). -</para> - <para>For example: -</para> - <para><command>example.com# rsync -PavhS --xattrs --ignore-existing /data/remote_dir/ client:/mnt/gluster</command></para> - <para>Verify that the data is synced by using the following command: -</para> - <para>On master volume, run the following command: -</para> - <para><programlisting>Client # ls | wc –l -100</programlisting></para> - <para>On the Slave run the following command: -</para> - <para><programlisting>example.com# ls /data/remote_dir/ | wc –l -100</programlisting></para> - <para>Now Master volume and Slave directory is synced. -</para> - </listitem> - <listitem> - <para>Restart geo-replication session from master to slave using the following command: -</para> - <para><command># gluster volume geo-replication <replaceable>MASTER SLAVE</replaceable> start </command></para> - <para>For example: -</para> - <para><programlisting>machine1# gluster volume geo-replication Volume1 -example.com:/data/remote_dir start -Starting geo-replication session between Volume1 & -example.com:/data/remote_dir has been successful</programlisting></para> - </listitem> - </orderedlist> - </section> - <section id="chap-Administration_Guide-Geo_Rep-Best_Practices"> - <title>Best Practices</title> - <para><emphasis role="bold">Manually Setting Time </emphasis></para> - <para>If you have to change the time on your bricks manually, then you must set uniform time on all bricks. This avoids the out-of-time sync issue described in <xref linkend="chap-Administration_Guide-Geo_Rep-Preparation-Settingup_Environment"/>. Setting time backward corrupts the geo-replication index, so the recommended way to set the time manually is: -</para> - <orderedlist> - <listitem> - <para>Stop geo-replication between the master and slave using the following command: -</para> - <para><command># gluster volume geo-replication <replaceable>MASTER SLAVE</replaceable> sto</command>p -</para> - </listitem> - <listitem> - <para>Stop the geo-replication indexing using the following command: -</para> - <para><command># gluster volume set <replaceable>MASTER</replaceable> geo-replication.indexing of</command>f</para> - </listitem> - <listitem> - <para>Set uniform time on - all bricks.s</para> - </listitem> - <listitem> - <para>Restart your geo-replication sessions by using the following command: -</para> - <para><command># gluster volume geo-replication <replaceable>MASTER SLAVE </replaceable>start </command></para> - </listitem> - </orderedlist> - <para><emphasis role="bold">Running Geo-replication commands in one system</emphasis> -</para> - <para>It is advisable to run the geo-replication commands in one of the bricks in the trusted storage pool. This is because, the log files for the geo-replication session would be stored in the *Server* where the Geo-replication start is initiated. Hence it would be easier to locate the log-files when required. -</para> - <para><emphasis role="bold">Isolation </emphasis></para> - <para>Geo-replication slave operation is not sandboxed as of now and is ran as a privileged service. So for the security reason, it is advised to create a sandbox environment (dedicated machine / dedicated virtual machine / chroot/container type solution) by the administrator to run the geo-replication slave in it. Enhancement in this regard will be available in follow-up minor release. -</para> - </section> -</chapter> diff --git a/doc/admin-guide/en-US/admin_managing_volumes.xml b/doc/admin-guide/en-US/admin_managing_volumes.xml deleted file mode 100644 index 0c4d2e922cf..00000000000 --- a/doc/admin-guide/en-US/admin_managing_volumes.xml +++ /dev/null @@ -1,735 +0,0 @@ -<?xml version='1.0' encoding='UTF-8'?> -<!-- This document was created with Syntext Serna Free. --><!DOCTYPE chapter PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN" "http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd" [ -<!ENTITY % BOOK_ENTITIES SYSTEM "Administration_Guide.ent"> -%BOOK_ENTITIES; -]> -<chapter id="chap-Administration_Guide-Managing_Volumes"> - <title>Managing GlusterFS Volumes</title> - <para>This section describes how to perform common GlusterFS management operations, including the following: </para> - <itemizedlist> - <listitem> - <para><xref linkend="sect-Administration_Guide-Managing_Volumes-Tuning"/></para> - </listitem> - <listitem> - <para><xref linkend="sect-Administration_Guide-Managing_Volumes-Expanding"/></para> - </listitem> - <listitem> - <para><xref linkend="sect-Administration_Guide-Managing_Volumes-Shrinking"/></para> - </listitem> - <listitem> - <para><xref linkend="sect-Administration_Guide-Managing_Volumes-Migrating"/></para> - </listitem> - <listitem> - <para><xref linkend="sect-Administration_Guide-Managing_Volumes-Rebalancing"/></para> - </listitem> - <listitem> - <para><xref linkend="sect-Administration_Guide-Managing_Volumes-Stop"/></para> - </listitem> - <listitem> - <para><xref linkend="sect-Administration_Guide-Managing_Volumes-Delete"/></para> - </listitem> - <listitem> - <para><xref linkend="sect-Administration_Guide-Managing_Volumes-Self_heal"/></para> - </listitem> - </itemizedlist> - <section id="sect-Administration_Guide-Managing_Volumes-Tuning"> - <title>Tuning Volume Options</title> - <para>You can tune volume options, as needed, while the cluster is online and available. </para> - <para><note> - <para>Red Hat recommends you to set server.allow-insecure option to ON if there are too many bricks in each volume or if there are too many services which have already utilized all the privileged ports in the system. Turning this option ON allows ports to accept/reject messages from insecure ports. So, use this option only if your deployment requires it. </para> - </note></para> - <para>To tune volume options </para> - <itemizedlist> - <listitem> - <para>Tune volume options using the following command:</para> - <para><command># gluster volume set <replaceable>VOLNAME OPTION PARAMETER</replaceable></command></para> - <para>For example, to specify the performance cache size for test-volume:</para> - <para><programlisting># gluster volume set test-volume performance.cache-size 256MB -Set volume successful</programlisting></para> - <para>The following table lists the Volume options along with its description and default value: </para> - <para><note> - <para>The default options given here are subject to modification at any given time and may not be the same for all versions.</para> - </note></para> - <informaltable frame="all"> - <tgroup cols="4"> - <colspec colname="c1"/> - <colspec colname="c2"/> - <colspec colname="c3"/> - <colspec colname="c4"/> - <thead> - <row> - <entry>Option</entry> - <entry>Description</entry> - <entry>Default Value</entry> - <entry>Available Options</entry> - </row> - </thead> - <tbody> - <row> - <entry>auth.allow</entry> - <entry>IP addresses of the clients which should be allowed to access the volume. </entry> - <entry>* (allow all)</entry> - <entry>Valid IP address which includes wild card patterns including *, such as 192.168.1.*</entry> - </row> - <row> - <entry>auth.reject</entry> - <entry>IP addresses of the clients which should be denied to access the volume. </entry> - <entry>NONE (reject none) </entry> - <entry>Valid IP address which includes wild card patterns including *, such as 192.168.2.*</entry> - </row> - <row> - <entry>client.grace-timeout</entry> - <entry>Specifies the duration for the lock state to be maintained on the client after a network disconnection.</entry> - <entry>10 </entry> - <entry>10 - 1800 secs</entry> - </row> - <row> - <entry>cluster.self-heal-window-size</entry> - <entry>Specifies the maximum number of blocks per file on which self-heal would happen simultaneously. </entry> - <entry>16 </entry> - <entry>0 - 1025 blocks</entry> - </row> - <row> - <entry>cluster.data-self-heal-algorithm</entry> - <entry>Specifies the type of self-heal. If you set the option as "full", the entire file is copied from source to destinations. If the option is set to "diff" the file blocks that are not in sync are copied to destinations. Reset uses a heuristic model. If the file does not exist on one of the subvolumes, or a zero-byte file exists (created by entry self-heal) the entire content has to be copied anyway, so there is no benefit from using the "diff" algorithm. If the file size is about the same as page size, the entire file can be read and written with a few operations, which will be faster than "diff" which has to read checksums and then read and write. </entry> - <entry>reset</entry> - <entry>full | diff | reset</entry> - </row> - <row> - <entry>cluster.min-free-disk</entry> - <entry>Specifies the percentage of disk space that must be kept free. Might be useful for non-uniform bricks. </entry> - <entry>10%</entry> - <entry>Percentage of required minimum free disk space</entry> - </row> - <row> - <entry>cluster.stripe-block-size</entry> - <entry>Specifies the size of the stripe unit that will be read from or written to. </entry> - <entry>128 KB (for all files)</entry> - <entry>size in bytes</entry> - </row> - <row> - <entry>cluster.self-heal-daemon</entry> - <entry>Allows you to turn-off proactive self-heal on replicated volumes.</entry> - <entry>on</entry> - <entry>On | Off</entry> - </row> - <row> - <entry>diagnostics.brick-log-level</entry> - <entry>Changes the log-level of the bricks. </entry> - <entry>INFO </entry> - <entry>DEBUG|WARNING|ERROR|CRITICAL|NONE|TRACE</entry> - </row> - <row> - <entry>diagnostics.client-log-level</entry> - <entry>Changes the log-level of the clients. </entry> - <entry>INFO </entry> - <entry>DEBUG|WARNING|ERROR|CRITICAL|NONE|TRACE</entry> - </row> - <row> - <entry>diagnostics.latency-measurement</entry> - <entry>Statistics related to the latency of each operation would be tracked. </entry> - <entry>off </entry> - <entry>On | Off</entry> - </row> - <row> - <entry>diagnostics.dump-fd-stats</entry> - <entry>Statistics related to file-operations would be tracked.</entry> - <entry>off </entry> - <entry>On | Off</entry> - </row> - <row> - <entry>feature.read-only</entry> - <entry>Enables you to mount the entire volume as read-only for all the clients (including NFS clients) accessing it.</entry> - <entry>off</entry> - <entry>On | Off</entry> - </row> - <row> - <entry>features.lock-heal</entry> - <entry>Enables self-healing of locks when the network disconnects.</entry> - <entry>on</entry> - <entry>On | Off</entry> - </row> - <row> - <entry>features.quota-timeout</entry> - <entry>For performance reasons, quota caches the directory sizes on client. You can set timeout indicating the maximum duration of directory sizes in cache, from the time they are populated, during which they are considered valid. </entry> - <entry>0</entry> - <entry>0 - 3600 secs</entry> - </row> - <row> - <entry>geo-replication.indexing</entry> - <entry>Use this option to automatically sync the changes in the filesystem from Master to Slave.</entry> - <entry>off </entry> - <entry>On | Off</entry> - </row> - <row> - <entry>network.frame-timeout</entry> - <entry>The time frame after which the operation has to be declared as dead, if the server does not respond for a particular operation. </entry> - <entry>1800 (30 mins) </entry> - <entry>1800 secs</entry> - </row> - <row> - <entry>network.ping-timeout</entry> - <entry>The time duration for which the client waits to check if the server is responsive. When a ping timeout happens, there is a network disconnect between the client and server. All resources held by server on behalf of the client get cleaned up. When a reconnection happens, all resources will need to be re-acquired before the client can resume its operations on the server. Additionally, the locks will be acquired and the lock tables updated. <para>This reconnect is a very expensive operation and should be avoided. -</para></entry> - <entry>42 Secs</entry> - <entry>42 Secs</entry> - </row> - <row> - <entry>nfs.enable-ino32</entry> - <entry>For 32-bit nfs clients or applications that do not support 64-bit inode numbers or large files, use this option from the CLI to make Gluster NFS return 32-bit inode numbers instead of 64-bit inode numbers. Applications that will benefit are those that were either: <para>* Built 32-bit and run on 32-bit machines.</para><para>* Built 32-bit on 64-bit systems.</para><para>* Built 64-bit but use a library built 32-bit, especially relevant for python and perl scripts.</para><para>Either of the conditions above can lead to application on Linux NFS clients failing with "Invalid argument" or "Value too large for defined data type" errors.</para></entry> - <entry>off</entry> - <entry>On | Off</entry> - </row> - <row> - <entry>nfs.volume-access </entry> - <entry>Set the access type for the specified sub-volume. </entry> - <entry>read-write </entry> - <entry>read-write|read-only </entry> - </row> - <row> - <entry>nfs.trusted-write </entry> - <entry>If there is an UNSTABLE write from the client, STABLE flag will be returned to force the client to not send a COMMIT request. <para>In some environments, combined with a replicated GlusterFS setup, this option can improve write performance. This flag allows users to trust Gluster replication logic to sync data to the disks and recover when required. COMMIT requests if received will be handled in a default manner by fsyncing. STABLE writes are still handled in a sync manner.</para></entry> - <entry> off </entry> - <entry>On | Off </entry> - </row> - <row> - <entry>nfs.trusted-sync</entry> - <entry> All writes and COMMIT requests are treated as async. This implies that no write requests are guaranteed to be on server disks when the write reply is received at the NFS client. Trusted sync includes trusted-write behavior. </entry> - <entry>off </entry> - <entry>On | Off </entry> - </row> - <row> - <entry>nfs.export-dir </entry> - <entry>By default, all sub-volumes of NFS are exported as individual exports. Now, this option allows you to export only the specified subdirectory or subdirectories in the volume. This option can also be used in conjunction with nfs3.export-volumes option to restrict exports only to the subdirectories specified through this option. You must provide an absolute path.</entry> - <entry>Enabled for all sub directories.</entry> - <entry>Enable | Disable </entry> - </row> - <row> - <entry>nfs.export-volumes </entry> - <entry>Enable/Disable exporting entire volumes, instead if used in conjunction with nfs3.export-dir, can allow setting up only subdirectories as exports. </entry> - <entry>on</entry> - <entry> On | Off </entry> - </row> - <row> - <entry>nfs.rpc-auth-unix </entry> - <entry>Enable/Disable the AUTH_UNIX authentication type. This option is enabled by default for better interoperability. However, you can disable it if required.</entry> - <entry>on </entry> - <entry> On | Off </entry> - </row> - <row> - <entry>nfs.rpc-auth-null </entry> - <entry>Enable/Disable the AUTH_NULL authentication type. It is not recommended to change the default value for this option. </entry> - <entry>on </entry> - <entry> On | Off </entry> - </row> - <row> - <entry>nfs.rpc-auth-allow<IP- Addresses> </entry> - <entry>Allow a comma separated list of addresses and/or hostnames to connect to the server. By default, all clients are disallowed. This allows you to define a general rule for all exported volumes.</entry> - <entry>Reject All </entry> - <entry>IP address or Host name </entry> - </row> - <row> - <entry>nfs.rpc-auth-reject IP- Addresses </entry> - <entry>Reject a comma separated list of addresses and/or hostnames from connecting to the server. By default, all connections are disallowed. This allows you to define a general rule for all exported volumes.</entry> - <entry>Reject All </entry> - <entry>IP address or Host name </entry> - </row> - <row> - <entry>nfs.ports-insecure </entry> - <entry>Allow client connections from unprivileged ports. By default only privileged ports are allowed. This is a global setting in case insecure ports are to be enabled for all exports using a single option. </entry> - <entry>off</entry> - <entry> On | Off </entry> - </row> - <row> - <entry>nfs.addr-namelookup </entry> - <entry>Turn-off name lookup for incoming client connections using this option. In some setups, the name server can take too long to reply to DNS queries resulting in timeouts of mount requests. Use this option to turn off name lookups during address authentication. Note, turning this off will prevent you from using hostnames in rpc-auth.addr.* filters. </entry> - <entry>on </entry> - <entry> On | Off </entry> - </row> - <row> - <entry>nfs.register-with- portmap </entry> - <entry>For systems that need to run multiple NFS servers, you need to prevent more than one from registering with portmap service. Use this option to turn off portmap registration for Gluster NFS. </entry> - <entry>on </entry> - <entry> On | Off </entry> - </row> - <row> - <entry>nfs.port <PORT- NUMBER> </entry> - <entry>Use this option on systems that need Gluster NFS to be associated with a non-default port number. </entry> - <entry>38465- 38467 </entry> - <entry/> - </row> - <row> - <entry>nfs.disable</entry> - <entry>Turn-off volume being exported by NFS</entry> - <entry> off </entry> - <entry>On | Off </entry> - </row> - <row> - <entry>performance.write-behind-window-size </entry> - <entry>Size of the per-file write-behind buffer.</entry> - <entry>1 MB </entry> - <entry>Write-behind cache size </entry> - </row> - <row> - <entry>performance.io-thread-count </entry> - <entry>The number of threads in IO threads translator. </entry> - <entry>16</entry> - <entry>0 - 65 </entry> - </row> - <row> - <entry>performance.flush-behind </entry> - <entry>If this option is set ON, instructs write-behind translator to perform flush in background, by returning success (or any errors, if any of previous writes were failed) to application even before flush is sent to backend filesystem. </entry> - <entry>On </entry> - <entry>On | Off </entry> - </row> - <row> - <entry>performance.cache-max-file-size </entry> - <entry>Sets the maximum file size cached by the io-cache translator. Can use the normal size descriptors of KB, MB, GB,TB or PB (for example, 6GB). Maximum size uint64. </entry> - <entry>2 ^ 64 -1 bytes </entry> - <entry>size in bytes </entry> - </row> - <row> - <entry>performance.cache-min-file-size </entry> - <entry> Sets the minimum file size cached by the io-cache translator. Values same as "max" above.</entry> - <entry>0B</entry> - <entry>size in bytes </entry> - </row> - <row> - <entry>performance.cache-refresh-timeout </entry> - <entry>The cached data for a file will be retained till 'cache-refresh-timeout' seconds, after which data re-validation is performed. </entry> - <entry>1 sec </entry> - <entry>0 - 61 </entry> - </row> - <row> - <entry>performance.cache-size </entry> - <entry>Size of the read cache.</entry> - <entry> 32 MB </entry> - <entry>size in bytes </entry> - </row> - <row> - <entry>server.allow-insecure </entry> - <entry>Allow client connections from unprivileged ports. By default only privileged ports are allowed. This is a global setting in case insecure ports are to be enabled for all exports using a single option. </entry> - <entry>on </entry> - <entry>On | Off </entry> - </row> - <row> - <entry>server.grace-timeout</entry> - <entry>Specifies the duration for the lock state to be maintained on the server after a network disconnection.</entry> - <entry>10</entry> - <entry>10 - 1800 secs</entry> - </row> - <row> - <entry>server.statedump-path </entry> - <entry>Location of the state dump file. </entry> - <entry>/tmp directory of the brick </entry> - <entry>New directory path</entry> - </row> - </tbody> - </tgroup> - </informaltable> - <para>You can view the changed volume options using the<command> # gluster volume info <replaceable>VOLNAME</replaceable></command> command. For more information, see <xref linkend="sect-Administration_Guide-Managing_Volumes-Delete"/>.</para> - </listitem> - </itemizedlist> - </section> - <section id="sect-Administration_Guide-Managing_Volumes-Expanding"> - <title>Expanding Volumes</title> - <para>You can expand volumes, as needed, while the cluster is online and available. For example, you might want to add a brick to a distributed volume, thereby increasing the distribution and adding to the capacity of the GlusterFS volume. </para> - <para>Similarly, you might want to add a group of bricks to a distributed replicated volume, increasing the capacity of the GlusterFS volume. </para> - <para><note> - <para>When expanding distributed replicated and distributed striped volumes, you need to add a number of bricks that is a multiple of the replica or stripe count. For example, to expand a distributed replicated volume with a replica count of 2, you need to add bricks in multiples of 2 (such as 4, 6, 8, etc.). </para> - </note></para> - <para><emphasis role="bold">To expand a volume</emphasis> </para> - <orderedlist> - <listitem> - <para>On the first server in the cluster, probe the server to which you want to add the new brick using the following command:</para> - <para><command># gluster peer probe <replaceable>HOSTNAME</replaceable></command></para> - <para>For example:</para> - <para><programlisting># gluster peer probe server4 -Probe successful</programlisting></para> - </listitem> - <listitem> - <para>Add the brick using the following command: </para> - <para><command># gluster volume add-brick <replaceable>VOLNAME NEW-BRICK</replaceable></command></para> - <para>For example:</para> - <para><programlisting># gluster volume add-brick test-volume server4:/exp4 -Add Brick successful</programlisting></para> - </listitem> - <listitem> - <para>Check the volume information using the following command: </para> - <para><command># gluster volume info </command></para> - <para>The command displays information similar to the following:</para> - <para><programlisting>Volume Name: test-volume -Type: Distribute -Status: Started -Number of Bricks: 4 -Bricks: -Brick1: server1:/exp1 -Brick2: server2:/exp2 -Brick3: server3:/exp3 -Brick4: server4:/exp4</programlisting></para> - </listitem> - <listitem> - <para>Rebalance the volume to ensure that all files are distributed to the new brick.</para> - <para>You can use the rebalance command as described in <xref linkend="sect-Administration_Guide-Managing_Volumes-Rebalancing"/>.</para> - </listitem> - </orderedlist> - </section> - <section id="sect-Administration_Guide-Managing_Volumes-Shrinking"> - <title>Shrinking Volumes</title> - <para>You can shrink volumes, as needed, while the cluster is online and available. For example, you might need to remove a brick that has become inaccessible in a distributed volume due to hardware or network failure. </para> - <para><note> - <para>Data residing on the brick that you are removing will no longer be accessible at the Gluster mount point. Note however that only the configuration information is removed - you can continue to access the data directly from the brick, as necessary. </para> - </note></para> - <para>When shrinking distributed replicated and distributed striped volumes, you need to remove a number of bricks that is a multiple of the replica or stripe count. For example, to shrink a distributed striped volume with a stripe count of 2, you need to remove bricks in multiples of 2 (such as 4, 6, 8, etc.). In addition, the bricks you are trying to remove must be from the same sub-volume (the same replica or stripe set). </para> - <para><emphasis role="bold">To shrink a volume</emphasis> </para> - <orderedlist> - <listitem> - <para>Remove the brick using the following command:</para> - <para><command># gluster volume remove-brick <varname>VOLNAME</varname><replaceable> BRICK</replaceable></command> <command>start</command></para> - <para>For example, to remove server2:/exp2:</para> - <para><programlisting># gluster volume remove-brick test-volume server2:/exp2 - -Removing brick(s) can result in data loss. Do you want to Continue? (y/n)</programlisting></para> - </listitem> - <listitem> - <para>Enter "y" to confirm the operation. The command displays the following message indicating that the remove brick operation is successfully started: </para> - <para><programlisting>Remove Brick successful </programlisting></para> - </listitem> - <listitem> - <para>(Optional) View the status of the remove brick operation using the following command:</para> - <para><command># gluster volume remove-brick <varname>VOLNAME</varname><replaceable> BRICK</replaceable></command><command> status</command></para> - <para>For example, to view the status of remove brick operation on server2:/exp2 brick:</para> - <para><screen># gluster volume remove-brick test-volume server2:/exp2 status - Node Rebalanced-files size scanned status - --------- ---------------- ---- ------- ----------- -617c923e-6450-4065-8e33-865e28d9428f 34 340 162 in progress</screen></para> - </listitem> - <listitem> - <para>Check the volume information using the following command: </para> - <para><command># gluster volume info </command></para> - <para>The command displays information similar to the following:</para> - <para><programlisting># gluster volume info -Volume Name: test-volume -Type: Distribute -Status: Started -Number of Bricks: 3 -Bricks: -Brick1: server1:/exp1 -Brick3: server3:/exp3 -Brick4: server4:/exp4</programlisting></para> - </listitem> - <listitem> - <para>Rebalance the volume to ensure that all files are distributed to the new brick.</para> - <para>You can use the rebalance command as described in <xref linkend="sect-Administration_Guide-Managing_Volumes-Rebalancing"/>.</para> - </listitem> - </orderedlist> - </section> - <section id="sect-Administration_Guide-Managing_Volumes-Migrating"> - <title>Migrating Volumes</title> - <para>You can migrate the data from one brick to another, as needed, while the cluster is online and available. </para> - <para><emphasis role="bold">To migrate a volume</emphasis> </para> - <orderedlist> - <listitem> - <para>Make sure the new brick, server5 in this example, is successfully added to the cluster.</para> - <para>For more information, see <xref linkend="sect-Administration_Guide-Storage_Pools-Adding_Servers"/>.</para> - </listitem> - <listitem> - <para>Migrate the data from one brick to another using the following command:</para> - <para><command> # gluster volume replace-brick <code>VOLNAME</code><code> BRICK</code><code>NEW-BRICK</code> start</command></para> - <para>For example, to migrate the data in server3:/exp3 to server5:/exp5 in test-volume:</para> - <para><programlisting># gluster volume replace-brick test-volume server3:/exp3 server5:exp5 start -Replace brick start operation successful</programlisting></para> - <para><note> - <para>You need to have the FUSE package installed on the server on which you are running the replace-brick command for the command to work.</para> - </note></para> - </listitem> - <listitem> - <para>To pause the migration operation, if needed, use the following command: </para> - <para><command># gluster volume replace-brick <varname>VOLNAME BRICK NEW-BRICK </varname> pause </command></para> - <para>For example, to pause the data migration from server3:/exp3 to server5:/exp5 in test-volume:</para> - <para><programlisting># gluster volume replace-brick test-volume server3:/exp3 server5:exp5 pause -Replace brick pause operation successful</programlisting></para> - </listitem> - <listitem> - <para>To abort the migration operation, if needed, use the following command: </para> - <para><command> # gluster volume replace-brick <varname>VOLNAME BRICK NEW-BRICK </varname>abort </command></para> - <para>For example, to abort the data migration from server3:/exp3 to server5:/exp5 in test-volume:</para> - <para><programlisting># gluster volume replace-brick test-volume server3:/exp3 server5:exp5 abort -Replace brick abort operation successful</programlisting></para> - </listitem> - <listitem> - <para>Check the status of the migration operation using the following command: </para> - <para><command> # gluster volume replace-brick <varname>VOLNAME BRICK NEW-BRICK </varname>status </command></para> - <para>For example, to check the data migration status from server3:/exp3 to server5:/exp5 in test-volume:</para> - <para><programlisting># gluster volume replace-brick test-volume server3:/exp3 server5:/exp5 status -Current File = /usr/src/linux-headers-2.6.31-14/block/Makefile -Number of files migrated = 10567 -Migration complete</programlisting></para> - <para>The status command shows the current file being migrated along with the current total number of files migrated. After completion of migration, it displays Migration complete.</para> - </listitem> - <listitem> - <para>Commit the migration of data from one brick to another using the following command: </para> - <para><command> # gluster volume replace-brick <varname>VOLNAME BRICK NEW-BRICK </varname>commit </command></para> - <para>For example, to commit the data migration from server3:/exp3 to server5:/exp5 in test-volume:</para> - <para><programlisting># gluster volume replace-brick test-volume server3:/exp3 server5:/exp5 commit -replace-brick commit successful</programlisting></para> - </listitem> - <listitem> - <para>Verify the migration of brick by viewing the volume info using the following command: </para> - <para><command># gluster volume info <code>VOLNAME</code></command></para> - <para>For example, to check the volume information of new brick server5:/exp5 in test-volume:</para> - <para><programlisting># gluster volume info test-volume -Volume Name: testvolume -Type: Replicate -Status: Started -Number of Bricks: 4 -Transport-type: tcp -Bricks: -Brick1: server1:/exp1 -Brick2: server2:/exp2 -Brick3: server4:/exp4 -Brick4: server5:/exp5 - -The new volume details are displayed. -</programlisting></para> - <para>The new volume details are displayed.</para> - <para>In the above example, previously, there were bricks; 1,2,3, and 4 and now brick 3 is replaced by brick 5.</para> - </listitem> - </orderedlist> - </section> - <section id="sect-Administration_Guide-Managing_Volumes-Rebalancing"> - <title>Rebalancing Volumes</title> - <para>After expanding or shrinking a volume (using the add-brick and remove-brick commands respectively), you need to rebalance the data among the servers. New directories created after expanding or shrinking of the volume will be evenly distributed automatically. For all the existing directories, the distribution can be fixed by rebalancing the layout and/or data. </para> - <para>This section describes how to rebalance GlusterFS volumes in your storage environment, using the following common scenarios: </para> - <itemizedlist> - <listitem> - <para>Fix Layout - Fixes the layout changes so that the files can actually go to newly added nodes. For more information, see <xref linkend="sect-Administration_Guide-Managing_Volumes-Rebalancing-Fix_Layout"/>. </para> - </listitem> - <listitem> - <para>Fix Layout and Migrate Data - Rebalances volume by fixing the layout changes and migrating the existing data. For more information, see <xref linkend="sect-Administration_Guide-Managing_Volumes-Rebalancing-Fix_Migrate"/>.</para> - </listitem> - </itemizedlist> - <section id="sect-Administration_Guide-Managing_Volumes-Rebalancing-Fix_Layout"> - <title>Rebalancing Volume to Fix Layout Changes</title> - <para>Fixing the layout is necessary because the layout structure is static for a given directory. In a scenario where new bricks have been added to the existing volume, newly created files in existing directories will still be distributed only among the old bricks. The <command># gluster volume rebalance<varname> VOLNAME</varname> fix-layout start </command>command will fix the layout information so that the files can also go to newly added nodes. When this command is issued, all the file stat information which is already cached will get revalidated. </para> - <para>A fix-layout rebalance will only fix the layout changes and does not migrate data. If you want to migrate the existing data, use<command># gluster volume rebalance <varname>VOLNAME</varname> start </command> command to rebalance data among the servers. </para> - <para><emphasis role="bold">To rebalance a volume to fix layout changes</emphasis></para> - <itemizedlist> - <listitem> - <para>Start the rebalance operation on any one of the server using the following command:</para> - <para><command># gluster volume rebalance<varname> VOLNAME</varname> fix-layout start</command></para> - <para>For example:</para> - <para><programlisting># gluster volume rebalance test-volume fix-layout start -Starting rebalance on volume test-volume has been successful</programlisting></para> - </listitem> - </itemizedlist> - </section> - <section id="sect-Administration_Guide-Managing_Volumes-Rebalancing-Fix_Migrate"> - <title>Rebalancing Volume to Fix Layout and Migrate Data</title> - <para>After expanding or shrinking a volume (using the add-brick and remove-brick commands respectively), you need to rebalance the data among the servers. </para> - <para><emphasis role="bold">To rebalance a volume to fix layout and migrate the existing data</emphasis></para> - <itemizedlist> - <listitem> - <para>Start the rebalance operation on any one of the server using the following command:</para> - <para><command># gluster volume rebalance<varname> VOLNAME</varname> start</command></para> - <para>For example:</para> - <para><programlisting># gluster volume rebalance test-volume start -Starting rebalancing on volume test-volume has been successful</programlisting></para> - </listitem> - <listitem> - <para>Start the migration operation forcefully on any one of the server using the following command:</para> - <para><command># gluster volume rebalance<varname> VOLNAME</varname> start force</command></para> - <para>For example:</para> - <para><programlisting># gluster volume rebalance test-volume start force -Starting rebalancing on volume test-volume has been successful</programlisting></para> - </listitem> - </itemizedlist> - </section> - <section> - <title>Displaying Status of Rebalance Operation</title> - <para>You can display the status information about rebalance volume operation, as needed. </para> - <para><emphasis role="bold">To view status of rebalance volume</emphasis></para> - <itemizedlist> - <listitem> - <para>Check the status of the rebalance operation, using the following command:</para> - <para><command># gluster volume rebalance <replaceable>VOLNAME</replaceable> status</command></para> - <para>For example:</para> - <para><screen># gluster volume rebalance test-volume status - Node Rebalanced-files size scanned status - --------- ---------------- ---- ------- ----------- -617c923e-6450-4065-8e33-865e28d9428f 416 1463 312 in progress</screen></para> - <para>The time to complete the rebalance operation depends on the number of files on the volume along with the corresponding file sizes. Continue checking the rebalance status, verifying that the number of files rebalanced or total files scanned keeps increasing.</para> - <para>For example, running the status command again might display a result similar to the following:</para> - <para><screen># gluster volume rebalance test-volume status - Node Rebalanced-files size scanned status - --------- ---------------- ---- ------- ----------- -617c923e-6450-4065-8e33-865e28d9428f 498 1783 378 in progress</screen></para> - <para>The rebalance status displays the following when the rebalance is complete:</para> - <para><screen># gluster volume rebalance test-volume status - Node Rebalanced-files size scanned status - --------- ---------------- ---- ------- ----------- -617c923e-6450-4065-8e33-865e28d9428f 502 1873 334 completed</screen></para> - </listitem> - </itemizedlist> - </section> - <section> - <title>Stopping Rebalance Operation</title> - <para>You can stop the rebalance operation, as needed.</para> - <para><emphasis role="bold">To stop rebalance</emphasis></para> - <itemizedlist> - <listitem> - <para>Stop the rebalance operation using the following command:</para> - <para><command># gluster volume rebalance <replaceable>VOLNAME</replaceable> stop</command></para> - <para>For example:</para> - <para><screen># gluster volume rebalance test-volume stop - Node Rebalanced-files size scanned status - --------- ---------------- ---- ------- ----------- -617c923e-6450-4065-8e33-865e28d9428f 59 590 244 stopped -Stopped rebalance process on volume test-volume </screen></para> - </listitem> - </itemizedlist> - </section> - </section> - <section id="sect-Administration_Guide-Managing_Volumes-Stop"> - <title>Stopping Volumes</title> - <para>To stop a volume</para> - <orderedlist> - <listitem> - <para>Stop the volume using the following command: - -</para> - <para><command># gluster volume stop <varname>VOLNAME </varname></command></para> - <para>For example, to stop test-volume:</para> - <para><programlisting># gluster volume stop test-volume -Stopping volume will make its data inaccessible. Do you want to continue? (y/n) -</programlisting></para> - </listitem> - <listitem> - <para>Enter <userinput>y</userinput> to confirm the operation. The output of the command displays the following: - -</para> - <programlisting>Stopping volume test-volume has been successful</programlisting> - </listitem> - </orderedlist> - </section> - <section id="sect-Administration_Guide-Managing_Volumes-Delete"> - <title>Deleting Volumes</title> - <para>To delete a volume </para> - <orderedlist> - <listitem> - <para>Delete the volume using the following command:</para> - <para><command># gluster volume delete <varname>VOLNAME</varname></command></para> - <para>For example, to delete test-volume:</para> - <para><programlisting># gluster volume delete test-volume -Deleting volume will erase all information about the volume. Do you want to continue? (y/n)</programlisting></para> - </listitem> - <listitem> - <para>Enter <userinput role="bold">y</userinput> to confirm the operation. The command displays the following:</para> - <para><programlisting>Deleting volume test-volume has been successful</programlisting></para> - </listitem> - </orderedlist> - </section> - <section id="sect-Administration_Guide-Managing_Volumes-Self_heal"> - <title>Triggering Self-Heal on Replicate</title> - <para>In replicate module, previously you had to manually trigger a self-heal when a brick goes offline and comes back online, to bring all the replicas in sync. Now the pro-active self-heal daemon runs in the background, diagnoses issues and automatically initiates self-healing every 10 minutes on the files which requires<emphasis role="italic"> healing</emphasis>. </para> - <para>You can view the list of files that need <emphasis role="italic">healing</emphasis>, the list of files which are currently/previously <emphasis role="italic">healed</emphasis>, list of files which are in split-brain state, and you can manually trigger self-heal on the entire volume or only on the files which need <emphasis role="italic">healing</emphasis>.</para> - <itemizedlist> - <listitem> - <para>Trigger self-heal only on the files which requires <emphasis role="italic">healing</emphasis>:</para> - <para><command># gluster volume heal <replaceable>VOLNAME</replaceable></command></para> - <para>For example, to trigger self-heal on files which requires <emphasis role="italic">healing</emphasis> of test-volume:</para> - <para><screen># gluster volume heal test-volume -Heal operation on volume test-volume has been successful</screen></para> - </listitem> - <listitem> - <para>Trigger self-heal on all the files of a volume:</para> - <para><command># gluster volume heal <replaceable>VOLNAME</replaceable></command> <command>full</command></para> - <para>For example, to trigger self-heal on all the files of of test-volume:</para> - <para><screen># gluster volume heal test-volume full -Heal operation on volume test-volume has been successful</screen></para> - </listitem> - <listitem> - <para>View the list of files that needs <emphasis role="italic">healing</emphasis>:</para> - <para><command># gluster volume heal <replaceable>VOLNAME</replaceable></command> <command>info</command></para> - <para>For example, to view the list of files on test-volume that needs <emphasis role="italic">healing</emphasis>:</para> - <para><screen># gluster volume heal test-volume info -Brick <emphasis role="italic">server1</emphasis>:/gfs/test-volume_0 -Number of entries: 0 - -Brick <emphasis role="italic">server2</emphasis>:/gfs/test-volume_1 -Number of entries: 101 -/95.txt -/32.txt -/66.txt -/35.txt -/18.txt -/26.txt -/47.txt -/55.txt -/85.txt -...</screen></para> - </listitem> - <listitem> - <para>View the list of files that are self-healed:</para> - <para><command># gluster volume heal <replaceable>VOLNAME</replaceable></command> <command>info healed</command> </para> - <para>For example, to view the list of files on test-volume that are self-healed:</para> - <para><screen># gluster volume heal test-volume info healed -Brick <emphasis role="italic">server1</emphasis>:/gfs/test-volume_0 -Number of entries: 0 - -Brick <emphasis role="italic">server2</emphasis>:/gfs/test-volume_1 -Number of entries: 69 -/99.txt -/93.txt -/76.txt -/11.txt -/27.txt -/64.txt -/80.txt -/19.txt -/41.txt -/29.txt -/37.txt -/46.txt -...</screen></para> - </listitem> - <listitem> - <para>View the list of files of a particular volume on which the self-heal failed:</para> - <para><command># gluster volume heal <replaceable>VOLNAME</replaceable></command> <command>info failed</command> </para> - <para>For example, to view the list of files of test-volume that are not self-healed:</para> - <para><screen># gluster volume heal test-volume info failed -Brick <emphasis role="italic">server1</emphasis>:/gfs/test-volume_0 -Number of entries: 0 - -Brick server2:/gfs/test-volume_3 -Number of entries: 72 -/90.txt -/95.txt -/77.txt -/71.txt -/87.txt -/24.txt -...</screen></para> - </listitem> - <listitem> - <para>View the list of files of a particular volume which are in split-brain state:</para> - <para><command># gluster volume heal <replaceable>VOLNAME</replaceable></command> <command>info split-brain</command> </para> - <para>For example, to view the list of files of test-volume which are in split-brain state:</para> - <para><screen># gluster volume heal test-volume info split-brain -Brick server1:/gfs/test-volume_2 -Number of entries: 12 -/83.txt -/28.txt -/69.txt -... - -Brick <emphasis role="italic">server2</emphasis>:/gfs/test-volume_2 -Number of entries: 12 -/83.txt -/28.txt -/69.txt -...</screen></para> - </listitem> - </itemizedlist> - </section> -</chapter> diff --git a/doc/admin-guide/en-US/admin_monitoring_workload.xml b/doc/admin-guide/en-US/admin_monitoring_workload.xml deleted file mode 100644 index e85bc51d896..00000000000 --- a/doc/admin-guide/en-US/admin_monitoring_workload.xml +++ /dev/null @@ -1,878 +0,0 @@ -<?xml version='1.0' encoding='UTF-8'?> -<!-- This document was created with Syntext Serna Free. --><!DOCTYPE chapter PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN" "docbookV4.5/docbookx.dtd" []> -<chapter id="chap-Administration_Guide-Monitor_Workload"> - <title>Monitoring your GlusterFS Workload</title> - <para>You can monitor the GlusterFS volumes on different parameters. Monitoring volumes helps in capacity planning and performance tuning tasks of the GlusterFS volume. Using these information, you can identify and troubleshoot issues. </para> - <para>You can use Volume Top and Profile commands to view the performance and identify bottlenecks/hotspots of each brick of a volume. This helps system administrators to get vital performance information whenever performance needs to be probed. </para> - <para>You can also perform statedump of the brick processes and nfs server process of a volume, and also view volume status and volume information. </para> - <section id="chap-Administration_Guide-Monitor_Workload-Profile"> - <title>Running GlusterFS Volume Profile Command</title> - <para>GlusterFS Volume Profile command provides an interface to get the per-brick I/O information for each File Operation (FOP) of a volume. The per brick information helps in identifying bottlenecks in the storage system. -</para> - <para>This section describes how to run GlusterFS Volume Profile command by performing the following operations: -</para> - <itemizedlist> - <listitem> - <para><xref linkend="chap-Administration_Guide-Monitor_Workload-Profile-Start"/></para> - </listitem> - <listitem> - <para><xref linkend="chap-Administration_Guide-Monitor_Workload-Profile-Display"/></para> - </listitem> - <listitem> - <para><xref linkend="chap-Administration_Guide-Monitor_Workload-Profile-Stop"/></para> - </listitem> - </itemizedlist> - <section id="chap-Administration_Guide-Monitor_Workload-Profile-Start"> - <title>Start Profiling</title> - <para>You must start the Profiling to view the File Operation information for each brick. -</para> - <para><emphasis role="bold">To start profiling: </emphasis></para> - <itemizedlist> - <listitem> - <para>Start profiling using the following command: - </para> - </listitem> - </itemizedlist> - <para><command># gluster volume profile <replaceable>VOLNAME</replaceable> start </command></para> - <para>For example, to start profiling on test-volume: -</para> - <para><programlisting># gluster volume profile test-volume start -Profiling started on test-volume</programlisting></para> - <para>When profiling on the volume is started, the following additional options are displayed in the Volume Info: -</para> - <para><programlisting>diagnostics.count-fop-hits: on - -diagnostics.latency-measurement: on</programlisting></para> - </section> - <section id="chap-Administration_Guide-Monitor_Workload-Profile-Display"> - <title>Displaying the I/0 Information</title> - <para>You can view the I/O information of each brick. -</para> - <para>To display I/O information: -</para> - <itemizedlist> - <listitem> - <para>Display the I/O information using the following command: -</para> - </listitem> - </itemizedlist> - <para><command># gluster volume profile <replaceable>VOLNAME</replaceable> info</command> - -</para> - <para>For example, to see the I/O information on test-volume: - -</para> - <para><programlisting># gluster volume profile test-volume info -Brick: Test:/export/2 -Cumulative Stats: - -Block 1b+ 32b+ 64b+ -Size: - Read: 0 0 0 - Write: 908 28 8 - -Block 128b+ 256b+ 512b+ -Size: - Read: 0 6 4 - Write: 5 23 16 - -Block 1024b+ 2048b+ 4096b+ -Size: - Read: 0 52 17 - Write: 15 120 846 - -Block 8192b+ 16384b+ 32768b+ -Size: - Read: 52 8 34 - Write: 234 134 286 - -Block 65536b+ 131072b+ -Size: - Read: 118 622 - Write: 1341 594 - - -%-latency Avg- Min- Max- calls Fop - latency Latency Latency -___________________________________________________________ -4.82 1132.28 21.00 800970.00 4575 WRITE -5.70 156.47 9.00 665085.00 39163 READDIRP -11.35 315.02 9.00 1433947.00 38698 LOOKUP -11.88 1729.34 21.00 2569638.00 7382 FXATTROP -47.35 104235.02 2485.00 7789367.00 488 FSYNC - ------------------- - ------------------- - -Duration : 335 - -BytesRead : 94505058 - -BytesWritten : 195571980</programlisting></para> - </section> - <section id="chap-Administration_Guide-Monitor_Workload-Profile-Stop"> - <title>Stop Profiling</title> - <para>You can stop profiling the volume, if you do not need profiling information anymore. -</para> - <para><emphasis role="bold">To stop profiling</emphasis> -</para> - <itemizedlist> - <listitem> - <para>Stop profiling using the following command: -</para> - <para><command># gluster volume profile <replaceable>VOLNAME</replaceable> stop</command> -</para> - <para>For example, to stop profiling on test-volume:</para> - <para><command># gluster volume profile <replaceable>test-volume</replaceable> stop</command> </para> - <para><computeroutput>Profiling stopped on test-volume</computeroutput></para> - </listitem> - </itemizedlist> - </section> - </section> - <section id="chap-Administration_Guide-Monitor_Workload-Top"> - <title> Running GlusterFS Volume TOP Command </title> - <para>GlusterFS Volume Top command allows you to view the glusterfs bricks’ performance metrics like -read, write, file open calls, file read calls, file write calls, directory open calls, and directory real -calls. The top command displays up to 100 results. -</para> - <para>This section describes how to run and view the results for the following GlusterFS Top commands: -</para> - <itemizedlist> - <listitem> - <para><xref linkend="chap-Administration_Guide-Monitor_Workload-Top-Open_FD_Count"/></para> - </listitem> - <listitem> - <para><xref linkend="chap-Administration_Guide-Monitor_Workload-Top-File_Read"/></para> - </listitem> - <listitem> - <para><xref linkend="chap-Administration_Guide-Monitor_Workload-Top-File_Write"/></para> - </listitem> - <listitem> - <para><xref linkend="chap-Administration_Guide-Monitor_Workload-Top-Open_Calls"/></para> - </listitem> - <listitem> - <para><xref linkend="chap-Administration_Guide-Monitor_Workload-Top-Read_Calls"/></para> - </listitem> - <listitem> - <para><xref linkend="chap-Administration_Guide-Monitor_Workload-Top-Read_Perf"/></para> - </listitem> - <listitem> - <para><xref linkend="chap-Administration_Guide-Monitor_Workload-Top-Write_Perf"/></para> - </listitem> - </itemizedlist> - <section id="chap-Administration_Guide-Monitor_Workload-Top-Open_FD_Count"> - <title>Viewing Open fd Count and Maximum fd Count </title> - <para>You can view both current open fd count (list of files that are currently the most opened and the -count) on the brick and the maximum open fd count (count of files that are the currently open and -the count of maximum number of files opened at any given point of time, since the servers are up -and running). If the brick name is not specified, then open fd metrics of all the bricks belonging to -the volume will be displayed. -</para> - <para><emphasis role="bold">To view open fd count and maximum fd count: </emphasis></para> - <itemizedlist> - <listitem> - <para>View open fd count and maximum fd count using the following command:</para> - <para><command># gluster volume top <replaceable>VOLNAME</replaceable> open [brick <replaceable>BRICK-NAME</replaceable>] [list-cnt <replaceable>cnt</replaceable>]</command> -</para> - <para>For example, to view open fd count and maximum fd count on brick <replaceable>server:/export</replaceable> of <replaceable>test-volume</replaceable> and list top 10 open calls: -</para> - <para><command># gluster volume top <replaceable>test-volume</replaceable> open brick <replaceable>server:/export/</replaceable> list-cnt <replaceable>10</replaceable></command></para> - <para><computeroutput>Brick: server:/export/dir1 </computeroutput></para> - <para><computeroutput>Current open fd's: 34 Max open fd's: 209 </computeroutput><programlisting> ==========Open file stats======== - -open file name -call count - -2 /clients/client0/~dmtmp/PARADOX/ - COURSES.DB - -11 /clients/client0/~dmtmp/PARADOX/ - ENROLL.DB - -11 /clients/client0/~dmtmp/PARADOX/ - STUDENTS.DB - -10 /clients/client0/~dmtmp/PWRPNT/ - TIPS.PPT - -10 /clients/client0/~dmtmp/PWRPNT/ - PCBENCHM.PPT - -9 /clients/client7/~dmtmp/PARADOX/ - STUDENTS.DB - -9 /clients/client1/~dmtmp/PARADOX/ - STUDENTS.DB - -9 /clients/client2/~dmtmp/PARADOX/ - STUDENTS.DB - -9 /clients/client0/~dmtmp/PARADOX/ - STUDENTS.DB - -9 /clients/client8/~dmtmp/PARADOX/ - STUDENTS.DB</programlisting></para> - </listitem> - </itemizedlist> - </section> - <section id="chap-Administration_Guide-Monitor_Workload-Top-File_Read"> - <title>Viewing Highest File Read Calls </title> - <para>You can view highest read calls on each brick. If brick name is not specified, then by default, list of -100 files will be displayed. -</para> - <para><emphasis role="bold">To view highest file Read calls:</emphasis> -</para> - <itemizedlist> - <listitem> - <para>View highest file Read calls using the following command: -</para> - <para><command># gluster volume top <replaceable>VOLNAME</replaceable> read [brick <replaceable>BRICK-NAME</replaceable>] [list-cnt <replaceable>cnt</replaceable>] </command></para> - <para>For example, to view highest Read calls on brick server:/export of test-volume: -</para> - <para><command># gluster volume top <replaceable>test-volume</replaceable> read brick <replaceable>server:/export</replaceable> list-cnt <replaceable>10</replaceable></command></para> - <para><computeroutput>Brick:</computeroutput> <replaceable>server:/export/dir1</replaceable><programlisting> ==========Read file stats======== - -read filename -call count - -116 /clients/client0/~dmtmp/SEED/LARGE.FIL - -64 /clients/client0/~dmtmp/SEED/MEDIUM.FIL - -54 /clients/client2/~dmtmp/SEED/LARGE.FIL - -54 /clients/client6/~dmtmp/SEED/LARGE.FIL - -54 /clients/client5/~dmtmp/SEED/LARGE.FIL - -54 /clients/client0/~dmtmp/SEED/LARGE.FIL - -54 /clients/client3/~dmtmp/SEED/LARGE.FIL - -54 /clients/client4/~dmtmp/SEED/LARGE.FIL - -54 /clients/client9/~dmtmp/SEED/LARGE.FIL - -54 /clients/client8/~dmtmp/SEED/LARGE.FIL</programlisting> </para> - </listitem> - </itemizedlist> - </section> - <section id="chap-Administration_Guide-Monitor_Workload-Top-File_Write"> - <title>Viewing Highest File Write Calls </title> - <para>You can view list of files which has highest file write calls on each brick. If brick name is not -specified, then by default, list of 100 files will be displayed. -</para> - <para><emphasis role="bold">To view highest file Write calls:</emphasis> -</para> - <itemizedlist> - <listitem> - <para>View highest file Write calls using the following command: -</para> - <para><command># gluster volume top <replaceable>VOLNAME</replaceable> write [brick <replaceable>BRICK-NAME</replaceable>] [list-cnt <replaceable>cnt</replaceable>] </command></para> - <para>For example, to view highest Write calls on brick <replaceable>server:/export</replaceable> of <replaceable>test-volume</replaceable>: -</para> - <para><command># gluster volume top <replaceable>test-volume</replaceable> write brick <replaceable>server:/export</replaceable> list-cnt <replaceable>10</replaceable></command></para> - <para><code>Brick: server:/export/dir1 </code><programlisting> ==========Write file stats======== -write call count filename - -83 /clients/client0/~dmtmp/SEED/LARGE.FIL - -59 /clients/client7/~dmtmp/SEED/LARGE.FIL - -59 /clients/client1/~dmtmp/SEED/LARGE.FIL - -59 /clients/client2/~dmtmp/SEED/LARGE.FIL - -59 /clients/client0/~dmtmp/SEED/LARGE.FIL - -59 /clients/client8/~dmtmp/SEED/LARGE.FIL - -59 /clients/client5/~dmtmp/SEED/LARGE.FIL - -59 /clients/client4/~dmtmp/SEED/LARGE.FIL - -59 /clients/client6/~dmtmp/SEED/LARGE.FIL - -59 /clients/client3/~dmtmp/SEED/LARGE.FIL</programlisting></para> - </listitem> - </itemizedlist> - </section> - <section id="chap-Administration_Guide-Monitor_Workload-Top-Open_Calls"> - <title>Viewing Highest Open Calls on Directories </title> - <para>You can view list of files which has highest open calls on directories of each brick. If brick name is -not specified, then the metrics of all the bricks belonging to that volume will be displayed. -</para> - <para>To view list of open calls on each directory</para> - <itemizedlist> - <listitem> - <para>View list of open calls on each directory using the following command: -</para> - <para><command># gluster volume top <replaceable>VOLNAME</replaceable> opendir [brick <replaceable>BRICK-NAME</replaceable>] [list-cnt <replaceable>cnt</replaceable>] </command></para> - <para>For example, to view open calls on brick server:/export/ of test-volume: -</para> - <para><command># gluster volume top <replaceable>test-volume</replaceable> opendir brick <replaceable>server:/export</replaceable> list-cnt <replaceable>10</replaceable></command></para> - <para><code>Brick: server:/export/dir1 </code><programlisting> ==========Directory open stats======== - -Opendir count directory name - -1001 /clients/client0/~dmtmp - -454 /clients/client8/~dmtmp - -454 /clients/client2/~dmtmp - -454 /clients/client6/~dmtmp - -454 /clients/client5/~dmtmp - -454 /clients/client9/~dmtmp - -443 /clients/client0/~dmtmp/PARADOX - -408 /clients/client1/~dmtmp - -408 /clients/client7/~dmtmp - -402 /clients/client4/~dmtmp</programlisting></para> - </listitem> - </itemizedlist> - </section> - <section id="chap-Administration_Guide-Monitor_Workload-Top-Read_Calls"> - <title>Viewing Highest Read Calls on Directory </title> - <para>You can view list of files which has highest directory read calls on each brick. If brick name is not -specified, then the metrics of all the bricks belonging to that volume will be displayed. -</para> - <para><emphasis role="bold">To view list of highest directory read calls on each brick</emphasis> -</para> - <itemizedlist> - <listitem> - <para>View list of highest directory read calls on each brick using the following command: -</para> - <para><command># gluster volume top <replaceable>VOLNAME</replaceable> readdir [brick <replaceable>BRICK-NAME</replaceable>] [list-cnt <replaceable>cnt</replaceable>] </command></para> - <para>For example, to view highest directory read calls on brick <replaceable>server:/export</replaceable> of <replaceable>test-volume</replaceable>:</para> - <para><command># gluster volume top <replaceable>test-volume</replaceable> readdir brick <replaceable>server:/export</replaceable> list-cnt <replaceable>10</replaceable></command> </para> - <para><code>Brick: <replaceable>server:/export/dir1</replaceable></code><programlisting>==========Directory readdirp stats======== - -readdirp count directory name - -1996 /clients/client0/~dmtmp - -1083 /clients/client0/~dmtmp/PARADOX - -904 /clients/client8/~dmtmp - -904 /clients/client2/~dmtmp - -904 /clients/client6/~dmtmp - -904 /clients/client5/~dmtmp - -904 /clients/client9/~dmtmp - -812 /clients/client1/~dmtmp - -812 /clients/client7/~dmtmp - -800 /clients/client4/~dmtmp</programlisting> -</para> - </listitem> - </itemizedlist> - </section> - <section id="chap-Administration_Guide-Monitor_Workload-Top-Read_Perf"> - <title>Viewing List of Read Performance on each Brick </title> - <para>You can view the read throughput of files on each brick. If brick name is not specified, then the -metrics of all the bricks belonging to that volume will be displayed. The output will be the read -throughput. -</para> - <para><programlisting> ==========Read throughput file stats======== - -read filename Time -through -put(MBp -s) - -2570.00 /clients/client0/~dmtmp/PWRPNT/ -2011-01-31 - TRIDOTS.POT 15:38:36.894610 -2570.00 /clients/client0/~dmtmp/PWRPNT/ -2011-01-31 - PCBENCHM.PPT 15:38:39.815310 -2383.00 /clients/client2/~dmtmp/SEED/ -2011-01-31 - MEDIUM.FIL 15:52:53.631499 - -2340.00 /clients/client0/~dmtmp/SEED/ -2011-01-31 - MEDIUM.FIL 15:38:36.926198 - -2299.00 /clients/client0/~dmtmp/SEED/ -2011-01-31 - LARGE.FIL 15:38:36.930445 - -2259.00 /clients/client0/~dmtmp/PARADOX/ -2011-01-31 - COURSES.X04 15:38:40.549919 - -2221.00 /clients/client0/~dmtmp/PARADOX/ -2011-01-31 - STUDENTS.VAL 15:52:53.298766 - -2221.00 /clients/client3/~dmtmp/SEED/ -2011-01-31 - COURSES.DB 15:39:11.776780 - -2184.00 /clients/client3/~dmtmp/SEED/ -2011-01-31 - MEDIUM.FIL 15:39:10.251764 - -2184.00 /clients/client5/~dmtmp/WORD/ -2011-01-31 - BASEMACH.DOC 15:39:09.336572 </programlisting>This command will initiate a dd for the specified count and block size and measures the -corresponding throughput. -</para> - <para><emphasis role="bold">To view list of read performance on each brick</emphasis> -</para> - <itemizedlist> - <listitem> - <para>View list of read performance on each brick using the following command: -</para> - <para><command># gluster volume top <replaceable>VOLNAME</replaceable> read-perf [bs <replaceable>blk-size</replaceable> count <replaceable>count</replaceable>] [brick <replaceable>BRICK-NAME</replaceable>] [list-cnt <replaceable>cnt</replaceable>]</command> -</para> - <para>For example, to view read performance on brick server:/export/ of test-volume, 256 block size -of count 1, and list count 10: -</para> - <para><command># gluster volume top <replaceable>test-volume</replaceable> read-perf bs 256 count 1 brick <replaceable>server:/export/ </replaceable>list-cnt <replaceable>10</replaceable></command></para> - <para><computeroutput>Brick: server:/export/dir1 256 bytes (256 B) copied, Throughput: 4.1 MB/s </computeroutput></para> - <programlisting> ==========Read throughput file stats======== - -read filename Time -through -put(MBp -s) - -2912.00 /clients/client0/~dmtmp/PWRPNT/ -2011-01-31 - TRIDOTS.POT 15:38:36.896486 - -2570.00 /clients/client0/~dmtmp/PWRPNT/ -2011-01-31 - PCBENCHM.PPT 15:38:39.815310 - -2383.00 /clients/client2/~dmtmp/SEED/ -2011-01-31 - MEDIUM.FIL 15:52:53.631499 - -2340.00 /clients/client0/~dmtmp/SEED/ -2011-01-31 - MEDIUM.FIL 15:38:36.926198 - -2299.00 /clients/client0/~dmtmp/SEED/ -2011-01-31 - LARGE.FIL 15:38:36.930445 - -2259.00 /clients/client0/~dmtmp/PARADOX/ -2011-01-31 - COURSES.X04 15:38:40.549919 - -2221.00 /clients/client9/~dmtmp/PARADOX/ -2011-01-31 - STUDENTS.VAL 15:52:53.298766 - -2221.00 /clients/client8/~dmtmp/PARADOX/ -2011-01-31 - COURSES.DB 15:39:11.776780 - -2184.00 /clients/client3/~dmtmp/SEED/ -2011-01-31 - MEDIUM.FIL 15:39:10.251764 - -2184.00 /clients/client5/~dmtmp/WORD/ -2011-01-31 - BASEMACH.DOC 15:39:09.336572 - </programlisting> - </listitem> - </itemizedlist> - </section> - <section id="chap-Administration_Guide-Monitor_Workload-Top-Write_Perf"> - <title>Viewing List of Write Performance on each Brick </title> - <para>You can view list of write throughput of files on each brick. If brick name is not specified, then the -metrics of all the bricks belonging to that volume will be displayed. The output will be the write -throughput. -</para> - <para>This command will initiate a dd for the specified count and block size and measures the -corresponding throughput. -To view list of write performance on each brick: -</para> - <itemizedlist> - <listitem> - <para>View list of write performance on each brick using the following command: -</para> - <para><command># gluster volume top <replaceable>VOLNAME</replaceable> write-perf [bs <replaceable>blk-size</replaceable> count <replaceable>count</replaceable>] [brick <replaceable>BRICK-NAME</replaceable>] [list-cnt <replaceable>cnt</replaceable>] </command></para> - <para>For example, to view write performance on brick <replaceable>server:/export/</replaceable> of <replaceable>test-volume</replaceable>, 256 block size -of count 1, and list count 10: -</para> - <para><command># gluster volume top <replaceable>test-volume</replaceable> write-perf bs 256 count 1 brick <replaceable>server:/export/ </replaceable>list-cnt <replaceable>10</replaceable></command></para> - <para><code>Brick</code>: <replaceable>server:/export/dir1</replaceable> -</para> - <para><code>256 bytes (256 B) copied, Throughput: 2.8 MB/s </code><programlisting> ==========Write throughput file stats======== - -write filename Time -throughput -(MBps) - -1170.00 /clients/client0/~dmtmp/SEED/ -2011-01-31 - SMALL.FIL 15:39:09.171494 - -1008.00 /clients/client6/~dmtmp/SEED/ -2011-01-31 - LARGE.FIL 15:39:09.73189 - -949.00 /clients/client0/~dmtmp/SEED/ -2011-01-31 - MEDIUM.FIL 15:38:36.927426 - -936.00 /clients/client0/~dmtmp/SEED/ -2011-01-31 - LARGE.FIL 15:38:36.933177 -897.00 /clients/client5/~dmtmp/SEED/ -2011-01-31 - MEDIUM.FIL 15:39:09.33628 - -897.00 /clients/client6/~dmtmp/SEED/ -2011-01-31 - MEDIUM.FIL 15:39:09.27713 - -885.00 /clients/client0/~dmtmp/SEED/ -2011-01-31 - SMALL.FIL 15:38:36.924271 - -528.00 /clients/client5/~dmtmp/SEED/ -2011-01-31 - LARGE.FIL 15:39:09.81893 - -516.00 /clients/client6/~dmtmp/ACCESS/ -2011-01-31 - FASTENER.MDB 15:39:01.797317 -</programlisting></para> - </listitem> - </itemizedlist> - </section> - </section> - <section id="sect-Administration_Guide-Monitor_Workload-Displaying_Volume_Information"> - <title>Displaying Volume Information </title> - <para>You can display information about a specific volume, or all volumes, as needed.</para> - <para><emphasis role="bold">To display volume information </emphasis></para> - <itemizedlist> - <listitem> - <para>Display information about a specific volume using the following command:</para> - <para><command># gluster volume info </command><varname>VOLNAME</varname></para> - <para>For example, to display information about test-volume:</para> - <para><programlisting># gluster volume info test-volume -Volume Name: test-volume -Type: Distribute -Status: Created -Number of Bricks: 4 -Bricks: -Brick1: server1:/exp1 -Brick2: server2:/exp2 -Brick3: server3:/exp3 -Brick4: server4:/exp4</programlisting></para> - </listitem> - <listitem> - <para>Display information about all volumes using the following command:</para> - <para><command># gluster volume info all</command></para> - <para><programlisting># gluster volume info all - -Volume Name: test-volume -Type: Distribute -Status: Created -Number of Bricks: 4 -Bricks: -Brick1: server1:/exp1 -Brick2: server2:/exp2 -Brick3: server3:/exp3 -Brick4: server4:/exp4 - -Volume Name: mirror -Type: Distributed-Replicate -Status: Started -Number of Bricks: 2 X 2 = 4 -Bricks: -Brick1: server1:/brick1 -Brick2: server2:/brick2 -Brick3: server3:/brick3 -Brick4: server4:/brick4 - -Volume Name: Vol -Type: Distribute -Status: Started -Number of Bricks: 1 -Bricks: -Brick: server:/brick6 - -</programlisting></para> - </listitem> - </itemizedlist> - </section> - <section id="sect-Administration_Guide-Monitor_Workload-Performing_Statedump"> - <title>Performing Statedump on a Volume </title> - <para>Statedump is a mechanism through which you can get details of all internal variables and state of the glusterfs process at the time of issuing the command.You can perform statedumps of the brick processes and nfs server process of a volume using the statedump command. The following options can be used to determine what information is to be dumped:</para> - <itemizedlist> - <listitem> - <para><emphasis role="bold">mem</emphasis> - Dumps the memory usage and memory pool details of the bricks.</para> - </listitem> - <listitem> - <para><emphasis role="bold">iobuf</emphasis> - Dumps iobuf details of the bricks.</para> - </listitem> - <listitem> - <para><emphasis role="bold">priv</emphasis> - Dumps private information of loaded translators.</para> - </listitem> - <listitem> - <para><emphasis role="bold">callpool</emphasis> - Dumps the pending calls of the volume.</para> - </listitem> - <listitem> - <para><emphasis role="bold">fd</emphasis> - Dumps the open fd tables of the volume.</para> - </listitem> - <listitem> - <para><emphasis role="bold">inode</emphasis> - Dumps the inode tables of the volume.</para> - </listitem> - </itemizedlist> - <para><emphasis role="bold">To display volume statedump </emphasis></para> - <itemizedlist> - <listitem> - <para>Display statedump of a volume or NFS server using the following command:</para> - <para> <command># gluster volume statedump <replaceable>VOLNAME</replaceable> [nfs] [all|mem|iobuf|callpool|priv|fd|inode]</command></para> - <para>For example, to display statedump of test-volume:</para> - <para><programlisting># gluster volume statedump test-volume -Volume statedump successful</programlisting></para> - <para>The statedump files are created on the brick servers in the<filename> /tmp</filename> directory or in the directory set using <command>server.statedump-path</command> volume option. The naming convention of the dump file is <filename><brick-path>.<brick-pid>.dump</filename>.</para> - </listitem> - <listitem> - <para>By defult, the output of the statedump is stored at <filename> /tmp/<brickname.PID.dump></filename> file on that particular server. Change the directory of the statedump file using the following command:</para> - <para><command># gluster volume set <replaceable>VOLNAME</replaceable> server.statedump-path <replaceable>path</replaceable></command></para> - <para>For example, to change the location of the statedump file of test-volume:</para> - <para><programlisting># gluster volume set test-volume server.statedump-path /usr/local/var/log/glusterfs/dumps/ -Set volume successful</programlisting></para> - <para>You can view the changed path of the statedump file using the following command:</para> - <para><command># gluster volume info <replaceable>VOLNAME</replaceable></command></para> - </listitem> - </itemizedlist> - </section> - <section id="sect-Administration_Guide-Monitor_Workload-Displaying_Volume_Status"> - <title>Displaying Volume Status </title> - <para>You can display the status information about a specific volume, brick or all volumes, as needed. Status information can be used to understand the current status of the brick, nfs processes, and overall file system. Status information can also be used to monitor and debug the volume information. You can view status of the volume along with the following details:</para> - <itemizedlist> - <listitem> - <para><emphasis role="bold">detail</emphasis> - Displays additional information about the bricks.</para> - </listitem> - <listitem> - <para><emphasis role="bold">clients</emphasis> - Displays the list of clients connected to the volume.</para> - </listitem> - <listitem> - <para><emphasis role="bold">mem</emphasis> - Displays the memory usage and memory pool details of the bricks.</para> - </listitem> - <listitem> - <para><emphasis role="bold">inode</emphasis> - Displays the inode tables of the volume.</para> - </listitem> - <listitem> - <para><emphasis role="bold">fd</emphasis> - Displays the open fd (file descriptors) tables of the volume.</para> - </listitem> - <listitem> - <para><emphasis role="bold">callpool</emphasis> - Displays the pending calls of the volume.</para> - </listitem> - </itemizedlist> - <para><emphasis role="bold">To display volume status </emphasis></para> - <itemizedlist> - <listitem> - <para>Display information about a specific volume using the following command:</para> - <para><command># gluster volume status [all|<replaceable>VOLNAME</replaceable> [<replaceable>BRICKNAME</replaceable>]] [detail|clients|mem|inode|fd|callpool]</command> </para> - <para>For example, to display information about test-volume:</para> - <para><programlisting># gluster volume status test-volume -STATUS OF VOLUME: test-volume -BRICK PORT ONLINE PID --------------------------------------------------------- -arch:/export/1 24009 Y 22445 --------------------------------------------------------- -arch:/export/2 24010 Y 22450</programlisting></para> - </listitem> - <listitem> - <para>Display information about all volumes using the following command:</para> - <para><command># gluster volume status all</command> -</para> - <para><programlisting># gluster volume status all -STATUS OF VOLUME: volume-test -BRICK PORT ONLINE PID --------------------------------------------------------- -arch:/export/4 24010 Y 22455 - -STATUS OF VOLUME: test-volume -BRICK PORT ONLINE PID --------------------------------------------------------- -arch:/export/1 24009 Y 22445 --------------------------------------------------------- -arch:/export/2 24010 Y 22450</programlisting></para> - </listitem> - <listitem> - <para>Display additional information about the bricks using the following command:</para> - <para><command># gluster volume status <replaceable>VOLNAME</replaceable> detail</command> -</para> - <para>For example, to display additional information about the bricks of test-volume:</para> - <para><programlisting># gluster volume status test-volume details -STATUS OF VOLUME: test-volume -------------------------------------------- -Brick : arch:/export/1 -Port : 24009 -Online : Y -Pid : 16977 -File System : rootfs -Device : rootfs -Mount Options : rw -Disk Space Free : 13.8GB -Total Disk Space : 46.5GB -Inode Size : N/A -Inode Count : N/A -Free Inodes : N/A - -Number of Bricks: 1 -Bricks: -Brick: server:/brick6</programlisting></para> - </listitem> - <listitem> - <para>Display the list of clients accessing the volumes using the following command:</para> - <para><command># gluster volume status <replaceable>VOLNAME</replaceable> clients</command> -</para> - <para>For example, to display the list of clients connected to test-volume:</para> - <para><programlisting># gluster volume status test-volume clients -Brick : arch:/export/1 -Clients connected : 2 -Hostname Bytes Read BytesWritten --------- --------- ------------ -127.0.0.1:1013 776 676 -127.0.0.1:1012 50440 51200</programlisting></para> - </listitem> - <listitem> - <para>Display the memory usage and memory pool details of the bricks using the following command:</para> - <para><command># gluster volume status <replaceable>VOLNAME</replaceable> mem</command> -</para> - <para>For example, to display the memory usage and memory pool details of the bricks of test-volume:</para> - <screen>Memory status for volume : test-volume ----------------------------------------------- -Brick : arch:/export/1 -Mallinfo --------- -Arena : 434176 -Ordblks : 2 -Smblks : 0 -Hblks : 12 -Hblkhd : 40861696 -Usmblks : 0 -Fsmblks : 0 -Uordblks : 332416 -Fordblks : 101760 -Keepcost : 100400 - -Mempool Stats -------------- -Name HotCount ColdCount PaddedSizeof AllocCount MaxAlloc ----- -------- --------- ------------ ---------- -------- -test-volume-server:fd_t 0 16384 92 57 5 -test-volume-server:dentry_t 59 965 84 59 59 -test-volume-server:inode_t 60 964 148 60 60 -test-volume-server:rpcsvc_request_t 0 525 6372 351 2 -glusterfs:struct saved_frame 0 4096 124 2 2 -glusterfs:struct rpc_req 0 4096 2236 2 2 -glusterfs:rpcsvc_request_t 1 524 6372 2 1 -glusterfs:call_stub_t 0 1024 1220 288 1 -glusterfs:call_stack_t 0 8192 2084 290 2 -glusterfs:call_frame_t 0 16384 172 1728 6</screen> - </listitem> - <listitem> - <para>Display the inode tables of the volume using the following command:</para> - <para><command># gluster volume status <replaceable>VOLNAME</replaceable> inode</command> -</para> - <para>For example, to display the inode tables of the test-volume:</para> - <para><programlisting># gluster volume status test-volume inode -inode tables for volume test-volume ----------------------------------------------- -Brick : arch:/export/1 -Active inodes: -GFID Lookups Ref IA type ----- ------- --- ------- -6f3fe173-e07a-4209-abb6-484091d75499 1 9 2 -370d35d7-657e-44dc-bac4-d6dd800ec3d3 1 1 2 - -LRU inodes: -GFID Lookups Ref IA type ----- ------- --- ------- -80f98abe-cdcf-4c1d-b917-ae564cf55763 1 0 1 -3a58973d-d549-4ea6-9977-9aa218f233de 1 0 1 -2ce0197d-87a9-451b-9094-9baa38121155 1 0 2</programlisting></para> - </listitem> - <listitem> - <para>Display the open fd tables of the volume using the following command:</para> - <para><command># gluster volume status <replaceable>VOLNAME</replaceable> fd</command> -</para> - <para>For example, to display the open fd tables of the test-volume:</para> - <para><screen># gluster volume status test-volume fd - -FD tables for volume test-volume ----------------------------------------------- -Brick : arch:/export/1 -Connection 1: -RefCount = 0 MaxFDs = 128 FirstFree = 4 -FD Entry PID RefCount Flags --------- --- -------- ----- -0 26311 1 2 -1 26310 3 2 -2 26310 1 2 -3 26311 3 2 - -Connection 2: -RefCount = 0 MaxFDs = 128 FirstFree = 0 -No open fds - -Connection 3: -RefCount = 0 MaxFDs = 128 FirstFree = 0 -No open fds</screen></para> - </listitem> - <listitem> - <para>Display the pending calls of the volume using the following command:</para> - <para><command># gluster volume status <replaceable>VOLNAME</replaceable> callpool</command> -</para> - <para>Each call has a call stack containing call frames.</para> - <para>For example, to display the pending calls of test-volume:</para> - <para><programlisting># gluster volume status test-volume - -Pending calls for volume test-volume ----------------------------------------------- -Brick : arch:/export/1 -Pending calls: 2 -Call Stack1 - UID : 0 - GID : 0 - PID : 26338 - Unique : 192138 - Frames : 7 - Frame 1 - Ref Count = 1 - Translator = test-volume-server - Completed = No - Frame 2 - Ref Count = 0 - Translator = test-volume-posix - Completed = No - Parent = test-volume-access-control - Wind From = default_fsync - Wind To = FIRST_CHILD(this)->fops->fsync - Frame 3 - Ref Count = 1 - Translator = test-volume-access-control - Completed = No - Parent = repl-locks - Wind From = default_fsync - Wind To = FIRST_CHILD(this)->fops->fsync - Frame 4 - Ref Count = 1 - Translator = test-volume-locks - Completed = No - Parent = test-volume-io-threads - Wind From = iot_fsync_wrapper - Wind To = FIRST_CHILD (this)->fops->fsync - Frame 5 - Ref Count = 1 - Translator = test-volume-io-threads - Completed = No - Parent = test-volume-marker - Wind From = default_fsync - Wind To = FIRST_CHILD(this)->fops->fsync - Frame 6 - Ref Count = 1 - Translator = test-volume-marker - Completed = No - Parent = /export/1 - Wind From = io_stats_fsync - Wind To = FIRST_CHILD(this)->fops->fsync - Frame 7 - Ref Count = 1 - Translator = /export/1 - Completed = No - Parent = test-volume-server - Wind From = server_fsync_resume - Wind To = bound_xl->fops->fsync</programlisting></para> - </listitem> - </itemizedlist> - </section> -</chapter> diff --git a/doc/admin-guide/en-US/admin_setting_volumes.xml b/doc/admin-guide/en-US/admin_setting_volumes.xml deleted file mode 100644 index 6a8468d5f11..00000000000 --- a/doc/admin-guide/en-US/admin_setting_volumes.xml +++ /dev/null @@ -1,325 +0,0 @@ -<?xml version='1.0' encoding='UTF-8'?> -<!-- This document was created with Syntext Serna Free. --><!DOCTYPE chapter PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN" "http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd" [ -<!ENTITY % BOOK_ENTITIES SYSTEM "Administration_Guide.ent"> -%BOOK_ENTITIES; -]> -<chapter id="chap-Administration_Guide-Setting_Volumes"> - <title>Setting up GlusterFS Server Volumes</title> - <para>A volume is a logical collection of bricks where each brick is an export directory on a server in the trusted storage pool. Most of the gluster management operations are performed on the volume. </para> - <para>To create a new volume in your storage environment, specify the bricks that comprise the volume. After you have created a new volume, you must start it before attempting to mount it. </para> - <itemizedlist> - <listitem> - <para>Volumes of the following types can be created in your storage environment: </para> - <itemizedlist> - <listitem> - <para>Distributed - Distributed volumes distributes files throughout the bricks in the volume. You can use distributed volumes where the requirement is to scale storage and the redundancy is either not important or is provided by other hardware/software layers. For more information, see <xref linkend="sect-Administration_Guide-Setting_Volumes-Distributed"/> .</para> - </listitem> - <listitem> - <para>Replicated – Replicated volumes replicates files across bricks in the volume. You can use replicated volumes in environments where high-availability and high-reliability are critical. For more information, see <xref linkend="sect-Administration_Guide-Setting_Volumes-Replicated"/>.</para> - </listitem> - <listitem> - <para>Striped – Striped volumes stripes data across bricks in the volume. For best results, you should use striped volumes only in high concurrency environments accessing very large files. For more information, see <xref linkend="sect-Administration_Guide-Setting_Volumes-Striped"/>.</para> - </listitem> - <listitem> - <para>Distributed Striped - Distributed striped volumes stripe data across two or more nodes in the cluster. You should use distributed striped volumes where the requirement is to scale storage and in high concurrency environments accessing very large files is critical. For more information, see <xref linkend="sect-Administration_Guide-Setting_Volumes-Distributed_Striped"/>.</para> - </listitem> - <listitem> - <para>Distributed Replicated - Distributed replicated volumes distributes files across replicated bricks in the volume. You can use distributed replicated volumes in environments where the requirement is to scale storage and high-reliability is critical. Distributed replicated volumes also offer improved read performance in most environments. For more information, see <xref linkend="sect-Administration_Guide-Setting_Volumes-Distributed_Replicated"/>. </para> - </listitem> - <listitem> - <para>Distributed Striped Replicated – Distributed striped replicated volumes distributes striped data across replicated bricks in the cluster. For best results, you should use distributed striped replicated volumes in highly concurrent environments where parallel access of very large files and performance is critical. In this release, configuration of this volume type is supported only for Map Reduce workloads. For more information, see <xref linkend="sect-Administration_Guide-Setting_Volumes-Distributed_Striped_Replicated"/>. -</para> - </listitem> - <listitem> - <para>Striped Replicated – Striped replicated volumes stripes data across replicated bricks in the cluster. For best results, you should use striped replicated volumes in highly concurrent environments where there is parallel access of very large files and performance is critical. In this release, configuration of this volume type is supported only for Map Reduce workloads. For more -information, see <xref linkend="sect-Administration_Guide-Setting_Volumes-Striped_Replicated"/>.</para> - </listitem> - </itemizedlist> - </listitem> - </itemizedlist> - <para><emphasis role="bold">To create a new volume </emphasis></para> - <itemizedlist> - <listitem> - <para>Create a new volume :</para> - <para><command># gluster volume create<replaceable> NEW-VOLNAME</replaceable> [stripe <replaceable>COUNT</replaceable> | replica <replaceable>COUNT</replaceable>] [transport tcp | rdma | tcp, rdma] <replaceable>NEW-BRICK1 NEW-BRICK2 NEW-BRICK3...</replaceable></command></para> - <para>For example, to create a volume called test-volume consisting of server3:/exp3 and server4:/exp4:</para> - <para><programlisting># gluster volume create test-volume server3:/exp3 server4:/exp4 -Creation of test-volume has been successful -Please start the volume to access data.</programlisting></para> - </listitem> - </itemizedlist> - <section id="sect-Administration_Guide-Setting_Volumes-Distributed"> - <title>Creating Distributed Volumes</title> - <para>In a distributed volumes files are spread randomly across the bricks in the volume. Use distributed volumes where you need to scale storage and redundancy is either not important or is provided by other hardware/software layers. </para> - <para><note> - <para>Disk/server failure in distributed volumes can result in a serious loss of data because directory contents are spread randomly across the bricks in the volume. </para> - </note></para> - <figure> - <title>Illustration of a Distributed Volume</title> - <mediaobject> - <imageobject> - <imagedata fileref="images/Distributed_Volume.png"/> - </imageobject> - </mediaobject> - </figure> - <para><emphasis role="bold">To create a distributed volume</emphasis></para> - <orderedlist> - <listitem> - <para>Create a trusted storage pool as described earlier in <xref linkend="sect-Administration_Guide-Storage_Pools-Adding_Servers"/>.</para> - </listitem> - <listitem> - <para>Create the distributed volume:</para> - <para><command># gluster volume create <replaceable>NEW-VOLNAME</replaceable> [transport tcp | rdma | tcp,rdma] <replaceable>NEW-BRICK...</replaceable></command></para> - <para>For example, to create a distributed volume with four storage servers using tcp:</para> - <para><programlisting># gluster volume create test-volume server1:/exp1 server2:/exp2 server3:/exp3 server4:/exp4 -Creation of test-volume has been successful -Please start the volume to access data.</programlisting></para> - <para>(Optional) You can display the volume information:</para> - <para><programlisting># gluster volume info -Volume Name: test-volume -Type: Distribute -Status: Created -Number of Bricks: 4 -Transport-type: tcp -Bricks: -Brick1: server1:/exp1 -Brick2: server2:/exp2 -Brick3: server3:/exp3 -Brick4: server4:/exp4</programlisting></para> - <para>For example, to create a distributed volume with four storage servers over InfiniBand:</para> - <para><programlisting># gluster volume create test-volume transport rdma server1:/exp1 server2:/exp2 server3:/exp3 server4:/exp4 -Creation of test-volume has been successful -Please start the volume to access data.</programlisting></para> - <para>If the transport type is not specified, <emphasis role="italic"> tcp</emphasis> is used as the default. You can also set additional options if required, such as auth.allow or auth.reject. For more information, see <xref linkend="sect-Administration_Guide-Managing_Volumes-Tuning"/></para> - <para><note> - <para>Make sure you start your volumes before you try to mount them or else client operations after the mount will hang, see <xref linkend="sect-Administration_Guide-Setting_Volumes-Starting"/> for details. </para> - </note></para> - </listitem> - </orderedlist> - </section> - <section id="sect-Administration_Guide-Setting_Volumes-Replicated"> - <title>Creating Replicated Volumes </title> - <para>Replicated volumes create copies of files across multiple bricks in the volume. You can use replicated volumes in environments where high-availability and high-reliability are critical. </para> - <para><note> - <para>The number of bricks should be equal to of the replica count for a replicated volume. -To protect against server and disk failures, it is recommended that the bricks of the volume are from different servers. </para> - </note></para> - <figure> - <title>Illustration of a Replicated Volume</title> - <mediaobject> - <imageobject> - <imagedata fileref="images/Replicated_Volume.png"/> - </imageobject> - </mediaobject> - </figure> - <para><emphasis role="bold">To create a replicated volume </emphasis></para> - <orderedlist> - <listitem> - <para>Create a trusted storage pool as described earlier in <xref linkend="sect-Administration_Guide-Storage_Pools-Adding_Servers"/>.</para> - </listitem> - <listitem> - <para>Create the replicated volume:</para> - <para><command># gluster volume create <replaceable>NEW-VOLNAME</replaceable> [replica <replaceable>COUNT</replaceable>] [transport tcp | rdma tcp,rdma] <replaceable>NEW-BRICK...</replaceable></command></para> - <para>For example, to create a replicated volume with two storage servers:</para> - <para><programlisting># gluster volume create test-volume replica 2 transport tcp server1:/exp1 server2:/exp2 -Creation of test-volume has been successful -Please start the volume to access data.</programlisting></para> - <para>If the transport type is not specified, <emphasis role="italic"> tcp</emphasis> is used as the default. You can also set additional options if required, such as auth.allow or auth.reject. For more information, see <xref linkend="sect-Administration_Guide-Managing_Volumes-Tuning"/></para> - <para><note> - <para>Make sure you start your volumes before you try to mount them or else client operations after the mount will hang, see <xref linkend="sect-Administration_Guide-Setting_Volumes-Starting"/> for details. </para> - </note></para> - </listitem> - </orderedlist> - </section> - <section id="sect-Administration_Guide-Setting_Volumes-Striped"> - <title>Creating Striped Volumes</title> - <para>Striped volumes stripes data across bricks in the volume. For best results, you should use striped volumes only in high concurrency environments accessing very large files.</para> - <para><note> - <para>The number of bricks should be a equal to the stripe count for a striped volume. </para> - </note></para> - <figure> - <title>Illustration of a Striped Volume</title> - <mediaobject> - <imageobject> - <imagedata fileref="images/Striped_Volume.png"/> - </imageobject> - </mediaobject> - </figure> - <para><emphasis role="bold">To create a striped volume </emphasis></para> - <orderedlist> - <listitem> - <para>Create a trusted storage pool as described earlier in <xref linkend="sect-Administration_Guide-Storage_Pools-Adding_Servers"/>.</para> - </listitem> - <listitem> - <para>Create the striped volume:</para> - <para><command># gluster volume create <replaceable>NEW-VOLNAME</replaceable> [stripe <replaceable>COUNT</replaceable>] [transport tcp | rdma | tcp,rdma] <replaceable>NEW-BRICK...</replaceable></command></para> - <para>For example, to create a striped volume across two storage servers:</para> - <para><programlisting># gluster volume create test-volume stripe 2 transport tcp server1:/exp1 server2:/exp2 -Creation of test-volume has been successful -Please start the volume to access data.</programlisting></para> - <para>If the transport type is not specified, <emphasis role="italic"> tcp</emphasis> is used as the default. You can also set additional options if required, such as auth.allow or auth.reject. For more information, see <xref linkend="sect-Administration_Guide-Managing_Volumes-Tuning"/></para> - <para><note> - <para>Make sure you start your volumes before you try to mount them or else client operations after the mount will hang, see <xref linkend="sect-Administration_Guide-Setting_Volumes-Starting"/> for details. </para> - </note></para> - </listitem> - </orderedlist> - </section> - <section id="sect-Administration_Guide-Setting_Volumes-Distributed_Striped"> - <title>Creating Distributed Striped Volumes </title> - <para>Distributed striped volumes stripes files across two or more nodes in the cluster. For best results, you should use distributed striped volumes where the requirement is to scale storage and in high concurrency environments accessing very large files is critical.</para> - <para><note> - <para>The number of bricks should be a multiple of the stripe count for a distributed striped volume. </para> - </note></para> - <figure> - <title>Illustration of a Distributed Striped Volume</title> - <mediaobject> - <imageobject> - <imagedata fileref="images/Distributed_Striped_Volume.png"/> - </imageobject> - </mediaobject> - </figure> - <para><emphasis role="bold">To create a distributed striped volume </emphasis></para> - <orderedlist> - <listitem> - <para>Create a trusted storage pool as described earlier in <xref linkend="sect-Administration_Guide-Storage_Pools-Adding_Servers"/>.</para> - </listitem> - <listitem> - <para>Create the distributed striped volume:</para> - <para><command># gluster volume create <replaceable>NEW-VOLNAME</replaceable> [stripe <replaceable>COUNT</replaceable>] [transport tcp | rdma | tcp,rdma] <replaceable>NEW-BRICK...</replaceable></command></para> - <para>For example, to create a distributed striped volume across eight storage servers:</para> - <para><programlisting># gluster volume create test-volume stripe 4 transport tcp server1:/exp1 server2:/exp2 server3:/exp3 server4:/exp4 server5:/exp5 server6:/exp6 server7:/exp7 server8:/exp8 -Creation of test-volume has been successful -Please start the volume to access data.</programlisting></para> - <para>If the transport type is not specified, <emphasis role="italic"> tcp</emphasis> is used as the default. You can also set additional options if required, such as auth.allow or auth.reject. For more information, see <xref linkend="sect-Administration_Guide-Managing_Volumes-Tuning"/></para> - <para><note> - <para>Make sure you start your volumes before you try to mount them or else client operations after the mount will hang, see <xref linkend="sect-Administration_Guide-Setting_Volumes-Starting"/> for details. </para> - </note></para> - </listitem> - </orderedlist> - </section> - <section id="sect-Administration_Guide-Setting_Volumes-Distributed_Replicated"> - <title>Creating Distributed Replicated Volumes </title> - <para>Distributes files across replicated bricks in the volume. You can use distributed replicated volumes in environments where the requirement is to scale storage and high-reliability is critical. Distributed replicated volumes also offer improved read performance in most environments.</para> - <para><note> - <para>The number of bricks should be a multiple of the replica count for a distributed replicated volume. Also, the order in which bricks are specified has a great effect on data protection. Each replica_count consecutive bricks in the list you give will form a replica set, with all replica sets combined into a volume-wide distribute set. To make sure that replica-set members are not placed on the same node, list the first brick on every server, then the second brick on every server in the same order, and so on. </para> - </note></para> - <figure> - <title>Illustration of a Distributed Replicated Volume</title> - <mediaobject> - <imageobject> - <imagedata fileref="images/Distributed_Replicated_Volume.png"/> - </imageobject> - </mediaobject> - </figure> - <para><emphasis role="bold">To create a distributed replicated volume </emphasis></para> - <orderedlist> - <listitem> - <para>Create a trusted storage pool as described earlier in <xref linkend="sect-Administration_Guide-Storage_Pools-Adding_Servers"/>.</para> - </listitem> - <listitem> - <para>Create the distributed replicated volume:</para> - <para><command># gluster volume create <replaceable>NEW-VOLNAME</replaceable> [replica <replaceable>COUNT</replaceable>] [transport tcp | rdma | tcp,rdma] <replaceable>NEW-BRICK...</replaceable></command></para> - <para>For example, four node distributed (replicated) volume with a two-way mirror: -</para> - <para><programlisting># gluster volume create test-volume replica 2 transport tcp server1:/exp1 server2:/exp2 server3:/exp3 server4:/exp4 -Creation of test-volume has been successful -Please start the volume to access data.</programlisting></para> - <para>For example, to create a six node distributed (replicated) volume with a two-way mirror:</para> - <para><programlisting># gluster volume create test-volume replica 2 transport tcp server1:/exp1 server2:/exp2 server3:/exp3 server4:/exp4 server5:/exp5 server6:/exp6 -Creation of test-volume has been successful -Please start the volume to access data.</programlisting></para> - <para>If the transport type is not specified, <emphasis role="italic"> tcp</emphasis> is used as the default. You can also set additional options if required, such as auth.allow or auth.reject. For more information, see <xref linkend="sect-Administration_Guide-Managing_Volumes-Tuning"/></para> - <para><note> - <para>Make sure you start your volumes before you try to mount them or else client operations after the mount will hang, see <xref linkend="sect-Administration_Guide-Setting_Volumes-Starting"/> for details. </para> - </note></para> - </listitem> - </orderedlist> - </section> - <section id="sect-Administration_Guide-Setting_Volumes-Distributed_Striped_Replicated"> - <title>Creating Distributed Striped Replicated Volumes </title> - <para>Distributed striped replicated volumes distributes striped data across replicated bricks in the cluster. For best results, you should use distributed striped replicated volumes in highly concurrent environments where parallel access of very large files and performance is critical. In this release, configuration of this volume type is supported only for Map Reduce workloads. </para> - <para><note> - <para>The number of bricks should be a multiples of number of stripe count and replica count for -a distributed striped replicated volume. - </para> - </note></para> - <para><emphasis role="bold">To create a distributed striped replicated volume</emphasis> -</para> - <orderedlist> - <listitem> - <para>Create a trusted storage pool as described earlier in <xref linkend="sect-Administration_Guide-Storage_Pools-Adding_Servers"/>.</para> - </listitem> - <listitem> - <para>Create a distributed striped replicated volume using the following command:</para> - <para><command># gluster volume create <replaceable>NEW-VOLNAME</replaceable> [stripe <replaceable>COUNT</replaceable>] [replica <replaceable>COUNT</replaceable>] [transport tcp | rdma | tcp,rdma] <replaceable>NEW-BRICK...</replaceable></command></para> - <para>For example, to create a distributed replicated striped volume across eight storage servers: -</para> - <para><programlisting># gluster volume create test-volume stripe 2 replica 2 transport tcp server1:/exp1 server2:/exp2 server3:/exp3 server4:/exp4 server5:/exp5 server6:/exp6 server7:/exp7 server8:/exp8 -Creation of test-volume has been successful -Please start the volume to access data.</programlisting></para> - <para>If the transport type is not specified, <emphasis role="italic"> tcp</emphasis> is used as the default. You can also set additional options if required, such as auth.allow or auth.reject. For more information, see <xref linkend="sect-Administration_Guide-Managing_Volumes-Tuning"/></para> - <para><note> - <para>Make sure you start your volumes before you try to mount them or else client operations after the mount will hang, see <xref linkend="sect-Administration_Guide-Setting_Volumes-Starting"/> for details. </para> - </note></para> - </listitem> - </orderedlist> - </section> - <section id="sect-Administration_Guide-Setting_Volumes-Striped_Replicated"> - <title>Creating Striped Replicated Volumes </title> - <para>Striped replicated volumes stripes data across replicated bricks in the cluster. For best results, you should use striped replicated volumes in highly concurrent environments where there is parallel access of very large files and performance is critical. In this release, configuration of this volume type is supported only for Map Reduce workloads.</para> - <para><note> - <para>The number of bricks should be a multiple of the replicate count and stripe count for a -striped replicated volume. -</para> - </note></para> - <figure> - <title>Illustration of a Striped Replicated Volume</title> - <mediaobject> - <imageobject> - <imagedata fileref="images/Striped_Replicated_Volume.png"/> - </imageobject> - </mediaobject> - </figure> - <para><emphasis role="bold">To create a striped replicated volume</emphasis> -</para> - <orderedlist> - <listitem> - <para>Create a trusted storage pool consisting of the storage servers that will comprise the volume.</para> - <para>For more information, see <xref linkend="sect-Administration_Guide-Storage_Pools-Adding_Servers"/>.</para> - </listitem> - <listitem> - <para>Create a striped replicated volume :</para> - <para><command># gluster volume create <replaceable>NEW-VOLNAME</replaceable> [stripe <replaceable>COUNT</replaceable>] [replica <replaceable>COUNT</replaceable>] [transport tcp | rdma | tcp,rdma] <replaceable>NEW-BRICK...</replaceable></command></para> - <para>For example, to create a striped replicated volume across four storage servers: - -</para> - <para><programlisting># gluster volume create test-volume stripe 2 replica 2 transport tcp server1:/exp1 server2:/exp2 server3:/exp3 server4:/exp4 -Creation of test-volume has been successful -Please start the volume to access data.</programlisting></para> - <para>To create a striped replicated volume across six storage servers: -</para> - <para><programlisting># gluster volume create test-volume stripe 3 replica 2 transport tcp server1:/exp1 server2:/exp2 server3:/exp3 server4:/exp4 server5:/exp5 server6:/exp6 -Creation of test-volume has been successful -Please start the volume to access data.</programlisting></para> - <para>If the transport type is not specified, <emphasis role="italic"> tcp</emphasis> is used as the default. You can also set additional options if required, such as auth.allow or auth.reject. For more information, see <xref linkend="sect-Administration_Guide-Managing_Volumes-Tuning"/></para> - <para><note> - <para>Make sure you start your volumes before you try to mount them or else client operations after the mount will hang, see <xref linkend="sect-Administration_Guide-Setting_Volumes-Starting"/> for details. </para> - </note></para> - </listitem> - </orderedlist> - </section> - <section id="sect-Administration_Guide-Setting_Volumes-Starting"> - <title>Starting Volumes </title> - <para>You must start your volumes before you try to mount them. </para> - <para><emphasis role="bold">To start a volume </emphasis></para> - <itemizedlist> - <listitem> - <para>Start a volume:</para> - <para><command># gluster volume start <replaceable>VOLNAME</replaceable></command></para> - <para>For example, to start test-volume:</para> - <para><programlisting># gluster volume start test-volume -Starting test-volume has been successful</programlisting></para> - </listitem> - </itemizedlist> - </section> -</chapter> diff --git a/doc/admin-guide/en-US/admin_settingup_clients.xml b/doc/admin-guide/en-US/admin_settingup_clients.xml deleted file mode 100644 index 22979acf477..00000000000 --- a/doc/admin-guide/en-US/admin_settingup_clients.xml +++ /dev/null @@ -1,511 +0,0 @@ -<?xml version='1.0' encoding='UTF-8'?> -<!-- This document was created with Syntext Serna Free. --><!DOCTYPE chapter PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN" "http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd" [ -<!ENTITY % BOOK_ENTITIES SYSTEM "Administration_Guide.ent"> -%BOOK_ENTITIES; -]> -<chapter id="chap-Administration_Guide-GlusterFS_Client"> - <title>Accessing Data - Setting Up GlusterFS Client</title> - <para>You can access gluster volumes in multiple ways. You can use Gluster Native Client method for high concurrency, performance and transparent failover in GNU/Linux clients. You can also use NFS v3 to access gluster volumes. Extensive testing has be done on GNU/Linux clients and NFS implementation in other operating system, such as FreeBSD, and Mac OS X, as well as Windows 7 (Professional and Up) and Windows Server 2003. Other NFS client implementations may work with gluster NFS server.</para> - <para>You can use CIFS to access volumes when using Microsoft Windows as well as SAMBA clients. For this access method, Samba packages need to be present on the client side. </para> - <section id="sect-Administration_Guide-Test_Chapter-GlusterFS_Client-Native"> - <title>Gluster Native Client</title> - <para>The Gluster Native Client is a FUSE-based client running in user space. Gluster Native Client is the recommended method for accessing volumes when high concurrency and high write performance is required.</para> - <para>This section introduces the Gluster Native Client and explains how to install the software on client machines. This section also describes how to mount volumes on clients (both manually and automatically) and how to verify that the volume has mounted successfully. </para> - <section> - <title>Installing the Gluster Native Client</title> - <para>Before you begin installing the Gluster Native Client, you need to verify that the FUSE module is loaded on the client and has access to the required modules as follows: </para> - <orderedlist> - <listitem> - <para>Add the FUSE loadable kernel module (LKM) to the Linux kernel:</para> - <para><command># modprobe fuse</command></para> - </listitem> - <listitem> - <para>Verify that the FUSE module is loaded:</para> - <para><command># dmesg | grep -i fuse </command></para> - <para><command>fuse init (API version 7.13)</command></para> - </listitem> - </orderedlist> - <section id="sect-Administration_Guide-Test_Chapter-GlusterFS_Client-Native-RPM"> - <title>Installing on Red Hat Package Manager (RPM) Distributions </title> - <para>To install Gluster Native Client on RPM distribution-based systems</para> - <orderedlist> - <listitem> - <para>Install required prerequisites on the client using the following command:</para> - <para><command>$ sudo yum -y install openssh-server wget fuse fuse-libs openib libibverbs</command></para> - </listitem> - <listitem> - <para>Ensure that TCP and UDP ports 24007 and 24008 are open on all Gluster servers. Apart from these ports, you need to open one port for each brick starting from port 24009. For example: if you have five bricks, you need to have ports 24009 to 24013 open.</para> - <para>You can use the following chains with iptables:</para> - <para><code>$ sudo iptables -A RH-Firewall-1-INPUT -m state --state NEW -m tcp -p tcp --dport 24007:24008 -j ACCEPT </code></para> - <para><code>$ sudo iptables -A RH-Firewall-1-INPUT -m state --state NEW -m tcp -p tcp --dport 24009:24014 -j ACCEPT</code></para> - <para><note> - <para>If you already have iptable chains, make sure that the above ACCEPT rules precede the DROP rules. This can be achieved by providing a lower rule number than the DROP rule.</para> - </note></para> - </listitem> - <listitem> - <para>Download the latest glusterfs, glusterfs-fuse, and glusterfs-rdma RPM files to each client. The glusterfs package contains the Gluster Native Client. The glusterfs-fuse package contains the FUSE translator required for mounting on client systems and the glusterfs-rdma packages contain OpenFabrics verbs RDMA module for Infiniband.</para> - <para>You can download the software at <ulink url="http://bits.gluster.com/gluster/glusterfs/3.3.0qa30/x86_64/"/>.</para> - </listitem> - <listitem> - <para>Install Gluster Native Client on the client.</para> - <para><command>$ sudo rpm -i glusterfs-3.3.0qa30-1.x86_64.rpm </command></para> - <para><command>$ sudo rpm -i glusterfs-fuse-3.3.0qa30-1.x86_64.rpm </command></para> - <para><command>$ sudo rpm -i glusterfs-rdma-3.3.0qa30-1.x86_64.rpm</command></para> - <para><note> - <para>The RDMA module is only required when using Infiniband.</para> - </note></para> - </listitem> - </orderedlist> - </section> - <section condition="gfs"> - <title>Installing on Debian-based Distributions</title> - <para>To install Gluster Native Client on Debian-based distributions</para> - <orderedlist> - <listitem> - <para>Install OpenSSH Server on each client using the following command:</para> - <para><command>$ sudo apt-get install openssh-server vim wget</command></para> - </listitem> - <listitem> - <para>Download the latest GlusterFS .deb file and checksum to each client.</para> - <para>You can download the software at <ulink url="http://www.gluster.org/download/"/>.</para> - </listitem> - <listitem> - <para>For each .deb file, get the checksum (using the following command) and compare it against the checksum for that file in the md5sum file.</para> - <para> -<command>$ md5sum GlusterFS_DEB_file.deb </command></para> - <para>The md5sum of the packages is available at: <ulink url="http://download.gluster.com/pub/gluster/glusterfs"/></para> - </listitem> - <listitem> - <para>Uninstall GlusterFS v3.1 (or an earlier version) from the client using the following command: -</para> - <para><command>$ sudo dpkg -r glusterfs </command></para> - <para>(Optional) Run <command>$ sudo dpkg -purge glusterfs </command>to purge the configuration files.</para> - </listitem> - <listitem> - <para>Install Gluster Native Client on the client using the following command: -</para> - <para><command>$ sudo dpkg -i GlusterFS_DEB_file </command></para> - <para>For example: -</para> - <para><command>$ sudo dpkg -i glusterfs-3.3.x.deb </command></para> - </listitem> - <listitem> - <para>Ensure that TCP and UDP ports 24007 and 24008 are open on all Gluster servers. Apart from these ports, you need to open one port for each brick starting from port 24009. For example: if you have five bricks, you need to have ports 24009 to 24013 open. -</para> - <para>You can use the following chains with iptables: -</para> - <para><code>$ sudo iptables -A RH-Firewall-1-INPUT -m state --state NEW -m tcp -p tcp --dport 24007:24008 -j ACCEPT </code></para> - <para><code>$ sudo iptables -A RH-Firewall-1-INPUT -m state --state NEW -m tcp -p tcp --dport 24009:24014 -j ACCEPT</code></para> - <para><note> - <para>If you already have iptable chains, make sure that the above ACCEPT rules precede the DROP rules. This can be achieved by providing a lower rule number than the DROP rule.</para> - </note></para> - </listitem> - </orderedlist> - </section> - <section> - <title>Performing a Source Installation</title> - <para>To build and install Gluster Native Client from the source code</para> - <orderedlist> - <listitem> - <para>Create a new directory using the following commands:</para> - <para><command># mkdir glusterfs </command></para> - <para><command># cd glusterfs</command></para> - </listitem> - <listitem> - <para>Download the source code. -</para> - <para>You can download the source at <ulink url="http://www.gluster.org/download/"/>.</para> - </listitem> - <listitem> - <para>Extract the source code using the following command: -</para> - <para><command># tar -xvzf SOURCE-FILE </command></para> - </listitem> - <listitem> - <para>Run the configuration utility using the following command: -</para> - <para><code># ./configure </code></para> - <para><code>GlusterFS configure summary </code></para> - <para><code>================== </code></para> - <para><code>FUSE client : yes </code></para> - <para><code>Infiniband verbs : yes </code></para> - <para><code>epoll IO multiplex : yes </code></para> - <para><code>argp-standalone : no </code></para> - <para><code>fusermount : no </code></para> - <para><code>readline : yes</code></para> - <para>The configuration summary shows the components that will be built with Gluster Native Client.</para> - </listitem> - <listitem> - <para>Build the Gluster Native Client software using the following commands: -</para> - <para><command># make </command></para> - <para><command># make install</command></para> - </listitem> - <listitem> - <para>Verify that the correct version of Gluster Native Client is installed, using the following command: -</para> - <para><command># glusterfs –-version</command></para> - </listitem> - </orderedlist> - </section> - </section> - <section id="sect-Administration_Guide-Test_Chapter-GlusterFS_Client-Mounting_Volumes"> - <title>Mounting Volumes</title> - <para>After installing the Gluster Native Client, you need to mount Gluster volumes to access data. There are two methods you can choose: </para> - <itemizedlist> - <listitem> - <para><xref linkend="sect-Administration_Guide-Test_Chapter-GlusterFS_Client-Manuall"/></para> - </listitem> - <listitem> - <para><xref linkend="sect-Administration_Guide-Test_Chapter-GlusterFS_Client-Automatic"/></para> - </listitem> - </itemizedlist> - <para>After mounting a volume, you can test the mounted volume using the procedure described in <xref linkend="sect-Administration_Guide-Test_Chapter-GlusterFS_Client-Native-Testing"/>. </para> - <para><note> - <para>Server names selected during creation of Volumes should be resolvable in the client machine. You can use appropriate /etc/hosts entries or DNS server to resolve server names to IP addresses. </para> - </note></para> - <section id="sect-Administration_Guide-Test_Chapter-GlusterFS_Client-Manuall"> - <title>Manually Mounting Volumes</title> - <para>To manually mount a Gluster volume </para> - <itemizedlist> - <listitem> - <para>To mount a volume, use the following command: -</para> - <para><command># mount -t glusterfs HOSTNAME-OR-IPADDRESS:/VOLNAME MOUNTDIR</command> -</para> - <para>For example: -</para> - <para><command># mount -t glusterfs server1:/test-volume /mnt/glusterfs</command></para> - <note> - <para>The server specified in the mount command is only used to fetch the gluster configuration volfile describing the volume name. Subsequently, the client will communicate directly with the servers mentioned in the volfile (which might not even include the one used for mount). - -</para> - <para>If you see a usage message like "Usage: mount.glusterfs", mount usually requires you to create a directory to be used as the mount point. Run "mkdir /mnt/glusterfs" before you attempt to run the mount command listed above.</para> - </note> - </listitem> - </itemizedlist> - <para><emphasis role="bold">Mounting Options</emphasis></para> - <para>You can specify the following options when using the <command>mount -t glusterfs</command> command. Note that you need to separate all options with commas. - -</para> - <para>backupvolfile-server=server-name</para> - <para>volfile-max-fetch-attempts=number of attempts</para> - <para>log-level=loglevel -</para> - <para>log-file=logfile -</para> - <para>transport=transport-type -</para> - <para>direct-io-mode=[enable|disable] - -</para> - <para>For example: -</para> - <para><code># mount -t glusterfs -o backupvolfile-server=volfile_server2 --volfile-max-fetch-attempts=2 log-level=WARNING,log-file=/var/log/gluster.log server1:/test-volume /mnt/glusterfs</code></para> - <para>If <option>backupvolfile-server</option> option is added while mounting fuse client, when the first -volfile server fails, then the server specified in <option>backupvolfile-server</option> option is used as volfile server to mount -the client.</para> - <para>In <code>--volfile-max-fetch-attempts=X</code> option, specify the number of attempts to fetch volume files while mounting a volume. This option is useful when you mount a server with multiple IP addresses or when round-robin DNS is configured for the server-name.. </para> - </section> - <section id="sect-Administration_Guide-Test_Chapter-GlusterFS_Client-Automatic" dir="lro"> - <title>Automatically Mounting Volumes</title> - <para>You can configure your system to automatically mount the Gluster volume each time your system starts. </para> - <para>The server specified in the mount command is only used to fetch the gluster configuration volfile describing the volume name. Subsequently, the client will communicate directly with the servers mentioned in the volfile (which might not even include the one used for mount). </para> - <para><emphasis role="bold">To automatically mount a Gluster volume</emphasis></para> - <itemizedlist> - <listitem> - <para>To mount a volume, edit the /etc/fstab file and add the following line: -</para> - <para><command>HOSTNAME-OR-IPADDRESS:/VOLNAME MOUNTDIR glusterfs defaults,_netdev 0 0 </command></para> - <para>For example: -</para> - <para><code>server1:/test-volume /mnt/glusterfs glusterfs defaults,_netdev 0 0</code></para> - </listitem> - </itemizedlist> - <para><emphasis role="bold">Mounting Options </emphasis></para> - <para>You can specify the following options when updating the /etc/fstab file. Note that you need to separate all options with commas. - -</para> - <para>log-level=loglevel -</para> - <para>log-file=logfile -</para> - <para>transport=transport-type -</para> - <para>direct-io-mode=[enable|disable] - -</para> - <para>For example: -</para> - <para><code>HOSTNAME-OR-IPADDRESS:/VOLNAME MOUNTDIR glusterfs defaults,_netdev,log-level=WARNING,log-file=/var/log/gluster.log 0 0 </code></para> - </section> - <section id="sect-Administration_Guide-Test_Chapter-GlusterFS_Client-Native-Testing"> - <title>Testing Mounted Volumes</title> - <para>To test mounted volumes</para> - <itemizedlist> - <listitem> - <para>Use the following command: -</para> - <para><command># mount </command></para> - <para>If the gluster volume was successfully mounted, the output of the mount command on the client will be similar to this example: - -</para> - <para><code>server1:/test-volume on /mnt/glusterfs type fuse.glusterfs (rw,allow_other,default_permissions,max_read=131072</code></para> - </listitem> - </itemizedlist> - <itemizedlist> - <listitem> - <para>Use the following command: -</para> - <para><command># df</command> -</para> - <para>The output of df command on the client will display the aggregated storage space from all the bricks in a volume similar to this example: -</para> - <para><code># df -h /mnt/glusterfs Filesystem Size Used Avail Use% Mounted on server1:/test-volume 28T 22T 5.4T 82% /mnt/glusterfs</code></para> - </listitem> - <listitem> - <para>Change to the directory and list the contents by entering the following: -</para> - <para><command># cd MOUNTDIR </command></para> - <para><command># ls</command></para> - </listitem> - <listitem> - <para>For example,</para> - <para><code># cd /mnt/glusterfs </code></para> - <para><code># ls</code></para> - </listitem> - </itemizedlist> - </section> - </section> - </section> - <section id="sect-Administration_Guide-GlusterFS_Client-NFS"> - <title>NFS</title> - <para>You can use NFS v3 to access to gluster volumes. Extensive testing has be done on GNU/Linux clients and NFS implementation in other operating system, such as FreeBSD, and Mac OS X, as well as Windows 7 (Professional and Up), Windows Server 2003, and others, may work with gluster NFS server implementation. </para> - <para>GlusterFS now includes network lock manager (NLM) v4. NLM enables applications on NFSv3 clients to do record locking on files on NFS server. It is started automatically whenever the NFS server is run.</para> - <para condition="gfs">You must install nfs-common package on both servers and clients (only for Debian-based) distribution.</para> - <para>This section describes how to use NFS to mount Gluster volumes (both manually and automatically) and how to verify that the volume has been mounted successfully. </para> - <section> - <title>Using NFS to Mount Volumes</title> - <para>You can use either of the following methods to mount Gluster volumes: </para> - <para><itemizedlist> - <listitem> - <para><xref linkend="sect-Administration_Guide-GlusterFS_Client-NFS-Manual"/></para> - </listitem> - <listitem> - <para><xref linkend="sect-Administration_Guide-GlusterFS_Client-NFS-Automatic"/></para> - </listitem> - </itemizedlist></para> - <para condition="gfs"><emphasis role="bold">Prerequisite</emphasis>: Install nfs-common package on both servers and clients (only for Debian-based distribution), using the following command: </para> - <para condition="gfs"><command>$ sudo aptitude install nfs-common </command></para> - <para>After mounting a volume, you can test the mounted volume using the procedure described in <xref linkend="sect-Administration_Guide-GlusterFS_Client-NFS-Testing"/>. </para> - <section id="sect-Administration_Guide-GlusterFS_Client-NFS-Manual"> - <title>Manually Mounting Volumes Using NFS </title> - <para>To manually mount a Gluster volume using NFS </para> - <itemizedlist> - <listitem> - <para>To mount a volume, use the following command: -</para> - <para><command># mount -t nfs -o vers=3 HOSTNAME-OR-IPADDRESS:/VOLNAME MOUNTDIR</command> -</para> - <para>For example:</para> - <para><command># mount -t nfs -o vers=3 server1:/test-volume /mnt/glusterfs</command></para> - <para><note> - <para> Gluster NFS server does not support UDP. If the NFS client you are using defaults to connecting using UDP, the following message appears: -</para> - <para><code>requested NFS version or transport protocol is not supported</code>. </para> - </note></para> - <para><emphasis role="bold">To connect using TCP</emphasis></para> - </listitem> - <listitem> - <para>Add the following option to the mount command: -</para> - <para><command>-o mountproto=tcp </command></para> - <para>For example: -</para> - <para><command># mount -o mountproto=tcp -t nfs server1:/test-volume /mnt/glusterfs</command></para> - </listitem> - </itemizedlist> - <para><emphasis role="bold">To mount Gluster NFS server from a Solaris client </emphasis></para> - <itemizedlist> - <listitem> - <para>Use the following command: -</para> - <para><command># mount -o proto=tcp,vers=3 nfs://HOSTNAME-OR-IPADDRESS:38467/VOLNAME MOUNTDIR</command></para> - <para> -For example:</para> - <para><command> # mount -o proto=tcp,vers=3 nfs://server1:38467/test-volume /mnt/glusterfs</command></para> - </listitem> - </itemizedlist> - </section> - <section id="sect-Administration_Guide-GlusterFS_Client-NFS-Automatic"> - <title>Automatically Mounting Volumes Using NFS</title> - <para>You can configure your system to automatically mount Gluster volumes using NFS each time the system starts.</para> - <para><emphasis role="bold">To automatically mount a Gluster volume using NFS </emphasis></para> - <itemizedlist> - <listitem> - <para>To mount a volume, edit the /etc/fstab file and add the following line:</para> - <para><command>HOSTNAME-OR-IPADDRESS:/VOLNAME MOUNTDIR nfs defaults,_netdev,vers=3 0 0</command></para> - <para>For example,</para> - <para><command>server1:/test-volume /mnt/glusterfs nfs defaults,_netdev,vers=3 0 0</command></para> - <note> - <para>Gluster NFS server does not support UDP. If the NFS client you are using defaults to connecting using UDP, the following message appears: </para> - <para><command>requested NFS version or transport protocol is not supported.</command></para> - </note> - <para/> - <para>To connect using TCP </para> - </listitem> - <listitem> - <para>Add the following entry in /etc/fstab file :</para> - <para><command>HOSTNAME-OR-IPADDRESS:/VOLNAME MOUNTDIR nfs defaults,_netdev,mountproto=tcp 0 0</command></para> - <para>For example,</para> - <para><command>server1:/test-volume /mnt/glusterfs nfs defaults,_netdev,mountproto=tcp 0 0</command></para> - </listitem> - </itemizedlist> - <para><emphasis role="bold">To automount NFS mounts</emphasis></para> - <para>Gluster supports *nix standard method of automounting NFS mounts. Update the /etc/auto.master and /etc/auto.misc and restart the autofs service. After that, whenever a user or process attempts to access the directory it will be mounted in the background. </para> - </section> - <section id="sect-Administration_Guide-GlusterFS_Client-NFS-Testing"> - <title>Testing Volumes Mounted Using NFS</title> - <para>You can confirm that Gluster directories are mounting successfully. </para> - <para><emphasis role="bold">To test mounted volumes</emphasis></para> - <itemizedlist> - <listitem> - <para>Use the mount command by entering the following:</para> - <para><command># mount</command></para> - <para>For example, the output of the mount command on the client will display an entry like the following:</para> - <para><command>server1:/test-volume on /mnt/glusterfs type nfs (rw,vers=3,addr=server1)</command></para> - </listitem> - </itemizedlist> - <itemizedlist> - <listitem> - <para>Use the df command by entering the following:</para> - <para><command># df</command></para> - <para>For example, the output of df command on the client will display the aggregated storage space from all the bricks in a volume.</para> - <para><screen># df -h /mnt/glusterfs -Filesystem Size Used Avail Use% Mounted on -server1:/test-volume 28T 22T 5.4T 82% /mnt/glusterfs</screen></para> - </listitem> - <listitem> - <para>Change to the directory and list the contents by entering the following:</para> - <para><command># cd MOUNTDIR</command></para> - <para><command># ls</command></para> - <para>For example,</para> - <para><command> - <command># cd /mnt/glusterfs</command> - </command></para> - <para><command># ls</command></para> - </listitem> - </itemizedlist> - </section> - </section> - </section> - <section id="sect-Administration_Guide-GlusterFS_Client-CIFS"> - <title>CIFS</title> - <para>You can use CIFS to access to volumes when using Microsoft Windows as well as SAMBA clients. For this access method, Samba packages need to be present on the client side. You can export glusterfs mount point as the samba export, and then mount it using CIFS protocol.</para> - <para>This section describes how to mount CIFS shares on Microsoft Windows-based clients (both manually and automatically) and how to verify that the volume has mounted successfully.</para> - <para><note> - <para> CIFS access using the Mac OS X Finder is not supported, however, you can use the Mac OS X command line to access Gluster volumes using CIFS.</para> - </note></para> - <section> - <title>Using CIFS to Mount Volumes</title> - <para>You can use either of the following methods to mount Gluster volumes: </para> - <itemizedlist> - <listitem> - <para><xref linkend="sect-Administration_Guide-GlusterFS_Client-CIFS-Manual"/></para> - </listitem> - <listitem> - <para><xref linkend="sect-Administration_Guide-GlusterFS_Client-CIFS-Automatic"/></para> - </listitem> - </itemizedlist> - <para>After mounting a volume, you can test the mounted volume using the procedure described in <xref linkend="sect-Administration_Guide-GlusterFS_Client-CIFS-Testing"/>.</para> - <para>You can also use Samba for exporting Gluster Volumes through CIFS protocol.</para> - <section> - <title>Exporting Gluster Volumes Through Samba</title> - <para>We recommend you to use Samba for exporting Gluster volumes through the CIFS protocol. </para> - <para><emphasis role="bold">To export volumes through CIFS protocol </emphasis></para> - <orderedlist> - <listitem> - <para>Mount a Gluster volume. For more information on mounting volumes, see <xref linkend="sect-Administration_Guide-Test_Chapter-GlusterFS_Client-Mounting_Volumes"/>.</para> - </listitem> - <listitem> - <para>Setup Samba configuration to export the mount point of the Gluster volume.</para> - <para>For example, if a Gluster volume is mounted on /mnt/gluster, you must edit smb.conf file to enable exporting this through CIFS. Open smb.conf file in an editor and add the following lines for a simple configuration:</para> - <para>[glustertest] - </para> - <para> comment = For testing a Gluster volume exported through CIFS - </para> - <para> path = /mnt/glusterfs - </para> - <para> read only = no - </para> - <para> guest ok = yes</para> - </listitem> - </orderedlist> - <para>Save the changes and start the smb service using your systems init scripts (/etc/init.d/smb [re]start).</para> - <para><note> - <para>To be able mount from any server in the trusted storage pool, you must repeat these steps on each Gluster node. For more advanced configurations, see Samba documentation. </para> - </note></para> - </section> - <section id="sect-Administration_Guide-GlusterFS_Client-CIFS-Manual"> - <title>Manually Mounting Volumes Using CIFS </title> - <para>You can manually mount Gluster volumes using CIFS on Microsoft Windows-based client machines. </para> - <para><emphasis role="bold">To manually mount a Gluster volume using CIFS </emphasis></para> - <orderedlist> - <listitem> - <para>Using Windows Explorer, choose <emphasis role="bold">Tools > Map Network Drive…</emphasis> from the menu. The <emphasis role="bold">Map Network Drive </emphasis>window appears. </para> - </listitem> - <listitem> - <para>Choose the drive letter using the <emphasis role="bold">Drive</emphasis> drop-down list. </para> - </listitem> - <listitem> - <para>Click <emphasis role="bold">Browse</emphasis>, select the volume to map to the network drive, and click <emphasis role="bold">OK</emphasis>. </para> - </listitem> - <listitem> - <para>Click <emphasis role="bold">Finish.</emphasis></para> - </listitem> - </orderedlist> - <para>The network drive (mapped to the volume) appears in the Computer window.</para> - <para><emphasis role="bold">Alternatively, to manually mount a Gluster volume using CIFS.</emphasis></para> - <itemizedlist> - <listitem> - <para>Click <emphasis role="bold">Start > Run</emphasis> and enter the following:</para> - <para><command> - <code>\\SERVERNAME\VOLNAME</code> - </command></para> - <para>For example:</para> - <para><command> - <code>\\server1\test-volume</code> - </command></para> - </listitem> - </itemizedlist> - </section> - <section id="sect-Administration_Guide-GlusterFS_Client-CIFS-Automatic"> - <title>Automatically Mounting Volumes Using CIFS</title> - <para>You can configure your system to automatically mount Gluster volumes using CIFS on Microsoft Windows-based clients each time the system starts.</para> - <para><emphasis role="bold">To automatically mount a Gluster volume using CIFS</emphasis></para> - <para>The network drive (mapped to the volume) appears in the Computer window and is reconnected each time the system starts.</para> - <orderedlist> - <listitem> - <para>Using Windows Explorer, choose <emphasis role="bold">Tools > Map Network Drive…</emphasis> from the menu. The <emphasis role="bold">Map Network Drive </emphasis>window appears. </para> - </listitem> - <listitem> - <para>Choose the drive letter using the <emphasis role="bold">Drive</emphasis> drop-down list. </para> - </listitem> - <listitem> - <para>Click <emphasis role="bold">Browse</emphasis>, select the volume to map to the network drive, and click <emphasis role="bold">OK</emphasis>. </para> - </listitem> - <listitem> - <para>Click the <emphasis role="bold">Reconnect</emphasis> at logon checkbox.</para> - </listitem> - <listitem> - <para>Click <emphasis role="bold">Finish.</emphasis></para> - </listitem> - </orderedlist> - </section> - <section id="sect-Administration_Guide-GlusterFS_Client-CIFS-Testing"> - <title>Testing Volumes Mounted Using CIFS</title> - <para>You can confirm that Gluster directories are mounting successfully by navigating to the directory using Windows Explorer. </para> - </section> - </section> - </section> -</chapter> diff --git a/doc/admin-guide/en-US/admin_start_stop_daemon.xml b/doc/admin-guide/en-US/admin_start_stop_daemon.xml deleted file mode 100644 index bdab0b8b608..00000000000 --- a/doc/admin-guide/en-US/admin_start_stop_daemon.xml +++ /dev/null @@ -1,56 +0,0 @@ -<?xml version='1.0' encoding='UTF-8'?> -<!-- This document was created with Syntext Serna Free. --><!DOCTYPE chapter PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN" "http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd" [ -<!ENTITY % BOOK_ENTITIES SYSTEM "Administration_Guide.ent"> -%BOOK_ENTITIES; -]> -<chapter id="chap-Administration_Guide-Start_Stop_Daemon"> - <title id="chap-Administration_Guide-Stop_Start_Daemon">Managing the glusterd Service</title> - <para>After installing GlusterFS, you must start glusterd service. The glusterd service serves as the Gluster elastic volume manager, overseeing glusterfs processes, and co-ordinating dynamic volume operations, such as adding and removing volumes across multiple storage servers non-disruptively.</para> - <para>This section describes how to start the glusterd service in the following ways: </para> - <itemizedlist> - <listitem> - <para><xref linkend="sect-Administration_Guide-Start_Stop_Daemon-Manually"/></para> - </listitem> - <listitem> - <para><xref linkend="sect-Administration_Guide-Start_Stop_Daemon-Automatically"/></para> - </listitem> - </itemizedlist> - <note> - <para>You must start glusterd on all GlusterFS servers.</para> - </note> - <section id="sect-Administration_Guide-Start_Stop_Daemon-Manually"> - <title>Starting and Stopping glusterd Manually</title> - <para>This section describes how to start and stop glusterd manually</para> - <itemizedlist> - <listitem> - <para>To start glusterd manually, enter the following command:</para> - <para><command># /etc/init.d/glusterd start </command></para> - </listitem> - <listitem> - <para>To stop glusterd manually, enter the following command: </para> - <para><command># /etc/init.d/glusterd stop</command></para> - </listitem> - </itemizedlist> - </section> - <section id="sect-Administration_Guide-Start_Stop_Daemon-Automatically"> - <title>Starting glusterd Automatically</title> - <para condition="gfs">This section describes how to configure the system to automatically start the glusterd service every time the system boots. </para> - <para condition="appliance">To automatically start the glusterd service every time the system boots, enter the following from the command line: </para> - <para condition="appliance"><command># chkconfig glusterd on </command></para> - <section condition="gfs"> - <title condition="gfs">Red Hat-based Systems</title> - <para>To configure Red Hat-based systems to automatically start the glusterd service every time the system boots, enter the following from the command line: </para> - <para><command># chkconfig glusterd on </command></para> - </section> - <section condition="gfs"> - <title condition="gfs">Debian-based Systems</title> - <para>To configure Debian-based systems to automatically start the glusterd service every time the system boots, enter the following from the command line:</para> - <para><command># update-rc.d glusterd defaults</command></para> - </section> - <section condition="gfs"> - <title condition="gfs">Systems Other than Red Hat and Debain</title> - <para>To configure systems other than Red Hat or Debian to automatically start the glusterd service every time the system boots, enter the following entry to the<emphasis role="italic"> /etc/rc.local</emphasis> file: </para> - <para><command># echo "glusterd" >> /etc/rc.local </command></para> - </section> - </section> -</chapter> diff --git a/doc/admin-guide/en-US/admin_storage_pools.xml b/doc/admin-guide/en-US/admin_storage_pools.xml deleted file mode 100644 index 87b6320bd4b..00000000000 --- a/doc/admin-guide/en-US/admin_storage_pools.xml +++ /dev/null @@ -1,57 +0,0 @@ -<?xml version='1.0' encoding='UTF-8'?> -<!-- This document was created with Syntext Serna Free. --><!DOCTYPE chapter PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN" "docbookV4.5/docbookx.dtd" []> -<chapter id="chap-Administration_Guide-Storage-pool"> - <title>Setting up Trusted Storage Pools</title> - <para>Before you can configure a GlusterFS volume, you must create a trusted storage pool consisting of the storage servers that provides bricks to a volume. </para> - <para>A storage pool is a trusted network of storage servers. When you start the first server, the storage pool consists of that server alone. To add additional storage servers to the storage pool, you can use the probe command from a storage server that is already trusted. </para> - <para><note> - <para>Do not self-probe the first server/localhost.</para> - </note></para> - <para>The GlusterFS service must be running on all storage servers that you want to add to the storage pool. See <xref linkend="chap-Administration_Guide-Start_Stop_Daemon"/> for more information.</para> - <section id="sect-Administration_Guide-Storage_Pools-Adding_Servers"> - <title>Adding Servers to Trusted Storage Pool</title> - <para>To create a trusted storage pool, add servers to the trusted storage pool</para> - <orderedlist> - <listitem> - <para>The hostnames used to create the storage pool must be resolvable by DNS.</para> - <para>To add a server to the storage pool:</para> - <para><command># gluster peer probe <replaceable>server</replaceable></command></para> - <para>For example, to create a trusted storage pool of four servers, add three servers to the storage pool from server1:</para> - <para><programlisting># gluster peer probe server2 -Probe successful - -# gluster peer probe server3 -Probe successful - -# gluster peer probe server4 -Probe successful -</programlisting></para> - </listitem> - <listitem> - <para>Verify the peer status from the first server using the following commands:</para> - <para><programlisting># gluster peer status -Number of Peers: 3 - -Hostname: server2 -Uuid: 5e987bda-16dd-43c2-835b-08b7d55e94e5 -State: Peer in Cluster (Connected) - -Hostname: server3 -Uuid: 1e0ca3aa-9ef7-4f66-8f15-cbc348f29ff7 -State: Peer in Cluster (Connected) - -Hostname: server4 -Uuid: 3e0caba-9df7-4f66-8e5d-cbc348f29ff7 -State: Peer in Cluster (Connected)</programlisting></para> - </listitem> - </orderedlist> - </section> - <section> - <title>Removing Servers from the Trusted Storage Pool</title> - <para>To remove a server from the storage pool:</para> - <para><command># gluster peer detach<replaceable> server</replaceable></command></para> - <para> For example, to remove server4 from the trusted storage pool:</para> - <para><programlisting># gluster peer detach server4 -Detach successful</programlisting></para> - </section> -</chapter> diff --git a/doc/admin-guide/en-US/admin_troubleshooting.xml b/doc/admin-guide/en-US/admin_troubleshooting.xml deleted file mode 100644 index dff182c5f1e..00000000000 --- a/doc/admin-guide/en-US/admin_troubleshooting.xml +++ /dev/null @@ -1,509 +0,0 @@ -<?xml version='1.0' encoding='UTF-8'?> -<!-- This document was created with Syntext Serna Free. --><!DOCTYPE chapter PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN" "docbookV4.5/docbookx.dtd" []> -<chapter id="chap-Administration_Guide-Troubleshooting"> - <title>Troubleshooting GlusterFS </title> - <para>This section describes how to manage GlusterFS logs and most common troubleshooting scenarios -related to GlusterFS. -</para> - <section> - <title>Managing GlusterFS Logs </title> - <para>This section describes how to manage GlusterFS logs by performing the following operation: - -</para> - <itemizedlist> - <listitem> - <para>Rotating Logs -</para> - </listitem> - </itemizedlist> - <section> - <title>Rotating Logs </title> - <para>Administrators can rotate the log file in a volume, as needed. -</para> - <para><emphasis role="bold">To rotate a log file </emphasis></para> - <itemizedlist> - <listitem> - <para>Rotate the log file using the following command: -</para> - <para><command># gluster volume log rotate <replaceable>VOLNAME</replaceable></command></para> - <para>For example, to rotate the log file on test-volume: -</para> - <programlisting># gluster volume log rotate test-volume -log rotate successful -</programlisting> - <note> - <para>When a log file is rotated, the contents of the current log file are moved to log-file- -name.epoch-time-stamp. -</para> - </note> - </listitem> - </itemizedlist> - </section> - </section> - <section> - <title>Troubleshooting Geo-replication </title> - <para>This section describes the most common troubleshooting scenarios related to GlusterFS Geo-replication. -</para> - <section> - <title>Locating Log Files </title> - <para>For every Geo-replication session, the following three log files are associated to it (four, if the slave is a -gluster volume): -</para> - <itemizedlist> - <listitem> - <para>Master-log-file - log file for the process which monitors the Master volume -</para> - </listitem> - <listitem> - <para>Slave-log-file - log file for process which initiates the changes in slave -</para> - </listitem> - <listitem> - <para>Master-gluster-log-file - log file for the maintenance mount point that Geo-replication module -uses to monitor the master volume -</para> - </listitem> - <listitem> - <para>Slave-gluster-log-file - is the slave's counterpart of it -</para> - </listitem> - </itemizedlist> - <para><emphasis role="bold">Master Log File</emphasis> -</para> - <para>To get the Master-log-file for geo-replication, use the following command: -</para> - <para><command>gluster volume geo-replication <code>MASTER SLAVE</code> config log-file</command> -</para> - <para>For example: -</para> - <para><command># gluster volume geo-replication Volume1 example.com:/data/remote_dir config log-file </command></para> - <para><emphasis role="bold">Slave Log File </emphasis></para> - <para>To get the log file for Geo-replication on slave (glusterd must be running on slave machine), use the -following commands: -</para> - <orderedlist> - <listitem> - <para>On master, run the following command: -</para> - <para><command># gluster volume geo-replication Volume1 example.com:/data/remote_dir config session-owner 5f6e5200-756f-11e0-a1f0-0800200c9a66 </command></para> - <para>Displays the session owner details. -</para> - </listitem> - <listitem> - <para>On slave, run the following command: -</para> - <para><command># gluster volume geo-replication /data/remote_dir config log-file /var/log/gluster/${session-owner}:remote-mirror.log </command></para> - </listitem> - <listitem> - <para>Replace the session owner details (output of Step 1) to the output of the Step 2 to get the -location of the log file. -</para> - <para><command>/var/log/gluster/5f6e5200-756f-11e0-a1f0-0800200c9a66:remote-mirror.log</command> -</para> - </listitem> - </orderedlist> - </section> - <section> - <title>Rotating Geo-replication Logs</title> - <para>Administrators can rotate the log file of a particular master-slave session, as needed. -When you run geo-replication's <command> log-rotate</command> command, the log file -is backed up with the current timestamp suffixed to the file -name and signal is sent to gsyncd to start logging to a new -log file.</para> - <para><emphasis role="bold">To rotate a geo-replication log file </emphasis></para> - <itemizedlist> - <listitem> - <para>Rotate log file for a particular master-slave session using the following command: -</para> - <para><command># gluster volume geo-replication <replaceable>master slave</replaceable> log-rotate</command> -</para> - <para>For example, to rotate the log file of master <filename>Volume1</filename> and slave <filename>example.com:/data/remote_dir</filename> -: -</para> - <programlisting># gluster volume geo-replication Volume1 example.com:/data/remote_dir log rotate -log rotate successful</programlisting> - </listitem> - <listitem> - <para>Rotate log file for all sessions for a master volume using the following command: -</para> - <para><command># gluster volume geo-replication <replaceable>master</replaceable> log-rotate</command> -</para> - <para>For example, to rotate the log file of master <filename>Volume1</filename>: -</para> - <programlisting># gluster volume geo-replication Volume1 log rotate -log rotate successful</programlisting> - </listitem> - <listitem> - <para>Rotate log file for all sessions using the following command: -</para> - <para><command># gluster volume geo-replication log-rotate</command> -</para> - <para>For example, to rotate the log file for all sessions:</para> - <programlisting># gluster volume geo-replication log rotate -log rotate successful</programlisting> - </listitem> - </itemizedlist> - </section> - <section> - <title>Synchronization is not complete </title> - <para><emphasis role="bold">Description</emphasis>: GlusterFS Geo-replication did not synchronize the data completely but still the geo- -replication status displayed is OK. -</para> - <para><emphasis role="bold">Solution</emphasis>: You can enforce a full sync of the data by erasing the index and restarting GlusterFS Geo- -replication. After restarting, GlusterFS Geo-replication begins synchronizing all the data. All files are compared using checksum, which can be a lengthy and high resource utilization operation on large -data sets. If the error situation persists, contact Red Hat Support. -</para> - <para>For more information about erasing index, see <xref linkend="sect-Administration_Guide-Managing_Volumes-Tuning"/>. -</para> - </section> - <section> - <title>Issues in Data Synchronization </title> - <para><emphasis role="bold">Description</emphasis>: Geo-replication display status as OK, but the files do not get synced, only -directories and symlink gets synced with the following error message in the log: -</para> - <para><errortext>[2011-05-02 13:42:13.467644] E [master:288:regjob] GMaster: failed to sync ./some_file` </errortext></para> - <para><emphasis role="bold">Solution</emphasis>: Geo-replication invokes rsync v3.0.0 or higher on the host and the remote machine. You must verify if -you have installed the required version. -</para> - </section> - <section> - <title>Geo-replication status displays Faulty very often </title> - <para><emphasis role="bold">Description</emphasis>: Geo-replication displays status as faulty very often with a backtrace similar to -the following: -</para> - <para><errortext>2011-04-28 14:06:18.378859] E [syncdutils:131:log_raise_exception] <top>: FAIL: Traceback (most recent call last): File "/usr/local/libexec/glusterfs/python/syncdaemon/syncdutils.py", line 152, in twraptf(*aa) File "/usr/local/libexec/glusterfs/python/syncdaemon/repce.py", line 118, in listen rid, exc, res = recv(self.inf) File "/usr/local/libexec/glusterfs/python/syncdaemon/repce.py", line 42, in recv return pickle.load(inf) EOFError </errortext></para> - <para><emphasis role="bold">Solution</emphasis>: This error indicates that the RPC communication between the master gsyncd module and slave -gsyncd module is broken and this can happen for various reasons. Check if it satisfies all the following -pre-requisites: -</para> - <itemizedlist> - <listitem> - <para>Password-less SSH is set up properly between the host and the remote machine. -</para> - </listitem> - <listitem> - <para>If FUSE is installed in the machine, because geo-replication module mounts the GlusterFS volume -using FUSE to sync data. -</para> - </listitem> - <listitem> - <para>If the <emphasis role="bold">Slave</emphasis> is a volume, check if that volume is started. -</para> - </listitem> - <listitem> - <para>If the Slave is a plain directory, verify if the directory has been created already with the -required permissions. -</para> - </listitem> - <listitem> - <para>If GlusterFS 3.2 or higher is not installed in the default location (in Master) and has been prefixed to be -installed in a custom location, configure the <command>gluster-command</command> for it to point to the exact -location. -</para> - </listitem> - <listitem> - <para>If GlusterFS 3.2 or higher is not installed in the default location (in slave) and has been prefixed to be -installed in a custom location, configure the <command>remote-gsyncd-command</command> for it to point to the -exact place where gsyncd is located. -</para> - </listitem> - </itemizedlist> - </section> - <section> - <title>Intermediate Master goes to Faulty State </title> - <para><emphasis role="bold">Description</emphasis>: In a cascading set-up, the intermediate master goes to faulty state with the following -log: -</para> - <para><errortext>raise RuntimeError ("aborting on uuid change from %s to %s" % \ RuntimeError: aborting on uuid change from af07e07c-427f-4586-ab9f- 4bf7d299be81 to de6b5040-8f4e-4575-8831-c4f55bd41154 </errortext></para> - <para><emphasis role="bold">Solution</emphasis>: In a cascading set-up the Intermediate master is loyal to the original primary master. The -above log means that the geo-replication module has detected change in primary master. -If this is the desired behavior, delete the config option volume-id in the session initiated from the -intermediate master. -</para> - </section> - </section> - <section> - <title>Troubleshooting POSIX ACLs </title> - <para>This section describes the most common troubleshooting issues related to POSIX ACLs. -</para> - <section> - <title>setfacl command fails with “setfacl: <file or directory name>: Operation not supported” error </title> - <para>You may face this error when the backend file systems in one of the servers is not mounted with -the "-o acl" option. The same can be confirmed by viewing the following error message in the log file -of the server "Posix access control list is not supported". -</para> - <para><emphasis role="bold">Solution</emphasis>: Remount the backend file system with "-o acl" option. For more information, see <xref linkend="sect-Administration_Guide-ACLs-Activating_ACLs-Server"/>. -</para> - </section> - </section> - <section> - <title>Troubleshooting Hadoop Compatible Storage </title> - <para>This section describes the most common troubleshooting issues related to Hadoop Compatible -Storage. - - </para> - <section id="sect-Administration_Guide-Troubleshooting-Test_Section_1"> - <title>Time Sync</title> - <para>Running MapReduce job may throw exceptions if the time is out-of-sync on the hosts in the cluster. - - </para> - <para><emphasis role="bold">Solution</emphasis>: Sync the time on all hosts using ntpd program. -</para> - </section> - </section> - <section> - <title>Troubleshooting NFS </title> - <para>This section describes the most common troubleshooting issues related to NFS . -</para> - <section> - <title>mount command on NFS client fails with “RPC Error: Program not registered” </title> - <para>Start portmap or rpcbind service on the NFS server. -</para> - <para>This error is encountered when the server has not started correctly. -</para> - <para>On most Linux distributions this is fixed by starting portmap: -</para> - <para><command>$ /etc/init.d/portmap start</command> -</para> - <para>On some distributions where portmap has been replaced by rpcbind, the following command is -required: -</para> - <para><command>$ /etc/init.d/rpcbind start </command></para> - <para>After starting portmap or rpcbind, gluster NFS server needs to be restarted. -</para> - </section> - <section> - <title>NFS server start-up fails with “Port is already in use” error in the log file." </title> - <para>Another Gluster NFS server is running on the same machine. -</para> - <para>This error can arise in case there is already a Gluster NFS server running on the same machine. -This situation can be confirmed from the log file, if the following error lines exist: -</para> - <para><screen>[2010-05-26 23:40:49] E [rpc-socket.c:126:rpcsvc_socket_listen] rpc-socket: binding socket failed:Address already in use -[2010-05-26 23:40:49] E [rpc-socket.c:129:rpcsvc_socket_listen] rpc-socket: Port is already in use -[2010-05-26 23:40:49] E [rpcsvc.c:2636:rpcsvc_stage_program_register] rpc-service: could not create listening connection -[2010-05-26 23:40:49] E [rpcsvc.c:2675:rpcsvc_program_register] rpc-service: stage registration of program failed -[2010-05-26 23:40:49] E [rpcsvc.c:2695:rpcsvc_program_register] rpc-service: Program registration failed: MOUNT3, Num: 100005, Ver: 3, Port: 38465 -[2010-05-26 23:40:49] E [nfs.c:125:nfs_init_versions] nfs: Program init failed -[2010-05-26 23:40:49] C [nfs.c:531:notify] nfs: Failed to initialize protocols</screen></para> - <para>To resolve this error one of the Gluster NFS servers will have to be shutdown. At this time, -Gluster NFS server does not support running multiple NFS servers on the same machine. -</para> - </section> - <section> - <title>mount command fails with “rpc.statd” related error message </title> - <para>If the mount command fails with the following error message: -</para> - <para><errortext>mount.nfs: rpc.statd is not running but is required for remote locking. mount.nfs: Either use '-o nolock' to keep locks local, or start statd. </errortext></para> - <para><errortext>Start rpc.statd </errortext></para> - <para>For NFS clients to mount the NFS server, rpc.statd service must be running on the clients. </para> - <para>Start -rpc.statd service by running the following command: -</para> - <para><command>$ rpc.statd </command></para> - </section> - <section> - <title>mount command takes too long to finish. </title> - <para>Start rpcbind service on the NFS client. -</para> - <para>The problem is that the rpcbind or portmap service is not running on the NFS client. The -resolution for this is to start either of these services by running the following command: -</para> - <para><command>$ /etc/init.d/portmap start</command> -</para> - <para>On some distributions where portmap has been replaced by rpcbind, the following command is -required: -</para> - <para><command>$ /etc/init.d/rpcbind start</command></para> - </section> - <section> - <title>NFS server glusterfsd starts but initialization fails with “nfsrpc- service: portmap registration of program failed” error message in the log. </title> - <para>NFS start-up can succeed but the initialization of the NFS service can still fail preventing clients -from accessing the mount points. Such a situation can be confirmed from the following error -messages in the log file: -</para> - <para><screen>[2010-05-26 23:33:47] E [rpcsvc.c:2598:rpcsvc_program_register_portmap] rpc-service: Could notregister with portmap -[2010-05-26 23:33:47] E [rpcsvc.c:2682:rpcsvc_program_register] rpc-service: portmap registration of program failed -[2010-05-26 23:33:47] E [rpcsvc.c:2695:rpcsvc_program_register] rpc-service: Program registration failed: MOUNT3, Num: 100005, Ver: 3, Port: 38465 -[2010-05-26 23:33:47] E [nfs.c:125:nfs_init_versions] nfs: Program init failed -[2010-05-26 23:33:47] C [nfs.c:531:notify] nfs: Failed to initialize protocols -[2010-05-26 23:33:49] E [rpcsvc.c:2614:rpcsvc_program_unregister_portmap] rpc-service: Could not unregister with portmap -[2010-05-26 23:33:49] E [rpcsvc.c:2731:rpcsvc_program_unregister] rpc-service: portmap unregistration of program failed -[2010-05-26 23:33:49] E [rpcsvc.c:2744:rpcsvc_program_unregister] rpc-service: Program unregistration failed: MOUNT3, Num: 100005, Ver: 3, Port: 38465</screen></para> - <orderedlist> - <listitem> - <para>Start portmap or rpcbind service on the NFS server. -</para> - <para>On most Linux distributions, portmap can be started using the following command: -</para> - <para><command>$ /etc/init.d/portmap start </command></para> - <para>On some distributions where portmap has been replaced by rpcbind, run the following command: -</para> - <para><command>$ /etc/init.d/rpcbind start </command></para> - <para>After starting portmap or rpcbind, gluster NFS server needs to be restarted. -</para> - </listitem> - <listitem> - <para>Stop another NFS server running on the same machine. -</para> - <para>Such an error is also seen when there is another NFS server running on the same machine but it is -not the Gluster NFS server. On Linux systems, this could be the kernel NFS server. Resolution -involves stopping the other NFS server or not running the Gluster NFS server on the machine. -Before stopping the kernel NFS server, ensure that no critical service depends on access to that -NFS server's exports. -</para> - <para>On Linux, kernel NFS servers can be stopped by using either of the following commands -depending on the distribution in use: -</para> - <para><command>$ /etc/init.d/nfs-kernel-server stop</command> -</para> - <para><command>$ /etc/init.d/nfs stop</command></para> - </listitem> - <listitem> - <para>Restart Gluster NFS server. -</para> - </listitem> - </orderedlist> - </section> - <section> - <title>mount command fails with NFS server failed error. </title> - <para>mount command fails with following error -</para> - <para><emphasis role="italic">mount: mount to NFS server '10.1.10.11' failed: timed out (retrying).</emphasis></para> - <para>Perform one of the following to resolve this issue: -</para> - <orderedlist> - <listitem> - <para>Disable name lookup requests from NFS server to a DNS server. -</para> - <para>The NFS server attempts to authenticate NFS clients by performing a reverse DNS lookup to -match hostnames in the volume file with the client IP addresses. There can be a situation where -the NFS server either is not able to connect to the DNS server or the DNS server is taking too long -to responsd to DNS request. These delays can result in delayed replies from the NFS server to the -NFS client resulting in the timeout error seen above. -</para> - <para>NFS server provides a work-around that disables DNS requests, instead relying only on the client -IP addresses for authentication. The following option can be added for successful mounting in -such situations: -</para> - <para><command>option rpc-auth.addr.namelookup off </command></para> - <para><note> - <para>Note: Remember that disabling the NFS server forces authentication of clients to use only IP -addresses and if the authentication rules in the volume file use hostnames, those authentication -rules will fail and disallow mounting for those clients. -</para> - </note></para> - <para>or</para> - </listitem> - <listitem> - <para>NFS version used by the NFS client is other than version 3. -</para> - <para>Gluster NFS server supports version 3 of NFS protocol. In recent Linux kernels, the default NFS -version has been changed from 3 to 4. It is possible that the client machine is unable to connect -to the Gluster NFS server because it is using version 4 messages which are not understood by -Gluster NFS server. The timeout can be resolved by forcing the NFS client to use version 3. The -<emphasis role="bold">vers</emphasis> option to mount command is used for this purpose: -</para> - <para><command>$ mount <replaceable>nfsserver</replaceable><replaceable>:export</replaceable> -o vers=3 <replaceable>mount-point</replaceable></command> -</para> - </listitem> - </orderedlist> - </section> - <section> - <title>showmount fails with clnt_create: RPC: Unable to receive </title> - <para>Check your firewall setting to open ports 111 for portmap requests/replies and Gluster NFS -server requests/replies. Gluster NFS server operates over the following port numbers: 38465, -38466, and 38467. -</para> - <para>For more information, see <xref linkend="sect-Administration_Guide-Test_Chapter-GlusterFS_Client-Native-RPM"/>. -</para> - </section> - <section> - <title>Application fails with "Invalid argument" or "Value too large for defined data type" error. </title> - <para>These two errors generally happen for 32-bit nfs clients or applications that do not support 64-bit -inode numbers or large files. -Use the following option from the CLI to make Gluster NFS return 32-bit inode numbers instead: -nfs.enable-ino32 <on|off> -</para> - <para>Applications that will benefit are those that were either: -</para> - <itemizedlist> - <listitem> - <para>built 32-bit and run on 32-bit machines such that they do not support large files by default</para> - </listitem> - <listitem> - <para>built 32-bit on 64-bit systems -</para> - </listitem> - </itemizedlist> - <para>This option is disabled by default so NFS returns 64-bit inode numbers by default. -</para> - <para>Applications which can be rebuilt from source are recommended to rebuild using the following -flag with gcc:</para> - <para><command> -D_FILE_OFFSET_BITS=64</command> -</para> - </section> - </section> - <section> - <title>Troubleshooting File Locks</title> - <para>In GlusterFS 3.3 you can use <command>statedump</command> command to list the locks held on files. The statedump output also provides information on each lock with its range, basename, PID of the application holding the lock, and so on. You can analyze the output to know about the locks whose owner/application is no longer running or interested in that lock. After ensuring that the no application is using the file, you can clear the lock using the following <command>clear lock</command> command:</para> - <para><command># <command>gluster volume clear-locks <replaceable>VOLNAME path</replaceable> kind {blocked | granted | all}{inode [range] | entry [basename] | posix [range]}</command></command></para> - <para>For more information on performing <command>statedump</command>, see <xref linkend="sect-Administration_Guide-Monitor_Workload-Performing_Statedump"/></para> - <para><emphasis role="bold">To identify locked file and clear locks</emphasis></para> - <orderedlist> - <listitem> - <para>Perform statedump on the volume to view the files that are locked using the following command:</para> - <para> <command># gluster volume statedump <replaceable>VOLNAME</replaceable> inode</command></para> - <para>For example, to display statedump of test-volume:</para> - <para><programlisting># gluster volume statedump test-volume -Volume statedump successful</programlisting></para> - <para>The statedump files are created on the brick servers in the<filename> /tmp</filename> directory or in the directory set using <command>server.statedump-path</command> volume option. The naming convention of the dump file is <filename><brick-path>.<brick-pid>.dump</filename>.</para> - <para>The following are the sample contents of the statedump file. It indicates that GlusterFS has entered into a state where there is an entry lock (entrylk) and an inode lock (inodelk). Ensure that those are stale locks and no resources own them. </para> - <para><screen>[xlator.features.locks.vol-locks.inode] -path=/ -mandatory=0 -entrylk-count=1 -lock-dump.domain.domain=vol-replicate-0 -xlator.feature.locks.lock-dump.domain.entrylk.entrylk[0](ACTIVE)=type=ENTRYLK_WRLCK on basename=file1, pid = 714782904, owner=ffffff2a3c7f0000, transport=0x20e0670, , granted at Mon Feb 27 16:01:01 2012 - -conn.2.bound_xl./gfs/brick1.hashsize=14057 -conn.2.bound_xl./gfs/brick1.name=/gfs/brick1/inode -conn.2.bound_xl./gfs/brick1.lru_limit=16384 -conn.2.bound_xl./gfs/brick1.active_size=2 -conn.2.bound_xl./gfs/brick1.lru_size=0 -conn.2.bound_xl./gfs/brick1.purge_size=0 - -[conn.2.bound_xl./gfs/brick1.active.1] -gfid=538a3d4a-01b0-4d03-9dc9-843cd8704d07 -nlookup=1 -ref=2 -ia_type=1 -[xlator.features.locks.vol-locks.inode] -path=/file1 -mandatory=0 -inodelk-count=1 -lock-dump.domain.domain=vol-replicate-0 -inodelk.inodelk[0](ACTIVE)=type=WRITE, whence=0, start=0, len=0, pid = 714787072, owner=00ffff2a3c7f0000, transport=0x20e0670, , granted at Mon Feb 27 16:01:01 2012</screen></para> - </listitem> - <listitem> - <para>Clear the lock using the following command:</para> - <para><command># <command>gluster volume clear-locks <replaceable>VOLNAME path</replaceable> kind granted entry basename</command></command></para> - <para>For example, to clear the entry lock on <filename>file1</filename> of test-volume: -</para> - <para><screen># gluster volume clear-locks test-volume / kind granted entry file1 -Volume clear-locks successful -vol-locks: entry blocked locks=0 granted locks=1</screen></para> - </listitem> - <listitem> - <para>Clear the inode lock using the following command:</para> - <para><command># <command>gluster volume clear-locks <replaceable>VOLNAME path</replaceable> kind granted inode range </command></command></para> - <para>For example, to clear the inode lock on <filename>file1</filename> of test-volume: -</para> - <para><screen># gluster volume clear-locks test-volume /file1 kind granted inode 0,0-0 -Volume clear-locks successful -vol-locks: inode blocked locks=0 granted locks=1</screen></para> - <para>You can perform statedump on test-volume again to verify that the above inode and entry locks are cleared.</para> - </listitem> - </orderedlist> - </section> -</chapter> diff --git a/doc/admin-guide/en-US/gfs_introduction.xml b/doc/admin-guide/en-US/gfs_introduction.xml deleted file mode 100644 index 5fd88730556..00000000000 --- a/doc/admin-guide/en-US/gfs_introduction.xml +++ /dev/null @@ -1,54 +0,0 @@ -<?xml version='1.0' encoding='UTF-8'?> -<!-- This document was created with Syntext Serna Free. --><!DOCTYPE chapter PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN" "docbookV4.5/docbookx.dtd" []> -<chapter> - <title>Introducing Gluster File System</title> - <para>GlusterFS is an open source, clustered file system capable of scaling to several petabytes and handling thousands of clients. GlusterFS can be flexibly combined with commodity physical, virtual, and cloud resources to deliver highly available and performant enterprise storage at a fraction of the cost of traditional solutions.</para> - <para>GlusterFS clusters together storage building blocks over Infiniband RDMA and/or TCP/IP interconnect, aggregating disk and memory resources and managing data in a single global namespace. GlusterFS is based on a stackable user space design, delivering exceptional performance for diverse workloads. -</para> - <figure> - <title>Virtualized Cloud Environments</title> - <mediaobject> - <textobject> - <phrase>Virtualized Cloud Environments</phrase> - </textobject> - <imageobject> - <imagedata align="center" fileref="images/640px-GlusterFS_3.2_Architecture.png"/> - </imageobject> - </mediaobject> - </figure> - <para>GlusterFS is designed for today's high-performance, virtualized cloud environments. Unlike traditional data centers, cloud environments require multi-tenancy along with the ability to grow or shrink resources on demand. Enterprises can scale capacity, performance, and availability on demand, with no vendor lock-in, across on-premise, public cloud, and hybrid environments. </para> - <para>GlusterFS is in production at thousands of enterprises spanning media, healthcare, government, education, web 2.0, and financial services. The following table lists the commercial offerings and its documentation location: -</para> - <informaltable frame="all"> - <tgroup cols="2"> - <colspec colname="c1" colwidth="16%"/> - <colspec colname="c2" colwidth="84%"/> - <thead> - <row> - <entry>Product</entry> - <entry>Documentation Location</entry> - </row> - </thead> - <tbody> - <row> - <entry>Red Hat Storage Software Appliance</entry> - <entry> - <ulink url="http://docs.redhat.com/docs/en-US/Red_Hat_Storage_Software_Appliance/index.html"/> - </entry> - </row> - <row> - <entry>Red Hat Virtual Storage Appliance</entry> - <entry> - <ulink url="http://docs.redhat.com/docs/en-US/Red_Hat_Virtual_Storage_Appliance/index.html"/> - </entry> - </row> - <row> - <entry>Red Hat Storage </entry> - <entry> - <ulink url="http://docs.redhat.com/docs/en-US/Red_Hat_Storage/index.html"/> - </entry> - </row> - </tbody> - </tgroup> - </informaltable> -</chapter> diff --git a/doc/admin-guide/en-US/glossary.xml b/doc/admin-guide/en-US/glossary.xml deleted file mode 100644 index 8c314feaad4..00000000000 --- a/doc/admin-guide/en-US/glossary.xml +++ /dev/null @@ -1,126 +0,0 @@ -<?xml version='1.0' encoding='UTF-8'?> -<!-- This document was created with Syntext Serna Free. --><!DOCTYPE chapter PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN" "docbookV4.5/docbookx.dtd" []> -<chapter> - <title>Glossary</title> - <glosslist> - <glossentry> - <glossterm>Brick</glossterm> - <glossdef> - <para>A Brick is the GlusterFS basic unit of storage, represented by an export directory on a server in the trusted storage pool. A Brick is expressed by combining a server with an export directory in the following format:</para> - <para><code>SERVER:EXPORT</code></para> - <para>For example:</para> - <para><filename>myhostname:/exports/myexportdir/</filename></para> - </glossdef> - </glossentry> - <glossentry> - <glossterm>Cluster</glossterm> - <glossdef> - <para>A cluster is a group of linked computers, working together closely thus in many respects forming a single computer.</para> - </glossdef> - </glossentry> - <glossentry> - <glossterm>Distributed File System</glossterm> - <glossdef> - <para>A file system that allows multiple clients to concurrently access data over a computer network.</para> - </glossdef> - </glossentry> - <glossentry> - <glossterm>Filesystem</glossterm> - <glossdef> - <para>A method of storing and organizing computer files and their data. Essentially, it organizes these files into a database for the storage, organization, manipulation, and retrieval by the computer's operating system.</para> - <para>Source: <ulink url="http://en.wikipedia.org/wiki/Filesystem">Wikipedia</ulink></para> - </glossdef> - </glossentry> - <glossentry> - <glossterm>FUSE</glossterm> - <glossdef> - <para>Filesystem in Userspace (<acronym>FUSE</acronym>) is a loadable kernel module for Unix-like computer operating systems that lets non-privileged users create their own file systems without editing kernel code. This is achieved by running file system code in user space while the <acronym>FUSE</acronym> module provides only a "bridge" to the actual kernel interfaces.</para> - <para>Source: <ulink url="http://en.wikipedia.org/wiki/Filesystem_in_Userspace">Wikipedia</ulink></para> - </glossdef> - </glossentry> - <glossentry> - <glossterm>Geo-Replication</glossterm> - <glossdef> - <para>Geo-replication provides a continuous, asynchronous, and incremental replication service from site to another over Local Area Networks (<acronym>LAN</acronym>), Wide Area Network (<acronym>WAN</acronym>), and across the Internet.</para> - </glossdef> - </glossentry> - <glossentry> - <glossterm>glusterd</glossterm> - <glossdef> - <para>The Gluster management daemon that needs to run on all servers in the trusted storage pool.</para> - </glossdef> - </glossentry> - <glossentry> - <glossterm>Metadata</glossterm> - <glossdef> - <para>Metadata is data providing information about one or more other pieces of data.</para> - </glossdef> - </glossentry> - <glossentry> - <glossterm>Namespace</glossterm> - <glossdef> - <para>Namespace is an abstract container or environment created to hold a logical grouping of unique identifiers or symbols. Each Gluster volume exposes a single namespace as a POSIX mount point that contains every file in the cluster.</para> - </glossdef> - </glossentry> - <glossentry> - <glossterm>Open Source</glossterm> - <glossdef> - <para>Open source describes practices in production and development that promote access to the end product's source materials. Some consider open source a philosophy, others consider it a pragmatic methodology.</para> - <para>Before the term open source became widely adopted, developers and producers used a variety of phrases to describe the concept; open source gained hold with the rise of the Internet, and the attendant need for massive retooling of the computing source code.</para> - <para>Opening the source code enabled a self-enhancing diversity of production models, communication paths, and interactive communities. Subsequently, a new, three-word phrase "open source software" was born to describe the environment that the new copyright, licensing, domain, and consumer issues created.</para> - <para>Source: <ulink url="http://en.wikipedia.org/wiki/Open_source">Wikipedia</ulink></para> - </glossdef> - </glossentry> - <glossentry> - <glossterm>Petabyte</glossterm> - <glossdef> - <para>A petabyte (derived from the SI prefix peta- ) is a unit of information equal to one quadrillion (short scale) bytes, or 1000 terabytes. The unit symbol for the petabyte is PB. The prefix peta- (P) indicates a power of 1000:</para> - <para>1 PB = 1,000,000,000,000,000 B = 10005 B = 1015 B.</para> - <para>The term "pebibyte" (<acronym>PiB</acronym>), using a binary prefix, is used for the corresponding power of 1024.</para> - <para>Source: <ulink url="http://en.wikipedia.org/wiki/Petabyte">Wikipedia</ulink></para> - </glossdef> - </glossentry> - <glossentry> - <glossterm>POSIX</glossterm> - <glossdef> - <para>Portable Operating System Interface (for Unix) is the name of a family of related standards specified by the IEEE to define the application programming interface (<acronym>API</acronym>), along with shell and utilities interfaces for software compatible with variants of the Unix operating system. Gluster exports a fully <acronym>POSIX</acronym> compliant file system.</para> - </glossdef> - </glossentry> - <glossentry> - <glossterm>RAID</glossterm> - <glossdef> - <para>Redundant Array of Inexpensive Disks (<acronym>RAID</acronym>) is a technology that provides increased storage reliability through redundancy, combining multiple low-cost, less-reliable disk drives components into a logical unit where all drives in the array are interdependent.</para> - </glossdef> - </glossentry> - <glossentry> - <glossterm>RRDNS</glossterm> - <glossdef> - <para>Round Robin Domain Name Service (<acronym>RRDNS</acronym>) is a method to distribute load across application servers. <acronym>RRDNS</acronym> is implemented by creating multiple A records with the same name and different IP addresses in the zone file of a DNS server.</para> - </glossdef> - </glossentry> - <glossentry> - <glossterm>Trusted Storage Pool</glossterm> - <glossdef> - <para>A storage pool is a trusted network of storage servers. When you start the first server, the storage pool consists of that server alone.</para> - </glossdef> - </glossentry> - <glossentry> - <glossterm>Userspace</glossterm> - <glossdef> - <para>Applications running in user space don’t directly interact with hardware, instead using the kernel to moderate access. Userspace applications are generally more portable than applications in kernel space. Gluster is a user space application.</para> - </glossdef> - </glossentry> - <glossentry> - <glossterm>Volfile</glossterm> - <glossdef> - <para>Volfile is a configuration file used by glusterfs process. Volfile will be usually located at <filename>/etc/glusterd/vols/VOLNAME</filename>.</para> - </glossdef> - </glossentry> - <glossentry> - <glossterm>Volume</glossterm> - <glossdef> - <para>A volume is a logical collection of bricks. Most of the gluster management operations happen on the volume.</para> - </glossdef> - </glossentry> - </glosslist> -</chapter> diff --git a/doc/admin-guide/en-US/images/640px-GlusterFS_3.2_Architecture.png b/doc/admin-guide/en-US/images/640px-GlusterFS_3.2_Architecture.png Binary files differdeleted file mode 100644 index 95f89ec8286..00000000000 --- a/doc/admin-guide/en-US/images/640px-GlusterFS_3.2_Architecture.png +++ /dev/null diff --git a/doc/admin-guide/en-US/images/Distributed_Replicated_Volume.png b/doc/admin-guide/en-US/images/Distributed_Replicated_Volume.png Binary files differdeleted file mode 100644 index dfc0a2c5600..00000000000 --- a/doc/admin-guide/en-US/images/Distributed_Replicated_Volume.png +++ /dev/null diff --git a/doc/admin-guide/en-US/images/Distributed_Striped_Replicated_Volume.png b/doc/admin-guide/en-US/images/Distributed_Striped_Replicated_Volume.png Binary files differdeleted file mode 100644 index d286fa99e94..00000000000 --- a/doc/admin-guide/en-US/images/Distributed_Striped_Replicated_Volume.png +++ /dev/null diff --git a/doc/admin-guide/en-US/images/Distributed_Striped_Volume.png b/doc/admin-guide/en-US/images/Distributed_Striped_Volume.png Binary files differdeleted file mode 100644 index 752fa982fa6..00000000000 --- a/doc/admin-guide/en-US/images/Distributed_Striped_Volume.png +++ /dev/null diff --git a/doc/admin-guide/en-US/images/Distributed_Volume.png b/doc/admin-guide/en-US/images/Distributed_Volume.png Binary files differdeleted file mode 100644 index 4386ca935b9..00000000000 --- a/doc/admin-guide/en-US/images/Distributed_Volume.png +++ /dev/null diff --git a/doc/admin-guide/en-US/images/Geo-Rep03_Internet.png b/doc/admin-guide/en-US/images/Geo-Rep03_Internet.png Binary files differdeleted file mode 100644 index 3cd0eaded02..00000000000 --- a/doc/admin-guide/en-US/images/Geo-Rep03_Internet.png +++ /dev/null diff --git a/doc/admin-guide/en-US/images/Geo-Rep04_Cascading.png b/doc/admin-guide/en-US/images/Geo-Rep04_Cascading.png Binary files differdeleted file mode 100644 index 54bf9f05cff..00000000000 --- a/doc/admin-guide/en-US/images/Geo-Rep04_Cascading.png +++ /dev/null diff --git a/doc/admin-guide/en-US/images/Geo-Rep_LAN.png b/doc/admin-guide/en-US/images/Geo-Rep_LAN.png Binary files differdeleted file mode 100644 index a74f6dbb50a..00000000000 --- a/doc/admin-guide/en-US/images/Geo-Rep_LAN.png +++ /dev/null diff --git a/doc/admin-guide/en-US/images/Geo-Rep_WAN.png b/doc/admin-guide/en-US/images/Geo-Rep_WAN.png Binary files differdeleted file mode 100644 index d72d72768bc..00000000000 --- a/doc/admin-guide/en-US/images/Geo-Rep_WAN.png +++ /dev/null diff --git a/doc/admin-guide/en-US/images/GlusterFS_3.2_Architecture.png b/doc/admin-guide/en-US/images/GlusterFS_3.2_Architecture.png Binary files differdeleted file mode 100644 index b506db1f4e7..00000000000 --- a/doc/admin-guide/en-US/images/GlusterFS_3.2_Architecture.png +++ /dev/null diff --git a/doc/admin-guide/en-US/images/Hadoop_Architecture.png b/doc/admin-guide/en-US/images/Hadoop_Architecture.png Binary files differdeleted file mode 100644 index 8725bd330bb..00000000000 --- a/doc/admin-guide/en-US/images/Hadoop_Architecture.png +++ /dev/null diff --git a/doc/admin-guide/en-US/images/Replicated_Volume.png b/doc/admin-guide/en-US/images/Replicated_Volume.png Binary files differdeleted file mode 100644 index 135a63f345a..00000000000 --- a/doc/admin-guide/en-US/images/Replicated_Volume.png +++ /dev/null diff --git a/doc/admin-guide/en-US/images/Striped_Replicated_Volume.png b/doc/admin-guide/en-US/images/Striped_Replicated_Volume.png Binary files differdeleted file mode 100644 index ee88af73134..00000000000 --- a/doc/admin-guide/en-US/images/Striped_Replicated_Volume.png +++ /dev/null diff --git a/doc/admin-guide/en-US/images/Striped_Volume.png b/doc/admin-guide/en-US/images/Striped_Volume.png Binary files differdeleted file mode 100644 index 63a84b242ab..00000000000 --- a/doc/admin-guide/en-US/images/Striped_Volume.png +++ /dev/null diff --git a/doc/admin-guide/en-US/images/UFO_Architecture.png b/doc/admin-guide/en-US/images/UFO_Architecture.png Binary files differdeleted file mode 100644 index be85d7b2825..00000000000 --- a/doc/admin-guide/en-US/images/UFO_Architecture.png +++ /dev/null diff --git a/doc/admin-guide/en-US/images/VSA_Architecture.png b/doc/admin-guide/en-US/images/VSA_Architecture.png Binary files differdeleted file mode 100644 index c3ab80cf3e8..00000000000 --- a/doc/admin-guide/en-US/images/VSA_Architecture.png +++ /dev/null diff --git a/doc/admin-guide/en-US/images/arhitecture.png b/doc/admin-guide/en-US/images/arhitecture.png deleted file mode 100644 index 4e5188bf8dd..00000000000 --- a/doc/admin-guide/en-US/images/arhitecture.png +++ /dev/null @@ -1,13 +0,0 @@ -<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"> -<html xmlns="http://www.w3.org/1999/xhtml"> -<head> -<title>HTTP Error 403</title> -</head> -<body> -<h1>Error 403</h1> -<p>We're sorry, but we could not fulfill your request for -/community/documentation/index.php/Image:GlusterFS_3.2_Architecture.png on this server.</p> -<p>An invalid request was received from your browser. This may be caused by a malfunctioning proxy server or browser privacy software.</p> -<p>Your technical support key is: <strong>7ab5-0b6a-1756-6707</strong></p> -<p>You can use this key to <a href="http://www.ioerror.us/bb2-support-key?key=7ab5-0b6a-1756-6707">fix this problem yourself</a>.</p> -<p>If you are unable to fix the problem yourself, please contact <a href="mailto:webmaster+nospam@nospam.gluster.com">webmaster at gluster.com</a> and be sure to provide the technical support key shown above.</p> diff --git a/doc/admin-guide/en-US/images/icon.svg b/doc/admin-guide/en-US/images/icon.svg deleted file mode 100644 index b2f16d0f61d..00000000000 --- a/doc/admin-guide/en-US/images/icon.svg +++ /dev/null @@ -1,19 +0,0 @@ -<?xml version="1.0" encoding="UTF-8" standalone="no"?> -<svg xmlns:svg="http://www.w3.org/2000/svg" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.0" width="32" height="32" id="svg3017"> - <defs id="defs3019"> - <linearGradient id="linearGradient2381"> - <stop id="stop2383" style="stop-color:#ffffff;stop-opacity:1" offset="0"/> - <stop id="stop2385" style="stop-color:#ffffff;stop-opacity:0" offset="1"/> - </linearGradient> - <linearGradient x1="296.4996" y1="188.81061" x2="317.32471" y2="209.69398" id="linearGradient2371" xlink:href="#linearGradient2381" gradientUnits="userSpaceOnUse" gradientTransform="matrix(0.90776,0,0,0.90776,24.35648,49.24131)"/> - </defs> - <g transform="matrix(0.437808,-0.437808,0.437808,0.437808,-220.8237,43.55311)" id="g5089"> - <path d="m 8.4382985,-6.28125 c -0.6073916,0 -4.3132985,5.94886271 -4.3132985,8.25 l 0,26.71875 c 0,0.846384 0.5818159,1.125 1.15625,1.125 l 25.5625,0 c 0.632342,0 1.125001,-0.492658 1.125,-1.125 l 0,-5.21875 0.28125,0 c 0.49684,0 0.906249,-0.409411 0.90625,-0.90625 l 0,-27.9375 c 0,-0.4968398 -0.40941,-0.90625 -0.90625,-0.90625 l -23.8117015,0 z" transform="translate(282.8327,227.1903)" id="path5091" style="fill:#5c5c4f;stroke:#000000;stroke-width:3.23021388;stroke-miterlimit:4;stroke-dasharray:none"/> - <rect width="27.85074" height="29.369793" rx="1.1414107" ry="1.1414107" x="286.96509" y="227.63805" id="rect5093" style="fill:#032c87"/> - <path d="m 288.43262,225.43675 25.2418,0 0,29.3698 -26.37615,0.0241 1.13435,-29.39394 z" id="rect5095" style="fill:#ffffff"/> - <path d="m 302.44536,251.73726 c 1.38691,7.85917 -0.69311,11.28365 -0.69311,11.28365 2.24384,-1.60762 3.96426,-3.47694 4.90522,-5.736 0.96708,2.19264 1.83294,4.42866 4.27443,5.98941 0,0 -1.59504,-7.2004 -1.71143,-11.53706 l -6.77511,0 z" id="path5097" style="fill:#a70000;fill-opacity:1;stroke-width:2"/> - <rect width="25.241802" height="29.736675" rx="0.89682275" ry="0.89682275" x="290.73544" y="220.92249" id="rect5099" style="fill:#809cc9"/> - <path d="m 576.47347,725.93939 6.37084,0.41502 0.4069,29.51809 c -1.89202,-1.31785 -6.85427,-3.7608 -8.26232,-1.68101 l 0,-26.76752 c 0,-0.82246 0.66212,-1.48458 1.48458,-1.48458 z" transform="matrix(0.499065,-0.866565,0,1,0,0)" id="rect5101" style="fill:#4573b3;fill-opacity:1"/> - <path d="m 293.2599,221.89363 20.73918,0 c 0.45101,0 0.8141,0.3631 0.8141,0.81411 0.21547,6.32836 -19.36824,21.7635 -22.36739,17.59717 l 0,-17.59717 c 0,-0.45101 0.3631,-0.81411 0.81411,-0.81411 z" id="path5103" style="opacity:0.65536726;fill:url(#linearGradient2371);fill-opacity:1"/> - </g> -</svg> diff --git a/doc/admin-guide/publican.cfg b/doc/admin-guide/publican.cfg deleted file mode 100644 index e42fa1b3dc8..00000000000 --- a/doc/admin-guide/publican.cfg +++ /dev/null @@ -1,12 +0,0 @@ -# Config::Simple 4.59 -# Thu Apr 5 11:09:15 2012 - -xml_lang: "en-US" -type: Book -brand: Gluster_Brand -prod_url: http://www.gluster.org -doc_url: http://www.gluster.com/community/documentation/index.php/Main_Page -condition: gfs -show_remarks: 1 - - diff --git a/doc/debugging/analyzing-regression-cores.md b/doc/debugging/analyzing-regression-cores.md new file mode 100644 index 00000000000..5e10f41c6eb --- /dev/null +++ b/doc/debugging/analyzing-regression-cores.md @@ -0,0 +1,54 @@ +# Analyzing Regression Cores +This document explains how to analyze core-dumps obtained from regression machines, with examples. +1. Download the core-tarball and extract it. +2. `cd` into directory where the tarball is extracted. +``` +[sh]# pwd +/home/user/Downloads +[sh]# ls +build build-install-20150625_05_42_39.tar.bz2 lib64 usr +``` +3. Determine the core file you need to examine. There can be more than one core file. You can list them from './build/install/cores' directory. +``` +[sh]# ls build/install/cores/ +core.9341 liblist.txt liblist.txt.tmp +``` +In case you are unsure which binary generated the core-file, executing 'file' command on it will help. +``` +[sh]# file ./build/install/cores/core.9341 +./build/install/cores/core.9341: ELF 64-bit LSB core file x86-64, version 1 (SYSV), SVR4-style, from '/build/install/sbin/glusterfsd -s slave26.cloud.gluster.org --volfile-id patchy' +``` +As seen, the core file was generated by glusterfsd binary, and path to it is provided (/build/install/sbin/glusterfsd). + +4. Now, run the following command on the core: +``` +gdb -ex 'set sysroot ./' -ex 'core-file ./build/install/cores/core.xxx' <target, say ./build/install/sbin/glusterd> +In this case, +gdb -ex 'set sysroot ./' -ex 'core-file ./build/install/cores/core.9341' ./build/install/sbin/glusterfsd +``` +5. You can cross check if all shared libraries are available and loaded by using 'info sharedlibrary' command from inside gdb. +6. Once verified, usual gdb commands based on requirement can be used to debug the core. + `bt` or `backtrace` from gdb of core used in examples: +``` +Core was generated by `/build/install/sbin/glusterfsd -s slave26.cloud.gluster.org --volfile-id patchy'. +Program terminated with signal SIGABRT, Aborted. +#0 0x00007f512a54e625 in raise () from ./lib64/libc.so.6 +(gdb) bt +#0 0x00007f512a54e625 in raise () from ./lib64/libc.so.6 +#1 0x00007f512a54fe05 in abort () from ./lib64/libc.so.6 +#2 0x00007f512a54774e in __assert_fail_base () from ./lib64/libc.so.6 +#3 0x00007f512a547810 in __assert_fail () from ./lib64/libc.so.6 +#4 0x00007f512b9fc434 in __gf_free (free_ptr=0x7f50f4000e50) at /home/jenkins/root/workspace/rackspace-regression-2GB-triggered/libglusterfs/src/mem-pool.c:304 +#5 0x00007f512b9b6657 in loc_wipe (loc=0x7f510c20d1a0) at /home/jenkins/root/workspace/rackspace-regression-2GB-triggered/libglusterfs/src/xlator.c:685 +#6 0x00007f511cb8201d in mq_start_quota_txn_v2 (this=0x7f5118019b60, loc=0x7f510c20d2b8, ctx=0x7f50f4000bf0, contri=0x7f50f4000d60) + at /home/jenkins/root/workspace/rackspace-regression-2GB-triggered/xlators/features/marker/src/marker-quota.c:2921 +#7 0x00007f511cb82c55 in mq_initiate_quota_task (opaque=0x7f510c20d2b0) at /home/jenkins/root/workspace/rackspace-regression-2GB-triggered/xlators/features/marker/src/marker-quota.c:3199 +#8 0x00007f511cb81820 in mq_synctask (this=0x7f5118019b60, task=0x7f511cb829fa <mq_initiate_quota_task>, spawn=_gf_false, loc=0x7f510c20d430, dict=0x0, buf=0x0, contri=0) + at /home/jenkins/root/workspace/rackspace-regression-2GB-triggered/xlators/features/marker/src/marker-quota.c:2789 +#9 0x00007f511cb82f82 in mq_initiate_quota_blocking_txn (this=0x7f5118019b60, loc=0x7f510c20d430) at /home/jenkins/root/workspace/rackspace-regression-2GB-triggered/xlators/features/marker/src/marker-quota.c:3230 +#10 0x00007f511cb82844 in mq_reduce_parent_size_task (opaque=0x7f510c000df0) at /home/jenkins/root/workspace/rackspace-regression-2GB-triggered/xlators/features/marker/src/marker-quota.c:3117 +#11 0x00007f512ba0f9dc in synctask_wrap (old_task=0x7f510c0053e0) at /home/jenkins/root/workspace/rackspace-regression-2GB-triggered/libglusterfs/src/syncop.c:370 +#12 0x00007f512a55f8f0 in ?? () from ./lib64/libc.so.6 +#13 0x0000000000000000 in ?? () +(gdb) +``` diff --git a/doc/debugging/gfid-to-path.md b/doc/debugging/gfid-to-path.md new file mode 100644 index 00000000000..1917bf2cca1 --- /dev/null +++ b/doc/debugging/gfid-to-path.md @@ -0,0 +1,68 @@ +# Convert GFID to Path + +GlusterFS internal file identifier (GFID) is a uuid that is unique to each +file across the entire cluster. This is analogous to inode number in a +normal filesystem. The GFID of a file is stored in its xattr named +`trusted.gfid`. + +#### Special mount using [gfid-access translator][1]: +``` +mount -t glusterfs -o aux-gfid-mount vm1:test /mnt/testvol +``` + +Assuming, you have `GFID` of a file from changelog (or somewhere else). +For trying this out, you can get `GFID` of a file from mountpoint: +``` +getfattr -n glusterfs.gfid.string /mnt/testvol/dir/file +``` + + +--- +### Get file path from GFID (Method 1): +**(Lists hardlinks delimited by `:`, returns path as seen from mountpoint)** + +#### Turn on build-pgfid option +``` +gluster volume set test build-pgfid on +``` +Read virtual xattr `glusterfs.ancestry.path` which contains the file path +``` +getfattr -n glusterfs.ancestry.path -e text /mnt/testvol/.gfid/<GFID> +``` + +**Example:** +``` +[root@vm1 glusterfs]# ls -il /mnt/testvol/dir/ +total 1 +10610563327990022372 -rw-r--r--. 2 root root 3 Jul 17 18:05 file +10610563327990022372 -rw-r--r--. 2 root root 3 Jul 17 18:05 file3 + +[root@vm1 glusterfs]# getfattr -n glusterfs.gfid.string /mnt/testvol/dir/file +getfattr: Removing leading '/' from absolute path names +# file: mnt/testvol/dir/file +glusterfs.gfid.string="11118443-1894-4273-9340-4b212fa1c0e4" + +[root@vm1 glusterfs]# getfattr -n glusterfs.ancestry.path -e text /mnt/testvol/.gfid/11118443-1894-4273-9340-4b212fa1c0e4 +getfattr: Removing leading '/' from absolute path names +# file: mnt/testvol/.gfid/11118443-1894-4273-9340-4b212fa1c0e4 +glusterfs.ancestry.path="/dir/file:/dir/file3" +``` + +--- +### Get file path from GFID (Method 2): +**(Does not list all hardlinks, returns backend brick path)** +``` +getfattr -n trusted.glusterfs.pathinfo -e text /mnt/testvol/.gfid/<GFID> +``` + +**Example:** +``` +[root@vm1 glusterfs]# getfattr -n trusted.glusterfs.pathinfo -e text /mnt/testvol/.gfid/11118443-1894-4273-9340-4b212fa1c0e4 +getfattr: Removing leading '/' from absolute path names +# file: mnt/testvol/.gfid/11118443-1894-4273-9340-4b212fa1c0e4 +trusted.glusterfs.pathinfo="(<DISTRIBUTE:test-dht> <POSIX(/mnt/brick-test/b):vm1:/mnt/brick-test/b/dir//file3>)" +``` + +--- +#### References and links: +[posix: placeholders for GFID to path conversion](http://review.gluster.org/5951) diff --git a/doc/debugging/mem-alloc-list.md b/doc/debugging/mem-alloc-list.md new file mode 100644 index 00000000000..1c68e65d323 --- /dev/null +++ b/doc/debugging/mem-alloc-list.md @@ -0,0 +1,19 @@ +## Viewing Memory Allocations + +While statedumps provide stats of the number of allocations, size etc for a +particular mem type, there is no easy way to examine all the allocated objects of that type +in memory.Being able to view this information could help with determining how an object is used, +and if there are any memory leaks. + +The mem_acct_rec structures have been updated to include lists to which the allocated object is +added. These can be examined in gdb using simple scripts. + +`gdb> plist xl->mem_acct.rec[$type]->obj_list` + +will print out the pointers of all allocations of $type. + +These changes are primarily targeted at developers and need to enabled +at compile-time using `configure --enable-debug`. + + + diff --git a/doc/debugging/split-brain.md b/doc/debugging/split-brain.md new file mode 100644 index 00000000000..6b122c40551 --- /dev/null +++ b/doc/debugging/split-brain.md @@ -0,0 +1,264 @@ +# Steps to recover from File split-brain +This document contains steps to recover from a file split-brain. +## Quick Start: +### Step 1. Get the path of the file that is in split-brain: +It can be obtained either by +1. The command `gluster volume heal info split-brain`. +2. Identify the files for which file operations performed from the client keep failing with Input/Output error. + +### Step 2. Close the applications that opened this file from the mount point. +In case of VMs, they need to be powered-off. + +### Step 3. Decide on the correct copy: +This is done by observing the afr changelog extended attributes of the file on +the bricks using the getfattr command; then identifying the type of split-brain +(data split-brain, metadata split-brain, entry split-brain or split-brain due to +gfid-mismatch); and finally determining which of the bricks contains the 'good copy' +of the file. +``` +getfattr -d -m . -e hex <file-path-on-brick> +``` + +It is also possible that one brick might contain the correct data while the +other might contain the correct metadata. + +### Step 4. Reset the relevant extended attribute on the brick(s) that contains the 'bad copy' of the file data/metadata using the setfattr command. +``` +setfattr -n <attribute-name> -v <attribute-value> <file-path-on-brick> +``` + +### Step 5. Trigger self-heal on the file by performing lookup from the client: +``` +ls -l <file-path-on-gluster-mount> +``` + +Detailed Instructions for steps 3 through 5: +=========================================== +To understand how to resolve split-brain we need to know how to interpret the +afr changelog extended attributes. + +Execute `getfattr -d -m . -e hex <file-path-on-brick>` + +Example: +``` +[root@store3 ~]# getfattr -d -e hex -m. brick-a/file.txt +\#file: brick-a/file.txt +security.selinux=0x726f6f743a6f626a6563745f723a66696c655f743a733000 +trusted.afr.vol-client-2=0x000000000000000000000000 +trusted.afr.vol-client-3=0x000000000200000000000000 +trusted.gfid=0x307a5c9efddd4e7c96e94fd4bcdcbd1b +``` + +The extended attributes with `trusted.afr.<volname>-client-<subvolume-index>` +are used by afr to maintain changelog of the file.The values of the +`trusted.afr.<volname>-client-<subvolume-index>` are calculated by the glusterfs +client (fuse or nfs-server) processes. When the glusterfs client modifies a file +or directory, the client contacts each brick and updates the changelog extended +attribute according to the response of the brick. + +`subvolume-index` is nothing but (brick number - 1) in +`gluster volume info <volname>` output. + +Example: +``` +[root@pranithk-laptop ~]# gluster volume info vol + Volume Name: vol + Type: Distributed-Replicate + Volume ID: 4f2d7849-fbd6-40a2-b346-d13420978a01 + Status: Created + Number of Bricks: 4 x 2 = 8 + Transport-type: tcp + Bricks: + brick-a: pranithk-laptop:/gfs/brick-a + brick-b: pranithk-laptop:/gfs/brick-b + brick-c: pranithk-laptop:/gfs/brick-c + brick-d: pranithk-laptop:/gfs/brick-d + brick-e: pranithk-laptop:/gfs/brick-e + brick-f: pranithk-laptop:/gfs/brick-f + brick-g: pranithk-laptop:/gfs/brick-g + brick-h: pranithk-laptop:/gfs/brick-h +``` + +In the example above: +``` +Brick | Replica set | Brick subvolume index +---------------------------------------------------------------------------- +-/gfs/brick-a | 0 | 0 +-/gfs/brick-b | 0 | 1 +-/gfs/brick-c | 1 | 2 +-/gfs/brick-d | 1 | 3 +-/gfs/brick-e | 2 | 4 +-/gfs/brick-f | 2 | 5 +-/gfs/brick-g | 3 | 6 +-/gfs/brick-h | 3 | 7 +``` + +Each file in a brick maintains the changelog of itself and that of the files +present in all the other bricks in it's replica set as seen by that brick. + +In the example volume given above, all files in brick-a will have 2 entries, +one for itself and the other for the file present in it's replica pair, i.e.brick-b: +``` +trusted.afr.vol-client-0=0x000000000000000000000000 -->changelog for itself (brick-a) +trusted.afr.vol-client-1=0x000000000000000000000000 -->changelog for brick-b as seen by brick-a +``` +Likewise, all files in brick-b will have: +``` +trusted.afr.vol-client-0=0x000000000000000000000000 -->changelog for brick-a as seen by brick-b +trusted.afr.vol-client-1=0x000000000000000000000000 -->changelog for itself (brick-b) +``` + +The same can be extended for other replica pairs. + +Interpreting Changelog (roughly pending operation count) Value: +Each extended attribute has a value which is 24 hexa decimal digits. +First 8 digits represent changelog of data. Second 8 digits represent changelog +of metadata. Last 8 digits represent Changelog of directory entries. + +Pictorially representing the same, we have: +``` +0x 000003d7 00000001 00000000 + | | | + | | \_ changelog of directory entries + | \_ changelog of metadata + \ _ changelog of data +``` + + +For Directories metadata and entry changelogs are valid. +For regular files data and metadata changelogs are valid. +For special files like device files etc metadata changelog is valid. +When a file split-brain happens it could be either data split-brain or +meta-data split-brain or both. When a split-brain happens the changelog of the +file would be something like this: + +Example:(Lets consider both data, metadata split-brain on same file). +``` +[root@pranithk-laptop vol]# getfattr -d -m . -e hex /gfs/brick-?/a +getfattr: Removing leading '/' from absolute path names +\#file: gfs/brick-a/a +trusted.afr.vol-client-0=0x000000000000000000000000 +trusted.afr.vol-client-1=0x000003d70000000100000000 +trusted.gfid=0x80acdbd886524f6fbefa21fc356fed57 +\#file: gfs/brick-b/a +trusted.afr.vol-client-0=0x000003b00000000100000000 +trusted.afr.vol-client-1=0x000000000000000000000000 +trusted.gfid=0x80acdbd886524f6fbefa21fc356fed57 +``` + +### Observations: + +#### According to changelog extended attributes on file /gfs/brick-a/a: +The first 8 digits of trusted.afr.vol-client-0 are all +zeros (0x00000000................), and the first 8 digits of +trusted.afr.vol-client-1 are not all zeros (0x000003d7................). +So the changelog on /gfs/brick-a/a implies that some data operations succeeded +on itself but failed on /gfs/brick-b/a. + +The second 8 digits of trusted.afr.vol-client-0 are +all zeros (0x........00000000........), and the second 8 digits of +trusted.afr.vol-client-1 are not all zeros (0x........00000001........). +So the changelog on /gfs/brick-a/a implies that some metadata operations succeeded +on itself but failed on /gfs/brick-b/a. + +#### According to Changelog extended attributes on file /gfs/brick-b/a: +The first 8 digits of trusted.afr.vol-client-0 are not all +zeros (0x000003b0................), and the first 8 digits of +trusted.afr.vol-client-1 are all zeros (0x00000000................). +So the changelog on /gfs/brick-b/a implies that some data operations succeeded +on itself but failed on /gfs/brick-a/a. + +The second 8 digits of trusted.afr.vol-client-0 are not +all zeros (0x........00000001........), and the second 8 digits of +trusted.afr.vol-client-1 are all zeros (0x........00000000........). +So the changelog on /gfs/brick-b/a implies that some metadata operations succeeded +on itself but failed on /gfs/brick-a/a. + +Since both the copies have data, metadata changes that are not on the other +file, it is in both data and metadata split-brain. + +Deciding on the correct copy: +----------------------------- +The user may have to inspect stat,getfattr output of the files to decide which +metadata to retain and contents of the file to decide which data to retain. +Continuing with the example above, lets say we want to retain the data +of /gfs/brick-a/a and metadata of /gfs/brick-b/a. + +Resetting the relevant changelogs to resolve the split-brain: +------------------------------------------------------------- +For resolving data-split-brain: +We need to change the changelog extended attributes on the files as if some data +operations succeeded on /gfs/brick-a/a but failed on /gfs/brick-b/a. But +/gfs/brick-b/a should NOT have any changelog which says some data operations +succeeded on /gfs/brick-b/a but failed on /gfs/brick-a/a. We need to reset the +data part of the changelog on trusted.afr.vol-client-0 of /gfs/brick-b/a. + +For resolving metadata-split-brain: +We need to change the changelog extended attributes on the files as if some +metadata operations succeeded on /gfs/brick-b/a but failed on /gfs/brick-a/a. +But /gfs/brick-a/a should NOT have any changelog which says some metadata +operations succeeded on /gfs/brick-a/a but failed on /gfs/brick-b/a. +We need to reset metadata part of the changelog on +trusted.afr.vol-client-1 of /gfs/brick-a/a + +So, the intended changes are: +On /gfs/brick-b/a: +For trusted.afr.vol-client-0 +0x000003b00000000100000000 to 0x000000000000000100000000 +(Note that the metadata part is still not all zeros) +Hence execute +`setfattr -n trusted.afr.vol-client-0 -v 0x000000000000000100000000 /gfs/brick-b/a` + +On /gfs/brick-a/a: +For trusted.afr.vol-client-1 +0x0000000000000000ffffffff to 0x000003d70000000000000000 +(Note that the data part is still not all zeros) +Hence execute +`setfattr -n trusted.afr.vol-client-1 -v 0x000003d70000000000000000 /gfs/brick-a/a` + +Thus after the above operations are done, the changelogs look like this: +``` +[root@pranithk-laptop vol]# getfattr -d -m . -e hex /gfs/brick-?/a +getfattr: Removing leading '/' from absolute path names +\#file: gfs/brick-a/a +trusted.afr.vol-client-0=0x000000000000000000000000 +trusted.afr.vol-client-1=0x000003d70000000000000000 +trusted.gfid=0x80acdbd886524f6fbefa21fc356fed57 + +\#file: gfs/brick-b/a +trusted.afr.vol-client-0=0x000000000000000100000000 +trusted.afr.vol-client-1=0x000000000000000000000000 +trusted.gfid=0x80acdbd886524f6fbefa21fc356fed57 +``` + +Triggering Self-heal: +--------------------- +Perform `ls -l <file-path-on-gluster-mount>` to trigger healing. + +Fixing Directory entry split-brain: +---------------------------------- +Afr has the ability to conservatively merge different entries in the directories +when there is a split-brain on directory. +If on one brick directory 'd' has entries '1', '2' and has entries '3', '4' on +the other brick then afr will merge all of the entries in the directory to have +'1', '2', '3', '4' entries in the same directory. +(Note: this may result in deleted files to re-appear in case the split-brain +happens because of deletion of files on the directory) +Split-brain resolution needs human intervention when there is at least one entry +which has same file name but different gfid in that directory. +Example: +On brick-a the directory has entries '1' (with gfid g1), '2' and on brick-b +directory has entries '1' (with gfid g2) and '3'. +These kinds of directory split-brains need human intervention to resolve. +The user needs to remove either file '1' on brick-a or the file '1' on brick-b +to resolve the split-brain. In addition, the corresponding gfid-link file also +needs to be removed.The gfid-link files are present in the .glusterfs folder +in the top-level directory of the brick. If the gfid of the file is +0x307a5c9efddd4e7c96e94fd4bcdcbd1b (the trusted.gfid extended attribute got +from the getfattr command earlier),the gfid-link file can be found at +`/gfs/brick-a/.glusterfs/30/7a/307a5c9efddd4e7c96e94fd4bcdcbd1b` + +#### Word of caution: +Before deleting the gfid-link, we have to ensure that there are no hard links +to the file present on that brick. If hard-links exist,they must be deleted as +well. diff --git a/doc/debugging/statedump.md b/doc/debugging/statedump.md new file mode 100644 index 00000000000..9dfdce15fad --- /dev/null +++ b/doc/debugging/statedump.md @@ -0,0 +1,414 @@ +# Statedump +Statedump is a file generated by glusterfs process with different data structure state which may contain the active inodes, fds, mempools, iobufs, memory allocation stats of different types of datastructures per xlator etc. + +## How to generate statedump +We can find the directory where statedump files are created using `gluster --print-statedumpdir` command. +Create that directory if not already present based on the type of installation. +Lets call this directory `statedump-directory`. + +We can generate statedump using `kill -USR1 <pid-of-gluster-process>`. +gluster-process is nothing but glusterd/glusterfs/glusterfsd process. + +There are also commands to generate statedumps for brick processes/nfs server/quotad + +For bricks: +``` +gluster volume statedump <volname> +``` + +For nfs server: +``` +gluster volume statedump <volname> nfs +``` + +For quotad: +``` +gluster volume statedump <volname> quotad +``` + +For brick-processes files will be created in `statedump-directory` with name of the file as `hyphenated-brick-path.<pid>.dump.timestamp`. For all other processes it will be `glusterdump.<pid>.dump.timestamp`. + +For applications using libgfapi, `SIGUSR1` cannot be used, eg: smbd/libvirtd +processes could have used the `SIGUSR1` signal already for other purposes. +To generate statedump for the processes, using libgfapi, below command can be +executed from one of the nodes in the gluster cluster to which the libgfapi +application is connected to. +``` +gluster volume statedump <volname> client <hostname>:<process id> +``` +The statedumps can be found in the `statedump-directory`, the name of the +statedumps being `glusterdump.<pid>.dump.timestamp`. For a process there can be +multiple such files created depending on the number of times the volume is +accessed by the process (related to the number of `glfs_init()` calls). + +## How to read statedump +We shall see snippets of each type of statedump. + +First and last lines of the file have starting and ending time of writing the statedump file. Times will be in UTC timezone. + +mallinfo return status is printed in the following format. Please read man mallinfo for more information about what each field means. +### Mallinfo +``` +[mallinfo] +mallinfo_arena=100020224 /* Non-mmapped space allocated (bytes) */ +mallinfo_ordblks=69467 /* Number of free chunks */ +mallinfo_smblks=449 /* Number of free fastbin blocks */ +mallinfo_hblks=13 /* Number of mmapped regions */ +mallinfo_hblkhd=20144128 /* Space allocated in mmapped regions (bytes) */ +mallinfo_usmblks=0 /* Maximum total allocated space (bytes) */ +mallinfo_fsmblks=39264 /* Space in freed fastbin blocks (bytes) */ +mallinfo_uordblks=96710112 /* Total allocated space (bytes) */ +mallinfo_fordblks=3310112 /* Total free space (bytes) */ +mallinfo_keepcost=133712 /* Top-most, releasable space (bytes) */ +``` + +### Data structure allocation stats +For every xlator data structure memory per translator loaded in the call-graph is displayed in the following format: + +For xlator with name: glusterfs +``` +[global.glusterfs - Memory usage] #[global.xlator-name - Memory usage] +num_types=119 #It shows the number of data types it is using +``` + +Now for each data-type it prints the memory usage. + +``` +[global.glusterfs - usage-type gf_common_mt_gf_timer_t memusage] +#[global.xlator-name - usage-type <tag associated with the data-type> memusage] +size=112 #num_allocs times the sizeof(data-type) i.e. num_allocs * sizeof (data-type) +num_allocs=2 #Number of allocations of the data-type which are active at the time of taking statedump. +max_size=168 #max_num_allocs times the sizeof(data-type) i.e. max_num_allocs * sizeof (data-type) +max_num_allocs=3 #Maximum number of active allocations at any point in the life of the process. +total_allocs=7 #Number of times this data is allocated in the life of the process. +``` + +### Mempools + +Mempools are optimization to reduce the number of allocations of a data type. If we create a mem-pool of lets say 1024 elements for a data-type, new elements will be allocated from heap using syscalls like calloc, only if all the 1024 elements in the pool are in active use. + +Memory pool allocated by each xlator is displayed in the following format: + +``` +[mempool] #Section name +-----=----- +pool-name=fuse:fd_t #pool-name=<xlator-name>:<data-type> +hot-count=1 #number of mempool elements that are in active use. i.e. for this pool it is the number of 'fd_t' s in active use. +cold-count=1023 #number of mempool elements that are not in use. If a new allocation is required it will be served from here until all the elements in the pool are in use i.e. cold-count becomes 0. +padded_sizeof=108 #Each mempool element is padded with a doubly-linked-list + ptr of mempool + is-in-use info to operate the pool of elements, this size is the element-size after padding +pool-misses=0 #Number of times the element had to be allocated from heap because all elements from the pool are in active use. +alloc-count=314 #Number of times this type of data is allocated through out the life of this process. This may include pool-misses as well. +max-alloc=3 #Maximum number of elements from the pool in active use at any point in the life of the process. This does *not* include pool-misses. +cur-stdalloc=0 #Denotes the number of allocations made from heap once cold-count reaches 0, that are yet to be released via mem_put(). +max-stdalloc=0 #Maximum number of allocations from heap that are in active use at any point in the life of the process. +``` + +### Iobufs +``` +[iobuf.global] +iobuf_pool=0x1f0d970 #The memory pool for iobufs +iobuf_pool.default_page_size=131072 #The default size of iobuf (if no iobuf size is specified the default size is allocated) +#iobuf_arena: One arena represents a group of iobufs of a particular size +iobuf_pool.arena_size=12976128 # The initial size of the iobuf pool (doesn't include the stdalloc'd memory or the newly added arenas) +iobuf_pool.arena_cnt=8 #Total number of arenas in the pool +iobuf_pool.request_misses=0 #The number of iobufs that were stdalloc'd (as they exceeded the default max page size provided by iobuf_pool). +``` + +There are 3 lists of arenas: + +1. Arena list: arenas allocated during iobuf pool creation and the arenas that are in use(active_cnt != 0) will be part of this list. +2. Purge list: arenas that can be purged(no active iobufs, active_cnt == 0). +3. Filled list: arenas without free iobufs. + +``` +[purge.1] #purge.<S.No.> +purge.1.mem_base=0x7fc47b35f000 #The address of the arena structure +purge.1.active_cnt=0 #The number of iobufs active in that arena +purge.1.passive_cnt=1024 #The number of unused iobufs in the arena +purge.1.alloc_cnt=22853 #Total allocs in this pool(number of times the iobuf was allocated from this arena) +purge.1.max_active=7 #Max active iobufs from this arena, at any point in the life of this process. +purge.1.page_size=128 #Size of all the iobufs in this arena. + +[arena.5] #arena.<S.No.> +arena.5.mem_base=0x7fc47af1f000 +arena.5.active_cnt=0 +arena.5.passive_cnt=64 +arena.5.alloc_cnt=0 +arena.5.max_active=0 +arena.5.page_size=32768 +``` + +If the active_cnt of any arena is non zero, then the statedump will also have the iobuf list. +``` +[arena.6.active_iobuf.1] #arena.<S.No>.active_iobuf.<iobuf.S.No.> +arena.6.active_iobuf.1.ref=1 #refcount of the iobuf +arena.6.active_iobuf.1.ptr=0x7fdb921a9000 #address of the iobuf + +[arena.6.active_iobuf.2] +arena.6.active_iobuf.2.ref=1 +arena.6.active_iobuf.2.ptr=0x7fdb92189000 +``` + +At any given point in time if there are lots of filled arenas then that could be a sign of iobuf leaks. + +### Call stack +All the fops received by gluster are handled using call-stacks. Call stack contains the information about uid/gid/pid etc of the process that is executing the fop. Each call-stack contains different call-frames per xlator which handles that fop. + +``` +[global.callpool.stack.3] #global.callpool.stack.<Serial-Number> +stack=0x7fc47a44bbe0 #Stack address +uid=0 #Uid of the process which is executing the fop +gid=0 #Gid of the process which is executing the fop +pid=6223 #Pid of the process which is executing the fop +unique=2778 #Xlators like afr do copy_frame and perform the operation in a different stack, this id is useful to find out the stacks that are inter-related because of copy-frame +lk-owner=0000000000000000 #Some of the fuse fops have lk-owner. +op=LOOKUP #Fop +type=1 #Type of the op i.e. FOP/MGMT-OP +cnt=9 #Number of frames in this stack. +``` +### Call-frame +Each frame will have information about which xlator the frame belongs to, what is the function it wound to/from and will be unwind to. It also mentions if the unwind happened or not. If we observe hangs in the system and want to find out which xlator is causing it. Take a statedump and see what is the final xlator which is yet to be unwound. + +``` +[global.callpool.stack.3.frame.2]#global.callpool.stack.<stack-serial-number>.frame.<frame-serial-number> +frame=0x7fc47a611dbc #Frame address +ref_count=0 #Incremented at the time of wind and decremented at the time of unwind. +translator=r2-client-1 #Xlator this frame belongs to +complete=0 #if this value is 1 that means this frame is already unwound. 0 if it is yet to unwind. +parent=r2-replicate-0 #Parent xlator of this frame +wind_from=afr_lookup #Parent xlator function from which the wind happened +wind_to=priv->children[i]->fops->lookup +unwind_to=afr_lookup_cbk #Parent xlator function to which unwind happened +``` + +### History of operations in Fuse + +Fuse maintains history of operations that happened in fuse. + +``` +[xlator.mount.fuse.history] +TIME=2014-07-09 16:44:57.523364 +message=[0] fuse_release: RELEASE(): 4590:, fd: 0x1fef0d8, gfid: 3afb4968-5100-478d-91e9-76264e634c9f + +TIME=2014-07-09 16:44:57.523373 +message=[0] send_fuse_err: Sending Success for operation 18 on inode 3afb4968-5100-478d-91e9-76264e634c9f + +TIME=2014-07-09 16:44:57.523394 +message=[0] fuse_getattr_resume: 4591, STAT, path: (/iozone.tmp), gfid: (3afb4968-5100-478d-91e9-76264e634c9f) +``` + +### Xlator configuration +``` +[cluster/replicate.r2-replicate-0] #Xlator type, name information +child_count=2 #Number of children to the xlator +#Xlator specific configuration below +child_up[0]=1 +pending_key[0]=trusted.afr.r2-client-0 +child_up[1]=1 +pending_key[1]=trusted.afr.r2-client-1 +data_self_heal=on +metadata_self_heal=1 +entry_self_heal=1 +data_change_log=1 +metadata_change_log=1 +entry-change_log=1 +read_child=1 +favorite_child=-1 +wait_count=1 +``` + +### Graph/inode table +``` +[active graph - 1] + +conn.1.bound_xl./data/brick01a/homegfs.hashsize=14057 +conn.1.bound_xl./data/brick01a/homegfs.name=/data/brick01a/homegfs/inode +conn.1.bound_xl./data/brick01a/homegfs.lru_limit=16384 #Least recently used size limit +conn.1.bound_xl./data/brick01a/homegfs.active_size=690 #Number of inodes undergoing some kind of fop to be precise on which there is at least one ref. +conn.1.bound_xl./data/brick01a/homegfs.lru_size=183 #Number of inodes present in lru list +conn.1.bound_xl./data/brick01a/homegfs.purge_size=0 #Number of inodes present in purge list +``` + +### Inode +``` +[conn.1.bound_xl./data/brick01a/homegfs.active.324] #324th inode in active inode list +gfid=e6d337cf-97eb-44b3-9492-379ba3f6ad42 #Gfid of the inode +nlookup=13 #Number of times lookups happened from the client or from fuse kernel +fd-count=4 #Number of fds opened on the inode +ref=11 #Number of refs taken on the inode +ia_type=1 #Type of the inode. This should be changed to some string :-( +Ref by xl:.patchy-md-cache=11 #Further this there will be a list of xlators, and the ref count taken by each of them on this inode at the time of statedump + +[conn.1.bound_xl./data/brick01a/homegfs.lru.1] #1st inode in lru list. Note that ref count is zero for these inodes. +gfid=5114574e-69bc-412b-9e52-f13ff087c6fc +nlookup=5 +fd-count=0 +ref=0 +ia_type=2 +Ref by xl:.fuse=1 +Ref by xl:.patchy-client-0=-1 +``` +### Inode context +For each inode per xlator some context could be stored. This context can also be printed in the statedump. Here is the inode ctx of locks xlator +``` +[xlator.features.locks.homegfs-locks.inode] +path=/homegfs/users/dfrobins/gfstest/r4/SCRATCH/fort.5102 - path of the file +mandatory=0 +inodelk-count=5 #Number of inode locks +lock-dump.domain.domain=homegfs-replicate-0:self-heal #Domain name where self-heals take locks to prevent more than one heal on the same file +inodelk.inodelk[0](ACTIVE)=type=WRITE, whence=0, start=0, len=0, pid = 18446744073709551615, owner=080b1ada117f0000, client=0xb7fc30, connection-id=compute-30-029.com-3505-2014/06/29-14:46:12:477358-homegfs-client-0-0-1, granted at Sun Jun 29 11:01:00 2014 #Active lock information + +inodelk.inodelk[1](BLOCKED)=type=WRITE, whence=0, start=0, len=0, pid = 18446744073709551615, owner=c0cb091a277f0000, client=0xad4f10, connection-id=gfs01a.com-4080-2014/06/29-14:41:36:917768-homegfs-client-0-0-0, blocked at Sun Jun 29 11:04:44 2014 #Blocked lock information + +lock-dump.domain.domain=homegfs-replicate-0:metadata #Domain name where metadata operations take locks to maintain replication consistency +lock-dump.domain.domain=homegfs-replicate-0 #Domain name where entry/data operations take locks to maintain replication consistency +inodelk.inodelk[0](ACTIVE)=type=WRITE, whence=0, start=11141120, len=131072, pid = 18446744073709551615, owner=080b1ada117f0000, client=0xb7fc30, connection-id=compute-30-029.com-3505-2014/06/29-14:46:12:477358-homegfs-client-0-0-1, granted at Sun Jun 29 11:10:36 2014 #Active lock information +``` + +## FAQ +### How to debug Memory leaks using statedump? + +#### Using memory accounting feature: + +[Bug 1120151](https://bugzilla.redhat.com/show_bug.cgi?id=1120151) is one of the bugs which was debugged using statedump to see which data-structure is leaking. Here is the process used to find what the leak is using statedump. According to the bug the observation is that the process memory usage is increasing whenever one of the bricks is wiped in a replicate volume and a `full` self-heal is invoked to heal the contents. Statedump of the process is taken using `kill -USR1 <pid-of-gluster-self-heal-daemon>`. +``` +grep -w num_allocs glusterdump.5225.dump.1405493251 +num_allocs=77078 +num_allocs=87070 +num_allocs=117376 +.... + +grep hot-count glusterdump.5225.dump.1405493251 +hot-count=16384 +hot-count=16384 +hot-count=4095 +.... + +Find the occurrences in the statedump file to figure out the tags. +``` +grep of the statedump revealed too many allocations for the following data-types under replicate, + +1. gf_common_mt_asprintf +2. gf_common_mt_char +3. gf_common_mt_mem_pool. + +After checking afr-code for allocations with tag `gf_common_mt_char` found `data-self-heal` code path does not free one such allocated memory. `gf_common_mt_mem_pool` suggests that there is a leak in pool memory. `replicate-0:dict_t`, `glusterfs:data_t` and `glusterfs:data_pair_t` pools are using lot of memory, i.e. cold_count is `0` and too many allocations. Checking source code of dict.c revealed that `key` in `dict` is allocated with `gf_common_mt_char` i.e. `2.` tag and value is created using gf_asprintf which in-turn uses `gf_common_mt_asprintf` i.e. `1.`. Browsing the code for leak in self-heal code paths lead to a line which over-writes a variable with new dictionary even when it was already holding a reference to another dictionary. After fixing these leaks, ran the same test to verify that none of the `num_allocs` are increasing even after healing 10,000 files directory hierarchy in statedump of self-heal daemon. +Please check this [patch](http://review.gluster.org/8316) for more info about the fix. + +#### Debugging leaks in memory pools: +Statedump output of memory pools was used to test and verify the fixes to [Bug 1134221](https://bugzilla.redhat.com/show_bug.cgi?id=1134221). On code analysis, dict_t objects were found to be leaking (in terms of not being unref'd enough number of times, during name self-heal. The test involved creating 100 files on plain replicate volume, removing them from one of the brick's backend, and then triggering lookup on them from the mount point. Statedump of the mount process was taken before executing the test case and after it, after compiling glusterfs with -DDEBUG flags (to have cold count set to 0 by default). + +Statedump output of the fuse mount process before the test case was executed: + +``` + +pool-name=glusterfs:dict_t +hot-count=0 +cold-count=0 +padded_sizeof=140 +alloc-count=33 +max-alloc=0 +pool-misses=33 +cur-stdalloc=14 +max-stdalloc=18 + +``` +Statedump output of the fuse mount process after the test case was executed: + +``` + +pool-name=glusterfs:dict_t +hot-count=0 +cold-count=0 +padded_sizeof=140 +alloc-count=2841 +max-alloc=0 +pool-misses=2841 +cur-stdalloc=214 +max-stdalloc=220 + +``` +Here, with cold count being 0 by default, `cur-stdalloc` indicated the number of `dict_t` objects that were allocated in heap using `mem_get()`, and yet to be freed using `mem_put()` (refer to this [page](../developer-guide/datastructure-mem-pool.md) for more details on how mempool works). After the test case (name selfheal of 100 files), there was a rise in the cur-stdalloc value (from 14 to 214) for `dict_t`. + +After these leaks were fixed, glusterfs was again compiled with -DDEBUG flags, and the same steps were performed again and statedump was taken before and after executing the test case, of the mount. This was done to ascertain the validity of the fix. And the following are the results: + +Statedump output of the fuse mount process before executing the test case: + +``` +pool-name=glusterfs:dict_t +hot-count=0 +cold-count=0 +padded_sizeof=140 +alloc-count=33 +max-alloc=0 +pool-misses=33 +cur-stdalloc=14 +max-stdalloc=18 + +``` +Statedump output of the fuse mount process after executing the test case: + +``` +pool-name=glusterfs:dict_t +hot-count=0 +cold-count=0 +padded_sizeof=140 +alloc-count=2837 +max-alloc=0 +pool-misses=2837 +cur-stdalloc=14 +max-stdalloc=119 + +``` +The value of cur-stdalloc remained 14 before and after the test, indicating that the fix indeed does what it's supposed to do. + +### How to debug hangs because of frame-loss? +[Bug 994959](https://bugzilla.redhat.com/show_bug.cgi?id=994959) is one of the bugs where statedump was helpful in finding where the frame was lost. Here is the process used to find where the hang is using statedump. +When the hang was observed, statedumps are taken for all the processes. On mount's statedump the following stack is shown: +``` +[global.callpool.stack.1.frame.1] +ref_count=1 +translator=fuse +complete=0 + +[global.callpool.stack.1.frame.2] +ref_count=0 +translator=r2-client-1 +complete=1 <<----- Client xlator completed the readdirp call and unwound to afr +parent=r2-replicate-0 +wind_from=afr_do_readdir +wind_to=children[call_child]->fops->readdirp +unwind_from=client3_3_readdirp_cbk +unwind_to=afr_readdirp_cbk + +[global.callpool.stack.1.frame.3] +ref_count=0 +translator=r2-replicate-0 +complete=0 <<---- Afr xlator is not unwinding for some reason. +parent=r2-dht +wind_from=dht_do_readdir +wind_to=xvol->fops->readdirp +unwind_to=dht_readdirp_cbk + +[global.callpool.stack.1.frame.4] +ref_count=1 +translator=r2-dht +complete=0 +parent=r2-io-cache +wind_from=ioc_readdirp +wind_to=FIRST_CHILD(this)->fops->readdirp +unwind_to=ioc_readdirp_cbk + +[global.callpool.stack.1.frame.5] +ref_count=1 +translator=r2-io-cache +complete=0 +parent=r2-quick-read +wind_from=qr_readdirp +wind_to=FIRST_CHILD (this)->fops->readdirp +unwind_to=qr_readdirp_cbk + +``` +`unwind_to` shows that call was unwound to `afr_readdirp_cbk` from client xlator. +Inspecting that function revealed that afr is not unwinding the stack when fop failed. +Check this [patch](http://review.gluster.org/5531) for more info about the fix. diff --git a/doc/developer-guide/Language-Bindings.md b/doc/developer-guide/Language-Bindings.md new file mode 100644 index 00000000000..951f5fae2f6 --- /dev/null +++ b/doc/developer-guide/Language-Bindings.md @@ -0,0 +1,45 @@ +# Language Bindings +GlusterFS 3.4 introduced the libgfapi client API for C programs. This +page lists bindings to the libgfapi C library from other languages. + +Go +-- + +- [gogfapi](https://github.com/gluster/gogfapi) - Go language bindings + for libgfapi, aiming to provide an api consistent with the default + Go file apis. + +Java +---- + +- [libgfapi-jni](https://github.com/semiosis/libgfapi-jni/) - Low + level JNI binding for libgfapi +- [glusterfs-java-filesystem](https://github.com/semiosis/glusterfs-java-filesystem) + - High level NIO.2 FileSystem Provider implementation for the Java + platform +- [libgfapi-java-io](https://github.com/gluster/libgfapi-java-io) - + Java bindings for libgfapi, similar to java.io + +Python +------ + +- [libgfapi-python](https://github.com/gluster/libgfapi-python) - + Libgfapi bindings for Python + +Ruby +---- + +- [libgfapi-ruby](https://github.com/spajus/libgfapi-ruby) - Libgfapi + bindings for Ruby using FFI + +Rust +---- + +- [gfapi-sys](https://github.com/cholcombe973/Gfapi-sys) - Libgfapi + bindings for Rust using FFI + +Perl +---- + +- [libgfapi-perl](https://github.com/gluster/libgfapi-perl) - Libgfapi + bindings for Perl using FFI diff --git a/doc/developer-guide/README.md b/doc/developer-guide/README.md new file mode 100644 index 00000000000..aaf9c7476b0 --- /dev/null +++ b/doc/developer-guide/README.md @@ -0,0 +1,81 @@ +Developers +========== + +### From GlusterDocumentation + +Contributing to the Gluster community +------------------------------------- + +Are you itching to send in patches and participate as a developer in the +Gluster community? Here are a number of starting points for getting +involved. We don't require a signed contributor license agreement or +copyright assignment, but we do require a "signed-off-by" line on each +code check-in. + +- [License + Change](http://www.gluster.org/2012/05/glusterfs-license-change/) - + we recently changed the client library code to a dual license under + the GPL v2 and the LGPL v3 or later +- [GlusterFS Coding Standards](./coding-standard.md) + +- If you are not sure of where to start, and what to do, we have a small + write-up on what you can pick. [Check it out](./options-to-contribute.md) + + +Adding File operations +---------------------- + +- [Steps to be followed when adding a new FOP to GlusterFS ](./adding-fops.md) + +Automatic File Replication +-------------------------- + +- [Cluster/afr translator](./afr.md) +- [History of Locking in AFR](./afr-locks-evolution.md) +- [Self heal Daemon](./afr-self-heal-daemon.md) + +Data Structures +--------------- + +- [inode data structure](./datastructure-inode.md) +- [iobuf data structure](./datastructure-iobuf.md) +- [mem-pool data structure](./datastructure-mem-pool.md) + +Find the gfapi symbol versions [here](./gfapi-symbol-versions.md) + +Daemon Management Framework +--------------------------- + +- [How to introduce new daemons using daemon management framework](./daemon-management-framework.md) + +Translators +----------- + +- [Performance/write-Behind Translator](./write-behind.md) +- [Translator Development](./translator-development.md) +- [Storage/posix Translator](./posix.md) + + +Brick multiplex +--------------- + +- [Brick mux resource reduction](./brickmux-thread-reduction.md) + +Fuse +---- + +- [Interrupt handling](./fuse-interrupt.md) + +Testing/Debugging +----------------- + +- [Unit Tests in GlusterFS](./unittest.md) +- [Using the Gluster Test + Framework](./Using-Gluster-Test-Framework.md) - Step by + step instructions for running the Gluster Test Framework +- [Coredump Analysis](../debugging/analyzing-regression-cores.md) - Steps to analize coredumps generated by regression machines. +- [Identifying Resource Leaks](./identifying-resource-leaks.md) + +Release Process +--------------- +- [Versioning](./versioning.md) diff --git a/doc/developer-guide/Using-Gluster-Test-Framework.md b/doc/developer-guide/Using-Gluster-Test-Framework.md new file mode 100644 index 00000000000..d2bb1c391da --- /dev/null +++ b/doc/developer-guide/Using-Gluster-Test-Framework.md @@ -0,0 +1,271 @@ +# USing Gluster Test Framwork +Description +----------- + +The Gluster Test Framework, is a suite of scripts used for regression +testing of Gluster. + +It runs well on RHEL and CentOS (possibly Fedora too, presently being +tested), and is automatically run against every patch submitted to +Gluster [for review](http://review.gluster.org). + +The Gluster Test Framework is part of the main Gluster code base, living +under the "tests" subdirectory: + + http://git.gluster.org/?p=glusterfs.git;a=summary + +WARNING +------- + +Running the Gluster Test Framework deletes “/var/lib/glusterd/\*”. + +**DO NOT run it on a server with any data.** + +Preparation steps for Ubuntu 14.04 LTS +-------------------------------------- + +1. \# apt-get install dbench git libacl1-dev mock nfs-common +nfs-kernel-server libtest-harness-perl libyajl-dev xfsprogs psmisc attr +acl lvm2 rpm + +2. \# apt-get install python-webob python-paste python-sphinx + +3. \# apt-get install autoconf automake bison dos2unix flex libfuse-dev +libaio-dev libibverbs-dev librdmacm-dev libtool libxml2-dev +libxml2-utils liblvm2-dev make libssl-dev pkg-config libpython-dev +python-eventlet python-netifaces python-simplejson python-pyxattr +libreadline-dev tar + +4) Install cmockery2 from github (https://github.com/lpabon/cmockery2) +and compile and make install as in Readme + +5) + + sudo groupadd mock + sudo useradd -g mock mock + +6) mkdir /var/run/gluster + +**Note**: redhat-rpm-config package is not found in ubuntu + +Preparation steps for CentOS 7 (only) +------------------------------------- + +1. Install EPEL: + + $ sudo yum install -y http://epel.mirror.net.in/epel/7/x86_64/e/epel-release-7-1.noarch.rpm + +2. Install the CentOS 7.x dependencies: + + $ sudo yum install -y --enablerepo=epel cmockery2-devel dbench git libacl-devel mock nfs-utils perl-Test-Harness yajl xfsprogs psmisc + + $ sudo yum install -y --enablerepo=epel python-webob1.0 python-paste-deploy1.5 python-sphinx10 redhat-rpm-config + +==\> Despite below missing packages it worked for me + + No package python-webob1.0 available. + No package python-paste-deploy1.5 available. + No package python-sphinx10 available. + + $ sudo yum install -y --enablerepo=epel autoconf automake bison dos2unix flex fuse-devel libaio-devel libibverbs-devel \ + librdmacm-devel libtool libxml2-devel lvm2-devel make openssl-devel pkgconfig \ + python-devel python-eventlet python-netifaces python-paste-deploy \ + python-simplejson python-sphinx python-webob pyxattr readline-devel rpm-build \ + tar + +3. Create the mock user + + $ sudo useradd -g mock mock + +Preparation steps for CentOS 6.3+ (only) +---------------------------------------- + +1. Install EPEL: + + $ sudo yum install -y http://download.fedoraproject.org/pub/epel/6/x86_64/epel-release-6-8.noarch.rpm + +2. Install the CentOS 6.x dependencies: + + $ sudo yum install -y --enablerepo=epel cmockery2-devel dbench git libacl-devel mock nfs-utils perl-Test-Harness yajl xfsprogs + $ sudo yum install -y --enablerepo=epel python-webob1.0 python-paste-deploy1.5 python-sphinx10 redhat-rpm-config + $ sudo yum install -y --enablerepo=epel autoconf automake bison dos2unix flex fuse-devel libaio-devel libibverbs-devel \ + librdmacm-devel libtool libxml2-devel lvm2-devel make openssl-devel pkgconfig \ + python-devel python-eventlet python-netifaces python-paste-deploy \ + python-simplejson python-sphinx python-webob pyxattr readline-devel rpm-build \ + tar + +3. Create the mock user + + $ sudo useradd -g mock mock + +Preparation steps for RHEL 6.3+ (only) +-------------------------------------- + +1. Ensure you have the "Scalable Filesystem Support" group installed + +This provides the xfsprogs package, which is required by the test +framework. + +2. Install EPEL: + + $ sudo yum install -y http://download.fedoraproject.org/pub/epel/6/x86_64/epel-release-6-8.noarch.rpm + +3. Install the CentOS 6.x dependencies: + + $ sudo yum install -y --enablerepo=epel cmockery2-devel dbench git libacl-devel mock nfs-utils yajl perl-Test-Harness + $ sudo yum install -y --enablerepo=rhel-6-server-optional-rpms python-webob1.0 python-paste-deploy1.5 python-sphinx10 redhat-rpm-config + $ sudo yum install -y --disablerepo=rhs* --enablerepo=*optional-rpms autoconf \ + automake bison dos2unix flex fuse-devel libaio-devel libibverbs-devel \ + librdmacm-devel libtool libxml2-devel lvm2-devel make openssl-devel pkgconfig \ + python-devel python-eventlet python-netifaces python-paste-deploy \ + python-simplejson python-sphinx python-webob pyxattr readline-devel rpm-build \ + tar + +4. Create the mock user + + $ sudo useradd -g mock mock + +Preparation steps for Fedora 16-19 (only) +----------------------------------------- + +**Still in development** + +1. Install the Fedora dependencies: + + $ sudo yum install -y attr cmockery2-devel dbench git mock nfs-utils perl-Test-Harness psmisc xfsprogs + $ sudo yum install -y python-webob1.0 python-paste-deploy1.5 python-sphinx10 redhat-rpm-config + $ sudo yum install -y autoconf automake bison dos2unix flex fuse-devel libaio-devel libibverbs-devel \ + librdmacm-devel libtool libxml2-devel lvm2-devel make openssl-devel pkgconfig \ + python-devel python-eventlet python-netifaces python-paste-deploy \ + python-simplejson python-sphinx python-webob pyxattr readline-devel rpm-build \ + tar + +3. Create the mock user + + $ sudo useradd -g mock mock + +Common steps +------------ + +1. Ensure DNS for your server is working + +The Gluster Test Framework fails miserably if the full domain name for +your server doesn't resolve back to itself. + +If you don't have a working DNS infrastructure in place, adding an entry +for your server to its /etc/hosts file will work. + +2. Install the version of Gluster you are testing + +Either install an existing set of rpms: + + $ sudo yum install [your gluster rpms here] + +Or compile your own ones (fairly easy): + + http://www.gluster.org/community/documentation/index.php/CompilingRPMS + +3. Clone the GlusterFS git repository + + $ git clone git://git.gluster.org/glusterfs + $ cd glusterfs + +Ensure mock can access the directory +------------------------------------ + +Some tests run as the user "mock". If the mock user can't access the +tests subdirectory directory, these tests fail. (rpm.t is one such test) + +This is a known gotcha when the git repo is cloned to your home +directory. Home directories generally don't have world readable +permissions. You can fix this by adjusting your home directory +permissions, or placing the git repo somewhere else (with access for the +mock user). + +Running the tests +----------------- + +The tests need to run as root, so they can mount volumes and manage +gluster processes as needed. + +It's also best to run them directly as the root user, instead of through +sudo. Strange things sporadicly happen (for me) when using the full test +framework through sudo, that haven't happened (yet) when running +directly as root. Hangs in dbench particularly, which are part of at +least one test. + + # ./run-tests.sh + +The test framework takes just over 45 minutes to run in a VM here (4 +cpu's assigned, 8GB ram, SSD storage). It may take significantly more or +less time for you, depending on the hardware and software you're using. + +Showing debug information +------------------------- + +To display verbose information while the tests are running, set the +DEBUG environment variable to 1 prior to running the tests. + + # DEBUG=1 ./run-tests.sh + +Log files +--------- + +Verbose output from the rpm.t test goes into "rpmbuild-mock.log", +located in the same directory the test is run from. + +Reporting bugs +-------------- + +If you hit a bug when running the test framework, **please** create a +bug report for it on Bugzilla so it gets fixed: + + https://bugzilla.redhat.com/enter_bug.cgi?product=GlusterFS&component=tests + +Creating your own tests +----------------------- + +The test scripts are written in bash, with their filenames ending in .t +instead of .sh. + +When creating your own test scripts, create them in an appropriate +subdirectory under "tests" (eg "bugs" or "features") and use descriptive +names like "bug-XXXXXXX-checking-feature-X.t" + +Also include the "include.rc" file, which defines the test types and +host/brick/volume defaults: + + . $(dirname $0)/../include.rc + +There are 5 test types available at present, but feel free to add more +if you need something that doesn't yet exist. The test types are +explained in more detail below. + +Also essential is the "cleanup" command, which removes any existing +Gluster configuration (**without backing it up**), and also kills any +running gluster processes. + +There is a basic test template you can copy, named bug-000000.t in the +bugs subdirectory: + + $ cp bugs/bug-000000.t somedir/descriptive-name.t + +### TEST + +- Example of usage in basic/volume.t + +### TEST\_IN\_LOOP + +- Example of usage in basic/rpm.t + +### EXPECT + +- Example of usage in basic/volume.t + +### EXPECT\_WITHIN + +- Example of usage in basic/volume-status.t + +### EXPECT\_KEYWORD + +- Defined in include.rc, but seems to be unused? diff --git a/doc/developer-guide/adding-fops.md b/doc/developer-guide/adding-fops.md new file mode 100644 index 00000000000..3f72ed3e23a --- /dev/null +++ b/doc/developer-guide/adding-fops.md @@ -0,0 +1,18 @@ +Adding a new FOP +================ + +Steps to be followed when adding a new FOP to GlusterFS: + +1. Edit `glusterfs.h` and add a `GF_FOP_*` constant. +2. Edit `xlator.[ch]` and: + * add the new prototype for fop and callback. + * edit `xlator_fops` structure. +3. Edit `xlator.c` and add to fill_defaults. +4. Edit `protocol.h` and add struct necessary for the new FOP. +5. Edit `defaults.[ch]` and provide default implementation. +6. Edit `call-stub.[ch]` and provide stub implementation. +7. Edit `common-utils.c` and add to gf_global_variable_init(). +8. Edit client-protocol and add your FOP. +9. Edit server-protocol and add your FOP. +10. Implement your FOP in any translator for which the default implementation + is not sufficient. diff --git a/doc/developer-guide/afr-locks-evolution.md b/doc/developer-guide/afr-locks-evolution.md new file mode 100644 index 00000000000..2dabbcfeb13 --- /dev/null +++ b/doc/developer-guide/afr-locks-evolution.md @@ -0,0 +1,91 @@ +History of locking in AFR +-------------------------- + +GlusterFS has **locks** translator which provides the following internal locking operations called `inodelk`, `entrylk` which are used by afr to achieve synchronization of operations on files or directories that conflict with each other. + +`Inodelk` gives the facility for translators in GlusterFS to obtain range (denoted by tuple with **offset**, **length**) locks in a given **domain** for an inode. +Full file lock is denoted by the tuple (offset: `0`, length: `0`) i.e. length `0` is considered as infinity. + +`Entrylk` enables translators of GlusterFS to obtain locks on `name` in a given **domain** for an inode, typically a directory. + +The **locks** translator provides both *blocking* and *nonblocking* variants and of these locks. + + +AFR makes use of locks xlator extensively: + +1)For FOPS (from clients) +----------------------- +* Data transactions take inode locks on data domain, Let's refer to this domain name as DATA_DOMAIN. + + So locking for writes would be something like this:`inodelk(offset,length, DATA_DOMAIN)` + For truncating a file to zero, it would be `inodelk(0,0,DATA_DOMAIN)` + +* Metadata transactions (chown/chmod) also take inode locks but on a special range on metadata domain, + i.e.`(LLONG_MAX-1 , 0, METADATA_DOMAIN).` + +* Entry transactions (create, mkdir, rmdir,unlink, symlink, link,rename) take entrylk on `(name, parent inode)`. + + +2)For self heal: +------------- +* For Metadata self-heal, it is the same. i.e.`inodelk(LLONG_MAX-1 , 0, METADATA_DOMAIN)`. +* For Entry self-heal, it is `entrylk(NULL name, parent inode)`. Specifying NULL for the name takes full lock on the directory referred to by the inode. +* For data self-heal, there is a bit of history as to how locks evolved: + +### Initial version (say version 1) : +There was no concept of selfheal daemon (shd). Only client lookups triggered heals. so AFR always took `inodelk(0,0,DATA_DOMAIN)` for healing. The issue with this approach was that when heal was in progress, I/O from clients was blocked . + +### version 2: +shd was introduced. We needed to allow I/O to go through when heal was going,provided the ranges did not overlap. To that extent, the following approach was adopted: + ++ 1.shd takes (full inodelk in DATA_DOMAIN). Thus client FOPS are blocked and cannot modify changelog-xattrs ++ 2.shd inspects xattrs to determine source/sink ++ 3.shd takes a chunk inodelk(0-128kb) again in DATA_DOMAIN (locks xlator allows overlapping locks if lock owner is the same). ++ 4.unlock full lock ++ 5.heal ++ 6.take next chunk lock(129-256kb) ++ 7.unlock 1st chunk lock, heal the second chunk and so on. + + +Thus after 4, any client FOP could write to regions that was not currently under heal. The exception was truncate (to size 0) because it needs full file lock and will always block because some chunk is always under lock by the shd until heal completes. + +Another issue was that 2 shds could run in parallel. Say SHD1 and SHD2 compete for step 1. Let SHD1 win. It proceeds and completes step 4. Now SHD2 also succeeds in step 1, continues all steps. Thus at the end both shds will decrement the changelog leading to negative values in it) + +### version 3 +To prevent parallel self heals, another domain was introduced, let us call it SELF_HEAL_DOMAIN. With this domain, the following approach was adopted and is **the approach currently in use**: + ++ 1.shd takes (full inodelk on SELF_HEAL_DOMAIN) ++ 2.shd takes (full inodelk on DATA_DOMAIN) ++ 3.shd inspects xattrs to determine source/sink ++ 4.unlock full lock on DATA_DOMAIN ++ 5.take chunk lock(0-128kb) on DATA_DOMAIN ++ 6.heal ++ 7.take next chunk lock(129-256kb) on DATA_DOMAIN ++ 8.unlock 1st chunk lock, heal and so on. ++ 9.Finally release full lock on SELF_HEAL_DOMAIN + +Thus until one shd completes step 9, another shd cannot start step 1, solving the problem of simultaneous heals. +Note that the issue of truncate (to zero) FOP hanging still remains. +Also there are multiple network calls involved in this scheme. (lock,heal(ie read+write), unlock) per chunk. i.e 4 calls per chunk. + +### version 4 (ToDo) +Some improvements that need to be made in version 3: +* Reduce network calls using piggy backing. +* After taking chunk lock and healing, we need to unlock the lock before locking the next chunk. This gives a window for any pending truncate FOPs to succeed. If truncate succeeds, the heal of next chunk will fail (read returns zero) +and heal is stopped. *BUT* there is **yet another** issue: + +* shd does steps 1 to 4. Let's assume source is brick b1, sink is brick b2 . i.e xattrs are (0,1) and (0,0) on b1 and b2 respectively. Now before shd takes (0-128kb) lock, a client FOP takes it. +It modifies data but the FOP succeeds only on brick 2. writev returns success, and the attrs now read (0,1) (1,0). SHD takes over and heals. It had observed (0,1),(0,0) earlier +and thus goes ahead and copies stale 128Kb from brick 1 to brick2. Thus as far as application is concerned, `writev` returned success but bricks have stale data. +What needs to be done is `writev` must return success only if it succeeded on atleast one source brick (brick b1 in this case). Otherwise The heal still happens in reverse direction but as far as the application is concerned, it received an error. + +### Note on lock **domains** +We have used conceptual names in this document like DATA_DOMAIN/ METADATA_DOMAIN/ SELF_HEAL_DOMAIN. In the code, these are mapped to strings that are based on the AFR xlator name like so: + +DATA_DOMAIN --->"vol_name-replicate-n" + +METADATA_DOMAIN --->"vol_name-replicate-n:metadata" + +SELF_HEAL_DOMAIN -->"vol_name-replicate-n:self-heal" + +where vol_name is the name of the volume and 'n' is the replica subvolume index (starting from 0). diff --git a/doc/developer-guide/afr-self-heal-daemon.md b/doc/developer-guide/afr-self-heal-daemon.md new file mode 100644 index 00000000000..65940d420b7 --- /dev/null +++ b/doc/developer-guide/afr-self-heal-daemon.md @@ -0,0 +1,92 @@ +Self-Heal Daemon +================ +The self-heal daemon (shd) is a glusterfs process that is responsible for healing files in a replicate/ disperse gluster volume. +Every server (brick) node of the volume runs one instance of the shd. So even if one node contains replicate/ disperse bricks of +multiple volumes, it would be healed by the same shd. + +This document only describes how the shd works for replicate (AFR) volumes. + +The shd is launched by glusterd when the volume starts (only if the volume includes a replicate configuration). The graph +of the shd process in every node contains the following: The io-stats which is the top most xlator, its children being the +replicate xlators (subvolumes) of *only* the bricks present in that particular node, and finally *all* the client xlators that are the children of the replicate xlators. + +The shd does two types of self-heal crawls: Index heal and Full heal. For both these types of crawls, the basic idea is the same: +For each file encountered while crawling, perform metadata, data and entry heals under appropriate locks. +* An overview of how each of these heals is performed is detailed in the 'Self-healing' section of *doc/features/afr-v1.md* +* The different file locks which the shd takes for each of these heals is detailed in *doc/developer +-guide/afr-locks-evolution.md* + +Metadata heal refers to healing extended attributes, mode and permissions of a file or directory. +Data heal refers to healing the file contents. +Entry self-heal refers to healing entries inside a directory. + +Index heal +========== +The index heal is done: + a) Every 600 seconds (can be changed via the `cluster.heal-timeout` volume option) + b) When it is explicitly triggered via the `gluster vol heal <VOLNAME>` command + c) Whenever a replica brick that was down comes back up. + +Only one heal can be in progress at one time, irrespective of reason why it was triggered. If another heal is triggered before the first one completes, it will be queued. +Only one heal can be queued while the first one is running. If an Index heal is queued, it can be overridden by queuing a Full heal and not vice-versa. Also, before processing +each entry in index heal, a check is made if a full heal is queued. If it is, then the index heal is aborted so that the full heal can proceed. + +In index heal, each shd reads the entries present inside .glusterfs/indices/xattrop/ folder and triggers heal on each entry with appropriate locks. +The .glusterfs/indices/xattrop/ directory contains a base entry of the name "xattrop-<virtual-gfid-string>". All other entries are hardlinks to the base entry. The +*names* of the hardlinks are the gfid strings of the files that may need heal. + +When a client (mount) performs an operation on the file, the index xlator present in each brick process adds the hardlinks in the pre-op phase of the FOP's transaction +and removes it in post-op phase if the operation is successful. Thus if an entry is present inside the .glusterfs/indices/xattrop/ directory when there is no I/O +happening on the file, it means the file needs healing (or atleast an examination if the brick crashed after the post-op completed but just before the removal of the hardlink). + +#### Index heal steps: +<pre><code> +In shd process of *each node* { + opendir +readdir (.glusterfs/indices/xattrop/) + for each entry inside it { + self_heal_entry() //Explained below. + } +} +</code></pre> + +<pre><code> +self_heal_entry() { + Call syncop_lookup(replicae subvolume) which eventually does { + take appropriate locks + determine source and sinks from AFR changelog xattrs + perform whatever heal is needed (any of metadata, data and entry heal in that order) + clear changelog xattrs and hardlink inside .glusterfs/indices/xattrop/ + } +} +</code></pre> + +Note: +* If the gfid hardlink is present in the .glusterfs/indices/xattrop/ of both replica bricks, then each shd will try to heal the file but only one of them will be able to proceed due to the self-heal domain lock. + +* While processing entries inside .glusterfs/indices/xattrop/, if shd encounters an entry whose parent is yet to be healed, it will skip it and it will be picked up in the next crawl. + +* If a file is in data/ metadata split-brain, it will not be healed. + +* If a directory is in entry split-brain, a conservative merge will be performed, wherein after the merge, the entries of the directory will be a union of the entries in the replica pairs. + +Full heal +========= +A full heal is triggered by running `gluster vol heal <VOLNAME> full`. This command is usually run in disk replacement scenarios where the entire data is to be copied from one of the healthy bricks of the replica to the brick that was just replaced. + +Unlike the index heal which runs on the shd of every node in a replicate subvolume, the full heal is run only on the shd of one node per replicate subvolume: the node having the highest UUID. +i.e In a 2x2 volume made of 4 nodes N1, N2, N3 and N4, If UUID of N1>N2 and UUID N4 >N3, then the full crawl is carried out by the shds of N1 and N4.(Node UUID can be found in `/var/lib/glusterd/glusterd.info`) + +The full heal steps are almost identical to the index heal, except the heal is performed on each replica starting from the root of the volume: +<pre><code> +In shd process of *highest UUID node per replica* { + opendir +readdir ("/") + for each entry inside it { + self_heal_entry() + if (entry == directory) { + /* Recurse*/ + again opendir+readdir (directory) followed by self_heal_entry() of each entry. + } + + } +} +</code></pre> diff --git a/doc/developer-guide/afr.md b/doc/developer-guide/afr.md new file mode 100644 index 00000000000..566573a4e26 --- /dev/null +++ b/doc/developer-guide/afr.md @@ -0,0 +1,191 @@ +cluster/afr translator +====================== + +Locking +------- + +Before understanding replicate, one must understand two internal FOPs: + +### `GF_FILE_LK` + +This is exactly like `fcntl(2)` locking, except the locks are in a +separate domain from locks held by applications. + +### `GF_DIR_LK (loc_t *loc, char *basename)` + +This allows one to lock a name under a directory. For example, +to lock /mnt/glusterfs/foo, one would use the call: + +``` +GF_DIR_LK ({loc_t for "/mnt/glusterfs"}, "foo") +``` + +If one wishes to lock *all* the names under a particular directory, +supply the basename argument as `NULL`. + +The locks can either be read locks or write locks; consult the +function prototype for more details. + +Both these operations are implemented by the features/locks (earlier +known as posix-locks) translator. + +Basic design +------------ + +All FOPs can be classified into four major groups: + +### inode-read + +Operations that read an inode's data (file contents) or metadata (perms, etc.). + +access, getxattr, fstat, readlink, readv, stat. + +### inode-write + +Operations that modify an inode's data or metadata. + +chmod, chown, truncate, writev, utimens. + +### dir-read + +Operations that read a directory's contents or metadata. + +readdir, getdents, checksum. + +### dir-write + +Operations that modify a directory's contents or metadata. + +create, link, mkdir, mknod, rename, rmdir, symlink, unlink. + +Some of these make a subgroup in that they modify *two* different entries: +link, rename, symlink. + +### Others + +Other operations. + +flush, lookup, open, opendir, statfs. + +Algorithms +---------- + +Each of the four major groups has its own algorithm: + +### inode-read, dir-read + +1. Send a request to the first child that is up: + * if it fails: + * try the next available child + * if we have exhausted all children: + * return failure + +### inode-write + + All operations are done in parallel unless specified otherwise. + +1. Send a ``GF_FILE_LK`` request on all children for a write lock on the + appropriate region + (for metadata operations: entire file (0, 0) for writev: + (offset, offset+size of buffer)) + * If a lock request fails on a child: + * unlock all children + * try to acquire a blocking lock (`F_SETLKW`) on each child, serially. + If this fails (due to `ENOTCONN` or `EINVAL`): + Consider this child as dead for rest of transaction. +2. Mark all children as "pending" on all (alive) children (see below for +meaning of "pending"). + * If it fails on any child: + * mark it as dead (in transaction local state). +3. Perform operation on all (alive) children. + * If it fails on any child: + * mark it as dead (in transaction local state). +4. Unmark all successful children as not "pending" on all nodes. +5. Unlock region on all (alive) children. + +### dir-write + + The algorithm for dir-write is same as above except instead of holding + `GF_FILE_LK` locks we hold a GF_DIR_LK lock on the name being operated upon. + In case of link-type calls, we hold locks on both the operand names. + +"pending" +--------- + +The "pending" number is like a journal entry. A pending entry is an +array of 32-bit integers stored in network byte-order as the extended +attribute of an inode (which can be a directory as well). + +There are three keys corresponding to three types of pending operations: + +### `AFR_METADATA_PENDING` + +There are some metadata operations pending on this inode (perms, ctime/mtime, +xattr, etc.). + +### `AFR_DATA_PENDING` + +There is some data pending on this inode (writev). + +### `AFR_ENTRY_PENDING` + +There are some directory operations pending on this directory +(create, unlink, etc.). + +Self heal +--------- + +* On lookup, gather extended attribute data: + * If entry is a regular file: + * If an entry is present on one child and not on others: + * create entry on others. + * If entries exist but have different metadata (perms, etc.): + * consider the entry with the highest `AFR_METADATA_PENDING` number as + definitive and replicate its attributes on children. + * If entry is a directory: + * Consider the entry with the highest `AFR_ENTRY_PENDING` number as + definitive and replicate its contents on all children. + * If any two entries have non-matching types (i.e., one is file and + other is directory): + * Announce to the user via log that a split-brain situation has been + detected, and do nothing. +* On open, gather extended attribute data: + * Consider the file with the highest `AFR_DATA_PENDING` number as + the definitive one and replicate its contents on all other + children. + +During all self heal operations, appropriate locks must be held on all +regions/entries being affected. + +Inode scaling +------------- + +Inode scaling is necessary because if a situation arises where an inode number +is returned for a directory (by lookup) which was previously the inode number +of a file (as per FUSE's table), then FUSE gets horribly confused (consult a +FUSE expert for more details). + +To avoid such a situation, we distribute the 64-bit inode space equally +among all children of replicate. + +To illustrate: + +If c1, c2, c3 are children of replicate, they each get 1/3 of the available +inode space: + +------------- -- -- -- -- -- -- -- -- -- -- -- --- +Child: c1 c2 c3 c1 c2 c3 c1 c2 c3 c1 c2 ... +Inode number: 1 2 3 4 5 6 7 8 9 10 11 ... +------------- -- -- -- -- -- -- -- -- -- -- -- --- + +Thus, if lookup on c1 returns an inode number "2", it is scaled to "4" +(which is the second inode number in c1's space). + +This way we ensure that there is never a collision of inode numbers from +two different children. + +This reduction of inode space doesn't really reduce the usability of +replicate since even if we assume replicate has 1024 children (which would be a +highly unusual scenario), each child still has a 54-bit inode space: +$2^{54} \sim 1.8 \times 10^{16}$, which is much larger than any real +world requirement. diff --git a/doc/developer-guide/brickmux-thread-reduction.md b/doc/developer-guide/brickmux-thread-reduction.md new file mode 100644 index 00000000000..7d76e8ff579 --- /dev/null +++ b/doc/developer-guide/brickmux-thread-reduction.md @@ -0,0 +1,64 @@ +# Resource usage reduction in brick multiplexing + +Each brick is regresented with a graph of translators in a brick process. +Each translator in the graph has its own set of threads and mem pools +and other system resources allocations. Most of the times all these +resources are not put to full use. Reducing the resource consumption +of each brick is a problem in itself that needs to be addressed. The other +aspect to it is, sharing of resources across brick graph, this becomes +critical in brick multiplexing scenario. In this document we will be discussing +only about the threads. + +If a brick mux process hosts 50 bricks there are atleast 600+ threads created +in that process. Some of these are global threads that are shared by all the +brick graphs, and others are per translator threads. The global threads like +synctask threads, timer threads, sigwaiter, poller etc. are configurable and +do not needs to be reduced. The per translator threads keeps growing as the +number of bricks in the process increases. Each brick spawns atleast 10+ +threads: +- io-threads +- posix threads: + 1. Janitor + 2. Fsyncer + 3. Helper + 4. aio-thread +- changelog and bitrot threads(even when the features are not enabled) + +## io-threads + +io-threads should be made global to the process, having 16+ threads for +each brick does not make sense. But io-thread translator is loaded in +the graph, and the position of io-thread translator decides from when +the fops will be parallelised across threads. We cannot entirely move +the io-threads to libglusterfs and say the multiplexing happens from +the master translator or so. Hence, the io-thread orchestrator code +is moved to libglusterfs, which ensures there is only one set of +io-threads that is shared among the io-threads translator in each brick. +This poses performance issues due to lock-contention in the io-threds +layer. This also shall be addressed by having multiple locks instead of +one global lock for io-threads. + +## Posix threads +Most of the posix threads execute tasks in a timely manner, hence it can be +replaced with a timer whose handler register a task to synctask framework, once +the task is complete, the timer is registered again. With this we can eliminate +the need of one thread for each task. The problem with using synctasks is +the performance impact it will have due to make/swapcontext. For task that +does not involve network wait, we need not do makecontext, instead the task +function with arg can be stored and executed when a synctask thread is free. +We need to implement an api in synctask to execute atomic tasks(no network wait) +without the overhead of make/swapcontext. This will solve the performance +impact associated with using synctask framework. + +And the other challenge, is to cancel all the tasks pending from a translator. +This is important to cleanly detach brick. For this, we need to implement an +api in synctask that can cancel all the tasks from a given translator. + +For future, this will be replced to use global thread-pool(once implemented). + +## Changelog and bitrot threads + +In the initial implementation, the threads are not created if the feature is +not enabled. We need to share threads across changelog instances if we plan +to enable these features in brick mux scenario. + diff --git a/doc/developer-guide/coding-standard.md b/doc/developer-guide/coding-standard.md new file mode 100644 index 00000000000..031c6c0da99 --- /dev/null +++ b/doc/developer-guide/coding-standard.md @@ -0,0 +1,697 @@ +GlusterFS Coding Standards +========================== + +Before you get started +---------------------- +Before starting with other part of coding standard, install `clang-format` + +On Fedora: +``` +$ dnf install clang +``` +On debian/Ubuntu: +``` +$ apt-get install clang +``` +Once you are done with all the local changes, you need to run below set of commands, +before submitting the patch for review. +``` +$ git add $file # if any +$ git commit -a -s -m "commit message" +$ git show --pretty="format:" --name-only | grep -v "contrib/" | egrep "*\.[ch]$" | xargs clang-format -i +$ git diff # see if there are any changes +$ git commit -a --amend # get the format changes done +$ ./submit-for-review.sh +``` + + +Structure definitions should have a comment per member +------------------------------------------------------ + +Every member in a structure definition must have a comment about its purpose. +The comment should be descriptive without being overly verbose. For pointer +members, lifecycle concerns for the pointed-to object should be noted. For lock +members, the relationship between the lock member and the other members it +protects should be explicit. + +*Bad:* + +``` +gf_lock_t lock; /* lock */ +``` + +*Good:* + +``` +DBTYPE access_mode; /* access mode for accessing + * the databases, can be + * DB_HASH, DB_BTREE + * (option access-mode <mode>) + */ +``` + +Structure members should be aligned based on the padding requirements +--------------------------------------------------------------------- + +The compiler will make sure that structure members have optimum alignment, +but at the expense of suboptimal padding. More important is to optimize the +padding. The compiler won't do that for you. + +This also will help utilize the memory better + +*Bad:* +``` +struct bad { + bool b; /* 0 */ + /* 1..7 pad */ + void *p; /* 8..15 */ + char c; /* 16 */ + char a[16]; /* 17..33 */ + /* 34..39 pad */ + int64_t ii; /* 40..47 */ + int32_t i; /* 48..51 */ + /* 52..55 pad */ + int64_t iii; /* 56..63 */ +}; +``` + +*Good:* +``` +struct good { + int64_t ii; /* explicit 64-bit types */ + void *p; /* may be 64- or 32-bit */ + long l; /* may be 64- or 32-bit */ + int i; /* 32-bit */ + short s; /* 16-bit */ + char c; /* 8-bit */ + bool b; /* 8-bit */ + char a[1024]; +); +``` +Make sure the items with the most stringent alignment requirements will need +to come earliest (ie, pointers and perhaps uint64_t etc), and those with less +stringent alignment requirements at the end (uint16/uint8 and char). Also note +that the long array (if any) should be at the end of the structure, regardless +of the type. + +Also note, if your structure's overall size is crossing 1k-4k limit, it is +recommended to mention the reason why the particular structure needs so much +memory as a comment at the top. + +Use \_typename for struct tags and typename\_t for typedefs +--------------------------------------------------------- + +Being consistent here makes it possible to automate navigation from use of a +type to its true definition (not just the typedef). + +*Bad:* + +``` +struct thing {...}; +struct thing_t {...}; +typedef struct _thing thing; +``` + +*Good:* + +``` +typedef struct _thing {...} thing_t; +``` + +No double underscores +--------------------- + +Identifiers beginning with double underscores are supposed to reserved for the +compiler. + +http://www.open-std.org/jtc1/sc22/wg14/www/docs/n1570.pdf + +When you need to define inner/outer functions, use a different prefix/suffix. + +*Bad:* + +``` +void __do_something (void); + +void +do_something (void) +{ + LOCK (); + __do_something (); + UNLOCK (); +} +``` + +*Good:* + +``` +void do_something_locked (void); +``` + +Only use safe pointers in initializers +---------------------------------------------------------- + +Some pointers, such as `this` in a fop function, can be assumed to be non-NULL. +However, other parameters and further-derived values might be NULL. + +*Good:* + +``` +pid_t pid = frame->root->pid; +``` + + +*Bad:* + +``` +data_t *my_data = dict_get (xdata, "fubar"); +``` + +No giant stack allocations +-------------------------- + +Synctasks have small finite stacks. To avoid overflowing these stacks, avoid +allocating any large data structures on the stack. Use dynamic allocation +instead. + +*Bad:* + +``` +gf_boolean_t port_inuse[65536]; /* 256KB, this actually happened */ +``` + +NOTE: Ideal is to limit the stack array to less than 256 bytes. + + +Character array initializing +---------------------------- + +It is recommended to keep the character array initializing to empty string. + +*Good:* +``` +char msg[1024] = ""; +``` + +Not so much recommended, even though it means the same. + +``` +char msg[1024] = {0,}; +``` + +We recommend above to structure initialization. + + + +Validate all arguments to a function +------------------------------------ + +All pointer arguments to a function must be checked for `NULL`. +A macro named `GF_VALIDATE_OR_GOTO` (in `common-utils.h`) +takes two arguments; if the first is `NULL`, it writes a log message and +jumps to a label specified by the second aergument after setting errno +appropriately. There are several variants of this function for more +specific purposes, and their use is recommended. + +*Bad:* + +``` +/* top of function */ +ret = dict_get (xdata, ...) +``` + +*Good:* + +``` +/* top of function */ +GF_VALIDATE_OR_GOTO(xdata,out); +ret = dict_get (xdata, ...) +``` + +Never rely on precedence of operators +------------------------------------- + +Never write code that relies on the precedence of operators to execute +correctly. Such code can be hard to read and someone else might not +know the precedence of operators as accurately as you do. This includes +precedence of increment/decrement vs. field/subscript. The only exceptions are +arithmetic operators (which have had defined precedence since before computers +even existed) and boolean negation. + +*Bad:* + +``` +if (op_ret == -1 && errno != ENOENT) +++foo->bar /* incrementing foo, or incrementing foo->bar? */ +a && b || !c +``` + +*Good:* + +``` +if ((op_ret == -1) && (errno != ENOENT)) +(++foo)->bar +++(foo->bar) +(a && b) || !c +a && (b || !c) +``` + +Use exactly matching types +-------------------------- + +Use a variable of the exact type declared in the manual to hold the +return value of a function. Do not use an 'equivalent' type. + + +*Bad:* + +``` +int len = strlen (path); +``` + +*Good:* + +``` +size_t len = strlen (path); +``` + +Avoid code such as `foo->bar->baz`; check every pointer +------------------------------------------------------------- + +Do not write code that blindly follows a chain of pointer references. Any +pointer in the chain may be `NULL` and thus cause a crash. Verify that each +pointer is non-null before following it. Even if `foo->bar` has been checked +and is known safe, repeating it can make code more verbose and less clear. + +This rule includes `[]` as well as `->` because both dereference pointers. + +*Bad:* + +``` +foo->bar->field1 = value1; +xyz = foo->bar->field2 + foo->bar->field3 * foo->bar->field4; +foo->bar[5].baz +``` + +*Good:* + +``` +my_bar = foo->bar; +if (!my_bar) ... return; +my_bar->field1 = value1; +xyz = my_bar->field2 + my_bar->field3 * my_bar->field4; +``` + +Document unchecked return values +---------------------------------------------------- + +In general, return values should be checked. If a function is being called +for its side effects and the return value really doesn't matter, an explicit +cast to void is required (to keep static analyzers happy) and a comment is +recommended. + +*Bad:* + +``` +close (fd); +do_important_thing (); +``` + +*Good (or at least OK):* + +``` +(void) sleep (1); +``` + +Gracefully handle failure of malloc (and other allocation functions) +-------------------------------------------------------------------- + +GlusterFS should never crash or exit due to lack of memory. If a +memory allocation fails, the call should be unwound and an error +returned to the user. + +*Use result args and reserve the return value to indicate success or failure:* + +The return value of every functions must indicate success or failure (unless +it is impossible for the function to fail --- e.g., boolean functions). If +the function needs to return additional data, it must be returned using a +result (pointer) argument. + +*Bad:* + +``` +int32_t dict_get_int32 (dict_t *this, char *key); +``` + +*Good:* + +``` +int dict_get_int32 (dict_t *this, char *key, int32_t *val); +``` + +Always use the 'n' versions of string functions +----------------------------------------------- + +Unless impossible, use the length-limited versions of the string functions. + +*Bad:* + +``` +strcpy (entry_path, real_path); +``` + +*Good:* + +``` +strncpy (entry_path, real_path, entry_path_len); +``` + +Do not use memset prior to sprintf/snprintf/vsnprintf etc... +------------------------------------------------------------ +snprintf(and other similar string functions) terminates the buffer with a +'\0'(null character). Hence, there is no need to do a memset before using +snprintf. (Of course you need to account one extra byte for the null character +in your allocation). + +Note: Similarly if you are doing pre-memory allocation for the buffer, use +GF_MALLOC instead of GF_CALLOC, since the later is bit costlier. + +*Bad:* + +``` +char buffer[x]; +memset (buffer, 0, x); +bytes_read = snprintf (buffer, sizeof buffer, "bad standard"); +``` + +*Good:* +``` +char buffer[x]; +bytes_read = snprintf (buffer, sizeof (buffer), "good standard"); +``` + +And it is always to good initialize the char array if the string is static. + +E.g. +``` +char buffer[] = "good standard"; +``` + +No dead or commented code +------------------------- + +There must be no dead code (code to which control can never be passed) or +commented out code in the codebase. + +Function length or Keep functions small +--------------------------------------- + +We live in the UNIX-world where modules do one thing and do it well. +This rule should apply to our functions also. If a function is very long, try splitting it +into many little helper functions. The question is, in a coding +spree, how do we know a function is long and unreadable. One rule of +thumb given by Linus Torvalds is that, a function should be broken-up +if you have 4 or more levels of indentation going on for more than 3-4 +lines. + +*Example for a helper function:* +``` +static int +same_owner (posix_lock_t *l1, posix_lock_t *l2) +{ + return ((l1->client_pid == l2->client_pid) && + (l1->transport == l2->transport)); +} +``` + +Define functions as static +-------------------------- + +Declare functions as static unless they're exposed via a module-level API for +use from other modules. + +No nested functions +------------------- + +Nested functions have proven unreliable, e.g. as callbacks in code that uses +ucontext (green) threads, + +Use inline functions instead of macros whenever possible +-------------------------------------------------------- + +Inline functions enforce type safety; macros do not. Use macros only for things +that explicitly need to be type-agnostic (e.g. cases where one might use +generics or templates in other languages), or that use other preprocessor +features such as `#` for stringification or `##` for token pasting. In general, +"static inline" is the preferred form. + +Avoid copypasta +--------------- + +Code that is copied and then pasted into multiple functions often creates +maintenance problems later, e.g. updating all but one instance for a subsequent +change. If you find yourself copying the same "boilerplate" many places, +consider refactoring to use helper functions (including inline) or macros, or +code generation. + +Ensure function calls wrap around after 80-columns +-------------------------------------------------- + +Place remaining arguments on the next line if needed. + +Functions arguments and function definition +------------------------------------------- + +Place all the arguments of a function definition on the same line +until the line goes beyond 80-cols. Arguments that extend beyind +80-cols should be placed on the next line. + +Style issues +------------ + +### Brace placement + +Use K&R/Linux style of brace placement for blocks. + +*Good:* + +``` +int some_function (...) +{ + if (...) { + /* ... */ + } else if (...) { + /* ... */ + } else { + /* ... */ + } + + do { + /* ... */ + } while (cond); +} +``` + +### Indentation + +Use *eight* spaces for indenting blocks. Ensure that your +file contains only spaces and not tab characters. You can do this +in Emacs by selecting the entire file (`C-x h`) and +running `M-x untabify`. + +To make Emacs indent lines automatically by eight spaces, add this +line to your `.emacs`: + +``` +(add-hook 'c-mode-hook (lambda () (c-set-style "linux"))) +``` + +### Comments + +Write a comment before every function describing its purpose (one-line), +its arguments, and its return value. Mention whether it is an internal +function or an exported function. + +Write a comment before every structure describing its purpose, and +write comments about each of its members. + +Follow the style shown below for comments, since such comments +can then be automatically extracted by doxygen to generate +documentation. + +*Good:* + +``` +/** +* hash_name -hash function for filenames +* @par: parent inode number +* @name: basename of inode +* @mod: number of buckets in the hashtable +* +* @return: success: bucket number +* failure: -1 +* +* Not for external use. +*/ +``` + +### Indicating critical sections + +To clearly show regions of code which execute with locks held, use +the following format: + +``` +pthread_mutex_lock (&mutex); +{ + /* code */ +} +pthread_mutex_unlock (&mutex); +``` + +### Always use braces + +Even around single statements. + +*Bad:* + +``` +if (condition) action (); + +if (condition) + action (); +``` + +*Good:* + +``` +if (condition) { + action (); +} +``` + +### Avoid multi-line conditionals + +These can be hard to read and even harder to modify later. Predicate functions +and helper variables are always better for maintainability. + +*Bad:* + +``` +if ((thing1 && other_complex_condition (thing1, lots, of, args)) + || (!thing2 || even_more_complex_condition (thing2)) + || all_sorts_of_stuff_with_thing3) { + return; +} + +``` + +*Better:* + +``` +thing1_ok = predicate1 (thing1, lots, of, args +thing2_ok = predicate2 (thing2); +thing3_ok = predicate3 (thing3); + +if (!thing1_ok || !thing2_ok || !thing3_ok) { + return; +} +``` + +*Best:* + +``` +if (thing1 && other_complex_condition (thing1, lots, of, args)) { + return; +} +if (!thing2 || even_more_complex_condition (thing2)) { + /* Note potential for a different message here. */ + return; +} +if (all_sorts_of_stuff_with_thing3) { + /* And here too. */ + return; +} +``` + +### Use 'const' liberally + +If a value isn't supposed/expected to change, there's no cost to adding a +'const' keyword and it will help prevent violation of expectations. + +### Avoid global variables (including 'static' auto variables) +Almost all state in Gluster is contextual and should be contained in the +appropriate structure reflecting its scope (e.g. `call\_frame\_t`, `call\_stack\_t`, +`xlator\_t`, `glusterfs\_ctx\_t`). With dynamic loading and graph switches in play, +each global requires careful consideration of when it should be initialized or +reinitialized, when it might _accidentally_ be reinitialized, when its value +might become stale, and so on. A few global variables are needed to serve as +'anchor points' for these structures, and more exceptions to the rule might be +approved in the future, but new globals should not be added to the codebase +without explicit approval. + +## A skeleton fop function + +This is the recommended template for any fop. In the beginning come the +initializations. After that, the 'success' control flow should be linear. Any +error conditions should cause a `goto` to a label at the end. By convention +this is 'out' if there is only one such label, but a cascade of such labels is +allowable to support multi-stage cleanup. At that point, the code should detect +the error that has occurred and do appropriate cleanup. + +``` +int32_t +sample_fop (call_frame_t *frame, xlator_t *this, ...) +{ + char * var1 = NULL; + int32_t op_ret = -1; + int32_t op_errno = 0; + DIR * dir = NULL; + struct posix_fd * pfd = NULL; + + VALIDATE_OR_GOTO (frame, out); + VALIDATE_OR_GOTO (this, out); + + /* other validations */ + + dir = opendir (...); + + if (dir == NULL) { + op_errno = errno; + gf_log (this->name, GF_LOG_ERROR, + "opendir failed on %s (%s)", loc->path, + strerror (op_errno)); + goto out; + } + + /* another system call */ + if (...) { + op_errno = ENOMEM; + gf_log (this->name, GF_LOG_ERROR, + "out of memory :("); + goto out; + } + + /* ... */ + + out: + if (op_ret == -1) { + + /* check for all the cleanup that needs to be + done */ + + if (dir) { + closedir (dir); + dir = NULL; + } + + if (pfd) { + FREE (pfd->path); + FREE (pfd); + pfd = NULL; + } + } + + STACK_UNWIND (frame, op_ret, op_errno, fd); + return 0; +} +``` diff --git a/doc/developer-guide/commit-guidelines.md b/doc/developer-guide/commit-guidelines.md new file mode 100644 index 00000000000..38bbe525cbd --- /dev/null +++ b/doc/developer-guide/commit-guidelines.md @@ -0,0 +1,136 @@ +## Git Commit Good Practice + +The following document is based on experience doing code development, bug troubleshooting and code review across a number of projects using Git. The document is mostly borrowed from [Open Stack](https://wiki.openstack.org/wiki/GitCommitMessages), but made more meaningful in the context of GlusterFS project. + +This topic can be split into two areas of concern + +* The structured set/split of the code changes +* The information provided in the commit message + +### Executive Summary +The points and examples that will be raised in this document ought to clearly demonstrate the value in splitting up changes into a sequence of individual commits, and the importance in writing good commit messages to go along with them. If these guidelines were widely applied it would result in a significant improvement in the quality of the GlusterFS Git history. Both a carrot & stick will be required to effect changes. This document intends to be the carrot by alerting people to the benefits, while anyone doing Gerrit code review can act as the stick ;-P + +In other words, when reviewing a change in Gerrit: +* Do not simply look at the correctness of the code. +* Review the commit message itself and request improvements to its content. +* Look out for commits which are mixing multiple logical changes and require the submitter to split them into separate commits. +* Ensure whitespace changes are not mixed in with functional changes. +* Ensure no-op code refactoring is done separately from functional changes. + +And so on. + +It might be mentioned that Gerrit's handling of patch series is not entirely perfect. Let that not become a valid reason to avoid creating patch series. The tools being used should be subservient to developers needs, and since they are open source they can be fixed / improved. Software source code is "read mostly, write occassionally" and thus the most important criteria is to improve the long term maintainability by the large pool of developers in the community, and not to sacrifice too much for the sake of the single author who may never touch the code again. + +And now the long detailed guidelines & examples of good & bad practice + +### Structural split of changes +The cardinal rule for creating good commits is to ensure there is only one "logical change" per commit. There are many reasons why this is an important rule: + +* The smaller the amount of code being changed, the quicker & easier it is to review & identify potential flaws. +* If a change is found to be flawed later, it may be necessary to revert the broken commit. This is much easier to do if there are not other unrelated code changes entangled with the original commit. +* When troubleshooting problems using Git's bisect capability, small well defined changes will aid in isolating exactly where the code problem was introduced. +* When browsing history using Git annotate/blame, small well defined changes also aid in isolating exactly where & why a piece of code came from. + +#### Things to avoid when creating commits +With the above points in mind, there are some commonly encountered examples of bad things to avoid + +* Mixing whitespace changes with functional code changes. + +The whitespace changes will obscure the important functional changes, making it harder for a reviewer to correctly determine whether the change is correct. Solution: Create 2 commits, one with the whitespace changes, one with the functional changes. Typically the whitespace change would be done first, but that need not be a hard rule. + +* Mixing two unrelated functional changes. + +Again the reviewer will find it harder to identify flaws if two unrelated changes are mixed together. If it becomes necessary to later revert a broken commit, the two unrelated changes will need to be untangled, with further risk of bug creation. + +* Sending large new features in a single giant commit. + +It may well be the case that the code for a new feature is only useful when all of it is present. This does not, however, imply that the entire feature should be provided in a single commit. New features often entail refactoring existing code. It is highly desirable that any refactoring is done in commits which are separate from those implementing the new feature. This helps reviewers and test suites validate that the refactoring has no unintentional functional changes. + +Even the newly written code can often be split up into multiple pieces that can be independently reviewed. For example, changes which add new internal fops or library functions, can be in self-contained commits. Again this leads to easier code review. It also allows other developers to cherry-pick small parts of the work, if the entire new feature is not immediately ready for merge. This will encourage the author & reviewers to think about the generic library functions' design, and not simply pick a design that is easier for their currently chosen internal implementation. + +The basic rule to follow is + +If a code change can be split into a sequence of patches/commits, then it should be split. Less is not more. More is more. + +##### Examples of bad practice + +TODO: Pick glusterfs specific example. + + +##### Examples of good practice + + +### Information in commit messages +As important as the content of the change, is the content of the commit message describing it. When writing a commit message there are some important things to remember + +* Do not assume the reviewer understands what the original problem was. + +When reading bug reports, after a number of back & forth comments, it is often as clear as mud, what the root cause problem is. The commit message should have a clear statement as to what the original problem is. The bug is merely interesting historical background on /how/ the problem was identified. It should be possible to review a proposed patch for correctness without needing to read the bug ticket. + +* Do not assume the reviewer has access to external web services/site. + +In 6 months time when someone is on a train/plane/coach/beach/pub troubleshooting a problem & browsing Git history, there is no guarantee they will have access to the online bug tracker, or online blueprint documents. The great step forward with distributed SCM is that you no longer need to be "online" to have access to all information about the code repository. The commit message should be totally self-contained, to maintain that benefit. + +* Do not assume the code is self-evident/self-documenting. + +What is self-evident to one person, might be clear as mud to another person. Always document what the original problem was and how it is being fixed, for any change except the most obvious typos, or whitespace only commits. + +* Describe why a change is being made. + +A common mistake is to just document how the code has been written, without describing /why/ the developer chose to do it that way. By all means describe the overall code structure, particularly for large changes, but more importantly describe the intent/motivation behind the changes. + +* Read the commit message to see if it hints at improved code structure. + +Often when describing a large commit message, it becomes obvious that a commit should have in fact been split into 2 or more parts. Don't be afraid to go back and rebase the change to split it up into separate commits. + +* Ensure sufficient information to decide whether to review. + +When Gerrit sends out email alerts for new patch submissions there is minimal information included, principally the commit message and the list of files changes. Given the high volume of patches, it is not reasonable to expect all reviewers to examine the patches in detail. The commit message must thus contain sufficient information to alert the potential reviewers to the fact that this is a patch they need to look at. + +* The first commit line is the most important. + +In Git commits the first line of the commit message has special significance. It is used as email subject line, git annotate messages, gitk viewer annotations, merge commit messages and many more places where space is at a premium. As well as summarizing the change itself, it should take care to detail what part of the code is affected. eg if it is 'afr', 'dht' or any translator. Or in some cases, it can be touching all these components, but the commit message can be 'coverity:', 'txn-framework:', 'new-fop: ', etc. + +* Describe any limitations of the current code. + +If the code being changed still has future scope for improvements, or any known limitations then mention these in the commit message. This demonstrates to the reviewer that the broader picture has been considered and what tradeoffs have been done in terms of short term goals vs. long term wishes. + +* Do not include patch set-specific comments. + +In other words, if you rebase your change please don't add "Patch set 2: rebased" to your commit message. That isn't going to be relevant once your change has merged. Please do make a note of that in Gerrit as a comment on your change, however. It helps reviewers know what changed between patch sets. This also applies to comments such as "Added unit tests", "Fixed localization problems", or any other such patch set to patch set changes that don't affect the overall intent of your commit. + +**The main rule to follow is:** + +The commit message must contain all the information required to fully understand & review the patch for correctness. Less is not more. More is more. + + +#### Including external references + +The commit message is primarily targeted towards human interpretation, but there is always some metadata provided for machine use. In the case of GlusterFS this includes at least the 'Change-id', "bug"/"feature" ID references and "Signed-off-by" tag (generated by 'git commit -s'). + +The 'Change-id' line is a unique hash describing the change, which is generated by a Git commit hook. This should not be changed when rebasing a commit following review feedback, since it is used by Gerrit, to track versions of a patch. + +The 'bug' line can reference a bug in a few ways. Gerrit creates a link to the bug when viewing the patch on review.gluster.org so that reviewers can quickly access the bug/issue on Bugzilla or Github. + +**Fixes: bz#1601166** -- use 'Fixes: bz#NNNNN' if the commit is intended to fully fix and close the bug being referenced. +**Fixes: #411** -- use 'Fixes: #NNN' if the patch fixes the github issue completely. + +**Updates: bz#1193929** -- use 'Updates: bz#NNNN' if the commit is only a partial fix and more work is needed. +**Updates: #175** -- use 'Updates: #NNNN' if the commit is only a partial fix and more work is needed for the feature completion. + +We encourage the use of `Co-Authored-By: name <name@example.com>` in commit messages to indicate people who worked on a particular patch. It's a convention for recognizing multiple authors, and our projects would encourage the stats tools to observe it when collecting statistics. + +### Summary of Git commit message structure + +* Provide a brief description of the change in the first line. +* The first line should be limited to 50 characters and should not end with a period. + +* Insert a single blank line after the first line. + +* Provide a detailed description of the change in the following lines, breaking paragraphs where needed. + +* Subsequent lines should be wrapped at 72 characters. + +Put the 'Change-id', 'Fixes bz#NNNNN' and 'Signed-off-by: <>' lines at the very end. + +TODO: Add good examples diff --git a/doc/developer-guide/daemon-management-framework.md b/doc/developer-guide/daemon-management-framework.md new file mode 100644 index 00000000000..592192e665d --- /dev/null +++ b/doc/developer-guide/daemon-management-framework.md @@ -0,0 +1,38 @@ + +How to introduce new daemons using daemon management framework +============================================================== +Glusterd manages GlusterFS daemons providing services like NFS, Proactive +self-heal, Quota, User servicable snapshots etc. Following are some of the +aspects that come under daemon management. + +Data members & functions of different management objects + +- **Connection Management** + - unix domain sockets based channel for internal communication + - rpc connection for the communication + - frame timeout value for UDS + - Methods - notify + - init, connect, termination, disconnect APIs can be invoked using the + connection management object + +- **Process Management** + - Name of the process + - pidfile to detect if the daemon is running + - loggging directory, log file, volfile, volfileserver & volfileid + - init, stop APIs can be invoked using the process management object + +- **Service Management** + - connection object + - process object + - online status + - Methods - manager, start, stop which can be abstracted as a common methods + or specific to service requirements + - init API can be invoked using the service management object + + The above structures defines the skeleton of the daemon management framework. + Introduction of new daemons in GlusterFS needs to inherit these properties. Any + requirement specific to a daemon needs to be implemented in its own service + (for eg : snapd defines its own type glusterd_snapdsvc_t using glusterd_svc_t + and snapd specific data). New daemons will need to have its own service specific + code written in glusterd-<feature>-svc.h{c} and need to reuse the existing + framework. diff --git a/doc/developer-guide/datastructure-inode.md b/doc/developer-guide/datastructure-inode.md new file mode 100644 index 00000000000..45d7a941e5f --- /dev/null +++ b/doc/developer-guide/datastructure-inode.md @@ -0,0 +1,221 @@ +# Inode and dentry management in GlusterFS: + +## Background +Filesystems internally refer to files and directories via inodes. Inodes +are unique identifiers of the entities stored in a filesystem. Whenever an +application has to operate on a file/directory (read/modify), the filesystem +maps that file/directory to the right inode and start referring to that inode +whenever an operation has to be performed on the file/directory. + +In GlusterFS a new inode gets created whenever a new file/directory is created +OR when a successful lookup is done on a file/directory for the first time. +Inodes in GlusterFS are maintained by the inode table which gets initiated when +the filesystem daemon is started (both for the brick process as well as the +mount process). Below are some important data structures for inode management. + +## Data-structure (inode-table) +``` +struct _inode_table { + pthread_mutex_t lock; + size_t hashsize; /* bucket size of inode hash and dentry hash */ + char *name; /* name of the inode table, just for gf_log() */ + inode_t *root; /* root directory inode, with inode + number and gfid 1 */ + xlator_t *xl; /* xlator to be called to do purge and + the xlator which maintains the inode table*/ + uint32_t lru_limit; /* maximum LRU cache size */ + struct list_head *inode_hash; /* buckets for inode hash table */ + struct list_head *name_hash; /* buckets for dentry hash table */ + struct list_head active; /* list of inodes currently active (in an fop) */ + uint32_t active_size; /* count of inodes in active list */ + struct list_head lru; /* list of inodes recently used. + lru.next most recent */ + uint32_t lru_size; /* count of inodes in lru list */ + struct list_head purge; /* list of inodes to be purged soon */ + uint32_t purge_size; /* count of inodes in purge list */ + + struct mem_pool *inode_pool; /* memory pool for inodes */ + struct mem_pool *dentry_pool; /* memory pool for dentrys */ + struct mem_pool *fd_mem_pool; /* memory pool for fd_t */ + int ctxcount; /* number of slots in inode->ctx */ +}; +``` + +# Life-cycle +``` +inode_table_new (size_t lru_limit, xlator_t *xl) +``` +This is a function which allocates a new inode table. Usually the top xlators in +the graph such as protocol/server (for bricks), fuse and nfs (for fuse and nfs +mounts) and libgfapi do inode managements. Hence they are the ones which will +allocate a new inode table by calling the above function. + +Each xlator graph in glusterfs maintains an inode table. So in fuse clients, +whenever there is a graph change due to add brick/remove brick or +addition/removal of some other xlators, a new graph is created which creates a +new inode table. + +Thus an allocated inode table is destroyed only when the filesystem daemon is +killed or unmounted. + + +# what it contains. +Inode table in glusterfs mainly contains a hash table for maintaining inodes. +In general a file/directory is considered to be existing if there is a +corresponding inode present in the inode table. If a inode for a file/directory +cannot be found in the inode table, glusterfs tries to resolve it by sending a +lookup on the entry for which the inode is needed. If lookup is successful, then +a new inode correponding to the entry is added to the hash table present in the +inode table. Thus an inode present in the hash-table means, its an existing +file/directory within the filesystem. The inode table also contains the hash +size of the hash table (as of now it is hard coded to 14057. The hash value of +a inode is calculated using its gfid). + +Apart from the hash table, inode table also maintains 3 important list of inodes +1. Active list: +Active list contains all the active inodes (i.e inodes which are currently part +of some fop). +2. Lru list: +Least recently used inodes list. A limit can be set for the size of the lru +list. For bricks it is 16384 and for clients it is infinity. +3. Purge list: +List of all the inodes which have to be purged (i.e inodes which have to be +deleted from the inode table due to unlink/rmdir/forget). + +And at last it also contains the mem-pool for allocating inodes, dentries so +that frequent malloc/calloc and free of the data structures can be avoided. + + +# Data structure (inode) +``` +struct _inode { + inode_table_t *table; /* the table this inode belongs to */ + uuid_t gfid; /* unique identifier of the inode */ + gf_lock_t lock; + uint64_t nlookup; + uint32_t fd_count; /* Open fd count */ + uint32_t ref; /* reference count on this inode */ + ia_type_t ia_type; /* what kind of file */ + struct list_head fd_list; /* list of open files on this inode */ + struct list_head dentry_list; /* list of directory entries for this inode */ + struct list_head hash; /* hash table pointers */ + struct list_head list; /* active/lru/purge */ + + struct _inode_ctx *_ctx; /* place holder for keeping the + information about the inode by different xlators */ +}; +``` +As said above, inodes are internal way of identifying the files/directories. A +inode uniquely represents a file/directory. A new inode is created whenever a +create/mkdir/symlink/mknod operations are performed. Apart from that a new inode +is created upon the successful fresh lookup of a file/directory. Say the +filesystem contained some file "a" within root and the filesystem was +unmounted. Now when glusterfs is mounted and some operation is perfomed on "/a", +glusterfs tries to get the inode for the entry "a" with parent inode as +root. But, since glusterfs just came up, it will not be able to find the inode +for "a" and will send a lookup on "/a". If the lookup operation succeeds (i.e. +the root of glusterfs contains an entry called "a"), then a new inode for "/a" +is created and added to the inode table. + +Depending upon the situation, an inode can be in one of the 3 lists maintained +by the inode table. If some fop is happening on the inode, then the inode will +be present in the active inodes list maintained by the inode table. Active +inodes are those inodes whose refcount is greater than zero. Whenever some +operation comes on a file/directory, and the resolver tries to find the inode +for it, it increments the refcount of the inode before returning the inode. The +refcount of an inode can be incremented by calling the below function +``` +inode_ref (inode_t *inode) +``` +Any xlator which wants to operate on a inode as part of some fop (or wants the +inode in the callback), should hold a ref on the inode. +Once the fop is completed before sending the reply of the fop to the above +layers , the inode has to be unrefed. When the refcount of an inode becomes +zero, it is removed from the active inodes list and put into LRU list maintained +by the inode table. Thus in short if some fop is happening on a file/directory, +the corresponding inode will be in the active list or it will be in the LRU +list. + + +# Life Cycle + +A new inode is created whenever a new file/directory/symlink is created OR a +successful lookup of an existing entry is done. The xlators which does inode +management (as of now protocol/server, fuse, nfs, gfapi) will perform inode_link +operation upon successful lookup or successful creation of a new entry. +``` +inode_link (inode_t *inode, inode_t *parent, const char *name, + struct iatt *buf); +``` +inode_link actually adds the inode to the inode table (to be precise it adds +the inode to the hash table maintained by the inode table. The hash value is +calculated based on the gfid). Copies the gfid to the inode (the gfid is +present in the iatt structure). Creates a dentry with the new name. + +A inode is removed from the inode table and eventually destroyed when unlink +or rmdir operation is performed on a file/directory, or the the lru limit of +the inode table has been exceeded. + +# Data structure (dentry) +``` + +struct _dentry { + struct list_head inode_list; /* list of dentries of inode */ + struct list_head hash; /* hash table pointers */ + inode_t *inode; /* inode of this directory entry */ + char *name; /* name of the directory entry */ + inode_t *parent; /* directory of the entry */ +}; +``` +A dentry is the presence of an entry for a file/directory within its parent +directory. A dentry usually points to the inode to which it belongs to. In +glusterfs a dentry contains the following fields. +1. a hook using which it can add itself to the list of +the dentries maintained by the inode to which it points to. +2. A hash table pointer. +3. Pointer to the inode to which it belongs to. +4. Name of the dentry +5. Pointer to the inode of the parent directory in which the dentry is present + +A new dentry is created when a new file/directory/symlink is created or a hard +link to an existing file is created. +``` +__dentry_create (inode_t *inode, inode_t *parent, const char *name); +``` +A dentry holds a refcount on the parent +directory so that the parent inode is never removed from the active inode's list +and put to the lru list (If the lru limit of the lru list is exceeded, there is +a chance of parent inode being destroyed. To avoid it, the dentries hold a +reference to the parent inode). A dentry is removed whenevern a unlink/rmdir +is perfomed on a file/directory. Or when the lru limit has been exceeded, the +oldest inodes are purged out of the inode table, during which all the dentries +of the inode are removed. + +Whenever a unlink/rmdir comes on a file/directory, the corresponding inode +should be removed from the inode table. So upon unlink/rmdir, the inode will +be moved to the purge list maintained by the inode table and from there it is +destroyed. To be more specific, if a inode has to be destroyed, its refcount +and nlookup count both should become 0. For refcount to become 0, the inode +should not be part of any fop (there should not be any open fds). Or if the +inode belongs to a directory, then there should not be any fop happening on the +directory and it should not contain any dentries within it. For nlookup count to +become zero, a forget has to be sent on the inode with nlookup count set to 0 as +an argument. For fuse clients, forget is sent by the kernel itself whenever a +unlink/rmdir is performed. But for brick processes, upon unlink/rmdir, the +protocol/server itself has to do inode_forget. Whenever the inode has to be +deleted due to file removal or lru limit being exceeded the inode is retired +(i.e. all the dentries of the inode are deleted and the inode is moved to the +purge list maintained by the inode table), the nlookup count is set to 0 via +inode_forget api. The inode table, then prunes all the inodes from the purge +list by destroying the inode contexts maintained by each xlator. +``` +unlinking of the dentry is done via inode_unlink; + +void +inode_unlink (inode_t *inode, inode_t *parent, const char *name); +``` +If the inode has multiple hard links, then the unlink operation performed by +the application results just in the removal of the dentry with the name provided +by the application. For the inode to be removed, all the dentries of the inode +should be unlinked. + diff --git a/doc/developer-guide/datastructure-iobuf.md b/doc/developer-guide/datastructure-iobuf.md new file mode 100644 index 00000000000..03604e3672c --- /dev/null +++ b/doc/developer-guide/datastructure-iobuf.md @@ -0,0 +1,259 @@ +# Iobuf-pool +## Datastructures +### iobuf +Short for IO Buffer. It is one allocatable unit for the consumers of the IOBUF +API, each unit hosts @page_size(defined in arena structure) bytes of memory. As +initial step of processing a fop, the IO buffer passed onto GlusterFS by the +other applications (FUSE VFS/ Applications using gfapi) is copied into GlusterFS +space i.e. iobufs. Hence Iobufs are mostly allocated/deallocated in Fuse, gfapi, +protocol xlators, and also in performance xlators to cache the IO buffers etc. +``` +struct iobuf { + union { + struct list_head list; + struct { + struct iobuf *next; + struct iobuf *prev; + }; + }; + struct iobuf_arena *iobuf_arena; + + gf_lock_t lock; /* for ->ptr and ->ref */ + int ref; /* 0 == passive, >0 == active */ + + void *ptr; /* usable memory region by the consumer */ + + void *free_ptr; /* in case of stdalloc, this is the + one to be freed not the *ptr */ +}; +``` + +### iobref +There may be need of multiple iobufs for a single fop, like in vectored read/write. +Hence multiple iobufs(default 16) are encapsulated under one iobref. +``` +struct iobref { + gf_lock_t lock; + int ref; + struct iobuf **iobrefs; /* list of iobufs */ + int alloced; /* 16 by default, grows as required */ + int used; /* number of iobufs added to this iobref */ +}; +``` +### iobuf_arenas +One region of memory MMAPed from the operating system. Each region MMAPs +@arena_size bytes of memory, and hosts @arena_size / @page_size IOBUFs. +The same sized iobufs are grouped into one arena, for sanity of access. + +``` +struct iobuf_arena { + union { + struct list_head list; + struct { + struct iobuf_arena *next; + struct iobuf_arena *prev; + }; + }; + + size_t page_size; /* size of all iobufs in this arena */ + size_t arena_size; /* this is equal to + (iobuf_pool->arena_size / page_size) + * page_size */ + size_t page_count; + + struct iobuf_pool *iobuf_pool; + + void *mem_base; + struct iobuf *iobufs; /* allocated iobufs list */ + + int active_cnt; + struct iobuf active; /* head node iobuf + (unused by itself) */ + int passive_cnt; + struct iobuf passive; /* head node iobuf + (unused by itself) */ + uint64_t alloc_cnt; /* total allocs in this pool */ + int max_active; /* max active buffers at a given time */ +}; + +``` +### iobuf_pool +Pool of Iobufs. As there may be many Io buffers required by the filesystem, +a pool of iobufs are preallocated and kept, if these preallocated ones are +exhausted only then the standard malloc/free is called, thus improving the +performance. Iobuf pool is generally one per process, allocated during +glusterfs_ctx_t init (glusterfs_ctx_defaults_init), currently the preallocated +iobuf pool memory is freed on process exit. Iobuf pool is globally accessible +across GlusterFs, hence iobufs allocated by any xlator can be accessed by any +other xlators(unless iobuf is not passed). +``` +struct iobuf_pool { + pthread_mutex_t mutex; + size_t arena_size; /* size of memory region in + arena */ + size_t default_page_size; /* default size of iobuf */ + + int arena_cnt; + struct list_head arenas[GF_VARIABLE_IOBUF_COUNT]; + /* array of arenas. Each element of the array is a list of arenas + holding iobufs of particular page_size */ + + struct list_head filled[GF_VARIABLE_IOBUF_COUNT]; + /* array of arenas without free iobufs */ + + struct list_head purge[GF_VARIABLE_IOBUF_COUNT]; + /* array of of arenas which can be purged */ + + uint64_t request_misses; /* mostly the requests for higher + value of iobufs */ +}; +``` +~~~ +The default size of the iobuf_pool(as of yet): +1024 iobufs of 128Bytes = 128KB +512 iobufs of 512Bytes = 256KB +512 iobufs of 2KB = 1MB +128 iobufs of 8KB = 1MB +64 iobufs of 32KB = 2MB +32 iobufs of 128KB = 4MB +8 iobufs of 256KB = 2MB +2 iobufs of 1MB = 2MB +Total ~13MB +~~~ +As seen in the datastructure iobuf_pool has 3 arena lists. + +- arenas: +The arenas allocated during iobuf_pool create, are part of this list. This list +also contains arenas that are partially filled i.e. contain few active and few +passive iobufs (passive_cnt !=0, active_cnt!=0 except for initially allocated +arenas). There will be by default 8 arenas of the sizes mentioned above. +- filled: +If all the iobufs in the arena are filled(passive_cnt = 0), the arena is moved +to the filled list. If any of the iobufs from the filled arena is iobuf_put, +then the arena moves back to the 'arenas' list. +- purge: +If there are no active iobufs in the arena(active_cnt = 0), the arena is moved +to purge list. iobuf_put() triggers destruction of the arenas in this list. The +arenas in the purge list are destroyed only if there is atleast one arena in +'arenas' list, that way there won't be spurious mmap/unmap of buffers. +(e.g: If there is an arena (page_size=128KB, count=32) in purge list, this arena +is destroyed(munmap) only if there is an arena in 'arenas' list with page_size=128KB). + +## APIs +### iobuf_get + +``` +struct iobuf *iobuf_get (struct iobuf_pool *iobuf_pool); +``` +Creates a new iobuf of the default page size(128KB hard coded as of yet). +Also takes a reference(increments ref count), hence no need of doing it +explicitly after getting iobuf. + +### iobuf_get2 + +``` +struct iobuf * iobuf_get2 (struct iobuf_pool *iobuf_pool, size_t page_size); +``` +Creates a new iobuf of a specified page size, if page_size=0 default page size +is considered. +``` +if (requested iobuf size > Max iobuf size in the pool(1MB as of yet)) + { + Perform standard allocation(CALLOC) of the requested size and + add it to the list iobuf_pool->arenas[IOBUF_ARENA_MAX_INDEX]. + } + else + { + -Round the page size to match the stndard sizes in iobuf pool. + (eg: if 3KB is requested, it is rounded to 8KB). + -Select the arena list corresponding to the rounded size + (eg: select 8KB arena) + If the selected arena has passive count > 0, then return the + iobuf from this arena, set the counters(passive/active/etc.) + appropriately. + else the arena is full, allocate new arena with rounded size + and standard page numbers and add to the arena list + (eg: 128 iobufs of 8KB is allocated). + } +``` +Also takes a reference(increments ref count), hence no need of doing it +explicitly after getting iobuf. + +### iobuf_ref + +``` +struct iobuf *iobuf_ref (struct iobuf *iobuf); +``` + Take a reference on the iobuf. If using an iobuf allocated by some other +xlator/function/, its a good practice to take a reference so that iobuf is not +deleted by the allocator. + +### iobuf_unref +``` +void iobuf_unref (struct iobuf *iobuf); +``` +Unreference the iobuf, if the ref count is zero iobuf is considered free. + +``` + -Delete the iobuf, if allocated from standard alloc and return. + -set the active/passive count appropriately. + -if passive count > 0 then add the arena to 'arena' list. + -if active count = 0 then add the arena to 'purge' list. +``` +Every iobuf_ref should have a corresponding iobuf_unref, and also every +iobuf_get/2 should have a correspondning iobuf_unref. + +### iobref_new +``` +struct iobref *iobref_new (); +``` +Creates a new iobref structure and returns its pointer. + +### iobref_ref +``` +struct iobref *iobref_ref (struct iobref *iobref); +``` +Take a reference on the iobref. + +### iobref_unref +``` +void iobref_unref (struct iobref *iobref); +``` +Decrements the reference count of the iobref. If the ref count is 0, then unref +all the iobufs(iobuf_unref) in the iobref, and destroy the iobref. + +### iobref_add +``` +int iobref_add (struct iobref *iobref, struct iobuf *iobuf); +``` +Adds the given iobuf into the iobref, it takes a ref on the iobuf before adding +it, hence explicit iobuf_ref is not required if adding to the iobref. + +### iobref_merge +``` +int iobref_merge (struct iobref *to, struct iobref *from); +``` +Adds all the iobufs in the 'from' iobref to the 'to' iobref. Merge will not +cause the delete of the 'from' iobref, therefore it will result in another ref +on all the iobufs added to the 'to' iobref. Hence iobref_unref should be +performed both on 'from' and 'to' iobrefs (performing iobref_unref only on 'to' +will not free the iobufs and may result in leak). + +### iobref_clear +``` +void iobref_clear (struct iobref *iobref); +``` +Unreference all the iobufs in the iobref, and also unref the iobref. + +## Iobuf Leaks +If all iobuf_refs/iobuf_new do not have correspondning iobuf_unref, then the +iobufs are not freed and recurring execution of such code path may lead to huge +memory leaks. The easiest way to identify if a memory leak is caused by iobufs +is to take a statedump. If the statedump shows a lot of filled arenas then it is +a sure sign of leak. Refer doc/debugging/statedump.md for more details. + +If iobufs are leaking, the next step is to find where the iobuf_unref went +missing. There is no standard/easy way of debugging this, code reading and logs +are the only ways. If there is a liberty to reproduce the memory leak at will, +then logs(gf_callinginfo) in iobuf_ref/unref might help. +TODO: A easier way to debug iobuf leaks. diff --git a/doc/developer-guide/datastructure-mem-pool.md b/doc/developer-guide/datastructure-mem-pool.md new file mode 100644 index 00000000000..225567cbf9f --- /dev/null +++ b/doc/developer-guide/datastructure-mem-pool.md @@ -0,0 +1,124 @@ +# Mem-pool +## Background +There was a time when every fop in glusterfs used to incur cost of allocations/de-allocations for every stack wind/unwind between xlators because stack/frame/*_localt_t in every wind/unwind was allocated and de-allocated. Because of all these system calls in the fop path there was lot of latency and the worst part is that most of the times the number of frames/stacks active at any time wouldn't cross a threshold. So it was decided that this threshold number of frames/stacks would be allocated in the beginning of the process only once. Get one of them from the pool of stacks/frames whenever `STACK_WIND` is performed and put it back into the pool in `STACK_UNWIND`/`STACK_DESTROY` without incurring any extra system calls. The data structures are allocated only when threshold number of such items are in active use i.e. pool is in complete use.% increase in the performance once this was added to all the common data structures (inode/fd/dict etc) in xlators throughout the stack was tremendous. + +## Data structure +``` +struct mem_pool { + struct list_head list; /*Each member in the mempool is element padded with a doubly-linked-list + ptr of mempool + is-in +-use info. This list is used to add the element to the list of free members in the mem-pool*/ + int hot_count;/*number of mempool elements that are in active use*/ + int cold_count;/*number of mempool elements that are not in use. If a new allocation is required it +will be served from here until all the elements in the pool are in use i.e. cold-count becomes 0.*/ + gf_lock_t lock;/*synchronization mechanism*/ + unsigned long padded_sizeof_type;/*Each mempool element is padded with a doubly-linked-list + ptr of mempool + is-in +-use info to operate the pool of elements, this size is the element-size after padding*/ + void *pool;/*Starting address of pool*/ + void *pool_end;/*Ending address of pool*/ +/* If an element address is in the range between pool, pool_end addresses then it is alloced from the pool otherwise it is 'calloced' this is very useful for functions like 'mem_put'*/ + int real_sizeof_type;/* size of just the element without any padding*/ + uint64_t alloc_count; /*Number of times this type of data is allocated through out the life of this process. This may include calloced elements as well*/ + uint64_t pool_misses; /*Number of times the element had to be allocated from heap because all elements from the pool are in active use.*/ + int max_alloc; /*Maximum number of elements from the pool in active use at any point in the life of the process. This does *not* include calloced elements*/ + int curr_stdalloc;/*Number of elements that are allocated from heap at the moment because the pool is in completed use. It should be '0' when pool is not in complete use*/ + int max_stdalloc;/*Maximum number of allocations from heap after the pool is completely used that are in active use at any point in the life of the process.*/ + char *name; /*Contains xlator-name:data-type as a string + struct list_head global_list;/*This is used to insert it into the global_list of mempools maintained in 'glusterfs-ctx' +}; +``` + +## Life-cycle +``` +mem_pool_new (data_type, unsigned long count) + +This is a macro which expands to mem_pool_new_fn (sizeof (data_type), count, string-rep-of-data_type) + +struct mem_pool * +mem_pool_new_fn (unsigned long sizeof_type, unsigned long count, char *name) + +Padded-element: + ---------------------------------------- +|list-ptr|mem-pool-address|in-use|Element| + ---------------------------------------- + ``` + +This function allocates the `mem-pool` structure and sets up the pool for use. +`name` parameter above is the `string` containing type of the datatype. This `name` is appended to `xlator-name + ':'` so that it can be easily identified in things like statedump. `count` is the number of elements that need to be allocated. `sizeof_type` is the size of each element. Ideally `('sizeof_type'*'count')` should be the size of the total pool. But to manage the pool using `mem_get`/`mem_put` (will be explained after this section) each element needs to be padded in the front with a `('list', 'mem-pool-address', 'in_use')`. So the actual size of the pool it allocates will be `('padded_sizeof_type'*'count')`. Why these extra elements are needed will be evident after understanding how `mem_get` and `mem_put` are implemented. In this function it just initializes all the `list` structures in front of each element and adds them to the `mem_pool->list` which represent the list of `cold` elements which can be allocated whenever `mem_get` is called on this mem_pool. It remembers mem_pool's start and end addresses in `mem_pool->pool`, `mem_pool->pool_end` respectively. Initializes `mem_pool->cold_count` to `count` and `mem_pool->hot_count` to `0`. This mem-pool will be added to the list of `global_list` maintained in `glusterfs-ctx` + + +``` +void* mem_get (struct mem_pool *mem_pool) + +Initial-list before mem-get +---------------- +| Pool | +| ----------- | ---------------------------------------- ---------------------------------------- +| | pool-list | |<---> |list-ptr|mem-pool-address|in-use|Element|<--->|list-ptr|mem-pool-address|in-use|Element| +| ----------- | ---------------------------------------- ---------------------------------------- +---------------- + +list after mem-get from the pool +---------------- +| Pool | +| ----------- | ---------------------------------------- +| | pool-list | |<--->|list-ptr|mem-pool-address|in-use|Element| +| ----------- | ---------------------------------------- +---------------- + +List when the pool is full: + ---------------- +| Pool | extra element that is allocated +| ----------- | ---------------------------------------- +| | pool-list | | |list-ptr|mem-pool-address|in-use|Element| +| ----------- | ---------------------------------------- + ---------------- +``` + +This function is similar to `malloc()` but it gives memory of type `element` of this pool. When this function is called it increments `mem_pool->alloc_count`, checks if there are any free elements in the pool that can be returned by inspecting `mem_pool->cold_count`. If `mem_pool->cold_count` is non-zero then it means there are elements in the pool which are not in active use. It deletes one element from the list of free elements and decrements `mem_pool->cold_count` and increments `mem_pool->hot_count` to indicate there is one more element in active use. Updates `mem_pool->max_alloc` accordingly. Sets `element->in_use` in the padded memory to `1`. Sets `element->mem_pool` address to this mem_pool also in the padded memory(It is useful for mem_put). Returns the address of the memory after the padded boundary to the caller of this function. In the cases where all the elements in the pool are in active use it `callocs` the element with padded size and sets mem_pool address in the padded memory. To indicate the pool-miss and give useful accounting information of the pool-usage it increments `mem_pool->pool_misses`, `mem_pool->curr_stdalloc`. Updates `mem_pool->max_stdalloc` accordingly. + +``` +void* mem_get0 (struct mem_pool *mem_pool) +``` +Just like `calloc` is to `malloc`, `mem_get0` is to `mem_get`. It memsets the memory to all '0' before returning the element. + + +``` +void mem_put (void *ptr) + +list before mem-put from the pool + ---------------- +| Pool | +| ----------- | ---------------------------------------- +| | pool-list | |<--->|list-ptr|mem-pool-address|in-use|Element| +| ----------- | ---------------------------------------- + ---------------- + +list after mem-put to the pool + ---------------- +| Pool | +| ----------- | ---------------------------------------- ---------------------------------------- +| | pool-list | |<---> |list-ptr|mem-pool-address|in-use|Element|<--->|list-ptr|mem-pool-address|in-use|Element| +| ----------- | ---------------------------------------- ---------------------------------------- + ---------------- + +If mem_put is putting an element not from pool then it is just freed so +no change to the pool + ---------------- +| Pool | +| ----------- | +| | pool-list | | +| ----------- | + ---------------- +``` + +This function is similar to `free()`. Remember that ptr passed to this function is the address of the element, so this function gets the ptr to its head of the padding in front of it. If this memory falls in bettween `mem_pool->pool`, `mem_pool->pool_end` then the memory is part of the 'pool' memory that is allocated so it does some sanity checks to see if the memory is indeed head of the element by checking if `in_use` is set to `1`. It resets `in_use` to `0`. It gets the mem_pool address stored in the padded region and adds this element to the list of free elements. Decreases `mem_pool->hot_count` increases `mem_pool->cold_count`. In the case where padded-element address does not fall in the range of `mem_pool->pool`, `mem_pool->pool_end` it just frees the element and decreases `mem_pool->curr_stdalloc`. + +``` +void +mem_pool_destroy (struct mem_pool *pool) +``` +Deletes this pool from the `global_list` maintained by `glusterfs-ctx` and frees all the memory allocated in `mem_pool_new`. + + +### How to pick pool-size +This varies from work-load to work-load. Create the mem-pool with some random size and run the work-load. Take the statedump after the work-load is complete. In the statedump if `max_alloc` is always less than `cold_count` may be reduce the size of the pool closer to `max_alloc`. On the otherhand if there are lots of `pool-misses` then increase the `pool_size` by `max_stdalloc` to achieve better 'hit-rate' of the pool. diff --git a/doc/developer-guide/dirops-transactions-in-dht.md b/doc/developer-guide/dirops-transactions-in-dht.md new file mode 100644 index 00000000000..909a97001aa --- /dev/null +++ b/doc/developer-guide/dirops-transactions-in-dht.md @@ -0,0 +1,273 @@ +# dirops transactions in dht +Need for transactions during operations on directories arise from two +basic design elements of DHT: + + 1. A directory is created on all subvolumes of dht. Since glusterfs + associates each file-system object with an unique gfid, every + subvolume should have the same unique mapping of (path of directory, + gfid). To elaborate, + * Each subvolume should've same gfid associated with a path to + directory. + * A gfid should not be associated with more than one path in any + subvolume. + + So, entire operations like mkdir, renamedir, rmdir and creation of + directories during self-heal need to be atomic in dht. In other words, + any of these operations shouldn't begin on an inode if one of them is + already in progress on the same inode, till it completes on all + subvolumes of dht. If not, more than one of these operations + happening in parallel can break any or all of the two requirements + listed above. This is referred in the rest of the document by the + name _Atomicity during namespace operations_. + + 2. Each directory has an independent layout persisted on + subvolumes. Each subvolume contains only part of the layout relevant + to it. For performance reasons _and_ since _only_ dht has aggregated + view, this layout is cached in memory of client. To make sure dht + reads or modifies a single complete layout while parallel modifications of the layout are in progress, we need atomicity during layout modification and reading. This is referred in the rest of the document as _Atomicity during layout modification and reading_. + +Rest of the document explains how atomicity is achieved for each of +the case above. + +**Atomicity during layout modification and reading** +File operations a.k.a fops can be classified into two categories based on how they consume layout. + + - Layout writer. Setting of layout during selfheal of a directory is + layout writer of _that_ directory. + - Layout reader. + * Any entry fop like create, unlink, rename, link, symlink, + unlink, mknod, rename, mkdir, rmdir, renamedir which needs layout of the parent directory. Each of these fops are readers of layout on parent directory. + * setting of layout during mkdir of a directory is considered as + a reader of the same directory's layout. The reason for this is that + only a parallel lookup on that directory can be a competing fop that modifies the layout (Other fops need gfid of the directory which can be got only after either lookup or mkdir completes). However, healing of layout is considered as a writer and a single writer blocks all readers. + +*Algorithm* +Atomicity is achieved by locking on the inode of directory whose +layout is being modified or read. The fop used is inodelk. + - Writer acquires blocking inodelk (directory-inode, write-lock) on + all subvolumes serially. The order of subvols in which they are + locked by different clients remains constant for a directory. If locking fails on any subvolume, layout modification is abandoned. + - Reader acquires an inodelk (directory-inode, read-lock) on _any_ + one subvolume. If locking fails on a subvolume (say with + ESTALE/ENOTCONN error), locking can be tried on other subvolumes till + we get one lock. If we cannot get lock on at least one subvolume, + consistency of layout is not guaranteed. Based on the consistency + requirements of fops, they can be failed or continued. + +Reasons why writer has to lock on _all_ subvols: + + - DHT don't have a metadata server and locking is implemented by brick. So, there is no well-defined subvol/brick that can be used as an arbitrator by different clients while acquiring locks. + - readers should acquire as minimum number of locks as possible. In + other words, the algorithm aims to have less synchronization cost to + readers. + - The subvolume to which a directory hashes could be used as a + lock server. However, in the case of an entry fop like create + (/a/b/c) where we want to block modification of layout of b for the + duration of create, we would be required to acquire lock on the + subvol to which /a/b hashes. To find out the hashed-subvol of + /a/b, we would need layout of /a. Note that how there is a dependency + of locking the layouts of ancestors all the way to root. So this + locking is not preferred. Also, note that only the immediate parent + inode is available in arguments of a fop like create. + +**Atomicity during namespace operations** + + - We use locks on inode of parent directory in the namespace of + _"basename"_ during mkdir, rmdir, renamedir and directory + creation phase of self-heal. The exact fop we use is _entrylk + (parent-inode, "basename")_. + - refresh in-memory layout of parent-inode from values stored on backend + - _entrylk (parent-inode, "basename")_ is done on subvolume to which + _"basename" hashes_. So, this operation is a _reader_ of the + layout on _parent-inode_. Which means an _inodelk (parent-inode, + read-lock)_ has to be completed before _entrylk (parent-inode, + "basename")_ is issued. Both the locks have to be held till the + operation is tried on all subvolumes. If acquiring of any/all of + these locks fail, the operation should be failed. + +With the above details, algorithms for mkdir, rmdir, renamedir, +self-heal of directory are explicitly detailed below. + +**Self-heal of a directory** + + - creation of directories on subvolumes is done only during + _named-lookup_ of a directory as we need < parent-inode, + "basename" >. + - If a directory is missing on one or more subvolumes, + * acquire _inodelk (parent-inode, read-lock)_ on _any one_ of the + subvolumes. + * refresh the in-memory layout of parent-inode from values stored on backend + * acquire _entrylk (parent-inode, "basename")_ on the subvolume + to which _"basename"_ hashes. + * If any/all of the locks fail, self-heal is aborted. + * create directories on missing subvolumes. + * release _entrylk (parent-inode, "basename")_. + * release _inodelk (parent-inode, read-lock)_. + + - If layout of a directory needs healing + * acquire _inodelk (directory-inode, write-lock)_ on _all_ the + subvolumes. If locking fails on any of the subvolumes, + self-heal is aborted. Blocking Locks are acquired serially across subvolumes in a _well-defined_ order which is _constant_ across all the healers of a directory. One order could be the order in which subvolumes are stored in the array _children_ of dht xlator. + * heal the layout. + * release _inodelk (directory-inode, write-lock)_ on _all_ the + subvolumes in parallel. + * Note that healing of layout can be done in both _named_ and + _nameless_ lookups of a directory as _only directory-inode_ is needed + for healing and it is available during both. + +**mkdir (parent-inode, "basename")** + +* while creating directory across subvolumes, + + - acquire _inodelk (parent-inode, read-lock)_ on _any one_ of the + subvolumes. + - refresh in-memory layout of parent-inode from values stored on backend + - acquire _entrylk (parent-inode, "basename")_ on the subvolume to + which _"basename"_ hashes. + - If any/all of the above two locks fail, release the locks that + were acquired successfully and mkdir should be failed (as perceived by application). + - do _mkdir (parent-inode, "basename")_ on the subvolume to which + _"basename"_ hashes. If this mkdir fails, mkdir is failed. + - do _mkdir (parent-inode, "basename")_ on the remaining subvolumes. + - release _entrylk (parent-inode, "basename")_. + - release _inodelk (parent-inode, "read-lock")_. +* while setting the layout of a directory, + - acquire _inodelk (directory-inode, read-lock)_ on _any one_ of the + subvolumes. + - If locking fails, cleanup the locks that were acquired + successfully and abort layout setting. Note that we'll have a + directory without a layout till a lookup happens on the + directory. This means entry operations within this directory fail + in this time window. We can also consider failing mkdir. The + problem of dealing with a directory without layout is out of the + scope of this document. + - set the layout on _directory-inode_. + - release _inodelk (directory-inode, read-lock)_. +* Note that during layout setting we consider mkdir as a _reader_ not + _writer_, though it is setting the layout. Reasons are: + - Before any of other readers like create, link etc that operate on + this directory to happen, _gfid_ of this directory has to be + resolved. But _gfid_ is only available only if either of following + conditions are true: + * after mkdir is complete. + * a lookup on the same path happens parallel to in-progress + mkdir. + + But, on completion of any of the above two operations, layout + will be healed. So, none of the _readers_ will happen on a + directory with partial layout. + +* Note that since we've an _entrylk (parent-inode, "basename")_ for + the entire duration of (attempting) creating directories, parallel + mkdirs will no longer contend on _mkdir_ on subvolume _to which "basename" hashes_. But instead, contend on _entrylk (parent-inode, "basename")_ on the subvolume _to which "basename" hashes_. So, we can attempt the _mkdir_ in _parallel_ on all subvolumes instead of two stage mkdir on hashed first and the rest of them in parallel later. However, we need to make sure that mkdir is successful on the subvolume _to which "basename" hashes_ for mkdir to be successful (as perceived by application). In the case of failed mkdir (as perceived by application), a cleanup should be performed on all the subvolumes before _entrylk (parent-inode, "basename")_ is released. + +**rmdir (parent-inode, "basename", directory-inode)** + + - acquire _inodelk (parent-inode, read-lock)_ on _any one_ + subvolume. + - refresh in-memory layout of parent-inode from values stored on backend + - acquire _entrylk (parent-inode, "basename")_ on the subvolume to + which _"basename" hashes_. + - If any/all of the above locks fail, rmdir is failed after cleanup + of the locks that were acquired successfully. + - do _rmdir (parent-inode, "basename")_ on the subvolumes to which + _"basename" doesn't hash to_. + * If successful, continue. + * Else, + * recreate directories on those subvolumes where rmdir + succeeded. + * heal the layout of _directory-inode_. Note that this will have + same synchronization requirements as discussed during layout + healing part of the section "Directoy self-heal" above. + * release _entrylk (parent-inode, "basename")_. + * release _inodelk (parent-inode, read-lock)_. + * fail _rmdir (parent-inode, "basename")_ to application. + - do _rmdir (parent-inode, "basename")_ on the subvolume to which + _"basename" hashes_. + - If successful, continue. + - Else, Go to the failure part of _rmdir (parent-inode, "basename")_ + on subvolumes to which "basename" _doesn't hash to_. + - release _entrylk (parent-inode, "basename")_. + - release _inodelk (parent-inode, read-lock)_. + - return success to application. + +**renamedir (src-parent-inode, "src-basename", src-directory-inode, dst-parent-inode, "dst-basename", dst-directory-inode)** + + - requirement is to prevent any operation in both _src-namespace_ + and _dst-namespace_. So, we need to acquire locks on both + namespaces.We also need to have constant ordering while acquiring + locks during parallel renames of the form _rename (src, dst)_ and + _rename (dst, src)_ to prevent deadlocks. We can sort gfids of + _src-parent-inode_ and _dst-parent-inode_ and use that order to + acquire locks. For the sake of explanation lets say we ended up + with order of _src_ followed by _dst_. + - acquire _inodelk (src-parent-inode, read-lock)_. + - refresh in-memory layout of src-parent-inode from values stored on backend + - acquire _entrylk (src-parent-inode, "src-basename")_. + - acquire _inodelk (dst-parent-inode, read-lock)_. + - refresh in-memory layout of dst-parent-inode from values stored on backend + - acquire _entrylk (dst-parent-inode, "dst-basename")_. + - If acquiring any/all of the locks above fail, + * release the locks that were successfully acquired. + * fail the renamedir operation to application + * done + - do _renamedir ("src", "dst")_ on the subvolume _to which "dst-basename" hashes_. + * If failure, Goto point _If acquiring any/all of the locks above fail_. + * else, continue. + - do _renamedir ("src", "dst")_ on rest of the subvolumes. + * If there is any failure, + * revert the successful renames. + * Goto to point _If acquiring any/all of the locks above fail_. + * else, + - release all the locks acquired. + - return renamedir as success to application. + +**Some examples of races** +This section gives concrete examples of races that can result in inconsistencies explained in the beginning of the document. + +Some assumptions are: + +* We consider an example distribute of three subvols s1, s2 and s3. +* For examples of renamedir ("src", "dst"), _src_ hashes to s1 and _dst_ hashes to s2. _src_ and _dst_ are associated with _gfid-src_ and _gfid-dst_ respectively +* For non renamedir examples, _dir_ is the name of directory and it hashes to s1. + +And the examples are: + + - mkdir vs rmdir - inconsistency in namespace. + * mkdir ("dir", gfid1) is complete on s1 + * rmdir is issued on same directory. Note that, since rmdir needs a gfid, a lookup should be complete before rmdir. lookup creates the directory on rest of the subvols as part of self-heal. + * rmdir (gfid1) deletes directory from all subvols. + * A new mkdir ("dir", gfid2) is issued. It is successful on s1 associating "dir" with gfid2. + * mkdir ("dir", gfid1) resumes and creates directory on s2 and s3 associating "dir" with gfid1. + * mkdir ("dir", gfid2) fails with EEXIST on s2 and s3. Since, EEXIST errors are ignored, mkdir is considered successful to application. + * In this example we have multiple inconsitencies + * "dir" is associated with gfid1 on s2, s3 and with gfid2 on s1 + * Even if mkdir ("dir", gfid2) was not issued, we would've a case of a directory magically reappearing after a successful rmdir. + - lookup heal vs rmdir + * rmdir ("dir", gfid1) is issued. It is successful on s2 and s3 (non-hashed subvols for name "dir") + * lookup ("dir") is issued. Since directory is present on s1 yet, it is created on s2 and s3 associating with gfid1 as part of self-heal + * rmdir ("dir", gfid1) is complete on s1 and it is successful + * Another lookup ("dir") creates the directory on s1 too + * "dir" magically reappears after a successful rmdir + - lookup heal (src) vs renamedir ("src", "dst") + * renamedir ("src", "dst") complete on s2 + * lookup ("src") recreates _src_ with _gfid-src_ on s2 + * renamedir ("src", "dst") completes on s1, s3. After rename is complete path _dst_ will be associated with gfid _gfid-src_ + * Another lookup ("src") recreates _src_ on subvols s1 and s3, associating it with gfid _gfid-src_ + * Inconsistencies are + * after a successful renamedir ("src", "dst"), both src and dst exist + * Two directories - src and dst - are associated with same gfid. One common symptom is that some entries (of the earlier _src_ and current _dst_ directory) being missed out in readdir listing as the gfid handle might be pointing to the empty healed directory than the actual directory containing entries + - lookup heal (dst) vs renamedir ("src", "dst") + * dst exists and empty when renamdir started + * dst doesn't exist when renamedir started + - renamedir ("src", "dst") complete on s2 and s3 + - lookup ("dst") creates _dst_ associating it with _gfid-src_ on s1 + - An entry is created in _dst_ on either s1 + - renamedir ("src", "dst") on s1 will result in a directory _dst/dst_ as _dst_ is no longer empty and _man 2 rename_ states that if _dst_ is not empty, _src_ is renamed _as a subdirectory of dst_ + - A lookup ( _dst/dst_) creates _dst/dst_ on s2 and s3 associating with _gfid-src_ as part of self-heal + - Inconsistencies are: + * Two directories - _dst_ and _dst/dst_ - exist even though both of them didn't exist at the beginning of renamedir + * Both _dst_ and _dst/dst_ have same gfid - _gfid-src_. As observed earlier, symptom might be directory listing being incomplete + - mkdir (dst) vs renamedir ("src", "dst") + - rmdir (src) vs renamedir ("src", "dst") + - rmdir (dst) vs renamedir ("src", "dst") diff --git a/doc/developer-guide/ec-implementation.md b/doc/developer-guide/ec-implementation.md new file mode 100644 index 00000000000..77e62583caa --- /dev/null +++ b/doc/developer-guide/ec-implementation.md @@ -0,0 +1,588 @@ +Erasure coding implementation +============================= + +This document provides information about how [erasure code][1] has +been implemented into ec translator. It describes the algorithm used +and the optimizations made, but it doesn't contain a full description +of the mathematical background needed to understand erasure coding in +general. It either describes the other parts of ec not directly +related to the encoding/decoding procedure, like synchronization or +fop management. + + +Introduction +------------ + +EC is based on [Reed-Solomon][2] erasure code. It's a very old code. +It's not considered the best one nowadays, but is good enough and it's +one of the few codes that is not covered by any patent and can be +freely used. + +To define the Reed-Solomon code we use 3 parameters: + + * __Key fragments (K)__ + It represents the minimum number of healthy fragments that will be + needed to be able to recover the original data. Any subset of K + out of the total number of fragments will serve. + + * __Redundancy fragments (R)__ + It represents the number of extra fragments to compute for each + original data block. This value determines how many fragments can + be lost before being unable to recover the original data. + + * __Fragment size (S)__ + This determines the size of each fragment. The original data + block size is computed as S * K. Currently this values is fixed + to 512 bytes. + + * __Total number of fragments (N = K + R)__ + This isn't a real parameter but it will be useful to simplify + the following descriptions. + +From the point of view of the implementation, it only consists on +matrix multiplications. There are two kinds of matrices to use for +Reed-Solomon: + + * __[Systematic][3]__ + This kind of matrix has the particularity that K of the encoded + fragments are simply a copy of the original data, divided into K + pieces. Thus no real encoding needs to be done for them and only + the R redundancy fragments need to be computed. + + This kind of matrices contain one KxK submatrix that is the + [identity matrix][4]. + + * __Non-systematic__ + This kind of matrix doesn't contain an identity submatrix. This + means that all of the N fragments need to be encoded, requiring + more computation. On the other hand, these matrices have some nice + properties that allow faster implementations of some algorithms, + like the matrix inversion used to decode the data. + + Another advantage of non-systematic matrices is that the decoding + time is constant, independently of how many fragments are lost, + while systematic approach can suffer from performance degradation + when one fragment is lost. + +All non-systematic matrices can be converted to systematic ones, but +then we lose the good properties of the non-systematic. We have to +choose betwee best peek performance (systematic) and performance +stability (non-systematic). + + +Encoding procedure +------------------ + +To encode a block of data we need a KxN matrix where each subset of K +rows is [linearly independent][5]. In other words, the determinant of +each KxK submatrix is not 0. + +There are some known ways to obtain this kind of matrices. EC uses a +small variation of a matrix known as [Vandermonde Matrix][6] where +each element of the matrix is defined as: + + a(i, j) = i ^ (K - j) + + where i is the row from 1 to N, and j is the column from 1 to K. + +This is exactly the Vandermonde Matrix but with the elements of each +row in reverse order. This change is made to be able to implement a +small optimization in the matrix multiplication. + +Once we have the matrix, we only need to compute the multiplication +of this matrix by a vector composed of K elements of data coming from +the original data block. + + / \ / \ + | 1 1 1 1 1 | / \ | a + b + c + d + e = t | + | 16 8 4 2 1 | | a | | 16a + 8b + 4c + 2d + e = u | + | 81 27 9 3 1 | | b | = | 81a + 27b + 9c + 3d + e = v | + | 256 64 16 4 1 | * | c | | 256a + 64b + 16c + 4d + e = w | + | 625 125 25 5 1 | | d | | 625a + 125b + 25c + 5d + e = x | + | 1296 216 36 6 1 | | e | | 1296a + 216b + 36c + 6d + e = y | + | 2401 343 49 7 1 | \ / | 2401a + 343b + 49c + 7d + e = z | + \ / \ / + +The optimization that can be done here is this: + + 16a + 8b + 4c + 2d + e = 2(2(2(2a + b) + c) + d) + e + +So all the multiplications are always by the number of the row (2 in +this case) and we don't need temporal storage for intermediate +results: + + a *= 2 + a += b + a *= 2 + a += c + a *= 2 + a += d + a *= 2 + a += e + +Once we have the result vector, each element is a fragment that needs +to be stored in a separate place. + + +Decoding procedure +------------------ + +To recover the data we need exactly K of the fragments. We need to +know which K fragments we have (i.e. the original row number from +which each fragment was calculated). Once we have this data we build +a square KxK matrix composed by the rows corresponding to the given +fragments and invert it. + +With the inverted matrix, we can recover the original data by +multiplying it with the vector composed by the K fragments. + +In our previous example, if we consider that we have recovered +fragments t, u, v, x and z, corresponding to rows 1, 2, 3, 5 and 7, +we can build the following matrix: + + / \ + | 1 1 1 1 1 | + | 16 8 4 2 1 | + | 81 27 9 3 1 | + | 625 125 25 5 1 | + | 2401 343 49 7 1 | + \ / + +And invert it: + + / \ + | 1/48 -1/15 1/16 -1/48 1/240 | + | -17/48 16/15 -15/16 13/48 -11/240 | + | 101/48 -86/15 73/16 -53/48 41/240 | + | -247/48 176/15 -129/16 83/48 -61/240 | + | 35/8 -7 35/8 -7/8 1/8 | + \ / + +Multiplying it by the vector (t, u, v, x, z) we recover the original +data (a, b, c, d, e): + + / \ / \ / \ + | 1/48 -1/15 1/16 -1/48 1/240 | | t | | a | + | -17/48 16/15 -15/16 13/48 -11/240 | | u | | b | + | 101/48 -86/15 73/16 -53/48 41/240 | * | v | = | c | + | -247/48 176/15 -129/16 83/48 -61/240 | | x | | d | + | 35/8 -7 35/8 -7/8 1/8 | | z | | e | + \ / \ / \ / + + +Galois Field +------------ + +This encoding/decoding procedure is quite complex to compute using +regular mathematical operations and it's not well suited for what +we want to do (note that matrix elements can grow unboundly). + +To solve this problem, exactly the same procedure is done inside a +[Galois Field][7] of characteristic 2, which is a finite field with +some interesting properties that make it specially useful for fast +operations using computers. + +There are two main differences when we use this specific Galois Field: + + * __All regular additions are replaced by bitwise xor's__ + For todays computers it's not really faster to execute an xor + compared to an addition, however replacing additions by xor's + inside a multiplication has many advantages (we will make use of + this to optimize the multiplication). + + Another consequence of this change is that additions and + substractions are really the same xor operation. + + * __The elements of the matrix are bounded__ + The field uses a modulus that keep all possible elements inside + a delimited region, avoiding really big numbers and fixing the + number of bits needed to represent each value. + + In the current implementation EC uses 8 bits per field element. + +It's very important to understand how multiplications are computed +inside a Galois Field to be able to understand how has it been +optimized. + +We'll start with a simple 'old school' multiplication but in base 2. +For example, if we want to multiply 7 * 5 (111b * 101b in binary), we +do the following: + + 1 1 1 (= 7) + * 1 0 1 (= 5) + ----------- + 1 1 1 (= 7) + + 0 0 0 (= 0) + + 1 1 1 (= 7) + ----------- + 1 0 0 0 1 1 (= 35) + +This is quite simple. Note that the addition of the third column +generates a carry that is propagated to all the other left columns. + +The next step is to define the modulus of the field. Suppose we use +11 as the modulus. Then we convert the result into an element of the +field by dividing by the modulus and taking the residue. We also use +the 'old school' method in binary: + + + 1 0 0 0 1 1 (= 35) | 1 0 1 1 (= 11) + - 0 0 0 0 ---------------- + --------- 0 1 1 (= 3) + 1 0 0 0 1 + - 1 0 1 1 + ----------- + 0 0 1 1 0 1 + - 1 0 1 1 + ------------- + 0 0 1 0 (= 2) + +So, 7 * 5 in a field with modulus 11 is 2. Note that the main +objective in each iteration of the division is to make higher bits +equal to 0 when possible (if it's not possible in one iteration, it +will be zeroed on the next). + +If we do the same but changing additions with xors we get this: + + 1 1 1 (= 7) + * 1 0 1 (= 5) + ----------- + 1 1 1 (= 7) + x 0 0 0 (= 0) + x 1 1 1 (= 7) + ----------- + 1 1 0 1 1 (= 27) + +In this case, the xor of the third column doesn't generate any carry. + +Now we need to divide by the modulus. We can also use 11 as the +modulus since it still satisfies the needed conditions to work on a +Galois Field of characteristic 2 with 3 bits: + + 1 1 0 1 1 (= 27) | 1 0 1 1 (= 11) + x 1 0 1 1 ---------------- + --------- 1 1 1 (= 7) + 0 1 1 0 1 + x 1 0 1 1 + ----------- + 0 1 1 0 1 + x 1 0 1 1 + ------------- + 0 1 1 0 (= 6) + +Note that, in this case, to make zero the higher bit we need to +consider the result of the xor operation, not the addition operation. + +So, 7 * 5 in a Galois Field of 3 bits with modulus 11 is 6. + + +Optimization +------------ + +To compute all these operations in a fast way some methods have been +traditionally used. Maybe the most common is the [lookup table][8]. + +The problem with this method is that it requires 3 lookups for each +byte multiplication, greatly amplifying the needed memory bandwidth +and making it difficult to take advantage of any SIMD support on the +processor. + +What EC does to improve the performance is based on the following +property (using the 3 bits Galois Field of the last example): + + A * B mod N = (A * b{2} * 4 mod N) x + (A * b{1} * 2 mod N) x + (A * b{0} mod N) + +This is basically a rewrite of the steps made in the previous example +to multiply two numbers but moving the modulus calculation inside each +intermediate result. What we can see here is that each term of the +xor can be zeroed if the corresponding bit of B is 0, so we can ignore +that factor. If the bit is 1, we need to compute A multiplied by a +power of two and take the residue of the division by the modulus. We +can precompute these values: + + A0 = A (we don't need to compute the modulus here) + A1 = A0 * 2 mod N + A2 = A1 * 2 mod N + +Having these values we only need to add those corresponding to bits +set to 1 in B. Using our previous example: + + A = 1 1 1 (= 7) + B = 1 0 1 (= 5) + + A0 = 1 1 1 (= 7) + A1 = 1 1 1 * 1 0 mod 1 0 1 1 = 1 0 1 (= 5) + A2 = 1 0 1 * 1 0 mod 1 0 1 1 = 0 0 1 (= 1) + + Since only bits 0 and 2 are 1 in B, we add A0 and A2: + + A0 + A2 = 1 1 1 x 0 0 1 = 1 1 0 (= 6) + +If we carefully look at what we are doing when computing each Ax, we +see that we do two basic things: + + - Shift the original value one bit to the left + - If the highest bit is 1, xor with the modulus + +Let's write this in a detailed way (representing each bit): + + Original value: a{2} a{1} a{0} + Shift 1 bit: a{2} a{1} a{0} 0 + + If a{2} is 0 we already have the result: + a{1} a{0} 0 + + If a{2} is 1 we need to xor with the modulus: + 1 a{1} a{0} 0 x 1 0 1 1 = a{1} (a{0} x 1) 1 + +An important thing to see here is that if a{2} is 0, we can get the +same result by xoring with all 0 instead of the modulus. For this +reason we can rewrite the modulus as this: + + Modulus: a{2} 0 a{2} a{2} + +This means that the modulus will be 0 0 0 0 is a{2} is 0, so the value +won't change, and it will be 1 0 1 1 if a{2} is 1, giving the correct +result. So, the computation is simply: + + Original value: a{2} a{1} a{0} + Shift 1 bit: a{2} a{1} a{0} 0 + Apply modulus: a{1} (a{0} x a{2}) a{2} + +We can compute all Ax using this method. We'll get this: + + A0 = a{2} a{1} a{0} + A1 = a{1} (a{0} x a{2}) a{2} + A2 = (a{0} x a{2}) (a{1} x a{2}) a{1} + +Once we have all terms, we xor the ones corresponding to the bits set +to 1 in B. In out example this will be A0 and A2: + + Result: (a{2} x a{0} x a{2}) (a{1} x a{1} x a{2}) (a{0} x a{1}) + +We can easily see that we can remove some redundant factors: + + Result: a{0} a{2} (a{0} x a{1}) + +This way we have come up with a simply set of equations to compute the +multiplication of any number by 5. If A is 1 1 1 (= 7), the result +must be 1 1 0 (= 6) using the equations, as we expected. If we try +another numbe for A, like 0 1 0 (= 2), the result must be 0 0 1 (= 1). + +This seems a really fast way to compute the multiplication without +using any table lookup. The problem is that this is only valid for +B = 5. For other values of B another set of equations will be found. +To solve this problem we can pregenerate the equations for all +possible values of B. Since the Galois Field we use is small, this is +feasible. + +One thing to be aware of is that, in general, two equations for +different bits of the same B can share common subexpressions. This +gives space for further optimizations to reduce the total number of +xors used in the final equations for a given B. However this is not +easy to find, since finding the smallest number of xors that give the +correct result is an NP-Problem. For EC an exhaustive search has been +made to find the best combinations for each possible value. + + +Implementation +-------------- + +All this seems great from the hardware point of view, but implementing +this using normal processor instructions is not so easy because we +would need a lot of shifts, ands, xors and ors to move the bits of +each number to the correct position to compute the equation and then +another shift to put each bit back to its final place. + +For example, to implement the functions to multiply by 5, we would +need something like this: + + Bit 2: T2 = (A & 1) << 2 + Bit 1: T1 = (A & 4) >> 1 + Bit 0: T0 = ((A >> 1) x A) & 1 + Result: T2 + T1 + T0 + +This doesn't look good. So here we make a change in the way we get +and process the data: instead of reading full numbers into variables +and operate with them afterwards, we use a single independent variable +for each bit of the number. + +Assume that we can read and write independent bits from memory (later +we'll see how to solve this problem when this is not possible). In +this case, the code would look something like this: + + Bit 2: T2 = Mem[2] + Bit 1: T1 = Mem[1] + Bit 0: T0 = Mem[0] + Computation: T1 ^= T0 + Store result: Mem[2] = T0 + Mem[1] = T2 + Mem[0] = T1 + +Note that in this case we handle the final reordering of bits simply +by storing the right variable to the right place, without any shifts, +ands nor ors. In fact we only have memory loads, memory stores and +xors. Note also that we can do all the computations directly using the +variables themselves, without additional storage. This true for most +of the values, but in some cases an additional one or two temporal +variables will be needed to store intermediate results. + +The drawback of this approach is that additions, that are simply a +xor of two numbers will need as many xors as bits are in each number. + + +SIMD optimization +----------------- + +So we have a good way to compute the multiplications, but even using +this we'll need several operations for each byte of the original data. +We can improve this by doing multiple multiplications using the same +set of instructions. + +With the approach taken in the implementation section, we can see that +in fact it's really easy to add SIMD support to this method. We only +need to store in each variable one bit from multiple numbers. For +example, when we load T2 from memory, instead of reading the bit 2 of +the first number, we can read the bit 2 of the first, second, third, +fourth, ... numbers. The same can be done when loading T1 and T0. + +Obviously this needs to have a special encoding of the numbers into +memory to be able to do that in a single operation, but since we can +choose whatever encoding we want for EC, we have chosen to have +exactly that. We interpret the original data as a stream of bits, and +we split it into subsequences of length L, each containing one bit of +a number. Every S subsequences form a set of numbers of S bits that +are encoded and decoded as a single group. This repeats for any +remaining data. + +For example, in a simple case with L = 8 and S = 3, the original data +would contain something like this (interpreted as a sequence of bits, +offsets are also bit-based): + + Offset 0: a{0} b{0} c{0} d{0} e{0} f{0} g{0} h{0} + Offset 8: a{1} b{1} c{1} d{1} e{1} f{1} g{1} h{1} + Offset 16: a{2} b{2} c{2} d{2} e{2} f{2} g{2} h{2} + Offset 24: i{0} j{0} k{0} l{0} m{0} n{0} o{0} p{0} + Offset 32: i{1} j{1} k{1} l{1} m{1} n{1} o{1} p{1} + Offset 40: i{2} j{2} k{2} l{2} m{2} n{2} o{2} p{2} + +Note: If the input file is not a multiple of S * L, 0-padding is done. + +Here we have 16 numbers encoded, from A to P. This way we can easily +see that reading the first byte of the file will read all bits 0 of +number A, B, C, D, E, F, G and H. The same happens with bits 1 and 2 +when we read the second and third bytes respectively. Using this +encoding and the implementation described above, we can see that the +same set of instructions will be computing the multiplication of 8 +numbers at the same time. + +This can be further improved if we use L = 64 with 64 bits variables +on 64-bits processor. It's even faster if we use L = 128 using SSE +registers or L = 256 using AVX registers on Intel processors. + +Currently EC uses L = 512 and S = 8. This means that numbers are +packed in blocks of 512 bytes and gives space for even bigger +processor registers up to 512 bits. + + +Conclusions +----------- + +This method requires a single variable/processor register for each +bit. This can be challenging if we want to avoid additional memory +accesses, even if we use modern processors that have many registers. +However, the implementation we chose for the Vandermonde Matrix +doesn't require temporary storage, so we don't need a full set of 8 +new registers (one for each bit) to store partial computations. +Additionally, the computation of the multiplications requires, at +most, 2 extra registers, but this is afordable. + +Xors are a really fast operation in modern processors. Intel CPU's +can dispatch up to 3 xors per CPU cycle if there are no dependencies +with ongoing previous instructions. Worst case is 1 xor per cycle. So, +in some configurations, this method could be very near to the memory +speed. + +Another interesting thing of this method is that all data it needs to +operate is packed in small sequential blocks of memory, meaning that +it can take advantage of the faster internal CPU caches. + + +Results +------- + +For the particular case of 8 bits, EC can compute each multiplication +using 12.8 xors on average (without counting 0 and 1 that do not +require any xor). Some numbers require less, like 2 that only requires +3 xors. + +Having all this, we can check some numbers to see the performance of +this method. + +Maybe the most interesting thing is the average number of xors needed +to encode a single byte of data. To compute this we'll need to define +some variables: + + * K: Number of data fragments + * R: Number of redundancy fragments + * N: K + R + * B: Number of bits per number + * A: Average number of xors per number + * Z: Bits per CPU register (can be up to 256 for AVX registers) + * X: Average number of xors per CPU cycle + * L: Average cycles per load + * S: Average cycles per store + * G: Core speed in Hz + +_Total number of bytes processed for a single matrix multiplication_: + + * __Read__: K * B * Z / 8 + * __Written__: N * B * Z / 8 + +_Total number of memory accesses_: + + * __Loads__: K * B * N + * __Stores__: B * N + +> We need to read the same K * B * Z bits, in registers of Z bits, N +> times, one for each row of the matrix. However the last N - 1 reads +> could be made from the internal CPU caches if conditions are good. + +_Total number of operations_: + + * __Additions__: (K - 1) * N + * __Multiplications__: K * N + +__Total number of xors__: B * (K - 1) * N + A * K * N = + N * ((A + B) * K - B) + +__Xors per byte__: 8 * N * ((A + B) * K - B) / (K * B * Z) + +__CPU cycles per byte__: 8 * N * ((A + B) * K - B) / (K * B * Z * X) + + 8 * L * N / Z + (loads) + 8 * S * N / (K * Z) (stores) + +__Bytes per second__: G / {CPU cycles per byte} + +Some xors per byte numbers for specific configurations (B=8): + + Z=64 Z=128 Z=256 + K=2/R=1 0.79 0.39 0.20 + K=4/R=2 1.76 0.88 0.44 + K=4/R=3 2.06 1.03 0.51 + K=8/R=3 3.40 1.70 0.85 + K=8/R=4 3.71 1.86 0.93 + K=16/R=4 6.34 3.17 1.59 + + + +[1]: https://en.wikipedia.org/wiki/Erasure_code +[2]: https://en.wikipedia.org/wiki/Reed%E2%80%93Solomon_error_correction +[3]: https://en.wikipedia.org/wiki/Systematic_code +[4]: https://en.wikipedia.org/wiki/Identity_matrix +[5]: https://en.wikipedia.org/wiki/Linear_independence +[6]: https://en.wikipedia.org/wiki/Vandermonde_matrix +[7]: https://en.wikipedia.org/wiki/Finite_field +[8]: https://en.wikipedia.org/wiki/Finite_field_arithmetic#Implementation_tricks diff --git a/doc/developer-guide/fuse-interrupt.md b/doc/developer-guide/fuse-interrupt.md new file mode 100644 index 00000000000..ec991b81ec5 --- /dev/null +++ b/doc/developer-guide/fuse-interrupt.md @@ -0,0 +1,211 @@ +# Fuse interrupt handling + +## Conventions followed + +- *FUSE* refers to the "wire protocol" between kernel and userspace and + related specifications. +- *fuse* refers to the kernel subsystem and also to the GlusterFs translator. + +## FUSE interrupt handling spec + +The [Linux kernel FUSE documentation](https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/Documentation/filesystems/fuse.txt?h=v4.18#n148) +desrcibes how interrupt handling happens in fuse. + +## Interrupt handling in the fuse translator + +### Declarations + +This document describes the internal API in the fuse translator with which +interrupt can be handled. + +The API being internal (to be used only in fuse-bridge.c; the functions are +not exported to a header file). + +``` +enum fuse_interrupt_state { + /* ... */ + INTERRUPT_SQUELCHED, + INTERRUPT_HANDLED, + /* ... */ +}; +typedef enum fuse_interrupt_state fuse_interrupt_state_t; +struct fuse_interrupt_record; +typedef struct fuse_interrupt_record fuse_interrupt_record_t; +typedef void (*fuse_interrupt_handler_t)(xlator_t *this, + fuse_interrupt_record_t *); +struct fuse_interrupt_record { + fuse_in_header_t fuse_in_header; + void *data; + /* + ... + */ +}; + +fuse_interrupt_record_t * +fuse_interrupt_record_new(fuse_in_header_t *finh, + fuse_interrupt_handler_t handler); + +void +fuse_interrupt_record_insert(xlator_t *this, fuse_interrupt_record_t *fir); + +gf_boolean_t +fuse_interrupt_finish_fop(call_frame_t *frame, xlator_t *this, + gf_boolean_t sync, void **datap); + +void +fuse_interrupt_finish_interrupt(xlator_t *this, fuse_interrupt_record_t *fir, + fuse_interrupt_state_t intstat, + gf_boolean_t sync, void **datap); +``` + +The code demonstrates the usage of the API through `fuse_flush()`. (It's a +dummy implementation only for demonstration purposes.) Flush is chosen +because a `FLUSH` interrupt is easy to trigger (see +*tests/features/interrupt.t*). Interrupt handling for flush is switched on +by `--fuse-flush-handle-interrupt` (a hidden glusterfs command line flag). +The implementation of flush interrupt is contained in the +`fuse_flush_interrupt_handler()` function and blocks guarded by the + +``` +if (priv->flush_handle_interrupt) { ... +``` + +conditional (where `priv` is a `*fuse_private_t`). + +### Overview + +"Regular" fuse fops and interrupt handlers interact via a list containing +interrupt records. + +If a fop wishes to have its interrupts handled, it needs to set up an +interrupt record and insert it into the list; also when it's to finish +(ie. in its "cbk" stage) it needs to delete the record from the list. + +If no interrupt happens, basically that's all to it - a list insertion +and deletion. + +However, if an interrupt comes for the fop, the interrupt FUSE request +will carry the data identifying an ongoing fop (that is, its `unique`), +and based on that, the interrupt record will be looked up in the list, and +the specific interrupt handler (a member of the interrupt record) will be +called. + +Usually the fop needs to share some data with the interrupt handler to +enable it to perform its task (also shared via the interrupt record). +The interrupt API offers two approaches to manage shared data: +- _Async or reference-counting strategy_: from the point on when the interrupt + record is inserted to the list, it's owned jointly by the regular fop and + the prospective interrupt handler. Both of them need to check before they + return if the other is still holding a reference; if not, then they are + responsible for reclaiming the shared data. +- _Sync or borrow strategy_: the interrupt handler is considered a borrower + of the shared data. The interrupt handler should not reclaim the shared + data. The fop will wait for the interrupt handler to finish (ie., the borrow + to be returned), then it has to reclaim the shared data. + +The user of the interrupt API need to call the following functions to +instrument this control flow: +- `fuse_interrupt_record_insert()` in the fop to insert the interrupt record to + the list; +- `fuse_interrupt_finish_fop()`in the fop (cbk) and +- `fuse_interrupt_finish_interrupt()`in the interrupt handler + +to perform needed synchronization at the end their tenure. The data management +strategies are implemented by the `fuse_interrupt_finish_*()` functions (which +have an argument to specify which strategy to use); these routines take care +of freeing the interrupt record itself, while the reclamation of the shared data +is left to the API user. + +### Usage + +A given FUSE fop can be enabled to handle interrupts via the following +steps: + +- Define a handler function (of type `fuse_interrupt_handler_t`). + It should implement the interrupt handling logic and in the end + call (directly or as async callback) `fuse_interrupt_finish_interrupt()`. + The `intstat` argument to `fuse_interrupt_finish_interrupt` should be + either `INTERRUPT_SQUELCHED` or `INTERRUPT_HANDLED`. + - `INTERRUPT_SQUELCHED` means that the interrupt could not be delivered + and the fop is going on uninterrupted. + - `INTERRUPT_HANDLED` means that the interrupt was actually handled. In + this case the fop will be answered from interrupt context with errno + `EINTR` (that is, the fop should not send a response to the kernel). + + (the enum `fuse_interrupt_state` includes further members, which are reserved + for internal use). + + We return to the `sync` and `datap` arguments later. +- In the `fuse_<FOP>` function create an interrupt record using + `fuse_interrupt_record_new()`, passing the incoming `fuse_in_header` and + the above handler function to it. + - Arbitrary further data can be referred to via the `data` member of the + interrupt record that is to be passed on from fop context to + interrupt context. +- When it's set up, pass the interrupt record to + `fuse_interrupt_record_insert()`. +- In `fuse_<FOP>_cbk` call `fuse_interrupt_finish_fop()`. + - `fuse_interrupt_finish_fop()` returns a Boolean according to whether the + interrupt was handled. If it was, then the FUSE request is already + answered and the stack gets destroyed in `fuse_interrupt_finish_fop` so + `fuse_<FOP>_cbk()` can just return (zero). Otherwise follow the standard + cbk logic (answer the FUSE request and destroy the stack -- these are + typically accomplished by `fuse_err_cbk()`). +- The last two argument of `fuse_interrupt_finish_fop()` and + `fuse_interrupt_finish_interrupt()` are `gf_boolean_t sync` and + `void **datap`. + - `sync` represents the strategy for freeing the interrupt record. The + interrupt handler and the fop handler are in race to get at the interrupt + record first (interrupt handler for purposes of doing the interrupt + handling, fop handler for purposes of deactivating the interrupt record + upon completion of the fop handling). + - If `sync` is true, then the fop handler will wait for the interrupt + handler to finish and it takes care of freeing. + - If `sync` is false, the loser of the above race will perform freeing. + + Freeing is done within the respective interrupt finish routines, except + for the `data` field of the interrupt record; with respect to that, see + the discussion of the `datap` parameter below. The strategy has to be + consensual, that is, `fuse_interrupt_finish_fop()` and + `fuse_interrupt_finish_interrupt()` must pass the same value for `sync`. + If dismantling the resources associated with the interrupt record is + simple, `sync = _gf_false` is the suggested choice; `sync = _gf_true` can + be useful in the opposite case, when dismantling those resources would + be inconvenient to implement in two places or to enact in non-fop context. + - If `datap` is `NULL`, the `data` member of the interrupt record will be + freed within the interrupt finish routine. If it points to a valid + `void *` pointer, and if caller is doing the cleanup (see `sync` above), + then that pointer will be directed to the `data` member of the interrupt + record and it's up to the caller what it's doing with it. + - If `sync` is true, interrupt handler can use `datap = NULL`, and + fop handler will have `datap` point to a valid pointer. + - If `sync` is false, and handlers pass a pointer to a pointer for + `datap`, they should check if the pointed pointer is NULL before + attempting to deal with the data. + +### FUSE answer for the interrupted fop + +The kernel acknowledges a successful interruption for a given FUSE request +if the filesystem daemon answers it with errno EINTR; upon that, the syscall +which induced the request will be abruptly terminated with an interrupt, rather +than returning a value. + +In glusterfs, this can be arranged in two ways. + +- If the interrupt handler wins the race for the interrupt record, ie. + `fuse_interrupt_finish_fop()` returns true to `fuse_<FOP>_cbk()`, then, as + said above, `fuse_<FOP>_cbk()` does not need to answer the FUSE request. + That's because then the interrupt handler will take care about answering + it (with errno EINTR). +- If `fuse_interrupt_finish_fop()` returns false to `fuse_<FOP>_cbk()`, then + this return value does not inform the fop handler whether there was an interrupt + or not. This return value occurs both when fop handler won the race for the + interrupt record against the interrupt handler, and when there was no interrupt + at all. + + However, the internal logic of the fop handler might detect from other + circumstances that an interrupt was delivered. For example, the fop handler + might be sleeping, waiting for some data to arrive, so that a premature + wakeup (with no data present) occurs if the interrupt handler intervenes. In + such cases it's the responsibility of the fop handler to reply the FUSE + request with errro EINTR. diff --git a/doc/developer-guide/gfapi-symbol-versions.md b/doc/developer-guide/gfapi-symbol-versions.md new file mode 100644 index 00000000000..e4f4fe9f052 --- /dev/null +++ b/doc/developer-guide/gfapi-symbol-versions.md @@ -0,0 +1,270 @@ + +## Symbol Versions and SO_NAMEs + + In general, adding new APIs to a shared library does not require that +symbol versions be used or the the SO_NAME be "bumped." These actions +are usually reserved for when a major change is introduced, e.g. many +APIs change or a signficant change in the functionality occurs. + + Over the normal lifetime of a When a new API is added, the library is +recompiled, consumers of the new API are able to do so, and existing, +legacy consumers of the original API continue as before. If by some +chance an old copy of the library is installed on a system, it's unlikely +that most applications will be affected. New applications that use the +new API will incur a run-time error terminate. + + Bumping the SO_NAME, i.e. changing the shared lib's file name, e.g. +from libfoo.so.0 to libfoo.so.1, which also changes the ELF SO_NAME +attribute inside the file, works a little differently. libfoo.so.0 +contains only the old APIs. libfoo.so.1 contains both the old and new +APIs. Legacy software that was linked with libfoo.so.0 continues to work +as libfoo.so.0 is usually left installed on the system. New software that +uses the new APIs is linked with libfoo.so.1, and works as long as +long as libfoo.so.1 is installed on the system. Accidentally (re)installing +libfoo.so.0 doesn't break new software as long as reinstalling doesn't +erase libfoo.so.1. + + Using symbol versions is somewhere in the middle. The shared library +file remains libfoo.so.0 forever. Legacy APIs may or may not have an +associated symbol version. New APIs may or may not have an associated +symbol version either. In general symbol versions are reserved for APIs +that have changed. Either the function's signature has changed, i.e. the +return time or the number of paramaters, and/or the parameter types have +changed. Another reason for using symbol versions on an API is when the +behaviour or functionality of the API changes dramatically. As with a +library that doesn't use versioned symbols, old and new applications +either find or don't find the versioned symbols they need. If the versioned +symbol doesn't exist in the installed library, the application incurs a +run-time error and terminates. + + GlusterFS wanted to keep tight control over the APIs in libgfapi. +Originally bumping the SO_NAME was considered, and GlusterFS-3.6.0 was +released with libgfapi.so.7. Not only was "7" a mistake (it should have +been "6"), but it was quickly pointed out that many dependent packages +that use libgfapi would be forced to be recompiled/relinked. Thus no +packages of 3.6.0 were ever released and 3.6.1 was quickly released with +libgfapi.so.0, but with symbol versions. There's no strong technical +reason for either; the APIs have not changed, only new APIs have been +added. It's merely being done in anticipation that some APIs might change +sometime in the future. + + Enough about that now, let's get into the nitty gritty—— + +## Adding new APIs + +### Adding a public API. + + This is the default, and the easiest thing to do. Public APIs have +declarations in either glfs.h, glfs-handles.h, or, at your discretion, +in a new header file intended for consumption by other developers. + +Here's what you need to do to add a new public API: + ++ Write the declaration, e.g. in glfs.h: + +```C + int glfs_dtrt (const char *volname, void *stuff) __THROW +``` + ++ Write the definition, e.g. in glfs-dtrt.c: + +```C + int + pub_glfs_dtrt (const char *volname, void *stuff) + { + ... + return 0; + } +``` + ++ Add the symbol version magic for ELF, gnu toolchain to the definition. + + following the definition of your new function in glfs-dtrtops.c, add a + line like this: + +```C + GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_dtrt, 3.7.0) +``` + + The whole thing should look like: + +```C + int + pub_glfs_dtrt (const char *volname, void *stuff) + { + ... + } + GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_dtrt, 3.7.0); +``` + + In this example, 3.7.0 refers to the Version the symbol will first + appear in. There's nothing magic about it, it's just a string token. + The current versions we have are 3.4.0, 3.4.2, 3.5.0, 3.5.1, and 3.6.0. + They are to be considered locked or closed. You can not, must not add + any new APIs and use these versions. Most new APIs will use 3.7.0. If + you add a new API appearing in 3.6.2 (and mainline) then you would use + 3.6.2. + ++ Add the symbol version magic for OS X to the declaration. + + following the declaration in glfs.h, add a line like this: + +```C + GFAPI_PUBLIC(glfs_dtrt, 3.7.0) +``` + + The whole thing should look like: + +```C + int glfs_dtrt (const char *volname, void *stuff) __THROW + GFAPI_PUBLIC(glfs_dtrt, 3.7.0); +``` + + The version here must match the version associated with the definition. + ++ Add the new API to the ELF, gnu toolchain link map file, gfapi.map + + Most new public APIs will probably be added to a new section that + looks like this: + +``` + GFAPI_3.7.0 { + global: + glfs_dtrt; + } GFAPI_PRIVATE_3.7.0; +``` + + if you're adding your new API to, e.g. 3.6.2, it'll look like this: + +``` + GFAPI_3.6.2 { + global: + glfs_dtrt; + } GFAPI_3.6.0; +``` + + and you must change the +``` + GFAPI_PRIVATE_3.7.0 { ...} GFAPI_3.6.0; +``` + section to: +``` + GFAPI_PRIVATE_3.7.0 { ...} GFAPI_3.6.2; +``` + ++ Add the new API to the OS X alias list file, gfapi.aliases. + + Most new APIs will use a line that looks like this: + +```C + _pub_glfs_dtrt _glfs_dtrt$GFAPI_3.7.0 +``` + + if you're adding your new API to, e.g. 3.6.2, it'll look like this: + +```C + _pub_glfs_dtrt _glfs_dtrt$GFAPI_3.6.2 +``` + +And that's it. + + +### Adding a private API. + + If you're thinking about adding a private API that isn't declared in +one of the header files, then you should seriously rethink what you're +doing and figure out how to put it in libglusterfs instead. + +If that hasn't convinced you, follow the instructions above, but use the +_PRIVATE versions of macros, symbol versions, and aliases. If you're 1337 +enough to ignore this advice, then you're 1337 enough to figure out how +to do it. + + +## Changing an API. + +### Changing a public API. + + There are two ways an API might change, 1) its signature has changed, or +2) its new functionality or behavior is substantially different than the +old. An APIs signature consists of the function return type, and the number +and/or type of its parameters. E.g. the original API: + +```C + int glfs_dtrt (const char *volname, void *stuff); +``` + +and the changed API: + +```C + void *glfs_dtrt (const char *volname, glfs_t *ctx, void *stuff); +``` + + One way to avoid a change like this, and which is preferable in many +ways, is to leave the legacy glfs_dtrt() function alone, document it as +deprecated, and simply add a new API, e.g. glfs_dtrt2(). Practically +speaking, that's effectively what we'll be doing anyway, the difference +is only that we'll use a versioned symbol to do it. + + On the assumption that adding a new API is undesirable for some reason, +perhaps the use of glfs_gnu() is just so pervasive that we really don't +want to add glfs_gnu2(). + ++ change the declaration in glfs.h: + +```C + glfs_t *glfs_gnu (const char *volname, void *stuff) __THROW + GFAPI_PUBLIC(glfs_gnu, 3.7.0); +```` + +Note that there is only the single, new declaration. + ++ change the old definition of glfs_gnu() in glfs.c: + +```C + struct glfs * + pub_glfs_gnu340 (const char * volname) + { + ... + } + GFAPI_SYMVER_PUBLIC(glfs_gnu340, glfs_gnu, 3.4.0); +``` + ++ create the new definition of glfs_gnu in glfs.c: + +```C + struct glfs * + pub_glfs_gnu (const char * volname, void *stuff) + { + ... + } + GFAPI_SYMVER_PUBLIC_DEFAULT(glfs_gnu, 3.7.0); +``` + ++ Add the new API to the ELF, gnu toolchain link map file, gfapi.map + +``` + GFAPI_3.7.0 { + global: + glfs_gnu; + } GFAPI_PRIVATE_3.7.0; +``` + ++ Update the OS X alias list file, gfapi.aliases, for both versions: + +Change the old line: +```C + _pub_glfs_gnu _glfs_gnu$GFAPI_3.4.0 +``` +to: +```C + _pub_glfs_gnu340 _glfs_gnu$GFAPI_3.4.0 +``` + +Add a new line: +```C + _pub_glfs_gnu _glfs_gnu$GFAPI_3.7.0 +``` + ++ Lastly, change all gfapi internal calls glfs_gnu to the new API. + diff --git a/doc/developer-guide/identifying-resource-leaks.md b/doc/developer-guide/identifying-resource-leaks.md new file mode 100644 index 00000000000..950cae79b0a --- /dev/null +++ b/doc/developer-guide/identifying-resource-leaks.md @@ -0,0 +1,200 @@ +# Identifying Resource Leaks + +Like most other pieces of software, GlusterFS is not perfect in how it manages +its resources like memory, threads and the like. Gluster developers try hard to +prevent leaking resources but releasing and unallocating the used structures. +Unfortunately every now and then some resource leaks are unintentionally added. + +This document tries to explain a few helpful tricks to identify resource leaks +so that they can be addressed. + + +## Debug Builds + +There are certain techniques used in GlusterFS that make it difficult to use +tools like Valgrind for memory leak detection. There are some build options +that make it more practical to use Valgrind and other tools. When running +Valgrind, it is important to have GlusterFS builds that contain the +debuginfo/symbols. Some distributions (try to) strip the debuginfo to get +smaller executables. Fedora and RHEL based distributions have sub-packages +called ...-debuginfo that need to be installed for symbol resolving. + + +### Memory Pools + +By using memory pools, there are no allocation/freeing of single structures +needed. This improves performance, but also makes it impossible to track the +allocation and freeing of srtuctures. + +It is possible to disable the use of memory pools, and use standard `malloc()` +and `free()` functions provided by the C library. Valgrind is then able to +track the allocated areas and verify if they have been free'd. In order to +disable memory pools, the Gluster sources needs to be configured with the +`--enable-debug` option: + +```shell +./configure --enable-debug +``` + +When building RPMs, the `.spec` handles the `--with=debug` option too: + +```shell +make dist +rpmbuild -ta --with=debug glusterfs-....tar.gz +``` + +### Dynamically Loaded xlators + +Valgrind tracks the call chain of functions that do memory allocations. The +addresses of the functions are stored and before Valgrind exits the addresses +are resolved into human readable function names and offsets (line numbers in +source files). Because Gluster loads xlators dynamically, and unloads then +before exiting, Valgrind is not able to resolve the function addresses into +symbols anymore. Whenever this happend, Valgrind shows `???` in the output, +like + +``` + ==25170== 344 bytes in 1 blocks are definitely lost in loss record 233 of 324 + ==25170== at 0x4C29975: calloc (vg_replace_malloc.c:711) + ==25170== by 0x52C7C0B: __gf_calloc (mem-pool.c:117) + ==25170== by 0x12B0638A: ??? + ==25170== by 0x528FCE6: __xlator_init (xlator.c:472) + ==25170== by 0x528FE16: xlator_init (xlator.c:498) + ... +``` + +These `???` can be prevented by not calling `dlclose()` for unloading the +xlator. This will cause a small leak of the handle that was returned with +`dlopen()`, but for improved debugging this can be acceptible. For this and +other Valgrind features, a `--enable-valgrind` option is available to +`./configure`. When GlusterFS is built with this option, Valgrind will be able +to resolve the symbol names of the functions that do memory allocations inside +xlators. + +```shell +./configure --enable-valgrind +``` + +When building RPMs, the `.spec` handles the `--with=valgrind` option too: + +```shell +make dist +rpmbuild -ta --with=valgrind glusterfs-....tar.gz +``` + +## Running Valgrind against a single xlator + +Debugging a single xlator is not trivial. But there are some tools to make it +easier. The `sink` xlator does not do any memory allocations itself, but +contains just enough functionality to mount a volume with only the `sink` +xlator. There is a little gfapi application under `tests/basic/gfapi/` in the +GlusterFS sources that can be used to run only gfapi and the core GlusterFS +infrastructure with the `sink` xlator. By extending the `.vol` file to load +more xlators, each xlator can be debugged pretty much separately (as long as +the xlators have no dependencies on each other). A basic Valgrind run with the +suitable configure options looks like this: + +```shell +./autogen.sh +./configure --enable-debug --enable-valgrind +make && make install +cd tests/basic/gfapi/ +make gfapi-load-volfile +valgrind ./gfapi-load-volfile sink.vol +``` + +Combined with other very useful options to Valgrind, the following execution +shows many more useful details: + +```shell +valgrind \ + --fullpath-after= --leak-check=full --show-leak-kinds=all \ + ./gfapi-load-volfile sink.vol +``` + +Note that the `--fullpath-after=` option is left empty, this makes Valgrind +print the full path and filename that contains the functions: + +``` +==2450== 80 bytes in 1 blocks are definitely lost in loss record 8 of 60 +==2450== at 0x4C29975: calloc (/builddir/build/BUILD/valgrind-3.11.0/coregrind/m_replacemalloc/vg_replace_malloc.c:711) +==2450== by 0x52C6F73: __gf_calloc (/usr/src/debug/glusterfs-3.11dev/libglusterfs/src/mem-pool.c:117) +==2450== by 0x12F10CDA: init (/usr/src/debug/glusterfs-3.11dev/xlators/meta/src/meta.c:231) +==2450== by 0x528EFD5: __xlator_init (/usr/src/debug/glusterfs-3.11dev/libglusterfs/src/xlator.c:472) +==2450== by 0x528F105: xlator_init (/usr/src/debug/glusterfs-3.11dev/libglusterfs/src/xlator.c:498) +==2450== by 0x52D9D8B: glusterfs_graph_init (/usr/src/debug/glusterfs-3.11dev/libglusterfs/src/graph.c:321) +... +``` + +In the above example, the `init` function in `xlators/meta/src/meta.c` does a +memory allocation on line 231. This memory is never free'd again, and hence +Valgrind logs this call stack. When looking in the code, it seems that the +allocation of `priv` is assigned to the `this->private` member of the +`xlator_t` structure. Because the allocation is done in `init()`, free'ing is +expected to happen in `fini()`. Both functions are shown below, with the +inclusion of the empty `fini()`: + + +``` +226 int +227 init (xlator_t *this) +228 { +229 meta_priv_t *priv = NULL; +230 +231 priv = GF_CALLOC (sizeof(*priv), 1, gf_meta_mt_priv_t); +232 if (!priv) +233 return -1; +234 +235 GF_OPTION_INIT ("meta-dir-name", priv->meta_dir_name, str, out); +236 +237 this->private = priv; +238 out: +239 return 0; +240 } +241 +242 +243 int +244 fini (xlator_t *this) +245 { +246 return 0; +247 } +``` + +In this case, the resource leak can be addressed by adding a single line to the +`fini()` function: + +``` +243 int +244 fini (xlator_t *this) +245 { +246 GF_FREE (this->private); +247 return 0; +248 } +``` + +Running the same Valgrind command and comparing the output will show that the +memory leak in `xlators/meta/src/meta.c:init` is not reported anymore. + +### Running DRD, the Valgrind thread error detector + +When configuring GlusterFS with: + +```shell +./configure --enable-valgrind +``` + +the default Valgrind tool (Memcheck) is enabled. But it's also possble to select +one of Memcheck or DRD by using: + +```shell +./configure --enable-valgrind=memcheck +``` + +or: + +```shell +./configure --enable-valgrind=drd +``` + +respectively. When using DRD, it's recommended to consult +https://valgrind.org/docs/manual/drd-manual.html before running. diff --git a/doc/developer-guide/logging-guidelines.md b/doc/developer-guide/logging-guidelines.md new file mode 100644 index 00000000000..0e6b2588535 --- /dev/null +++ b/doc/developer-guide/logging-guidelines.md @@ -0,0 +1,132 @@ +Guidelines on using the logging framework within a component +============================================================ +Gluster library libglusterfs.so provides message logging abstractions that +are intended to be used across all code/components within gluster. + +There could be potentially 2 major cases how the logging infrastructure is +used, + - A new gluster service daemon or end point is created + - The service daemon infrastructure itself initlializes the logging + infrastructure (i.e calling gf_log_init and related set functions) + - See, glusterfsd.c:logging_init + - Alternatively there could be a case where an end point service (say + gfapi) may need to do the required initialization + - This document does not (yet?) cover guidelines for these cases. Best + bet would be to look at code in glusterfsd.c:logging_init (or equivalent) + in case a need arises and you reach this document. + - A new xlator or subcomponent is written as a part of the stack + - Primarily in this case, the consumer of the logging APIs would only + invoke an API to *log* a particular message at a certain severity + - This document elaborates on this use of the message logging framework + in this context + +There are 2 interfaces provided to log messages, +1. The older gf_log* interface +2. The newer gf_msg* interface + +1. Older _to_be_deprecated_ gf_log* interface + - Not much documentation is provided for this interface here, as these are + meant to be deprecated and all code should be moved to the newer gf_msg* + interface + +2. New gf_msg* interface + - The set of interfaces provided by gf_msg are, + - NOTE: It is best to consult logging.h for the latest interfaces, as + this document may get out of sync with the code + + *gf_msg(dom, levl, errnum, msgid, fmt...)* + - Used predominantly to log a message above TRACE and DEBUG levels + - See further guidelines below + + *gf_msg_debug(dom, errnum, fmt...)* + *gf_msg_trace(dom, errnum, fmt...)* + - Above are handy shortcuts for TRACE and DEBUG level messages + - See further guidelines below + + *gf_msg_callingfn(dom, levl, errnum, msgid, fmt...)* + - Useful function when a backtrace to detect calling routines that + reached this execution point needs to be logged in addition to the + message + - This is very handy for framework libraries or code, as there could + be many callers to the same, and the real failure would need the call + stack to determine who the caller actually was + - A good example is the dict interfaces using this function to log who + called, when a particular error/warning needs to be displayed + - See further guidelines below + + *gf_msg_plain(levl, fmt...)* + *gf_msg_plain_nomem(levl, msg)* + *gf_msg_vplain(levl, fmt, va)* + *gf_msg_backtrace_nomem* + - The above interfaces are provided to log messages without any typical + headers (like the time stamp, dom, errnum etc.). The primary users of + the above interfaces are, when printing the final graph, or printing + the configuration when a process is about dump core or abort, or + printing the backtrace when a process receives a critical signal + - These interfaces should not be used outside the scope of the users + above, unless you know what you are doing + + *gf_msg_nomem(dom, levl, size)* + - Very crisp messages, throwing out file, function, line numbers, in + addition to the passed in size + - These are used in the memory allocation abstractions interfaces in + gluster, when the allocation fails (hence a no-mem log message, so that + further allocation is not attempted by the logging infrastructure) + - If you are contemplating using these, then you know why and how + + - Guidelines for the various arguments to the interfaces + **dom** + - The domain from which this message appears, IOW for xlators it should + be the name of the xlator (as in this->name) + + - The intention of this string is to distinguish from which + component/xlator the message is being printed + + - This information can also be gleaned from the FILE:LINE:FUNCTION + information printed in the message anyway, but is retained to provide + backward compatability to the messages as seen by admins earlier + + **levl** + - Consult logging.h:gf_loglevel_t for logging levels (they pretty much + map to syslog levels) + + **errnum** + - errno should be passed in here, if available, 0 otherwise. This auto + enables the logging infrastructure to print (man 3) strerror form of the + same at the end of the message in a consistent format + + - If any message is already adding the strerror as a parameter to the + message, it is suggested/encouraged to remove the same and pass it as + errnum + + - The intention is that, if all messages did this using errnum and not + as a part of their own argument list, the output would look consistent + for the admin and cross component developers when reading logs + + **msgid** + - This is a unique message ID per message (which now is more a message + ID per group of messages in the implementation) + + - Rules for generating this ID can be found in dht-messages.h (it used + to be template-common-messages.h, but that template is not updated, + this comment should be changed once that is done) and glfs-message-id.h + + - Every message that is *above* TRACE and DEBUG should get a message + ID, as this helps generating message catalogs that can help admins to + understand the context of messages better. Another intention is that + automated message parsers could detect a class of message IDs and send + out notifications on the same, rather than parse the message string + itself (which if it changes, the input to the parser has to change, etc. + and hence is better to retain message IDs) + + - Ok so if intention is not yet clear, look at journald MESSAGE ID + motivation, as that coupled with ident/dom above is expected to provide + us with similar advantages + + - Bottomline: Every message gets its own ID, in case a message is + *not* DEBUG or TRACE and still is developer centric, a generic message + ID per component *maybe* assigned to the same to provide ease of use + of the API + + **fmt** + - As in format argument of (man 3) printf diff --git a/doc/developer-guide/network_compression.md b/doc/developer-guide/network_compression.md new file mode 100644 index 00000000000..1222a765276 --- /dev/null +++ b/doc/developer-guide/network_compression.md @@ -0,0 +1,71 @@ +# On-Wire Compression + Decompression + +The 'compression translator' compresses and decompresses data in-flight +between client and bricks. + +### Working +When a writev call occurs, the client compresses the data before sending it to +brick. On the brick, compressed data is decompressed. Similarly, when a readv +call occurs, the brick compresses the data before sending it to client. On the +client, the compressed data is decompressed. Thus, the amount of data sent over +the wire is minimized. Compression/Decompression is done using Zlib library. + +During normal operation, this is the format of data sent over wire: + +~~~ +<compressed-data> + trailer(8 bytes) +~~~ + +The trailer contains the CRC32 checksum and length of original uncompressed +data. This is used for validation. + +### Usage + +Turning on compression xlator: + +~~~ +gluster volume set <vol_name> network.compression on +~~~ + +### Configurable parameters (optional) + +**Compression level** +~~~ +gluster volume set <vol_name> network.compression.compression-level 8 +~~~ + +~~~ + 0 : no compression + 1 : best speed + 9 : best compression +-1 : default compression +~~~ + +**Minimum file size** + +~~~ +gluster volume set <vol_name> network.compression.min-size 50 +~~~ + +Data is compressed only when its size exceeds the above value in bytes. + +**Other paramaters** + +Other less frequently used parameters include `network.compression.mem-level` +and `network.compression.window-size`. More details can about these options +can be found by running `gluster volume set help` command. + +### Known Issues and Limitations + +* Compression translator cannot work with striped volumes. +* Mount point hangs when writing a file with write-behind xlator turned on. To +overcome this, turn off `performance.write-behind` entirely OR +set`performance.strict-write-ordering` to on. +* For glusterfs versions <= 3.5, compression traslator can ONLY work with pure +distribute volumes. This limitation is caused by AFR not being able to +propagate xdata. This issue has been fixed in glusterfs versions > 3.5 + +### TODO +Although zlib offers high compression ratio, it is very slow. We can make the +translator pluggable to add support for other compression methods such as +[lz4 compression](https://code.google.com/p/lz4/) diff --git a/doc/developer-guide/options-to-contribute.md b/doc/developer-guide/options-to-contribute.md new file mode 100644 index 00000000000..3f0d84e7645 --- /dev/null +++ b/doc/developer-guide/options-to-contribute.md @@ -0,0 +1,212 @@ +# A guide for contributors + +While you have gone through 'how to contribute' guides, if you are +not sure what to work on, but really want to help the project, you +have now landed on the right document :-) + +### Basic + +Instead of planning to fix **all** the below issues in one patch, +we recommend you to have a a constant, continuous flow of improvements +for the project. We recommend you to pick 1 file (or just few files) at +a time to address below issues. +Pick any `.c` (or `.h`) file, and you can send a patch which fixes **any** +of the below themes. Ideally, fix all such occurrences in the file, even +though, the reviewers would review even a single line change patch +from you. + +1. Check for variable definitions, and if there is an array definition, +which is very large at the top of the function, see if you can re-scope +the variable to relevant sections (if it helps). + +Most of the time, some of these arrays may be used for 'error' handling, +and it is possible to use them only in that scope. + +Reference: https://review.gluster.org/20846/ + + +2. Check for complete string initialization at the beginning of a function. +Ideally, there is no reason to initialize a string. Fix it across the file. + +Example: + +`char new_path_name[PATH_MAX] = {0};` to `char new_path_name[PATH_MAX];` + + +3. Change `calloc()` to `malloc()` wherever it makes sense. + +In a case of allocating a structures, where you expect certain (or most of) +variables to be 0 (or NULL), it makes sense to use calloc(). But otherwise, +there is an extra cost to `memset()` the whole object after allocating it. +While it is not a significant improvement in performance, code which gets +hit 1000s of times in a second, it would add some value. + +Reference: https://review.gluster.org/20878/ + + +4. You can consider using `snprintf()`, instead of `strncpy()` while dealing +with strings. + +strncpy() won't null terminate if the dest buffer isn't big enough; snprintf() +does. While most of the string operations in the code is on array, and larger +size than required, strncpy() does an extra copy of 0s at the end of +string till the size of the array. It makes sense to use `snprintf()`, +which doesn't suffer from that behavior. + +Also check the return value from snprintf() for buffer overflow and handle +accordingly + +Reference: https://review.gluster.org/20925/ + + +5. Now, pick a `.h` file, and see if a structure is very large, and see +if re-aligning them as per [coding-standard](./coding-standard.md) gives any size benefit, +if yes, go ahead and change it. Make sure you check all the structures +in the file for similar pattern. + +Reference: [Check this section](https://github.com/gluster/glusterfs/blob/master/doc/developer-guide/coding-standard.md#structure-members-should-be-aligned-based-on-the-padding-requirements + + +### If you are up for more :-) + +Good progress! Glad you are interested to know more. We are surely interested +in next level of contributions from you! + +#### Coverity + +Visit [Coverity Dashboard](https://scan.coverity.com/projects/gluster-glusterfs?tab=overview). + +Now, if the number of defect is not 0, you have an opportunity to contribute. + +You get all the detail on why the particular defect is mentioned there, and +most probable hint on how to fix it. Do it! + +Reference: https://review.gluster.org/21394/ + +Use the same reference Id (789278) as the patch, so we can capture it is in +single bugzilla. + +#### Clang-Scan + +Clang-Scan is a tool which scans the .c files and reports the possible issues, +similar to coverity, but a different tool. Over the years we have seen, they +both report very different set of issues, and hence there is a value in fixing it. + +GlusterFS project gets tested with clang-scan job every night, and the report is +posted in the [job details page](https://build.gluster.org/job/clang-scan/lastCompletedBuild/clangScanBuildBugs/). +As long as the number is not 0 in the report here, you have an opportunity to +contribute! Similar to coverity dashboard, click on 'Details' to find out the +reason behind that report, and send a patch. + +Reference: https://review.gluster.org/21025 + +Again, you can use reference Id (1622665) for these patches! + + +### I am good with programming, I would like to do more than above! + +#### Locked regions / Critical sections + +In the file you open, see if the lock is taken only to increment or decrement +a flag, counter etc. If yes, then recommend you to convert it to ATOMIC locks. +It is simple activity, but, if you know programing, you would know the benefit +here. + +NOTE: There may not always a possibility to do this! You may have to check +with developers first before going ahead. + +Reference: https://review.gluster.org/21221/ + + +#### ASan (address sanitizer) + +[The job](https://build.gluster.org/job/asan/) runs regression with asan builds, +and you can also run glusterfs with asan on your workload to identify the leaks. +If there are any leaks reported, feel free to check it, and send us patch. + +You can also run `valgrind` and let us know what it reports. + +Reference: https://review.gluster.org/21397 + + +#### Porting to different architecture + +This is something which we are not focusing right now, happy to collaborate! + +Reference: https://review.gluster.org/21276 + + +#### Fix 'TODO/FIXME' in codebase + +There are few cases of pending features, or pending validations, which are +pending from sometime. You can pick them in the given file, and choose to +fix it. + + +### I don't know C, but I am interested to contribute in some way! + +You are most welcome! Our community is open for your contribution! First thing +which comes to our mind is **documentation**. Next is, **testing** or validation. + +If you have some hardware, and want to run some performance comparisons with +different version, or options, and help us to tune better is also a great help. + + +#### Documentation + +1. We have some documentation in [glusterfs repo](../), go through these, and +see if you can help us to keep up-to-date. + +2. The https://docs.gluster.org is powered by https://github.com/gluster/glusterdocs +repo. You can check out the repo, and help in keeping that up-to-date. + +3. [Our website](https://gluster.org) is maintained by https://github.com/gluster/glusterweb +repo. Help us to keep this up-to-date, and add content there. + +4. Write blogs about Gluster, and your experience, and make world know little +more about Gluster, and your use-case, and how it helped to solve the problem. + + +#### Testing + +1. There is a regression test suite in glusterfs, which runs with every patch, and is +triggered by just running `./run-tests.sh` from the root of the project repo. + +You can add more test case to match your use-case, and send it as a patch, so you +can make sure all future patches in glusterfs would keep your usecase intact. + +2. [Glusto-Tests](https://github.com/gluster/glusto-tests): This is another testing +framework written for gluster, and makes use of clustered setup to test different +use-cases, and helps to validate many bugs. + + +#### Ansible + +Gluster Organization has rich set of ansible roles, which are actively maintained. +Feel free to check them out here - https://github.com/gluster/gluster-ansible + + +#### Monitoring + +We have prometheus repo, and are actively working on adding more metrics. Add what +you need @ https://github.com/gluster/gluster-prometheus + + +#### Health-Report + +This is a project, where at any given point in time, you want to run some set of +commands locally, and get an output to analyze the status, it can be added. +Contribute @ https://github.com/gluster/gluster-health-report + + +### All these C/bash/python is old-school, I want something in containers. + +We have something for you too :-) + +Please visit our https://github.com/gluster/gcs repo for checking how you can help, +and how gluster can help you in container world. + + +### Note + +For any queries, best way is to contact us through mailing-list, <mailto:gluster-devel@gluster.org> diff --git a/doc/developer-guide/posix.md b/doc/developer-guide/posix.md new file mode 100644 index 00000000000..84c813e55a2 --- /dev/null +++ b/doc/developer-guide/posix.md @@ -0,0 +1,59 @@ +storage/posix translator +======================== + +Notes +----- + +### `SET_FS_ID` + +This is so that all filesystem checks are done with the user's +uid/gid and not GlusterFS's uid/gid. + +### `MAKE_REAL_PATH` + +This macro concatenates the base directory of the posix volume +('option directory') with the given path. + +### `need_xattr` in lookup + +If this flag is passed, lookup returns a xattr dictionary that contains +the file's create time, the file's contents, and the version number +of the file. + +This is a hack to increase small file performance. If an application +wants to read a small file, it can finish its job with just a lookup +call instead of a lookup followed by read. + +### `getdents`/`setdents` + +These are used by unify to set and get directory entries. + +### `ALIGN_BUF` + +Macro to align an address to a page boundary (4K). + +### `priv->export_statfs` + +In some cases, two exported volumes may reside on the same +partition on the server. Sending statvfs info for both +the volumes will lead to erroneous df output at the client, +since free space on the partition will be counted twice. + +In such cases, user can disable exporting statvfs info +on one of the volumes by setting this option. + +### `xattrop` + +This fop is used by replicate to set version numbers on files. + +### `getxattr`/`setxattr` hack to read/write files + +A key, `GLUSTERFS_FILE_CONTENT_STRING`, is handled in a special way by +`getxattr`/`setxattr`. A getxattr with the key will return the entire +content of the file as the value. A `setxattr` with the key will write +the value as the entire content of the file. + +### `posix_checksum` + +This calculates a simple XOR checksum on all entry names in a +directory that is used by unify to compare directory contents. diff --git a/doc/legacy/rpc-for-glusterfs.changes-done.txt b/doc/developer-guide/rpc-for-glusterfs.changes-done.txt index 6bbbca78826..6bbbca78826 100644 --- a/doc/legacy/rpc-for-glusterfs.changes-done.txt +++ b/doc/developer-guide/rpc-for-glusterfs.changes-done.txt diff --git a/doc/developer-guide/rpc-for-glusterfs.new-versions.md b/doc/developer-guide/rpc-for-glusterfs.new-versions.md new file mode 100644 index 00000000000..e3da5efa4a2 --- /dev/null +++ b/doc/developer-guide/rpc-for-glusterfs.new-versions.md @@ -0,0 +1,32 @@ +# GlusterFS RPC program versions + +## Compatibility + +RPC layer of glusterfs is implemented with possible changes over the protocol layers in mind. If there are any changes in the FOPs from what is assumed to be client side, and whats in serverside, they are to be added as a separate program table. + +### Program tables and Versions + +A given RPC program has a specific Task, and Version along with actors belonging to the program. In any of the programs, if a new actor is added, it is very important to define one more program with different version, and then keep both, if both are supported. Or else, it is important to handle the 'handshake' properly. + +#### Server details + +More info on RPC program is at `rpc/rpc-lib/src/rpcsvc.h` and check for structure `rpcsvc_actor_t` and `struct rpcsvc_program`. For usage, check `xlators/protocol/server/src/server-rpc-fops.c` + +#### Client details + +For details on client structures check `rpc/rpc-lib/src/rpc-clnt.h` for `rpc_clnt_procedure_t` and `rpc_clnt_program_t`. For usage, check `xlators/protocol/client/src/client-rpc-fops.c` + +## Protocol + +A protocol is what is agreed between two parties. In glusterfs, a RPC protocol is defined as .x file, which then gets converted to .c/.h file using `rpcgen`. There are different protocols defined for communication between `xlators/mgmt/glusterd <==> glusterfsd`, `gluster CLI <==> glusterd`, and `client-protocol <==> server-protocol` + +Once a protocol is defined and a release is made with that protocol, make sure no one changes it. Any edits to a given structure there should be a new version of the structure, and also it should get used in new actor, and thus new program version. + +## Server and Client Handshake + +When a client succeeds to establish a connect (it can be any transport, socket, ib-verbs or unix), client sends a dump (GF_DUMP_DUMP) request to server, which will respond back with all the supported versions of the server RPC (the supported programs which are registered with `rpcsvc_program_register()`). + +A client which expects certain programs to be present in server, it should be taking care of looking for it in the handshake methods, and take appropriate action depending on what to do next. In general a compatibility issue should be handled at handshake level itself, thus we can clearly let user/admin know of any 'in-compatibilities'. +As a developer of GlusterFS protocol layer, one just has to make sure *never to make changes to existing program structures*, but they have to add new programs if required. New programs can have the same actors as present in existing, and also little more. Or it can even have same actor behave differently, take different parameter. + +If this is followed properly, there would be smooth upgrade / downgrade of versions. If not, technically, it is 100% guarantee of getting compatibility related issues. diff --git a/doc/developer-guide/syncop.md b/doc/developer-guide/syncop.md new file mode 100644 index 00000000000..bcc8bd08e01 --- /dev/null +++ b/doc/developer-guide/syncop.md @@ -0,0 +1,72 @@ +# syncop framework +A coroutines-based, cooperative multi-tasking framework. + +## Topics + +- Glossary +- Lifecycle of a synctask +- Existing usage + + +## Glossary + +### syncenv + +syncenv is an object that provides access to a pool of worker threads. +synctasks execute in a syncenv. + +### synctask + +synctask can be informally defined as a pair of function pointers, namely _the +call_ and _the callback_ (see syncop.h for more details). + + synctask_fn_t - 'the call' + synctask_cbk_t - 'the callback' + +synctask has two modes of operation, + +1. The calling thread waits for the synctask to complete. +2. The calling thread schedules the synctask and continues. + +synctask guarantees that the callback is called _after_ the call completes. + +### Lifecycle of a synctask + +A synctask could go into the following stages while in execution. + +- CREATED - On calling synctask_create/synctask_new. + +- RUNNABLE - synctask is queued in env->runq. + +- RUNNING - When one of syncenv's worker threads calls synctask_switch_to. + +- WAITING - When a synctask calls synctask_yield. + +- DONE - When a synctask has run to completion. + + + +-------------------------------+ + | CREATED | + +-------------------------------+ + | + | synctask_new/synctask_create + v + +-------------------------------+ + | RUNNABLE (in env->runq) | <+ + +-------------------------------+ | + | | + | synctask_switch_to | + v | + +------+ on task completion +-------------------------------+ | + | DONE | <-------------------- | RUNNING | | synctask_wake/wake + +------+ +-------------------------------+ | + | | + | synctask_yield/yield | + v | + +-------------------------------+ | + | WAITING (in env->waitq) | -+ + +-------------------------------+ + +Note: A synctask is not guaranteed to run on the same thread throughout its +lifetime. Every time a synctask yields, it is possible for it to run on a +different thread. diff --git a/doc/developer-guide/thread-naming.md b/doc/developer-guide/thread-naming.md new file mode 100644 index 00000000000..513140d4437 --- /dev/null +++ b/doc/developer-guide/thread-naming.md @@ -0,0 +1,104 @@ +Thread Naming +================ +Gluster processes spawn many threads; some threads are created by libglusterfs +library, while others are created by xlators. When gfapi library is used in an +application, some threads belong to the application and some are spawned by +gluster libraries. We also have features where n number of threads are spawned +to act as worker threads for same operation. + +In all the above cases, it is useful to be able to determine the list of threads +that exist in runtime. Naming threads when you create them is the easiest way to +provide that information to kernel so that it can then be queried by any means. + +How to name threads +------------------- +We have two wrapper functions in libglusterfs for creating threads. They take +name as an argument and set thread name after its creation. + +```C +gf_thread_create (pthread_t *thread, const pthread_attr_t *attr, + void *(*start_routine)(void *), void *arg, const char *name) +``` + +```C +gf_thread_create_detached (pthread_t *thread, + void *(*start_routine)(void *), void *arg, + const char *name) +``` + +As max name length for a thread in POSIX is only 16 characters including the +'\0' character, you have to be a little creative with naming. Also, it is +important that all Gluster threads have common prefix. Considering these +conditions, we have "glfs_" as prefix for all the threads created by these +wrapper functions. It is responsibility of the owner of thread to provide the +suffix part of the name. It does not have to be a descriptive name, as it has +only 10 letters to work with. However, it should be unique enough such that it +can be matched with a table which describes it. + +If n number of threads are spwaned to perform same function, it is must that the +threads are numbered. + +Table of thread names +--------------------- +Thread names don't have to be a descriptive; however, it should be unique enough +such that it can be matched with a table below without ambiguity. + +- bdaio - block device aio +- brfsscan - bit rot fs scanner +- brhevent - bit rot event handler +- brmon - bit rot monitor +- brosign - bit rot one shot signer +- brpobj - bit rot object processor +- brsproc - bit rot scrubber +- brssign - bit rot stub signer +- brswrker - bit rot worker +- clogc - changelog consumer +- clogcbki - changelog callback invoker +- clogd - changelog dispatcher +- clogecon - changelog reverse connection +- clogfsyn - changelog fsync +- cloghcon - changelog history consumer +- clogjan - changelog janitor +- clogpoll - changelog poller +- clogproc - changelog process +- clogro - changelog rollover +- ctrcomp - change time recorder compaction +- dhtdf - dht defrag task +- dhtdg - dht defrag start +- dhtfcnt - dht rebalance file counter +- ecshd - ec heal daemon +- epollN - epoll thread +- fdlwrker - fdl worker +- fusenoti - fuse notify +- fuseproc - fuse main thread +- gdhooks - glusterd hooks +- glfspoll - gfapi poller thread +- idxwrker - index worker +- iosdump - io stats dump +- iotwr - io thread worker +- jbrflush - jbr flush +- leasercl - lease recall +- memsweep - sweeper thread for mem pools +- nfsauth - nfs auth +- nfsnsm - nfs nsm +- nfsudp - nfs udp mount +- nlmmon - nfs nlm/nsm mon +- posixaio - posix aio +- posixfsy - posix fsync +- posixhc - posix heal +- posixjan - posix janitor +- posixrsv - posix reserve +- quiesce - quiesce dequeue +- rdmaAsyn - rdma async event handler +- rdmaehan - rdma completion handler +- rdmarcom - rdma receive completion handler +- rdmascom - rdma send completion handler +- rpcsvcrh - rpcsvc request handler +- scleanup - socket cleanup +- shdheal - self heal daemon +- sigwait - glusterfsd sigwaiter +- spoller - socket poller +- sprocN - syncop worker thread +- tbfclock - token bucket filter token generator thread +- timer - timer thread +- upreaper - upcall reaper diff --git a/doc/developer-guide/translator-development.md b/doc/developer-guide/translator-development.md new file mode 100644 index 00000000000..f75935519f6 --- /dev/null +++ b/doc/developer-guide/translator-development.md @@ -0,0 +1,683 @@ +Translator development +====================== + +Setting the Stage +----------------- + +This is the first post in a series that will explain some of the details of +writing a GlusterFS translator, using some actual code to illustrate. + +Before we begin, a word about environments. GlusterFS is over 300K lines of +code spread across a few hundred files. That's no Linux kernel or anything, but + you're still going to be navigating through a lot of code in every +code-editing session, so some kind of cross-referencing is *essential*. I use +cscope with the vim bindings, and if I couldn't do Crtl+G and such to jump +between definitions all the time my productivity would be cut in half. You may +prefer different tools, but as I go through these examples you'll need +something functionally similar to follow on. OK, on with the show. + +The first thing you need to know is that translators are not just bags of +functions and variables. They need to have a very definite internal structure +so that the translator-loading code can figure out where all the pieces are. +The way it does this is to use dlsym to look for specific names within your +shared-object file, as follow (from `xlator.c`): + +``` +if (!(xl->fops = dlsym (handle, "fops"))) { + gf_log ("xlator", GF_LOG_WARNING, "dlsym(fops) on %s", + dlerror ()); + goto out; +} + +if (!(xl->cbks = dlsym (handle, "cbks"))) { + gf_log ("xlator", GF_LOG_WARNING, "dlsym(cbks) on %s", + dlerror ()); + goto out; +} + +if (!(xl->init = dlsym (handle, "init"))) { + gf_log ("xlator", GF_LOG_WARNING, "dlsym(init) on %s", + dlerror ()); + goto out; +} + +if (!(xl->fini = dlsym (handle, "fini"))) { + gf_log ("xlator", GF_LOG_WARNING, "dlsym(fini) on %s", + dlerror ()); + goto out; +} +``` + +In this example, `xl` is a pointer to the in-memory object for the translator +we're loading. As you can see, it's looking up various symbols *by name* in the + shared object it just loaded, and storing pointers to those symbols. Some of +them (e.g. init) are functions, while others (e.g. fops) are dispatch tables +containing pointers to many functions. Together, these make up the translator's + public interface. + +Most of this glue or boilerplate can easily be found at the bottom of one of +the source files that make up each translator. We're going to use the `rot-13` +translator just for fun, so in this case you'd look in `rot-13.c` to see this: + +``` +struct xlator_fops fops = { + .readv = rot13_readv, + .writev = rot13_writev +}; + +struct xlator_cbks cbks = { +}; + +struct volume_options options[] = { +{ .key = {"encrypt-write"}, + .type = GF_OPTION_TYPE_BOOL +}, +{ .key = {"decrypt-read"}, + .type = GF_OPTION_TYPE_BOOL +}, +{ .key = {NULL} }, +}; +``` + +The `fops` table, defined in `xlator.h`, is one of the most important pieces. +This table contains a pointer to each of the filesystem functions that your +translator might implement -- `open`, `read`, `stat`, `chmod`, and so on. There +are 82 such functions in all, but don't worry; any that you don't specify here +will be see as null and filled with defaults from `defaults.c` when your +translator is loaded. In this particular example, since `rot-13` is an +exceptionally simple translator, we only fill in two entries for `readv` and +`writev`. + +There are actually two other tables, also required to have predefined names, +that are also used to find translator functions: `cbks` (which is empty in this + snippet) and `dumpops` (which is missing entirely). The first of these specify + entry points for when inodes are forgotten or file descriptors are released. +In other words, they're destructors for objects in which your translator might + have an interest. Mostly you can ignore them, because the default behavior +handles even the simpler cases of translator-specific inode/fd context +automatically. However, if the context you attach is a complex structure +requiring complex cleanup, you'll need to supply these functions. As for +dumpops, that's just used if you want to provide functions to pretty-print +various structures in logs. I've never used it myself, though I probably +should. What's noteworthy here is that we don't even define dumpops. That's +because all of the functions that might use these dispatch functions will check + for `xl->dumpops` being `NULL` before calling through it. This is in sharp +contrast to the behavior for `fops` and `cbks`, which *must* be present. If +they're not, translator loading will fail because these pointers are not +checked every time and if they're `NULL` then we'll segfault. That's why we +provide an empty definition for cbks; it's OK for the individual function +pointers to be NULL, but not for the whole table to be absent. + +The last piece I'll cover today is options. As you can see, this is a table of +translator-specific option names and some information about their types. +GlusterFS actually provides a pretty rich set of types (`volume_option_type_t` +in `options.`h) which includes paths, translator names, percentages, and times +in addition to the obvious integers and strings. Also, the `volume_option_t` +structure can include information about alternate names, min/max/default +values, enumerated string values, and descriptions. We don't see any of these +here, so let's take a quick look at some more complex examples from afr.c and +then come back to `rot-13`. + +``` +{ .key = {"data-self-heal-algorithm"}, + .type = GF_OPTION_TYPE_STR, + .default_value = "", + .description = "Select between \"full\", \"diff\". The " + "\"full\" algorithm copies the entire file from " + "source to sink. The \"diff\" algorithm copies to " + "sink only those blocks whose checksums don't match " + "with those of source.", + .value = { "diff", "full", "" } +}, +{ .key = {"data-self-heal-window-size"}, + .type = GF_OPTION_TYPE_INT, + .min = 1, + .max = 1024, + .default_value = "1", + .description = "Maximum number blocks per file for which " + "self-heal process would be applied simultaneously." +}, +``` + +When your translator is loaded, all of this information is used to parse the +options actually provided in the volfile, and then the result is turned into a +dictionary and stored as `xl->options`. This dictionary is then processed by +your init function, which you can see being looked up in the first code +fragment above. We're only going to look at a small part of the `rot-13`'s +init for now. + +``` +priv->decrypt_read = 1; +priv->encrypt_write = 1; + +data = dict_get (this->options, "encrypt-write"); +if (data) { + if (gf_string2boolean (data->data, &priv->encrypt_write + == -1) { + gf_log (this->name, GF_LOG_ERROR, + "encrypt-write takes only boolean options"); + return -1; + } +} +``` + +What we can see here is that we're setting some defaults in our priv structure, +then looking to see if an `encrypt-write` option was actually provided. If so, +we convert and store it. This is a pretty classic use of dict_get to fetch a +field from a dictionary, and of using one of many conversion functions in +`common-utils.c` to convert `data->data` into something we can use. + +So far we've covered the basic of how a translator gets loaded, how we find its +various parts, and how we process its options. In my next Translator 101 post, +we'll go a little deeper into other things that init and its companion fini +might do, and how some other fields in our `xlator_t` structure (commonly +referred to as this) are commonly used. + +`init`, `fini`, and private context +----------------------------------- + +In the previous Translator 101 post, we looked at some of the dispatch tables +and options processing in a translator. This time we're going to cover the rest + of the "shell" of a translator -- i.e. the other global parts not specific to +handling a particular request. + +Let's start by looking at the relationship between a translator and its shared +library. At a first approximation, this is the relationship between an object +and a class in just about any object-oriented programming language. The class +defines behaviors, but has to be instantiated as an object to have any kind of +existence. In our case the object is an `xlator_t`. Several of these might be +created within the same daemon, sharing all of the same code through init/fini +and dispatch tables, but sharing *no data*. You could implement shared data (as + static variables in your shared libraries) but that's strongly discouraged. +Every function in your shared library will get an `xlator_t` as an argument, +and should use it. This lack of class-level data is one of the points where +the analogy to common OOP systems starts to break down. Another place is the +complete lack of inheritance. Translators inherit behavior (code) from exactly +one shared library -- looked up and loaded using the `type` field in a volfile +`volume ... end-volume` block -- and that's it -- not even single inheritance, +no subclasses or superclasses, no mixins or prototypes, just the relationship +between an object and its class. With that in mind, let's turn to the init +function that we just barely touched on last time. + +``` +int32_t +init (xlator_t *this) +{ + data_t *data = NULL; + rot_13_private_t *priv = NULL; + + if (!this->children || this->children->next) { + gf_log ("rot13", GF_LOG_ERROR, + "FATAL: rot13 should have exactly one child"); + return -1; + } + + if (!this->parents) { + gf_log (this->name, GF_LOG_WARNING, + "dangling volume. check volfile "); + } + + priv = GF_CALLOC (sizeof (rot_13_private_t), 1, 0); + if (!priv) + return -1; +``` + +At the very top, we see the function signature -- we get a pointer to the +`xlator_t` object that we're initializing, and we return an `int32_t` status. +As with most functions in the translator API, this should be zero to indicate +success. In this case it's safe to return -1 for failure, but watch out: in +dispatch-table functions, the return value means the status of the *function +call* rather than the *request*. A request error should be reflected as a +callback with a non-zero `op_re`t value, but the dispatch function itself +should still return zero. In fact, the handling of a non-zero return from a +dispatch function is not all that robust (we recently had a bug report in +HekaFS related to this) so it's something you should probably avoid +altogether. This only underscores the difference between dispatch functions +and `init`/`fini` functions, where non-zero returns *are* expected and handled +logically by aborting the translator setup. We can see that down at the +bottom, where we return -1 to indicate that we couldn't allocate our +private-data area (more about that later). + +The first thing this init function does is check that the translator is being +set up in the right kind of environment. Translators are called by parents and +in turn call children. Some translators are "initial" translators that inject +requests into the system from elsewhere -- e.g. mount/fuse injecting requests +from the kernel, protocol/server injecting requests from the network. Those +translators don't need parents, but `rot-13` does and so we check for that. +Similarly, some translators are "final" translators that (from the perspective +of the current process) terminate requests instead of passing them on -- e.g. +`protocol/client` passing them to another node, `storage/posix` passing them to +a local filesystem. Other translators "multiplex" between multiple children -- + passing each parent request on to one (`cluster/dht`), some +(`cluster/stripe`), or all (`cluster/afr`) of those children. `rot-13` fits +into none of those categories either, so it checks that it has *exactly one* +child. It might be more convenient or robust if translator shared libraries +had standard variables describing these requirements, to be checked in a +consistent way by the translator-loading infrastructure itself instead of by +each separate init function, but this is the way translators work today. + +The last thing we see in this fragment is allocating our private data area. +This can literally be anything we want; the infrastructure just provides the +priv pointer as a convenience but takes no responsibility for how it's used. In + this case we're using `GF_CALLOC` to allocate our own `rot_13_private_t` +structure. This gets us all the benefits of GlusterFS's memory-leak detection +infrastructure, but the way we're calling it is not quite ideal. For one thing, + the first two arguments -- from `calloc(3)` -- are kind of reversed. For +another, notice how the last argument is zero. That can actually be an +enumerated value, to tell the GlusterFS allocator *what* type we're +allocating. This can be very useful information for memory profiling and leak +detection, so it's recommended that you follow the example of any +x`xx-mem-types.h` file elsewhere in the source tree instead of just passing +zero here (even though that works). + +To finish our tour of standard initialization/termination, let's look at the +end of `init` and the beginning of `fini`: + +``` + this->private = priv; + gf_log ("rot13", GF_LOG_DEBUG, "rot13 xlator loaded"); + return 0; +} + +void +fini (xlator_t *this) +{ + rot_13_private_t *priv = this->private; + + if (!priv) + return; + this->private = NULL; + GF_FREE (priv); +``` + +At the end of init we're just storing our private-data pointer in the `priv` +field of our `xlator_t`, then returning zero to indicate that initialization +succeeded. As is usually the case, our fini is even simpler. All it really has +to do is `GF_FREE` our private-data pointer, which we do in a slightly +roundabout way here. Notice how we don't even have a return value here, since +there's nothing obvious and useful that the infrastructure could do if `fini` +failed. + +That's practically everything we need to know to get our translator through +loading, initialization, options processing, and termination. If we had defined + no dispatch functions, we could actually configure a daemon to use our +translator and it would work as a basic pass-through from its parent to a +single child. In the next post I'll cover how to build the translator and +configure a daemon to use it, so that we can actually step through it in a +debugger and see how it all fits together before we actually start adding +functionality. + +This Time For Real +------------------ + +In the first two parts of this series, we learned how to write a basic +translator skeleton that can get through loading, initialization, and option +processing. This time we'll cover how to build that translator, configure a +volume to use it, and run the glusterfs daemon in debug mode. + +Unfortunately, there's not much direct support for writing new translators. You +can check out a GlusterFS tree and splice in your own translator directory, but + that's a bit painful because you'll have to update multiple makefiles plus a +bunch of autoconf garbage. As part of the HekaFS project, I basically reverse +engineered the truly necessary parts of the translator-building process and +then pestered one of the Fedora glusterfs package maintainers (thanks +daMaestro!) to add a `glusterfs-devel` package with the required headers. Since + then the complexity level in the HekaFS tree has crept back up a bit, but I +still remember the simple method and still consider it the easiest way to get +started on a new translator. For the sake of those not using Fedora, I'm going +to describe a method that doesn't depend on that header package. What it does +depend on is a GlusterFS source tree, much as you might have cloned from GitHub + or the Gluster review site. This tree doesn't have to be fully built, but you +do need to run `autogen.sh` and configure in it. Then you can take the +following simple makefile and put it in a directory with your actual source. + +``` +# Change these to match your source code. +TARGET = rot-13.so +OBJECTS = rot-13.o + +# Change these to match your environment. +GLFS_SRC = /srv/glusterfs +GLFS_LIB = /usr/lib64 +HOST_OS = GF_LINUX_HOST_OS + +# You shouldn't need to change anything below here. + +CFLAGS = -fPIC -Wall -O0 -g \ + -DHAVE_CONFIG_H -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE \ + -D$(HOST_OS) -I$(GLFS_SRC) -I$(GLFS_SRC)/contrib/uuid \ + -I$(GLFS_SRC)/libglusterfs/src +LDFLAGS = -shared -nostartfiles -L$(GLFS_LIB) +LIBS = -lglusterfs -lpthread + +$(TARGET): $(OBJECTS) + $(CC) $(OBJECTS) $(LDFLAGS) -o $(TARGET) $(OBJECTS) $(LIBS) +``` + +Yes, it's still Linux-specific. Mea culpa. As you can see, we're sticking with +the `rot-13` example, so you can just copy the files from +`xlators/encryption/rot-13/src` in your GlusterFS tree to follow on. Type +`make` and you should be rewarded with a nice little `.so` file. + +``` +xlator_example$ ls -l rot-13.so +-rwxr-xr-x. 1 jeff jeff 40784 Nov 16 16:41 rot-13.so +``` + +Notice that we've built with optimization level zero and debugging symbols +included, which would not typically be the case for a packaged version of +GlusterFS. Let's put our version of `rot-13.so` into a slightly different file +on our system, so that it doesn't stomp on the installed version (not that +you'd ever want to use that anyway). + +``` +xlator_example# ls /usr/lib64/glusterfs/3git/xlator/encryption/ +crypt.so crypt.so.0 crypt.so.0.0.0 rot-13.so rot-13.so.0 +rot-13.so.0.0.0 +xlator_example# cp rot-13.so \ + /usr/lib64/glusterfs/3git/xlator/encryption/my-rot-13.so +``` + +These paths represent the current Gluster filesystem layout, which is likely to +be deprecated in favor of the Fedora layout; your paths may vary. At this point + we're ready to configure a volume using our new translator. To do that, I'm +going to suggest something that's strongly discouraged except during +development (the Gluster guys are going to hate me for this): write our own +volfile. Here's just about the simplest volfile you'll ever see. + +``` +volume my-posix + type storage/posix + option directory /srv/export +end-volume + +volume my-rot13 + type encryption/my-rot-13 + subvolumes my-posix +end-volume +``` + +All we have here is a basic brick using `/srv/export` for its data, and then +an instance of our translator layered on top -- no client or server is +necessary for what we're doing, and the system will automatically push a +mount/fuse translator on top if there's no server translator. To try this out, +all we need is the following command (assuming the directories involved already + exist). + +``` +xlator_example$ glusterfs --debug -f my.vol /srv/import +``` + +You should be rewarded with a whole lot of log output, including the text of +the volfile (this is very useful for debugging problems in the field). If you +go to another window on the same machine, you can see that you have a new +filesystem mounted. + +``` +~$ df /srv/import +Filesystem 1K-blocks Used Available Use% Mounted on +/srv/xlator_example/my.vol + 114506240 2706176 105983488 3% /srv/import +``` + +Just for fun, write something into a file in `/srv/import`, then look at the +corresponding file in `/srv/export` to see it all `rot-13`'ed for you. + +``` +~$ echo hello > /srv/import/a_file +~$ cat /srv/export/a_file +uryyb +``` + +There you have it -- functionality you control, implemented easily, layered on +top of local storage. Now you could start adding functionality -- real +encryption, perhaps -- and inevitably having to debug it. You could do that the + old-school way, with `gf_log` (preferred) or even plain old `printf`, or you +could run daemons under `gdb` instead. Alternatively, you could wait for the +next Translator 101 post, where we'll be doing exactly that. + +Debugging a Translator +---------------------- + +Now that we've learned what a translator looks like and how to build one, it's +time to run one and actually watch it work. The best way to do this is good +old-fashioned `gdb`, as follows (using some of the examples from last time). + +``` +xlator_example# gdb glusterfs +GNU gdb (GDB) Red Hat Enterprise Linux (7.2-50.el6) +... +(gdb) r --debug -f my.vol /srv/import +Starting program: /usr/sbin/glusterfs --debug -f my.vol /srv/import +... +[2011-11-23 11:23:16.495516] I [fuse-bridge.c:2971:fuse_init] + 0-glusterfs-fuse: FUSE inited with protocol versions: + glusterfs 7.13 kernel 7.13 +``` + +If you get to this point, your glusterfs client process is already running. You +can go to another window to see the mountpoint, do file operations, etc. + +``` +~# df /srv/import +Filesystem 1K-blocks Used Available Use% Mounted on +/root/xlator_example/my.vol + 114506240 2643968 106045568 3% /srv/import +~# ls /srv/import +a_file +~# cat /srv/import/a_file +hello +``` + +Now let's interrupt the process and see where we are. + +``` + +Program received signal SIGINT, Interrupt. +0x0000003a0060b3dc in pthread_cond_wait@@GLIBC_2.3.2 () + from /lib64/libpthread.so.0 +(gdb) info threads + 5 Thread 0x7fffeffff700 (LWP 27206) 0x0000003a002dd8c7 + in readv () + from /lib64/libc.so.6 + 4 Thread 0x7ffff50e3700 (LWP 27205) 0x0000003a0060b75b + in pthread_cond_timedwait@@GLIBC_2.3.2 () + from /lib64/libpthread.so.0 + 3 Thread 0x7ffff5f02700 (LWP 27204) 0x0000003a0060b3dc + in pthread_cond_wait@@GLIBC_2.3.2 () + from /lib64/libpthread.so.0 + 2 Thread 0x7ffff6903700 (LWP 27203) 0x0000003a0060f245 + in sigwait () + from /lib64/libpthread.so.0 +* 1 Thread 0x7ffff7957700 (LWP 27196) 0x0000003a0060b3dc + in pthread_cond_wait@@GLIBC_2.3.2 () + from /lib64/libpthread.so.0 +``` + +Like any non-toy server, this one has multiple threads. What are they all +doing? Honestly, even I don't know. Thread 1 turns out to be in +`event_dispatch_epoll`, which means it's the one handling all of our network +I/O. Note that with socket multi-threading patch this will change, with one +thread in `socket_poller` per connection. Thread 2 is in `glusterfs_sigwaiter` +which means signals will be isolated to that thread. Thread 3 is in +`syncenv_task`, so it's a worker process for synchronous requests such as +those used by the rebalance and repair code. Thread 4 is in +`janitor_get_next_fd`, so it's waiting for a chance to close no-longer-needed +file descriptors on the local filesystem. (I admit I had to look that one up, +BTW.) Lastly, thread 5 is in `fuse_thread_proc`, so it's the one fetching +requests from our FUSE interface. You'll often see many more threads than +this, but it's a pretty good basic set. Now, let's set a breakpoint so we can +actually watch a request. + +``` +(gdb) b rot13_writev +Breakpoint 1 at 0x7ffff50e4f0b: file rot-13.c, line 119. +(gdb) c +Continuing. +``` + +At this point we go into our other window and do something that will involve a write. + +``` +~# echo goodbye > /srv/import/another_file +(back to the first window) +[Switching to Thread 0x7fffeffff700 (LWP 27206)] + +Breakpoint 1, rot13_writev (frame=0x7ffff6e4402c, this=0x638440, + fd=0x7ffff409802c, vector=0x7fffe8000cd8, count=1, offset=0, + iobref=0x7fffe8001070) at rot-13.c:119 +119 rot_13_private_t *priv = (rot_13_private_t *)this->private; +``` + +Remember how we built with debugging symbols enabled and no optimization? That +will be pretty important for the next few steps. As you can see, we're in +`rot13_writev`, with several parameters. + +* `frame` is our always-present frame pointer for this request. Also, + `frame->local` will point to any local data we created and attached to the + request ourselves. +* `this` is a pointer to our instance of the `rot-13` translator. You can examine + it if you like to see the name, type, options, parent/children, inode table, + and other stuff associated with it. +* `fd` is a pointer to a file-descriptor *object* (`fd_t`, not just a + file-descriptor index which is what most people use "fd" for). This in turn + points to an inode object (`inode_t`) and we can associate our own + `rot-13`-specific data with either of these. +* `vector` and `count` together describe the data buffers for this write, which + we'll get to in a moment. +* `offset` is the offset into the file at which we're writing. +* `iobref` is a buffer-reference object, which is used to track the life cycle + of buffers containing read/write data. If you look closely, you'll notice that + `vector[0].iov_base` points to the same address as `iobref->iobrefs[0].ptr`, which + should give you some idea of the inter-relationships between vector and iobref. + +OK, now what about that `vector`? We can use it to examine the data being +written, like this. + +``` +(gdb) p vector[0] +$2 = {iov_base = 0x7ffff7936000, iov_len = 8} +(gdb) x/s 0x7ffff7936000 +0x7ffff7936000: "goodbye\n" +``` + +It's not always safe to view this data as a string, because it might just as +well be binary data, but since we're generating the write this time it's safe +and convenient. With that knowledge, let's step through things a bit. + +``` +(gdb) s +120 if (priv->encrypt_write) +(gdb) +121 rot13_iovec (vector, count); +(gdb) +rot13_iovec (vector=0x7fffe8000cd8, count=1) at rot-13.c:57 +57 for (i = 0; i < count; i++) { +(gdb) +58 rot13 (vector[i].iov_base, vector[i].iov_len); +(gdb) +rot13 (buf=0x7ffff7936000 "goodbye\n", len=8) at rot-13.c:45 +45 for (i = 0; i < len; i++) { +(gdb) +46 if (buf[i] >= 'a' && buf[i] <= 'z') +(gdb) +47 buf[i] = 'a' + ((buf[i] - 'a' + 13) % 26); +``` + +Here we've stepped into `rot13_iovec`, which iterates through our vector +calling `rot13`, which in turn iterates through the characters in that chunk +doing the `rot-13` operation if/as appropriate. This is pretty straightforward +stuff, so let's skip to the next interesting bit. + +``` +(gdb) fin +Run till exit from #0 rot13 (buf=0x7ffff7936000 "goodbye\n", + len=8) at rot-13.c:47 +rot13_iovec (vector=0x7fffe8000cd8, count=1) at rot-13.c:57 +57 for (i = 0; i < count; i++) { +(gdb) fin +Run till exit from #0 rot13_iovec (vector=0x7fffe8000cd8, + count=1) at rot-13.c:57 +rot13_writev (frame=0x7ffff6e4402c, this=0x638440, + fd=0x7ffff409802c, vector=0x7fffe8000cd8, count=1, + offset=0, iobref=0x7fffe8001070) at rot-13.c:123 +123 STACK_WIND (frame, +(gdb) b 129 +Breakpoint 2 at 0x7ffff50e4f35: file rot-13.c, line 129. +(gdb) b rot13_writev_cbk +Breakpoint 3 at 0x7ffff50e4db3: file rot-13.c, line 106. +(gdb) c +``` + +So we've set breakpoints on both the callback and the statement following the +`STACK_WIND`. Which one will we hit first? + +``` +Breakpoint 3, rot13_writev_cbk (frame=0x7ffff6e4402c, + cookie=0x7ffff6e440d8, this=0x638440, op_ret=8, op_errno=0, + prebuf=0x7fffefffeca0, postbuf=0x7fffefffec30) + at rot-13.c:106 +106 STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, + prebuf, postbuf); +(gdb) bt +#0 rot13_writev_cbk (frame=0x7ffff6e4402c, + cookie=0x7ffff6e440d8, this=0x638440, op_ret=8, op_errno=0, + prebuf=0x7fffefffeca0, postbuf=0x7fffefffec30) + at rot-13.c:106 +#1 0x00007ffff52f1b37 in posix_writev (frame=0x7ffff6e440d8, + this=<value optimized out>, fd=<value optimized out>, + vector=<value optimized out>, count=1, + offset=<value optimized out>, iobref=0x7fffe8001070) + at posix.c:2217 +#2 0x00007ffff50e513e in rot13_writev (frame=0x7ffff6e4402c, + this=0x638440, fd=0x7ffff409802c, vector=0x7fffe8000cd8, + count=1, offset=0, iobref=0x7fffe8001070) at rot-13.c:123 +``` + +Surprise! We're in `rot13_writev_cbk` now, called (indirectly) while we're +still in `rot13_writev` before `STACK_WIND` returns (still at rot-13.c:123). If + you did any request cleanup here, then you need to be careful about what you +do in the remainder of `rot13_writev` because data may have been freed etc. +It's tempting to say you should just do the cleanup in `rot13_writev` after +the `STACK_WIND,` but that's not valid because it's also possible that some +other translator returned without calling `STACK_UNWIND` -- i.e. before +`rot13_writev` is called, so then it would be the one getting null-pointer +errors instead. To put it another way, the callback and the return from +`STACK_WIND` can occur in either order or even simultaneously on different +threads. Even if you were to use reference counts, you'd have to make sure to +use locking or atomic operations to avoid races, and it's not worth it. Unless +you *really* understand the possible flows of control and know what you're +doing, it's better to do cleanup in the callback and nothing after +`STACK_WIND.` + +At this point all that's left is a `STACK_UNWIND` and a return. The +`STACK_UNWIND` invokes our parent's completion callback, and in this case our +parent is FUSE so at that point the VFS layer is notified of the write being +complete. Finally, we return through several levels of normal function calls +until we come back to fuse_thread_proc, which waits for the next request. + +So that's it. For extra fun, you might want to repeat this exercise by stepping +through some other call -- stat or setxattr might be good choices -- but you'll + have to use a translator that actually implements those calls to see much +that's interesting. Then you'll pretty much know everything I knew when I +started writing my first for-real translators, and probably even a bit more. I +hope you've enjoyed this series, or at least found it useful, and if you have +any suggestions for other topics I should cover please let me know (via +comments or email, IRC or Twitter). + +Other versions +-------------- + +Original author's site: + + * [Translator 101 - Setting the Stage](http://pl.atyp.us/hekafs.org/index.php/2011/11/translator-101-class-1-setting-the-stage/) + + * [Translator 101 - Init, Fini and Private Context](http://pl.atyp.us/hekafs.org/index.php/2011/11/translator-101-lesson-2-init-fini-and-private-context/) + + * [Translator 101 - This Time for Real](http://pl.atyp.us/hekafs.org/index.php/2011/11/translator-101-lesson-3-this-time-for-real/) + + * [Translator 101 - Debugging a Translator](http://pl.atyp.us/hekafs.org/index.php/2011/11/translator-101-lesson-4-debugging-a-translator/) + +Gluster community site: + + * [Translators](https://docs.gluster.org/en/latest/Quick-Start-Guide/Architecture/#translators) diff --git a/doc/developer-guide/unittest.md b/doc/developer-guide/unittest.md new file mode 100644 index 00000000000..5c6c0a8a039 --- /dev/null +++ b/doc/developer-guide/unittest.md @@ -0,0 +1,228 @@ +# Unit Tests in GlusterFS + +## Overview +[Art-of-unittesting][definitionofunittest] provides a good definition for unit tests. A good unit test is: + +* Able to be fully automated +* Has full control over all the pieces running (Use mocks or stubs to achieve this isolation when needed) +* Can be run in any order if part of many other tests +* Runs in memory (no DB or File access, for example) +* Consistently returns the same result (You always run the same test, so no random numbers, for example. save those for integration or range tests) +* Runs fast +* Tests a single logical concept in the system +* Readable +* Maintainable +* Trustworthy (when you see its result, you don’t need to debug the code just to be sure) + +## cmocka +GlusterFS unit test framework is based on [cmocka][]. cmocka provides +developers with methods to isolate and test modules written in C language. It +also provides integration with Jenkins by providing JUnit XML compliant unit +test results. + +cmocka + +## Running Unit Tests +To execute the unit tests, all you need is to type `make check`. Here is a step-by-step example assuming you just cloned a GlusterFS tree: + +``` +$ ./autogen.sh +$ ./configure --enable-debug +$ make check +``` + +Sample output: + +``` +PASS: mem_pool_unittest +============================================================================ +Testsuite summary for glusterfs 3git +============================================================================ +# TOTAL: 1 +# PASS: 1 +# SKIP: 0 +# XFAIL: 0 +# FAIL: 0 +# XPASS: 0 +# ERROR: 0 +============================================================================ +``` + +In this example, `mem_pool_unittest` has multiple tests inside, but `make check` assumes that the program itself is the test, and that is why it only shows one test. Here is the output when we run `mem_pool_unittest` directly: + +``` +$ ./libglusterfs/src/mem_pool_unittest +[==========] Running 10 test(s). +[ RUN ] test_gf_mem_acct_enable_set +Expected assertion data != ((void *)0) occurred +[ OK ] test_gf_mem_acct_enable_set +[ RUN ] test_gf_mem_set_acct_info_asserts +Expected assertion xl != ((void *)0) occurred +Expected assertion size > ((4 + sizeof (size_t) + sizeof (xlator_t *) + 4 + 8) + 8) occurred +Expected assertion type <= xl->mem_acct.num_types occurred +[ OK ] test_gf_mem_set_acct_info_asserts +[ RUN ] test_gf_mem_set_acct_info_memory +[ OK ] test_gf_mem_set_acct_info_memory +[ RUN ] test_gf_calloc_default_calloc +[ OK ] test_gf_calloc_default_calloc +[ RUN ] test_gf_calloc_mem_acct_enabled +[ OK ] test_gf_calloc_mem_acct_enabled +[ RUN ] test_gf_malloc_default_malloc +[ OK ] test_gf_malloc_default_malloc +[ RUN ] test_gf_malloc_mem_acct_enabled +[ OK ] test_gf_malloc_mem_acct_enabled +[ RUN ] test_gf_realloc_default_realloc +[ OK ] test_gf_realloc_default_realloc +[ RUN ] test_gf_realloc_mem_acct_enabled +[ OK ] test_gf_realloc_mem_acct_enabled +[ RUN ] test_gf_realloc_ptr +Expected assertion ((void *)0) != ptr occurred +[ OK ] test_gf_realloc_ptr +[==========] 10 test(s) run. +[ PASSED ] 10 test(s). +[ FAILED ] 0 test(s). +[ REPORT ] Created libglusterfs_mem_pool_xunit.xml report +``` + + +## Writing Unit Tests + +### Enhancing your C functions + +#### Programming by Contract +Add the following to your C file: + +```c +#include <cmocka_pbc.h> +``` + +```c +/* + * Programming by Contract is a programming methodology + * which binds the caller and the function called to a + * contract. The contract is represented using Hoare Triple: + * {P} C {Q} + * where {P} is the precondition before executing command C, + * and {Q} is the postcondition. + * + * See also: + * http://en.wikipedia.org/wiki/Design_by_contract + * http://en.wikipedia.org/wiki/Hoare_logic + * http://dlang.org/dbc.html + */ + #ifndef CMOCKERY_PBC_H_ +#define CMOCKERY_PBC_H_ + +#if defined(UNIT_TESTING) || defined (DEBUG) + +#include <assert.h> + +/* + * Checks caller responsibility against contract + */ +#define REQUIRE(cond) assert(cond) + +/* + * Checks function reponsability against contract. + */ +#define ENSURE(cond) assert(cond) + +/* + * While REQUIRE and ENSURE apply to functions, INVARIANT + * applies to classes/structs. It ensures that intances + * of the class/struct are consistent. In other words, + * that the instance has not been corrupted. + */ +#define INVARIANT(invariant_fnc) do{ (invariant_fnc) } while (0); + +#else +#define REQUIRE(cond) do { } while (0); +#define ENSURE(cond) do { } while (0); +#define INVARIANT(invariant_fnc) do{ } while (0); + +#endif /* defined(UNIT_TESTING) || defined (DEBUG) */ +#endif /* CMOCKERY_PBC_H_ */ +``` + +##### Example +This is an _extremely_ simple example: + +```c +int divide (int n, int d) +{ + int ans; + + REQUIRE(d != 0); + + ans = n / d; + + // As code is added to this function throughout its lifetime, + // ENSURE will assert that data will be returned + // according to the contract. Again this is an + // extremely simple example. :-D + ENSURE( ans == (n / d) ); + + return ans; +} + +``` + +##### Important Note +`REQUIRE`, `ENSURE`, and `INVARIANT` are only available when `DEBUG` or `UNIT_TESTING` are set in the CFLAGS. You must pass `--enable-debug` to `./configure` to enable PBC on your non-unittest builds. + +#### Overriding functions +Cmockery2 provides its own memory allocation functions which check for buffer overrun and memory leaks. The following header file must be included **last** to be able to override any of the memory allocation functions: + +```c +#include <cmocka.h> +``` + +This file will only take effect with the `UNIT_TESTING` CFLAG is set. + +### Creating a unit test +Once you identify the C file you would like to test, first create a `unittest` directory under the directory where the C file is located. This will isolate the unittests to a different directory. + +Next, you need to edit the `Makefile.am` file in the directory where your C file is located. Initialize the +`Makefile.am` if it does not already have the following sections: + +``` +#### UNIT TESTS ##### +CLEANFILES += *.gcda *.gcno *_xunit.xml +noinst_PROGRAMS = +TESTS = +``` + +Now you can add the following for each of the unit tests that you would like to build: + +``` +### UNIT TEST xxx_unittest ### +xxx_unittest_CPPFLAGS = $(xxx_CPPFLAGS) +xxx_unittest_SOURCES = xxx.c \ + unittest/xxx_unittest.c +xxx_unittest_CFLAGS = $(UNITTEST_CFLAGS) +xxx_unittest_LDFLAGS = $(UNITTEST_LDFLAGS) +noinst_PROGRAMS += xxx_unittest +TESTS += xxx_unittest +``` + +Where `xxx` is the name of your C file. For example, look at `libglusterfs/src/Makefile.am`. + +Copy the simple unit test from the [cmocka API][cmockaapi] to `unittest/xxx_unittest.c`. If you would like to see an example of a unit test, please refer to `libglusterfs/src/unittest/mem_pool_unittest.c`. + +#### Mocking +You may see that the linker will complain about missing functions needed by the C file you would like to test. Identify the required functions, then place their stubs in a file called `unittest/xxx_mock.c`, then include this file in `Makefile.am` in `xxx_unittest_SOURCES`. This will allow you to you Cmockery2's mocking functions. + +#### Running the unit test +You can type `make` in the directory where the C file is located. Once you built it and there are no errors, you can execute the test either by directly executing the program (in our example above it is called `xxx_unittest` ), or by running `make check`. + +#### Debugging +Sometimes you may need to debug your unit test. To do that, you will have to point `gdb` to the binary which is located in the same directory as the source. For example, you can do the following from the root of the source tree to debug `mem_pool_unittest`: + +``` +$ gdb libglusterfs/src/mem_pool_unittest +``` + + +[cmocka]: https://cmocka.org +[definitionofunittest]: http://artofunittesting.com/definition-of-a-unit-test/ +[cmockapi]: https://api.cmocka.org diff --git a/doc/developer-guide/versioning.md b/doc/developer-guide/versioning.md new file mode 100644 index 00000000000..10c1511b79b --- /dev/null +++ b/doc/developer-guide/versioning.md @@ -0,0 +1,44 @@ +Versioning +========== + +### current + +The number of the current interface exported by the library. A current value +of '1', means that you are calling the interface exported by this library +interface 1. + +### revision + +The implementation number of the most recent interface exported by this library. +In this case, a revision value of `0` means that this is the first +implementation of the interface. + +If the next release of this library exports the same interface, but has a +different implementation (perhaps some bugs have been fixed), the revision +number will be higher, but current number will be the same. In that case, when +given a choice, the library with the highest revision will always be used by +the runtime loader. + +### age + +The number of previous additional interfaces supported by this library. If age +were '2', then this library can be linked into executables which were built with +a release of this library that exported the current interface number, current, +or any of the previous two interfaces. By definition age must be less than or +equal to current. At the outset, only the first ever interface is implemented, +so age can only be `0'. + +For every release of the library `-version-info` argument needs to be set +correctly depending on any interface changes you have made. + +This is quite straightforward when you understand what the three numbers mean: + +If you have changed any of the sources for this library, the revision number +must be incremented. This is a new revision of the current interface. If the +interface has changed, then current must be incremented, and revision reset +to '0'. + +This is the first revision of a new interface. If the new interface is a +superset of the previous interface (that is, if the previous interface has not +been broken by the changes in this new release), then age must be incremented. +This release is backwards compatible with the previous release. diff --git a/doc/developer-guide/write-behind.md b/doc/developer-guide/write-behind.md new file mode 100644 index 00000000000..0d78964fa20 --- /dev/null +++ b/doc/developer-guide/write-behind.md @@ -0,0 +1,56 @@ +performance/write-behind translator +=================================== + +Basic working +-------------- + +Write behind is basically a translator to lie to the application that the +write-requests are finished, even before it is actually finished. + +On a regular translator tree without write-behind, control flow is like this: + +1. application makes a `write()` system call. +2. VFS ==> FUSE ==> `/dev/fuse`. +3. fuse-bridge initiates a glusterfs `writev()` call. +4. `writev()` is `STACK_WIND()`ed up to client-protocol or storage translator. +5. client-protocol, on receiving reply from server, starts `STACK_UNWIND()` towards the fuse-bridge. + +On a translator tree with write-behind, control flow is like this: + +1. application makes a `write()` system call. +2. VFS ==> FUSE ==> `/dev/fuse`. +3. fuse-bridge initiates a glusterfs `writev()` call. +4. `writev()` is `STACK_WIND()`ed up to write-behind translator. +5. write-behind adds the write buffer to its internal queue and does a `STACK_UNWIND()` towards the fuse-bridge. + +write call is completed in application's percepective. after +`STACK_UNWIND()`ing towards the fuse-bridge, write-behind initiates a fresh +writev() call to its child translator, whose replies will be consumed by +write-behind itself. Write-behind _doesn't_ cache the write buffer, unless +`option flush-behind on` is specified in volume specification file. + +Windowing +--------- + +With respect to write-behind, each write-buffer has three flags: `stack_wound`, `write_behind` and `got_reply`. + +* `stack_wound`: if set, indicates that write-behind has initiated `STACK_WIND()` towards child translator. +* `write_behind`: if set, indicates that write-behind has done `STACK_UNWIND()` towards fuse-bridge. +* `got_reply`: if set, indicates that write-behind has received reply from child translator for a `writev()` `STACK_WIND()`. a request will be destroyed by write-behind only if this flag is set. + +Currently pending write requests = aggregate size of requests with write_behind = 1 and got_reply = 0. + +window size limits the aggregate size of currently pending write requests. once +the pending requests' size has reached the window size, write-behind blocks +writev() calls from fuse-bridge. Blocking is only from application's +perspective. Write-behind does `STACK_WIND()` to child translator +straight-away, but hold behind the `STACK_UNWIND()` towards fuse-bridge. +`STACK_UNWIND()` is done only once write-behind gets enough replies to +accommodate for currently blocked request. + +Flush behind +------------ + +If `option flush-behind on` is specified in volume specification file, then +write-behind sends aggregate write requests to child translator, instead of +regular per request `STACK_WIND()`s. diff --git a/doc/developer-guide/writing-a-cloudsync-plugin.md b/doc/developer-guide/writing-a-cloudsync-plugin.md new file mode 100644 index 00000000000..907860aaed8 --- /dev/null +++ b/doc/developer-guide/writing-a-cloudsync-plugin.md @@ -0,0 +1,164 @@ +## How to write your Cloudsync Plugin + +### Background + +Cloudsync translator is part of the archival feature in Gluster. This translator +does the retrieval/download part. Each cold file will be archived to a remote +storage (public or private cloud). On future access to the file, it will be +retrieved from the remote storage by Cloudsync translator. Each remote storage +would need a unique plugin. Cloudsync translator will load this plugin and +call the necessary plugin functions. + +Upload can be done by a script or program. There are some basic mandatory steps +for uploading the data. There is a sample script for crawl and upload given at +the end of this guide. + +### Necessary changes to create a plugin + +1. Define store_methods: + +* This structure is the container of basic functions that will be called by + cloudsync xlator. + + typedef struct store_methodds { + int (*fop_download) (call_frame_t *frame, void *config); + /* return type should be the store config */ + void *(*fop_init) (xlator_t *this); + int (*fop_reconfigure) (xlator_t *this, dict_t *options); + void (*fop_fini) (void *config); + } store_methods_t; + + + Member details: + fop_download: + This is the download function pointer. + + frame: This will have the fd to write data downloaded from + cloud to GlusterFS.(frame->local->fd) + + config: This is the plugin configuration variable. + + Note: Structure cs_local_t has member dlfd and dloffset which + can be used to manage the writes to Glusterfs. + Include cloudsync-common.h to access these structures. + + fop_init: + This is similar to xlator init. But here the return value is + the plugin configuration pointer. This pointer will be stored + in the cloudsync private object (priv->stores->config). And + the cloudsync private object can be accessed by "this->private" + where "this" is of type xlator_t. + + fop_reconfigure: + This is similar to xlator_reconfigure. + + fop_fini: + Free plugin resources. + + Note: Store_methods_t is part of cs_private_t which in turn part of + xlator_t. Create a store_methods_t object named "store_ops" in + your plugin. For example + + store_methods_t store_ops = { + .fop_download = aws_download_s3, + .fop_init = aws_init, + .fop_reconfigure = aws_reconfigure, + .fop_fini = aws_fini, + }; + + +2 - Making Cloudsync xlator aware of the plugin: + + Add an entry in to the cs_plugin structure. For example + struct cs_plugin plugins[] = { + { + .name = "amazons3", + .library = "libamazons3.so", + .description = "amazon s3 store." + }, + + {.name = NULL}, + }; + + Description about individual members: + name: name of the plugin + library: This is the shared object created. Cloudsync will load + this library during init. + description: Describe about the plugin. + +3- Makefile Changes in Cloudsync: + + Add <plugin.la> to cloudsync_la_LIBADD variable. + +4 - Configure.ac changes: + + In cloudsync section add the necessary dependency checks for + the plugin. + +5 - Export symbols: + + Cloudsync needs "store_ops" to resolve all plugin functions. + Create a file <plugin>.sym and add write "store_ops" to it. + + +### Sample script for upload +This script assumes amazon s3 is the target cloud and bucket name is +gluster-bucket. User can do necessary aws configuration using command +"aws configure". Currently for amazons3 there are four gluster settings +available. +1- features.s3plugin-seckey -> s3 secret key +2- features.s3plugin-keyid -> s3 key id +3- features.s3plugin-bucketid -> bucketid +4- features.s3plugin-hostname -> hostname e.g. s3.amazonaws.com + +Additionally set cloudsync storetype to amazons3. + +gluster v set <VOLNAME> cloudsync-storetype amazons3 + +Now create a mount dedicated for this upload task. + +That covers necessary configurations needed. + +Below is the sample script for upload. The script will crawl directly on the +brick and will upload those files which are not modified for last one month. +It needs two arguments. +1st arguement - Gluster Brick path +2nd arguement - coldness that is how many days since the file was modified. +3rd argument - dedicated gluster mount point created for uploading. + +Once the cloud setup is done, run the following script on individual bricks. +Note: For an AFR volume, pick only the fully synchronized brick among the +replica bricks. + +``` +target_folder=$1 +coldness=$2 +mnt=$3 + +cd $target_folder +for i in `find . -type f | grep -v "glusterfs" | sed 's/..//'` +do + echo "processing $mnt/$i" + + #check whether the file is already archived + getfattr -n trusted.glusterfs.cs.remote $i &> /dev/null + if [ $? -eq 0 ] + then + echo "file $mnt/$i is already archived" + else + #upload to cloud + aws s3 cp $mnt/$i s3://gluster-bucket/ + mtime=`stat -c "%Y" $mnt/$i` + + #post processing of upload + setfattr -n trusted.glusterfs.csou.complete -v $mtime $mnt/$i + if [ $? -ne 0 ] + then + echo "archiving of file $mnt/$i failed" + else + echo "archiving of file $mnt/$i succeeded" + fi + + fi +done +``` diff --git a/doc/developer-guide/xlator-classification.md b/doc/developer-guide/xlator-classification.md new file mode 100644 index 00000000000..6073df9375f --- /dev/null +++ b/doc/developer-guide/xlator-classification.md @@ -0,0 +1,221 @@ +# xlator categories and expectations + +The purpose of the document is to define a category for various xlators +and expectations around what each category means from a perspective of +health and maintenance of a xlator. + +The need to do this is to ensure certain categories are kept in good +health, and helps the community and contributors focus their efforts around the +same. + +This document also provides implementation details for xlator developers to +declare a category for any xlator. + +## Table of contents +1. Audience +2. Categories (and expectations of each category) +3. Implementation and usage details + +## Audience + +This document is intended for the following community participants, +- New xlator contributors +- Existing xlator maintainers +- Packaging and gluster management stack maintainers + +For a more user facing understanding it is recommended to read section (TBD) +in the gluster documentation. + +## Categories +1. Experimental (E) +2. TechPreview (TP) +3. Maintained (M) +4. Deprecated (D) +5. Obsolete (O) + +### Experimental (E) + +Developed in the experimental branch, for exploring new features. These xlators +are NEVER packaged as a part of releases, interested users and contributors can +build and work with these from sources. In the future, these maybe available as +an package based on a weekly build of the same. + +#### Quality expectations +- Compiles or passes smoke tests +- Does not break nightly experimental regressions + - NOTE: If a nightly is broken, then all patches that were merged are reverted + till the errant patch is found and subsequently fixed + +### TechPreview (TP) + +Xlators in master or release branches that are not deemed fit to be in +production deployments, but are feature complete to invite feedback and host +user data. + +These xlators will be worked upon with priority by maintainers/authors who are +involved in making them more stable than xlators in the Experimental/Deprecated/ +Obsolete categories. + +There is no guarantee that these xlators will move to the Maintained state, and +may just get Obsoleted based on feedback, or other project goals or technical +alternatives. + +#### Quality expectations +- Same as Maintained, minus + - Performance, Scale, other(?) + - *TBD* *NOTE* Need inputs, Intention is all quality goals as in Maintained, + other than the list above (which for now has scale and performance) + +### Maintained (M) + +These xltors are part of the core Gluster functionality and are maintained +actively. These are part of master and release branches and are higher in +priority of maintainers and other interested contributors. + +#### Quality expectations + +NOTE: A short note on what each of these mean are added here, details to follow. + +NOTE: Out of the gate all of the following are not mandated, consider the +following a desirable state to reach as we progress on each + +- Bug backlog: Actively address bug backlog +- Enhancement backlog: Actively maintain outstanding enhancement backlog (need + not be acted on, but should be visible to all) +- Review backlog: Actively keep this below desired counts and states +- Static code health: Actively meet near-zero issues in this regard + - Coverity, spellcheck and other checks +- Runtime code health: Actively meet defined coverage levels in this regard + - Coverage, others? + - Per-patch regressions + - Glusto runs + - Performance + - Scalability +- Technical specifications: Implementation details should be documented and + updated at regular cadence (even per patch that change assumptions in + here) +- User documentation: User facing details should be maintained to current + status in the documentation +- Debuggability: Steps, tools, procedures should be documented and maintained + each release/patch as applicable +- Troubleshooting: Steps, tools, procedures should be documented and maintained + each release/patch as applicable + - Steps/guides for self service + - Knowledge base for problems +- Other common criteria that will apply: Required metrics/desired states to be + defined per criteria + - Monitoring, usability, statedump, and other such xlator expectations + +### Deprecated (D) + +Xlators on master or release branches that would be obsoleted and/or replaced +with similar or other functionality in the next major release. + +#### Quality expectations +- Retain status-quo when moved to this state, till it is moved to obsoleted +- Provide migration steps if feature provided by the xlator is replaced with +other xlators + +### Obsolete (O) + +Xlator/code still in tree, but not packaged or shipped or maintained in any +form. This is noted as a category till the code is removed from the tree. + +These xlators and their corresponding code and test health will not be executed. + +#### Quality expectations +- None + +## Implementation and usage details + +### How to specify an xlators category + +While defining 'xlator_api_t' structure for the corresponding xlator, add a +flag like below: + +``` +diff --git a/xlators/performance/nl-cache/src/nl-cache.c b/xlators/performance/nl-cache/src/nl-cache.c +index 0f0e53bac2..8267d6897c 100644 +--- a/xlators/performance/nl-cache/src/nl-cache.c ++++ b/xlators/performance/nl-cache/src/nl-cache.c +@@ -869,4 +869,5 @@ xlator_api_t xlator_api = { + .cbks = &nlc_cbks, + .options = nlc_options, + .identifier = "nl-cache", ++ .category = GF_TECH_PREVIEW, + }; +diff --git a/xlators/performance/quick-read/src/quick-read.c b/xlators/performance/quick-read/src/quick-read.c +index 8d39720e7f..235de27c19 100644 +--- a/xlators/performance/quick-read/src/quick-read.c ++++ b/xlators/performance/quick-read/src/quick-read.c +@@ -1702,4 +1702,5 @@ xlator_api_t xlator_api = { + .cbks = &qr_cbks, + .options = qr_options, + .identifier = "quick-read", ++ .category = GF_MAINTAINED, + }; +``` + +Similarly, if a particular option is in different state other than +the xlator state, one can add the same flag in options structure too. + +``` +diff --git a/xlators/cluster/afr/src/afr.c b/xlators/cluster/afr/src/afr.c +index 0e86e33d03..81996743d1 100644 +--- a/xlators/cluster/afr/src/afr.c ++++ b/xlators/cluster/afr/src/afr.c +@@ -772,6 +772,7 @@ struct volume_options options[] = { + .description = "Maximum latency for shd halo replication in msec." + }, + { .key = {"halo-enabled"}, ++ .category = GF_TECH_PREVIEW, + .type = GF_OPTION_TYPE_BOOL, + .default_value = "False", + +``` + + +### User experience using the categories + +#### Ability to use a category + +This section details which category of xlators can be used when and specifics +around when each category is enabled. + +1. Maintained category xlators can be used by default, this implies, volumes +created with these xlators enabled will throw no warnings, or need no user +intervention to use the xlator. + +2. Tech Preview category xlators needs cluster configuration changes to allow +these xlatorss to be used in volumes, further, logs will contain a message +stating TP xlators are in use. Without the cluster configured to allow TP +xlators, volumes created or edited to use such xlators would result in errors. + - (TBD) Cluster configuration option + - (TBD) Warning message + - (TBD) Code mechanics on how this is achieved + +3. Deprecated category xlators can be used by default, but will throw a warning +in the logs that such are in use and will be deprecated in the future. + - (TBD) Warning message + +4. Obsolete category xlators will not be packaged and hence cannot be used from +release builds. + +5. Experimental category xlators will not be packaged and hence cannot be used +from release builds, if running experimental (weekly or other such) builds, +these will throw a warning in the logs stating experimental xlators are in use. + - (TBD) Warning message + +#### Ability to query xlator category + +(TBD) Need to provide the ability to query xlator categories, or list xlators +and their respective categories. + +#### User facing changes + +User facing changes that are expected due to this change include the following, +- Cluster wide option to enable TP xlators, or more generically a category +level of xlators +- Errors in commands that fail due to invalid categories +- Warning messages in logs to denote certain categories of xlators are in use +- (TBD) Ability to query xlators and their respective categories diff --git a/doc/examples/legacy/Makefile.am b/doc/examples/legacy/Makefile.am deleted file mode 100644 index 49c9701efb3..00000000000 --- a/doc/examples/legacy/Makefile.am +++ /dev/null @@ -1,8 +0,0 @@ -EXTRA = README replicate.vol stripe.vol protocol-client.vol protocol-server.vol posix-locks.vol trash.vol write-behind.vol io-threads.vol io-cache.vol read-ahead.vol filter.vol trace.vol -EXTRA_DIST = $(EXTRA) - -docdir = $(datadir)/doc/$(PACKAGE_NAME) -Examplesdir = $(docdir)/examples -Examples_DATA = $(EXTRA) - -CLEANFILES = diff --git a/doc/examples/legacy/README b/doc/examples/legacy/README deleted file mode 100644 index 73275157160..00000000000 --- a/doc/examples/legacy/README +++ /dev/null @@ -1,13 +0,0 @@ -GlusterFS's translator feature is very flexible and there are quite a lot of -ways one can configure their filesystem to behave like. - -Volume Specification is a way in which GlusterFS understands how it has to work, -based on what is written there. - -Going through the following URLs may give you more idea about all these. - -* http://www.gluster.org/docs/index.php/GlusterFS -* http://www.gluster.org/docs/index.php/GlusterFS_Volume_Specification -* http://www.gluster.org/docs/index.php/GlusterFS_Translators - -Mail us any doubts, suggestions on 'gluster-devel(at)nongnu.org' diff --git a/doc/examples/legacy/filter.vol b/doc/examples/legacy/filter.vol deleted file mode 100644 index 59bb23ecf2c..00000000000 --- a/doc/examples/legacy/filter.vol +++ /dev/null @@ -1,23 +0,0 @@ -volume client - type protocol/client - option transport-type tcp # for TCP/IP transport - option remote-host 192.168.1.10 # IP address of the remote brick - option remote-subvolume brick # name of the remote volume -end-volume - -## In normal clustered storage type, any of the cluster translators can come here. -# -# Definition of other clients -# -# Definition of cluster translator (may be unify, afr, or unify over afr) -# - -### 'Filter' translator is used on client side (or server side according to needs). This traslator makes all the below translators, (or say volumes) as read-only. Hence if one wants a 'read-only' filesystem, using filter as the top most volume will make it really fast as the fops are returned from this level itself. - -volume filter-ro - type features/filter - option root-squashing enable -# option completely-read-only yes -# translate-uid 1-99=0 - subvolumes client -end-volume diff --git a/doc/examples/legacy/io-cache.vol b/doc/examples/legacy/io-cache.vol deleted file mode 100644 index a71745017a2..00000000000 --- a/doc/examples/legacy/io-cache.vol +++ /dev/null @@ -1,31 +0,0 @@ -volume client - type protocol/client - option transport-type tcp # for TCP/IP transport - option remote-host 192.168.1.10 # IP address of the remote brick - option remote-subvolume brick # name of the remote volume -end-volume - -## In normal clustered storage type, any of the cluster translators can come -# here. -# -# Definition of other clients -# -# Definition of cluster translator (may be distribute, replicate, or distribute -# over replicate) -# - -### 'IO-Cache' translator is best used on client side when a filesystem has file -# which are not modified frequently but read several times. For example, while -# compiling a kernel, *.h files are read while compiling every *.c file, in -# these case, io-cache translator comes very handy, as it keeps the whole file -# content in the cache, and serves from the cache. -# One can provide the priority of the cache too. - -volume ioc - type performance/io-cache - subvolumes client # In this example it is 'client' you may have to - # change it according to your spec file. - option cache-size 64MB # 32MB is default - option force-revalidate-timeout 5 # 1second is default - option priority *.html:2,*:1 # default is *:0 -end-volume diff --git a/doc/examples/legacy/io-threads.vol b/doc/examples/legacy/io-threads.vol deleted file mode 100644 index 236f5b8b188..00000000000 --- a/doc/examples/legacy/io-threads.vol +++ /dev/null @@ -1,22 +0,0 @@ -volume brick - type storage/posix # POSIX FS translator - option directory /home/export # Export this directory -end-volume - -### 'IO-threads' translator gives a threading behaviour to File I/O calls. All -# other normal fops are having default behaviour. Loading this on server side -# helps to reduce the contension of network. (Which is assumed as a GlusterFS -# hang). - -volume iot - type performance/io-threads - subvolumes brick - option thread-count 4 # default value is 1 -end-volume - -volume server - type protocol/server - subvolumes iot - option transport-type tcp # For TCP/IP transport - option auth.addr.iot.allow 192.168.* -end-volume diff --git a/doc/examples/legacy/posix-locks.vol b/doc/examples/legacy/posix-locks.vol deleted file mode 100644 index 673afa3f8ae..00000000000 --- a/doc/examples/legacy/posix-locks.vol +++ /dev/null @@ -1,19 +0,0 @@ -volume brick - type storage/posix # POSIX FS translator - option directory /home/export # Export this directory -end-volume - -# 'Posix-locks' feature should be added on the server side. - -volume p-locks - type features/posix-locks - subvolumes brick - option mandatory on -end-volume - -volume server - type protocol/server - subvolumes p-locks - option transport-type tcp - option auth.addr.p-locks.allow 192.168.* # Allow access to "p-locks" volume -end-volume diff --git a/doc/examples/legacy/protocol-client.vol b/doc/examples/legacy/protocol-client.vol deleted file mode 100644 index c34ef790d0e..00000000000 --- a/doc/examples/legacy/protocol-client.vol +++ /dev/null @@ -1,12 +0,0 @@ -volume client - type protocol/client - option transport-type tcp # for TCP/IP transport - option remote-host 192.168.1.10 # IP address of the remote brick -# option transport.socket.remote-port 24016 - -# option transport-type rdma # for Infiniband verbs transport -# option transport.rdma.work-request-send-count 16 -# option transport.rdma.work-request-recv-count 16 -# option transport.rdma.remote-port 24016 - option remote-subvolume brick # name of the remote volume -end-volume diff --git a/doc/examples/legacy/protocol-server.vol b/doc/examples/legacy/protocol-server.vol deleted file mode 100644 index 195e4965782..00000000000 --- a/doc/examples/legacy/protocol-server.vol +++ /dev/null @@ -1,21 +0,0 @@ -### Export volume "brick" with the contents of "/home/export" directory. -volume brick - type storage/posix # POSIX FS translator - option directory /home/export # Export this directory -end-volume - -### Add network serving capability to above brick. -volume server - type protocol/server - option transport-type tcp # For TCP/IP transport -# option transport.socket.listen-port 24016 - -# option transport-type rdma -# option transport.rdma.work-request-send-count 64 -# option transport.rdma.work-request-recv-count 64 -# option transport.rdma.listen-port 24016 - -# option bind-address 192.168.1.10 # Default is to listen on all interfaces - subvolumes brick - option auth.addr.brick.allow 192.168.* # Allow access to "brick" volume -end-volume diff --git a/doc/examples/legacy/read-ahead.vol b/doc/examples/legacy/read-ahead.vol deleted file mode 100644 index 9e4dba55627..00000000000 --- a/doc/examples/legacy/read-ahead.vol +++ /dev/null @@ -1,24 +0,0 @@ -volume client - type protocol/client - option transport-type tcp # for TCP/IP transport - option remote-host 192.168.1.10 # IP address of the remote brick - option remote-subvolume brick # name of the remote volume -end-volume - -## In normal clustered storage type, any of the cluster translators can come here. -# -# Definition of other clients -# -# Definition of cluster translator (may be distribute, replicate, or distribute -# over replicate) -# - -# 'Read-Ahead' translator is best utilized on client side, as it prefetches -# the file contents when the first read() call is issued. - -volume ra - type performance/read-ahead - subvolumes client - option page-count 4 # default is 2 - option force-atime-update no # defalut is 'no' -end-volume diff --git a/doc/examples/legacy/replicate.vol b/doc/examples/legacy/replicate.vol deleted file mode 100644 index 10626d46f05..00000000000 --- a/doc/examples/legacy/replicate.vol +++ /dev/null @@ -1,118 +0,0 @@ -### 'NOTE' -# This file has both server spec and client spec to get an understanding of -# replicate spec file. Hence can't be used as it is, as a GlusterFS spec file. -# One need to seperate out server spec and client spec to get it working. - -#========================================================================= - -# **** server1 spec file **** - -### Export volume "brick" with the contents of "/home/export" directory. -volume posix1 - type storage/posix # POSIX FS translator - option directory /home/export1 # Export this directory -end-volume - -### Add POSIX record locking support to the storage brick -volume brick1 - type features/posix-locks - option mandatory on # enables mandatory locking on all files - subvolumes posix1 -end-volume - -### Add network serving capability to above brick. -volume server - type protocol/server - option transport-type tcp # For TCP/IP transport - option transport.socket.listen-port 24016 - subvolumes brick1 - option auth.addr.brick1.allow * # access to "brick" volume -end-volume - - -#========================================================================= - -# **** server2 spec file **** -volume posix2 - type storage/posix # POSIX FS translator - option directory /home/export2 # Export this directory -end-volume - -### Add POSIX record locking support to the storage brick -volume brick2 - type features/posix-locks - option mandatory on # enables mandatory locking on all files - subvolumes posix2 -end-volume - -### Add network serving capability to above brick. -volume server - type protocol/server - option transport-type tcp # For TCP/IP transport - option transport.socket.listen-port 24017 - subvolumes brick2 - option auth.addr.brick2.allow * # Allow access to "brick" volume -end-volume - - -#========================================================================= - -# **** server3 spec file **** - -volume posix3 - type storage/posix # POSIX FS translator - option directory /home/export3 # Export this directory -end-volume - -### Add POSIX record locking support to the storage brick -volume brick3 - type features/posix-locks - option mandatory on # enables mandatory locking on all files - subvolumes posix3 -end-volume - -### Add network serving capability to above brick. -volume server - type protocol/server - option transport-type tcp # For TCP/IP transport - option transport.socket.listen-port 24018 - subvolumes brick3 - option auth.addr.brick3.allow * # access to "brick" volume -end-volume - - -#========================================================================= - -# **** Clustered Client config file **** - -### Add client feature and attach to remote subvolume of server1 -volume client1 - type protocol/client - option transport-type tcp # for TCP/IP transport - option remote-host 127.0.0.1 # IP address of the remote brick - option transport.socket.remote-port 24016 - option remote-subvolume brick1 # name of the remote volume -end-volume - -### Add client feature and attach to remote subvolume of server2 -volume client2 - type protocol/client - option transport-type tcp # for TCP/IP transport - option remote-host 127.0.0.1 # IP address of the remote brick - option transport.socket.remote-port 24017 - option remote-subvolume brick2 # name of the remote volume -end-volume - -volume client3 - type protocol/client - option transport-type tcp # for TCP/IP transport - option remote-host 127.0.0.1 # IP address of the remote brick - option transport.socket.remote-port 24018 - option remote-subvolume brick3 # name of the remote volume -end-volume - -## Add replicate feature. -volume replicate - type cluster/replicate - subvolumes client1 client2 client3 -end-volume diff --git a/doc/examples/legacy/stripe.vol b/doc/examples/legacy/stripe.vol deleted file mode 100644 index 9524e8198bf..00000000000 --- a/doc/examples/legacy/stripe.vol +++ /dev/null @@ -1,120 +0,0 @@ - -### 'NOTE' -# This file has both server spec and client spec to get an understanding of -# stripe's spec file. Hence can't be used as it is, as a GlusterFS spec file. -# One need to seperate out server spec and client spec to get it working. - -#========================================================================= - -# **** server1 spec file **** - -### Export volume "brick" with the contents of "/home/export" directory. -volume posix1 - type storage/posix # POSIX FS translator - option directory /home/export1 # Export this directory -end-volume - -### Add POSIX record locking support to the storage brick -volume brick1 - type features/posix-locks - option mandatory on # enables mandatory locking on all files - subvolumes posix1 -end-volume - -### Add network serving capability to above brick. -volume server - type protocol/server - option transport-type tcp # For TCP/IP transport - option transport.socket.listen-port 24016 - subvolumes brick1 - option auth.addr.brick1.allow * # access to "brick" volume -end-volume - - -#========================================================================= - -# **** server2 spec file **** -volume posix2 - type storage/posix # POSIX FS translator - option directory /home/export2 # Export this directory -end-volume - -### Add POSIX record locking support to the storage brick -volume brick2 - type features/posix-locks - option mandatory on # enables mandatory locking on all files - subvolumes posix2 -end-volume - -### Add network serving capability to above brick. -volume server - type protocol/server - option transport-type tcp # For TCP/IP transport - option transport.socket.listen-port 24017 - subvolumes brick2 - option auth.addr.brick2.allow * # Allow access to "brick" volume -end-volume - - -#========================================================================= - -# **** server3 spec file **** - -volume posix3 - type storage/posix # POSIX FS translator - option directory /home/export3 # Export this directory -end-volume - -### Add POSIX record locking support to the storage brick -volume brick3 - type features/posix-locks - option mandatory on # enables mandatory locking on all files - subvolumes posix3 -end-volume - -### Add network serving capability to above brick. -volume server - type protocol/server - option transport-type tcp # For TCP/IP transport - option transport.socket.listen-port 24018 - subvolumes brick3 - option auth.addr.brick3.allow * # access to "brick" volume -end-volume - - -#========================================================================= - -# **** Clustered Client config file **** - -### Add client feature and attach to remote subvolume of server1 -volume client1 - type protocol/client - option transport-type tcp # for TCP/IP transport - option remote-host 127.0.0.1 # IP address of the remote brick - option transport.socket.remote-port 24016 - option remote-subvolume brick1 # name of the remote volume -end-volume - -### Add client feature and attach to remote subvolume of server2 -volume client2 - type protocol/client - option transport-type tcp # for TCP/IP transport - option remote-host 127.0.0.1 # IP address of the remote brick - option transport.socket.remote-port 24017 - option remote-subvolume brick2 # name of the remote volume -end-volume - -volume client3 - type protocol/client - option transport-type tcp # for TCP/IP transport - option remote-host 127.0.0.1 # IP address of the remote brick - option transport.socket.remote-port 24018 - option remote-subvolume brick3 # name of the remote volume -end-volume - -## Add Stripe Feature. -volume stripe - type cluster/stripe - subvolumes client1 client2 client3 - option block-size 1MB -end-volume diff --git a/doc/examples/legacy/trace.vol b/doc/examples/legacy/trace.vol deleted file mode 100644 index 59830f26a9d..00000000000 --- a/doc/examples/legacy/trace.vol +++ /dev/null @@ -1,21 +0,0 @@ -volume client - type protocol/client - option transport-type tcp # for TCP/IP transport - option remote-host 192.168.1.10 # IP address of the remote brick - option remote-subvolume brick # name of the remote volume -end-volume - -### 'Trace' translator is a very handy debug tool for GlusterFS, as it can be -# loaded between any of the two volumes without changing the behaviour of the -# filesystem. -# On client side it can be the top most volume in spec (like now) to understand -# what calls are made on FUSE filesystem, when a mounted filesystem is -# accessed. - -volume trace - type debug/trace - subvolumes client -end-volume - -# 'NOTE:' By loading 'debug/trace' translator, filesystem will be very slow as -# it logs each and every calls to the log file. diff --git a/doc/examples/legacy/trash.vol b/doc/examples/legacy/trash.vol deleted file mode 100644 index 3fcf315af08..00000000000 --- a/doc/examples/legacy/trash.vol +++ /dev/null @@ -1,20 +0,0 @@ -volume brick - type storage/posix # POSIX FS translator - option directory /home/export # Export this directory -end-volume - -### 'Trash' translator is best used on server side as it just renames the -# deleted file inside 'trash-dir', and it makes 4 seperate fops for one unlink -# call. -volume trashcan - type features/trash - subvolumes brick - option trash-dir /.trashcan -end-volume - -volume server - type protocol/server - subvolumes trashcan - option transport-type tcp # For TCP/IP transport - option auth.addr.brick.allow 192.168.* # Allow access to "brick" volume -end-volume diff --git a/doc/examples/legacy/write-behind.vol b/doc/examples/legacy/write-behind.vol deleted file mode 100644 index 2b5ed413921..00000000000 --- a/doc/examples/legacy/write-behind.vol +++ /dev/null @@ -1,27 +0,0 @@ -volume client - type protocol/client - option transport-type tcp # for TCP/IP transport - option remote-host 192.168.1.10 # IP address of the remote brick - option remote-subvolume brick # name of the remote volume -end-volume - -## In normal clustered storage type, any of the cluster translators can come here. -# -# Definition of other clients -# -# Definition of cluster translator (may be unify, replicate, or unify over replicate) -# - - -# 'Write-behind' translator is a performance booster for write operation. Best -# used on client side, as its main intension is to reduce the network latency -# caused for each write operation. - -volume wb - type performance/write-behind - subvolumes client - option flush-behind on # default value is 'off' - option window-size 2MB - option enable-O_SYNC no # default is no - option disable-for-first-nbytes 128KB #default is 1 -end-volume diff --git a/doc/features/ctime.md b/doc/features/ctime.md new file mode 100644 index 00000000000..74a77abed4b --- /dev/null +++ b/doc/features/ctime.md @@ -0,0 +1,68 @@ +# Consistent time attributes in gluster across replica/distribute + + +#### Problem: +Traditionally gluster has been using time attributes (ctime, atime, mtime) of files/dirs from bricks. The problem with this approach is that, it is not consisteant across replica and distribute bricks. And applications which depend on it breaks as replica might not always return time attributes from same brick. + +Tar especially gives "file changed as we read it" whenever it detects ctime differences when stat is served from different bricks. The way we have been trying to solve it is to serve the stat structures from same brick in afr, max-time in dht. But it doesn't avoid the problem completely. Because there is no way to change ctime at the moment(lutimes() only allows mtime, atime), there is little we can do to make sure ctimes match after self-heals/xattr updates/rebalance. + +#### Solution Proposed: +Store time attribues (ctime, mtime, atime) as an xattr of the file. The xattr is updated based +on the fop. If a filesystem fop changes only mtime and ctime, update only those in xattr for +that file. + +#### Design Overview: +1) As part of each fop, top layer will generate a time stamp and pass it to the down along + with other information + - This will bring a dependency for NTP synced clients along with servers + - There can be a diff in time if the fop stuck in the xlator for various reason, +for ex: because of locks. + + 2) On the server, posix layer stores the value in the memory (inode ctx) and will sync the data periodically to the disk as an extended attr + - Of course sync call also will force it. And fop comes for an inode which is not linked, we do the sync immediately. + + 3) Each time when inodes are created or initialized it read the data from disk and store in inode ctx. + + 4) Before setting to inode_ctx we compare the timestamp stored and the timestamp received, and only store if the stored value is lesser than the current value. + + 5) So in best case data will be stored and retrieved from the memory. We replace the values in iatt with the values in inode_ctx. + + 6) File ops that changes the parent directory attr time need to be consistent across all the distributed directories across the subvolumes. (for eg: a create call will change ctime and mtime of parent dir) + + - This has to handle separately because we only send the fop to the hashed subvolume. + - We can asynchronously send the timeupdate setattr fop to the other subvoumes and change the values for parent directory if the file fops is successful on hashed subvolume. + - This will have a window where the times are inconsistent across dht subvolume (Please provide your suggestions) + +7) Currently we have couple of mount options for time attributes like noatime, relatime , nodiratime etc. But we are not explicitly handled those options even if it is given as mount option when gluster mount. + + +#### Implementation Overview: +This features involves changes in following xlators. + - utime xlator + - posix xlator + +##### utime xlator: +This is a new client side xlator which does following tasks. + +1. It will generate a time stamp and passes it down in frame->root->ctime and over the network. +2. Based on fop, it also decides the time attributes to be updated and this passed using "frame->root->flags" + + Patches: + 1. https://review.gluster.org/#/c/19857/ + +##### posix xlator: +Following tasks are done in posix xlator: + +1. Provides APIs to set and get the xattr from backend. It also caches the xattr in inode context. During get, it updates time attributes stored in xattr into iatt structure. +2. Based on the flags from utime xlator, relevant fops update the time attributes in the xattr. + + Patches: + 1. https://review.gluster.org/#/c/19267/ + 2. https://review.gluster.org/#/c/19795/ + 3. https://review.gluster.org/#/c/19796/ + +#### Pending Work: +1. Handling of time related mount options (noatime, realatime,etc) +2. flag based create (depending on flags in open, create behaviour might change) +3. Changes in dht for direcotory sync acrosss multiple subvolumes +4. readdirp stat need to be worked on. diff --git a/doc/features/ganesha-ha.md b/doc/features/ganesha-ha.md new file mode 100644 index 00000000000..4b226a22ccf --- /dev/null +++ b/doc/features/ganesha-ha.md @@ -0,0 +1,43 @@ +# Overview of Ganesha HA Resource Agents in GlusterFS 3.7 + +The ganesha_mon RA monitors its ganesha.nfsd daemon. While the +daemon is running, it creates two attributes: ganesha-active and +grace-active. When the daemon stops for any reason, the attributes +are deleted. Deleting the ganesha-active attribute triggers the +failover of the virtual IP (the IPaddr RA) to another node — +according to constraint location rules — where ganesha.nfsd is +still running. + +The ganesha_grace RA monitors the grace-active attribute. When +the grace-active attibute is deleted, the ganesha_grace RA stops, +and will not restart. This triggers pacemaker to invoke the notify +action in the ganesha_grace RAs on the other nodes in the cluster; +which send a DBUS message to their respective ganesha.nfsd. + +(N.B. grace-active is a bit of a misnomer. while the grace-active +attribute exists, everything is normal and healthy. Deleting the +attribute triggers putting the surviving ganesha.nfsds into GRACE.) + +To ensure that the remaining/surviving ganesha.nfsds are put into + NFS-GRACE before the IPaddr (virtual IP) fails over there is a +short delay (sleep) between deleting the grace-active attribute +and the ganesha-active attribute. To summarize, e.g. in a four +node cluster: + +1. on node 2 ganesha_mon::monitor notices that ganesha.nfsd has died + +2. on node 2 ganesha_mon::monitor deletes its grace-active attribute + +3. on node 2 ganesha_grace::monitor notices that grace-active is gone +and returns OCF_ERR_GENERIC, a.k.a. new error. When pacemaker tries +to (re)start ganesha_grace, its start action will return +OCF_NOT_RUNNING, a.k.a. known error, don't attempt further restarts. + +4. on nodes 1, 3, and 4, ganesha_grace::notify receives a post-stop +notification indicating that node 2 is gone, and sends a DBUS message +to its ganesha.nfsd, putting it into NFS-GRACE. + +5. on node 2 ganesha_mon::monitor waits a short period, then deletes +its ganesha-active attribute. This triggers the IPaddr (virt IP) +failover according to constraint location rules. + diff --git a/doc/gluster.8 b/doc/gluster.8 index 87c81208175..ba595edca15 100644 --- a/doc/gluster.8 +++ b/doc/gluster.8 @@ -1,18 +1,11 @@ -.\" Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com> + +.\" Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com> .\" This file is part of GlusterFS. .\" -.\" GlusterFS is free software; you can redistribute it and/or modify -.\" it under the terms of the GNU General Public License as published -.\" by the Free Software Foundation; either version 3 of the License, -.\" or (at your option) any later version. -.\" -.\" GlusterFS is distributed in the hope that it will be useful, but -.\" WITHOUT ANY WARRANTY; without even the implied warranty of -.\" MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -.\" General Public License for more details. -.\" -.\" You should have received a copy of the GNU General Public License -.\" along with this program. If not, see " <http://www.gnu.org/licenses/>. +.\" This file is licensed to you under your choice of the GNU Lesser +.\" General Public License, version 3 or any later version (LGPLv3 or +.\" later), or the GNU General Public License, version 2 (GPLv2), in all +.\" cases as published by the Free Software Foundation. .\" .\" .TH Gluster 8 "Gluster command line utility" "07 March 2011" "Gluster Inc." @@ -23,15 +16,14 @@ gluster - Gluster Console Manager (command line utility) .PP To run the program and display gluster prompt: .PP -.B gluster +.B gluster [--remote-host=<gluster_node>] [--mode=script] [--xml] .PP (or) .PP To specify a command directly: .PP .B gluster -.I [commands] [options] - +.I [commands] [options] [--remote-host=<gluster_node>] [--mode=script] [--xml] .SH DESCRIPTION The Gluster Console Manager is a command line utility for elastic volume management. You can run the gluster command on any export server. The command enables administrators to perform cloud operations, such as creating, expanding, shrinking, rebalancing, and migrating volumes without needing to schedule server downtime. .SH COMMANDS @@ -43,7 +35,13 @@ The Gluster Console Manager is a command line utility for elastic volume managem \fB\ volume info [all|<VOLNAME>] \fR Display information about all volumes, or the specified volume. .TP -\fB\ volume create <NEW-VOLNAME> [stripe <COUNT>] [replica <COUNT>] [transport <tcp|rdma|tcp,rdma>] <NEW-BRICK> ... \fR +\fB\ volume list \fR +List all volumes in cluster +.TP +\fB\ volume status [all | <VOLNAME> [nfs|shd|<BRICK>|quotad]] [detail|clients|mem|inode|fd|callpool|tasks|client-list] \fR +Display status of all or specified volume(s)/brick +.TP +\fB\ volume create <NEW-VOLNAME> [stripe <COUNT>] [[replica <COUNT> [arbiter <COUNT>]]|[replica 2 thin-arbiter 1]] [disperse [<COUNT>]] [disperse-data <COUNT>] [redundancy <COUNT>] [transport <tcp|rdma|tcp,rdma>] <NEW-BRICK> ... <TA-BRICK> \fR Create a new volume of the specified type using the specified bricks and transport type (the default transport type is tcp). To create a volume with both transports (tcp and rdma), give 'transport tcp,rdma' as an option. .TP @@ -56,12 +54,21 @@ Start the specified volume. \fB\ volume stop <VOLNAME> [force] \fR Stop the specified volume. .TP -\fB\ volume rename <VOLNAME> <NEW-VOLNAME> \fR -Rename the specified volume. -.TP \fB\ volume set <VOLNAME> <OPTION> <PARAMETER> [<OPTION> <PARAMETER>] ... \fR Set the volume options. .TP +\fB\ volume get <VOLNAME/all> <OPTION/all> \fR +Get the value of the all options or given option for volume <VOLNAME> or all option. gluster volume get all all is to get all global options +.TP +\fB\ volume reset <VOLNAME> [option] [force] \fR +Reset all the reconfigured options +.TP +\fB\ volume barrier <VOLNAME> {enable|disable} \fR +Barrier/unbarrier file operations on a volume +.TP +\fB\ volume clear-locks <VOLNAME> <path> kind {blocked|granted|all}{inode [range]|entry [basename]|posix [range]} \fR +Clear locks held on path +.TP \fB\ volume help \fR Display help for the volume command. .SS "Brick Commands" @@ -78,7 +85,13 @@ If you remove the brick, the data stored in that brick will not be available. Yo .B replace-brick option. .TP -\fB\ volume rebalance-brick <VOLNAME>(<BRICK> <NEW-BRICK>) start \fR +\fB\ volume reset-brick <VOLNAME> <SOURCE-BRICK> {{start} | {<NEW-BRICK> commit}} \fR +Brings down or replaces the specified source brick with the new brick. +.TP +\fB\ volume replace-brick <VOLNAME> <SOURCE-BRICK> <NEW-BRICK> commit force \fR +Replace the specified source brick with a new brick. +.TP +\fB\ volume rebalance <VOLNAME> start \fR Start rebalancing the specified volume. .TP \fB\ volume rebalance <VOLNAME> stop \fR @@ -86,9 +99,6 @@ Stop rebalancing the specified volume. .TP \fB\ volume rebalance <VOLNAME> status \fR Display the rebalance status of the specified volume. -.TP -\fB\ volume replace-brick <VOLNAME> (<BRICK> <NEW-BRICK>) start|pause|abort|status|commit \fR -Replace the specified brick. .SS "Log Commands" .TP \fB\ volume log filename <VOLNAME> [BRICK] <DIRECTORY> \fB @@ -99,10 +109,22 @@ Locate the log file for corresponding volume/brick. .TP \fB\ volume log rotate <VOLNAME> [BRICK] \fB Rotate the log file for corresponding volume/brick. +.TP +\fB\ volume profile <VOLNAME> {start|info [peek|incremental [peek]|cumulative|clear]|stop} [nfs] \fR +Profile operations on the volume. Once started, volume profile <volname> info provides cumulative statistics of the FOPs performed. +.TP +\fB\ volume top <VOLNAME> {open|read|write|opendir|readdir|clear} [nfs|brick <brick>] [list-cnt <value>] | {read-perf|write-perf} [bs <size> count <count>] [brick <brick>] [list-cnt <value>] \fR +Generates a profile of a volume representing the performance and bottlenecks/hotspots of each brick. +.TP +\fB\ volume statedump <VOLNAME> [[nfs|quotad] [all|mem|iobuf|callpool|priv|fd|inode|history]... | [client <hostname:process-id>]] \fR +Dumps the in memory state of the specified process or the bricks of the volume. +.TP +\fB\ volume sync <HOSTNAME> [all|<VOLNAME>] \fR +Sync the volume information from a peer .SS "Peer Commands" .TP \fB\ peer probe <HOSTNAME> \fR -Probe the specified peer. +Probe the specified peer. In case the <HOSTNAME> given belongs to an already probed peer, the peer probe command will add the hostname to the peer if required. .TP \fB\ peer detach <HOSTNAME> \fR Detach the specified peer. @@ -110,10 +132,216 @@ Detach the specified peer. \fB\ peer status \fR Display the status of peers. .TP +\fB\ pool list \fR +List all the nodes in the pool (including localhost) +.TP \fB\ peer help \fR Display help for the peer command. +.SS "Quota Commands" +.TP +\fB\ volume quota <VOLNAME> enable \fR +Enable quota on the specified volume. This will cause all the directories in the filesystem hierarchy to be accounted and updated thereafter on each operation in the the filesystem. To kick start this accounting, a crawl is done over the hierarchy with an auxiliary client. +.TP +\fB\ volume quota <VOLNAME> disable \fR +Disable quota on the volume. This will disable enforcement and accounting in the filesystem. Any configured limits will be lost. +.TP +\fB\ volume quota <VOLNAME> limit-usage <PATH> <SIZE> [<PERCENT>] \fR +Set a usage limit on the given path. Any previously set limit is overridden to the new value. The soft limit can optionally be specified (as a percentage of hard limit). If soft limit percentage is not provided the default soft limit value for the volume is used to decide the soft limit. +.TP +\fB\ volume quota <VOLNAME> limit-objects <PATH> <SIZE> [<PERCENT>] \fR +Set an inode limit on the given path. Any previously set limit is overridden to the new value. The soft limit can optionally be specified (as a percentage of hard limit). If soft limit percentage is not provided the default soft limit value for the volume is used to decide the soft limit. +.TP +NOTE: valid units of SIZE are : B, KB, MB, GB, TB, PB. If no unit is specified, the unit defaults to bytes. +.TP +\fB\ volume quota <VOLNAME> remove <PATH> \fR +Remove any usage limit configured on the specified directory. Note that if any limit is configured on the ancestors of this directory (previous directories along the path), they will still be honored and enforced. +.TP +\fB\ volume quota <VOLNAME> remove-objects <PATH> \fR +Remove any inode limit configured on the specified directory. Note that if any limit is configured on the ancestors of this directory (previous directories along the path), they will still be honored and enforced. +.TP +\fB\ volume quota <VOLNAME> list <PATH> \fR +Lists the usage and limits configured on directory(s). If a path is given only the limit that has been configured on the directory(if any) is displayed along with the directory's usage. If no path is given, usage and limits are displayed for all directories that has limits configured. +.TP +\fB\ volume quota <VOLNAME> list-objects <PATH> \fR +Lists the inode usage and inode limits configured on directory(s). If a path is given only the limit that has been configured on the directory(if any) is displayed along with the directory's inode usage. If no path is given, usage and limits are displayed for all directories that has limits configured. +.TP +\fB\ volume quota <VOLNAME> default-soft-limit <PERCENT> \fR +Set the percentage value for default soft limit for the volume. +.TP +\fB\ volume quota <VOLNAME> soft-timeout <TIME> \fR +Set the soft timeout for the volume. The interval in which limits are retested before the soft limit is breached. +.TP +\fB\ volume quota <VOLNAME> hard-timeout <TIME> \fR +Set the hard timeout for the volume. The interval in which limits are retested after the soft limit is breached. +.TP +\fB\ volume quota <VOLNAME> alert-time <TIME> \fR +Set the frequency in which warning messages need to be logged (in the brick logs) once soft limit is breached. +.TP +\fB\ volume inode-quota <VOLNAME> enable/disable \fR +Enable/disable inode-quota for <VOLNAME> +.TP +\fB\ volume quota help \fR +Display help for volume quota commands +.TP +NOTE: valid units of time and their symbols are : hours(h/hr), minutes(m/min), seconds(s/sec), weeks(w/wk), Days(d/days). +.SS "Geo-replication Commands" +.TP +\fI\ Note\fR: password-less ssh, from the master node (where these commands are executed) to the slave node <SLAVE_HOST>, is a prerequisite for the geo-replication commands. +.TP +\fB\ system:: execute gsec_create\fR +Generates pem keys which are required for push-pem +.TP +\fB\ volume geo-replication <MASTER_VOL> <SLAVE_HOST>::<SLAVE_VOL> create [[ssh-port n][[no-verify]|[push-pem]]] [force]\fR +Create a new geo-replication session from <MASTER_VOL> to <SLAVE_HOST> host machine having <SLAVE_VOL>. +Use ssh-port n if custom SSH port is configured in slave nodes. +Use no-verify if the rsa-keys of nodes in master volume is distributed to slave nodes through an external agent. +Use push-pem to push the keys automatically. +.TP +\fB\ volume geo-replication <MASTER_VOL> <SLAVE_HOST>::<SLAVE_VOL> {start|stop} [force] \fR +Start/stop the geo-replication session from <MASTER_VOL> to <SLAVE_HOST> host machine having <SLAVE_VOL>. +.TP +\fB\ volume geo-replication [<MASTER_VOL> [<SLAVE_HOST>::<SLAVE_VOL>]] status [detail] \fR +Query status of the geo-replication session from <MASTER_VOL> to <SLAVE_HOST> host machine having <SLAVE_VOL>. +.TP +\fB\ volume geo-replication <MASTER_VOL> <SLAVE_HOST>::<SLAVE_VOL> {pause|resume} [force] \fR +Pause/resume the geo-replication session from <MASTER_VOL> to <SLAVE_HOST> host machine having <SLAVE_VOL>. +.TP +\fB\ volume geo-replication <MASTER_VOL> <SLAVE_HOST>::<SLAVE_VOL> delete [reset-sync-time]\fR +Delete the geo-replication session from <MASTER_VOL> to <SLAVE_HOST> host machine having <SLAVE_VOL>. +Optionally you can also reset the sync time in case you need to resync the entire volume on session recreate. +.TP +\fB\ volume geo-replication <MASTER_VOL> <SLAVE_HOST>::<SLAVE_VOL> config [[!]<options> [<value>]] \fR +View (when no option provided) or set configuration for this geo-replication session. +Use "!<OPTION>" to reset option <OPTION> to default value. +.SS "Bitrot Commands" +.TP +\fB\ volume bitrot <VOLNAME> {enable|disable} \fR +Enable/disable bitrot for volume <VOLNAME> +.TP +\fB\ volume bitrot <VOLNAME> signing-time <time-in-secs> \fR +Waiting time for an object after last fd is closed to start signing process. +.TP +\fB\ volume bitrot <VOLNAME> signer-threads <count> \fR +Number of signing process threads. Usually set to number of available cores. +.TP +\fB\ volume bitrot <VOLNAME> scrub-throttle {lazy|normal|aggressive} \fR +Scrub-throttle value is a measure of how fast or slow the scrubber scrubs the filesystem for volume <VOLNAME> +.TP +\fB\ volume bitrot <VOLNAME> scrub-frequency {hourly|daily|weekly|biweekly|monthly} \fR +Scrub frequency for volume <VOLNAME> +.TP +\fB\ volume bitrot <VOLNAME> scrub {pause|resume|status|ondemand} \fR +Pause/Resume scrub. Upon resume, scrubber continues where it left off. status option shows the statistics of scrubber. ondemand option starts the scrubbing immediately if the scrubber is not paused or already running. +.TP +\fB\ volume bitrot help \fR +Display help for volume bitrot commands +.TP +.SS "Snapshot Commands" +.TP +\fB\ snapshot create <snapname> <volname> [no-timestamp] [description <description>] [force] \fR +Creates a snapshot of a GlusterFS volume. User can provide a snap-name and a description to identify the snap. Snap will be created by appending timestamp in GMT. User can override this behaviour using "no-timestamp" option. The description cannot be more than 1024 characters. To be able to take a snapshot, volume should be present and it should be in started state. +.TP +\fB\ snapshot restore <snapname> \fR +Restores an already taken snapshot of a GlusterFS volume. Snapshot restore is an offline activity therefore if the volume is online (in started state) then the restore operation will fail. Once the snapshot is restored it will not be available in the list of snapshots. +.TP +\fB\ snapshot clone <clonename> <snapname> \fR +Create a clone of a snapshot volume, the resulting volume will be GlusterFS volume. User can provide a clone-name. To be able to take a clone, snapshot should be present and it should be in activated state. +.TP +\fB\ snapshot delete ( all | <snapname> | volume <volname> ) \fR +If snapname is specified then mentioned snapshot is deleted. If volname is specified then all snapshots belonging to that particular volume is deleted. If keyword *all* is used then all snapshots belonging to the system is deleted. +.TP +\fB\ snapshot list [volname] \fR +Lists all snapshots taken. If volname is provided, then only the snapshots belonging to that particular volume is listed. +.TP +\fB\ snapshot info [snapname | (volume <volname>)] \fR +This command gives information such as snapshot name, snapshot UUID, time at which snapshot was created, and it lists down the snap-volume-name, number of snapshots already taken and number of snapshots still available for that particular volume, and the state of the snapshot. If snapname is specified then info of the mentioned snapshot is displayed. If volname is specified then info of all snapshots belonging to that volume is displayed. If both snapname and volname is not specified then info of all the snapshots present in the system are displayed. +.TP +\fB\ snapshot status [snapname | (volume <volname>)] \fR +This command gives status of the snapshot. The details included are snapshot brick path, volume group(LVM details), status of the snapshot bricks, PID of the bricks, data percentage filled for that particular volume group to which the snapshots belong to, and total size of the logical volume. + +If snapname is specified then status of the mentioned snapshot is displayed. If volname is specified then status of all snapshots belonging to that volume is displayed. If both snapname and volname is not specified then status of all the snapshots present in the system are displayed. +.TP +\fB\ snapshot config [volname] ([snap-max-hard-limit <count>] [snap-max-soft-limit <percent>]) | ([auto-delete <enable|disable>]) | ([activate-on-create <enable|disable>]) +Displays and sets the snapshot config values. + +snapshot config without any keywords displays the snapshot config values of all volumes in the system. If volname is provided, then the snapshot config values of that volume is displayed. + +Snapshot config command along with keywords can be used to change the existing config values. If volname is provided then config value of that volume is changed, else it will set/change the system limit. + +snap-max-soft-limit and auto-delete are global options, that will be inherited by all volumes in the system and cannot be set to individual volumes. + +snap-max-hard-limit can be set globally, as well as per volume. The lowest limit between the global system limit and the volume specific limit, becomes the +"Effective snap-max-hard-limit" for a volume. + +snap-max-soft-limit is a percentage value, which is applied on the "Effective snap-max-hard-limit" to get the "Effective snap-max-soft-limit". + +When auto-delete feature is enabled, then upon reaching the "Effective snap-max-soft-limit", with every successful snapshot creation, the oldest snapshot will be deleted. + +When auto-delete feature is disabled, then upon reaching the "Effective snap-max-soft-limit", the user gets a warning with every successful snapshot creation. + +When auto-delete feature is disabled, then upon reaching the "Effective snap-max-hard-limit", further snapshot creations will not be allowed. + +activate-on-create is disabled by default. If you enable activate-on-create, then further snapshot will be activated during the time of snapshot creation. +.TP +\fB\ snapshot activate <snapname> \fR +Activates the mentioned snapshot. + +Note : By default the snapshot is activated during snapshot creation. +.TP +\fB\ snapshot deactivate <snapname> \fR +Deactivates the mentioned snapshot. +.TP +\fB\ snapshot help \fR +Display help for the snapshot commands. +.SS "Self-heal Commands" +.TP +\fB\ volume heal <VOLNAME>\fR +Triggers index self heal for the files that need healing. + +.TP +\fB\ volume heal <VOLNAME> [enable | disable]\fR +Enable/disable self-heal-daemon for volume <VOLNAME>. + +.TP +\fB\ volume heal <VOLNAME> full\fR +Triggers self heal on all the files. + +.TP +\fB\ volume heal <VOLNAME> info \fR +Lists the files that need healing. + +.TP +\fB\ volume heal <VOLNAME> info split-brain \fR +Lists the files which are in split-brain state. + +.TP +\fB\ volume heal <VOLNAME> statistics \fR +Lists the crawl statistics. + +.TP +\fB\ volume heal <VOLNAME> statistics heal-count \fR +Displays the count of files to be healed. + +.TP +\fB\ volume heal <VOLNAME> statistics heal-count replica <HOSTNAME:BRICKNAME> \fR +Displays the number of files to be healed from a particular replica subvolume to which the brick <HOSTNAME:BRICKNAME> belongs. + +.TP +\fB\ volume heal <VOLNAME> split-brain bigger-file <FILE> \fR +Performs healing of <FILE> which is in split-brain by choosing the bigger file in the replica as source. + +.TP +\fB\ volume heal <VOLNAME> split-brain source-brick <HOSTNAME:BRICKNAME> \fR +Selects <HOSTNAME:BRICKNAME> as the source for all the files that are in split-brain in that replica and heals them. + +.TP +\fB\ volume heal <VOLNAME> split-brain source-brick <HOSTNAME:BRICKNAME> <FILE> \fR +Selects the split-brained <FILE> present in <HOSTNAME:BRICKNAME> as source and completes heal. .SS "Other Commands" .TP +\fB\ get-state [<daemon>] [[odir </path/to/output/dir/>] [file <filename>]] [detail|volumeoptions] \fR +Get local state representation of mentioned daemon and store data in provided path information +.TP \fB\ help \fR Display the command options. .TP @@ -121,7 +349,7 @@ Display the command options. Exit the gluster command line interface. .SH FILES -/etc/glusterd/* +/var/lib/glusterd/* .SH SEE ALSO .nf \fBfusermount\fR(1), \fBmount.glusterfs\fR(8), \fBglusterfs\fR(8), \fBglusterd\fR(8) diff --git a/doc/glusterd.8 b/doc/glusterd.8 index 267a7e00f48..e3768c78761 100644 --- a/doc/glusterd.8 +++ b/doc/glusterd.8 @@ -1,20 +1,11 @@ .\" -.\" Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com> +.\" Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com> .\" This file is part of GlusterFS. .\" -.\" GlusterFS is free software; you can redistribute it and/or modify -.\" it under the terms of the GNU General Public License as published -.\" by the Free Software Foundation; either version 3 of the License, -.\" or (at your option) any later version. -.\" -.\" GlusterFS is distributed in the hope that it will be useful, but -.\" WITHOUT ANY WARRANTY; without even the implied warranty of -.\" MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -.\" General Public License for more details. -.\" -.\" You should have received a copy of the GNU General Public License -.\" along with this program. If not, see -.\" <http://www.gnu.org/licenses/>. +.\" This file is licensed to you under your choice of the GNU Lesser +.\" General Public License, version 3 or any later version (LGPLv3 or +.\" later), or the GNU General Public License, version 2 (GPLv2), in all +.\" cases as published by the Free Software Foundation. .\" .\" @@ -39,6 +30,9 @@ File to use for logging. \fB\-L <LOGLEVEL>, \fB\-\-log\-level=<LOGLEVEL>\fR Logging severity. Valid options are TRACE, DEBUG, INFO, WARNING, ERROR and CRITICAL (the default is INFO). .TP +\fB\-\-localtime\-logging\fR +Enable localtime log timestamps. +.TP \fB\-\-debug\fR Run the program in debug mode. This option sets \fB\-\-no\-daemon\fR, \fB\-\-log\-level\fR to DEBUG and \fB\-\-log\-file\fR to console. @@ -59,7 +53,7 @@ Print the program version. .PP .SH FILES -/etc/glusterd/* +/var/lib/glusterd/* .SH SEE ALSO .nf diff --git a/doc/glusterd.vol b/doc/glusterd.vol deleted file mode 100644 index 809042cbdd9..00000000000 --- a/doc/glusterd.vol +++ /dev/null @@ -1,8 +0,0 @@ -volume management - type mgmt/glusterd - option working-directory /etc/glusterd - option transport-type socket,rdma - option transport.socket.keepalive-time 10 - option transport.socket.keepalive-interval 2 - option transport.socket.read-fail-log off -end-volume diff --git a/doc/glusterfs.8 b/doc/glusterfs.8 index 37bf67d1821..3d359ea85e4 100644 --- a/doc/glusterfs.8 +++ b/doc/glusterfs.8 @@ -1,19 +1,10 @@ -.\" Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com> +.\" Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com> .\" This file is part of GlusterFS. .\" -.\" GlusterFS is free software; you can redistribute it and/or modify -.\" it under the terms of the GNU General Public License as published -.\" by the Free Software Foundation; either version 3 of the License, -.\" or (at your option) any later version. -.\" -.\" GlusterFS is distributed in the hope that it will be useful, but -.\" WITHOUT ANY WARRANTY; without even the implied warranty of -.\" MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -.\" General Public License for more details. -.\" -.\" You should have received a copy of the GNU General Public License -.\" long with this program. If not, see -.\" <http://www.gnu.org/licenses/>. +.\" This file is licensed to you under your choice of the GNU Lesser +.\" General Public License, version 3 or any later version (LGPLv3 or +.\" later), or the GNU General Public License, version 2 (GPLv2), in all +.\" cases as published by the Free Software Foundation. .\" .\" .\" @@ -32,7 +23,8 @@ be made of any commodity hardware, such as x86-64 server with SATA-II RAID and Infiniband HBA. GlusterFS is fully POSIX compliant file system. On client side, it has dependency -on FUSE package, on server side, it works seemlessly on different operating systems. Currently supported on GNU/Linux and Solaris. +on FUSE package, on server side, it works seemlessly on different operating systems. +Currently supported on GNU/Linux and Solaris. .SH OPTIONS @@ -40,33 +32,58 @@ on FUSE package, on server side, it works seemlessly on different operating syst .PP .TP \fB\-f, \fB\-\-volfile=VOLUME-FILE\fR -File to use as VOLUME-FILE (the default is /etc/glusterfs/glusterfs.vol). +File to use as VOLUME-FILE. .TP \fB\-l, \fB\-\-log\-file=LOGFILE\fR -File to use for logging. +File to use for logging (the default is <INSTALL-DIR>/var/log/glusterfs/<MOUNT-POINT>.log). .TP \fB\-L, \fB\-\-log\-level=LOGLEVEL\fR -Logging severity. Valid options are TRACE, DEBUG, INFO, WARNING, ERROR and CRITICAL (the default is WARNING). +Logging severity. Valid options are TRACE, DEBUG, INFO, WARNING, ERROR and CRITICAL (the default is INFO). .TP \fB\-s, \fB\-\-volfile\-server=SERVER\fR Server to get the volume from. This option overrides \fB\-\-volfile \fR option. +.TP +\fB\-\-volfile\-max\-fetch\-attempts=MAX\-ATTEMPTS\fR +Maximum number of connect attempts to server. This option should be provided with +\fB\-\-volfile\-server\fR option (the default is 1). .SS "Advanced options" .PP .TP +\fB\-\-acl\fR +Mount the filesystem with POSIX ACL support. +.TP +\fB\-\-localtime\-logging\fR +Enable localtime log timestamps. +.TP \fB\-\-debug\fR Run in debug mode. This option sets \fB\-\-no\-daemon\fR, \fB\-\-log\-level\fR to DEBUG, and \fB\-\-log\-file\fR to console. .TP +\fB\-\-enable\-ino32=BOOL\fR +Use 32-bit inodes when mounting to workaround application that doesn't support 64-bit inodes. +.TP +\fB\-\-fopen\-keep\-cache[=BOOL]\fR +Do not purge the cache on file open (default: false). +.TP +\fB\-\-mac\-compat=BOOL\fR +Provide stubs for attributes needed for seamless operation on Macs (the default is off). +.TP \fB\-N, \fB\-\-no\-daemon\fR Run in the foreground. .TP -\fB\-\-read\-only\fR -Make the file system read-only. -.TP \fB\-p, \fB\-\-pid\-file=PIDFILE\fR File to use as PID file. .TP +\fB\-\-read\-only\fR +Mount the file system in 'read-only' mode. +.TP +\fB\-\-selinux\fR +Enable SELinux label (extended attributes) support on inodes. +.TP +\fB\-S, \fB\-\-socket\-file=SOCKFILE\fR +File to use as unix-socket. +.TP \fB\-\-volfile\-id=KEY\fR Key of the volume file to be fetched from the server. .TP @@ -74,26 +91,61 @@ Key of the volume file to be fetched from the server. Port number of volfile server. .TP \fB\-\-volfile\-server\-transport=TRANSPORT\fR -Transport type to get volume file from server (the default is socket). +Transport type to get volume file from server (the default is tcp). .TP \fB\-\-volume\-name=VOLUME\-NAME\fR Volume name to be used for MOUNT-POINT (the default is top most volume in VOLUME-FILE). .TP +\fB\-\-worm\fR +Mount the filesystem in 'worm' mode. +.TP \fB\-\-xlator\-option=VOLUME\-NAME.OPTION=VALUE\fR Add/Override a translator option for a volume with the specified value. +.TP +\fB\-\-subdir\-mount=SUBDIR\-MOUNT\-PATH\fR +Mount subdirectory instead of the '/' of volume. .SS "Fuse options" .PP .TP +\fB\-\-attr\-times\-granularity=NANOSECONDS\fR +Declare supported granularity of file attribute times (default is 0 which kernel handles as unspecified; valid real values are between 1 and 1000000000). +.TP \fB\-\-attribute\-timeout=SECONDS\fR Set attribute timeout to SECONDS for inodes in fuse kernel module (the default is 1). .TP +\fB\-\-background\-qlen=N\fR +Set fuse module's background queue length to N (the default is 64). +.TP +\fB\-\-congestion\-threshold=N\fR +Set fuse module's congestion threshold to N (the default is 48). +.TP +\fB\-\-direct\-io\-mode=BOOL|auto\fR +Specify fuse direct I/O strategy (the default is auto). +.TP +\fB\-\-dump-fuse=PATH\f\R +Dump fuse traffic to PATH +.TP \fB\-\-entry\-timeout=SECONDS\fR Set entry timeout to SECONDS in fuse kernel module (the default is 1). .TP -\fB\-\-direct\-io\-mode=BOOL\fR -Enable/Disable the direct-I/O mode in fuse module (the default is enable). +\fB\-\-gid\-timeout=SECONDS\fR +Set auxiliary group list timeout to SECONDS for fuse translator (the default is 0). +.TP +\fB\-\-kernel-writeback-cache=BOOL\fR +Enable fuse in-kernel writeback cache. +.TP +\fB\-\-negative\-timeout=SECONDS\fR +Set negative timeout to SECONDS in fuse kernel module (the default is 0). +.TP +\fB\-\-auto\-invalidation=BOOL\fR +controls whether fuse-kernel can auto-invalidate attribute, dentry and +page-cache. Disable this only if same files/directories are not +accessed across two different mounts concurrently [default: on]. +.TP +\fB\-\-volfile-check\fR +Enable strict volume file checking. .SS "Miscellaneous Options" .PP @@ -110,7 +162,7 @@ Print the program version. .PP .SH FILES -/etc/glusterfs/*.vol, /etc/glusterd/vols/*/*.vol +/var/lib/glusterd/vols/*/*.vol .SH EXAMPLES mount a volume named foo on server bar with log level DEBUG on mount point /mnt/foo @@ -124,6 +176,6 @@ mount a volume named foo on server bar with log level DEBUG on mount point .fi .SH COPYRIGHT .nf -Copyright(c) 2006-2011 Gluster, Inc. <http://www.gluster.com> +Copyright(c) 2006-2011 Red Hat, Inc. <http://www.redhat.com> \fR .fi diff --git a/doc/glusterfs.vol.sample b/doc/glusterfs.vol.sample deleted file mode 100644 index 977363b921b..00000000000 --- a/doc/glusterfs.vol.sample +++ /dev/null @@ -1,53 +0,0 @@ -### file: client-volume.vol.sample - -##################################### -### GlusterFS Client Volume File ## -##################################### - -#### CONFIG FILE RULES: -### "#" is comment character. -### - Config file is case sensitive -### - Options within a volume block can be in any order. -### - Spaces or tabs are used as delimitter within a line. -### - Each option should end within a line. -### - Missing or commented fields will assume default values. -### - Blank/commented lines are allowed. -### - Sub-volumes should already be defined above before referring. - -### Add client feature and attach to remote subvolume -volume client - type protocol/client - option transport-type tcp -# option transport-type unix -# option transport-type ib-sdp - option remote-host 127.0.0.1 # IP address of the remote brick -# option transport.socket.remote-port 24016 - -# option transport-type rdma -# option transport.rdma.remote-port 24016 -# option transport.rdma.work-request-send-count 16 -# option transport.rdma.work-request-recv-count 16 - - option remote-subvolume brick # name of the remote volume -end-volume - -### Add readahead feature -#volume readahead -# type performance/read-ahead -# option page-count 2 # cache per file = (page-count x page-size) -# subvolumes client -#end-volume - -### Add IO-Cache feature -#volume iocache -# type performance/io-cache -# subvolumes readahead -#end-volume - -### Add writeback feature -#volume writeback -# type performance/write-behind -# option window-size 2MB -# option flush-behind off -# subvolumes iocache -#end-volume diff --git a/doc/glusterfsd.8 b/doc/glusterfsd.8 index 17d053a5c07..bc1de2a8c80 100644 --- a/doc/glusterfsd.8 +++ b/doc/glusterfsd.8 @@ -1,19 +1,10 @@ -.\" Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com> +.\" Copyright (c) 20088888888-2012 Red Hat, Inc. <http://www.redhat.com> .\" This file is part of GlusterFS. .\" -.\" GlusterFS is free software; you can redistribute it and/or modify -.\" it under the terms of the GNU General Public License as published -.\" by the Free Software Foundation; either version 3 of the License, -.\" or (at your option) any later version. -.\" -.\" GlusterFS is distributed in the hope that it will be useful, but -.\" WITHOUT ANY WARRANTY; without even the implied warranty of -.\" MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -.\" General Public License for more details. -.\" -.\" You should have received a copy of the GNU General Public License -.\" long with this program. If not, see -.\" <http://www.gnu.org/licenses/>. +.\" This file is licensed to you under your choice of the GNU Lesser +.\" General Public License, version 3 or any later version (LGPLv3 or +.\" later), or the GNU General Public License, version 2 (GPLv2), in all +.\" cases as published by the Free Software Foundation. .\" .\" .\" @@ -60,6 +51,9 @@ Server to get the volume from. This option overrides \fB\-\-volfile option .PP .TP +\fB\-\-localtime\-logging\fR +Enable localtime log timestamps. +.TP \fB\-\-debug\fR Run in debug mode. This option sets \fB\-\-no\-daemon\fR, \fB\-\-log\-level\fR to DEBUG and \fB\-\-log\-file\fR to console @@ -89,7 +83,7 @@ KEY of the volume file to be fetched from server Port number of volfile server .TP \fB\-\-volfile\-server\-transport=TRANSPORT\fR -Transport type to get volume file from server [default: socket] +Transport type to get volume file from server [default: tcp] .TP \fB\-\-volume\-name=VOLUME\-NAME\fR Volume name to be used for MOUNT-POINT [default: top most volume in @@ -110,6 +104,14 @@ Set entry timeout to SECONDS in fuse kernel module [default: 1] .TP \fB\-\-direct\-io\-mode=BOOL\fR Enable/Disable direct-io mode in fuse module [default: enable] +.TP +\fB\-\-resolve-gids\fR +Resolve all auxiliary groups in fuse translator (max 32 otherwise) +.TP +\fB\-\-auto\-invalidation=BOOL\fR +controls whether fuse-kernel can auto-invalidate attribute, dentry and +page-cache. Disable this only if same files/directories are not +accessed across two different mounts concurrently [default: on] .SS "Miscellaneous Options" .PP @@ -131,7 +133,7 @@ Print program version .SH EXAMPLES Start a GlusterFS server on localhost with volume name foo -glusterfsd \-s localhost \-\-volfile\-id foo.server.media-disk\-1 \-p /etc/glusterd/vols/foo/run/server\-media\-disk\-1.pid \-S /tmp/<uniqueid>.socket \-\-brick-name /media/disk\-1 \-l /var/log/glusterfs/bricks/media\-disk\-1.log \-\-brick\-port 24009 \-\-xlator\-option foo\-server.listen-port=24009 +glusterfsd \-s localhost \-\-volfile\-id foo.server.media-disk\-1 \-p /var/lib/glusterd/vols/foo/run/server\-media\-disk\-1.pid \-S /tmp/<uniqueid>.socket \-\-brick-name /media/disk\-1 \-l /var/log/glusterfs/bricks/media\-disk\-1.log \-\-brick\-port 24009 \-\-xlator\-option foo\-server.listen-port=24009 .SH SEE ALSO .nf diff --git a/doc/glusterfsd.vol.sample b/doc/glusterfsd.vol.sample deleted file mode 100644 index ec2fd341ef0..00000000000 --- a/doc/glusterfsd.vol.sample +++ /dev/null @@ -1,44 +0,0 @@ -### file: server-volume.vol.sample - -##################################### -### GlusterFS Server Volume File ## -##################################### - -#### CONFIG FILE RULES: -### "#" is comment character. -### - Config file is case sensitive -### - Options within a volume block can be in any order. -### - Spaces or tabs are used as delimitter within a line. -### - Multiple values to options will be : delimited. -### - Each option should end within a line. -### - Missing or commented fields will assume default values. -### - Blank/commented lines are allowed. -### - Sub-volumes should already be defined above before referring. - -### Export volume "brick" with the contents of "/home/export" directory. -volume brick - type storage/posix # POSIX FS translator - option directory /home/export # Export this directory -end-volume - -### Add network serving capability to above brick. -volume server - type protocol/server - option transport-type tcp -# option transport-type unix -# option transport-type ib-sdp -# option transport.socket.bind-address 192.168.1.10 # Default is to listen - # on all interfaces -# option transport.socket.listen-port 24016 - -# option transport-type rdma -# option transport.rdma.listen-port 24016 -# option transport.rdma.work-request-send-count 64 -# option transport.rdma.work-request-recv-count 64 - - subvolumes brick -# NOTE: Access to any volume through protocol/server is denied by -# default. You need to explicitly grant access through # "auth" -# option. - option auth.addr.brick.allow * # Allow access to "brick" volume -end-volume diff --git a/doc/legacy/authentication.txt b/doc/legacy/authentication.txt deleted file mode 100644 index 73cb21d73ba..00000000000 --- a/doc/legacy/authentication.txt +++ /dev/null @@ -1,112 +0,0 @@ - -* Authentication is provided by two modules addr and login. Login based authentication uses username/password from client for authentication. Each module returns either ACCEPT, REJCET or DONT_CARE. DONT_CARE is returned if the input authentication information to the module is not concerned to its working. The theory behind authentication is that "none of the auth modules should return REJECT and atleast one of them should return ACCEPT" - -* Currently all the authentication related information is passed un-encrypted over the network from client to server. - ----------------------------------------------------------------------------------------------------- -* options provided in protocol/client: - * for username/password based authentication: - option username <username> - option password <password> - * client can have only one set of username/password - * for addr based authentication: - * no options required in protocol/client. Client has to bind to privileged port (port < 1024 ) which means the process in which protocol/client is loaded has to be run as root. - ----------------------------------------------------------------------------------------------------- -* options provided in protocol/server: - * for username/password based authentication: - option auth.login.<brick>.allow [comma seperated list of usernames using which clients can connect to volume <brick>] - option auth.login.<username>.password <password> #specify password <password> for username <username> - * for addr based authentication: - option auth.addr.<brick>.allow [comma seperated list of ip-addresses/unix-paths from which clients are allowed to connect to volume <brick>] - option auth.addr.<brick>.reject [comma seperated list of ip-addresses/unix-paths from which clients are not allowed to connect to volume <brick>] - * negation operator '!' is used to invert the sense of matching. - Eg., option auth.addr.brick.allow !a.b.c.d #do not allow client from a.b.c.d to connect to volume brick - option auth.addr.brick.reject !w.x.y.z #allow client from w.x.y.z to connect to volume brick - * wildcard '*' can be used to match any ip-address/unix-path - ----------------------------------------------------------------------------------------------------- - -* Usecases: - -* username/password based authentication only - protocol/client: - option username foo - option password foo-password - option remote-subvolume foo-brick - - protocol/server: - option auth.login.foo-brick.allow foo,who #,other users allowed to connect to foo-brick - option auth.login.foo.password foo-password - option auth.login.who.password who-password - - * in protocol/server, dont specify ip from which client is connecting in auth.addr.foo-brick.reject list - -**************************************************************************************************** - -* ip based authentication only - protocol/client: - option remote-subvolume foo-brick - * Client is connecting from a.b.c.d - - protocol/server: - option auth.addr.foo-brick.allow a.b.c.d,e.f.g.h,i.j.k.l #, other ip addresses from which clients are allowed to connect to foo-brick - -**************************************************************************************************** -* ip and username/password based authentication - * allow only "user foo from a.b.c.d" - protocol/client: - option username foo - option password foo-password - option remote-subvolume foo-brick - - protocol/server: - option auth.login.foo-brick.allow foo - option auth.login.foo.password foo-password - option auth.addr.foo-brick.reject !a.b.c.d - - * allow only "user foo" from a.b.c.d i.e., only user foo is allowed from a.b.c.d, but anyone is allowed from ip addresses other than a.b.c.d - protocol/client: - option username foo - option password foo-password - option remote-subvolume foo-brick - - protocol/server: - option auth.login.foo-brick.allow foo - option auth.login.foo.password foo-password - option auth.addr.foo-brick.allow !a.b.c.d - - * reject only "user shoo from a.b.c.d" - protcol/client: - option remote-subvolume shoo-brick - - protocol/server: - # observe that no "option auth.login.shoo-brick.allow shoo" given - # Also other users from a.b.c.d have to be explicitly allowed using auth.login.shoo-brick.allow ... - option auth.addr.shoo-brick.allow !a.b.c.d - - * reject only "user shoo" from a.b.c.d i.e., user shoo from a.b.c.d has to be rejected. - * same as reject only "user shoo from a.b.c.d" above, but rules have to be added whether to allow ip addresses (and users from those ips) other than a.b.c.d - -**************************************************************************************************** - -* ip or username/password based authentication - - * allow user foo or clients from a.b.c.d - protocol/client: - option remote-subvolume foo-brick - - protocol/server: - option auth.login.foo-brick.allow foo - option auth.login.foo.password foo-password - option auth.addr.foo-brick.allow a.b.c.d - - * reject user shoo or clients from a.b.c.d - protocol/client: - option remote-subvolume shoo-brick - - protocol/server: - option auth.login.shoo-brick.allow <usernames other than shoo> - #for each username mentioned in the above <usernames other than shoo> list, specify password as below - option auth.login.<username other than shoo>.password password - option auth.addr.shoo-brick.reject a.b.c.d diff --git a/doc/legacy/booster.txt b/doc/legacy/booster.txt deleted file mode 100644 index 6f215c63848..00000000000 --- a/doc/legacy/booster.txt +++ /dev/null @@ -1,54 +0,0 @@ -Introduction -============ -* booster is a LD_PRELOADable library which boosts read/write performance by bypassing fuse for - read() and write() calls. - -Requirements -============ -* fetch volfile from glusterfs. -* identify whether multiple files are from the same mount point. If so, use only one context. - -Design -====== -* for a getxattr, along with other attributes, fuse returns following attributes. - * contents of client volume-file. - * mount point. - -* LD_PRELOADed booster.so maintains an hash table storing mount-points and libglusterfsclient handles - so that handles are reused for files from same mount point. - -* it also maintains a fdtable. fdtable maps the fd (integer) returned to application to fd (pointer to fd struct) - used by libglusterfsclient. application is returned the same fd as the one returned from libc apis. - -* During fork, these tables are overwritten to enable creation of fresh glusterfs context in child. - -Working -======= -* application willing to use booster LD_PRELOADs booster.so which is a wrapper library implementing - open, read and write. - -* application should specify the path to logfile through the environment variable GLFS_BOOSTER_LOGFILE. If - not specified, logging is done to /dev/stderr. - -* open call does, - * real_open on the file. - * fgetxattr(fd). - * store the volume-file content got in the dictionary to a temparory file. - * look in the hashtable for the mount-point, if already present get the libglusterfsclient handle from the - hashtable. Otherwise get a new handle from libglusterfsclient (be careful about mount point not present in - the hashtable and multiple glusterfs_inits running simultaneously for the same mount-point there by using - multiple handles for the same mount point). - * real_close (fd). - * delete temporary volume-volfile. - * glusterfs_open (handle, path, mode). - * store the fd returned by glusterfs_open in the fdtable at the same index as the fd returned by real_open. - * return the index as fd. - -* read/write calls do, - * get the libglusterfsclient fd from fdtable. - * if found use glusterfs_read/glusterfs_write, else use real_read/real_write. - -* close call does, - * remove the fd from the fdtable. - -* other calls use real_calls. diff --git a/doc/legacy/coding-standard.pdf b/doc/legacy/coding-standard.pdf Binary files differdeleted file mode 100644 index bc9cb56202c..00000000000 --- a/doc/legacy/coding-standard.pdf +++ /dev/null diff --git a/doc/legacy/coding-standard.tex b/doc/legacy/coding-standard.tex deleted file mode 100644 index abaedb69c3c..00000000000 --- a/doc/legacy/coding-standard.tex +++ /dev/null @@ -1,385 +0,0 @@ -\documentclass{article}[12pt] -\usepackage{color} - -\begin{document} - - -\hrule -\begin{center}\textbf{\Large{GlusterFS Coding Standards}}\end{center} -\begin{center}\textbf{\large{\textcolor{red}{Z} Research}}\end{center} -\begin{center}{July 14, 2008}\end{center} -\hrule - -\vspace{8ex} - -\section*{$\bullet$ Structure definitions should have a comment per member} - -Every member in a structure definition must have a comment about its -purpose. The comment should be descriptive without being overly verbose. - -\vspace{2ex} -\textsl{Bad}: - -\begin{verbatim} - gf_lock_t lock; /* lock */ -\end{verbatim} - -\textsl{Good}: - -\begin{verbatim} - DBTYPE access_mode; /* access mode for accessing - * the databases, can be - * DB_HASH, DB_BTREE - * (option access-mode <mode>) - */ -\end{verbatim} - -\section*{$\bullet$ Declare all variables at the beginning of the function} -All local variables in a function must be declared immediately after the -opening brace. This makes it easy to keep track of memory that needs to be freed -during exit. It also helps debugging, since gdb cannot handle variables -declared inside loops or other such blocks. - -\section*{$\bullet$ Always initialize local variables} -Every local variable should be initialized to a sensible default value -at the point of its declaration. All pointers should be initialized to NULL, -and all integers should be zero or (if it makes sense) an error value. - -\vspace{2ex} - -\textsl{Good}: - -\begin{verbatim} - int ret = 0; - char *databuf = NULL; - int _fd = -1; -\end{verbatim} - -\section*{$\bullet$ Initialization should always be done with a constant value} -Never use a non-constant expression as the initialization value for a variable. - -\vspace{2ex} - -\textsl{Bad}: - -\begin{verbatim} - pid_t pid = frame->root->pid; - char *databuf = malloc (1024); -\end{verbatim} - -\section*{$\bullet$ Validate all arguments to a function} -All pointer arguments to a function must be checked for \texttt{NULL}. -A macro named \texttt{VALIDATE} (in \texttt{common-utils.h}) -takes one argument, and if it is \texttt{NULL}, writes a log message and -jumps to a label called \texttt{err} after setting op\_ret and op\_errno -appropriately. It is recommended to use this template. - -\vspace{2ex} - -\textsl{Good}: - -\begin{verbatim} - VALIDATE(frame); - VALIDATE(this); - VALIDATE(inode); -\end{verbatim} - -\section*{$\bullet$ Never rely on precedence of operators} -Never write code that relies on the precedence of operators to execute -correctly. Such code can be hard to read and someone else might not -know the precedence of operators as accurately as you do. -\vspace{2ex} - -\textsl{Bad}: - -\begin{verbatim} - if (op_ret == -1 && errno != ENOENT) -\end{verbatim} - -\textsl{Good}: - -\begin{verbatim} - if ((op_ret == -1) && (errno != ENOENT)) -\end{verbatim} - -\section*{$\bullet$ Use exactly matching types} -Use a variable of the exact type declared in the manual to hold the -return value of a function. Do not use an ``equivalent'' type. - -\vspace{2ex} - -\textsl{Bad}: - -\begin{verbatim} - int len = strlen (path); -\end{verbatim} - -\textsl{Good}: - -\begin{verbatim} - size_t len = strlen (path); -\end{verbatim} - -\section*{$\bullet$ Never write code such as \texttt{foo->bar->baz}; check every pointer} -Do not write code that blindly follows a chain of pointer -references. Any pointer in the chain may be \texttt{NULL} and thus -cause a crash. Verify that each pointer is non-null before following -it. - -\section*{$\bullet$ Check return value of all functions and system calls} -The return value of all system calls and API functions must be checked -for success or failure. - -\vspace{2ex} -\textsl{Bad}: - -\begin{verbatim} - close (fd); -\end{verbatim} - -\textsl{Good}: - -\begin{verbatim} - op_ret = close (_fd); - if (op_ret == -1) { - gf_log (this->name, GF_LOG_ERROR, - "close on file %s failed (%s)", real_path, - strerror (errno)); - op_errno = errno; - goto out; - } -\end{verbatim} - - -\section*{$\bullet$ Gracefully handle failure of malloc} -GlusterFS should never crash or exit due to lack of memory. If a -memory allocation fails, the call should be unwound and an error -returned to the user. - -\section*{$\bullet$ Use result args and reserve the return value to indicate success or failure} -The return value of every functions must indicate success or failure (unless -it is impossible for the function to fail --- e.g., boolean functions). If -the function needs to return additional data, it must be returned using a -result (pointer) argument. - -\vspace{2ex} -\textsl{Bad}: - -\begin{verbatim} - int32_t dict_get_int32 (dict_t *this, char *key); -\end{verbatim} - -\textsl{Good}: - -\begin{verbatim} - int dict_get_int32 (dict_t *this, char *key, int32_t *val); -\end{verbatim} - -\section*{$\bullet$ Always use the `n' versions of string functions} -Unless impossible, use the length-limited versions of the string functions. - -\vspace{2ex} -\textsl{Bad}: - -\begin{verbatim} - strcpy (entry_path, real_path); -\end{verbatim} - -\textsl{Good}: - -\begin{verbatim} - strncpy (entry_path, real_path, entry_path_len); -\end{verbatim} - -\section*{$\bullet$ No dead or commented code} -There must be no dead code (code to which control can never be passed) or -commented out code in the codebase. - -\section*{$\bullet$ Only one unwind and return per function} -There must be only one exit out of a function. \texttt{UNWIND} and return -should happen at only point in the function. - -\section*{$\bullet$ Function length or Keep functions small} -We live in the UNIX-world where modules do one thing and do it well. -This rule should apply to our functions also. If a function is very long, try splitting it -into many little helper functions. The question is, in a coding -spree, how do we know a function is long and unreadable. One rule of -thumb given by Linus Torvalds is that, a function should be broken-up -if you have 4 or more levels of indentation going on for more than 3-4 -lines. - -\vspace{2ex} -\textsl{Example for a helper function}: -\begin{verbatim} - static int - same_owner (posix_lock_t *l1, posix_lock_t *l2) - { - return ((l1->client_pid == l2->client_pid) && - (l1->transport == l2->transport)); - } -\end{verbatim} - -\section*{$\bullet$ Defining functions as static} -Define internal functions as static only if you're -very sure that there will not be a crash(..of any kind..) emanating in -that function. If there is even a remote possibility, perhaps due to -pointer derefering, etc, declare the function as non-static. This -ensures that when a crash does happen, the function name shows up the -in the back-trace generated by libc. However, doing so has potential -for polluting the function namespace, so to avoid conflicts with other -components in other parts, ensure that the function names are -prepended with a prefix that identify the component to which it -belongs. For eg. non-static functions in io-threads translator start -with iot\_. - -\section*{$\bullet$ Ensure function calls wrap around after -80-columns.} -Place remaining arguments on the next line if needed. - -\section*{$\bullet$ Functions arguments and function definition} -Place all the arguments of a function definition on the same line -until the line goes beyond 80-cols. Arguments that extend beyind -80-cols should be placed on the next line. - -\section*{Style issues} - -\subsection*{Brace placement} -Use K\&R/Linux style of brace placement for blocks. - -\textsl{Example}: -\begin{verbatim} - int some_function (...) - { - if (...) { - /* ... */ - } else if (...) { - /* ... */ - } else { - /* ... */ - } - - do { - /* ... */ - } while (cond); - } -\end{verbatim} - -\subsection*{Indentation} -Use \textbf{eight} spaces for indenting blocks. Ensure that your -file contains only spaces and not tab characters. You can do this -in Emacs by selecting the entire file (\texttt{C-x h}) and -running \texttt{M-x untabify}. - -To make Emacs indent lines automatically by eight spaces, add this -line to your \texttt{.emacs}: - -\begin{verbatim} - (add-hook 'c-mode-hook (lambda () (c-set-style "linux"))) -\end{verbatim} - -\subsection*{Comments} -Write a comment before every function describing its purpose (one-line), -its arguments, and its return value. Mention whether it is an internal -function or an exported function. - -Write a comment before every structure describing its purpose, and -write comments about each of its members. - -Follow the style shown below for comments, since such comments -can then be automatically extracted by doxygen to generate -documentation. - -\textsl{Example}: -\begin{verbatim} -/** - * hash_name -hash function for filenames - * @par: parent inode number - * @name: basename of inode - * @mod: number of buckets in the hashtable - * - * @return: success: bucket number - * failure: -1 - * - * Not for external use. - */ -\end{verbatim} - -\subsection*{Indicating critical sections} -To clearly show regions of code which execute with locks held, use -the following format: - -\begin{verbatim} - pthread_mutex_lock (&mutex); - { - /* code */ - } - pthread_mutex_unlock (&mutex); -\end{verbatim} - -\section*{A skeleton fop function} -This is the recommended template for any fop. In the beginning come -the initializations. After that, the `success' control flow should be -linear. Any error conditions should cause a \texttt{goto} to a single -point, \texttt{out}. At that point, the code should detect the error -that has occured and do appropriate cleanup. - -\begin{verbatim} -int32_t -sample_fop (call_frame_t *frame, xlator_t *this, ...) -{ - char * var1 = NULL; - int32_t op_ret = -1; - int32_t op_errno = 0; - DIR * dir = NULL; - struct posix_fd * pfd = NULL; - - VALIDATE_OR_GOTO (frame, out); - VALIDATE_OR_GOTO (this, out); - - /* other validations */ - - dir = opendir (...); - - if (dir == NULL) { - op_errno = errno; - gf_log (this->name, GF_LOG_ERROR, - "opendir failed on %s (%s)", loc->path, - strerror (op_errno)); - goto out; - } - - /* another system call */ - if (...) { - op_errno = ENOMEM; - gf_log (this->name, GF_LOG_ERROR, - "out of memory :("); - goto out; - } - - /* ... */ - - out: - if (op_ret == -1) { - - /* check for all the cleanup that needs to be - done */ - - if (dir) { - closedir (dir); - dir = NULL; - } - - if (pfd) { - if (pfd->path) - FREE (pfd->path); - FREE (pfd); - pfd = NULL; - } - } - - STACK_UNWIND (frame, op_ret, op_errno, fd); - return 0; -} -\end{verbatim} - -\end{document} diff --git a/doc/legacy/errno.list.bsd.txt b/doc/legacy/errno.list.bsd.txt deleted file mode 100644 index 350af25e4ab..00000000000 --- a/doc/legacy/errno.list.bsd.txt +++ /dev/null @@ -1,376 +0,0 @@ -/*- - * Copyright (c) 1982, 1986, 1989, 1993 - * The Regents of the University of California. All rights reserved. - * (c) UNIX System Laboratories, Inc. - * All or some portions of this file are derived from material licensed - * to the University of California by American Telephone and Telegraph - * Co. or Unix System Laboratories, Inc. and are reproduced herein with - * the permission of UNIX System Laboratories, Inc. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * 4. Neither the name of the University nor the names of its contributors - * may be used to endorse or promote products derived from this software - * without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - * - * @(#)errno.h 8.5 (Berkeley) 1/21/94 - * $FreeBSD: src/sys/sys/errno.h,v 1.28 2005/04/02 12:33:28 das Exp $ - */ - -#ifndef _SYS_ERRNO_H_ -#define _SYS_ERRNO_H_ - -#ifndef _KERNEL -#include <sys/cdefs.h> -__BEGIN_DECLS -int * __error(void); -__END_DECLS -#define errno (* __error()) -#endif - -#define EPERM 1 /* Operation not permitted */ -#define ENOENT 2 /* No such file or directory */ -#define ESRCH 3 /* No such process */ -#define EINTR 4 /* Interrupted system call */ -#define EIO 5 /* Input/output error */ -#define ENXIO 6 /* Device not configured */ -#define E2BIG 7 /* Argument list too long */ -#define ENOEXEC 8 /* Exec format error */ -#define EBADF 9 /* Bad file descriptor */ -#define ECHILD 10 /* No child processes */ -#define EDEADLK 11 /* Resource deadlock avoided */ - /* 11 was EAGAIN */ -#define ENOMEM 12 /* Cannot allocate memory */ -#define EACCES 13 /* Permission denied */ -#define EFAULT 14 /* Bad address */ -#ifndef _POSIX_SOURCE -#define ENOTBLK 15 /* Block device required */ -#endif -#define EBUSY 16 /* Device busy */ -#define EEXIST 17 /* File exists */ -#define EXDEV 18 /* Cross-device link */ -#define ENODEV 19 /* Operation not supported by device */ -#define ENOTDIR 20 /* Not a directory */ -#define EISDIR 21 /* Is a directory */ -#define EINVAL 22 /* Invalid argument */ -#define ENFILE 23 /* Too many open files in system */ -#define EMFILE 24 /* Too many open files */ -#define ENOTTY 25 /* Inappropriate ioctl for device */ -#ifndef _POSIX_SOURCE -#define ETXTBSY 26 /* Text file busy */ -#endif -#define EFBIG 27 /* File too large */ -#define ENOSPC 28 /* No space left on device */ -#define ESPIPE 29 /* Illegal seek */ -#define EROFS 30 /* Read-only filesystem */ -#define EMLINK 31 /* Too many links */ -#define EPIPE 32 /* Broken pipe */ - -/* math software */ -#define EDOM 33 /* Numerical argument out of domain */ -#define ERANGE 34 /* Result too large */ - -/* non-blocking and interrupt i/o */ -#define EAGAIN 35 /* Resource temporarily unavailable */ -#ifndef _POSIX_SOURCE -#define EWOULDBLOCK EAGAIN /* Operation would block */ -#define EINPROGRESS 36 /* Operation now in progress */ -#define EALREADY 37 /* Operation already in progress */ - -/* ipc/network software -- argument errors */ -#define ENOTSOCK 38 /* Socket operation on non-socket */ -#define EDESTADDRREQ 39 /* Destination address required */ -#define EMSGSIZE 40 /* Message too long */ -#define EPROTOTYPE 41 /* Protocol wrong type for socket */ -#define ENOPROTOOPT 42 /* Protocol not available */ -#define EPROTONOSUPPORT 43 /* Protocol not supported */ -#define ESOCKTNOSUPPORT 44 /* Socket type not supported */ -#define EOPNOTSUPP 45 /* Operation not supported */ -#define ENOTSUP EOPNOTSUPP /* Operation not supported */ -#define EPFNOSUPPORT 46 /* Protocol family not supported */ -#define EAFNOSUPPORT 47 /* Address family not supported by protocol family */ -#define EADDRINUSE 48 /* Address already in use */ -#define EADDRNOTAVAIL 49 /* Can't assign requested address */ - -/* ipc/network software -- operational errors */ -#define ENETDOWN 50 /* Network is down */ -#define ENETUNREACH 51 /* Network is unreachable */ -#define ENETRESET 52 /* Network dropped connection on reset */ -#define ECONNABORTED 53 /* Software caused connection abort */ -#define ECONNRESET 54 /* Connection reset by peer */ -#define ENOBUFS 55 /* No buffer space available */ -#define EISCONN 56 /* Socket is already connected */ -#define ENOTCONN 57 /* Socket is not connected */ -#define ESHUTDOWN 58 /* Can't send after socket shutdown */ -#define ETOOMANYREFS 59 /* Too many references: can't splice */ -#define ETIMEDOUT 60 /* Operation timed out */ -#define ECONNREFUSED 61 /* Connection refused */ - -#define ELOOP 62 /* Too many levels of symbolic links */ -#endif /* _POSIX_SOURCE */ -#define ENAMETOOLONG 63 /* File name too long */ - -/* should be rearranged */ -#ifndef _POSIX_SOURCE -#define EHOSTDOWN 64 /* Host is down */ -#define EHOSTUNREACH 65 /* No route to host */ -#endif /* _POSIX_SOURCE */ -#define ENOTEMPTY 66 /* Directory not empty */ - -/* quotas & mush */ -#ifndef _POSIX_SOURCE -#define EPROCLIM 67 /* Too many processes */ -#define EUSERS 68 /* Too many users */ -#define EDQUOT 69 /* Disc quota exceeded */ - -/* Network File System */ -#define ESTALE 70 /* Stale NFS file handle */ -#define EREMOTE 71 /* Too many levels of remote in path */ -#define EBADRPC 72 /* RPC struct is bad */ -#define ERPCMISMATCH 73 /* RPC version wrong */ -#define EPROGUNAVAIL 74 /* RPC prog. not avail */ -#define EPROGMISMATCH 75 /* Program version wrong */ -#define EPROCUNAVAIL 76 /* Bad procedure for program */ -#endif /* _POSIX_SOURCE */ - -#define ENOLCK 77 /* No locks available */ -#define ENOSYS 78 /* Function not implemented */ - -#ifndef _POSIX_SOURCE -#define EFTYPE 79 /* Inappropriate file type or format */ -#define EAUTH 80 /* Authentication error */ -#define ENEEDAUTH 81 /* Need authenticator */ -#define EIDRM 82 /* Identifier removed */ -#define ENOMSG 83 /* No message of desired type */ -#define EOVERFLOW 84 /* Value too large to be stored in data type */ -#define ECANCELED 85 /* Operation canceled */ -#define EILSEQ 86 /* Illegal byte sequence */ -#define ENOATTR 87 /* Attribute not found */ - -#define EDOOFUS 88 /* Programming error */ -#endif /* _POSIX_SOURCE */ - -#define EBADMSG 89 /* Bad message */ -#define EMULTIHOP 90 /* Multihop attempted */ -#define ENOLINK 91 /* Link has been severed */ -#define EPROTO 92 /* Protocol error */ - -#ifndef _POSIX_SOURCE -#define ELAST 92 /* Must be equal largest errno */ -#endif /* _POSIX_SOURCE */ - -#ifdef _KERNEL -/* pseudo-errors returned inside kernel to modify return to process */ -#define ERESTART (-1) /* restart syscall */ -#define EJUSTRETURN (-2) /* don't modify regs, just return */ -#define ENOIOCTL (-3) /* ioctl not handled by this layer */ -#define EDIRIOCTL (-4) /* do direct ioctl in GEOM */ -#endif - -#endif -/*- - * Copyright (c) 1982, 1986, 1989, 1993 - * The Regents of the University of California. All rights reserved. - * (c) UNIX System Laboratories, Inc. - * All or some portions of this file are derived from material licensed - * to the University of California by American Telephone and Telegraph - * Co. or Unix System Laboratories, Inc. and are reproduced herein with - * the permission of UNIX System Laboratories, Inc. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * 4. Neither the name of the University nor the names of its contributors - * may be used to endorse or promote products derived from this software - * without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - * - * @(#)errno.h 8.5 (Berkeley) 1/21/94 - * $FreeBSD: src/sys/sys/errno.h,v 1.28 2005/04/02 12:33:28 das Exp $ - */ - -#ifndef _SYS_ERRNO_H_ -#define _SYS_ERRNO_H_ - -#ifndef _KERNEL -#include <sys/cdefs.h> -__BEGIN_DECLS -int * __error(void); -__END_DECLS -#define errno (* __error()) -#endif - -#define EPERM 1 /* Operation not permitted */ -#define ENOENT 2 /* No such file or directory */ -#define ESRCH 3 /* No such process */ -#define EINTR 4 /* Interrupted system call */ -#define EIO 5 /* Input/output error */ -#define ENXIO 6 /* Device not configured */ -#define E2BIG 7 /* Argument list too long */ -#define ENOEXEC 8 /* Exec format error */ -#define EBADF 9 /* Bad file descriptor */ -#define ECHILD 10 /* No child processes */ -#define EDEADLK 11 /* Resource deadlock avoided */ - /* 11 was EAGAIN */ -#define ENOMEM 12 /* Cannot allocate memory */ -#define EACCES 13 /* Permission denied */ -#define EFAULT 14 /* Bad address */ -#ifndef _POSIX_SOURCE -#define ENOTBLK 15 /* Block device required */ -#endif -#define EBUSY 16 /* Device busy */ -#define EEXIST 17 /* File exists */ -#define EXDEV 18 /* Cross-device link */ -#define ENODEV 19 /* Operation not supported by device */ -#define ENOTDIR 20 /* Not a directory */ -#define EISDIR 21 /* Is a directory */ -#define EINVAL 22 /* Invalid argument */ -#define ENFILE 23 /* Too many open files in system */ -#define EMFILE 24 /* Too many open files */ -#define ENOTTY 25 /* Inappropriate ioctl for device */ -#ifndef _POSIX_SOURCE -#define ETXTBSY 26 /* Text file busy */ -#endif -#define EFBIG 27 /* File too large */ -#define ENOSPC 28 /* No space left on device */ -#define ESPIPE 29 /* Illegal seek */ -#define EROFS 30 /* Read-only filesystem */ -#define EMLINK 31 /* Too many links */ -#define EPIPE 32 /* Broken pipe */ - -/* math software */ -#define EDOM 33 /* Numerical argument out of domain */ -#define ERANGE 34 /* Result too large */ - -/* non-blocking and interrupt i/o */ -#define EAGAIN 35 /* Resource temporarily unavailable */ -#ifndef _POSIX_SOURCE -#define EWOULDBLOCK EAGAIN /* Operation would block */ -#define EINPROGRESS 36 /* Operation now in progress */ -#define EALREADY 37 /* Operation already in progress */ - -/* ipc/network software -- argument errors */ -#define ENOTSOCK 38 /* Socket operation on non-socket */ -#define EDESTADDRREQ 39 /* Destination address required */ -#define EMSGSIZE 40 /* Message too long */ -#define EPROTOTYPE 41 /* Protocol wrong type for socket */ -#define ENOPROTOOPT 42 /* Protocol not available */ -#define EPROTONOSUPPORT 43 /* Protocol not supported */ -#define ESOCKTNOSUPPORT 44 /* Socket type not supported */ -#define EOPNOTSUPP 45 /* Operation not supported */ -#define ENOTSUP EOPNOTSUPP /* Operation not supported */ -#define EPFNOSUPPORT 46 /* Protocol family not supported */ -#define EAFNOSUPPORT 47 /* Address family not supported by protocol family */ -#define EADDRINUSE 48 /* Address already in use */ -#define EADDRNOTAVAIL 49 /* Can't assign requested address */ - -/* ipc/network software -- operational errors */ -#define ENETDOWN 50 /* Network is down */ -#define ENETUNREACH 51 /* Network is unreachable */ -#define ENETRESET 52 /* Network dropped connection on reset */ -#define ECONNABORTED 53 /* Software caused connection abort */ -#define ECONNRESET 54 /* Connection reset by peer */ -#define ENOBUFS 55 /* No buffer space available */ -#define EISCONN 56 /* Socket is already connected */ -#define ENOTCONN 57 /* Socket is not connected */ -#define ESHUTDOWN 58 /* Can't send after socket shutdown */ -#define ETOOMANYREFS 59 /* Too many references: can't splice */ -#define ETIMEDOUT 60 /* Operation timed out */ -#define ECONNREFUSED 61 /* Connection refused */ - -#define ELOOP 62 /* Too many levels of symbolic links */ -#endif /* _POSIX_SOURCE */ -#define ENAMETOOLONG 63 /* File name too long */ - -/* should be rearranged */ -#ifndef _POSIX_SOURCE -#define EHOSTDOWN 64 /* Host is down */ -#define EHOSTUNREACH 65 /* No route to host */ -#endif /* _POSIX_SOURCE */ -#define ENOTEMPTY 66 /* Directory not empty */ - -/* quotas & mush */ -#ifndef _POSIX_SOURCE -#define EPROCLIM 67 /* Too many processes */ -#define EUSERS 68 /* Too many users */ -#define EDQUOT 69 /* Disc quota exceeded */ - -/* Network File System */ -#define ESTALE 70 /* Stale NFS file handle */ -#define EREMOTE 71 /* Too many levels of remote in path */ -#define EBADRPC 72 /* RPC struct is bad */ -#define ERPCMISMATCH 73 /* RPC version wrong */ -#define EPROGUNAVAIL 74 /* RPC prog. not avail */ -#define EPROGMISMATCH 75 /* Program version wrong */ -#define EPROCUNAVAIL 76 /* Bad procedure for program */ -#endif /* _POSIX_SOURCE */ - -#define ENOLCK 77 /* No locks available */ -#define ENOSYS 78 /* Function not implemented */ - -#ifndef _POSIX_SOURCE -#define EFTYPE 79 /* Inappropriate file type or format */ -#define EAUTH 80 /* Authentication error */ -#define ENEEDAUTH 81 /* Need authenticator */ -#define EIDRM 82 /* Identifier removed */ -#define ENOMSG 83 /* No message of desired type */ -#define EOVERFLOW 84 /* Value too large to be stored in data type */ -#define ECANCELED 85 /* Operation canceled */ -#define EILSEQ 86 /* Illegal byte sequence */ -#define ENOATTR 87 /* Attribute not found */ - -#define EDOOFUS 88 /* Programming error */ -#endif /* _POSIX_SOURCE */ - -#define EBADMSG 89 /* Bad message */ -#define EMULTIHOP 90 /* Multihop attempted */ -#define ENOLINK 91 /* Link has been severed */ -#define EPROTO 92 /* Protocol error */ - -#ifndef _POSIX_SOURCE -#define ELAST 92 /* Must be equal largest errno */ -#endif /* _POSIX_SOURCE */ - -#ifdef _KERNEL -/* pseudo-errors returned inside kernel to modify return to process */ -#define ERESTART (-1) /* restart syscall */ -#define EJUSTRETURN (-2) /* don't modify regs, just return */ -#define ENOIOCTL (-3) /* ioctl not handled by this layer */ -#define EDIRIOCTL (-4) /* do direct ioctl in GEOM */ -#endif - -#endif diff --git a/doc/legacy/errno.list.linux.txt b/doc/legacy/errno.list.linux.txt deleted file mode 100644 index cc868644b36..00000000000 --- a/doc/legacy/errno.list.linux.txt +++ /dev/null @@ -1,1586 +0,0 @@ -#define ICONV_SUPPORTS_ERRNO 1 -#include <errno.h> -/* Error constants. Linux specific version. - Copyright (C) 1996, 1997, 1998, 1999, 2005 Free Software Foundation, Inc. - This file is part of the GNU C Library. - - The GNU C Library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. - - The GNU C Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public - License along with the GNU C Library; if not, write to the Free - Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA - 02111-1307 USA. */ - -#ifdef _ERRNO_H - -# undef EDOM -# undef EILSEQ -# undef ERANGE -# include <linux/errno.h> - -/* Linux has no ENOTSUP error code. */ -# define ENOTSUP EOPNOTSUPP - -/* Older Linux versions also had no ECANCELED error code. */ -# ifndef ECANCELED -# define ECANCELED 125 -# endif - -/* Support for error codes to support robust mutexes was added later, too. */ -# ifndef EOWNERDEAD -# define EOWNERDEAD 130 -# define ENOTRECOVERABLE 131 -# endif - -# ifndef __ASSEMBLER__ -/* Function to get address of global `errno' variable. */ -extern int *__errno_location (void) __THROW __attribute__ ((__const__)); - -# if !defined _LIBC || defined _LIBC_REENTRANT -/* When using threads, errno is a per-thread value. */ -# define errno (*__errno_location ()) -# endif -# endif /* !__ASSEMBLER__ */ -#endif /* _ERRNO_H */ - -#if !defined _ERRNO_H && defined __need_Emath -/* This is ugly but the kernel header is not clean enough. We must - define only the values EDOM, EILSEQ and ERANGE in case __need_Emath is - defined. */ -# define EDOM 33 /* Math argument out of domain of function. */ -# define EILSEQ 84 /* Illegal byte sequence. */ -# define ERANGE 34 /* Math result not representable. */ -#endif /* !_ERRNO_H && __need_Emath */ -/* Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef APR_ERRNO_H -#define APR_ERRNO_H - -/** - * @file apr_errno.h - * @brief APR Error Codes - */ - -#include "apr.h" - -#if APR_HAVE_ERRNO_H -#include <errno.h> -#endif - -#ifdef __cplusplus -extern "C" { -#endif /* __cplusplus */ - -/** - * @defgroup apr_errno Error Codes - * @ingroup APR - * @{ - */ - -/** - * Type for specifying an error or status code. - */ -typedef int apr_status_t; - -/** - * Return a human readable string describing the specified error. - * @param statcode The error code the get a string for. - * @param buf A buffer to hold the error string. - * @param bufsize Size of the buffer to hold the string. - */ -APR_DECLARE(char *) apr_strerror(apr_status_t statcode, char *buf, - apr_size_t bufsize); - -#if defined(DOXYGEN) -/** - * @def APR_FROM_OS_ERROR(os_err_type syserr) - * Fold a platform specific error into an apr_status_t code. - * @return apr_status_t - * @param e The platform os error code. - * @warning macro implementation; the syserr argument may be evaluated - * multiple times. - */ -#define APR_FROM_OS_ERROR(e) (e == 0 ? APR_SUCCESS : e + APR_OS_START_SYSERR) - -/** - * @def APR_TO_OS_ERROR(apr_status_t statcode) - * @return os_err_type - * Fold an apr_status_t code back to the native platform defined error. - * @param e The apr_status_t folded platform os error code. - * @warning macro implementation; the statcode argument may be evaluated - * multiple times. If the statcode was not created by apr_get_os_error - * or APR_FROM_OS_ERROR, the results are undefined. - */ -#define APR_TO_OS_ERROR(e) (e == 0 ? APR_SUCCESS : e - APR_OS_START_SYSERR) - -/** @def apr_get_os_error() - * @return apr_status_t the last platform error, folded into apr_status_t, on most platforms - * @remark This retrieves errno, or calls a GetLastError() style function, and - * folds it with APR_FROM_OS_ERROR. Some platforms (such as OS2) have no - * such mechanism, so this call may be unsupported. Do NOT use this - * call for socket errors from socket, send, recv etc! - */ - -/** @def apr_set_os_error(e) - * Reset the last platform error, unfolded from an apr_status_t, on some platforms - * @param e The OS error folded in a prior call to APR_FROM_OS_ERROR() - * @warning This is a macro implementation; the statcode argument may be evaluated - * multiple times. If the statcode was not created by apr_get_os_error - * or APR_FROM_OS_ERROR, the results are undefined. This macro sets - * errno, or calls a SetLastError() style function, unfolding statcode - * with APR_TO_OS_ERROR. Some platforms (such as OS2) have no such - * mechanism, so this call may be unsupported. - */ - -/** @def apr_get_netos_error() - * Return the last socket error, folded into apr_status_t, on all platforms - * @remark This retrieves errno or calls a GetLastSocketError() style function, - * and folds it with APR_FROM_OS_ERROR. - */ - -/** @def apr_set_netos_error(e) - * Reset the last socket error, unfolded from an apr_status_t - * @param e The socket error folded in a prior call to APR_FROM_OS_ERROR() - * @warning This is a macro implementation; the statcode argument may be evaluated - * multiple times. If the statcode was not created by apr_get_os_error - * or APR_FROM_OS_ERROR, the results are undefined. This macro sets - * errno, or calls a WSASetLastError() style function, unfolding - * socketcode with APR_TO_OS_ERROR. - */ - -#endif /* defined(DOXYGEN) */ - -/** - * APR_OS_START_ERROR is where the APR specific error values start. - */ -#define APR_OS_START_ERROR 20000 -/** - * APR_OS_ERRSPACE_SIZE is the maximum number of errors you can fit - * into one of the error/status ranges below -- except for - * APR_OS_START_USERERR, which see. - */ -#define APR_OS_ERRSPACE_SIZE 50000 -/** - * APR_OS_START_STATUS is where the APR specific status codes start. - */ -#define APR_OS_START_STATUS (APR_OS_START_ERROR + APR_OS_ERRSPACE_SIZE) -/** - * APR_OS_START_USERERR are reserved for applications that use APR that - * layer their own error codes along with APR's. Note that the - * error immediately following this one is set ten times farther - * away than usual, so that users of apr have a lot of room in - * which to declare custom error codes. - */ -#define APR_OS_START_USERERR (APR_OS_START_STATUS + APR_OS_ERRSPACE_SIZE) -/** - * APR_OS_START_USEERR is obsolete, defined for compatibility only. - * Use APR_OS_START_USERERR instead. - */ -#define APR_OS_START_USEERR APR_OS_START_USERERR -/** - * APR_OS_START_CANONERR is where APR versions of errno values are defined - * on systems which don't have the corresponding errno. - */ -#define APR_OS_START_CANONERR (APR_OS_START_USERERR \ - + (APR_OS_ERRSPACE_SIZE * 10)) -/** - * APR_OS_START_EAIERR folds EAI_ error codes from getaddrinfo() into - * apr_status_t values. - */ -#define APR_OS_START_EAIERR (APR_OS_START_CANONERR + APR_OS_ERRSPACE_SIZE) -/** - * APR_OS_START_SYSERR folds platform-specific system error values into - * apr_status_t values. - */ -#define APR_OS_START_SYSERR (APR_OS_START_EAIERR + APR_OS_ERRSPACE_SIZE) - -/** no error. */ -#define APR_SUCCESS 0 - -/** - * @defgroup APR_Error APR Error Values - * <PRE> - * <b>APR ERROR VALUES</b> - * APR_ENOSTAT APR was unable to perform a stat on the file - * APR_ENOPOOL APR was not provided a pool with which to allocate memory - * APR_EBADDATE APR was given an invalid date - * APR_EINVALSOCK APR was given an invalid socket - * APR_ENOPROC APR was not given a process structure - * APR_ENOTIME APR was not given a time structure - * APR_ENODIR APR was not given a directory structure - * APR_ENOLOCK APR was not given a lock structure - * APR_ENOPOLL APR was not given a poll structure - * APR_ENOSOCKET APR was not given a socket - * APR_ENOTHREAD APR was not given a thread structure - * APR_ENOTHDKEY APR was not given a thread key structure - * APR_ENOSHMAVAIL There is no more shared memory available - * APR_EDSOOPEN APR was unable to open the dso object. For more - * information call apr_dso_error(). - * APR_EGENERAL General failure (specific information not available) - * APR_EBADIP The specified IP address is invalid - * APR_EBADMASK The specified netmask is invalid - * APR_ESYMNOTFOUND Could not find the requested symbol - * </PRE> - * - * <PRE> - * <b>APR STATUS VALUES</b> - * APR_INCHILD Program is currently executing in the child - * APR_INPARENT Program is currently executing in the parent - * APR_DETACH The thread is detached - * APR_NOTDETACH The thread is not detached - * APR_CHILD_DONE The child has finished executing - * APR_CHILD_NOTDONE The child has not finished executing - * APR_TIMEUP The operation did not finish before the timeout - * APR_INCOMPLETE The operation was incomplete although some processing - * was performed and the results are partially valid - * APR_BADCH Getopt found an option not in the option string - * APR_BADARG Getopt found an option that is missing an argument - * and an argument was specified in the option string - * APR_EOF APR has encountered the end of the file - * APR_NOTFOUND APR was unable to find the socket in the poll structure - * APR_ANONYMOUS APR is using anonymous shared memory - * APR_FILEBASED APR is using a file name as the key to the shared memory - * APR_KEYBASED APR is using a shared key as the key to the shared memory - * APR_EINIT Ininitalizer value. If no option has been found, but - * the status variable requires a value, this should be used - * APR_ENOTIMPL The APR function has not been implemented on this - * platform, either because nobody has gotten to it yet, - * or the function is impossible on this platform. - * APR_EMISMATCH Two passwords do not match. - * APR_EABSOLUTE The given path was absolute. - * APR_ERELATIVE The given path was relative. - * APR_EINCOMPLETE The given path was neither relative nor absolute. - * APR_EABOVEROOT The given path was above the root path. - * APR_EBUSY The given lock was busy. - * APR_EPROC_UNKNOWN The given process wasn't recognized by APR - * </PRE> - * @{ - */ -/** @see APR_STATUS_IS_ENOSTAT */ -#define APR_ENOSTAT (APR_OS_START_ERROR + 1) -/** @see APR_STATUS_IS_ENOPOOL */ -#define APR_ENOPOOL (APR_OS_START_ERROR + 2) -/* empty slot: +3 */ -/** @see APR_STATUS_IS_EBADDATE */ -#define APR_EBADDATE (APR_OS_START_ERROR + 4) -/** @see APR_STATUS_IS_EINVALSOCK */ -#define APR_EINVALSOCK (APR_OS_START_ERROR + 5) -/** @see APR_STATUS_IS_ENOPROC */ -#define APR_ENOPROC (APR_OS_START_ERROR + 6) -/** @see APR_STATUS_IS_ENOTIME */ -#define APR_ENOTIME (APR_OS_START_ERROR + 7) -/** @see APR_STATUS_IS_ENODIR */ -#define APR_ENODIR (APR_OS_START_ERROR + 8) -/** @see APR_STATUS_IS_ENOLOCK */ -#define APR_ENOLOCK (APR_OS_START_ERROR + 9) -/** @see APR_STATUS_IS_ENOPOLL */ -#define APR_ENOPOLL (APR_OS_START_ERROR + 10) -/** @see APR_STATUS_IS_ENOSOCKET */ -#define APR_ENOSOCKET (APR_OS_START_ERROR + 11) -/** @see APR_STATUS_IS_ENOTHREAD */ -#define APR_ENOTHREAD (APR_OS_START_ERROR + 12) -/** @see APR_STATUS_IS_ENOTHDKEY */ -#define APR_ENOTHDKEY (APR_OS_START_ERROR + 13) -/** @see APR_STATUS_IS_EGENERAL */ -#define APR_EGENERAL (APR_OS_START_ERROR + 14) -/** @see APR_STATUS_IS_ENOSHMAVAIL */ -#define APR_ENOSHMAVAIL (APR_OS_START_ERROR + 15) -/** @see APR_STATUS_IS_EBADIP */ -#define APR_EBADIP (APR_OS_START_ERROR + 16) -/** @see APR_STATUS_IS_EBADMASK */ -#define APR_EBADMASK (APR_OS_START_ERROR + 17) -/* empty slot: +18 */ -/** @see APR_STATUS_IS_EDSOPEN */ -#define APR_EDSOOPEN (APR_OS_START_ERROR + 19) -/** @see APR_STATUS_IS_EABSOLUTE */ -#define APR_EABSOLUTE (APR_OS_START_ERROR + 20) -/** @see APR_STATUS_IS_ERELATIVE */ -#define APR_ERELATIVE (APR_OS_START_ERROR + 21) -/** @see APR_STATUS_IS_EINCOMPLETE */ -#define APR_EINCOMPLETE (APR_OS_START_ERROR + 22) -/** @see APR_STATUS_IS_EABOVEROOT */ -#define APR_EABOVEROOT (APR_OS_START_ERROR + 23) -/** @see APR_STATUS_IS_EBADPATH */ -#define APR_EBADPATH (APR_OS_START_ERROR + 24) -/** @see APR_STATUS_IS_EPATHWILD */ -#define APR_EPATHWILD (APR_OS_START_ERROR + 25) -/** @see APR_STATUS_IS_ESYMNOTFOUND */ -#define APR_ESYMNOTFOUND (APR_OS_START_ERROR + 26) -/** @see APR_STATUS_IS_EPROC_UNKNOWN */ -#define APR_EPROC_UNKNOWN (APR_OS_START_ERROR + 27) -/** @see APR_STATUS_IS_ENOTENOUGHENTROPY */ -#define APR_ENOTENOUGHENTROPY (APR_OS_START_ERROR + 28) -/** @} */ - -/** - * @defgroup APR_STATUS_IS Status Value Tests - * @warning For any particular error condition, more than one of these tests - * may match. This is because platform-specific error codes may not - * always match the semantics of the POSIX codes these tests (and the - * corresponding APR error codes) are named after. A notable example - * are the APR_STATUS_IS_ENOENT and APR_STATUS_IS_ENOTDIR tests on - * Win32 platforms. The programmer should always be aware of this and - * adjust the order of the tests accordingly. - * @{ - */ -/** - * APR was unable to perform a stat on the file - * @warning always use this test, as platform-specific variances may meet this - * more than one error code - */ -#define APR_STATUS_IS_ENOSTAT(s) ((s) == APR_ENOSTAT) -/** - * APR was not provided a pool with which to allocate memory - * @warning always use this test, as platform-specific variances may meet this - * more than one error code - */ -#define APR_STATUS_IS_ENOPOOL(s) ((s) == APR_ENOPOOL) -/** APR was given an invalid date */ -#define APR_STATUS_IS_EBADDATE(s) ((s) == APR_EBADDATE) -/** APR was given an invalid socket */ -#define APR_STATUS_IS_EINVALSOCK(s) ((s) == APR_EINVALSOCK) -/** APR was not given a process structure */ -#define APR_STATUS_IS_ENOPROC(s) ((s) == APR_ENOPROC) -/** APR was not given a time structure */ -#define APR_STATUS_IS_ENOTIME(s) ((s) == APR_ENOTIME) -/** APR was not given a directory structure */ -#define APR_STATUS_IS_ENODIR(s) ((s) == APR_ENODIR) -/** APR was not given a lock structure */ -#define APR_STATUS_IS_ENOLOCK(s) ((s) == APR_ENOLOCK) -/** APR was not given a poll structure */ -#define APR_STATUS_IS_ENOPOLL(s) ((s) == APR_ENOPOLL) -/** APR was not given a socket */ -#define APR_STATUS_IS_ENOSOCKET(s) ((s) == APR_ENOSOCKET) -/** APR was not given a thread structure */ -#define APR_STATUS_IS_ENOTHREAD(s) ((s) == APR_ENOTHREAD) -/** APR was not given a thread key structure */ -#define APR_STATUS_IS_ENOTHDKEY(s) ((s) == APR_ENOTHDKEY) -/** Generic Error which can not be put into another spot */ -#define APR_STATUS_IS_EGENERAL(s) ((s) == APR_EGENERAL) -/** There is no more shared memory available */ -#define APR_STATUS_IS_ENOSHMAVAIL(s) ((s) == APR_ENOSHMAVAIL) -/** The specified IP address is invalid */ -#define APR_STATUS_IS_EBADIP(s) ((s) == APR_EBADIP) -/** The specified netmask is invalid */ -#define APR_STATUS_IS_EBADMASK(s) ((s) == APR_EBADMASK) -/* empty slot: +18 */ -/** - * APR was unable to open the dso object. - * For more information call apr_dso_error(). - */ -#if defined(WIN32) -#define APR_STATUS_IS_EDSOOPEN(s) ((s) == APR_EDSOOPEN \ - || APR_TO_OS_ERROR(s) == ERROR_MOD_NOT_FOUND) -#else -#define APR_STATUS_IS_EDSOOPEN(s) ((s) == APR_EDSOOPEN) -#endif -/** The given path was absolute. */ -#define APR_STATUS_IS_EABSOLUTE(s) ((s) == APR_EABSOLUTE) -/** The given path was relative. */ -#define APR_STATUS_IS_ERELATIVE(s) ((s) == APR_ERELATIVE) -/** The given path was neither relative nor absolute. */ -#define APR_STATUS_IS_EINCOMPLETE(s) ((s) == APR_EINCOMPLETE) -/** The given path was above the root path. */ -#define APR_STATUS_IS_EABOVEROOT(s) ((s) == APR_EABOVEROOT) -/** The given path was bad. */ -#define APR_STATUS_IS_EBADPATH(s) ((s) == APR_EBADPATH) -/** The given path contained wildcards. */ -#define APR_STATUS_IS_EPATHWILD(s) ((s) == APR_EPATHWILD) -/** Could not find the requested symbol. - * For more information call apr_dso_error(). - */ -#if defined(WIN32) -#define APR_STATUS_IS_ESYMNOTFOUND(s) ((s) == APR_ESYMNOTFOUND \ - || APR_TO_OS_ERROR(s) == ERROR_PROC_NOT_FOUND) -#else -#define APR_STATUS_IS_ESYMNOTFOUND(s) ((s) == APR_ESYMNOTFOUND) -#endif -/** The given process was not recognized by APR. */ -#define APR_STATUS_IS_EPROC_UNKNOWN(s) ((s) == APR_EPROC_UNKNOWN) - -/** APR could not gather enough entropy to continue. */ -#define APR_STATUS_IS_ENOTENOUGHENTROPY(s) ((s) == APR_ENOTENOUGHENTROPY) - -/** @} */ - -/** - * @addtogroup APR_Error - * @{ - */ -/** @see APR_STATUS_IS_INCHILD */ -#define APR_INCHILD (APR_OS_START_STATUS + 1) -/** @see APR_STATUS_IS_INPARENT */ -#define APR_INPARENT (APR_OS_START_STATUS + 2) -/** @see APR_STATUS_IS_DETACH */ -#define APR_DETACH (APR_OS_START_STATUS + 3) -/** @see APR_STATUS_IS_NOTDETACH */ -#define APR_NOTDETACH (APR_OS_START_STATUS + 4) -/** @see APR_STATUS_IS_CHILD_DONE */ -#define APR_CHILD_DONE (APR_OS_START_STATUS + 5) -/** @see APR_STATUS_IS_CHILD_NOTDONE */ -#define APR_CHILD_NOTDONE (APR_OS_START_STATUS + 6) -/** @see APR_STATUS_IS_TIMEUP */ -#define APR_TIMEUP (APR_OS_START_STATUS + 7) -/** @see APR_STATUS_IS_INCOMPLETE */ -#define APR_INCOMPLETE (APR_OS_START_STATUS + 8) -/* empty slot: +9 */ -/* empty slot: +10 */ -/* empty slot: +11 */ -/** @see APR_STATUS_IS_BADCH */ -#define APR_BADCH (APR_OS_START_STATUS + 12) -/** @see APR_STATUS_IS_BADARG */ -#define APR_BADARG (APR_OS_START_STATUS + 13) -/** @see APR_STATUS_IS_EOF */ -#define APR_EOF (APR_OS_START_STATUS + 14) -/** @see APR_STATUS_IS_NOTFOUND */ -#define APR_NOTFOUND (APR_OS_START_STATUS + 15) -/* empty slot: +16 */ -/* empty slot: +17 */ -/* empty slot: +18 */ -/** @see APR_STATUS_IS_ANONYMOUS */ -#define APR_ANONYMOUS (APR_OS_START_STATUS + 19) -/** @see APR_STATUS_IS_FILEBASED */ -#define APR_FILEBASED (APR_OS_START_STATUS + 20) -/** @see APR_STATUS_IS_KEYBASED */ -#define APR_KEYBASED (APR_OS_START_STATUS + 21) -/** @see APR_STATUS_IS_EINIT */ -#define APR_EINIT (APR_OS_START_STATUS + 22) -/** @see APR_STATUS_IS_ENOTIMPL */ -#define APR_ENOTIMPL (APR_OS_START_STATUS + 23) -/** @see APR_STATUS_IS_EMISMATCH */ -#define APR_EMISMATCH (APR_OS_START_STATUS + 24) -/** @see APR_STATUS_IS_EBUSY */ -#define APR_EBUSY (APR_OS_START_STATUS + 25) -/** @} */ - -/** - * @addtogroup APR_STATUS_IS - * @{ - */ -/** - * Program is currently executing in the child - * @warning - * always use this test, as platform-specific variances may meet this - * more than one error code */ -#define APR_STATUS_IS_INCHILD(s) ((s) == APR_INCHILD) -/** - * Program is currently executing in the parent - * @warning - * always use this test, as platform-specific variances may meet this - * more than one error code - */ -#define APR_STATUS_IS_INPARENT(s) ((s) == APR_INPARENT) -/** - * The thread is detached - * @warning - * always use this test, as platform-specific variances may meet this - * more than one error code - */ -#define APR_STATUS_IS_DETACH(s) ((s) == APR_DETACH) -/** - * The thread is not detached - * @warning - * always use this test, as platform-specific variances may meet this - * more than one error code - */ -#define APR_STATUS_IS_NOTDETACH(s) ((s) == APR_NOTDETACH) -/** - * The child has finished executing - * @warning - * always use this test, as platform-specific variances may meet this - * more than one error code - */ -#define APR_STATUS_IS_CHILD_DONE(s) ((s) == APR_CHILD_DONE) -/** - * The child has not finished executing - * @warning - * always use this test, as platform-specific variances may meet this - * more than one error code - */ -#define APR_STATUS_IS_CHILD_NOTDONE(s) ((s) == APR_CHILD_NOTDONE) -/** - * The operation did not finish before the timeout - * @warning - * always use this test, as platform-specific variances may meet this - * more than one error code - */ -#define APR_STATUS_IS_TIMEUP(s) ((s) == APR_TIMEUP) -/** - * The operation was incomplete although some processing was performed - * and the results are partially valid. - * @warning - * always use this test, as platform-specific variances may meet this - * more than one error code - */ -#define APR_STATUS_IS_INCOMPLETE(s) ((s) == APR_INCOMPLETE) -/* empty slot: +9 */ -/* empty slot: +10 */ -/* empty slot: +11 */ -/** - * Getopt found an option not in the option string - * @warning - * always use this test, as platform-specific variances may meet this - * more than one error code - */ -#define APR_STATUS_IS_BADCH(s) ((s) == APR_BADCH) -/** - * Getopt found an option not in the option string and an argument was - * specified in the option string - * @warning - * always use this test, as platform-specific variances may meet this - * more than one error code - */ -#define APR_STATUS_IS_BADARG(s) ((s) == APR_BADARG) -/** - * APR has encountered the end of the file - * @warning - * always use this test, as platform-specific variances may meet this - * more than one error code - */ -#define APR_STATUS_IS_EOF(s) ((s) == APR_EOF) -/** - * APR was unable to find the socket in the poll structure - * @warning - * always use this test, as platform-specific variances may meet this - * more than one error code - */ -#define APR_STATUS_IS_NOTFOUND(s) ((s) == APR_NOTFOUND) -/* empty slot: +16 */ -/* empty slot: +17 */ -/* empty slot: +18 */ -/** - * APR is using anonymous shared memory - * @warning - * always use this test, as platform-specific variances may meet this - * more than one error code - */ -#define APR_STATUS_IS_ANONYMOUS(s) ((s) == APR_ANONYMOUS) -/** - * APR is using a file name as the key to the shared memory - * @warning - * always use this test, as platform-specific variances may meet this - * more than one error code - */ -#define APR_STATUS_IS_FILEBASED(s) ((s) == APR_FILEBASED) -/** - * APR is using a shared key as the key to the shared memory - * @warning - * always use this test, as platform-specific variances may meet this - * more than one error code - */ -#define APR_STATUS_IS_KEYBASED(s) ((s) == APR_KEYBASED) -/** - * Ininitalizer value. If no option has been found, but - * the status variable requires a value, this should be used - * @warning - * always use this test, as platform-specific variances may meet this - * more than one error code - */ -#define APR_STATUS_IS_EINIT(s) ((s) == APR_EINIT) -/** - * The APR function has not been implemented on this - * platform, either because nobody has gotten to it yet, - * or the function is impossible on this platform. - * @warning - * always use this test, as platform-specific variances may meet this - * more than one error code - */ -#define APR_STATUS_IS_ENOTIMPL(s) ((s) == APR_ENOTIMPL) -/** - * Two passwords do not match. - * @warning - * always use this test, as platform-specific variances may meet this - * more than one error code - */ -#define APR_STATUS_IS_EMISMATCH(s) ((s) == APR_EMISMATCH) -/** - * The given lock was busy - * @warning always use this test, as platform-specific variances may meet this - * more than one error code - */ -#define APR_STATUS_IS_EBUSY(s) ((s) == APR_EBUSY) - -/** @} */ - -/** - * @addtogroup APR_Error APR Error Values - * @{ - */ -/* APR CANONICAL ERROR VALUES */ -/** @see APR_STATUS_IS_EACCES */ -#ifdef EACCES -#define APR_EACCES EACCES -#else -#define APR_EACCES (APR_OS_START_CANONERR + 1) -#endif - -/** @see APR_STATUS_IS_EXIST */ -#ifdef EEXIST -#define APR_EEXIST EEXIST -#else -#define APR_EEXIST (APR_OS_START_CANONERR + 2) -#endif - -/** @see APR_STATUS_IS_ENAMETOOLONG */ -#ifdef ENAMETOOLONG -#define APR_ENAMETOOLONG ENAMETOOLONG -#else -#define APR_ENAMETOOLONG (APR_OS_START_CANONERR + 3) -#endif - -/** @see APR_STATUS_IS_ENOENT */ -#ifdef ENOENT -#define APR_ENOENT ENOENT -#else -#define APR_ENOENT (APR_OS_START_CANONERR + 4) -#endif - -/** @see APR_STATUS_IS_ENOTDIR */ -#ifdef ENOTDIR -#define APR_ENOTDIR ENOTDIR -#else -#define APR_ENOTDIR (APR_OS_START_CANONERR + 5) -#endif - -/** @see APR_STATUS_IS_ENOSPC */ -#ifdef ENOSPC -#define APR_ENOSPC ENOSPC -#else -#define APR_ENOSPC (APR_OS_START_CANONERR + 6) -#endif - -/** @see APR_STATUS_IS_ENOMEM */ -#ifdef ENOMEM -#define APR_ENOMEM ENOMEM -#else -#define APR_ENOMEM (APR_OS_START_CANONERR + 7) -#endif - -/** @see APR_STATUS_IS_EMFILE */ -#ifdef EMFILE -#define APR_EMFILE EMFILE -#else -#define APR_EMFILE (APR_OS_START_CANONERR + 8) -#endif - -/** @see APR_STATUS_IS_ENFILE */ -#ifdef ENFILE -#define APR_ENFILE ENFILE -#else -#define APR_ENFILE (APR_OS_START_CANONERR + 9) -#endif - -/** @see APR_STATUS_IS_EBADF */ -#ifdef EBADF -#define APR_EBADF EBADF -#else -#define APR_EBADF (APR_OS_START_CANONERR + 10) -#endif - -/** @see APR_STATUS_IS_EINVAL */ -#ifdef EINVAL -#define APR_EINVAL EINVAL -#else -#define APR_EINVAL (APR_OS_START_CANONERR + 11) -#endif - -/** @see APR_STATUS_IS_ESPIPE */ -#ifdef ESPIPE -#define APR_ESPIPE ESPIPE -#else -#define APR_ESPIPE (APR_OS_START_CANONERR + 12) -#endif - -/** - * @see APR_STATUS_IS_EAGAIN - * @warning use APR_STATUS_IS_EAGAIN instead of just testing this value - */ -#ifdef EAGAIN -#define APR_EAGAIN EAGAIN -#elif defined(EWOULDBLOCK) -#define APR_EAGAIN EWOULDBLOCK -#else -#define APR_EAGAIN (APR_OS_START_CANONERR + 13) -#endif - -/** @see APR_STATUS_IS_EINTR */ -#ifdef EINTR -#define APR_EINTR EINTR -#else -#define APR_EINTR (APR_OS_START_CANONERR + 14) -#endif - -/** @see APR_STATUS_IS_ENOTSOCK */ -#ifdef ENOTSOCK -#define APR_ENOTSOCK ENOTSOCK -#else -#define APR_ENOTSOCK (APR_OS_START_CANONERR + 15) -#endif - -/** @see APR_STATUS_IS_ECONNREFUSED */ -#ifdef ECONNREFUSED -#define APR_ECONNREFUSED ECONNREFUSED -#else -#define APR_ECONNREFUSED (APR_OS_START_CANONERR + 16) -#endif - -/** @see APR_STATUS_IS_EINPROGRESS */ -#ifdef EINPROGRESS -#define APR_EINPROGRESS EINPROGRESS -#else -#define APR_EINPROGRESS (APR_OS_START_CANONERR + 17) -#endif - -/** - * @see APR_STATUS_IS_ECONNABORTED - * @warning use APR_STATUS_IS_ECONNABORTED instead of just testing this value - */ - -#ifdef ECONNABORTED -#define APR_ECONNABORTED ECONNABORTED -#else -#define APR_ECONNABORTED (APR_OS_START_CANONERR + 18) -#endif - -/** @see APR_STATUS_IS_ECONNRESET */ -#ifdef ECONNRESET -#define APR_ECONNRESET ECONNRESET -#else -#define APR_ECONNRESET (APR_OS_START_CANONERR + 19) -#endif - -/** @see APR_STATUS_IS_ETIMEDOUT - * @deprecated */ -#ifdef ETIMEDOUT -#define APR_ETIMEDOUT ETIMEDOUT -#else -#define APR_ETIMEDOUT (APR_OS_START_CANONERR + 20) -#endif - -/** @see APR_STATUS_IS_EHOSTUNREACH */ -#ifdef EHOSTUNREACH -#define APR_EHOSTUNREACH EHOSTUNREACH -#else -#define APR_EHOSTUNREACH (APR_OS_START_CANONERR + 21) -#endif - -/** @see APR_STATUS_IS_ENETUNREACH */ -#ifdef ENETUNREACH -#define APR_ENETUNREACH ENETUNREACH -#else -#define APR_ENETUNREACH (APR_OS_START_CANONERR + 22) -#endif - -/** @see APR_STATUS_IS_EFTYPE */ -#ifdef EFTYPE -#define APR_EFTYPE EFTYPE -#else -#define APR_EFTYPE (APR_OS_START_CANONERR + 23) -#endif - -/** @see APR_STATUS_IS_EPIPE */ -#ifdef EPIPE -#define APR_EPIPE EPIPE -#else -#define APR_EPIPE (APR_OS_START_CANONERR + 24) -#endif - -/** @see APR_STATUS_IS_EXDEV */ -#ifdef EXDEV -#define APR_EXDEV EXDEV -#else -#define APR_EXDEV (APR_OS_START_CANONERR + 25) -#endif - -/** @see APR_STATUS_IS_ENOTEMPTY */ -#ifdef ENOTEMPTY -#define APR_ENOTEMPTY ENOTEMPTY -#else -#define APR_ENOTEMPTY (APR_OS_START_CANONERR + 26) -#endif - -/** @} */ - -#if defined(OS2) && !defined(DOXYGEN) - -#define APR_FROM_OS_ERROR(e) (e == 0 ? APR_SUCCESS : e + APR_OS_START_SYSERR) -#define APR_TO_OS_ERROR(e) (e == 0 ? APR_SUCCESS : e - APR_OS_START_SYSERR) - -#define INCL_DOSERRORS -#define INCL_DOS - -/* Leave these undefined. - * OS2 doesn't rely on the errno concept. - * The API calls always return a result codes which - * should be filtered through APR_FROM_OS_ERROR(). - * - * #define apr_get_os_error() (APR_FROM_OS_ERROR(GetLastError())) - * #define apr_set_os_error(e) (SetLastError(APR_TO_OS_ERROR(e))) - */ - -/* A special case, only socket calls require this; - */ -#define apr_get_netos_error() (APR_FROM_OS_ERROR(errno)) -#define apr_set_netos_error(e) (errno = APR_TO_OS_ERROR(e)) - -/* And this needs to be greped away for good: - */ -#define APR_OS2_STATUS(e) (APR_FROM_OS_ERROR(e)) - -/* These can't sit in a private header, so in spite of the extra size, - * they need to be made available here. - */ -#define SOCBASEERR 10000 -#define SOCEPERM (SOCBASEERR+1) /* Not owner */ -#define SOCESRCH (SOCBASEERR+3) /* No such process */ -#define SOCEINTR (SOCBASEERR+4) /* Interrupted system call */ -#define SOCENXIO (SOCBASEERR+6) /* No such device or address */ -#define SOCEBADF (SOCBASEERR+9) /* Bad file number */ -#define SOCEACCES (SOCBASEERR+13) /* Permission denied */ -#define SOCEFAULT (SOCBASEERR+14) /* Bad address */ -#define SOCEINVAL (SOCBASEERR+22) /* Invalid argument */ -#define SOCEMFILE (SOCBASEERR+24) /* Too many open files */ -#define SOCEPIPE (SOCBASEERR+32) /* Broken pipe */ -#define SOCEOS2ERR (SOCBASEERR+100) /* OS/2 Error */ -#define SOCEWOULDBLOCK (SOCBASEERR+35) /* Operation would block */ -#define SOCEINPROGRESS (SOCBASEERR+36) /* Operation now in progress */ -#define SOCEALREADY (SOCBASEERR+37) /* Operation already in progress */ -#define SOCENOTSOCK (SOCBASEERR+38) /* Socket operation on non-socket */ -#define SOCEDESTADDRREQ (SOCBASEERR+39) /* Destination address required */ -#define SOCEMSGSIZE (SOCBASEERR+40) /* Message too long */ -#define SOCEPROTOTYPE (SOCBASEERR+41) /* Protocol wrong type for socket */ -#define SOCENOPROTOOPT (SOCBASEERR+42) /* Protocol not available */ -#define SOCEPROTONOSUPPORT (SOCBASEERR+43) /* Protocol not supported */ -#define SOCESOCKTNOSUPPORT (SOCBASEERR+44) /* Socket type not supported */ -#define SOCEOPNOTSUPP (SOCBASEERR+45) /* Operation not supported on socket */ -#define SOCEPFNOSUPPORT (SOCBASEERR+46) /* Protocol family not supported */ -#define SOCEAFNOSUPPORT (SOCBASEERR+47) /* Address family not supported by protocol family */ -#define SOCEADDRINUSE (SOCBASEERR+48) /* Address already in use */ -#define SOCEADDRNOTAVAIL (SOCBASEERR+49) /* Can't assign requested address */ -#define SOCENETDOWN (SOCBASEERR+50) /* Network is down */ -#define SOCENETUNREACH (SOCBASEERR+51) /* Network is unreachable */ -#define SOCENETRESET (SOCBASEERR+52) /* Network dropped connection on reset */ -#define SOCECONNABORTED (SOCBASEERR+53) /* Software caused connection abort */ -#define SOCECONNRESET (SOCBASEERR+54) /* Connection reset by peer */ -#define SOCENOBUFS (SOCBASEERR+55) /* No buffer space available */ -#define SOCEISCONN (SOCBASEERR+56) /* Socket is already connected */ -#define SOCENOTCONN (SOCBASEERR+57) /* Socket is not connected */ -#define SOCESHUTDOWN (SOCBASEERR+58) /* Can't send after socket shutdown */ -#define SOCETOOMANYREFS (SOCBASEERR+59) /* Too many references: can't splice */ -#define SOCETIMEDOUT (SOCBASEERR+60) /* Connection timed out */ -#define SOCECONNREFUSED (SOCBASEERR+61) /* Connection refused */ -#define SOCELOOP (SOCBASEERR+62) /* Too many levels of symbolic links */ -#define SOCENAMETOOLONG (SOCBASEERR+63) /* File name too long */ -#define SOCEHOSTDOWN (SOCBASEERR+64) /* Host is down */ -#define SOCEHOSTUNREACH (SOCBASEERR+65) /* No route to host */ -#define SOCENOTEMPTY (SOCBASEERR+66) /* Directory not empty */ - -/* APR CANONICAL ERROR TESTS */ -#define APR_STATUS_IS_EACCES(s) ((s) == APR_EACCES \ - || (s) == APR_OS_START_SYSERR + ERROR_ACCESS_DENIED \ - || (s) == APR_OS_START_SYSERR + ERROR_SHARING_VIOLATION) -#define APR_STATUS_IS_EEXIST(s) ((s) == APR_EEXIST \ - || (s) == APR_OS_START_SYSERR + ERROR_OPEN_FAILED \ - || (s) == APR_OS_START_SYSERR + ERROR_FILE_EXISTS \ - || (s) == APR_OS_START_SYSERR + ERROR_ALREADY_EXISTS \ - || (s) == APR_OS_START_SYSERR + ERROR_ACCESS_DENIED) -#define APR_STATUS_IS_ENAMETOOLONG(s) ((s) == APR_ENAMETOOLONG \ - || (s) == APR_OS_START_SYSERR + ERROR_FILENAME_EXCED_RANGE \ - || (s) == APR_OS_START_SYSERR + SOCENAMETOOLONG) -#define APR_STATUS_IS_ENOENT(s) ((s) == APR_ENOENT \ - || (s) == APR_OS_START_SYSERR + ERROR_FILE_NOT_FOUND \ - || (s) == APR_OS_START_SYSERR + ERROR_PATH_NOT_FOUND \ - || (s) == APR_OS_START_SYSERR + ERROR_NO_MORE_FILES \ - || (s) == APR_OS_START_SYSERR + ERROR_OPEN_FAILED) -#define APR_STATUS_IS_ENOTDIR(s) ((s) == APR_ENOTDIR) -#define APR_STATUS_IS_ENOSPC(s) ((s) == APR_ENOSPC \ - || (s) == APR_OS_START_SYSERR + ERROR_DISK_FULL) -#define APR_STATUS_IS_ENOMEM(s) ((s) == APR_ENOMEM) -#define APR_STATUS_IS_EMFILE(s) ((s) == APR_EMFILE \ - || (s) == APR_OS_START_SYSERR + ERROR_TOO_MANY_OPEN_FILES) -#define APR_STATUS_IS_ENFILE(s) ((s) == APR_ENFILE) -#define APR_STATUS_IS_EBADF(s) ((s) == APR_EBADF \ - || (s) == APR_OS_START_SYSERR + ERROR_INVALID_HANDLE) -#define APR_STATUS_IS_EINVAL(s) ((s) == APR_EINVAL \ - || (s) == APR_OS_START_SYSERR + ERROR_INVALID_PARAMETER \ - || (s) == APR_OS_START_SYSERR + ERROR_INVALID_FUNCTION) -#define APR_STATUS_IS_ESPIPE(s) ((s) == APR_ESPIPE \ - || (s) == APR_OS_START_SYSERR + ERROR_NEGATIVE_SEEK) -#define APR_STATUS_IS_EAGAIN(s) ((s) == APR_EAGAIN \ - || (s) == APR_OS_START_SYSERR + ERROR_NO_DATA \ - || (s) == APR_OS_START_SYSERR + SOCEWOULDBLOCK \ - || (s) == APR_OS_START_SYSERR + ERROR_LOCK_VIOLATION) -#define APR_STATUS_IS_EINTR(s) ((s) == APR_EINTR \ - || (s) == APR_OS_START_SYSERR + SOCEINTR) -#define APR_STATUS_IS_ENOTSOCK(s) ((s) == APR_ENOTSOCK \ - || (s) == APR_OS_START_SYSERR + SOCENOTSOCK) -#define APR_STATUS_IS_ECONNREFUSED(s) ((s) == APR_ECONNREFUSED \ - || (s) == APR_OS_START_SYSERR + SOCECONNREFUSED) -#define APR_STATUS_IS_EINPROGRESS(s) ((s) == APR_EINPROGRESS \ - || (s) == APR_OS_START_SYSERR + SOCEINPROGRESS) -#define APR_STATUS_IS_ECONNABORTED(s) ((s) == APR_ECONNABORTED \ - || (s) == APR_OS_START_SYSERR + SOCECONNABORTED) -#define APR_STATUS_IS_ECONNRESET(s) ((s) == APR_ECONNRESET \ - || (s) == APR_OS_START_SYSERR + SOCECONNRESET) -/* XXX deprecated */ -#define APR_STATUS_IS_ETIMEDOUT(s) ((s) == APR_ETIMEDOUT \ - || (s) == APR_OS_START_SYSERR + SOCETIMEDOUT) -#undef APR_STATUS_IS_TIMEUP -#define APR_STATUS_IS_TIMEUP(s) ((s) == APR_TIMEUP \ - || (s) == APR_OS_START_SYSERR + SOCETIMEDOUT) -#define APR_STATUS_IS_EHOSTUNREACH(s) ((s) == APR_EHOSTUNREACH \ - || (s) == APR_OS_START_SYSERR + SOCEHOSTUNREACH) -#define APR_STATUS_IS_ENETUNREACH(s) ((s) == APR_ENETUNREACH \ - || (s) == APR_OS_START_SYSERR + SOCENETUNREACH) -#define APR_STATUS_IS_EFTYPE(s) ((s) == APR_EFTYPE) -#define APR_STATUS_IS_EPIPE(s) ((s) == APR_EPIPE \ - || (s) == APR_OS_START_SYSERR + ERROR_BROKEN_PIPE \ - || (s) == APR_OS_START_SYSERR + SOCEPIPE) -#define APR_STATUS_IS_EXDEV(s) ((s) == APR_EXDEV \ - || (s) == APR_OS_START_SYSERR + ERROR_NOT_SAME_DEVICE) -#define APR_STATUS_IS_ENOTEMPTY(s) ((s) == APR_ENOTEMPTY \ - || (s) == APR_OS_START_SYSERR + ERROR_DIR_NOT_EMPTY \ - || (s) == APR_OS_START_SYSERR + ERROR_ACCESS_DENIED) - -/* - Sorry, too tired to wrap this up for OS2... feel free to - fit the following into their best matches. - - { ERROR_NO_SIGNAL_SENT, ESRCH }, - { SOCEALREADY, EALREADY }, - { SOCEDESTADDRREQ, EDESTADDRREQ }, - { SOCEMSGSIZE, EMSGSIZE }, - { SOCEPROTOTYPE, EPROTOTYPE }, - { SOCENOPROTOOPT, ENOPROTOOPT }, - { SOCEPROTONOSUPPORT, EPROTONOSUPPORT }, - { SOCESOCKTNOSUPPORT, ESOCKTNOSUPPORT }, - { SOCEOPNOTSUPP, EOPNOTSUPP }, - { SOCEPFNOSUPPORT, EPFNOSUPPORT }, - { SOCEAFNOSUPPORT, EAFNOSUPPORT }, - { SOCEADDRINUSE, EADDRINUSE }, - { SOCEADDRNOTAVAIL, EADDRNOTAVAIL }, - { SOCENETDOWN, ENETDOWN }, - { SOCENETRESET, ENETRESET }, - { SOCENOBUFS, ENOBUFS }, - { SOCEISCONN, EISCONN }, - { SOCENOTCONN, ENOTCONN }, - { SOCESHUTDOWN, ESHUTDOWN }, - { SOCETOOMANYREFS, ETOOMANYREFS }, - { SOCELOOP, ELOOP }, - { SOCEHOSTDOWN, EHOSTDOWN }, - { SOCENOTEMPTY, ENOTEMPTY }, - { SOCEPIPE, EPIPE } -*/ - -#elif defined(WIN32) && !defined(DOXYGEN) /* !defined(OS2) */ - -#define APR_FROM_OS_ERROR(e) (e == 0 ? APR_SUCCESS : e + APR_OS_START_SYSERR) -#define APR_TO_OS_ERROR(e) (e == 0 ? APR_SUCCESS : e - APR_OS_START_SYSERR) - -#define apr_get_os_error() (APR_FROM_OS_ERROR(GetLastError())) -#define apr_set_os_error(e) (SetLastError(APR_TO_OS_ERROR(e))) - -/* A special case, only socket calls require this: - */ -#define apr_get_netos_error() (APR_FROM_OS_ERROR(WSAGetLastError())) -#define apr_set_netos_error(e) (WSASetLastError(APR_TO_OS_ERROR(e))) - -/* APR CANONICAL ERROR TESTS */ -#define APR_STATUS_IS_EACCES(s) ((s) == APR_EACCES \ - || (s) == APR_OS_START_SYSERR + ERROR_ACCESS_DENIED \ - || (s) == APR_OS_START_SYSERR + ERROR_CANNOT_MAKE \ - || (s) == APR_OS_START_SYSERR + ERROR_CURRENT_DIRECTORY \ - || (s) == APR_OS_START_SYSERR + ERROR_DRIVE_LOCKED \ - || (s) == APR_OS_START_SYSERR + ERROR_FAIL_I24 \ - || (s) == APR_OS_START_SYSERR + ERROR_LOCK_VIOLATION \ - || (s) == APR_OS_START_SYSERR + ERROR_LOCK_FAILED \ - || (s) == APR_OS_START_SYSERR + ERROR_NOT_LOCKED \ - || (s) == APR_OS_START_SYSERR + ERROR_NETWORK_ACCESS_DENIED \ - || (s) == APR_OS_START_SYSERR + ERROR_SHARING_VIOLATION) -#define APR_STATUS_IS_EEXIST(s) ((s) == APR_EEXIST \ - || (s) == APR_OS_START_SYSERR + ERROR_FILE_EXISTS \ - || (s) == APR_OS_START_SYSERR + ERROR_ALREADY_EXISTS) -#define APR_STATUS_IS_ENAMETOOLONG(s) ((s) == APR_ENAMETOOLONG \ - || (s) == APR_OS_START_SYSERR + ERROR_FILENAME_EXCED_RANGE \ - || (s) == APR_OS_START_SYSERR + WSAENAMETOOLONG) -#define APR_STATUS_IS_ENOENT(s) ((s) == APR_ENOENT \ - || (s) == APR_OS_START_SYSERR + ERROR_FILE_NOT_FOUND \ - || (s) == APR_OS_START_SYSERR + ERROR_PATH_NOT_FOUND \ - || (s) == APR_OS_START_SYSERR + ERROR_OPEN_FAILED \ - || (s) == APR_OS_START_SYSERR + ERROR_NO_MORE_FILES) -#define APR_STATUS_IS_ENOTDIR(s) ((s) == APR_ENOTDIR \ - || (s) == APR_OS_START_SYSERR + ERROR_PATH_NOT_FOUND \ - || (s) == APR_OS_START_SYSERR + ERROR_BAD_NETPATH \ - || (s) == APR_OS_START_SYSERR + ERROR_BAD_NET_NAME \ - || (s) == APR_OS_START_SYSERR + ERROR_BAD_PATHNAME \ - || (s) == APR_OS_START_SYSERR + ERROR_INVALID_DRIVE) -#define APR_STATUS_IS_ENOSPC(s) ((s) == APR_ENOSPC \ - || (s) == APR_OS_START_SYSERR + ERROR_DISK_FULL) -#define APR_STATUS_IS_ENOMEM(s) ((s) == APR_ENOMEM \ - || (s) == APR_OS_START_SYSERR + ERROR_ARENA_TRASHED \ - || (s) == APR_OS_START_SYSERR + ERROR_NOT_ENOUGH_MEMORY \ - || (s) == APR_OS_START_SYSERR + ERROR_INVALID_BLOCK \ - || (s) == APR_OS_START_SYSERR + ERROR_NOT_ENOUGH_QUOTA \ - || (s) == APR_OS_START_SYSERR + ERROR_OUTOFMEMORY) -#define APR_STATUS_IS_EMFILE(s) ((s) == APR_EMFILE \ - || (s) == APR_OS_START_SYSERR + ERROR_TOO_MANY_OPEN_FILES) -#define APR_STATUS_IS_ENFILE(s) ((s) == APR_ENFILE) -#define APR_STATUS_IS_EBADF(s) ((s) == APR_EBADF \ - || (s) == APR_OS_START_SYSERR + ERROR_INVALID_HANDLE \ - || (s) == APR_OS_START_SYSERR + ERROR_INVALID_TARGET_HANDLE) -#define APR_STATUS_IS_EINVAL(s) ((s) == APR_EINVAL \ - || (s) == APR_OS_START_SYSERR + ERROR_INVALID_ACCESS \ - || (s) == APR_OS_START_SYSERR + ERROR_INVALID_DATA \ - || (s) == APR_OS_START_SYSERR + ERROR_INVALID_FUNCTION \ - || (s) == APR_OS_START_SYSERR + ERROR_INVALID_HANDLE \ - || (s) == APR_OS_START_SYSERR + ERROR_INVALID_PARAMETER \ - || (s) == APR_OS_START_SYSERR + ERROR_NEGATIVE_SEEK) -#define APR_STATUS_IS_ESPIPE(s) ((s) == APR_ESPIPE \ - || (s) == APR_OS_START_SYSERR + ERROR_SEEK_ON_DEVICE \ - || (s) == APR_OS_START_SYSERR + ERROR_NEGATIVE_SEEK) -#define APR_STATUS_IS_EAGAIN(s) ((s) == APR_EAGAIN \ - || (s) == APR_OS_START_SYSERR + ERROR_NO_DATA \ - || (s) == APR_OS_START_SYSERR + ERROR_NO_PROC_SLOTS \ - || (s) == APR_OS_START_SYSERR + ERROR_NESTING_NOT_ALLOWED \ - || (s) == APR_OS_START_SYSERR + ERROR_MAX_THRDS_REACHED \ - || (s) == APR_OS_START_SYSERR + ERROR_LOCK_VIOLATION \ - || (s) == APR_OS_START_SYSERR + WSAEWOULDBLOCK) -#define APR_STATUS_IS_EINTR(s) ((s) == APR_EINTR \ - || (s) == APR_OS_START_SYSERR + WSAEINTR) -#define APR_STATUS_IS_ENOTSOCK(s) ((s) == APR_ENOTSOCK \ - || (s) == APR_OS_START_SYSERR + WSAENOTSOCK) -#define APR_STATUS_IS_ECONNREFUSED(s) ((s) == APR_ECONNREFUSED \ - || (s) == APR_OS_START_SYSERR + WSAECONNREFUSED) -#define APR_STATUS_IS_EINPROGRESS(s) ((s) == APR_EINPROGRESS \ - || (s) == APR_OS_START_SYSERR + WSAEINPROGRESS) -#define APR_STATUS_IS_ECONNABORTED(s) ((s) == APR_ECONNABORTED \ - || (s) == APR_OS_START_SYSERR + WSAECONNABORTED) -#define APR_STATUS_IS_ECONNRESET(s) ((s) == APR_ECONNRESET \ - || (s) == APR_OS_START_SYSERR + ERROR_NETNAME_DELETED \ - || (s) == APR_OS_START_SYSERR + WSAECONNRESET) -/* XXX deprecated */ -#define APR_STATUS_IS_ETIMEDOUT(s) ((s) == APR_ETIMEDOUT \ - || (s) == APR_OS_START_SYSERR + WSAETIMEDOUT \ - || (s) == APR_OS_START_SYSERR + WAIT_TIMEOUT) -#undef APR_STATUS_IS_TIMEUP -#define APR_STATUS_IS_TIMEUP(s) ((s) == APR_TIMEUP \ - || (s) == APR_OS_START_SYSERR + WSAETIMEDOUT \ - || (s) == APR_OS_START_SYSERR + WAIT_TIMEOUT) -#define APR_STATUS_IS_EHOSTUNREACH(s) ((s) == APR_EHOSTUNREACH \ - || (s) == APR_OS_START_SYSERR + WSAEHOSTUNREACH) -#define APR_STATUS_IS_ENETUNREACH(s) ((s) == APR_ENETUNREACH \ - || (s) == APR_OS_START_SYSERR + WSAENETUNREACH) -#define APR_STATUS_IS_EFTYPE(s) ((s) == APR_EFTYPE \ - || (s) == APR_OS_START_SYSERR + ERROR_EXE_MACHINE_TYPE_MISMATCH \ - || (s) == APR_OS_START_SYSERR + ERROR_INVALID_DLL \ - || (s) == APR_OS_START_SYSERR + ERROR_INVALID_MODULETYPE \ - || (s) == APR_OS_START_SYSERR + ERROR_BAD_EXE_FORMAT \ - || (s) == APR_OS_START_SYSERR + ERROR_INVALID_EXE_SIGNATURE \ - || (s) == APR_OS_START_SYSERR + ERROR_FILE_CORRUPT \ - || (s) == APR_OS_START_SYSERR + ERROR_BAD_FORMAT) -#define APR_STATUS_IS_EPIPE(s) ((s) == APR_EPIPE \ - || (s) == APR_OS_START_SYSERR + ERROR_BROKEN_PIPE) -#define APR_STATUS_IS_EXDEV(s) ((s) == APR_EXDEV \ - || (s) == APR_OS_START_SYSERR + ERROR_NOT_SAME_DEVICE) -#define APR_STATUS_IS_ENOTEMPTY(s) ((s) == APR_ENOTEMPTY \ - || (s) == APR_OS_START_SYSERR + ERROR_DIR_NOT_EMPTY) - -#elif defined(NETWARE) && defined(USE_WINSOCK) && !defined(DOXYGEN) /* !defined(OS2) && !defined(WIN32) */ - -#define APR_FROM_OS_ERROR(e) (e == 0 ? APR_SUCCESS : e + APR_OS_START_SYSERR) -#define APR_TO_OS_ERROR(e) (e == 0 ? APR_SUCCESS : e - APR_OS_START_SYSERR) - -#define apr_get_os_error() (errno) -#define apr_set_os_error(e) (errno = (e)) - -/* A special case, only socket calls require this: */ -#define apr_get_netos_error() (APR_FROM_OS_ERROR(WSAGetLastError())) -#define apr_set_netos_error(e) (WSASetLastError(APR_TO_OS_ERROR(e))) - -/* APR CANONICAL ERROR TESTS */ -#define APR_STATUS_IS_EACCES(s) ((s) == APR_EACCES) -#define APR_STATUS_IS_EEXIST(s) ((s) == APR_EEXIST) -#define APR_STATUS_IS_ENAMETOOLONG(s) ((s) == APR_ENAMETOOLONG) -#define APR_STATUS_IS_ENOENT(s) ((s) == APR_ENOENT) -#define APR_STATUS_IS_ENOTDIR(s) ((s) == APR_ENOTDIR) -#define APR_STATUS_IS_ENOSPC(s) ((s) == APR_ENOSPC) -#define APR_STATUS_IS_ENOMEM(s) ((s) == APR_ENOMEM) -#define APR_STATUS_IS_EMFILE(s) ((s) == APR_EMFILE) -#define APR_STATUS_IS_ENFILE(s) ((s) == APR_ENFILE) -#define APR_STATUS_IS_EBADF(s) ((s) == APR_EBADF) -#define APR_STATUS_IS_EINVAL(s) ((s) == APR_EINVAL) -#define APR_STATUS_IS_ESPIPE(s) ((s) == APR_ESPIPE) - -#define APR_STATUS_IS_EAGAIN(s) ((s) == APR_EAGAIN \ - || (s) == EWOULDBLOCK \ - || (s) == APR_OS_START_SYSERR + WSAEWOULDBLOCK) -#define APR_STATUS_IS_EINTR(s) ((s) == APR_EINTR \ - || (s) == APR_OS_START_SYSERR + WSAEINTR) -#define APR_STATUS_IS_ENOTSOCK(s) ((s) == APR_ENOTSOCK \ - || (s) == APR_OS_START_SYSERR + WSAENOTSOCK) -#define APR_STATUS_IS_ECONNREFUSED(s) ((s) == APR_ECONNREFUSED \ - || (s) == APR_OS_START_SYSERR + WSAECONNREFUSED) -#define APR_STATUS_IS_EINPROGRESS(s) ((s) == APR_EINPROGRESS \ - || (s) == APR_OS_START_SYSERR + WSAEINPROGRESS) -#define APR_STATUS_IS_ECONNABORTED(s) ((s) == APR_ECONNABORTED \ - || (s) == APR_OS_START_SYSERR + WSAECONNABORTED) -#define APR_STATUS_IS_ECONNRESET(s) ((s) == APR_ECONNRESET \ - || (s) == APR_OS_START_SYSERR + WSAECONNRESET) -/* XXX deprecated */ -#define APR_STATUS_IS_ETIMEDOUT(s) ((s) == APR_ETIMEDOUT \ - || (s) == APR_OS_START_SYSERR + WSAETIMEDOUT \ - || (s) == APR_OS_START_SYSERR + WAIT_TIMEOUT) -#undef APR_STATUS_IS_TIMEUP -#define APR_STATUS_IS_TIMEUP(s) ((s) == APR_TIMEUP \ - || (s) == APR_OS_START_SYSERR + WSAETIMEDOUT \ - || (s) == APR_OS_START_SYSERR + WAIT_TIMEOUT) -#define APR_STATUS_IS_EHOSTUNREACH(s) ((s) == APR_EHOSTUNREACH \ - || (s) == APR_OS_START_SYSERR + WSAEHOSTUNREACH) -#define APR_STATUS_IS_ENETUNREACH(s) ((s) == APR_ENETUNREACH \ - || (s) == APR_OS_START_SYSERR + WSAENETUNREACH) -#define APR_STATUS_IS_ENETDOWN(s) ((s) == APR_OS_START_SYSERR + WSAENETDOWN) -#define APR_STATUS_IS_EFTYPE(s) ((s) == APR_EFTYPE) -#define APR_STATUS_IS_EPIPE(s) ((s) == APR_EPIPE) -#define APR_STATUS_IS_EXDEV(s) ((s) == APR_EXDEV) -#define APR_STATUS_IS_ENOTEMPTY(s) ((s) == APR_ENOTEMPTY) - -#else /* !defined(NETWARE) && !defined(OS2) && !defined(WIN32) */ - -/* - * os error codes are clib error codes - */ -#define APR_FROM_OS_ERROR(e) (e) -#define APR_TO_OS_ERROR(e) (e) - -#define apr_get_os_error() (errno) -#define apr_set_os_error(e) (errno = (e)) - -/* A special case, only socket calls require this: - */ -#define apr_get_netos_error() (errno) -#define apr_set_netos_error(e) (errno = (e)) - -/** - * @addtogroup APR_STATUS_IS - * @{ - */ - -/** permission denied */ -#define APR_STATUS_IS_EACCES(s) ((s) == APR_EACCES) -/** file exists */ -#define APR_STATUS_IS_EEXIST(s) ((s) == APR_EEXIST) -/** path name is too long */ -#define APR_STATUS_IS_ENAMETOOLONG(s) ((s) == APR_ENAMETOOLONG) -/** - * no such file or directory - * @remark - * EMVSCATLG can be returned by the automounter on z/OS for - * paths which do not exist. - */ -#ifdef EMVSCATLG -#define APR_STATUS_IS_ENOENT(s) ((s) == APR_ENOENT \ - || (s) == EMVSCATLG) -#else -#define APR_STATUS_IS_ENOENT(s) ((s) == APR_ENOENT) -#endif -/** not a directory */ -#define APR_STATUS_IS_ENOTDIR(s) ((s) == APR_ENOTDIR) -/** no space left on device */ -#ifdef EDQUOT -#define APR_STATUS_IS_ENOSPC(s) ((s) == APR_ENOSPC \ - || (s) == EDQUOT) -#else -#define APR_STATUS_IS_ENOSPC(s) ((s) == APR_ENOSPC) -#endif -/** not enough memory */ -#define APR_STATUS_IS_ENOMEM(s) ((s) == APR_ENOMEM) -/** too many open files */ -#define APR_STATUS_IS_EMFILE(s) ((s) == APR_EMFILE) -/** file table overflow */ -#define APR_STATUS_IS_ENFILE(s) ((s) == APR_ENFILE) -/** bad file # */ -#define APR_STATUS_IS_EBADF(s) ((s) == APR_EBADF) -/** invalid argument */ -#define APR_STATUS_IS_EINVAL(s) ((s) == APR_EINVAL) -/** illegal seek */ -#define APR_STATUS_IS_ESPIPE(s) ((s) == APR_ESPIPE) - -/** operation would block */ -#if !defined(EWOULDBLOCK) || !defined(EAGAIN) -#define APR_STATUS_IS_EAGAIN(s) ((s) == APR_EAGAIN) -#elif (EWOULDBLOCK == EAGAIN) -#define APR_STATUS_IS_EAGAIN(s) ((s) == APR_EAGAIN) -#else -#define APR_STATUS_IS_EAGAIN(s) ((s) == APR_EAGAIN \ - || (s) == EWOULDBLOCK) -#endif - -/** interrupted system call */ -#define APR_STATUS_IS_EINTR(s) ((s) == APR_EINTR) -/** socket operation on a non-socket */ -#define APR_STATUS_IS_ENOTSOCK(s) ((s) == APR_ENOTSOCK) -/** Connection Refused */ -#define APR_STATUS_IS_ECONNREFUSED(s) ((s) == APR_ECONNREFUSED) -/** operation now in progress */ -#define APR_STATUS_IS_EINPROGRESS(s) ((s) == APR_EINPROGRESS) - -/** - * Software caused connection abort - * @remark - * EPROTO on certain older kernels really means ECONNABORTED, so we need to - * ignore it for them. See discussion in new-httpd archives nh.9701 & nh.9603 - * - * There is potentially a bug in Solaris 2.x x<6, and other boxes that - * implement tcp sockets in userland (i.e. on top of STREAMS). On these - * systems, EPROTO can actually result in a fatal loop. See PR#981 for - * example. It's hard to handle both uses of EPROTO. - */ -#ifdef EPROTO -#define APR_STATUS_IS_ECONNABORTED(s) ((s) == APR_ECONNABORTED \ - || (s) == EPROTO) -#else -#define APR_STATUS_IS_ECONNABORTED(s) ((s) == APR_ECONNABORTED) -#endif - -/** Connection Reset by peer */ -#define APR_STATUS_IS_ECONNRESET(s) ((s) == APR_ECONNRESET) -/** Operation timed out - * @deprecated */ -#define APR_STATUS_IS_ETIMEDOUT(s) ((s) == APR_ETIMEDOUT) -/** no route to host */ -#define APR_STATUS_IS_EHOSTUNREACH(s) ((s) == APR_EHOSTUNREACH) -/** network is unreachable */ -#define APR_STATUS_IS_ENETUNREACH(s) ((s) == APR_ENETUNREACH) -/** inappropiate file type or format */ -#define APR_STATUS_IS_EFTYPE(s) ((s) == APR_EFTYPE) -/** broken pipe */ -#define APR_STATUS_IS_EPIPE(s) ((s) == APR_EPIPE) -/** cross device link */ -#define APR_STATUS_IS_EXDEV(s) ((s) == APR_EXDEV) -/** Directory Not Empty */ -#define APR_STATUS_IS_ENOTEMPTY(s) ((s) == APR_ENOTEMPTY || \ - (s) == APR_EEXIST) -/** @} */ - -#endif /* !defined(NETWARE) && !defined(OS2) && !defined(WIN32) */ - -/** @} */ - -#ifdef __cplusplus -} -#endif - -#endif /* ! APR_ERRNO_H */ -#ifndef _LINUX_ERRNO_H -#define _LINUX_ERRNO_H - -#include <asm/errno.h> - -#ifdef __KERNEL__ - -/* Should never be seen by user programs */ -#define ERESTARTSYS 512 -#define ERESTARTNOINTR 513 -#define ERESTARTNOHAND 514 /* restart if no handler.. */ -#define ENOIOCTLCMD 515 /* No ioctl command */ -#define ERESTART_RESTARTBLOCK 516 /* restart by calling sys_restart_syscall */ - -/* Defined for the NFSv3 protocol */ -#define EBADHANDLE 521 /* Illegal NFS file handle */ -#define ENOTSYNC 522 /* Update synchronization mismatch */ -#define EBADCOOKIE 523 /* Cookie is stale */ -#define ENOTSUPP 524 /* Operation is not supported */ -#define ETOOSMALL 525 /* Buffer or request is too small */ -#define ESERVERFAULT 526 /* An untranslatable error occurred */ -#define EBADTYPE 527 /* Type not supported by server */ -#define EJUKEBOX 528 /* Request initiated, but will not complete before timeout */ -#define EIOCBQUEUED 529 /* iocb queued, will get completion event */ -#define EIOCBRETRY 530 /* iocb queued, will trigger a retry */ - -#endif - -#endif -// Copyright (c) 1994 James Clark -// See the file COPYING for copying permission. - -#ifndef ErrnoMessageArg_INCLUDED -#define ErrnoMessageArg_INCLUDED 1 - -#include "MessageArg.h" -#include "rtti.h" - -#ifdef SP_NAMESPACE -namespace SP_NAMESPACE { -#endif - -class SP_API ErrnoMessageArg : public OtherMessageArg { - RTTI_CLASS -public: - ErrnoMessageArg(int errnum) : errno_(errnum) { } - MessageArg *copy() const; - // errno might be a macro so we must use a different name - int errnum() const; -private: - int errno_; -}; - -inline -int ErrnoMessageArg::errnum() const -{ - return errno_; -} - -#ifdef SP_NAMESPACE -} -#endif - -#endif /* not ErrnoMessageArg_INCLUDED */ -/* Copyright (C) 1991,92,93,94,95,96,97,2002 Free Software Foundation, Inc. - This file is part of the GNU C Library. - - The GNU C Library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. - - The GNU C Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public - License along with the GNU C Library; if not, write to the Free - Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA - 02111-1307 USA. */ - -/* - * ISO C99 Standard: 7.5 Errors <errno.h> - */ - -#ifndef _ERRNO_H - -/* The includer defined __need_Emath if he wants only the definitions - of EDOM and ERANGE, and not everything else. */ -#ifndef __need_Emath -# define _ERRNO_H 1 -# include <features.h> -#endif - -__BEGIN_DECLS - -/* Get the error number constants from the system-specific file. - This file will test __need_Emath and _ERRNO_H. */ -#include <bits/errno.h> -#undef __need_Emath - -#ifdef _ERRNO_H - -/* Declare the `errno' variable, unless it's defined as a macro by - bits/errno.h. This is the case in GNU, where it is a per-thread - variable. This redeclaration using the macro still works, but it - will be a function declaration without a prototype and may trigger - a -Wstrict-prototypes warning. */ -#ifndef errno -extern int errno; -#endif - -#ifdef __USE_GNU - -/* The full and simple forms of the name with which the program was - invoked. These variables are set up automatically at startup based on - the value of ARGV[0] (this works only if you use GNU ld). */ -extern char *program_invocation_name, *program_invocation_short_name; -#endif /* __USE_GNU */ -#endif /* _ERRNO_H */ - -__END_DECLS - -#endif /* _ERRNO_H */ - -/* The Hurd <bits/errno.h> defines `error_t' as an enumerated type so - that printing `error_t' values in the debugger shows the names. We - might need this definition sometimes even if this file was included - before. */ -#if defined __USE_GNU || defined __need_error_t -# ifndef __error_t_defined -typedef int error_t; -# define __error_t_defined 1 -# endif -# undef __need_error_t -#endif -#ifndef _I386_ERRNO_H -#define _I386_ERRNO_H - -#include <asm-generic/errno.h> - -#endif -#ifndef _ASM_GENERIC_ERRNO_BASE_H -#define _ASM_GENERIC_ERRNO_BASE_H - -#define EPERM 1 /* Operation not permitted */ -#define ENOENT 2 /* No such file or directory */ -#define ESRCH 3 /* No such process */ -#define EINTR 4 /* Interrupted system call */ -#define EIO 5 /* I/O error */ -#define ENXIO 6 /* No such device or address */ -#define E2BIG 7 /* Argument list too long */ -#define ENOEXEC 8 /* Exec format error */ -#define EBADF 9 /* Bad file number */ -#define ECHILD 10 /* No child processes */ -#define EAGAIN 11 /* Try again */ -#define ENOMEM 12 /* Out of memory */ -#define EACCES 13 /* Permission denied */ -#define EFAULT 14 /* Bad address */ -#define ENOTBLK 15 /* Block device required */ -#define EBUSY 16 /* Device or resource busy */ -#define EEXIST 17 /* File exists */ -#define EXDEV 18 /* Cross-device link */ -#define ENODEV 19 /* No such device */ -#define ENOTDIR 20 /* Not a directory */ -#define EISDIR 21 /* Is a directory */ -#define EINVAL 22 /* Invalid argument */ -#define ENFILE 23 /* File table overflow */ -#define EMFILE 24 /* Too many open files */ -#define ENOTTY 25 /* Not a typewriter */ -#define ETXTBSY 26 /* Text file busy */ -#define EFBIG 27 /* File too large */ -#define ENOSPC 28 /* No space left on device */ -#define ESPIPE 29 /* Illegal seek */ -#define EROFS 30 /* Read-only file system */ -#define EMLINK 31 /* Too many links */ -#define EPIPE 32 /* Broken pipe */ -#define EDOM 33 /* Math argument out of domain of func */ -#define ERANGE 34 /* Math result not representable */ - -#endif -#ifndef _ASM_GENERIC_ERRNO_H -#define _ASM_GENERIC_ERRNO_H - -#include <asm-generic/errno-base.h> - -#define EDEADLK 35 /* Resource deadlock would occur */ -#define ENAMETOOLONG 36 /* File name too long */ -#define ENOLCK 37 /* No record locks available */ -#define ENOSYS 38 /* Function not implemented */ -#define ENOTEMPTY 39 /* Directory not empty */ -#define ELOOP 40 /* Too many symbolic links encountered */ -#define EWOULDBLOCK EAGAIN /* Operation would block */ -#define ENOMSG 42 /* No message of desired type */ -#define EIDRM 43 /* Identifier removed */ -#define ECHRNG 44 /* Channel number out of range */ -#define EL2NSYNC 45 /* Level 2 not synchronized */ -#define EL3HLT 46 /* Level 3 halted */ -#define EL3RST 47 /* Level 3 reset */ -#define ELNRNG 48 /* Link number out of range */ -#define EUNATCH 49 /* Protocol driver not attached */ -#define ENOCSI 50 /* No CSI structure available */ -#define EL2HLT 51 /* Level 2 halted */ -#define EBADE 52 /* Invalid exchange */ -#define EBADR 53 /* Invalid request descriptor */ -#define EXFULL 54 /* Exchange full */ -#define ENOANO 55 /* No anode */ -#define EBADRQC 56 /* Invalid request code */ -#define EBADSLT 57 /* Invalid slot */ - -#define EDEADLOCK EDEADLK - -#define EBFONT 59 /* Bad font file format */ -#define ENOSTR 60 /* Device not a stream */ -#define ENODATA 61 /* No data available */ -#define ETIME 62 /* Timer expired */ -#define ENOSR 63 /* Out of streams resources */ -#define ENONET 64 /* Machine is not on the network */ -#define ENOPKG 65 /* Package not installed */ -#define EREMOTE 66 /* Object is remote */ -#define ENOLINK 67 /* Link has been severed */ -#define EADV 68 /* Advertise error */ -#define ESRMNT 69 /* Srmount error */ -#define ECOMM 70 /* Communication error on send */ -#define EPROTO 71 /* Protocol error */ -#define EMULTIHOP 72 /* Multihop attempted */ -#define EDOTDOT 73 /* RFS specific error */ -#define EBADMSG 74 /* Not a data message */ -#define EOVERFLOW 75 /* Value too large for defined data type */ -#define ENOTUNIQ 76 /* Name not unique on network */ -#define EBADFD 77 /* File descriptor in bad state */ -#define EREMCHG 78 /* Remote address changed */ -#define ELIBACC 79 /* Can not access a needed shared library */ -#define ELIBBAD 80 /* Accessing a corrupted shared library */ -#define ELIBSCN 81 /* .lib section in a.out corrupted */ -#define ELIBMAX 82 /* Attempting to link in too many shared libraries */ -#define ELIBEXEC 83 /* Cannot exec a shared library directly */ -#define EILSEQ 84 /* Illegal byte sequence */ -#define ERESTART 85 /* Interrupted system call should be restarted */ -#define ESTRPIPE 86 /* Streams pipe error */ -#define EUSERS 87 /* Too many users */ -#define ENOTSOCK 88 /* Socket operation on non-socket */ -#define EDESTADDRREQ 89 /* Destination address required */ -#define EMSGSIZE 90 /* Message too long */ -#define EPROTOTYPE 91 /* Protocol wrong type for socket */ -#define ENOPROTOOPT 92 /* Protocol not available */ -#define EPROTONOSUPPORT 93 /* Protocol not supported */ -#define ESOCKTNOSUPPORT 94 /* Socket type not supported */ -#define EOPNOTSUPP 95 /* Operation not supported on transport endpoint */ -#define EPFNOSUPPORT 96 /* Protocol family not supported */ -#define EAFNOSUPPORT 97 /* Address family not supported by protocol */ -#define EADDRINUSE 98 /* Address already in use */ -#define EADDRNOTAVAIL 99 /* Cannot assign requested address */ -#define ENETDOWN 100 /* Network is down */ -#define ENETUNREACH 101 /* Network is unreachable */ -#define ENETRESET 102 /* Network dropped connection because of reset */ -#define ECONNABORTED 103 /* Software caused connection abort */ -#define ECONNRESET 104 /* Connection reset by peer */ -#define ENOBUFS 105 /* No buffer space available */ -#define EISCONN 106 /* Transport endpoint is already connected */ -#define ENOTCONN 107 /* Transport endpoint is not connected */ -#define ESHUTDOWN 108 /* Cannot send after transport endpoint shutdown */ -#define ETOOMANYREFS 109 /* Too many references: cannot splice */ -#define ETIMEDOUT 110 /* Connection timed out */ -#define ECONNREFUSED 111 /* Connection refused */ -#define EHOSTDOWN 112 /* Host is down */ -#define EHOSTUNREACH 113 /* No route to host */ -#define EALREADY 114 /* Operation already in progress */ -#define EINPROGRESS 115 /* Operation now in progress */ -#define ESTALE 116 /* Stale NFS file handle */ -#define EUCLEAN 117 /* Structure needs cleaning */ -#define ENOTNAM 118 /* Not a XENIX named type file */ -#define ENAVAIL 119 /* No XENIX semaphores available */ -#define EISNAM 120 /* Is a named type file */ -#define EREMOTEIO 121 /* Remote I/O error */ -#define EDQUOT 122 /* Quota exceeded */ - -#define ENOMEDIUM 123 /* No medium found */ -#define EMEDIUMTYPE 124 /* Wrong medium type */ -#define ECANCELED 125 /* Operation Canceled */ -#define ENOKEY 126 /* Required key not available */ -#define EKEYEXPIRED 127 /* Key has expired */ -#define EKEYREVOKED 128 /* Key has been revoked */ -#define EKEYREJECTED 129 /* Key was rejected by service */ - -/* for robust mutexes */ -#define EOWNERDEAD 130 /* Owner died */ -#define ENOTRECOVERABLE 131 /* State not recoverable */ - -#endif diff --git a/doc/legacy/errno.list.macosx.txt b/doc/legacy/errno.list.macosx.txt deleted file mode 100644 index 4954e03d855..00000000000 --- a/doc/legacy/errno.list.macosx.txt +++ /dev/null @@ -1,1513 +0,0 @@ -/* Copyright 2000-2005 The Apache Software Foundation or its licensors, as - * applicable. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef APR_ERRNO_H -#define APR_ERRNO_H - -/** - * @file apr_errno.h - * @brief APR Error Codes - */ - -#include "apr.h" - -#if APR_HAVE_ERRNO_H -#include <errno.h> -#endif - -#ifdef __cplusplus -extern "C" { -#endif /* __cplusplus */ - -/** - * @defgroup apr_errno Error Codes - * @ingroup APR - * @{ - */ - -/** - * Type for specifying an error or status code. - */ -typedef int apr_status_t; - -/** - * Return a human readable string describing the specified error. - * @param statcode The error code the get a string for. - * @param buf A buffer to hold the error string. - * @param bufsize Size of the buffer to hold the string. - */ -APR_DECLARE(char *) apr_strerror(apr_status_t statcode, char *buf, - apr_size_t bufsize); - -#if defined(DOXYGEN) -/** - * @def APR_FROM_OS_ERROR(os_err_type syserr) - * Fold a platform specific error into an apr_status_t code. - * @return apr_status_t - * @param e The platform os error code. - * @warning macro implementation; the syserr argument may be evaluated - * multiple times. - */ -#define APR_FROM_OS_ERROR(e) (e == 0 ? APR_SUCCESS : e + APR_OS_START_SYSERR) - -/** - * @def APR_TO_OS_ERROR(apr_status_t statcode) - * @return os_err_type - * Fold an apr_status_t code back to the native platform defined error. - * @param e The apr_status_t folded platform os error code. - * @warning macro implementation; the statcode argument may be evaluated - * multiple times. If the statcode was not created by apr_get_os_error - * or APR_FROM_OS_ERROR, the results are undefined. - */ -#define APR_TO_OS_ERROR(e) (e == 0 ? APR_SUCCESS : e - APR_OS_START_SYSERR) - -/** @def apr_get_os_error() - * @return apr_status_t the last platform error, folded into apr_status_t, on most platforms - * @remark This retrieves errno, or calls a GetLastError() style function, and - * folds it with APR_FROM_OS_ERROR. Some platforms (such as OS2) have no - * such mechanism, so this call may be unsupported. Do NOT use this - * call for socket errors from socket, send, recv etc! - */ - -/** @def apr_set_os_error(e) - * Reset the last platform error, unfolded from an apr_status_t, on some platforms - * @param e The OS error folded in a prior call to APR_FROM_OS_ERROR() - * @warning This is a macro implementation; the statcode argument may be evaluated - * multiple times. If the statcode was not created by apr_get_os_error - * or APR_FROM_OS_ERROR, the results are undefined. This macro sets - * errno, or calls a SetLastError() style function, unfolding statcode - * with APR_TO_OS_ERROR. Some platforms (such as OS2) have no such - * mechanism, so this call may be unsupported. - */ - -/** @def apr_get_netos_error() - * Return the last socket error, folded into apr_status_t, on all platforms - * @remark This retrieves errno or calls a GetLastSocketError() style function, - * and folds it with APR_FROM_OS_ERROR. - */ - -/** @def apr_set_netos_error(e) - * Reset the last socket error, unfolded from an apr_status_t - * @param e The socket error folded in a prior call to APR_FROM_OS_ERROR() - * @warning This is a macro implementation; the statcode argument may be evaluated - * multiple times. If the statcode was not created by apr_get_os_error - * or APR_FROM_OS_ERROR, the results are undefined. This macro sets - * errno, or calls a WSASetLastError() style function, unfolding - * socketcode with APR_TO_OS_ERROR. - */ - -#endif /* defined(DOXYGEN) */ - -/** - * APR_OS_START_ERROR is where the APR specific error values start. - */ -#define APR_OS_START_ERROR 20000 -/** - * APR_OS_ERRSPACE_SIZE is the maximum number of errors you can fit - * into one of the error/status ranges below -- except for - * APR_OS_START_USERERR, which see. - */ -#define APR_OS_ERRSPACE_SIZE 50000 -/** - * APR_OS_START_STATUS is where the APR specific status codes start. - */ -#define APR_OS_START_STATUS (APR_OS_START_ERROR + APR_OS_ERRSPACE_SIZE) -/** - * APR_OS_START_USERERR are reserved for applications that use APR that - * layer their own error codes along with APR's. Note that the - * error immediately following this one is set ten times farther - * away than usual, so that users of apr have a lot of room in - * which to declare custom error codes. - */ -#define APR_OS_START_USERERR (APR_OS_START_STATUS + APR_OS_ERRSPACE_SIZE) -/** - * APR_OS_START_USEERR is obsolete, defined for compatibility only. - * Use APR_OS_START_USERERR instead. - */ -#define APR_OS_START_USEERR APR_OS_START_USERERR -/** - * APR_OS_START_CANONERR is where APR versions of errno values are defined - * on systems which don't have the corresponding errno. - */ -#define APR_OS_START_CANONERR (APR_OS_START_USERERR \ - + (APR_OS_ERRSPACE_SIZE * 10)) -/** - * APR_OS_START_EAIERR folds EAI_ error codes from getaddrinfo() into - * apr_status_t values. - */ -#define APR_OS_START_EAIERR (APR_OS_START_CANONERR + APR_OS_ERRSPACE_SIZE) -/** - * APR_OS_START_SYSERR folds platform-specific system error values into - * apr_status_t values. - */ -#define APR_OS_START_SYSERR (APR_OS_START_EAIERR + APR_OS_ERRSPACE_SIZE) - -/** no error. */ -#define APR_SUCCESS 0 - -/** - * @defgroup APR_Error APR Error Values - * <PRE> - * <b>APR ERROR VALUES</b> - * APR_ENOSTAT APR was unable to perform a stat on the file - * APR_ENOPOOL APR was not provided a pool with which to allocate memory - * APR_EBADDATE APR was given an invalid date - * APR_EINVALSOCK APR was given an invalid socket - * APR_ENOPROC APR was not given a process structure - * APR_ENOTIME APR was not given a time structure - * APR_ENODIR APR was not given a directory structure - * APR_ENOLOCK APR was not given a lock structure - * APR_ENOPOLL APR was not given a poll structure - * APR_ENOSOCKET APR was not given a socket - * APR_ENOTHREAD APR was not given a thread structure - * APR_ENOTHDKEY APR was not given a thread key structure - * APR_ENOSHMAVAIL There is no more shared memory available - * APR_EDSOOPEN APR was unable to open the dso object. For more - * information call apr_dso_error(). - * APR_EGENERAL General failure (specific information not available) - * APR_EBADIP The specified IP address is invalid - * APR_EBADMASK The specified netmask is invalid - * APR_ESYMNOTFOUND Could not find the requested symbol - * </PRE> - * - * <PRE> - * <b>APR STATUS VALUES</b> - * APR_INCHILD Program is currently executing in the child - * APR_INPARENT Program is currently executing in the parent - * APR_DETACH The thread is detached - * APR_NOTDETACH The thread is not detached - * APR_CHILD_DONE The child has finished executing - * APR_CHILD_NOTDONE The child has not finished executing - * APR_TIMEUP The operation did not finish before the timeout - * APR_INCOMPLETE The operation was incomplete although some processing - * was performed and the results are partially valid - * APR_BADCH Getopt found an option not in the option string - * APR_BADARG Getopt found an option that is missing an argument - * and an argument was specified in the option string - * APR_EOF APR has encountered the end of the file - * APR_NOTFOUND APR was unable to find the socket in the poll structure - * APR_ANONYMOUS APR is using anonymous shared memory - * APR_FILEBASED APR is using a file name as the key to the shared memory - * APR_KEYBASED APR is using a shared key as the key to the shared memory - * APR_EINIT Ininitalizer value. If no option has been found, but - * the status variable requires a value, this should be used - * APR_ENOTIMPL The APR function has not been implemented on this - * platform, either because nobody has gotten to it yet, - * or the function is impossible on this platform. - * APR_EMISMATCH Two passwords do not match. - * APR_EABSOLUTE The given path was absolute. - * APR_ERELATIVE The given path was relative. - * APR_EINCOMPLETE The given path was neither relative nor absolute. - * APR_EABOVEROOT The given path was above the root path. - * APR_EBUSY The given lock was busy. - * APR_EPROC_UNKNOWN The given process wasn't recognized by APR - * </PRE> - * @{ - */ -/** @see APR_STATUS_IS_ENOSTAT */ -#define APR_ENOSTAT (APR_OS_START_ERROR + 1) -/** @see APR_STATUS_IS_ENOPOOL */ -#define APR_ENOPOOL (APR_OS_START_ERROR + 2) -/* empty slot: +3 */ -/** @see APR_STATUS_IS_EBADDATE */ -#define APR_EBADDATE (APR_OS_START_ERROR + 4) -/** @see APR_STATUS_IS_EINVALSOCK */ -#define APR_EINVALSOCK (APR_OS_START_ERROR + 5) -/** @see APR_STATUS_IS_ENOPROC */ -#define APR_ENOPROC (APR_OS_START_ERROR + 6) -/** @see APR_STATUS_IS_ENOTIME */ -#define APR_ENOTIME (APR_OS_START_ERROR + 7) -/** @see APR_STATUS_IS_ENODIR */ -#define APR_ENODIR (APR_OS_START_ERROR + 8) -/** @see APR_STATUS_IS_ENOLOCK */ -#define APR_ENOLOCK (APR_OS_START_ERROR + 9) -/** @see APR_STATUS_IS_ENOPOLL */ -#define APR_ENOPOLL (APR_OS_START_ERROR + 10) -/** @see APR_STATUS_IS_ENOSOCKET */ -#define APR_ENOSOCKET (APR_OS_START_ERROR + 11) -/** @see APR_STATUS_IS_ENOTHREAD */ -#define APR_ENOTHREAD (APR_OS_START_ERROR + 12) -/** @see APR_STATUS_IS_ENOTHDKEY */ -#define APR_ENOTHDKEY (APR_OS_START_ERROR + 13) -/** @see APR_STATUS_IS_EGENERAL */ -#define APR_EGENERAL (APR_OS_START_ERROR + 14) -/** @see APR_STATUS_IS_ENOSHMAVAIL */ -#define APR_ENOSHMAVAIL (APR_OS_START_ERROR + 15) -/** @see APR_STATUS_IS_EBADIP */ -#define APR_EBADIP (APR_OS_START_ERROR + 16) -/** @see APR_STATUS_IS_EBADMASK */ -#define APR_EBADMASK (APR_OS_START_ERROR + 17) -/* empty slot: +18 */ -/** @see APR_STATUS_IS_EDSOPEN */ -#define APR_EDSOOPEN (APR_OS_START_ERROR + 19) -/** @see APR_STATUS_IS_EABSOLUTE */ -#define APR_EABSOLUTE (APR_OS_START_ERROR + 20) -/** @see APR_STATUS_IS_ERELATIVE */ -#define APR_ERELATIVE (APR_OS_START_ERROR + 21) -/** @see APR_STATUS_IS_EINCOMPLETE */ -#define APR_EINCOMPLETE (APR_OS_START_ERROR + 22) -/** @see APR_STATUS_IS_EABOVEROOT */ -#define APR_EABOVEROOT (APR_OS_START_ERROR + 23) -/** @see APR_STATUS_IS_EBADPATH */ -#define APR_EBADPATH (APR_OS_START_ERROR + 24) -/** @see APR_STATUS_IS_EPATHWILD */ -#define APR_EPATHWILD (APR_OS_START_ERROR + 25) -/** @see APR_STATUS_IS_ESYMNOTFOUND */ -#define APR_ESYMNOTFOUND (APR_OS_START_ERROR + 26) -/** @see APR_STATUS_IS_EPROC_UNKNOWN */ -#define APR_EPROC_UNKNOWN (APR_OS_START_ERROR + 27) -/** @see APR_STATUS_IS_ENOTENOUGHENTROPY */ -#define APR_ENOTENOUGHENTROPY (APR_OS_START_ERROR + 28) -/** @} */ - -/** - * @defgroup APR_STATUS_IS Status Value Tests - * @warning For any particular error condition, more than one of these tests - * may match. This is because platform-specific error codes may not - * always match the semantics of the POSIX codes these tests (and the - * corresponding APR error codes) are named after. A notable example - * are the APR_STATUS_IS_ENOENT and APR_STATUS_IS_ENOTDIR tests on - * Win32 platforms. The programmer should always be aware of this and - * adjust the order of the tests accordingly. - * @{ - */ -/** - * APR was unable to perform a stat on the file - * @warning always use this test, as platform-specific variances may meet this - * more than one error code - */ -#define APR_STATUS_IS_ENOSTAT(s) ((s) == APR_ENOSTAT) -/** - * APR was not provided a pool with which to allocate memory - * @warning always use this test, as platform-specific variances may meet this - * more than one error code - */ -#define APR_STATUS_IS_ENOPOOL(s) ((s) == APR_ENOPOOL) -/** APR was given an invalid date */ -#define APR_STATUS_IS_EBADDATE(s) ((s) == APR_EBADDATE) -/** APR was given an invalid socket */ -#define APR_STATUS_IS_EINVALSOCK(s) ((s) == APR_EINVALSOCK) -/** APR was not given a process structure */ -#define APR_STATUS_IS_ENOPROC(s) ((s) == APR_ENOPROC) -/** APR was not given a time structure */ -#define APR_STATUS_IS_ENOTIME(s) ((s) == APR_ENOTIME) -/** APR was not given a directory structure */ -#define APR_STATUS_IS_ENODIR(s) ((s) == APR_ENODIR) -/** APR was not given a lock structure */ -#define APR_STATUS_IS_ENOLOCK(s) ((s) == APR_ENOLOCK) -/** APR was not given a poll structure */ -#define APR_STATUS_IS_ENOPOLL(s) ((s) == APR_ENOPOLL) -/** APR was not given a socket */ -#define APR_STATUS_IS_ENOSOCKET(s) ((s) == APR_ENOSOCKET) -/** APR was not given a thread structure */ -#define APR_STATUS_IS_ENOTHREAD(s) ((s) == APR_ENOTHREAD) -/** APR was not given a thread key structure */ -#define APR_STATUS_IS_ENOTHDKEY(s) ((s) == APR_ENOTHDKEY) -/** Generic Error which can not be put into another spot */ -#define APR_STATUS_IS_EGENERAL(s) ((s) == APR_EGENERAL) -/** There is no more shared memory available */ -#define APR_STATUS_IS_ENOSHMAVAIL(s) ((s) == APR_ENOSHMAVAIL) -/** The specified IP address is invalid */ -#define APR_STATUS_IS_EBADIP(s) ((s) == APR_EBADIP) -/** The specified netmask is invalid */ -#define APR_STATUS_IS_EBADMASK(s) ((s) == APR_EBADMASK) -/* empty slot: +18 */ -/** - * APR was unable to open the dso object. - * For more information call apr_dso_error(). - */ -#if defined(WIN32) -#define APR_STATUS_IS_EDSOOPEN(s) ((s) == APR_EDSOOPEN \ - || APR_TO_OS_ERROR(s) == ERROR_MOD_NOT_FOUND) -#else -#define APR_STATUS_IS_EDSOOPEN(s) ((s) == APR_EDSOOPEN) -#endif -/** The given path was absolute. */ -#define APR_STATUS_IS_EABSOLUTE(s) ((s) == APR_EABSOLUTE) -/** The given path was relative. */ -#define APR_STATUS_IS_ERELATIVE(s) ((s) == APR_ERELATIVE) -/** The given path was neither relative nor absolute. */ -#define APR_STATUS_IS_EINCOMPLETE(s) ((s) == APR_EINCOMPLETE) -/** The given path was above the root path. */ -#define APR_STATUS_IS_EABOVEROOT(s) ((s) == APR_EABOVEROOT) -/** The given path was bad. */ -#define APR_STATUS_IS_EBADPATH(s) ((s) == APR_EBADPATH) -/** The given path contained wildcards. */ -#define APR_STATUS_IS_EPATHWILD(s) ((s) == APR_EPATHWILD) -/** Could not find the requested symbol. - * For more information call apr_dso_error(). - */ -#if defined(WIN32) -#define APR_STATUS_IS_ESYMNOTFOUND(s) ((s) == APR_ESYMNOTFOUND \ - || APR_TO_OS_ERROR(s) == ERROR_PROC_NOT_FOUND) -#else -#define APR_STATUS_IS_ESYMNOTFOUND(s) ((s) == APR_ESYMNOTFOUND) -#endif -/** The given process was not recognized by APR. */ -#define APR_STATUS_IS_EPROC_UNKNOWN(s) ((s) == APR_EPROC_UNKNOWN) - -/** APR could not gather enough entropy to continue. */ -#define APR_STATUS_IS_ENOTENOUGHENTROPY(s) ((s) == APR_ENOTENOUGHENTROPY) - -/** @} */ - -/** - * @addtogroup APR_Error - * @{ - */ -/** @see APR_STATUS_IS_INCHILD */ -#define APR_INCHILD (APR_OS_START_STATUS + 1) -/** @see APR_STATUS_IS_INPARENT */ -#define APR_INPARENT (APR_OS_START_STATUS + 2) -/** @see APR_STATUS_IS_DETACH */ -#define APR_DETACH (APR_OS_START_STATUS + 3) -/** @see APR_STATUS_IS_NOTDETACH */ -#define APR_NOTDETACH (APR_OS_START_STATUS + 4) -/** @see APR_STATUS_IS_CHILD_DONE */ -#define APR_CHILD_DONE (APR_OS_START_STATUS + 5) -/** @see APR_STATUS_IS_CHILD_NOTDONE */ -#define APR_CHILD_NOTDONE (APR_OS_START_STATUS + 6) -/** @see APR_STATUS_IS_TIMEUP */ -#define APR_TIMEUP (APR_OS_START_STATUS + 7) -/** @see APR_STATUS_IS_INCOMPLETE */ -#define APR_INCOMPLETE (APR_OS_START_STATUS + 8) -/* empty slot: +9 */ -/* empty slot: +10 */ -/* empty slot: +11 */ -/** @see APR_STATUS_IS_BADCH */ -#define APR_BADCH (APR_OS_START_STATUS + 12) -/** @see APR_STATUS_IS_BADARG */ -#define APR_BADARG (APR_OS_START_STATUS + 13) -/** @see APR_STATUS_IS_EOF */ -#define APR_EOF (APR_OS_START_STATUS + 14) -/** @see APR_STATUS_IS_NOTFOUND */ -#define APR_NOTFOUND (APR_OS_START_STATUS + 15) -/* empty slot: +16 */ -/* empty slot: +17 */ -/* empty slot: +18 */ -/** @see APR_STATUS_IS_ANONYMOUS */ -#define APR_ANONYMOUS (APR_OS_START_STATUS + 19) -/** @see APR_STATUS_IS_FILEBASED */ -#define APR_FILEBASED (APR_OS_START_STATUS + 20) -/** @see APR_STATUS_IS_KEYBASED */ -#define APR_KEYBASED (APR_OS_START_STATUS + 21) -/** @see APR_STATUS_IS_EINIT */ -#define APR_EINIT (APR_OS_START_STATUS + 22) -/** @see APR_STATUS_IS_ENOTIMPL */ -#define APR_ENOTIMPL (APR_OS_START_STATUS + 23) -/** @see APR_STATUS_IS_EMISMATCH */ -#define APR_EMISMATCH (APR_OS_START_STATUS + 24) -/** @see APR_STATUS_IS_EBUSY */ -#define APR_EBUSY (APR_OS_START_STATUS + 25) -/** @} */ - -/** - * @addtogroup APR_STATUS_IS - * @{ - */ -/** - * Program is currently executing in the child - * @warning - * always use this test, as platform-specific variances may meet this - * more than one error code */ -#define APR_STATUS_IS_INCHILD(s) ((s) == APR_INCHILD) -/** - * Program is currently executing in the parent - * @warning - * always use this test, as platform-specific variances may meet this - * more than one error code - */ -#define APR_STATUS_IS_INPARENT(s) ((s) == APR_INPARENT) -/** - * The thread is detached - * @warning - * always use this test, as platform-specific variances may meet this - * more than one error code - */ -#define APR_STATUS_IS_DETACH(s) ((s) == APR_DETACH) -/** - * The thread is not detached - * @warning - * always use this test, as platform-specific variances may meet this - * more than one error code - */ -#define APR_STATUS_IS_NOTDETACH(s) ((s) == APR_NOTDETACH) -/** - * The child has finished executing - * @warning - * always use this test, as platform-specific variances may meet this - * more than one error code - */ -#define APR_STATUS_IS_CHILD_DONE(s) ((s) == APR_CHILD_DONE) -/** - * The child has not finished executing - * @warning - * always use this test, as platform-specific variances may meet this - * more than one error code - */ -#define APR_STATUS_IS_CHILD_NOTDONE(s) ((s) == APR_CHILD_NOTDONE) -/** - * The operation did not finish before the timeout - * @warning - * always use this test, as platform-specific variances may meet this - * more than one error code - */ -#define APR_STATUS_IS_TIMEUP(s) ((s) == APR_TIMEUP) -/** - * The operation was incomplete although some processing was performed - * and the results are partially valid. - * @warning - * always use this test, as platform-specific variances may meet this - * more than one error code - */ -#define APR_STATUS_IS_INCOMPLETE(s) ((s) == APR_INCOMPLETE) -/* empty slot: +9 */ -/* empty slot: +10 */ -/* empty slot: +11 */ -/** - * Getopt found an option not in the option string - * @warning - * always use this test, as platform-specific variances may meet this - * more than one error code - */ -#define APR_STATUS_IS_BADCH(s) ((s) == APR_BADCH) -/** - * Getopt found an option not in the option string and an argument was - * specified in the option string - * @warning - * always use this test, as platform-specific variances may meet this - * more than one error code - */ -#define APR_STATUS_IS_BADARG(s) ((s) == APR_BADARG) -/** - * APR has encountered the end of the file - * @warning - * always use this test, as platform-specific variances may meet this - * more than one error code - */ -#define APR_STATUS_IS_EOF(s) ((s) == APR_EOF) -/** - * APR was unable to find the socket in the poll structure - * @warning - * always use this test, as platform-specific variances may meet this - * more than one error code - */ -#define APR_STATUS_IS_NOTFOUND(s) ((s) == APR_NOTFOUND) -/* empty slot: +16 */ -/* empty slot: +17 */ -/* empty slot: +18 */ -/** - * APR is using anonymous shared memory - * @warning - * always use this test, as platform-specific variances may meet this - * more than one error code - */ -#define APR_STATUS_IS_ANONYMOUS(s) ((s) == APR_ANONYMOUS) -/** - * APR is using a file name as the key to the shared memory - * @warning - * always use this test, as platform-specific variances may meet this - * more than one error code - */ -#define APR_STATUS_IS_FILEBASED(s) ((s) == APR_FILEBASED) -/** - * APR is using a shared key as the key to the shared memory - * @warning - * always use this test, as platform-specific variances may meet this - * more than one error code - */ -#define APR_STATUS_IS_KEYBASED(s) ((s) == APR_KEYBASED) -/** - * Ininitalizer value. If no option has been found, but - * the status variable requires a value, this should be used - * @warning - * always use this test, as platform-specific variances may meet this - * more than one error code - */ -#define APR_STATUS_IS_EINIT(s) ((s) == APR_EINIT) -/** - * The APR function has not been implemented on this - * platform, either because nobody has gotten to it yet, - * or the function is impossible on this platform. - * @warning - * always use this test, as platform-specific variances may meet this - * more than one error code - */ -#define APR_STATUS_IS_ENOTIMPL(s) ((s) == APR_ENOTIMPL) -/** - * Two passwords do not match. - * @warning - * always use this test, as platform-specific variances may meet this - * more than one error code - */ -#define APR_STATUS_IS_EMISMATCH(s) ((s) == APR_EMISMATCH) -/** - * The given lock was busy - * @warning always use this test, as platform-specific variances may meet this - * more than one error code - */ -#define APR_STATUS_IS_EBUSY(s) ((s) == APR_EBUSY) - -/** @} */ - -/** - * @addtogroup APR_Error APR Error Values - * @{ - */ -/* APR CANONICAL ERROR VALUES */ -/** @see APR_STATUS_IS_EACCES */ -#ifdef EACCES -#define APR_EACCES EACCES -#else -#define APR_EACCES (APR_OS_START_CANONERR + 1) -#endif - -/** @see APR_STATUS_IS_EXIST */ -#ifdef EEXIST -#define APR_EEXIST EEXIST -#else -#define APR_EEXIST (APR_OS_START_CANONERR + 2) -#endif - -/** @see APR_STATUS_IS_ENAMETOOLONG */ -#ifdef ENAMETOOLONG -#define APR_ENAMETOOLONG ENAMETOOLONG -#else -#define APR_ENAMETOOLONG (APR_OS_START_CANONERR + 3) -#endif - -/** @see APR_STATUS_IS_ENOENT */ -#ifdef ENOENT -#define APR_ENOENT ENOENT -#else -#define APR_ENOENT (APR_OS_START_CANONERR + 4) -#endif - -/** @see APR_STATUS_IS_ENOTDIR */ -#ifdef ENOTDIR -#define APR_ENOTDIR ENOTDIR -#else -#define APR_ENOTDIR (APR_OS_START_CANONERR + 5) -#endif - -/** @see APR_STATUS_IS_ENOSPC */ -#ifdef ENOSPC -#define APR_ENOSPC ENOSPC -#else -#define APR_ENOSPC (APR_OS_START_CANONERR + 6) -#endif - -/** @see APR_STATUS_IS_ENOMEM */ -#ifdef ENOMEM -#define APR_ENOMEM ENOMEM -#else -#define APR_ENOMEM (APR_OS_START_CANONERR + 7) -#endif - -/** @see APR_STATUS_IS_EMFILE */ -#ifdef EMFILE -#define APR_EMFILE EMFILE -#else -#define APR_EMFILE (APR_OS_START_CANONERR + 8) -#endif - -/** @see APR_STATUS_IS_ENFILE */ -#ifdef ENFILE -#define APR_ENFILE ENFILE -#else -#define APR_ENFILE (APR_OS_START_CANONERR + 9) -#endif - -/** @see APR_STATUS_IS_EBADF */ -#ifdef EBADF -#define APR_EBADF EBADF -#else -#define APR_EBADF (APR_OS_START_CANONERR + 10) -#endif - -/** @see APR_STATUS_IS_EINVAL */ -#ifdef EINVAL -#define APR_EINVAL EINVAL -#else -#define APR_EINVAL (APR_OS_START_CANONERR + 11) -#endif - -/** @see APR_STATUS_IS_ESPIPE */ -#ifdef ESPIPE -#define APR_ESPIPE ESPIPE -#else -#define APR_ESPIPE (APR_OS_START_CANONERR + 12) -#endif - -/** - * @see APR_STATUS_IS_EAGAIN - * @warning use APR_STATUS_IS_EAGAIN instead of just testing this value - */ -#ifdef EAGAIN -#define APR_EAGAIN EAGAIN -#elif defined(EWOULDBLOCK) -#define APR_EAGAIN EWOULDBLOCK -#else -#define APR_EAGAIN (APR_OS_START_CANONERR + 13) -#endif - -/** @see APR_STATUS_IS_EINTR */ -#ifdef EINTR -#define APR_EINTR EINTR -#else -#define APR_EINTR (APR_OS_START_CANONERR + 14) -#endif - -/** @see APR_STATUS_IS_ENOTSOCK */ -#ifdef ENOTSOCK -#define APR_ENOTSOCK ENOTSOCK -#else -#define APR_ENOTSOCK (APR_OS_START_CANONERR + 15) -#endif - -/** @see APR_STATUS_IS_ECONNREFUSED */ -#ifdef ECONNREFUSED -#define APR_ECONNREFUSED ECONNREFUSED -#else -#define APR_ECONNREFUSED (APR_OS_START_CANONERR + 16) -#endif - -/** @see APR_STATUS_IS_EINPROGRESS */ -#ifdef EINPROGRESS -#define APR_EINPROGRESS EINPROGRESS -#else -#define APR_EINPROGRESS (APR_OS_START_CANONERR + 17) -#endif - -/** - * @see APR_STATUS_IS_ECONNABORTED - * @warning use APR_STATUS_IS_ECONNABORTED instead of just testing this value - */ - -#ifdef ECONNABORTED -#define APR_ECONNABORTED ECONNABORTED -#else -#define APR_ECONNABORTED (APR_OS_START_CANONERR + 18) -#endif - -/** @see APR_STATUS_IS_ECONNRESET */ -#ifdef ECONNRESET -#define APR_ECONNRESET ECONNRESET -#else -#define APR_ECONNRESET (APR_OS_START_CANONERR + 19) -#endif - -/** @see APR_STATUS_IS_ETIMEDOUT - * @deprecated */ -#ifdef ETIMEDOUT -#define APR_ETIMEDOUT ETIMEDOUT -#else -#define APR_ETIMEDOUT (APR_OS_START_CANONERR + 20) -#endif - -/** @see APR_STATUS_IS_EHOSTUNREACH */ -#ifdef EHOSTUNREACH -#define APR_EHOSTUNREACH EHOSTUNREACH -#else -#define APR_EHOSTUNREACH (APR_OS_START_CANONERR + 21) -#endif - -/** @see APR_STATUS_IS_ENETUNREACH */ -#ifdef ENETUNREACH -#define APR_ENETUNREACH ENETUNREACH -#else -#define APR_ENETUNREACH (APR_OS_START_CANONERR + 22) -#endif - -/** @see APR_STATUS_IS_EFTYPE */ -#ifdef EFTYPE -#define APR_EFTYPE EFTYPE -#else -#define APR_EFTYPE (APR_OS_START_CANONERR + 23) -#endif - -/** @see APR_STATUS_IS_EPIPE */ -#ifdef EPIPE -#define APR_EPIPE EPIPE -#else -#define APR_EPIPE (APR_OS_START_CANONERR + 24) -#endif - -/** @see APR_STATUS_IS_EXDEV */ -#ifdef EXDEV -#define APR_EXDEV EXDEV -#else -#define APR_EXDEV (APR_OS_START_CANONERR + 25) -#endif - -/** @see APR_STATUS_IS_ENOTEMPTY */ -#ifdef ENOTEMPTY -#define APR_ENOTEMPTY ENOTEMPTY -#else -#define APR_ENOTEMPTY (APR_OS_START_CANONERR + 26) -#endif - -/** @} */ - -#if defined(OS2) && !defined(DOXYGEN) - -#define APR_FROM_OS_ERROR(e) (e == 0 ? APR_SUCCESS : e + APR_OS_START_SYSERR) -#define APR_TO_OS_ERROR(e) (e == 0 ? APR_SUCCESS : e - APR_OS_START_SYSERR) - -#define INCL_DOSERRORS -#define INCL_DOS - -/* Leave these undefined. - * OS2 doesn't rely on the errno concept. - * The API calls always return a result codes which - * should be filtered through APR_FROM_OS_ERROR(). - * - * #define apr_get_os_error() (APR_FROM_OS_ERROR(GetLastError())) - * #define apr_set_os_error(e) (SetLastError(APR_TO_OS_ERROR(e))) - */ - -/* A special case, only socket calls require this; - */ -#define apr_get_netos_error() (APR_FROM_OS_ERROR(errno)) -#define apr_set_netos_error(e) (errno = APR_TO_OS_ERROR(e)) - -/* And this needs to be greped away for good: - */ -#define APR_OS2_STATUS(e) (APR_FROM_OS_ERROR(e)) - -/* These can't sit in a private header, so in spite of the extra size, - * they need to be made available here. - */ -#define SOCBASEERR 10000 -#define SOCEPERM (SOCBASEERR+1) /* Not owner */ -#define SOCESRCH (SOCBASEERR+3) /* No such process */ -#define SOCEINTR (SOCBASEERR+4) /* Interrupted system call */ -#define SOCENXIO (SOCBASEERR+6) /* No such device or address */ -#define SOCEBADF (SOCBASEERR+9) /* Bad file number */ -#define SOCEACCES (SOCBASEERR+13) /* Permission denied */ -#define SOCEFAULT (SOCBASEERR+14) /* Bad address */ -#define SOCEINVAL (SOCBASEERR+22) /* Invalid argument */ -#define SOCEMFILE (SOCBASEERR+24) /* Too many open files */ -#define SOCEPIPE (SOCBASEERR+32) /* Broken pipe */ -#define SOCEOS2ERR (SOCBASEERR+100) /* OS/2 Error */ -#define SOCEWOULDBLOCK (SOCBASEERR+35) /* Operation would block */ -#define SOCEINPROGRESS (SOCBASEERR+36) /* Operation now in progress */ -#define SOCEALREADY (SOCBASEERR+37) /* Operation already in progress */ -#define SOCENOTSOCK (SOCBASEERR+38) /* Socket operation on non-socket */ -#define SOCEDESTADDRREQ (SOCBASEERR+39) /* Destination address required */ -#define SOCEMSGSIZE (SOCBASEERR+40) /* Message too long */ -#define SOCEPROTOTYPE (SOCBASEERR+41) /* Protocol wrong type for socket */ -#define SOCENOPROTOOPT (SOCBASEERR+42) /* Protocol not available */ -#define SOCEPROTONOSUPPORT (SOCBASEERR+43) /* Protocol not supported */ -#define SOCESOCKTNOSUPPORT (SOCBASEERR+44) /* Socket type not supported */ -#define SOCEOPNOTSUPP (SOCBASEERR+45) /* Operation not supported on socket */ -#define SOCEPFNOSUPPORT (SOCBASEERR+46) /* Protocol family not supported */ -#define SOCEAFNOSUPPORT (SOCBASEERR+47) /* Address family not supported by protocol family */ -#define SOCEADDRINUSE (SOCBASEERR+48) /* Address already in use */ -#define SOCEADDRNOTAVAIL (SOCBASEERR+49) /* Can't assign requested address */ -#define SOCENETDOWN (SOCBASEERR+50) /* Network is down */ -#define SOCENETUNREACH (SOCBASEERR+51) /* Network is unreachable */ -#define SOCENETRESET (SOCBASEERR+52) /* Network dropped connection on reset */ -#define SOCECONNABORTED (SOCBASEERR+53) /* Software caused connection abort */ -#define SOCECONNRESET (SOCBASEERR+54) /* Connection reset by peer */ -#define SOCENOBUFS (SOCBASEERR+55) /* No buffer space available */ -#define SOCEISCONN (SOCBASEERR+56) /* Socket is already connected */ -#define SOCENOTCONN (SOCBASEERR+57) /* Socket is not connected */ -#define SOCESHUTDOWN (SOCBASEERR+58) /* Can't send after socket shutdown */ -#define SOCETOOMANYREFS (SOCBASEERR+59) /* Too many references: can't splice */ -#define SOCETIMEDOUT (SOCBASEERR+60) /* Connection timed out */ -#define SOCECONNREFUSED (SOCBASEERR+61) /* Connection refused */ -#define SOCELOOP (SOCBASEERR+62) /* Too many levels of symbolic links */ -#define SOCENAMETOOLONG (SOCBASEERR+63) /* File name too long */ -#define SOCEHOSTDOWN (SOCBASEERR+64) /* Host is down */ -#define SOCEHOSTUNREACH (SOCBASEERR+65) /* No route to host */ -#define SOCENOTEMPTY (SOCBASEERR+66) /* Directory not empty */ - -/* APR CANONICAL ERROR TESTS */ -#define APR_STATUS_IS_EACCES(s) ((s) == APR_EACCES \ - || (s) == APR_OS_START_SYSERR + ERROR_ACCESS_DENIED \ - || (s) == APR_OS_START_SYSERR + ERROR_SHARING_VIOLATION) -#define APR_STATUS_IS_EEXIST(s) ((s) == APR_EEXIST \ - || (s) == APR_OS_START_SYSERR + ERROR_OPEN_FAILED \ - || (s) == APR_OS_START_SYSERR + ERROR_FILE_EXISTS \ - || (s) == APR_OS_START_SYSERR + ERROR_ALREADY_EXISTS \ - || (s) == APR_OS_START_SYSERR + ERROR_ACCESS_DENIED) -#define APR_STATUS_IS_ENAMETOOLONG(s) ((s) == APR_ENAMETOOLONG \ - || (s) == APR_OS_START_SYSERR + ERROR_FILENAME_EXCED_RANGE \ - || (s) == APR_OS_START_SYSERR + SOCENAMETOOLONG) -#define APR_STATUS_IS_ENOENT(s) ((s) == APR_ENOENT \ - || (s) == APR_OS_START_SYSERR + ERROR_FILE_NOT_FOUND \ - || (s) == APR_OS_START_SYSERR + ERROR_PATH_NOT_FOUND \ - || (s) == APR_OS_START_SYSERR + ERROR_NO_MORE_FILES \ - || (s) == APR_OS_START_SYSERR + ERROR_OPEN_FAILED) -#define APR_STATUS_IS_ENOTDIR(s) ((s) == APR_ENOTDIR) -#define APR_STATUS_IS_ENOSPC(s) ((s) == APR_ENOSPC \ - || (s) == APR_OS_START_SYSERR + ERROR_DISK_FULL) -#define APR_STATUS_IS_ENOMEM(s) ((s) == APR_ENOMEM) -#define APR_STATUS_IS_EMFILE(s) ((s) == APR_EMFILE \ - || (s) == APR_OS_START_SYSERR + ERROR_TOO_MANY_OPEN_FILES) -#define APR_STATUS_IS_ENFILE(s) ((s) == APR_ENFILE) -#define APR_STATUS_IS_EBADF(s) ((s) == APR_EBADF \ - || (s) == APR_OS_START_SYSERR + ERROR_INVALID_HANDLE) -#define APR_STATUS_IS_EINVAL(s) ((s) == APR_EINVAL \ - || (s) == APR_OS_START_SYSERR + ERROR_INVALID_PARAMETER \ - || (s) == APR_OS_START_SYSERR + ERROR_INVALID_FUNCTION) -#define APR_STATUS_IS_ESPIPE(s) ((s) == APR_ESPIPE \ - || (s) == APR_OS_START_SYSERR + ERROR_NEGATIVE_SEEK) -#define APR_STATUS_IS_EAGAIN(s) ((s) == APR_EAGAIN \ - || (s) == APR_OS_START_SYSERR + ERROR_NO_DATA \ - || (s) == APR_OS_START_SYSERR + SOCEWOULDBLOCK \ - || (s) == APR_OS_START_SYSERR + ERROR_LOCK_VIOLATION) -#define APR_STATUS_IS_EINTR(s) ((s) == APR_EINTR \ - || (s) == APR_OS_START_SYSERR + SOCEINTR) -#define APR_STATUS_IS_ENOTSOCK(s) ((s) == APR_ENOTSOCK \ - || (s) == APR_OS_START_SYSERR + SOCENOTSOCK) -#define APR_STATUS_IS_ECONNREFUSED(s) ((s) == APR_ECONNREFUSED \ - || (s) == APR_OS_START_SYSERR + SOCECONNREFUSED) -#define APR_STATUS_IS_EINPROGRESS(s) ((s) == APR_EINPROGRESS \ - || (s) == APR_OS_START_SYSERR + SOCEINPROGRESS) -#define APR_STATUS_IS_ECONNABORTED(s) ((s) == APR_ECONNABORTED \ - || (s) == APR_OS_START_SYSERR + SOCECONNABORTED) -#define APR_STATUS_IS_ECONNRESET(s) ((s) == APR_ECONNRESET \ - || (s) == APR_OS_START_SYSERR + SOCECONNRESET) -/* XXX deprecated */ -#define APR_STATUS_IS_ETIMEDOUT(s) ((s) == APR_ETIMEDOUT \ - || (s) == APR_OS_START_SYSERR + SOCETIMEDOUT) -#undef APR_STATUS_IS_TIMEUP -#define APR_STATUS_IS_TIMEUP(s) ((s) == APR_TIMEUP \ - || (s) == APR_OS_START_SYSERR + SOCETIMEDOUT) -#define APR_STATUS_IS_EHOSTUNREACH(s) ((s) == APR_EHOSTUNREACH \ - || (s) == APR_OS_START_SYSERR + SOCEHOSTUNREACH) -#define APR_STATUS_IS_ENETUNREACH(s) ((s) == APR_ENETUNREACH \ - || (s) == APR_OS_START_SYSERR + SOCENETUNREACH) -#define APR_STATUS_IS_EFTYPE(s) ((s) == APR_EFTYPE) -#define APR_STATUS_IS_EPIPE(s) ((s) == APR_EPIPE \ - || (s) == APR_OS_START_SYSERR + ERROR_BROKEN_PIPE \ - || (s) == APR_OS_START_SYSERR + SOCEPIPE) -#define APR_STATUS_IS_EXDEV(s) ((s) == APR_EXDEV \ - || (s) == APR_OS_START_SYSERR + ERROR_NOT_SAME_DEVICE) -#define APR_STATUS_IS_ENOTEMPTY(s) ((s) == APR_ENOTEMPTY \ - || (s) == APR_OS_START_SYSERR + ERROR_DIR_NOT_EMPTY \ - || (s) == APR_OS_START_SYSERR + ERROR_ACCESS_DENIED) - -/* - Sorry, too tired to wrap this up for OS2... feel free to - fit the following into their best matches. - - { ERROR_NO_SIGNAL_SENT, ESRCH }, - { SOCEALREADY, EALREADY }, - { SOCEDESTADDRREQ, EDESTADDRREQ }, - { SOCEMSGSIZE, EMSGSIZE }, - { SOCEPROTOTYPE, EPROTOTYPE }, - { SOCENOPROTOOPT, ENOPROTOOPT }, - { SOCEPROTONOSUPPORT, EPROTONOSUPPORT }, - { SOCESOCKTNOSUPPORT, ESOCKTNOSUPPORT }, - { SOCEOPNOTSUPP, EOPNOTSUPP }, - { SOCEPFNOSUPPORT, EPFNOSUPPORT }, - { SOCEAFNOSUPPORT, EAFNOSUPPORT }, - { SOCEADDRINUSE, EADDRINUSE }, - { SOCEADDRNOTAVAIL, EADDRNOTAVAIL }, - { SOCENETDOWN, ENETDOWN }, - { SOCENETRESET, ENETRESET }, - { SOCENOBUFS, ENOBUFS }, - { SOCEISCONN, EISCONN }, - { SOCENOTCONN, ENOTCONN }, - { SOCESHUTDOWN, ESHUTDOWN }, - { SOCETOOMANYREFS, ETOOMANYREFS }, - { SOCELOOP, ELOOP }, - { SOCEHOSTDOWN, EHOSTDOWN }, - { SOCENOTEMPTY, ENOTEMPTY }, - { SOCEPIPE, EPIPE } -*/ - -#elif defined(WIN32) && !defined(DOXYGEN) /* !defined(OS2) */ - -#define APR_FROM_OS_ERROR(e) (e == 0 ? APR_SUCCESS : e + APR_OS_START_SYSERR) -#define APR_TO_OS_ERROR(e) (e == 0 ? APR_SUCCESS : e - APR_OS_START_SYSERR) - -#define apr_get_os_error() (APR_FROM_OS_ERROR(GetLastError())) -#define apr_set_os_error(e) (SetLastError(APR_TO_OS_ERROR(e))) - -/* A special case, only socket calls require this: - */ -#define apr_get_netos_error() (APR_FROM_OS_ERROR(WSAGetLastError())) -#define apr_set_netos_error(e) (WSASetLastError(APR_TO_OS_ERROR(e))) - -/* APR CANONICAL ERROR TESTS */ -#define APR_STATUS_IS_EACCES(s) ((s) == APR_EACCES \ - || (s) == APR_OS_START_SYSERR + ERROR_ACCESS_DENIED \ - || (s) == APR_OS_START_SYSERR + ERROR_CANNOT_MAKE \ - || (s) == APR_OS_START_SYSERR + ERROR_CURRENT_DIRECTORY \ - || (s) == APR_OS_START_SYSERR + ERROR_DRIVE_LOCKED \ - || (s) == APR_OS_START_SYSERR + ERROR_FAIL_I24 \ - || (s) == APR_OS_START_SYSERR + ERROR_LOCK_VIOLATION \ - || (s) == APR_OS_START_SYSERR + ERROR_LOCK_FAILED \ - || (s) == APR_OS_START_SYSERR + ERROR_NOT_LOCKED \ - || (s) == APR_OS_START_SYSERR + ERROR_NETWORK_ACCESS_DENIED \ - || (s) == APR_OS_START_SYSERR + ERROR_SHARING_VIOLATION) -#define APR_STATUS_IS_EEXIST(s) ((s) == APR_EEXIST \ - || (s) == APR_OS_START_SYSERR + ERROR_FILE_EXISTS \ - || (s) == APR_OS_START_SYSERR + ERROR_ALREADY_EXISTS) -#define APR_STATUS_IS_ENAMETOOLONG(s) ((s) == APR_ENAMETOOLONG \ - || (s) == APR_OS_START_SYSERR + ERROR_FILENAME_EXCED_RANGE \ - || (s) == APR_OS_START_SYSERR + WSAENAMETOOLONG) -#define APR_STATUS_IS_ENOENT(s) ((s) == APR_ENOENT \ - || (s) == APR_OS_START_SYSERR + ERROR_FILE_NOT_FOUND \ - || (s) == APR_OS_START_SYSERR + ERROR_PATH_NOT_FOUND \ - || (s) == APR_OS_START_SYSERR + ERROR_OPEN_FAILED \ - || (s) == APR_OS_START_SYSERR + ERROR_NO_MORE_FILES) -#define APR_STATUS_IS_ENOTDIR(s) ((s) == APR_ENOTDIR \ - || (s) == APR_OS_START_SYSERR + ERROR_PATH_NOT_FOUND \ - || (s) == APR_OS_START_SYSERR + ERROR_BAD_NETPATH \ - || (s) == APR_OS_START_SYSERR + ERROR_BAD_NET_NAME \ - || (s) == APR_OS_START_SYSERR + ERROR_BAD_PATHNAME \ - || (s) == APR_OS_START_SYSERR + ERROR_INVALID_DRIVE) -#define APR_STATUS_IS_ENOSPC(s) ((s) == APR_ENOSPC \ - || (s) == APR_OS_START_SYSERR + ERROR_DISK_FULL) -#define APR_STATUS_IS_ENOMEM(s) ((s) == APR_ENOMEM \ - || (s) == APR_OS_START_SYSERR + ERROR_ARENA_TRASHED \ - || (s) == APR_OS_START_SYSERR + ERROR_NOT_ENOUGH_MEMORY \ - || (s) == APR_OS_START_SYSERR + ERROR_INVALID_BLOCK \ - || (s) == APR_OS_START_SYSERR + ERROR_NOT_ENOUGH_QUOTA \ - || (s) == APR_OS_START_SYSERR + ERROR_OUTOFMEMORY) -#define APR_STATUS_IS_EMFILE(s) ((s) == APR_EMFILE \ - || (s) == APR_OS_START_SYSERR + ERROR_TOO_MANY_OPEN_FILES) -#define APR_STATUS_IS_ENFILE(s) ((s) == APR_ENFILE) -#define APR_STATUS_IS_EBADF(s) ((s) == APR_EBADF \ - || (s) == APR_OS_START_SYSERR + ERROR_INVALID_HANDLE \ - || (s) == APR_OS_START_SYSERR + ERROR_INVALID_TARGET_HANDLE) -#define APR_STATUS_IS_EINVAL(s) ((s) == APR_EINVAL \ - || (s) == APR_OS_START_SYSERR + ERROR_INVALID_ACCESS \ - || (s) == APR_OS_START_SYSERR + ERROR_INVALID_DATA \ - || (s) == APR_OS_START_SYSERR + ERROR_INVALID_FUNCTION \ - || (s) == APR_OS_START_SYSERR + ERROR_INVALID_HANDLE \ - || (s) == APR_OS_START_SYSERR + ERROR_INVALID_PARAMETER \ - || (s) == APR_OS_START_SYSERR + ERROR_NEGATIVE_SEEK) -#define APR_STATUS_IS_ESPIPE(s) ((s) == APR_ESPIPE \ - || (s) == APR_OS_START_SYSERR + ERROR_SEEK_ON_DEVICE \ - || (s) == APR_OS_START_SYSERR + ERROR_NEGATIVE_SEEK) -#define APR_STATUS_IS_EAGAIN(s) ((s) == APR_EAGAIN \ - || (s) == APR_OS_START_SYSERR + ERROR_NO_DATA \ - || (s) == APR_OS_START_SYSERR + ERROR_NO_PROC_SLOTS \ - || (s) == APR_OS_START_SYSERR + ERROR_NESTING_NOT_ALLOWED \ - || (s) == APR_OS_START_SYSERR + ERROR_MAX_THRDS_REACHED \ - || (s) == APR_OS_START_SYSERR + ERROR_LOCK_VIOLATION \ - || (s) == APR_OS_START_SYSERR + WSAEWOULDBLOCK) -#define APR_STATUS_IS_EINTR(s) ((s) == APR_EINTR \ - || (s) == APR_OS_START_SYSERR + WSAEINTR) -#define APR_STATUS_IS_ENOTSOCK(s) ((s) == APR_ENOTSOCK \ - || (s) == APR_OS_START_SYSERR + WSAENOTSOCK) -#define APR_STATUS_IS_ECONNREFUSED(s) ((s) == APR_ECONNREFUSED \ - || (s) == APR_OS_START_SYSERR + WSAECONNREFUSED) -#define APR_STATUS_IS_EINPROGRESS(s) ((s) == APR_EINPROGRESS \ - || (s) == APR_OS_START_SYSERR + WSAEINPROGRESS) -#define APR_STATUS_IS_ECONNABORTED(s) ((s) == APR_ECONNABORTED \ - || (s) == APR_OS_START_SYSERR + WSAECONNABORTED) -#define APR_STATUS_IS_ECONNRESET(s) ((s) == APR_ECONNRESET \ - || (s) == APR_OS_START_SYSERR + ERROR_NETNAME_DELETED \ - || (s) == APR_OS_START_SYSERR + WSAECONNRESET) -/* XXX deprecated */ -#define APR_STATUS_IS_ETIMEDOUT(s) ((s) == APR_ETIMEDOUT \ - || (s) == APR_OS_START_SYSERR + WSAETIMEDOUT \ - || (s) == APR_OS_START_SYSERR + WAIT_TIMEOUT) -#undef APR_STATUS_IS_TIMEUP -#define APR_STATUS_IS_TIMEUP(s) ((s) == APR_TIMEUP \ - || (s) == APR_OS_START_SYSERR + WSAETIMEDOUT \ - || (s) == APR_OS_START_SYSERR + WAIT_TIMEOUT) -#define APR_STATUS_IS_EHOSTUNREACH(s) ((s) == APR_EHOSTUNREACH \ - || (s) == APR_OS_START_SYSERR + WSAEHOSTUNREACH) -#define APR_STATUS_IS_ENETUNREACH(s) ((s) == APR_ENETUNREACH \ - || (s) == APR_OS_START_SYSERR + WSAENETUNREACH) -#define APR_STATUS_IS_EFTYPE(s) ((s) == APR_EFTYPE \ - || (s) == APR_OS_START_SYSERR + ERROR_EXE_MACHINE_TYPE_MISMATCH \ - || (s) == APR_OS_START_SYSERR + ERROR_INVALID_DLL \ - || (s) == APR_OS_START_SYSERR + ERROR_INVALID_MODULETYPE \ - || (s) == APR_OS_START_SYSERR + ERROR_BAD_EXE_FORMAT \ - || (s) == APR_OS_START_SYSERR + ERROR_INVALID_EXE_SIGNATURE \ - || (s) == APR_OS_START_SYSERR + ERROR_FILE_CORRUPT \ - || (s) == APR_OS_START_SYSERR + ERROR_BAD_FORMAT) -#define APR_STATUS_IS_EPIPE(s) ((s) == APR_EPIPE \ - || (s) == APR_OS_START_SYSERR + ERROR_BROKEN_PIPE) -#define APR_STATUS_IS_EXDEV(s) ((s) == APR_EXDEV \ - || (s) == APR_OS_START_SYSERR + ERROR_NOT_SAME_DEVICE) -#define APR_STATUS_IS_ENOTEMPTY(s) ((s) == APR_ENOTEMPTY \ - || (s) == APR_OS_START_SYSERR + ERROR_DIR_NOT_EMPTY) - -#elif defined(NETWARE) && defined(USE_WINSOCK) && !defined(DOXYGEN) /* !defined(OS2) && !defined(WIN32) */ - -#define APR_FROM_OS_ERROR(e) (e == 0 ? APR_SUCCESS : e + APR_OS_START_SYSERR) -#define APR_TO_OS_ERROR(e) (e == 0 ? APR_SUCCESS : e - APR_OS_START_SYSERR) - -#define apr_get_os_error() (errno) -#define apr_set_os_error(e) (errno = (e)) - -/* A special case, only socket calls require this: */ -#define apr_get_netos_error() (APR_FROM_OS_ERROR(WSAGetLastError())) -#define apr_set_netos_error(e) (WSASetLastError(APR_TO_OS_ERROR(e))) - -/* APR CANONICAL ERROR TESTS */ -#define APR_STATUS_IS_EACCES(s) ((s) == APR_EACCES) -#define APR_STATUS_IS_EEXIST(s) ((s) == APR_EEXIST) -#define APR_STATUS_IS_ENAMETOOLONG(s) ((s) == APR_ENAMETOOLONG) -#define APR_STATUS_IS_ENOENT(s) ((s) == APR_ENOENT) -#define APR_STATUS_IS_ENOTDIR(s) ((s) == APR_ENOTDIR) -#define APR_STATUS_IS_ENOSPC(s) ((s) == APR_ENOSPC) -#define APR_STATUS_IS_ENOMEM(s) ((s) == APR_ENOMEM) -#define APR_STATUS_IS_EMFILE(s) ((s) == APR_EMFILE) -#define APR_STATUS_IS_ENFILE(s) ((s) == APR_ENFILE) -#define APR_STATUS_IS_EBADF(s) ((s) == APR_EBADF) -#define APR_STATUS_IS_EINVAL(s) ((s) == APR_EINVAL) -#define APR_STATUS_IS_ESPIPE(s) ((s) == APR_ESPIPE) - -#define APR_STATUS_IS_EAGAIN(s) ((s) == APR_EAGAIN \ - || (s) == EWOULDBLOCK \ - || (s) == APR_OS_START_SYSERR + WSAEWOULDBLOCK) -#define APR_STATUS_IS_EINTR(s) ((s) == APR_EINTR \ - || (s) == APR_OS_START_SYSERR + WSAEINTR) -#define APR_STATUS_IS_ENOTSOCK(s) ((s) == APR_ENOTSOCK \ - || (s) == APR_OS_START_SYSERR + WSAENOTSOCK) -#define APR_STATUS_IS_ECONNREFUSED(s) ((s) == APR_ECONNREFUSED \ - || (s) == APR_OS_START_SYSERR + WSAECONNREFUSED) -#define APR_STATUS_IS_EINPROGRESS(s) ((s) == APR_EINPROGRESS \ - || (s) == APR_OS_START_SYSERR + WSAEINPROGRESS) -#define APR_STATUS_IS_ECONNABORTED(s) ((s) == APR_ECONNABORTED \ - || (s) == APR_OS_START_SYSERR + WSAECONNABORTED) -#define APR_STATUS_IS_ECONNRESET(s) ((s) == APR_ECONNRESET \ - || (s) == APR_OS_START_SYSERR + WSAECONNRESET) -/* XXX deprecated */ -#define APR_STATUS_IS_ETIMEDOUT(s) ((s) == APR_ETIMEDOUT \ - || (s) == APR_OS_START_SYSERR + WSAETIMEDOUT \ - || (s) == APR_OS_START_SYSERR + WAIT_TIMEOUT) -#undef APR_STATUS_IS_TIMEUP -#define APR_STATUS_IS_TIMEUP(s) ((s) == APR_TIMEUP \ - || (s) == APR_OS_START_SYSERR + WSAETIMEDOUT \ - || (s) == APR_OS_START_SYSERR + WAIT_TIMEOUT) -#define APR_STATUS_IS_EHOSTUNREACH(s) ((s) == APR_EHOSTUNREACH \ - || (s) == APR_OS_START_SYSERR + WSAEHOSTUNREACH) -#define APR_STATUS_IS_ENETUNREACH(s) ((s) == APR_ENETUNREACH \ - || (s) == APR_OS_START_SYSERR + WSAENETUNREACH) -#define APR_STATUS_IS_ENETDOWN(s) ((s) == APR_OS_START_SYSERR + WSAENETDOWN) -#define APR_STATUS_IS_EFTYPE(s) ((s) == APR_EFTYPE) -#define APR_STATUS_IS_EPIPE(s) ((s) == APR_EPIPE) -#define APR_STATUS_IS_EXDEV(s) ((s) == APR_EXDEV) -#define APR_STATUS_IS_ENOTEMPTY(s) ((s) == APR_ENOTEMPTY) - -#else /* !defined(NETWARE) && !defined(OS2) && !defined(WIN32) */ - -/* - * os error codes are clib error codes - */ -#define APR_FROM_OS_ERROR(e) (e) -#define APR_TO_OS_ERROR(e) (e) - -#define apr_get_os_error() (errno) -#define apr_set_os_error(e) (errno = (e)) - -/* A special case, only socket calls require this: - */ -#define apr_get_netos_error() (errno) -#define apr_set_netos_error(e) (errno = (e)) - -/** - * @addtogroup APR_STATUS_IS - * @{ - */ - -/** permission denied */ -#define APR_STATUS_IS_EACCES(s) ((s) == APR_EACCES) -/** file exists */ -#define APR_STATUS_IS_EEXIST(s) ((s) == APR_EEXIST) -/** path name is too long */ -#define APR_STATUS_IS_ENAMETOOLONG(s) ((s) == APR_ENAMETOOLONG) -/** - * no such file or directory - * @remark - * EMVSCATLG can be returned by the automounter on z/OS for - * paths which do not exist. - */ -#ifdef EMVSCATLG -#define APR_STATUS_IS_ENOENT(s) ((s) == APR_ENOENT \ - || (s) == EMVSCATLG) -#else -#define APR_STATUS_IS_ENOENT(s) ((s) == APR_ENOENT) -#endif -/** not a directory */ -#define APR_STATUS_IS_ENOTDIR(s) ((s) == APR_ENOTDIR) -/** no space left on device */ -#ifdef EDQUOT -#define APR_STATUS_IS_ENOSPC(s) ((s) == APR_ENOSPC \ - || (s) == EDQUOT) -#else -#define APR_STATUS_IS_ENOSPC(s) ((s) == APR_ENOSPC) -#endif -/** not enough memory */ -#define APR_STATUS_IS_ENOMEM(s) ((s) == APR_ENOMEM) -/** too many open files */ -#define APR_STATUS_IS_EMFILE(s) ((s) == APR_EMFILE) -/** file table overflow */ -#define APR_STATUS_IS_ENFILE(s) ((s) == APR_ENFILE) -/** bad file # */ -#define APR_STATUS_IS_EBADF(s) ((s) == APR_EBADF) -/** invalid argument */ -#define APR_STATUS_IS_EINVAL(s) ((s) == APR_EINVAL) -/** illegal seek */ -#define APR_STATUS_IS_ESPIPE(s) ((s) == APR_ESPIPE) - -/** operation would block */ -#if !defined(EWOULDBLOCK) || !defined(EAGAIN) -#define APR_STATUS_IS_EAGAIN(s) ((s) == APR_EAGAIN) -#elif (EWOULDBLOCK == EAGAIN) -#define APR_STATUS_IS_EAGAIN(s) ((s) == APR_EAGAIN) -#else -#define APR_STATUS_IS_EAGAIN(s) ((s) == APR_EAGAIN \ - || (s) == EWOULDBLOCK) -#endif - -/** interrupted system call */ -#define APR_STATUS_IS_EINTR(s) ((s) == APR_EINTR) -/** socket operation on a non-socket */ -#define APR_STATUS_IS_ENOTSOCK(s) ((s) == APR_ENOTSOCK) -/** Connection Refused */ -#define APR_STATUS_IS_ECONNREFUSED(s) ((s) == APR_ECONNREFUSED) -/** operation now in progress */ -#define APR_STATUS_IS_EINPROGRESS(s) ((s) == APR_EINPROGRESS) - -/** - * Software caused connection abort - * @remark - * EPROTO on certain older kernels really means ECONNABORTED, so we need to - * ignore it for them. See discussion in new-httpd archives nh.9701 & nh.9603 - * - * There is potentially a bug in Solaris 2.x x<6, and other boxes that - * implement tcp sockets in userland (i.e. on top of STREAMS). On these - * systems, EPROTO can actually result in a fatal loop. See PR#981 for - * example. It's hard to handle both uses of EPROTO. - */ -#ifdef EPROTO -#define APR_STATUS_IS_ECONNABORTED(s) ((s) == APR_ECONNABORTED \ - || (s) == EPROTO) -#else -#define APR_STATUS_IS_ECONNABORTED(s) ((s) == APR_ECONNABORTED) -#endif - -/** Connection Reset by peer */ -#define APR_STATUS_IS_ECONNRESET(s) ((s) == APR_ECONNRESET) -/** Operation timed out - * @deprecated */ -#define APR_STATUS_IS_ETIMEDOUT(s) ((s) == APR_ETIMEDOUT) -/** no route to host */ -#define APR_STATUS_IS_EHOSTUNREACH(s) ((s) == APR_EHOSTUNREACH) -/** network is unreachable */ -#define APR_STATUS_IS_ENETUNREACH(s) ((s) == APR_ENETUNREACH) -/** inappropiate file type or format */ -#define APR_STATUS_IS_EFTYPE(s) ((s) == APR_EFTYPE) -/** broken pipe */ -#define APR_STATUS_IS_EPIPE(s) ((s) == APR_EPIPE) -/** cross device link */ -#define APR_STATUS_IS_EXDEV(s) ((s) == APR_EXDEV) -/** Directory Not Empty */ -#define APR_STATUS_IS_ENOTEMPTY(s) ((s) == APR_ENOTEMPTY || \ - (s) == APR_EEXIST) -/** @} */ - -#endif /* !defined(NETWARE) && !defined(OS2) && !defined(WIN32) */ - -/** @} */ - -#ifdef __cplusplus -} -#endif - -#endif /* ! APR_ERRNO_H */ -/* - * Copyright (c) 2000 Apple Computer, Inc. All rights reserved. - * - * @APPLE_LICENSE_HEADER_START@ - * - * This file contains Original Code and/or Modifications of Original Code - * as defined in and that are subject to the Apple Public Source License - * Version 2.0 (the 'License'). You may not use this file except in - * compliance with the License. Please obtain a copy of the License at - * http://www.opensource.apple.com/apsl/ and read it before using this - * file. - * - * The Original Code and all software distributed under the License are - * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER - * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, - * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. - * Please see the License for the specific language governing rights and - * limitations under the License. - * - * @APPLE_LICENSE_HEADER_END@ - */ -#include <sys/errno.h> - - -/* - * Copyright (c) 2000-2002 Apple Computer, Inc. All rights reserved. - * - * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ - * - * This file contains Original Code and/or Modifications of Original Code - * as defined in and that are subject to the Apple Public Source License - * Version 2.0 (the 'License'). You may not use this file except in - * compliance with the License. The rights granted to you under the License - * may not be used to create, or enable the creation or redistribution of, - * unlawful or unlicensed copies of an Apple operating system, or to - * circumvent, violate, or enable the circumvention or violation of, any - * terms of an Apple operating system software license agreement. - * - * Please obtain a copy of the License at - * http://www.opensource.apple.com/apsl/ and read it before using this file. - * - * The Original Code and all software distributed under the License are - * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER - * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, - * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. - * Please see the License for the specific language governing rights and - * limitations under the License. - * - * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ - */ -/* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */ -/* - * Copyright (c) 1982, 1986, 1989, 1993 - * The Regents of the University of California. All rights reserved. - * (c) UNIX System Laboratories, Inc. - * All or some portions of this file are derived from material licensed - * to the University of California by American Telephone and Telegraph - * Co. or Unix System Laboratories, Inc. and are reproduced herein with - * the permission of UNIX System Laboratories, Inc. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * 3. All advertising materials mentioning features or use of this software - * must display the following acknowledgement: - * This product includes software developed by the University of - * California, Berkeley and its contributors. - * 4. Neither the name of the University nor the names of its contributors - * may be used to endorse or promote products derived from this software - * without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - * - * @(#)errno.h 8.5 (Berkeley) 1/21/94 - */ - -#ifndef _SYS_ERRNO_H_ -#define _SYS_ERRNO_H_ - -#include <sys/cdefs.h> -__BEGIN_DECLS -extern int * __error(void); -#define errno (*__error()) -__END_DECLS - -/* - * Error codes - */ - -#define EPERM 1 /* Operation not permitted */ -#define ENOENT 2 /* No such file or directory */ -#define ESRCH 3 /* No such process */ -#define EINTR 4 /* Interrupted system call */ -#define EIO 5 /* Input/output error */ -#define ENXIO 6 /* Device not configured */ -#define E2BIG 7 /* Argument list too long */ -#define ENOEXEC 8 /* Exec format error */ -#define EBADF 9 /* Bad file descriptor */ -#define ECHILD 10 /* No child processes */ -#define EDEADLK 11 /* Resource deadlock avoided */ - /* 11 was EAGAIN */ -#define ENOMEM 12 /* Cannot allocate memory */ -#define EACCES 13 /* Permission denied */ -#define EFAULT 14 /* Bad address */ -#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE) -#define ENOTBLK 15 /* Block device required */ -#endif -#define EBUSY 16 /* Device / Resource busy */ -#define EEXIST 17 /* File exists */ -#define EXDEV 18 /* Cross-device link */ -#define ENODEV 19 /* Operation not supported by device */ -#define ENOTDIR 20 /* Not a directory */ -#define EISDIR 21 /* Is a directory */ -#define EINVAL 22 /* Invalid argument */ -#define ENFILE 23 /* Too many open files in system */ -#define EMFILE 24 /* Too many open files */ -#define ENOTTY 25 /* Inappropriate ioctl for device */ -#define ETXTBSY 26 /* Text file busy */ -#define EFBIG 27 /* File too large */ -#define ENOSPC 28 /* No space left on device */ -#define ESPIPE 29 /* Illegal seek */ -#define EROFS 30 /* Read-only file system */ -#define EMLINK 31 /* Too many links */ -#define EPIPE 32 /* Broken pipe */ - -/* math software */ -#define EDOM 33 /* Numerical argument out of domain */ -#define ERANGE 34 /* Result too large */ - -/* non-blocking and interrupt i/o */ -#define EAGAIN 35 /* Resource temporarily unavailable */ -#define EWOULDBLOCK EAGAIN /* Operation would block */ -#define EINPROGRESS 36 /* Operation now in progress */ -#define EALREADY 37 /* Operation already in progress */ - -/* ipc/network software -- argument errors */ -#define ENOTSOCK 38 /* Socket operation on non-socket */ -#define EDESTADDRREQ 39 /* Destination address required */ -#define EMSGSIZE 40 /* Message too long */ -#define EPROTOTYPE 41 /* Protocol wrong type for socket */ -#define ENOPROTOOPT 42 /* Protocol not available */ -#define EPROTONOSUPPORT 43 /* Protocol not supported */ -#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE) -#define ESOCKTNOSUPPORT 44 /* Socket type not supported */ -#endif /* (!_POSIX_C_SOURCE || _DARWIN_C_SOURCE) */ -#define ENOTSUP 45 /* Operation not supported */ -#if !__DARWIN_UNIX03 && !defined(KERNEL) -/* - * This is the same for binary and source copmpatability, unless compiling - * the kernel itself, or compiling __DARWIN_UNIX03; if compiling for the - * kernel, the correct value will be returned. If compiling non-POSIX - * source, the kernel return value will be converted by a stub in libc, and - * if compiling source with __DARWIN_UNIX03, the conversion in libc is not - * done, and the caller gets the expected (discrete) value. - */ -#define EOPNOTSUPP ENOTSUP /* Operation not supported on socket */ -#endif /* !__DARWIN_UNIX03 && !KERNEL */ - -#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE) -#define EPFNOSUPPORT 46 /* Protocol family not supported */ -#endif /* (_POSIX_C_SOURCE && !_DARWIN_C_SOURCE) */ -#define EAFNOSUPPORT 47 /* Address family not supported by protocol family */ -#define EADDRINUSE 48 /* Address already in use */ -#define EADDRNOTAVAIL 49 /* Can't assign requested address */ - -/* ipc/network software -- operational errors */ -#define ENETDOWN 50 /* Network is down */ -#define ENETUNREACH 51 /* Network is unreachable */ -#define ENETRESET 52 /* Network dropped connection on reset */ -#define ECONNABORTED 53 /* Software caused connection abort */ -#define ECONNRESET 54 /* Connection reset by peer */ -#define ENOBUFS 55 /* No buffer space available */ -#define EISCONN 56 /* Socket is already connected */ -#define ENOTCONN 57 /* Socket is not connected */ -#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE) -#define ESHUTDOWN 58 /* Can't send after socket shutdown */ -#define ETOOMANYREFS 59 /* Too many references: can't splice */ -#endif /* (_POSIX_C_SOURCE && !_DARWIN_C_SOURCE) */ -#define ETIMEDOUT 60 /* Operation timed out */ -#define ECONNREFUSED 61 /* Connection refused */ - -#define ELOOP 62 /* Too many levels of symbolic links */ -#define ENAMETOOLONG 63 /* File name too long */ - -/* should be rearranged */ -#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE) -#define EHOSTDOWN 64 /* Host is down */ -#endif /* (_POSIX_C_SOURCE && !_DARWIN_C_SOURCE) */ -#define EHOSTUNREACH 65 /* No route to host */ -#define ENOTEMPTY 66 /* Directory not empty */ - -/* quotas & mush */ -#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE) -#define EPROCLIM 67 /* Too many processes */ -#define EUSERS 68 /* Too many users */ -#endif /* (_POSIX_C_SOURCE && !_DARWIN_C_SOURCE) */ -#define EDQUOT 69 /* Disc quota exceeded */ - -/* Network File System */ -#define ESTALE 70 /* Stale NFS file handle */ -#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE) -#define EREMOTE 71 /* Too many levels of remote in path */ -#define EBADRPC 72 /* RPC struct is bad */ -#define ERPCMISMATCH 73 /* RPC version wrong */ -#define EPROGUNAVAIL 74 /* RPC prog. not avail */ -#define EPROGMISMATCH 75 /* Program version wrong */ -#define EPROCUNAVAIL 76 /* Bad procedure for program */ -#endif /* (_POSIX_C_SOURCE && !_DARWIN_C_SOURCE) */ - -#define ENOLCK 77 /* No locks available */ -#define ENOSYS 78 /* Function not implemented */ - -#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE) -#define EFTYPE 79 /* Inappropriate file type or format */ -#define EAUTH 80 /* Authentication error */ -#define ENEEDAUTH 81 /* Need authenticator */ - -/* Intelligent device errors */ -#define EPWROFF 82 /* Device power is off */ -#define EDEVERR 83 /* Device error, e.g. paper out */ -#endif /* (_POSIX_C_SOURCE && !_DARWIN_C_SOURCE) */ - -#define EOVERFLOW 84 /* Value too large to be stored in data type */ - -/* Program loading errors */ -#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE) -#define EBADEXEC 85 /* Bad executable */ -#define EBADARCH 86 /* Bad CPU type in executable */ -#define ESHLIBVERS 87 /* Shared library version mismatch */ -#define EBADMACHO 88 /* Malformed Macho file */ -#endif /* (_POSIX_C_SOURCE && !_DARWIN_C_SOURCE) */ - -#define ECANCELED 89 /* Operation canceled */ - -#define EIDRM 90 /* Identifier removed */ -#define ENOMSG 91 /* No message of desired type */ -#define EILSEQ 92 /* Illegal byte sequence */ -#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE) -#define ENOATTR 93 /* Attribute not found */ -#endif /* (_POSIX_C_SOURCE && !_DARWIN_C_SOURCE) */ - -#define EBADMSG 94 /* Bad message */ -#define EMULTIHOP 95 /* Reserved */ -#define ENODATA 96 /* No message available on STREAM */ -#define ENOLINK 97 /* Reserved */ -#define ENOSR 98 /* No STREAM resources */ -#define ENOSTR 99 /* Not a STREAM */ -#define EPROTO 100 /* Protocol error */ -#define ETIME 101 /* STREAM ioctl timeout */ - -#if __DARWIN_UNIX03 || defined(KERNEL) -/* This value is only discrete when compiling __DARWIN_UNIX03, or KERNEL */ -#define EOPNOTSUPP 102 /* Operation not supported on socket */ -#endif /* __DARWIN_UNIX03 || KERNEL */ - -#define ENOPOLICY 103 /* No such policy registered */ - -#if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE) -#define ELAST 103 /* Must be equal largest errno */ -#endif /* (_POSIX_C_SOURCE && !_DARWIN_C_SOURCE) */ - -#endif /* _SYS_ERRNO_H_ */ diff --git a/doc/legacy/errno.list.solaris.txt b/doc/legacy/errno.list.solaris.txt deleted file mode 100644 index 23601e9d374..00000000000 --- a/doc/legacy/errno.list.solaris.txt +++ /dev/null @@ -1,206 +0,0 @@ -/* - * CDDL HEADER START - * - * The contents of this file are subject to the terms of the - * Common Development and Distribution License, Version 1.0 only - * (the "License"). You may not use this file except in compliance - * with the License. - * - * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE - * or http://www.opensolaris.org/os/licensing. - * See the License for the specific language governing permissions - * and limitations under the License. - * - * When distributing Covered Code, include this CDDL HEADER in each - * file and include the License file at usr/src/OPENSOLARIS.LICENSE. - * If applicable, add the following below this CDDL HEADER, with the - * fields enclosed by brackets "[]" replaced with your own identifying - * information: Portions Copyright [yyyy] [name of copyright owner] - * - * CDDL HEADER END - */ -/* - * Copyright 2000 Sun Microsystems, Inc. All rights reserved. - * Use is subject to license terms. - */ - -/* Copyright (c) 1983, 1984, 1985, 1986, 1987, 1988, 1989 AT&T */ -/* All Rights Reserved */ - -/* - * University Copyright- Copyright (c) 1982, 1986, 1988 - * The Regents of the University of California - * All Rights Reserved - * - * University Acknowledgment- Portions of this document are derived from - * software developed by the University of California, Berkeley, and its - * contributors. - */ - -#ifndef _SYS_ERRNO_H -#define _SYS_ERRNO_H - -#pragma ident "@(#)errno.h 1.22 05/06/08 SMI" - -#ifdef __cplusplus -extern "C" { -#endif - -/* - * Error codes - */ - -#define EPERM 1 /* Not super-user */ -#define ENOENT 2 /* No such file or directory */ -#define ESRCH 3 /* No such process */ -#define EINTR 4 /* interrupted system call */ -#define EIO 5 /* I/O error */ -#define ENXIO 6 /* No such device or address */ -#define E2BIG 7 /* Arg list too long */ -#define ENOEXEC 8 /* Exec format error */ -#define EBADF 9 /* Bad file number */ -#define ECHILD 10 /* No children */ -#define EAGAIN 11 /* Resource temporarily unavailable */ -#define ENOMEM 12 /* Not enough core */ -#define EACCES 13 /* Permission denied */ -#define EFAULT 14 /* Bad address */ -#define ENOTBLK 15 /* Block device required */ -#define EBUSY 16 /* Mount device busy */ -#define EEXIST 17 /* File exists */ -#define EXDEV 18 /* Cross-device link */ -#define ENODEV 19 /* No such device */ -#define ENOTDIR 20 /* Not a directory */ -#define EISDIR 21 /* Is a directory */ -#define EINVAL 22 /* Invalid argument */ -#define ENFILE 23 /* File table overflow */ -#define EMFILE 24 /* Too many open files */ -#define ENOTTY 25 /* Inappropriate ioctl for device */ -#define ETXTBSY 26 /* Text file busy */ -#define EFBIG 27 /* File too large */ -#define ENOSPC 28 /* No space left on device */ -#define ESPIPE 29 /* Illegal seek */ -#define EROFS 30 /* Read only file system */ -#define EMLINK 31 /* Too many links */ -#define EPIPE 32 /* Broken pipe */ -#define EDOM 33 /* Math arg out of domain of func */ -#define ERANGE 34 /* Math result not representable */ -#define ENOMSG 35 /* No message of desired type */ -#define EIDRM 36 /* Identifier removed */ -#define ECHRNG 37 /* Channel number out of range */ -#define EL2NSYNC 38 /* Level 2 not synchronized */ -#define EL3HLT 39 /* Level 3 halted */ -#define EL3RST 40 /* Level 3 reset */ -#define ELNRNG 41 /* Link number out of range */ -#define EUNATCH 42 /* Protocol driver not attached */ -#define ENOCSI 43 /* No CSI structure available */ -#define EL2HLT 44 /* Level 2 halted */ -#define EDEADLK 45 /* Deadlock condition. */ -#define ENOLCK 46 /* No record locks available. */ -#define ECANCELED 47 /* Operation canceled */ -#define ENOTSUP 48 /* Operation not supported */ - -/* Filesystem Quotas */ -#define EDQUOT 49 /* Disc quota exceeded */ - -/* Convergent Error Returns */ -#define EBADE 50 /* invalid exchange */ -#define EBADR 51 /* invalid request descriptor */ -#define EXFULL 52 /* exchange full */ -#define ENOANO 53 /* no anode */ -#define EBADRQC 54 /* invalid request code */ -#define EBADSLT 55 /* invalid slot */ -#define EDEADLOCK 56 /* file locking deadlock error */ - -#define EBFONT 57 /* bad font file fmt */ - -/* Interprocess Robust Locks */ -#define EOWNERDEAD 58 /* process died with the lock */ -#define ENOTRECOVERABLE 59 /* lock is not recoverable */ - -/* stream problems */ -#define ENOSTR 60 /* Device not a stream */ -#define ENODATA 61 /* no data (for no delay io) */ -#define ETIME 62 /* timer expired */ -#define ENOSR 63 /* out of streams resources */ - -#define ENONET 64 /* Machine is not on the network */ -#define ENOPKG 65 /* Package not installed */ -#define EREMOTE 66 /* The object is remote */ -#define ENOLINK 67 /* the link has been severed */ -#define EADV 68 /* advertise error */ -#define ESRMNT 69 /* srmount error */ - -#define ECOMM 70 /* Communication error on send */ -#define EPROTO 71 /* Protocol error */ - -/* Interprocess Robust Locks */ -#define ELOCKUNMAPPED 72 /* locked lock was unmapped */ - -#define ENOTACTIVE 73 /* Facility is not active */ -#define EMULTIHOP 74 /* multihop attempted */ -#define EBADMSG 77 /* trying to read unreadable message */ -#define ENAMETOOLONG 78 /* path name is too long */ -#define EOVERFLOW 79 /* value too large to be stored in data type */ -#define ENOTUNIQ 80 /* given log. name not unique */ -#define EBADFD 81 /* f.d. invalid for this operation */ -#define EREMCHG 82 /* Remote address changed */ - -/* shared library problems */ -#define ELIBACC 83 /* Can't access a needed shared lib. */ -#define ELIBBAD 84 /* Accessing a corrupted shared lib. */ -#define ELIBSCN 85 /* .lib section in a.out corrupted. */ -#define ELIBMAX 86 /* Attempting to link in too many libs. */ -#define ELIBEXEC 87 /* Attempting to exec a shared library. */ -#define EILSEQ 88 /* Illegal byte sequence. */ -#define ENOSYS 89 /* Unsupported file system operation */ -#define ELOOP 90 /* Symbolic link loop */ -#define ERESTART 91 /* Restartable system call */ -#define ESTRPIPE 92 /* if pipe/FIFO, don't sleep in stream head */ -#define ENOTEMPTY 93 /* directory not empty */ -#define EUSERS 94 /* Too many users (for UFS) */ - -/* BSD Networking Software */ - /* argument errors */ -#define ENOTSOCK 95 /* Socket operation on non-socket */ -#define EDESTADDRREQ 96 /* Destination address required */ -#define EMSGSIZE 97 /* Message too long */ -#define EPROTOTYPE 98 /* Protocol wrong type for socket */ -#define ENOPROTOOPT 99 /* Protocol not available */ -#define EPROTONOSUPPORT 120 /* Protocol not supported */ -#define ESOCKTNOSUPPORT 121 /* Socket type not supported */ -#define EOPNOTSUPP 122 /* Operation not supported on socket */ -#define EPFNOSUPPORT 123 /* Protocol family not supported */ -#define EAFNOSUPPORT 124 /* Address family not supported by */ - /* protocol family */ -#define EADDRINUSE 125 /* Address already in use */ -#define EADDRNOTAVAIL 126 /* Can't assign requested address */ - /* operational errors */ -#define ENETDOWN 127 /* Network is down */ -#define ENETUNREACH 128 /* Network is unreachable */ -#define ENETRESET 129 /* Network dropped connection because */ - /* of reset */ -#define ECONNABORTED 130 /* Software caused connection abort */ -#define ECONNRESET 131 /* Connection reset by peer */ -#define ENOBUFS 132 /* No buffer space available */ -#define EISCONN 133 /* Socket is already connected */ -#define ENOTCONN 134 /* Socket is not connected */ -/* XENIX has 135 - 142 */ -#define ESHUTDOWN 143 /* Can't send after socket shutdown */ -#define ETOOMANYREFS 144 /* Too many references: can't splice */ -#define ETIMEDOUT 145 /* Connection timed out */ -#define ECONNREFUSED 146 /* Connection refused */ -#define EHOSTDOWN 147 /* Host is down */ -#define EHOSTUNREACH 148 /* No route to host */ -#define EWOULDBLOCK EAGAIN -#define EALREADY 149 /* operation already in progress */ -#define EINPROGRESS 150 /* operation now in progress */ - -/* SUN Network File System */ -#define ESTALE 151 /* Stale NFS file handle */ - - -#ifdef __cplusplus -} -#endif - -#endif /* _SYS_ERRNO_H */ diff --git a/doc/legacy/get_put_api_using_xattr.txt b/doc/legacy/get_put_api_using_xattr.txt deleted file mode 100644 index 243f9f1aec2..00000000000 --- a/doc/legacy/get_put_api_using_xattr.txt +++ /dev/null @@ -1,22 +0,0 @@ -GlusterFS get/put API interface provided through extended attributes: - -API usage: - int put(dirpath/filename, data): setfattr -n glusterfs.file.<filename> -v <data> <dirpath> - void *get(dirpath/filename): getfattr -n glusterfs.file.<filename> <dirpath> - - -internals: -* unify handling setxattr/getxattr - - setxattr - unify's setxattr forwards setxattr call to all the child nodes with XATTR_REPLACE flag, except namespace. setxattr will succeeds only on the child node on which the file already exists. if the setxattr operation fails on all child nodes, it indicates that the file does not already exist on any of the child nodes. unify follows the same rules as it follows for create, but using setxattr call itself with XATTR_CREATE flag. unify sends a setxattr to namespace first, with zero length data. if namespace setxattr succeeds, unify schedules setxattr to one of the child nodes. - - - getxattr - unify's getxattr forwards getxattr call to all the child nodes. wait for completion of operation on all the child nodes, and returns success if getxattr succeeded one child node. - -* posix handling setxattr/getxattr - - setxattr - posix setxattr does a open with O_CREAT|O_TRUNC on the <path>/<name>, writes value of the setxattr as data into the file and closes the file. when data is null, posix setxattr avoids doing write. file is closed after write. - - - getxattr - posix getxattr does open with O_RDONLY on the <path>/<name>, reads the complete content of the file. file is closed after read. - diff --git a/doc/legacy/hacker-guide/Makefile.am b/doc/legacy/hacker-guide/Makefile.am deleted file mode 100644 index 65c92ac235e..00000000000 --- a/doc/legacy/hacker-guide/Makefile.am +++ /dev/null @@ -1,8 +0,0 @@ -EXTRA_DIST = replicate.txt bdb.txt posix.txt call-stub.txt write-behind.txt - -#EXTRA_DIST = hacker-guide.tex afr.txt bdb.txt posix.txt call-stub.txt write-behind.txt -#hacker_guidedir = $(docdir) -#hacker_guide_DATA = hacker-guide.pdf - -#hacker-guide.pdf: $(EXTRA_DIST) -# pdflatex $(srcdir)/hacker-guide.tex diff --git a/doc/legacy/hacker-guide/adding-fops.txt b/doc/legacy/hacker-guide/adding-fops.txt deleted file mode 100644 index e70dbbdc829..00000000000 --- a/doc/legacy/hacker-guide/adding-fops.txt +++ /dev/null @@ -1,33 +0,0 @@ - HOW TO ADD A NEW FOP TO GlusterFS - ================================= - -Steps to be followed when adding a new FOP to GlusterFS: - -1. Edit glusterfs.h and add a GF_FOP_* constant. - -2. Edit xlator.[ch] and: - 2a. add the new prototype for fop and callback. - 2b. edit xlator_fops structure. - -3. Edit xlator.c and add to fill_defaults. - -4. Edit protocol.h and add struct necessary for the new FOP. - -5. Edit defaults.[ch] and provide default implementation. - -6. Edit call-stub.[ch] and provide stub implementation. - -7. Edit common-utils.c and add to gf_global_variable_init(). - -8. Edit client-protocol and add your FOP. - -9. Edit server-protocol and add your FOP. - -10. Implement your FOP in any translator for which the default implementation - is not sufficient. - -========================================== -Last updated: Mon Oct 27 21:35:49 IST 2008 - -Author: Vikas Gorur <vikas@gluster.com> -========================================== diff --git a/doc/legacy/hacker-guide/bdb.txt b/doc/legacy/hacker-guide/bdb.txt deleted file mode 100644 index 1a80be813f6..00000000000 --- a/doc/legacy/hacker-guide/bdb.txt +++ /dev/null @@ -1,70 +0,0 @@ - -* How does file translates to key/value pair? ---------------------------------------------- - - in bdb a file is identified by key (obtained by taking basename() of the path of -the file) and file contents are stored as value corresponding to the key in database -file (defaults to glusterfs_storage.db under dirname() directory). - -* symlinks, directories ------------------------ - - symlinks and directories are stored as is. - -* db (database) files ---------------------- - - every directory, including root directory, contains a database file called -glusterfs_storage.db. all the regular files contained in the directory are stored -as key/value pair inside the glusterfs_storage.db. - -* internal data cache ---------------------- - - db does not provide a way to find out the size of the value corresponding to a key. -so, bdb makes DB->get() call for key and takes the length of the value returned. -since DB->get() also returns file contents for key, bdb maintains an internal cache and -stores the file contents in the cache. - every directory maintains a seperate cache. - -* inode number transformation ------------------------------ - - bdb allocates a inode number to each file and directory on its own. bdb maintains a -global counter and increments it after allocating inode number for each file -(regular, symlink or directory). NOTE: bdb does not guarantee persistent inode numbers. - -* checkpoint thread -------------------- - - bdb creates a checkpoint thread at the time of init(). checkpoint thread does a -periodic checkpoint on the DB_ENV. checkpoint is the mechanism, provided by db, to -forcefully commit the logged transactions to the storage. - -NOTES ABOUT FOPS: ------------------ - -lookup() - - 1> do lstat() on the path, if lstat fails, we assume that the file being looked up - is either a regular file or doesn't exist. - 2> lookup in the DB of parent directory for key corresponding to path. if key exists, - return key, with. - NOTE: 'struct stat' stat()ed from DB file is used as a container for 'struct stat' - of the regular file. st_ino, st_size, st_blocks are updated with file's values. - -readv() - - 1> do a lookup in bctx cache. if successful, return the requested data from cache. - 2> if cache missed, do a DB->get() the entire file content and insert to cache. - -writev(): - 1> flush any cached content of this file. - 2> do a DB->put(), with DB_DBT_PARTIAL flag. - NOTE: DB_DBT_PARTIAL is used to do partial update of a value in DB. - -readdir(): - 1> regular readdir() in a loop, and vomit all DB_ENV log files and DB files that - we encounter. - 2> if the readdir() buffer still has space, open a DB cursor and do a sequential - DBC->get() to fill the reaadir buffer. - - diff --git a/doc/legacy/hacker-guide/call-stub.txt b/doc/legacy/hacker-guide/call-stub.txt deleted file mode 100644 index 021037a3512..00000000000 --- a/doc/legacy/hacker-guide/call-stub.txt +++ /dev/null @@ -1,1033 +0,0 @@ -creating a call stub and pausing a call ---------------------------------------- -libglusterfs provides seperate API to pause each of the fop. parameters to each API is -@frame - call frame which has to be used to resume the call at call_resume(). -@fn - procedure to call during call_resume(). - NOTE: @fn should exactly take the same type and number of parameters that - the corresponding regular fop takes. -rest will be the regular parameters to corresponding fop. - -NOTE: @frame can never be NULL. fop_<operation>_stub() fails with errno - set to EINVAL, if @frame is NULL. also wherever @loc is applicable, - @loc cannot be NULL. - -refer to individual stub creation API to know about call-stub creation's behaviour with -specific parameters. - -here is the list of stub creation APIs for xlator fops. - -@frame - call frame which has to be used to resume the call at call_resume(). -@fn - procedure to call during call_resume(). -@loc - pointer to location structure. - NOTE: @loc will be copied to a different location, with inode_ref() to - @loc->inode and @loc->parent, if not NULL. also @loc->path will be - copied to a different location. -@need_xattr - flag to specify if xattr should be returned or not. -call_stub_t * -fop_lookup_stub (call_frame_t *frame, - fop_lookup_t fn, - loc_t *loc, - int32_t need_xattr); - -@frame - call frame which has to be used to resume the call at call_resume(). -@fn - procedure to call during call_resume(). -@loc - pointer to location structure. - NOTE: @loc will be copied to a different location, with inode_ref() to - @loc->inode and @loc->parent, if not NULL. also @loc->path will be - copied to a different location. -call_stub_t * -fop_stat_stub (call_frame_t *frame, - fop_stat_t fn, - loc_t *loc); - -@frame - call frame which has to be used to resume the call at call_resume(). -@fn - procedure to call during call_resume(). -@fd - file descriptor parameter to lk fop. - NOTE: @fd is stored with a fd_ref(). -call_stub_t * -fop_fstat_stub (call_frame_t *frame, - fop_fstat_t fn, - fd_t *fd); - -@frame - call frame which has to be used to resume the call at call_resume(). -@fn - procedure to call during call_resume(). -@loc - pointer to location structure. - NOTE: @loc will be copied to a different location, with inode_ref() to @loc->inode and - @loc->parent, if not NULL. also @loc->path will be copied to a different location. -@mode - mode parameter to chmod. -call_stub_t * -fop_chmod_stub (call_frame_t *frame, - fop_chmod_t fn, - loc_t *loc, - mode_t mode); - -@frame - call frame which has to be used to resume the call at call_resume(). -@fn - procedure to call during call_resume(). -@fd - file descriptor parameter to lk fop. - NOTE: @fd is stored with a fd_ref(). -@mode - mode parameter for fchmod fop. -call_stub_t * -fop_fchmod_stub (call_frame_t *frame, - fop_fchmod_t fn, - fd_t *fd, - mode_t mode); - -@frame - call frame which has to be used to resume the call at call_resume(). -@fn - procedure to call during call_resume(). -@loc - pointer to location structure. - NOTE: @loc will be copied to a different location, with inode_ref() to @loc->inode and - @loc->parent, if not NULL. also @loc->path will be copied to a different location. -@uid - uid parameter to chown. -@gid - gid parameter to chown. -call_stub_t * -fop_chown_stub (call_frame_t *frame, - fop_chown_t fn, - loc_t *loc, - uid_t uid, - gid_t gid); - -@frame - call frame which has to be used to resume the call at call_resume(). -@fn - procedure to call during call_resume(). -@fd - file descriptor parameter to lk fop. - NOTE: @fd is stored with a fd_ref(). -@uid - uid parameter to fchown. -@gid - gid parameter to fchown. -call_stub_t * -fop_fchown_stub (call_frame_t *frame, - fop_fchown_t fn, - fd_t *fd, - uid_t uid, - gid_t gid); - -@frame - call frame which has to be used to resume the call at call_resume(). -@fn - procedure to call during call_resume(). -@loc - pointer to location structure. - NOTE: @loc will be copied to a different location, with inode_ref() to - @loc->inode and @loc->parent, if not NULL. also @loc->path will be - copied to a different location, if not NULL. -@off - offset parameter to truncate fop. -call_stub_t * -fop_truncate_stub (call_frame_t *frame, - fop_truncate_t fn, - loc_t *loc, - off_t off); - -@frame - call frame which has to be used to resume the call at call_resume(). -@fn - procedure to call during call_resume(). -@fd - file descriptor parameter to lk fop. - NOTE: @fd is stored with a fd_ref(). -@off - offset parameter to ftruncate fop. -call_stub_t * -fop_ftruncate_stub (call_frame_t *frame, - fop_ftruncate_t fn, - fd_t *fd, - off_t off); - -@frame - call frame which has to be used to resume the call at call_resume(). -@fn - procedure to call during call_resume(). -@loc - pointer to location structure. - NOTE: @loc will be copied to a different location, with inode_ref() to - @loc->inode and @loc->parent, if not NULL. also @loc->path will be - copied to a different location. -@tv - tv parameter to utimens fop. -call_stub_t * -fop_utimens_stub (call_frame_t *frame, - fop_utimens_t fn, - loc_t *loc, - struct timespec tv[2]); - -@frame - call frame which has to be used to resume the call at call_resume(). -@fn - procedure to call during call_resume(). -@loc - pointer to location structure. - NOTE: @loc will be copied to a different location, with inode_ref() to - @loc->inode and @loc->parent, if not NULL. also @loc->path will be - copied to a different location. -@mask - mask parameter for access fop. -call_stub_t * -fop_access_stub (call_frame_t *frame, - fop_access_t fn, - loc_t *loc, - int32_t mask); - -@frame - call frame which has to be used to resume the call at call_resume(). -@fn - procedure to call during call_resume(). -@loc - pointer to location structure. - NOTE: @loc will be copied to a different location, with inode_ref() to - @loc->inode and @loc->parent, if not NULL. also @loc->path will be - copied to a different location. -@size - size parameter to readlink fop. -call_stub_t * -fop_readlink_stub (call_frame_t *frame, - fop_readlink_t fn, - loc_t *loc, - size_t size); - -@frame - call frame which has to be used to resume the call at call_resume(). -@fn - procedure to call during call_resume(). -@loc - pointer to location structure. - NOTE: @loc will be copied to a different location, with inode_ref() to - @loc->inode and @loc->parent, if not NULL. also @loc->path will be - copied to a different location. -@mode - mode parameter to mknod fop. -@rdev - rdev parameter to mknod fop. -call_stub_t * -fop_mknod_stub (call_frame_t *frame, - fop_mknod_t fn, - loc_t *loc, - mode_t mode, - dev_t rdev); - -@frame - call frame which has to be used to resume the call at call_resume(). -@fn - procedure to call during call_resume(). -@loc - pointer to location structure. - NOTE: @loc will be copied to a different location, with inode_ref() to - @loc->inode and @loc->parent, if not NULL. also @loc->path will be - copied to a different location. -@mode - mode parameter to mkdir fop. -call_stub_t * -fop_mkdir_stub (call_frame_t *frame, - fop_mkdir_t fn, - loc_t *loc, - mode_t mode); - -@frame - call frame which has to be used to resume the call at call_resume(). -@fn - procedure to call during call_resume(). -@loc - pointer to location structure. - NOTE: @loc will be copied to a different location, with inode_ref() to - @loc->inode and @loc->parent, if not NULL. also @loc->path will be - copied to a different location. -call_stub_t * -fop_unlink_stub (call_frame_t *frame, - fop_unlink_t fn, - loc_t *loc); - -@frame - call frame which has to be used to resume the call at call_resume(). -@fn - procedure to call during call_resume(). -@loc - pointer to location structure. - NOTE: @loc will be copied to a different location, with inode_ref() to - @loc->inode and @loc->parent, if not NULL. also @loc->path will be - copied to a different location. -call_stub_t * -fop_rmdir_stub (call_frame_t *frame, - fop_rmdir_t fn, - loc_t *loc); - -@frame - call frame which has to be used to resume the call at call_resume(). -@fn - procedure to call during call_resume(). -@linkname - linkname parameter to symlink fop. -@loc - pointer to location structure. - NOTE: @loc will be copied to a different location, with inode_ref() to - @loc->inode and @loc->parent, if not NULL. also @loc->path will be - copied to a different location. -call_stub_t * -fop_symlink_stub (call_frame_t *frame, - fop_symlink_t fn, - const char *linkname, - loc_t *loc); - -@frame - call frame which has to be used to resume the call at call_resume(). -@fn - procedure to call during call_resume(). -@oldloc - pointer to location structure. - NOTE: @oldloc will be copied to a different location, with inode_ref() to - @oldloc->inode and @oldloc->parent, if not NULL. also @oldloc->path will - be copied to a different location, if not NULL. -@newloc - pointer to location structure. - NOTE: @newloc will be copied to a different location, with inode_ref() to - @newloc->inode and @newloc->parent, if not NULL. also @newloc->path will - be copied to a different location, if not NULL. -call_stub_t * -fop_rename_stub (call_frame_t *frame, - fop_rename_t fn, - loc_t *oldloc, - loc_t *newloc); - -@frame - call frame which has to be used to resume the call at call_resume(). -@fn - procedure to call during call_resume(). -@loc - pointer to location structure. - NOTE: @loc will be copied to a different location, with inode_ref() to - @loc->inode and @loc->parent, if not NULL. also @loc->path will be - copied to a different location. -@newpath - newpath parameter to link fop. -call_stub_t * -fop_link_stub (call_frame_t *frame, - fop_link_t fn, - loc_t *oldloc, - const char *newpath); - -@frame - call frame which has to be used to resume the call at call_resume(). -@fn - procedure to call during call_resume(). -@loc - pointer to location structure. - NOTE: @loc will be copied to a different location, with inode_ref() to - @loc->inode and @loc->parent, if not NULL. also @loc->path will be - copied to a different location. -@flags - flags parameter to create fop. -@mode - mode parameter to create fop. -@fd - file descriptor parameter to create fop. - NOTE: @fd is stored with a fd_ref(). -call_stub_t * -fop_create_stub (call_frame_t *frame, - fop_create_t fn, - loc_t *loc, - int32_t flags, - mode_t mode, fd_t *fd); - -@frame - call frame which has to be used to resume the call at call_resume(). -@fn - procedure to call during call_resume(). -@flags - flags parameter to open fop. -@loc - pointer to location structure. - NOTE: @loc will be copied to a different location, with inode_ref() to - @loc->inode and @loc->parent, if not NULL. also @loc->path will be - copied to a different location. -call_stub_t * -fop_open_stub (call_frame_t *frame, - fop_open_t fn, - loc_t *loc, - int32_t flags, - fd_t *fd); - -@frame - call frame which has to be used to resume the call at call_resume(). -@fn - procedure to call during call_resume(). -@fd - file descriptor parameter to lk fop. - NOTE: @fd is stored with a fd_ref(). -@size - size parameter to readv fop. -@off - offset parameter to readv fop. -call_stub_t * -fop_readv_stub (call_frame_t *frame, - fop_readv_t fn, - fd_t *fd, - size_t size, - off_t off); - -@frame - call frame which has to be used to resume the call at call_resume(). -@fn - procedure to call during call_resume(). -@fd - file descriptor parameter to lk fop. - NOTE: @fd is stored with a fd_ref(). -@vector - vector parameter to writev fop. - NOTE: @vector is iov_dup()ed while creating stub. and frame->root->req_refs - dictionary is dict_ref()ed. -@count - count parameter to writev fop. -@off - off parameter to writev fop. -call_stub_t * -fop_writev_stub (call_frame_t *frame, - fop_writev_t fn, - fd_t *fd, - struct iovec *vector, - int32_t count, - off_t off); - -@frame - call frame which has to be used to resume the call at call_resume(). -@fn - procedure to call during call_resume(). -@fd - file descriptor parameter to flush fop. - NOTE: @fd is stored with a fd_ref(). -call_stub_t * -fop_flush_stub (call_frame_t *frame, - fop_flush_t fn, - fd_t *fd); - - -@frame - call frame which has to be used to resume the call at call_resume(). -@fn - procedure to call during call_resume(). -@fd - file descriptor parameter to lk fop. - NOTE: @fd is stored with a fd_ref(). -@datasync - datasync parameter to fsync fop. -call_stub_t * -fop_fsync_stub (call_frame_t *frame, - fop_fsync_t fn, - fd_t *fd, - int32_t datasync); - -@frame - call frame which has to be used to resume the call at call_resume(). -@fn - procedure to call during call_resume(). -@loc - pointer to location structure. - NOTE: @loc will be copied to a different location, with inode_ref() to @loc->inode and - @loc->parent, if not NULL. also @loc->path will be copied to a different location. -@fd - file descriptor parameter to opendir fop. - NOTE: @fd is stored with a fd_ref(). -call_stub_t * -fop_opendir_stub (call_frame_t *frame, - fop_opendir_t fn, - loc_t *loc, - fd_t *fd); - -@frame - call frame which has to be used to resume the call at call_resume(). -@fn - procedure to call during call_resume(). -@fd - file descriptor parameter to getdents fop. - NOTE: @fd is stored with a fd_ref(). -@size - size parameter to getdents fop. -@off - off parameter to getdents fop. -@flags - flags parameter to getdents fop. -call_stub_t * -fop_getdents_stub (call_frame_t *frame, - fop_getdents_t fn, - fd_t *fd, - size_t size, - off_t off, - int32_t flag); - -@frame - call frame which has to be used to resume the call at call_resume(). -@fn - procedure to call during call_resume(). -@fd - file descriptor parameter to setdents fop. - NOTE: @fd is stored with a fd_ref(). -@flags - flags parameter to setdents fop. -@entries - entries parameter to setdents fop. -call_stub_t * -fop_setdents_stub (call_frame_t *frame, - fop_setdents_t fn, - fd_t *fd, - int32_t flags, - dir_entry_t *entries, - int32_t count); - -@frame - call frame which has to be used to resume the call at call_resume(). -@fn - procedure to call during call_resume(). -@fd - file descriptor parameter to setdents fop. - NOTE: @fd is stored with a fd_ref(). -@datasync - datasync parameter to fsyncdir fop. -call_stub_t * -fop_fsyncdir_stub (call_frame_t *frame, - fop_fsyncdir_t fn, - fd_t *fd, - int32_t datasync); - -@frame - call frame which has to be used to resume the call at call_resume(). -@fn - procedure to call during call_resume(). -@loc - pointer to location structure. - NOTE: @loc will be copied to a different location, with inode_ref() to - @loc->inode and @loc->parent, if not NULL. also @loc->path will be - copied to a different location. -call_stub_t * -fop_statfs_stub (call_frame_t *frame, - fop_statfs_t fn, - loc_t *loc); - -@frame - call frame which has to be used to resume the call at call_resume(). -@fn - procedure to call during call_resume(). -@loc - pointer to location structure. - NOTE: @loc will be copied to a different location, with inode_ref() to - @loc->inode and @loc->parent, if not NULL. also @loc->path will be - copied to a different location. -@dict - dict parameter to setxattr fop. - NOTE: stub creation procedure stores @dict pointer with dict_ref() to it. -call_stub_t * -fop_setxattr_stub (call_frame_t *frame, - fop_setxattr_t fn, - loc_t *loc, - dict_t *dict, - int32_t flags); - -@frame - call frame which has to be used to resume the call at call_resume(). -@fn - procedure to call during call_resume(). -@loc - pointer to location structure. - NOTE: @loc will be copied to a different location, with inode_ref() to - @loc->inode and @loc->parent, if not NULL. also @loc->path will be - copied to a different location. -@name - name parameter to getxattr fop. -call_stub_t * -fop_getxattr_stub (call_frame_t *frame, - fop_getxattr_t fn, - loc_t *loc, - const char *name); - -@frame - call frame which has to be used to resume the call at call_resume(). -@fn - procedure to call during call_resume(). -@loc - pointer to location structure. - NOTE: @loc will be copied to a different location, with inode_ref() to - @loc->inode and @loc->parent, if not NULL. also @loc->path will be - copied to a different location. -@name - name parameter to removexattr fop. - NOTE: name string will be copied to a different location while creating stub. -call_stub_t * -fop_removexattr_stub (call_frame_t *frame, - fop_removexattr_t fn, - loc_t *loc, - const char *name); - -@frame - call frame which has to be used to resume the call at call_resume(). -@fn - procedure to call during call_resume(). -@fd - file descriptor parameter to lk fop. - NOTE: @fd is stored with a fd_ref(). -@cmd - command parameter to lk fop. -@lock - lock parameter to lk fop. - NOTE: lock will be copied to a different location while creating stub. -call_stub_t * -fop_lk_stub (call_frame_t *frame, - fop_lk_t fn, - fd_t *fd, - int32_t cmd, - struct flock *lock); - -@frame - call frame which has to be used to resume the call at call_resume(). -@fn - procedure to call during call_resume(). -@fd - fd parameter to gf_lk fop. - NOTE: @fd is fd_ref()ed while creating stub, if not NULL. -@cmd - cmd parameter to gf_lk fop. -@lock - lock paramater to gf_lk fop. - NOTE: @lock is copied to a different memory location while creating - stub. -call_stub_t * -fop_gf_lk_stub (call_frame_t *frame, - fop_gf_lk_t fn, - fd_t *fd, - int32_t cmd, - struct flock *lock); - -@frame - call frame which has to be used to resume the call at call_resume(). -@fn - procedure to call during call_resume(). -@fd - file descriptor parameter to readdir fop. - NOTE: @fd is stored with a fd_ref(). -@size - size parameter to readdir fop. -@off - offset parameter to readdir fop. -call_stub_t * -fop_readdir_stub (call_frame_t *frame, - fop_readdir_t fn, - fd_t *fd, - size_t size, - off_t off); - -@frame - call frame which has to be used to resume the call at call_resume(). -@fn - procedure to call during call_resume(). -@loc - pointer to location structure. - NOTE: @loc will be copied to a different location, with inode_ref() to - @loc->inode and @loc->parent, if not NULL. also @loc->path will be - copied to a different location. -@flags - flags parameter to checksum fop. -call_stub_t * -fop_checksum_stub (call_frame_t *frame, - fop_checksum_t fn, - loc_t *loc, - int32_t flags); - -@frame - call frame which has to be used to resume the call at call_resume(). -@fn - procedure to call during call_resume(). -@op_ret - op_ret parameter to @fn. -@op_errno - op_errno parameter to @fn. -@inode - inode parameter to @fn. - NOTE: @inode pointer is stored with a inode_ref(). -@buf - buf parameter to @fn. - NOTE: @buf is copied to a different memory location, if not NULL. -@dict - dict parameter to @fn. - NOTE: @dict pointer is stored with dict_ref(). -call_stub_t * -fop_lookup_cbk_stub (call_frame_t *frame, - fop_lookup_cbk_t fn, - int32_t op_ret, - int32_t op_errno, - inode_t *inode, - struct stat *buf, - dict_t *dict); -@frame - call frame which has to be used to resume the call at call_resume(). -@fn - procedure to call during call_resume(). -@op_ret - op_ret parameter to @fn. -@op_errno - op_errno parameter to @fn. -@buf - buf parameter to @fn. - NOTE: @buf is copied to a different memory location, if not NULL. -call_stub_t * -fop_stat_cbk_stub (call_frame_t *frame, - fop_stat_cbk_t fn, - int32_t op_ret, - int32_t op_errno, - struct stat *buf); - -@frame - call frame which has to be used to resume the call at call_resume(). -@fn - procedure to call during call_resume(). -@op_ret - op_ret parameter to @fn. -@op_errno - op_errno parameter to @fn. -@buf - buf parameter to @fn. - NOTE: @buf is copied to a different memory location, if not NULL. -call_stub_t * -fop_fstat_cbk_stub (call_frame_t *frame, - fop_fstat_cbk_t fn, - int32_t op_ret, - int32_t op_errno, - struct stat *buf); - -@frame - call frame which has to be used to resume the call at call_resume(). -@fn - procedure to call during call_resume(). -@op_ret - op_ret parameter to @fn. -@op_errno - op_errno parameter to @fn. -@buf - buf parameter to @fn. - NOTE: @buf is copied to a different memory location, if not NULL. -call_stub_t * -fop_chmod_cbk_stub (call_frame_t *frame, - fop_chmod_cbk_t fn, - int32_t op_ret, - int32_t op_errno, - struct stat *buf); - -@frame - call frame which has to be used to resume the call at call_resume(). -@fn - procedure to call during call_resume(). -@op_ret - op_ret parameter to @fn. -@op_errno - op_errno parameter to @fn. -@buf - buf parameter to @fn. - NOTE: @buf is copied to a different memory location, if not NULL. -call_stub_t * -fop_fchmod_cbk_stub (call_frame_t *frame, - fop_fchmod_cbk_t fn, - int32_t op_ret, - int32_t op_errno, - struct stat *buf); - -@frame - call frame which has to be used to resume the call at call_resume(). -@fn - procedure to call during call_resume(). -@op_ret - op_ret parameter to @fn. -@op_errno - op_errno parameter to @fn. -@buf - buf parameter to @fn. - NOTE: @buf is copied to a different memory location, if not NULL. -call_stub_t * -fop_chown_cbk_stub (call_frame_t *frame, - fop_chown_cbk_t fn, - int32_t op_ret, - int32_t op_errno, - struct stat *buf); - - -@frame - call frame which has to be used to resume the call at call_resume(). -@fn - procedure to call during call_resume(). -@op_ret - op_ret parameter to @fn. -@op_errno - op_errno parameter to @fn. -@buf - buf parameter to @fn. - NOTE: @buf is copied to a different memory location, if not NULL. -call_stub_t * -fop_fchown_cbk_stub (call_frame_t *frame, - fop_fchown_cbk_t fn, - int32_t op_ret, - int32_t op_errno, - struct stat *buf); - - -@frame - call frame which has to be used to resume the call at call_resume(). -@fn - procedure to call during call_resume(). -@op_ret - op_ret parameter to @fn. -@op_errno - op_errno parameter to @fn. -@buf - buf parameter to @fn. - NOTE: @buf is copied to a different memory location, if not NULL. -call_stub_t * -fop_truncate_cbk_stub (call_frame_t *frame, - fop_truncate_cbk_t fn, - int32_t op_ret, - int32_t op_errno, - struct stat *buf); - - -@frame - call frame which has to be used to resume the call at call_resume(). -@fn - procedure to call during call_resume(). -@op_ret - op_ret parameter to @fn. -@op_errno - op_errno parameter to @fn. -@buf - buf parameter to @fn. - NOTE: @buf is copied to a different memory location, if not NULL. -call_stub_t * -fop_ftruncate_cbk_stub (call_frame_t *frame, - fop_ftruncate_cbk_t fn, - int32_t op_ret, - int32_t op_errno, - struct stat *buf); - - -@frame - call frame which has to be used to resume the call at call_resume(). -@fn - procedure to call during call_resume(). -@op_ret - op_ret parameter to @fn. -@op_errno - op_errno parameter to @fn. -@buf - buf parameter to @fn. - NOTE: @buf is copied to a different memory location, if not NULL. -call_stub_t * -fop_utimens_cbk_stub (call_frame_t *frame, - fop_utimens_cbk_t fn, - int32_t op_ret, - int32_t op_errno, - struct stat *buf); - - -@frame - call frame which has to be used to resume the call at call_resume(). -@fn - procedure to call during call_resume(). -@op_ret - op_ret parameter to @fn. -@op_errno - op_errno parameter to @fn. -call_stub_t * -fop_access_cbk_stub (call_frame_t *frame, - fop_access_cbk_t fn, - int32_t op_ret, - int32_t op_errno); - - -@frame - call frame which has to be used to resume the call at call_resume(). -@fn - procedure to call during call_resume(). -@op_ret - op_ret parameter to @fn. -@op_errno - op_errno parameter to @fn. -@path - path parameter to @fn. - NOTE: @path is copied to a different memory location, if not NULL. -call_stub_t * -fop_readlink_cbk_stub (call_frame_t *frame, - fop_readlink_cbk_t fn, - int32_t op_ret, - int32_t op_errno, - const char *path); - - -@frame - call frame which has to be used to resume the call at call_resume(). -@fn - procedure to call during call_resume(). -@op_ret - op_ret parameter to @fn. -@op_errno - op_errno parameter to @fn. -@inode - inode parameter to @fn. - NOTE: @inode pointer is stored with a inode_ref(). -@buf - buf parameter to @fn. - NOTE: @buf is copied to a different memory location, if not NULL. -call_stub_t * -fop_mknod_cbk_stub (call_frame_t *frame, - fop_mknod_cbk_t fn, - int32_t op_ret, - int32_t op_errno, - inode_t *inode, - struct stat *buf); - - -@frame - call frame which has to be used to resume the call at call_resume(). -@fn - procedure to call during call_resume(). -@op_ret - op_ret parameter to @fn. -@op_errno - op_errno parameter to @fn. -@inode - inode parameter to @fn. - NOTE: @inode pointer is stored with a inode_ref(). -@buf - buf parameter to @fn. - NOTE: @buf is copied to a different memory location, if not NULL. -call_stub_t * -fop_mkdir_cbk_stub (call_frame_t *frame, - fop_mkdir_cbk_t fn, - int32_t op_ret, - int32_t op_errno, - inode_t *inode, - struct stat *buf); - - -@frame - call frame which has to be used to resume the call at call_resume(). -@fn - procedure to call during call_resume(). -@op_ret - op_ret parameter to @fn. -@op_errno - op_errno parameter to @fn. -call_stub_t * -fop_unlink_cbk_stub (call_frame_t *frame, - fop_unlink_cbk_t fn, - int32_t op_ret, - int32_t op_errno); - - -@frame - call frame which has to be used to resume the call at call_resume(). -@fn - procedure to call during call_resume(). -@op_ret - op_ret parameter to @fn. -@op_errno - op_errno parameter to @fn. -call_stub_t * -fop_rmdir_cbk_stub (call_frame_t *frame, - fop_rmdir_cbk_t fn, - int32_t op_ret, - int32_t op_errno); - - -@frame - call frame which has to be used to resume the call at call_resume(). -@fn - procedure to call during call_resume(). -@op_ret - op_ret parameter to @fn. -@op_errno - op_errno parameter to @fn. -@inode - inode parameter to @fn. - NOTE: @inode pointer is stored with a inode_ref(). -@buf - buf parameter to @fn. - NOTE: @buf is copied to a different memory location, if not NULL. -call_stub_t * -fop_symlink_cbk_stub (call_frame_t *frame, - fop_symlink_cbk_t fn, - int32_t op_ret, - int32_t op_errno, - inode_t *inode, - struct stat *buf); - - -@frame - call frame which has to be used to resume the call at call_resume(). -@fn - procedure to call during call_resume(). -@op_ret - op_ret parameter to @fn. -@op_errno - op_errno parameter to @fn. -@buf - buf parameter to @fn. - NOTE: @buf is copied to a different memory location, if not NULL. -call_stub_t * -fop_rename_cbk_stub (call_frame_t *frame, - fop_rename_cbk_t fn, - int32_t op_ret, - int32_t op_errno, - struct stat *buf); - - -@frame - call frame which has to be used to resume the call at call_resume(). -@fn - procedure to call during call_resume(). -@op_ret - op_ret parameter to @fn. -@op_errno - op_errno parameter to @fn. -@inode - inode parameter to @fn. - NOTE: @inode pointer is stored with a inode_ref(). -@buf - buf parameter to @fn. - NOTE: @buf is copied to a different memory location, if not NULL. -call_stub_t * -fop_link_cbk_stub (call_frame_t *frame, - fop_link_cbk_t fn, - int32_t op_ret, - int32_t op_errno, - inode_t *inode, - struct stat *buf); - -@frame - call frame which has to be used to resume the call at call_resume(). -@fn - procedure to call during call_resume(). -@op_ret - op_ret parameter to @fn. -@op_errno - op_errno parameter to @fn. -@fd - fd parameter to @fn. - NOTE: @fd pointer is stored with a fd_ref(). -@inode - inode parameter to @fn. - NOTE: @inode pointer is stored with a inode_ref(). -@buf - buf parameter to @fn. - NOTE: @buf is copied to a different memory location, if not NULL. -call_stub_t * -fop_create_cbk_stub (call_frame_t *frame, - fop_create_cbk_t fn, - int32_t op_ret, - int32_t op_errno, - fd_t *fd, - inode_t *inode, - struct stat *buf); - - -@frame - call frame which has to be used to resume the call at call_resume(). -@fn - procedure to call during call_resume(). -@op_ret - op_ret parameter to @fn. -@op_errno - op_errno parameter to @fn. -@fd - fd parameter to @fn. - NOTE: @fd pointer is stored with a fd_ref(). -call_stub_t * -fop_open_cbk_stub (call_frame_t *frame, - fop_open_cbk_t fn, - int32_t op_ret, - int32_t op_errno, - fd_t *fd); - - -@frame - call frame which has to be used to resume the call at call_resume(). -@fn - procedure to call during call_resume(). -@op_ret - op_ret parameter to @fn. -@op_errno - op_errno parameter to @fn. -@vector - vector parameter to @fn. - NOTE: @vector is copied to a different memory location, if not NULL. also - frame->root->rsp_refs is dict_ref()ed. -@stbuf - stbuf parameter to @fn. - NOTE: @stbuf is copied to a different memory location, if not NULL. -call_stub_t * -fop_readv_cbk_stub (call_frame_t *frame, - fop_readv_cbk_t fn, - int32_t op_ret, - int32_t op_errno, - struct iovec *vector, - int32_t count, - struct stat *stbuf); - - -@frame - call frame which has to be used to resume the call at call_resume(). -@fn - procedure to call during call_resume(). -@op_ret - op_ret parameter to @fn. -@op_errno - op_errno parameter to @fn. -@stbuf - stbuf parameter to @fn. - NOTE: @stbuf is copied to a different memory location, if not NULL. -call_stub_t * -fop_writev_cbk_stub (call_frame_t *frame, - fop_writev_cbk_t fn, - int32_t op_ret, - int32_t op_errno, - struct stat *stbuf); - - -@frame - call frame which has to be used to resume the call at call_resume(). -@fn - procedure to call during call_resume(). -@op_ret - op_ret parameter to @fn. -@op_errno - op_errno parameter to @fn. -call_stub_t * -fop_flush_cbk_stub (call_frame_t *frame, - fop_flush_cbk_t fn, - int32_t op_ret, - int32_t op_errno); - -@frame - call frame which has to be used to resume the call at call_resume(). -@fn - procedure to call during call_resume(). -@op_ret - op_ret parameter to @fn. -@op_errno - op_errno parameter to @fn. -call_stub_t * -fop_fsync_cbk_stub (call_frame_t *frame, - fop_fsync_cbk_t fn, - int32_t op_ret, - int32_t op_errno); - - -@frame - call frame which has to be used to resume the call at call_resume(). -@fn - procedure to call during call_resume(). -@op_ret - op_ret parameter to @fn. -@op_errno - op_errno parameter to @fn. -@fd - fd parameter to @fn. - NOTE: @fd pointer is stored with a fd_ref(). -call_stub_t * -fop_opendir_cbk_stub (call_frame_t *frame, - fop_opendir_cbk_t fn, - int32_t op_ret, - int32_t op_errno, - fd_t *fd); - - -@frame - call frame which has to be used to resume the call at call_resume(). -@fn - procedure to call during call_resume(). -@op_ret - op_ret parameter to @fn. -@op_errno - op_errno parameter to @fn. -@entries - entries parameter to @fn. -@count - count parameter to @fn. -call_stub_t * -fop_getdents_cbk_stub (call_frame_t *frame, - fop_getdents_cbk_t fn, - int32_t op_ret, - int32_t op_errno, - dir_entry_t *entries, - int32_t count); - - -@frame - call frame which has to be used to resume the call at call_resume(). -@fn - procedure to call during call_resume(). -@op_ret - op_ret parameter to @fn. -@op_errno - op_errno parameter to @fn. -call_stub_t * -fop_setdents_cbk_stub (call_frame_t *frame, - fop_setdents_cbk_t fn, - int32_t op_ret, - int32_t op_errno); - -@frame - call frame which has to be used to resume the call at call_resume(). -@fn - procedure to call during call_resume(). -@op_ret - op_ret parameter to @fn. -@op_errno - op_errno parameter to @fn. -call_stub_t * -fop_fsyncdir_cbk_stub (call_frame_t *frame, - fop_fsyncdir_cbk_t fn, - int32_t op_ret, - int32_t op_errno); - - -@frame - call frame which has to be used to resume the call at call_resume(). -@fn - procedure to call during call_resume(). -@op_ret - op_ret parameter to @fn. -@op_errno - op_errno parameter to @fn. -@buf - buf parameter to @fn. - NOTE: @buf is copied to a different memory location, if not NULL. -call_stub_t * -fop_statfs_cbk_stub (call_frame_t *frame, - fop_statfs_cbk_t fn, - int32_t op_ret, - int32_t op_errno, - struct statvfs *buf); - - -@frame - call frame which has to be used to resume the call at call_resume(). -@fn - procedure to call during call_resume(). -@op_ret - op_ret parameter to @fn. -@op_errno - op_errno parameter to @fn. -call_stub_t * -fop_setxattr_cbk_stub (call_frame_t *frame, - fop_setxattr_cbk_t fn, - int32_t op_ret, - int32_t op_errno); - - -@frame - call frame which has to be used to resume the call at call_resume(). -@fn - procedure to call during call_resume(). -@op_ret - op_ret parameter to @fn. -@op_errno - op_errno parameter to @fn. -@value - value dictionary parameter to @fn. - NOTE: @value pointer is stored with a dict_ref(). -call_stub_t * -fop_getxattr_cbk_stub (call_frame_t *frame, - fop_getxattr_cbk_t fn, - int32_t op_ret, - int32_t op_errno, - dict_t *value); - - -@frame - call frame which has to be used to resume the call at call_resume(). -@fn - procedure to call during call_resume(). -@op_ret - op_ret parameter to @fn. -@op_errno - op_errno parameter to @fn. -call_stub_t * -fop_removexattr_cbk_stub (call_frame_t *frame, - fop_removexattr_cbk_t fn, - int32_t op_ret, - int32_t op_errno); - - -@frame - call frame which has to be used to resume the call at call_resume(). -@fn - procedure to call during call_resume(). -@op_ret - op_ret parameter to @fn. -@op_errno - op_errno parameter to @fn. -@lock - lock parameter to @fn. - NOTE: @lock is copied to a different memory location while creating - stub. -call_stub_t * -fop_lk_cbk_stub (call_frame_t *frame, - fop_lk_cbk_t fn, - int32_t op_ret, - int32_t op_errno, - struct flock *lock); - -@frame - call frame which has to be used to resume the call at call_resume(). -@fn - procedure to call during call_resume(). -@op_ret - op_ret parameter to @fn. -@op_errno - op_errno parameter to @fn. -@lock - lock parameter to @fn. - NOTE: @lock is copied to a different memory location while creating - stub. -call_stub_t * -fop_gf_lk_cbk_stub (call_frame_t *frame, - fop_gf_lk_cbk_t fn, - int32_t op_ret, - int32_t op_errno, - struct flock *lock); - - -@frame - call frame which has to be used to resume the call at call_resume(). -@fn - procedure to call during call_resume(). -@op_ret - op_ret parameter to @fn. -@op_errno - op_errno parameter to @fn. -@entries - entries parameter to @fn. -call_stub_t * -fop_readdir_cbk_stub (call_frame_t *frame, - fop_readdir_cbk_t fn, - int32_t op_ret, - int32_t op_errno, - gf_dirent_t *entries); - - -@frame - call frame which has to be used to resume the call at call_resume(). -@fn - procedure to call during call_resume(). -@op_ret - op_ret parameter to @fn. -@op_errno - op_errno parameter to @fn. -@file_checksum - file_checksum parameter to @fn. - NOTE: file_checksum will be copied to a different memory location - while creating stub. -@dir_checksum - dir_checksum parameter to @fn. - NOTE: file_checksum will be copied to a different memory location - while creating stub. -call_stub_t * -fop_checksum_cbk_stub (call_frame_t *frame, - fop_checksum_cbk_t fn, - int32_t op_ret, - int32_t op_errno, - uint8_t *file_checksum, - uint8_t *dir_checksum); - -resuming a call: ---------------- - call can be resumed using call stub through call_resume API. - - void call_resume (call_stub_t *stub); - - stub - call stub created during pausing a call. - - NOTE: call_resume() will decrease reference count of any fd_t, dict_t and inode_t that it finds - in stub->args.<operation>.<fd_t-or-inode_t-or-dict_t>. so, if any fd_t, dict_t or - inode_t pointers are assigned at stub->args.<operation>.<fd_t-or-inode_t-or-dict_t> after - fop_<operation>_stub() call, they must be <fd_t-or-inode_t-or-dict_t>_ref()ed. - - call_resume does not STACK_DESTROY() for any fop. - - if stub->fn is NULL, call_resume does STACK_WIND() or STACK_UNWIND() using the stub->frame. - - return - call resume fails only if stub is NULL. call resume fails with errno set to EINVAL. diff --git a/doc/legacy/hacker-guide/hacker-guide.tex b/doc/legacy/hacker-guide/hacker-guide.tex deleted file mode 100644 index 11101e7a87a..00000000000 --- a/doc/legacy/hacker-guide/hacker-guide.tex +++ /dev/null @@ -1,309 +0,0 @@ -\documentclass{book}[12pt] -\usepackage{graphicx} -% \usepackage{fancyhdr} - -% \pagestyle{fancy} -\begin{document} - -% \headheight 117pt -% \rhead{\includegraphics{zr-logo.eps}} - -\author{Gluster} -\title{GlusterFS 1.3 Hacker's Guide} -\date{June 1, 2007} - -\maketitle -\frontmatter -\tableofcontents - -\mainmatter -\chapter{Introduction} - -\section{Coding guidelines} -GlusterFS uses Git for version control. To get the latest source do: -\begin{verbatim} - $ git clone git://git.gluster.com/glusterfs.git glusterfs -\end{verbatim} -\noindent -GlusterFS follows the GNU coding -standards\footnote{http://www.gnu.org/prep/standards\_toc.html} for the -most part. - -\chapter{Major components} -\section{libglusterfs} -\texttt{libglusterfs} contains supporting code used by all the other components. -The important files here are: - -\texttt{dict.c}: This is an implementation of a serializable dictionary type. It is -used by the protocol code to send requests and replies. It is also used to pass options -to translators. - -\texttt{logging.c}: This is a thread-safe logging library. The log messages go to a -file (default \texttt{/usr/local/var/log/glusterfs/*}). - -\texttt{protocol.c}: This file implements the GlusterFS on-the-wire -protocol. The protocol itself is a simple ASCII protocol, designed to -be easy to parse and be human readable. - -A sample GlusterFS protocol block looks like this: -\begin{verbatim} - Block Start header - 0000000000000023 callid - 00000001 type - 00000016 op - xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx human-readable name - 00000000000000000000000000000ac3 block size - <...> block - Block End -\end{verbatim} - -\texttt{stack.h}: This file defines the \texttt{STACK\_WIND} and -\texttt{STACK\_UNWIND} macros which are used to implement the parallel -stack that is maintained for inter-xlator calls. See the \textsl{Taking control -of the stack} section below for more details. - -\texttt{spec.y}: This contains the Yacc grammar for the GlusterFS -specification file, and the parsing code. - - -Draw diagrams of trees -Two rules: -(1) directory structure is same -(2) file can exist only on one node - -\section{glusterfs-fuse} -\section{glusterfsd} -\section{transport} -\section{scheduler} -\section{xlator} - -\chapter{xlators} -\section{Taking control of the stack} -One can think of STACK\_WIND/UNWIND as a very specific RPC mechanism. - -% \includegraphics{stack.eps} - -\section{Overview of xlators} - -\flushleft{\LARGE\texttt{cluster/}} -\vskip 2ex -\flushleft{\Large\texttt{afr}} -\vskip 2ex -\flushleft{\Large\texttt{stripe}} -\vskip 2ex -\flushleft{\Large\texttt{unify}} - -\vskip 4ex -\flushleft{\LARGE\texttt{debug/}} -\vskip 2ex -\flushleft{\Large\texttt{trace}} -\vskip 2ex -The trace xlator simply logs all fops and mops, and passes them through to its child. - -\vskip 4ex -\flushleft{\LARGE\texttt{features/}} -\flushleft{\Large\texttt{posix-locks}} -\vskip 2ex -This xlator implements \textsc{posix} record locking semantics over -any kind of storage. - -\vskip 4ex -\flushleft{\LARGE\texttt{performance/}} - -\flushleft{\Large\texttt{io-threads}} -\vskip 2ex -\flushleft{\Large\texttt{read-ahead}} -\vskip 2ex -\flushleft{\Large\texttt{stat-prefetch}} -\vskip 2ex -\flushleft{\Large\texttt{write-behind}} -\vskip 2ex - -\vskip 4ex -\flushleft{\LARGE\texttt{protocol/}} -\vskip 2ex - -\flushleft{\Large\texttt{client}} -\vskip 2ex - -\flushleft{\Large\texttt{server}} -\vskip 2ex - -\vskip 4ex -\flushleft{\LARGE\texttt{storage/}} -\flushleft{\Large\texttt{posix}} -\vskip 2ex -The \texttt{posix} xlator is the one which actually makes calls to the -on-disk filesystem. Currently this is the only storage xlator available. However, -plans to develop other storage xlators, such as one for Amazon's S3 service, are -on the roadmap. - -\chapter{Writing a simple xlator} -\noindent -In this section we're going to write a rot13 xlator. ``Rot13'' is a -simple substitution cipher which obscures a text by replacing each -letter with the letter thirteen places down the alphabet. So `a' (0) -would become `n' (12), `b' would be 'm', and so on. Rot13 applied to -a piece of ciphertext yields the plaintext again, because rot13 is its -own inverse, since: - -\[ -x_c = x + 13\; (mod\; 26) -\] -\[ -x_c + 13\; (mod\; 26) = x + 13 + 13\; (mod\; 26) = x -\] - -First we include the requisite headers. - -\begin{verbatim} -#include <ctype.h> -#include <sys/uio.h> - -#include "glusterfs.h" -#include "xlator.h" -#include "logging.h" - -/* - * This is a rot13 ``encryption'' xlator. It rot13's data when - * writing to disk and rot13's it back when reading it. - * This xlator is meant as an example, not for production - * use ;) (hence no error-checking) - */ - -\end{verbatim} - -Then we write the rot13 function itself. For simplicity, we only transform lower case -letters. Any other byte is passed through as it is. - -\begin{verbatim} -/* We only handle lower case letters for simplicity */ -static void -rot13 (char *buf, int len) -{ - int i; - for (i = 0; i < len; i++) { - if (isalpha (buf[i])) - buf[i] = (buf[i] - 'a' + 13) % 26; - else if (buf[i] <= 26) - buf[i] = (buf[i] + 13) % 26 + 'a'; - } -} -\end{verbatim} - -Next comes a utility function whose purpose will be clear after looking at the code -below. - -\begin{verbatim} -static void -rot13_iovec (struct iovec *vector, int count) -{ - int i; - for (i = 0; i < count; i++) { - rot13 (vector[i].iov_base, vector[i].iov_len); - } -} -\end{verbatim} - -\begin{verbatim} -static int32_t -rot13_readv_cbk (call_frame_t *frame, - call_frame_t *prev_frame, - xlator_t *this, - int32_t op_ret, - int32_t op_errno, - struct iovec *vector, - int32_t count) -{ - rot13_iovec (vector, count); - - STACK_UNWIND (frame, op_ret, op_errno, vector, count); - return 0; -} - -static int32_t -rot13_readv (call_frame_t *frame, - xlator_t *this, - dict_t *ctx, - size_t size, - off_t offset) -{ - STACK_WIND (frame, - rot13_readv_cbk, - FIRST_CHILD (this), - FIRST_CHILD (this)->fops->readv, - ctx, size, offset); - return 0; -} - -static int32_t -rot13_writev_cbk (call_frame_t *frame, - call_frame_t *prev_frame, - xlator_t *this, - int32_t op_ret, - int32_t op_errno) -{ - STACK_UNWIND (frame, op_ret, op_errno); - return 0; -} - -static int32_t -rot13_writev (call_frame_t *frame, - xlator_t *this, - dict_t *ctx, - struct iovec *vector, - int32_t count, - off_t offset) -{ - rot13_iovec (vector, count); - - STACK_WIND (frame, - rot13_writev_cbk, - FIRST_CHILD (this), - FIRST_CHILD (this)->fops->writev, - ctx, vector, count, offset); - return 0; -} - -\end{verbatim} - -Every xlator must define two functions and two external symbols. The functions are -\texttt{init} and \texttt{fini}, and the symbols are \texttt{fops} and \texttt{mops}. -The \texttt{init} function is called when the xlator is loaded by GlusterFS, and -contains code for the xlator to initialize itself. Note that if an xlator is present -multiple times in the spec tree, the \texttt{init} function will be called each time -the xlator is loaded. - -\begin{verbatim} -int32_t -init (xlator_t *this) -{ - if (!this->children) { - gf_log ("rot13", GF_LOG_ERROR, - "FATAL: rot13 should have exactly one child"); - return -1; - } - - gf_log ("rot13", GF_LOG_DEBUG, "rot13 xlator loaded"); - return 0; -} -\end{verbatim} - -\begin{verbatim} - -void -fini (xlator_t *this) -{ - return; -} - -struct xlator_fops fops = { - .readv = rot13_readv, - .writev = rot13_writev -}; - - -\end{verbatim} - -\end{document} - diff --git a/doc/legacy/hacker-guide/lock-ahead.txt b/doc/legacy/hacker-guide/lock-ahead.txt deleted file mode 100644 index 70aa452d3de..00000000000 --- a/doc/legacy/hacker-guide/lock-ahead.txt +++ /dev/null @@ -1,80 +0,0 @@ - Lock-ahead translator - --------------------- - -The objective of the lock-ahead translator is to speculatively -hold locks (inodelk and entrylk) on the universal set (0 - infinity -in case of inodelk and all basenames in case of entrylk) even -when a lock is requested only on a subset, in anticipation that -further locks will be requested within the same universal set. - -So, for example, when cluster/replicate locks a region before -writing to it, lock-ahead would instead lock the entire file. -On further writes, lock-ahead can immediately return success for -the lock requests, since the entire file has been previously locked. - -To avoid starvation of other clients/mountpoints, we employ a -notify mechanism, described below. - -typedef struct { - struct list_head subset_locks; -} la_universal_lock_t; - -Universal lock structure is stored in the inode context. - -typedef struct { - enum {LOCK_AHEAD_ENTRYLK, LOCK_AHEAD_FENTRYLK, - LOCK_AHEAD_INODELK, LOCK_AHEAD_FINODELK}; - - union { - fd_t *fd; - loc_t loc; - }; - - off_t l_start; - off_t l_len; - - const char *basename; - - struct list_head universal_lock; -} la_subset_lock_t; - - -fops implemented: - -* inodelk/finodelk/entrylk/fentrylk: - -lock: - if universal lock held: - add subset to it (save loc_t or fd) and return success - else: - send lock-notify fop - hold universal lock and return - (set inode context, add subset to it, save loc_t or fd) - - if this fails: - forward the lock request - -unlock: - if subset exists in universal lock: - delete subset lock from list - else: - forward it - -* release: - hold subset locks (each subset lock using the saved loc_t or fd) - and release universal lock - -* lock-notify (on unwind) (new fop) - hold subset locks and release universal lock - - -lock-notify in locks translator: - -if a subset lock in entrylk/inodelk cannot be satisfied -because of a universal lock held by someone else: - unwind the lock-notify fop - -============================================== -$ Last updated: Tue Feb 17 11:31:18 IST 2009 $ -$ Author: Vikas Gorur <vikas@gluster.com> $ -============================================== diff --git a/doc/legacy/hacker-guide/posix.txt b/doc/legacy/hacker-guide/posix.txt deleted file mode 100644 index 7958af2ea7d..00000000000 --- a/doc/legacy/hacker-guide/posix.txt +++ /dev/null @@ -1,59 +0,0 @@ ---------------- -* storage/posix ---------------- - -- SET_FS_ID - - This is so that all filesystem checks are done with the user's - uid/gid and not GlusterFS's uid/gid. - -- MAKE_REAL_PATH - - This macro concatenates the base directory of the posix volume - ('option directory') with the given path. - -- need_xattr in lookup - - If this flag is passed, lookup returns a xattr dictionary that contains - the file's create time, the file's contents, and the version number - of the file. - - This is a hack to increase small file performance. If an application - wants to read a small file, it can finish its job with just a lookup - call instead of a lookup followed by read. - -- getdents/setdents - - These are used by unify to set and get directory entries. - -- ALIGN_BUF - - Macro to align an address to a page boundary (4K). - -- priv->export_statfs - - In some cases, two exported volumes may reside on the same - partition on the server. Sending statvfs info for both - the volumes will lead to erroneous df output at the client, - since free space on the partition will be counted twice. - - In such cases, user can disable exporting statvfs info - on one of the volumes by setting this option. - -- xattrop - - This fop is used by replicate to set version numbers on files. - -- getxattr/setxattr hack to read/write files - - A key, GLUSTERFS_FILE_CONTENT_STRING, is handled in a special way by - getxattr/setxattr. A getxattr with the key will return the entire - content of the file as the value. A setxattr with the key will write - the value as the entire content of the file. - -- posix_checksum - - This calculates a simple XOR checksum on all entry names in a - directory that is used by unify to compare directory contents. - - diff --git a/doc/legacy/hacker-guide/replicate.txt b/doc/legacy/hacker-guide/replicate.txt deleted file mode 100644 index 133c72afa91..00000000000 --- a/doc/legacy/hacker-guide/replicate.txt +++ /dev/null @@ -1,206 +0,0 @@ ---------------- -* cluster/replicate ---------------- - -Before understanding replicate, one must understand two internal FOPs: - -GF_FILE_LK: - This is exactly like fcntl(2) locking, except the locks are in a - separate domain from locks held by applications. - -GF_DIR_LK (loc_t *loc, char *basename): - This allows one to lock a name under a directory. For example, - to lock /mnt/glusterfs/foo, one would use the call: - - GF_DIR_LK ({loc_t for "/mnt/glusterfs"}, "foo") - - If one wishes to lock *all* the names under a particular directory, - supply the basename argument as NULL. - - The locks can either be read locks or write locks; consult the - function prototype for more details. - -Both these operations are implemented by the features/locks (earlier -known as posix-locks) translator. - --------------- -* Basic design --------------- - -All FOPs can be classified into four major groups: - - - inode-read - Operations that read an inode's data (file contents) or metadata (perms, etc.). - - access, getxattr, fstat, readlink, readv, stat. - - - inode-write - Operations that modify an inode's data or metadata. - - chmod, chown, truncate, writev, utimens. - - - dir-read - Operations that read a directory's contents or metadata. - - readdir, getdents, checksum. - - - dir-write - Operations that modify a directory's contents or metadata. - - create, link, mkdir, mknod, rename, rmdir, symlink, unlink. - - Some of these make a subgroup in that they modify *two* different entries: - link, rename, symlink. - - - Others - Other operations. - - flush, lookup, open, opendir, statfs. - ------------- -* Algorithms ------------- - -Each of the four major groups has its own algorithm: - - ---------------------- - - inode-read, dir-read - ---------------------- - - = Send a request to the first child that is up: - - if it fails: - try the next available child - - if we have exhausted all children: - return failure - - ------------- - - inode-write - ------------- - - All operations are done in parallel unless specified otherwise. - - (1) Send a GF_FILE_LK request on all children for a write lock on - the appropriate region - (for metadata operations: entire file (0, 0) - for writev: (offset, offset+size of buffer)) - - - If a lock request fails on a child: - unlock all children - try to acquire a blocking lock (F_SETLKW) on each child, serially. - - If this fails (due to ENOTCONN or EINVAL): - Consider this child as dead for rest of transaction. - - (2) Mark all children as "pending" on all (alive) children - (see below for meaning of "pending"). - - - If it fails on any child: - mark it as dead (in transaction local state). - - (3) Perform operation on all (alive) children. - - - If it fails on any child: - mark it as dead (in transaction local state). - - (4) Unmark all successful children as not "pending" on all nodes. - - (5) Unlock region on all (alive) children. - - ----------- - - dir-write - ----------- - - The algorithm for dir-write is same as above except instead of holding - GF_FILE_LK locks we hold a GF_DIR_LK lock on the name being operated upon. - In case of link-type calls, we hold locks on both the operand names. - ------------ -* "pending" ------------ - - The "pending" number is like a journal entry. A pending entry is an - array of 32-bit integers stored in network byte-order as the extended - attribute of an inode (which can be a directory as well). - - There are three keys corresponding to three types of pending operations: - - - AFR_METADATA_PENDING - There are some metadata operations pending on this inode (perms, ctime/mtime, - xattr, etc.). - - - AFR_DATA_PENDING - There is some data pending on this inode (writev). - - - AFR_ENTRY_PENDING - There are some directory operations pending on this directory - (create, unlink, etc.). - ------------ -* Self heal ------------ - - - On lookup, gather extended attribute data: - - If entry is a regular file: - - If an entry is present on one child and not on others: - - create entry on others. - - If entries exist but have different metadata (perms, etc.): - - consider the entry with the highest AFR_METADATA_PENDING number as - definitive and replicate its attributes on children. - - - If entry is a directory: - - Consider the entry with the higest AFR_ENTRY_PENDING number as - definitive and replicate its contents on all children. - - - If any two entries have non-matching types (i.e., one is file and - other is directory): - - Announce to the user via log that a split-brain situation has been - detected, and do nothing. - - - On open, gather extended attribute data: - - Consider the file with the highest AFR_DATA_PENDING number as - the definitive one and replicate its contents on all other - children. - - During all self heal operations, appropriate locks must be held on all - regions/entries being affected. - ---------------- -* Inode scaling ---------------- - -Inode scaling is necessary because if a situation arises where: - - An inode number is returned for a directory (by lookup) which was - previously the inode number of a file (as per FUSE's table), then - FUSE gets horribly confused (consult a FUSE expert for more details). - -To avoid such a situation, we distribute the 64-bit inode space equally -among all children of replicate. - -To illustrate: - -If c1, c2, c3 are children of replicate, they each get 1/3 of the available -inode space: - -Child: c1 c2 c3 c1 c2 c3 c1 c2 c3 c1 c2 ... -Inode number: 1 2 3 4 5 6 7 8 9 10 11 ... - -Thus, if lookup on c1 returns an inode number "2", it is scaled to "4" -(which is the second inode number in c1's space). - -This way we ensure that there is never a collision of inode numbers from -two different children. - -This reduction of inode space doesn't really reduce the usability of -replicate since even if we assume replicate has 1024 children (which would be a -highly unusual scenario), each child still has a 54-bit inode space. - -2^54 ~ 1.8 * 10^16 - -which is much larger than any real world requirement. - - -============================================== -$ Last updated: Sun Oct 12 23:17:01 IST 2008 $ -$ Author: Vikas Gorur <vikas@gluster.com> $ -============================================== - diff --git a/doc/legacy/hacker-guide/write-behind.txt b/doc/legacy/hacker-guide/write-behind.txt deleted file mode 100644 index 50b7d2a1d07..00000000000 --- a/doc/legacy/hacker-guide/write-behind.txt +++ /dev/null @@ -1,45 +0,0 @@ -basic working --------------- - - write behind is basically a translator to lie to the application that the write-requests are finished, even before it is actually finished. - - on a regular translator tree without write-behind, control flow is like this: - - 1. application makes a write() system call. - 2. VFS ==> FUSE ==> /dev/fuse. - 3. fuse-bridge initiates a glusterfs writev() call. - 4. writev() is STACK_WIND()ed upto client-protocol or storage translator. - 5. client-protocol, on receiving reply from server, starts STACK_UNWIND() towards the fuse-bridge. - - on a translator tree with write-behind, control flow is like this: - - 1. application makes a write() system call. - 2. VFS ==> FUSE ==> /dev/fuse. - 3. fuse-bridge initiates a glusterfs writev() call. - 4. writev() is STACK_WIND()ed upto write-behind translator. - 5. write-behind adds the write buffer to its internal queue and does a STACK_UNWIND() towards the fuse-bridge. - - write call is completed in application's percepective. after STACK_UNWIND()ing towards the fuse-bridge, write-behind initiates a fresh writev() call to its child translator, whose replies will be consumed by write-behind itself. write-behind _doesn't_ cache the write buffer, unless 'option flush-behind on' is specified in volume specification file. - -windowing ---------- - - write respect to write-behind, each write-buffer has three flags: 'stack_wound', 'write_behind' and 'got_reply'. - - stack_wound: if set, indicates that write-behind has initiated STACK_WIND() towards child translator. - - write_behind: if set, indicates that write-behind has done STACK_UNWIND() towards fuse-bridge. - - got_reply: if set, indicates that write-behind has received reply from child translator for a writev() STACK_WIND(). a request will be destroyed by write-behind only if this flag is set. - - currently pending write requests = aggregate size of requests with write_behind = 1 and got_reply = 0. - - window size limits the aggregate size of currently pending write requests. once the pending requests' size has reached the window size, write-behind blocks writev() calls from fuse-bridge. - blocking is only from application's perspective. write-behind does STACK_WIND() to child translator straight-away, but hold behind the STACK_UNWIND() towards fuse-bridge. STACK_UNWIND() is done only once write-behind gets enough replies to accomodate for currently blocked request. - -flush behind ------------- - - if 'option flush-behind on' is specified in volume specification file, then write-behind sends aggregate write requests to child translator, instead of regular per request STACK_WIND()s. - - diff --git a/doc/legacy/handling-options.txt b/doc/legacy/handling-options.txt deleted file mode 100644 index 9a3b2510acb..00000000000 --- a/doc/legacy/handling-options.txt +++ /dev/null @@ -1,13 +0,0 @@ - -How to add a new option to a given volume ? -=========================================== - -* Add a entry in 'struct volume_options options[]' with your key, what is - the type of the 'key', etc. - -* The 'key' and corresponding 'value' given for the same by user are validated - before calling init() of the translator/transport/scheduler/auth-module. - -* Once the complete init() is successful, user will get a warning if he has - given a 'key' which is not defined in these modules. - diff --git a/doc/legacy/mac-related-xattrs.txt b/doc/legacy/mac-related-xattrs.txt deleted file mode 100644 index 92bb2ceef2d..00000000000 --- a/doc/legacy/mac-related-xattrs.txt +++ /dev/null @@ -1,21 +0,0 @@ - -This document is intended to briefly explain how the Extended Attributes on -Darwin 10.5.x releases works ----- - -On Darwin other than all the normal filesystem operations, 'Finder' (like -Explorer in Windows but a little more) keeps its information in two extended -attributes named 'com.apple.FinderInfo' and 'com.apple.ResourceFork'. If these -xattrs are not implemented the filesystem won't be shown on Finder, and if they -are not implemented properly there may be issues when some of the file operations -are done through GUI of Finder. But when a filesystem is used over mountpoint in a -terminal, everything is fine and these xattrs are not required. - -Currently the way these xattrs are implemented is simple. All the xattr calls -(getxattr, setxattr, listxattr, removexattr) are passed down to underlaying filesystem, -most of the cases when exported FS is on MacOS X itself, these keys are supported, hence -the fops succeed. But in the case of using exports of different OS on Darwin the issue is -extended attribute prefix like 'com.apple.' may not be supported, hence the problem with -Finder. To solve this issue, GlusterFS returns virtual default values to these keys, which -works fine on most of the cases. - diff --git a/doc/legacy/porting_guide.txt b/doc/legacy/porting_guide.txt deleted file mode 100644 index 5705cd96461..00000000000 --- a/doc/legacy/porting_guide.txt +++ /dev/null @@ -1,45 +0,0 @@ - GlusterFS Porting Guide - ----------------------- - -* General setup - -The configure script will detect the target platform for the build. -All platform-specific CFLAGS, macro definitions should be done -in configure.ac - -Platform-specific code can be written like this: - -#ifdef GF_DARWIN_HOST_OS - /* some code specific to Darwin */ -#endif - -* Coding guidelines - -In general, avoid glibc extensions. For example, nested functions don't work -on Mac OS X. It is best to stick to C99. - -When using library calls and system calls, pay attention to the -portability notes. As far as possible stick to POSIX-specified behavior. -Do not use anything expressly permitted by the specification. For example, -some fields in structures may be present only on certain platforms. Avoid -use of such things. - -Do not pass values of constants such as F_*, O_*, errno values, etc. across -platforms. - -Please refer compat-errno.h for more details about errno handling inside -glusterfs for cross platform. - -* Specific issues - -- The argp library is available only on Linux through glibc, but for other - platforms glusterfs has already included argp-standalone library which will - statically linked during the glusterfs build. - -- Extended attribute calls (setxattr, listxattr, etc.) have differing prototypes - on different platforms. See compat.h for macro definitions to resolve this, also - read out the specific extended attribute documentation for your platforms. - ------------------------------------------- -Last revised: Thu Feb 28 13:58:07 IST 2008 ------------------------------------------- diff --git a/doc/legacy/replicate.lyx b/doc/legacy/replicate.lyx deleted file mode 100644 index 58ba6b2e00e..00000000000 --- a/doc/legacy/replicate.lyx +++ /dev/null @@ -1,797 +0,0 @@ -#LyX 1.4.2 created this file. For more info see http://www.lyx.org/ -\lyxformat 245 -\begin_document -\begin_header -\textclass article -\language english -\inputencoding auto -\fontscheme default -\graphics default -\paperfontsize default -\spacing single -\papersize default -\use_geometry false -\use_amsmath 1 -\cite_engine basic -\use_bibtopic false -\paperorientation portrait -\secnumdepth 3 -\tocdepth 3 -\paragraph_separation skip -\defskip medskip -\quotes_language english -\papercolumns 1 -\papersides 1 -\paperpagestyle default -\tracking_changes false -\output_changes false -\end_header - -\begin_body - -\begin_layout Title - -\size larger -Automatic File Replication (replicate) in GlusterFS -\end_layout - -\begin_layout Author -Vikas Gorur -\family typewriter -\size larger -<vikas@gluster.com> -\end_layout - -\begin_layout Standard -\begin_inset ERT -status open - -\begin_layout Standard - - -\backslash -hrule -\end_layout - -\end_inset - - -\end_layout - -\begin_layout Section* -Overview -\end_layout - -\begin_layout Standard -This document describes the design and usage of the replicate translator in GlusterFS. - This document is valid for the 1.4.x releases, and not earlier ones. -\end_layout - -\begin_layout Standard -The replicate translator of GlusterFS aims to keep identical copies of a file - on all its subvolumes, as far as possible. - It tries to do this by performing all filesystem mutation operations (writing - data, creating files, changing ownership, etc.) on all its subvolumes in - such a way that if an operation succeeds on atleast one subvolume, all - other subvolumes can later be brought up to date. -\end_layout - -\begin_layout Standard -In the rest of the document the terms -\begin_inset Quotes eld -\end_inset - -subvolume -\begin_inset Quotes erd -\end_inset - - and -\begin_inset Quotes eld -\end_inset - -server -\begin_inset Quotes erd -\end_inset - - are used interchangeably, trusting that it will cause no confusion to the - reader. -\end_layout - -\begin_layout Section* -Usage -\end_layout - -\begin_layout Standard -A sample volume declaration for replicate looks like this: -\end_layout - -\begin_layout Standard -\begin_inset ERT -status open - -\begin_layout Standard - - -\backslash -begin{verbatim} -\end_layout - -\begin_layout Standard - -volume replicate -\end_layout - -\begin_layout Standard - - type cluster/replicate -\end_layout - -\begin_layout Standard - - # options, see below for description -\end_layout - -\begin_layout Standard - - subvolumes brick1 brick2 -\end_layout - -\begin_layout Standard - -end-volume -\end_layout - -\begin_layout Standard - - -\backslash -end{verbatim} -\end_layout - -\begin_layout Standard - -\end_layout - -\begin_layout Standard - -\end_layout - -\begin_layout Standard - -\end_layout - -\end_inset - - -\end_layout - -\begin_layout Standard -This defines an replicate volume with two subvolumes, brick1, and brick2. - For replicate to work properly, it is essential that its subvolumes support -\series bold -extended attributes -\series default -. - This means that you should choose a backend filesystem that supports extended - attributes, like XFS, ReiserFS, or Ext3. -\end_layout - -\begin_layout Standard -The storage volumes used as backend for replicate -\emph on -must -\emph default - have a posix-locks volume loaded above them. -\end_layout - -\begin_layout Standard -\begin_inset ERT -status open - -\begin_layout Standard - - -\backslash -begin{verbatim} -\end_layout - -\begin_layout Standard - -volume brick1 -\end_layout - -\begin_layout Standard - - type features/posix-locks -\end_layout - -\begin_layout Standard - - subvolumes brick1-ds -\end_layout - -\begin_layout Standard - -end-volume -\end_layout - -\begin_layout Standard - - -\backslash -end{verbatim} -\end_layout - -\end_inset - - -\end_layout - -\begin_layout Section* -Design -\end_layout - -\begin_layout Subsection* -Read algorithm -\end_layout - -\begin_layout Standard -All operations that do not modify the file or directory are sent to all - the subvolumes and the first successful reply is returned to the application. -\end_layout - -\begin_layout Standard -The read() system call (reading data from a file) is an exception. - For read() calls, replicate tries to do load balancing by sending all reads from - a particular file to a particular server. -\end_layout - -\begin_layout Standard -The read algorithm is also affected by the option read-subvolume; see below - for details. -\end_layout - -\begin_layout Subsection* -Classes of file operations -\end_layout - -\begin_layout Standard -replicate divides all filesystem write operations into three classes: -\end_layout - -\begin_layout Itemize - -\series bold -data: -\series default -Operations that modify the contents of a file (write, truncate). -\end_layout - -\begin_layout Itemize - -\series bold -metadata: -\series default -Operations that modify attributes of a file or directory (permissions, ownership -, etc.). -\end_layout - -\begin_layout Itemize - -\series bold -entry: -\series default -Operations that create or delete directory entries (mkdir, create, rename, - rmdir, unlink, etc.). -\end_layout - -\begin_layout Subsection* -Locking and Change Log -\end_layout - -\begin_layout Standard -To ensure consistency across subvolumes, replicate holds a lock whenever a modificatio -n is being made to a file or directory. - By default, replicate considers the first subvolume as the sole lock server. - However, the number of lock servers can be increased upto the total number - of subvolumes. -\end_layout - -\begin_layout Standard -The change log is a set of extended attributes associated with files and - directories that replicate maintains. - The change log keeps track of the changes made to files and directories - (data, metadata, entry) so that the self-heal algorithm knows which copy - of a file or directory is the most recent one. -\end_layout - -\begin_layout Subsection* -Write algorithm -\end_layout - -\begin_layout Standard -The algorithm for all write operations (data, metadata, entry) is: -\end_layout - -\begin_layout Enumerate -Lock the file (or directory) on all of the lock servers (see options below). -\end_layout - -\begin_layout Enumerate -Write change log entries on all servers. -\end_layout - -\begin_layout Enumerate -Perform the operation. -\end_layout - -\begin_layout Enumerate -Erase change log entries. -\end_layout - -\begin_layout Enumerate -Unlock the file (or directory) on all of the lock servers. -\end_layout - -\begin_layout Standard -The above algorithm is a simplified version intended for general users. - Please refer to the source code for the full details. -\end_layout - -\begin_layout Subsection* -Self-Heal -\end_layout - -\begin_layout Standard -replicate automatically tries to fix any inconsistencies it detects among different - copies of a file. - It uses information in the change log to determine which copy is the -\begin_inset Quotes eld -\end_inset - -correct -\begin_inset Quotes erd -\end_inset - - version. -\end_layout - -\begin_layout Standard -Self-heal is triggered when a file or directory is first -\begin_inset Quotes eld -\end_inset - -accessed -\begin_inset Quotes erd -\end_inset - -, that is, the first time any operation is attempted on it. - The self-heal algorithm does the following things: -\end_layout - -\begin_layout Standard -If the entry being accessed is a directory: -\end_layout - -\begin_layout Itemize -The contents of the -\begin_inset Quotes eld -\end_inset - -correct -\begin_inset Quotes erd -\end_inset - - version is replicated on all subvolumes, by deleting entries and creating - entries as necessary. -\end_layout - -\begin_layout Standard -If the entry being accessed is a file: -\end_layout - -\begin_layout Itemize -If the file does not exist on some subvolumes, it is created. -\end_layout - -\begin_layout Itemize -If there is a mismatch in the size of the file, or ownership, or permission, - it is fixed. -\end_layout - -\begin_layout Itemize -If the change log indicates that some copies need updating, they are updated. -\end_layout - -\begin_layout Subsection* -Split-brain -\end_layout - -\begin_layout Standard -It may happen that one replicate client can access only some of the servers in - a cluster and another replicate client can access the remaining servers. - Or it may happen that in a cluster of two servers, one server goes down - and comes back up, but the other goes down immediately. - Both these scenarios result in a -\begin_inset Quotes eld -\end_inset - -split-brain -\begin_inset Quotes erd -\end_inset - -. -\end_layout - -\begin_layout Standard -In a split-brain situation, there will be two or more copies of a file, - all of which are -\begin_inset Quotes eld -\end_inset - -correct -\begin_inset Quotes erd -\end_inset - - in some sense. - replicate without manual intervention has no way of knowing what to do, since - it cannot consider any single copy as definitive, nor does it know of any - meaningful way to merge the copies. -\end_layout - -\begin_layout Standard -If replicate detects that a split-brain has happened on a file, it disallows opening - of that file. - You will have to manually resolve the conflict by deleting all but one - copy of the file. - Alternatively you can set an automatic split-brain resolution policy by - using the `favorite-child' option (see below). -\end_layout - -\begin_layout Section* -Translator Options -\end_layout - -\begin_layout Standard -replicate accepts the following options: -\end_layout - -\begin_layout Subsection* -read-subvolume (default: none) -\end_layout - -\begin_layout Standard -The value of this option must be the name of a subvolume. - If given, all read operations are sent to only the specified subvolume, - instead of being balanced across all subvolumes. -\end_layout - -\begin_layout Subsection* -favorite-child (default: none) -\end_layout - -\begin_layout Standard -The value of this option must be the name of a subvolume. - If given, the specified subvolume will be preferentially used in resolving - conflicts ( -\begin_inset Quotes eld -\end_inset - -split-brain -\begin_inset Quotes erd -\end_inset - -). - This means if a discrepancy is noticed in the attributes or content of - a file, the copy on the `favorite-child' will be considered the definitive - version and its contents will -\emph on -overwrite -\emph default -the contents of all other copies. - Use this option with caution! It is possible to -\emph on -lose data -\emph default - with this option. - If you are in doubt, do not specify this option. -\end_layout - -\begin_layout Subsection* -Self-heal options -\end_layout - -\begin_layout Standard -Setting any of these options to -\begin_inset Quotes eld -\end_inset - -off -\begin_inset Quotes erd -\end_inset - - prevents that kind of self-heal from being done on a file or directory. - For example, if metadata self-heal is turned off, permissions and ownership - are no longer fixed automatically. -\end_layout - -\begin_layout Subsubsection* -data-self-heal (default: on) -\end_layout - -\begin_layout Standard -Enable/disable self-healing of file contents. -\end_layout - -\begin_layout Subsubsection* -metadata-self-heal (default: off) -\end_layout - -\begin_layout Standard -Enable/disable self-healing of metadata (permissions, ownership, modification - times). -\end_layout - -\begin_layout Subsubsection* -entry-self-heal (default: on) -\end_layout - -\begin_layout Standard -Enable/disable self-healing of directory entries. -\end_layout - -\begin_layout Subsection* -Change Log options -\end_layout - -\begin_layout Standard -If any of these options is turned off, it disables writing of change log - entries for that class of file operations. - That is, steps 2 and 4 of the write algorithm (see above) are not done. - Note that if the change log is not written, the self-heal algorithm cannot - determine the -\begin_inset Quotes eld -\end_inset - -correct -\begin_inset Quotes erd -\end_inset - - version of a file and hence self-heal will only be able to fix -\begin_inset Quotes eld -\end_inset - -obviously -\begin_inset Quotes erd -\end_inset - - wrong things (such as a file existing on only one node). -\end_layout - -\begin_layout Subsubsection* -data-change-log (default: on) -\end_layout - -\begin_layout Standard -Enable/disable writing of change log for data operations. -\end_layout - -\begin_layout Subsubsection* -metadata-change-log (default: on) -\end_layout - -\begin_layout Standard -Enable/disable writing of change log for metadata operations. -\end_layout - -\begin_layout Subsubsection* -entry-change-log (default: on) -\end_layout - -\begin_layout Standard -Enable/disable writing of change log for entry operations. -\end_layout - -\begin_layout Subsection* -Locking options -\end_layout - -\begin_layout Standard -These options let you specify the number of lock servers to use for each - class of file operations. - The default values are satisfactory in most cases. - If you are extra paranoid, you may want to increase the values. - However, be very cautious if you set the data- or entry- lock server counts - to zero, since this can result in -\emph on -lost data. - -\emph default - For example, if you set the data-lock-server-count to zero, and two application -s write to the same region of a file, there is a possibility that none of - your servers will have all the data. - In other words, the copies will be -\emph on -inconsistent -\emph default -, and -\emph on -incomplete -\emph default -. - Do not set data- and entry- lock server counts to zero unless you absolutely - know what you are doing and agree to not hold GlusterFS responsible for - any lost data. -\end_layout - -\begin_layout Subsubsection* -data-lock-server-count (default: 1) -\end_layout - -\begin_layout Standard -Number of lock servers to use for data operations. -\end_layout - -\begin_layout Subsubsection* -metadata-lock-server-count (default: 0) -\end_layout - -\begin_layout Standard -Number of lock servers to use for metadata operations. -\end_layout - -\begin_layout Subsubsection* -entry-lock-server-count (default: 1) -\end_layout - -\begin_layout Standard -Number of lock servers to use for entry operations. -\end_layout - -\begin_layout Section* -Known Issues -\end_layout - -\begin_layout Subsection* -Self-heal of file with more than one link (hard links): -\end_layout - -\begin_layout Standard -Consider two servers, A and B. - Assume A is down, and the user creates a file `new' as a hard link to a - file `old'. - When A comes back up, replicate will see that the file `new' does not exist on - A, and self-heal will create the file and copy the contents from B. - However, now on server A the file `new' is not a link to the file `old' - but an entirely different file. -\end_layout - -\begin_layout Standard -We know of no easy way to fix this problem, but we will try to fix it in - forthcoming releases. -\end_layout - -\begin_layout Subsection* -File re-opening after a server comes back up: -\end_layout - -\begin_layout Standard -If a server A goes down and comes back up, any files which were opened while - A was down and are still open will not have their writes replicated on - A. - In other words, data replication only happens on those servers which were - alive when the file was opened. -\end_layout - -\begin_layout Standard -This is a rather tricky issue but we hope to fix it very soon. -\end_layout - -\begin_layout Section* -Frequently Asked Questions -\end_layout - -\begin_layout Subsection* -1. - How can I force self-heal to happen? -\end_layout - -\begin_layout Standard -You can force self-heal to happen on your cluster by running a script or - a command that accesses every file. - A simple way to do it would be: -\end_layout - -\begin_layout Standard -\begin_inset ERT -status open - -\begin_layout Standard - -\end_layout - -\begin_layout Standard - - -\backslash -begin{verbatim} -\end_layout - -\begin_layout Standard - -$ ls -lR -\end_layout - -\begin_layout Standard - - -\backslash -end{verbatim} -\end_layout - -\begin_layout Standard - -\end_layout - -\end_inset - - -\end_layout - -\begin_layout Standard -Run the command in all directories which you want to forcibly self-heal. -\end_layout - -\begin_layout Subsection* -2. - Which backend filesystem should I use for replicate? -\end_layout - -\begin_layout Standard -You can use any backend filesystem that supports extended attributes. - We know of users successfully using XFR, ReiserFS, and Ext3. -\end_layout - -\begin_layout Subsection* -3. - What can I do to improve replicate performance? -\end_layout - -\begin_layout Standard -Try loading performance translators such as io-threads, write-behind, io-cache, - and read-ahead depending on your workload. - If you are willing to sacrifice correctness in corner cases, you can experiment - with the lock-server-count and the change-log options (see above). - As warned earlier, be very careful! -\end_layout - -\begin_layout Subsection* -4. - How can I selectively replicate files? -\end_layout - -\begin_layout Standard -There is no support for selective replication in replicate itself. - You can achieve selective replication by loading the unify translator over - replicate, and using the switch scheduler. - Configure unify with two subvolumes, one of them being replicate. - Using the switch scheduler, schedule all files for which you need replication - to the replicate subvolume. - Consult unify and switch documentation for more details. -\end_layout - -\begin_layout Section* -Contact -\end_layout - -\begin_layout Standard -If you need more assistance on replicate, contact us on the mailing list <gluster-user -s@gluster.org> (visit gluster.org for details on how to subscribe). -\end_layout - -\begin_layout Standard -Send you comments and suggestions about this document to <vikas@gluster.com>. -\end_layout - -\end_body -\end_document diff --git a/doc/legacy/replicate.pdf b/doc/legacy/replicate.pdf Binary files differdeleted file mode 100644 index b7212af2b4e..00000000000 --- a/doc/legacy/replicate.pdf +++ /dev/null diff --git a/doc/legacy/solaris-related-xattrs.txt b/doc/legacy/solaris-related-xattrs.txt deleted file mode 100644 index 3a4643948c0..00000000000 --- a/doc/legacy/solaris-related-xattrs.txt +++ /dev/null @@ -1,44 +0,0 @@ - Solaris Extended Attributes - -In solaris extended attributes are logically supported as files -within the filesystem. The file system is therefore augmented -with an orthogonal namespace of file attributes. Attribute values -are accessed by file descriptors obtained through a special attribute -interface. This type of logical view of "attributes as files" allows -the leveraging of existing file system interface functionality to -support the construction, deletion and manipulation of attributes. - -But as we have tested through this functionality provided by Solaris -we have come accross two major issues as written below. - -1. Symlink XATTR_NOFOLLOW not present for creating extended attributes - directly on the symlinks like other platforms Linux,MAC-OSX,BSD etc. - An implementation is present for O_NOFOLLOW for "openat()" call sets - up errno ELOOP whenever encountered with a symlink and also another - implementation AT_SYMLINK_NOFOLLOW which is not present for calls like - "attropen(), openat()" - - a snippet of test code which helped us understand this behaviour - -------------------------------------- - attrfd = attropen (path, key, - flags|AT_SYMLINK_NOFOLLOW|O_CREAT|O_WRONLY|O_NOFOLLOW, 0777); - if (attrfd >= 0) { - ftruncate (attrfd, 0); - ret = write (attrfd, value, size); - close (attrfd); - } else { - fprintf (stderr, "Couldn't set extended attribute for %s (%d)\n", - path, errno); - } - -------------------------------------- - -2. Extended attribute support for special files like device files, fifo files - is not supported under solaris. - -Apart from these glitches almost everything regarding porting functionality -for extended attribute calls has been properly implemented in compat.c -with writing wrapper around functions over -"attropen()", "openat()", "unlinkat()" - - - diff --git a/doc/legacy/stat-prefetch-design.txt b/doc/legacy/stat-prefetch-design.txt deleted file mode 100644 index 68ed423d3dd..00000000000 --- a/doc/legacy/stat-prefetch-design.txt +++ /dev/null @@ -1,154 +0,0 @@ -what is stat-prefetch? -====================== -It is a translator which caches the dentries read in readdir. This dentry -list is stored in the context of fd. Later when lookup happens on -[parent-inode, basename (path)] combination, this list is searched for the -basename. The dentry thus searched is used to fill up the stat corresponding -to path being looked upon, thereby short-cutting lookup calls. This cache is -preserved till closedir is called on the fd. The purpose of this translator -is to optimize operations like 'ls -l', where a readdir is followed by -lookup (stat) calls on each directory entry. - -1. stat-prefetch harnesses the efficiency of short lookup calls - (saves network roundtrip time for lookup calls from being accounted to - the stat call). -2. To maintain the correctness, it does lookup-behind - lookup is winded to - underlying translators after it is unwound to upper translators. - lookup-behind is necessary as inode gets populated in server inode table - only in lookup-cbk and also because various translators store their - contexts in inode contexts during lookup calls. - -fops to be implemented: -======================= -* lookup - 1. check the dentry cache stored in context of fds opened by the same process - on parent inode for basename. If found unwind with cached stat, else wind - the lookup call to underlying translators. - 2. stat is stored in the context of inode if the path being looked upon - happens to be directory. This stat will be used to fill postparent stat - when lookup happens on any of the directory contents. - -* readdir - 1. cache the direntries returned in readdir_cbk in the context of fd. - 2. if the readdir is happening on non-expected offsets (means a seekdir/rewinddir - has happened), cache has to be flushed. - 3. delete the entry corresponding to basename of path on which fd is opened - from cache stored in parent. - -* chmod/fchmod - delete the entry corresponding to basename from cache stored in context of - fds opened on parent inode, since these calls change st_mode and st_ctime of - stat. - -* chown/fchown - delete the entry corresponding to basename from cache stored in context of - fds opened on parent inode, since these calls change st_uid/st_gid and - st_ctime of stat. - -* truncate/ftruncate - delete the entry corresponding to basename from cache stored in context of - fds opened on parent inode, since these calls change st_size/st_mtime of stat. - -* utimens - delete the entry corresponding to basename from cache stored in context of - fds opened on parent inode, since this call changes st_atime/st_mtime of stat. - -* readlink - delete the entry corresponding to basename from cache stored in context of fds - opened on parent inode, since this call changes st_atime of stat. - -* unlink - 1. delete the entry corresponding to basename from cache stored in context of - fds, opened on parent directory containing the file being unlinked. - 2. delete the entry corresponding to basename of parent directory from cache - of grand-parent. - -* rmdir - 1. delete the entry corresponding to basename from cache stored in context of - fds opened on parent inode. - 2. remove the entire cache from all fds opened on inode corresponding to - directory being removed. - 3. delete the entry correspondig to basename of parent from cache stored in - grand-parent. - -* readv - delete the entry corresponding to basename from cache stored in context of fds - opened on parent inode, since readv changes st_atime of file. - -* writev - delete the entry corresponding to basename from cache stored in context of fds - opened on parent inode, since writev can possibly change st_size and definitely - changes st_mtime of file. - -* fsync - there is a confusion here as to whether fsync updates mtime/ctimes. Disk based - filesystems (atleast ext2) just writes the times stored in inode to disk - during fsync and not the time at which fsync is being done. But in glusterfs, - a translator like write-behind actually sends writes during fsync which will - change mtime/ctime. Hence stat-prefetch implements fsync to delete the entry - corresponding to basename from cache stored in context of fds opened on parent - inode. - -* rename - 1. remove entry corresponding to oldname from cache stored in fd contexts of - oldparent. - 2. remove entry corresponding to newname from cache stored in fd contexts of - newparent. - 3. remove entry corresponding to oldparent from cache stored in - old-grand-parent, since removing oldname changes st_mtime and st_ctime - of oldparent stat. - 4. remove entry corresponding to newparent from cache stored in - new-grand-parent, since adding newname changes st_mtime and st_ctime - of newparent stat. - 5. if oldname happens to be a directory, remove entire cache from all fds - opened on it. - -* create/mknod/mkdir/symlink/link - delete entry corresponding to basename of parent directory in which these - operations are happening, from cache stored in context of fds opened on - grand-parent, since adding a new entry to a directory changes st_mtime - and st_ctime of parent directory. - -* setxattr/removexattr - delete the entry corresponding to basename from cache stored in context of - fds opened on parent inode, since setxattr changes st_ctime of file. - -* setdents - 1. remove entry corresponding to basename of path on which fd is opened from - cache stored in context of fds opened on parent. - 2. for each of the entry in the direntry list, delete from cache stored in - context of fd, the entry corresponding to basename of path being passed. - -* getdents - 1. remove entry corresponding to basename of path on which fd is opened from - cache stored in parent, since getdents changes st_atime. - 2. remove entries corresponding to symbolic links from cache, since readlink - would've changed st_atime. - -* checksum - delete the entry corresponding to basename from cache stored in context of - fds opened on parent inode, since st_atime is changed during this call. - -* xattrop/fxattrop - delete the entry corresponding to basename from cache stored in context of fds - opened on parent inode, since these calls modify st_ctime of file. - -callbacks to be implemented: -============================ -* releasedir - free the context stored in fd. - -* forget - dree the stat if the inode corresponds to a directory. - -limitations: -============ -* since a readdir does not return extended attributes of file, if need_xattr is - set, short-cutting of lookup does not happen and lookup is passed to - underlying translators. - -* posix_readdir does not check whether the dentries are spanning across multiple - mount points. Hence it is not transforming inode numbers in stat buffers if - posix is configured to allow export directory spanning on multiple mountpoints. - This is a bug which needs to be fixed. posix_readdir should treat dentries the - same way as if lookup is happening on dentries. diff --git a/doc/legacy/translator-options.txt b/doc/legacy/translator-options.txt deleted file mode 100644 index 3422c058a5d..00000000000 --- a/doc/legacy/translator-options.txt +++ /dev/null @@ -1,224 +0,0 @@ -mount/fuse: - * direct-io-mode GF_OPTION_TYPE_BOOL on|off|yes|no - * mount-point (mountpoint) GF_OPTION_TYPE_PATH <any-posix-valid-path> - * attribute-timeout GF_OPTION_TYPE_DOUBLE 0.0 - * entry-timeout GF_OPTION_TYPE_DOUBLE 0.0 - -protocol/server: - * transport-type GF_OPTION_TYPE_STR tcp|socket|ib-verbs|unix|ib-sdp| - tcp/client|ib-verbs/client - * volume-filename.* GF_OPTION_TYPE_PATH - * inode-lru-limit GF_OPTION_TYPE_INT 0-(1 * GF_UNIT_MB) - * client-volume-filename GF_OPTION_TYPE_PATH - -protocol/client: - * username GF_OPTION_TYPE_ANY - * password GF_OPTION_TYPE_ANY - * transport-type GF_OPTION_TYPE_STR tcp|socket|ib-verbs|unix|ib-sdp| - tcp/client|ib-verbs/client - * remote-host GF_OPTION_TYPE_ANY - * remote-subvolume GF_OPTION_TYPE_ANY - * transport-timeout GF_OPTION_TYPE_TIME 5-1013 - -cluster/replicate: - * read-subvolume GF_OPTION_TYPE_XLATOR - * favorite-child GF_OPTION_TYPE_XLATOR - * data-self-heal GF_OPTION_TYPE_BOOL - * metadata-self-heal GF_OPTION_TYPE_BOOL - * entry-self-heal GF_OPTION_TYPE_BOOL - * data-change-log GF_OPTION_TYPE_BOOL - * metadata-change-log GF_OPTION_TYPE_BOOL - * entry-change-log GF_OPTION_TYPE_BOOL - * data-lock-server-count GF_OPTION_TYPE_INT 0 - * metadata-lock-server-count GF_OPTION_TYPE_INT 0 - * entry-lock-server-count GF_OPTION_TYPE_INT 0 - -cluster/distribute: - * lookup-unhashed GF_OPTION_TYPE_BOOL - -cluster/unify: - * namespace GF_OPTION_TYPE_XLATOR - * scheduler GF_OPTION_TYPE_STR alu|rr|random|nufa|switch - * self-heal GF_OPTION_TYPE_STR foreground|background|off - * optimist GF_OPTION_TYPE_BOOL - -cluster/nufa: - local-volume-name GF_OPTION_TYPE_XLATOR - -cluster/stripe: - * block-size GF_OPTION_TYPE_ANY - * use-xattr GF_OPTION_TYPE_BOOL - -debug/trace: - * include-ops (include) GF_OPTION_TYPE_STR - * exclude-ops (exclude) GF_OPTION_TYPE_STR - -encryption/rot-13: - * encrypt-write GF_OPTION_TYPE_BOOL - * decrypt-read GF_OPTION_TYPE_BOOL - -features/path-convertor: - * start-offset GF_OPTION_TYPE_INT 0-4095 - * end-offset GF_OPTION_TYPE_INT 1-4096 - * replace-with GF_OPTION_TYPE_ANY - -features/trash: - * trash-dir GF_OPTION_TYPE_PATH - -features/locks: - * mandatory-locks (mandatory) GF_OPTION_TYPE_BOOL - -features/filter: - * root-squashing GF_OPTION_TYPE_BOOL - * read-only GF_OPTION_TYPE_BOOL - * fixed-uid GF_OPTION_TYPE_INT - * fixed-gid GF_OPTION_TYPE_INT - * translate-uid GF_OPTION_TYPE_ANY - * translate-gid GF_OPTION_TYPE_ANY - * filter-uid GF_OPTION_TYPE_ANY - * filter-gid GF_OPTION_TYPE_ANY - -features/quota: - * min-free-disk-limit GF_OPTION_TYPE_PERCENT - * refresh-interval GF_OPTION_TYPE_TIME - * disk-usage-limit GF_OPTION_TYPE_SIZET - -storage/posix: - * o-direct GF_OPTION_TYPE_BOOL - * directory GF_OPTION_TYPE_PATH - * export-statfs-size GF_OPTION_TYPE_BOOL - * mandate-attribute GF_OPTION_TYPE_BOOL - -storage/bdb: - * directory GF_OPTION_TYPE_PATH - * logdir GF_OPTION_TYPE_PATH - * errfile GF_OPTION_TYPE_PATH - * dir-mode GF_OPTION_TYPE_ANY - * file-mode GF_OPTION_TYPE_ANY - * page-size GF_OPTION_TYPE_SIZET - * lru-limit GF_OPTION_TYPE_INT - * lock-timeout GF_OPTION_TYPE_TIME - * checkpoint-timeout GF_OPTION_TYPE_TIME - * transaction-timeout GF_OPTION_TYPE_TIME - * mode GF_OPTION_TYPE_BOOL - * access-mode GF_OPTION_TYPE_STR - -performance/read-ahead: - * force-atime-update GF_OPTION_TYPE_BOOL - * page-size GF_OPTION_TYPE_SIZET (64 * GF_UNIT_KB)-(2 * GF_UNIT_MB) - * page-count GF_OPTION_TYPE_INT 1-16 - -performance/write-behind: - * flush-behind GF_OPTION_TYPE_BOOL - * aggregate-size GF_OPTION_TYPE_SIZET (128 * GF_UNIT_KB)-(4 * GF_UNIT_MB) - * window-size GF_OPTION_TYPE_SIZET (512 * GF_UNIT_KB)-(1 * GF_UNIT_GB) - * enable-O_SYNC GF_OPTION_TYPE_BOOL - * disable-for-first-nbytes GF_OPTION_TYPE_SIZET 1 - (1 * GF_UNIT_MB) - -performance/symlink-cache: - -performance/io-threads: - * thread-count GF_OPTION_TYPE_INT 1-32 - -performance/io-cache: - * priority GF_OPTION_TYPE_ANY - * cache-timeout (force-revalidate-timeout) GF_OPTION_TYPE_INT 0-60 - * page-size GF_OPTION_TYPE_SIZET (16 * GF_UNIT_KB)-(4 * GF_UNIT_MB) - * cache-size GF_OPTION_TYPE_SIZET (4 * GF_UNIT_MB)-(6 * GF_UNIT_GB) - -performance/quick-read: - * cache-timeout GF_OPTION_TYPE_INT 1-60 - * max-file-size GF_OPTION_TYPE_SIZET 0-(1000 * GF_UNIT_KB) - -auth: -- addr: - * auth.addr.*.allow GF_OPTION_TYPE_ANY - * auth.addr.*.reject GF_OPTION_TYPE_ANY - -- login: - * auth.login.*.allow GF_OPTION_TYPE_ANY - * auth.login.*.password GF_OPTION_TYPE_ANY - -scheduler/alu: - * scheduler.alu.order (alu.order) - GF_OPTION_TYPE_ANY - * scheduler.alu.disk-usage.entry-threshold (alu.disk-usage.entry-threshold) - GF_OPTION_TYPE_SIZET - * scheduler.alu.disk-usage.exit-threshold (alu.disk-usage.exit-threshold) - GF_OPTION_TYPE_SIZET - * scheduler.alu.write-usage.entry-threshold (alu.write-usage.entry-threshold) - GF_OPTION_TYPE_SIZET - * scheduler.alu.write-usage.exit-threshold (alu.write-usage.exit-threshold) - GF_OPTION_TYPE_SIZET - * scheduler.alu.read-usage.entry-threshold (alu.read-usage.entry-threshold) - GF_OPTION_TYPE_SIZET - * scheduler.alu.read-usage.exit-threshold (alu.read-usage.exit-threshold) - GF_OPTION_TYPE_SIZET - * scheduler.alu.open-files-usage.entry-threshold (alu.open-files-usage.entry-threshold) - GF_OPTION_TYPE_INT - * scheduler.alu.open-files-usage.exit-threshold (alu.open-files-usage.exit-threshold) - GF_OPTION_TYPE_INT - * scheduler.read-only-subvolumes (alu.read-only-subvolumes) - GF_OPTION_TYPE_ANY - * scheduler.refresh-interval (alu.refresh-interval) - GF_OPTION_TYPE_TIME - * scheduler.limits.min-free-disk (alu.limits.min-free-disk) - GF_OPTION_TYPE_PERCENT - * scheduler.alu.stat-refresh.num-file-create (alu.stat-refresh.num-file-create) - GF_OPTION_TYPE_INT - -scheduler/nufa: - * scheduler.refresh-interval (nufa.refresh-interval) - GF_OPTION_TYPE_TIME - * scheduler.limits.min-free-disk (nufa.limits.min-free-disk) - GF_OPTION_TYPE_PERCENT - * scheduler.local-volume-name (nufa.local-volume-name) - GF_OPTION_TYPE_XLATOR - -scheduler/random: - * scheduler.refresh-interval (random.refresh-interval) GF_OPTION_TYPE_TIME - * scheduler.limits.min-free-disk (random.limits.min-free-disk) GF_OPTION_TYPE_PERCENT - -scheduler/rr: - * scheduler.refresh-interval (rr.refresh-interval) GF_OPTION_TYPE_TIME - * scheduler.limits.min-free-disk (rr.limits.min-free-disk) GF_OPTION_TYPE_PERCENT - * scheduler.read-only-subvolumes (rr.read-only-subvolumes) GF_OPTION_TYPE_ANY - -scheduler/switch: - * scheduler.read-only-subvolumes (switch.read-only-subvolumes) GF_OPTION_TYPE_ANY - * scheduler.local-volume-name (switch.nufa.local-volume-name) GF_OPTION_TYPE_XLATOR - * scheduler.switch.case (switch.case) GF_OPTION_TYPE_ANY - -transport/ib-verbs: - * transport.ib-verbs.port (ib-verbs-port) GF_OPTION_TYPE_INT 1-4 - check the option by 'ibv_devinfo' - * transport.ib-verbs.mtu (ib-verbs-mtu) GF_OPTION_TYPE_INT - * transport.ib-verbs.device-name (ib-verbs-device-name) GF_OPTION_TYPE_ANY, - check by 'ibv_devinfo' - * transport.ib-verbs.work-request-send-size (ib-verbs-work-request-send-size) - GF_OPTION_TYPE_INT, - * transport.ib-verbs.work-request-recv-size (ib-verbs-work-request-recv-size) - GF_OPTION_TYPE_INT - * transport.ib-verbs.work-request-send-count (ib-verbs-work-request-send-count) - GF_OPTION_TYPE_INT - * transport.ib-verbs.work-request-recv-count (ib-verbs-work-request-recv-count) - GF_OPTION_TYPE_INT - * remote-port (transport.remote-port,transport.ib-verbs.remote-port) - GF_OPTION_TYPE_INT - * transport.ib-verbs.listen-port GF_OPTION_TYPE_INT - * transport.ib-verbs.connect-path (connect-path) GF_OPTION_TYPE_ANY - * transport.ib-verbs.bind-path (bind-path) GF_OPTION_TYPE_ANY - * transport.ib-verbs.listen-path (listen-path) GF_OPTION_TYPE_ANY - * transport.address-family (address-family) GF_OPTION_TYPE_STR inet|inet6|inet/inet6| - inet6/inet|unix|inet-sdp - -transport/socket: - * transport.remote-port (remote-port,transport.socket.remote-port) GF_OPTION_TYPE_INT - * transport.socket.listen-port (listen-port) GF_OPTION_TYPE_INT - * transport.socket.bind-address (bind-address) GF_OPTION_TYPE_ANY - * transport.socket.connect-path (connect-path) GF_OPTION_TYPE_ANY - * transport.socket.bind-path (bind-path) GF_OPTION_TYPE_ANY - * transport.socket.listen-path (listen-path) GF_OPTION_TYPE_ANY - * transport.address-family (address-family) GF_OPTION_TYPE_STR inet|inet6| - inet/inet6|inet6/inet| - unix|inet-sdp diff --git a/doc/mount.glusterfs.8 b/doc/mount.glusterfs.8 index 10e1d59f055..ce16e9e40b7 100644 --- a/doc/mount.glusterfs.8 +++ b/doc/mount.glusterfs.8 @@ -1,37 +1,31 @@ -.\" Copyright (c) 2008-2011 Gluster, Inc. <http://www.gluster.com> +.\" Copyright (c) 2008-2013 Red Hat, Inc. <http://www.redhat.com> .\" This file is part of GlusterFS. .\" -.\" GlusterFS is free software; you can redistribute it and/or modify -.\" it under the terms of the GNU General Public License as published -.\" by the Free Software Foundation; either version 3 of the License, -.\" or (at your option) any later version. +.\" This file is licensed to you under your choice of the GNU Lesser +.\" General Public License, version 3 or any later version (LGPLv3 or +.\" later), or the GNU General Public License, version 2 (GPLv2), in all +.\" cases as published by the Free Software Foundation. .\" -.\" GlusterFS is distributed in the hope that it will be useful, but -.\" WITHOUT ANY WARRANTY; without even the implied warranty of -.\" MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -.\" General Public License for more details. .\" -.\" You should have received a copy of the GNU General Public License -.\" long with this program. If not, see -.\" <http://www.gnu.org/licenses/>. .\" -.\" -.\" -.TH GlusterFS 8 "Cluster Filesystem" "18 March 2010" "Gluster Inc." +.TH GlusterFS 8 "Cluster Filesystem" "14 September 2013" "Red Hat, Inc." .SH NAME -mount.glusterfs - script to mount native GlusterFS volume +.B mount.glusterfs - script to mount native GlusterFS volume .SH SYNOPSIS -.B mount -t glusterfs -.I [-o <options>] <volumeserver>:<volumeid> <mountpoint> +.B mount -t glusterfs [-o <options>] <volumeserver>:/<volume>[/<subdir>] +.B <mountpoint> +.TP +.B mount -t glusterfs [-o <options>] <server1>,<server2>, +.B <server3>,..<serverN>:/<volname>[/<subdir>] <mount_point> .TP -.B mount -t glusterfs -.I [-o <options>] <path/to/volumefile> <mountpoint> +.TP +.B mount -t glusterfs [-o <options>] <path/to/volumefile> <mountpoint> .PP .SH DESCRIPTION This tool is part of \fBglusterfs\fR(8) package, which is used to mount using GlusterFS native binary. -\fBmount.glusterfs\fR is meant to be used by the mount(8) command for mounting +\fBmount.glusterfs\fR is meant to be used by the mount(8) command for mounting native GlusterFS client. This subcommand, however, can also be used as a standalone command with limited functionality. @@ -47,41 +41,137 @@ File to use for logging [default:/var/log/glusterfs/glusterfs.log] Logging severity. Valid options are TRACE, DEBUG, WARNING, ERROR, CRITICAL INFO and NONE [default: INFO] .TP +\fBacl +Mount the filesystem with POSIX ACL support +.TP +\fBfopen\-keep\-cache[=BOOL] +Do not purge the cache on file open (default: false) +.TP +\fBworm +Mount the filesystem in 'worm' mode +.TP +\fBaux\-gfid\-mount +Enable access to filesystem through gfid directly +.TP \fBro\fR Mount the filesystem read-only +.TP +\fBenable\-ino32=\fRBOOL +Use 32-bit inodes when mounting to workaround broken applications that don't +support 64-bit inodes +.TP +\fBmem\-accounting +Enable internal memory accounting +.TP +\fBcapability +Enable file capability setting and retrival +.TP +\fBthin-client +Enables thin mount and connects via gfproxyd daemon + .PP .SS "Advanced options" .PP .TP -\fBvolfile\-id=\fRKEY -Volume key or name of the volume file to be fetched from server +\fBattribute\-timeout=\fRSECONDS +Set attribute timeout to SECONDS for inodes in fuse kernel module [default: 1] +.TP +\fBentry\-timeout=\fRSECONDS +Set entry timeout to SECONDS in fuse kernel module [default: 1] +.TP +\fBbackground\-qlen=\fRN +Set fuse module's background queue length to N [default: 64] .TP -\fBtransport=\fRTRANSPORT-TYPE -Transport type to get volume file from server [default: socket] +\fBgid\-timeout=\fRSECONDS +Set auxiliary group list timeout to SECONDS for fuse translator [default: 0] +.TP +\fBnegative\-timeout=\fRSECONDS +Set negative timeout to SECONDS in fuse kernel module [default: 0] .TP \fBvolume\-name=\fRVOLUME-NAME Volume name to be used for MOUNT-POINT [default: top most volume in VOLUME-FILE] .TP -\fBdirect\-io\-mode=\fRdisable -Disable direct I/O mode in fuse kernel module +\fBdirect\-io\-mode=\fRBOOL|auto +Specify fuse direct I/O strategy [default: auto] +.TP +\fBcongestion\-threshold=\fRN +Set fuse module's congestion threshold to N [default: 48] +.TP +\fsubdir\-mount=\fRN +Set the subdirectory mount option [default: NULL, ie, no subdirectory mount] +.TP +.TP +\fBbackup\-volfile\-servers=\fRSERVERLIST +Provide list of backup volfile servers in the following format [default: None] + +\fB$ mount \-t glusterfs \-obackup\-volfile\-servers=<server2>:\fR +\fB <server3>:...:<serverN> <server1>:/<volname> <mount_point>\fR + +.TP +.TP +\fBbackupvolfile\-server=\fRSERVER +Provide list of backup volfile servers in the following format [default: None] + +\fB $ mount \-t glusterfs \-obackupvolfile\-server=<server2> +\fB <server1>:/<volname> <mount_point> + +.TP +.TP +\fBfetch-attempts=\fRN +\fBDeprecated\fR option - placed here for backward compatibility [default: 1] +.TP +.TP +\fBlru-limit=\fRN +Set fuse module's limit for number of inodes kept in LRU list to N [default: 65536] +.TP +.TP +\fBinvalidate-limit=\fRN +Suspend fuse invalidations implied by 'lru-limit' if number of outstanding +invalidations reaches N +.TP +.TP +\fBbackground-qlen=\fRN +Set fuse module's background queue length to N [default: 64] +.TP +\fBno\-root\-squash=\fRBOOL +disable root squashing for the trusted client [default: off] +.TP +\fBroot\-squash=\fRBOOL +enable root squashing for the trusted client [default: on] +.TP +\fBuse\-readdirp=\fRBOOL +Use readdirp() mode in fuse kernel module [default: on] +.TP +\fBdump\-fuse=\fRPATH +Dump fuse traffic to PATH +.TP +\fBkernel\-writeback\-cache=\fRBOOL +Enable fuse in-kernel writeback cache [default: off] +.TP +\fBattr\-times\-granularity=\fRNS +Declare supported granularity of file attribute [default: 0] .TP +\fBauto\-invalidation=\fRBOOL +controls whether fuse-kernel can auto-invalidate attribute, dentry and +page-cache. Disable this only if same files/directories are not +accessed across two different mounts concurrently [default: on] .PP .SH FILES .TP .I /etc/fstab A typical GlusterFS entry in /etc/fstab looks like below -server1.gluster.com:mirror /mnt/mirror glusterfs log-file=/var/log/mirror.vol,ro,defaults 0 0 +\fBserver1:/mirror /mnt/mirror glusterfs log-file=/var/log/mirror.log,acl 0 0\fR .TP -.I /etc/mtab -An example entry of a GlusterFS mountpoint in /etc/mtab looks like below +.I /proc/mounts +An example entry of a GlusterFS mountpoint in /proc/mounts looks like below -mirror.vol /mnt/glusterfs fuse.glusterfs rw,allow_other,default_permissions,max_read=131072 0 0 +\fBserver1:/mirror /mnt/glusterfs fuse.glusterfs rw,allow_other,default_permissions,max_read=131072 0 0\fR .SH SEE ALSO \fBglusterfs\fR(8), \fBmount\fR(8), \fBgluster\fR(8) .SH COPYRIGHT -Copyright(c) 2006-2011 Gluster, Inc. <http://www.gluster.com> +Copyright(c) 2006-2013 Red Hat, Inc. <http://www.redhat.com> diff --git a/doc/qa/legacy/qa-client.vol b/doc/qa/legacy/qa-client.vol deleted file mode 100644 index bcf242347d3..00000000000 --- a/doc/qa/legacy/qa-client.vol +++ /dev/null @@ -1,170 +0,0 @@ -# This spec file should be used for testing before any release -# - -# 1st client -volume client1 - type protocol/client - option transport-type tcp # for TCP/IP transport -# option transport-type ib-sdp # for Infiniband transport -# option transport-type ib-verbs # for ib-verbs transport -# option transport.ib-verbs.work-request-send-size 131072 -# option transport.ib-verbs.work-request-send-count 64 -# option transport.ib-verbs.work-request-recv-size 131072 -# option transport.ib-verbs.work-request-recv-count 64 - option remote-host 127.0.0.1 - option remote-subvolume ra1 -end-volume - -# 2nd client -volume client2 - type protocol/client - option transport-type tcp # for TCP/IP transport -# option transport-type ib-sdp # for Infiniband transport -# option transport-type ib-verbs # for ib-verbs transport - option remote-host 127.0.0.1 - option remote-subvolume ra2 -end-volume - -# 3rd client -volume client3 - type protocol/client - option transport-type tcp # for TCP/IP transport -# option transport-type ib-sdp # for Infiniband transport -# option transport-type ib-verbs # for ib-verbs transport - option remote-host 127.0.0.1 - option remote-subvolume ra3 -end-volume - -# 4th client -volume client4 - type protocol/client - option transport-type tcp # for TCP/IP transport -# option transport-type ib-sdp # for Infiniband transport -# option transport-type ib-verbs # for ib-verbs transport - option remote-host 127.0.0.1 - option remote-subvolume ra4 -end-volume - -# 5th client -volume client5 - type protocol/client - option transport-type tcp # for TCP/IP transport -# option transport-type ib-sdp # for Infiniband transport -# option transport-type ib-verbs # for ib-verbs transport - option remote-host 127.0.0.1 - option remote-subvolume ra5 -end-volume - -# 6th client -volume client6 - type protocol/client - option transport-type tcp # for TCP/IP transport -# option transport-type ib-sdp # for Infiniband transport -# option transport-type ib-verbs # for ib-verbs transport - option remote-host 127.0.0.1 - option remote-subvolume ra6 -end-volume - -# 7th client -volume client7 - type protocol/client - option transport-type tcp # for TCP/IP transport -# option transport-type ib-sdp # for Infiniband transport -# option transport-type ib-verbs # for ib-verbs transport - option remote-host 127.0.0.1 - option remote-subvolume ra7 -end-volume - -# 8th client -volume client8 - type protocol/client - option transport-type tcp # for TCP/IP transport -# option transport-type ib-sdp # for Infiniband transport -# option transport-type ib-verbs # for ib-verbs transport - option remote-host 127.0.0.1 - option remote-subvolume ra8 -end-volume - -# 1st Stripe (client1 client2) -volume stripe1 - type cluster/stripe - subvolumes client1 client2 - option block-size 128KB # all striped in 128kB block -end-volume - -# 2st Stripe (client3 client4) -volume stripe2 - type cluster/stripe - subvolumes client3 client4 - option block-size 128KB # all striped in 128kB block -end-volume - -# 3st Stripe (client5 client6) -volume stripe3 - type cluster/stripe - subvolumes client5 client6 - option block-size 128KB # all striped in 128kB block -end-volume - -# 4st Stripe (client7 client8) -volume stripe4 - type cluster/stripe - subvolumes client7 client8 - option block-size 128KB # all striped in 128kB block -end-volume - - -# 1st replicate -volume replicate1 - type cluster/replicate - subvolumes stripe1 stripe2 -end-volume - -# 2nd replicate -volume replicate2 - type cluster/replicate - subvolumes stripe3 stripe4 -end-volume - -volume ns - type protocol/client - option transport-type tcp - option remote-host 127.0.0.1 - option remote-subvolume brick-ns -end-volume - -# Unify -volume unify0 - type cluster/unify - subvolumes replicate1 replicate2 -# subvolumes stripe1 stripe3 - option namespace ns - option scheduler rr # random # alu # nufa - option rr.limits.min-free-disk 1GB -# option alu.order x -# option alu.x.entry-threshold -# option alu.x.exit-threshold -end-volume - - -# ==== Performance Translators ==== -# The default options for performance translators should be the best for 90+% of the cases -volume iot - type performance/io-threads - subvolumes unify0 -end-volume - -volume wb - type performance/write-behind - subvolumes iot -end-volume - -volume ioc - type performance/io-cache - subvolumes wb -end-volume - -volume ra - type performance/read-ahead - subvolumes ioc -end-volume diff --git a/doc/qa/legacy/qa-high-avail-client.vol b/doc/qa/legacy/qa-high-avail-client.vol deleted file mode 100644 index 69cb8dd30f1..00000000000 --- a/doc/qa/legacy/qa-high-avail-client.vol +++ /dev/null @@ -1,17 +0,0 @@ -volume client - type protocol/client - option transport-type tcp - option remote-host localhost - option transport.socket.remote-port 7001 - option remote-subvolume server1-iot -end-volume - -volume ra - type performance/read-ahead - subvolumes client -end-volume - -volume wb - type performance/write-behind - subvolumes ra -end-volume diff --git a/doc/qa/legacy/qa-high-avail-server.vol b/doc/qa/legacy/qa-high-avail-server.vol deleted file mode 100644 index 784e8d208e4..00000000000 --- a/doc/qa/legacy/qa-high-avail-server.vol +++ /dev/null @@ -1,344 +0,0 @@ - -# -- server 1 -- -volume server1-posix1 - type storage/posix - option directory /tmp/ha-export1/ -end-volume - -volume server1-ns1 - type storage/posix - option directory /tmp/ha-export-ns1/ -end-volume - -volume server1-client2 - type protocol/client - option transport-type tcp - option remote-host 127.0.0.1 - option transport.socket.remote-port 7002 - option remote-subvolume server2-posix2 -end-volume - -volume server1-ns2 - type protocol/client - option transport-type tcp - option remote-host 127.0.0.1 - option transport.socket.remote-port 7002 - option remote-subvolume server2-ns2 -end-volume - -volume server1-client3 - type protocol/client - option transport-type tcp - option remote-host 127.0.0.1 - option transport.socket.remote-port 7003 - option remote-subvolume server3-posix3 -end-volume - -volume server1-ns3 - type protocol/client - option transport-type tcp - option remote-host 127.0.0.1 - option transport.socket.remote-port 7003 - option remote-subvolume server3-ns3 -end-volume - -volume server1-io1 - type performance/io-threads - option thread-count 8 - subvolumes server1-posix1 -end-volume - - -volume server1-io2 - type performance/io-threads - option thread-count 8 - subvolumes server1-client2 -end-volume - -volume server1-io3 - type performance/io-threads - option thread-count 8 - subvolumes server1-client3 -end-volume - -volume server1-ns-io1 - type performance/io-threads - option thread-count 8 - subvolumes server1-ns1 -end-volume - -volume server1-ns-io2 - type performance/io-threads - option thread-count 8 - subvolumes server1-ns2 -end-volume - -volume server1-ns-io3 - type performance/io-threads - option thread-count 8 - subvolumes server1-ns3 -end-volume - -volume server1-ns-replicate - type cluster/replicate - subvolumes server1-ns-io1 server1-ns-io2 server1-ns-io3 -end-volume - -volume server1-storage-replicate - type cluster/replicate - subvolumes server1-io1 server1-io2 server1-io3 -end-volume - -volume server1-unify - type cluster/unify - #option self-heal off - subvolumes server1-storage-replicate - option namespace server1-ns-replicate - option scheduler rr -end-volume - -volume server1-iot - type performance/io-threads - option thread-count 8 - subvolumes server1-unify -end-volume - -volume server1 - type protocol/server - option transport-type tcp - subvolumes server1-iot - option transport.socket.listen-port 7001 - option auth.addr.server1-posix1.allow * - option auth.addr.server1-ns1.allow * - option auth.addr.server1-iot.allow * -end-volume - - -# == Server2 == -volume server2-client1 - type protocol/client - option transport-type tcp - option remote-host 127.0.0.1 - option transport.socket.remote-port 7001 - option remote-subvolume server1-posix1 -end-volume - -volume server2-ns1 - type protocol/client - option transport-type tcp - option remote-host 127.0.0.1 - option transport.socket.remote-port 7001 - option remote-subvolume server1-ns1 -end-volume - -volume server2-posix2 - type storage/posix - option directory /tmp/ha-export2/ -end-volume - -volume server2-ns2 - type storage/posix - option directory /tmp/ha-export-ns2/ -end-volume - -volume server2-client3 - type protocol/client - option transport-type tcp - option remote-host 127.0.0.1 - option transport.socket.remote-port 7003 - option remote-subvolume server3-posix3 -end-volume - -volume server2-ns3 - type protocol/client - option transport-type tcp - option remote-host 127.0.0.1 - option transport.socket.remote-port 7003 - option remote-subvolume server3-ns3 -end-volume - -volume server2-io1 - type performance/io-threads - option thread-count 8 - subvolumes server2-client1 -end-volume - - -volume server2-io2 - type performance/io-threads - option thread-count 8 - subvolumes server2-posix2 -end-volume - -volume server2-io3 - type performance/io-threads - option thread-count 8 - subvolumes server2-client3 -end-volume - -volume server2-ns-io1 - type performance/io-threads - option thread-count 8 - subvolumes server2-ns1 -end-volume - -volume server2-ns-io2 - type performance/io-threads - option thread-count 8 - subvolumes server2-ns2 -end-volume - -volume server2-ns-io3 - type performance/io-threads - option thread-count 8 - subvolumes server2-ns3 -end-volume - -volume server2-ns-replicate - type cluster/replicate - subvolumes server2-ns-io1 server2-ns-io2 server2-ns-io3 -end-volume - -volume server2-storage-replicate - type cluster/replicate - subvolumes server2-io2 server2-io3 server2-io1 -end-volume - -volume server2-unify - type cluster/unify - option self-heal off - subvolumes server2-storage-replicate - option namespace server2-ns-replicate - option scheduler rr -end-volume - -volume server2-iot - type performance/io-threads - option thread-count 8 - subvolumes server2-unify -end-volume - -volume server2 - type protocol/server - option transport-type tcp - subvolumes server2-iot - option transport.socket.listen-port 7002 - option auth.addr.server2-posix2.allow * - option auth.addr.server2-ns2.allow * - option auth.addr.server2-iot.allow * -end-volume - -# == server 3 == -volume server3-client1 - type protocol/client - option transport-type tcp - option remote-host 127.0.0.1 - option transport.socket.remote-port 7001 - option remote-subvolume server1-posix1 -end-volume - -volume server3-ns1 - type protocol/client - option transport-type tcp - option remote-host 127.0.0.1 - option transport.socket.remote-port 7001 - option remote-subvolume server1-ns1 -end-volume - -volume server3-client2 - type protocol/client - option transport-type tcp - option remote-host 127.0.0.1 - option transport.socket.remote-port 7002 - option remote-subvolume server2-posix2 -end-volume - -volume server3-ns2 - type protocol/client - option transport-type tcp - option remote-host 127.0.0.1 - option transport.socket.remote-port 7002 - option remote-subvolume server2-ns2 -end-volume - -volume server3-posix3 - type storage/posix - option directory /tmp/ha-export3/ -end-volume - -volume server3-ns3 - type storage/posix - option directory /tmp/ha-export-ns3/ -end-volume - -volume server3-io1 - type performance/io-threads - option thread-count 8 - subvolumes server3-client1 -end-volume - - -volume server3-io2 - type performance/io-threads - option thread-count 8 - subvolumes server3-client2 -end-volume - -volume server3-io3 - type performance/io-threads - option thread-count 8 - subvolumes server3-posix3 -end-volume - -volume server3-ns-io1 - type performance/io-threads - option thread-count 8 - subvolumes server3-ns1 -end-volume - -volume server3-ns-io2 - type performance/io-threads - option thread-count 8 - subvolumes server3-ns2 -end-volume - -volume server3-ns-io3 - type performance/io-threads - option thread-count 8 - subvolumes server3-ns3 -end-volume - -volume server3-ns-replicate - type cluster/replicate - subvolumes server3-ns-io1 server3-ns-io2 server3-ns-io3 -end-volume - -volume server3-storage-replicate - type cluster/replicate - subvolumes server3-io3 server3-io2 server3-io1 -end-volume - -volume server3-unify - type cluster/unify - option self-heal off - subvolumes server3-storage-replicate - option namespace server3-ns-replicate - option scheduler rr -end-volume - -volume server3-iot - type performance/io-threads - option thread-count 8 - subvolumes server3-unify -end-volume - -volume server3 - type protocol/server - option transport-type tcp - subvolumes server3-iot - option transport.socket.listen-port 7003 - option auth.addr.server3-posix3.allow * - option auth.addr.server3-ns3.allow * - option auth.addr.server3-iot.allow * -end-volume - diff --git a/doc/qa/legacy/qa-server.vol b/doc/qa/legacy/qa-server.vol deleted file mode 100644 index d948f701f1b..00000000000 --- a/doc/qa/legacy/qa-server.vol +++ /dev/null @@ -1,284 +0,0 @@ -# This spec file should be used for testing before any release -# - -# Namespace posix -volume brick-ns - type storage/posix # POSIX FS translator - option directory /tmp/export-ns # Export this directory -end-volume - -# 1st server - -volume brick1 - type storage/posix # POSIX FS translator - option directory /tmp/export1 # Export this directory -end-volume - -# == Posix-Locks == - volume plocks1 - type features/posix-locks -# option mandatory on - subvolumes brick1 - end-volume - -volume iot1 - type performance/io-threads - subvolumes plocks1 # change properly if above commented volumes needs to be included -# option <key> <value> -end-volume - -volume wb1 - type performance/write-behind - subvolumes iot1 -# option <key> <value> -end-volume - -volume ra1 - type performance/read-ahead - subvolumes wb1 -# option <key> <value> -end-volume - -volume brick2 - type storage/posix # POSIX FS translator - option directory /tmp/export2 # Export this directory -end-volume - -# == TrashCan Translator == -# volume trash2 -# type features/trash -# option trash-dir /.trashcan -# subvolumes brick2 -# end-volume - -# == Posix-Locks == -volume plocks2 - type features/posix-locks -# option <something> <something> - subvolumes brick2 -end-volume - -volume iot2 - type performance/io-threads - subvolumes plocks2 # change properly if above commented volumes needs to be included -# option <key> <value> -end-volume - -volume wb2 - type performance/write-behind - subvolumes iot2 -# option <key> <value> -end-volume - -volume ra2 - type performance/read-ahead - subvolumes wb2 -# option <key> <value> -end-volume - -volume brick3 - type storage/posix # POSIX FS translator - option directory /tmp/export3 # Export this directory -end-volume - -# == TrashCan Translator == -# volume trash3 -# type features/trash -# option trash-dir /.trashcan -# subvolumes brick3 -# end-volume - -# == Posix-Locks == -volume plocks3 - type features/posix-locks -# option <something> <something> - subvolumes brick3 -end-volume - -volume iot3 - type performance/io-threads - subvolumes plocks3 # change properly if above commented volumes needs to be included -# option <key> <value> -end-volume - -volume wb3 - type performance/write-behind - subvolumes iot3 -# option <key> <value> -end-volume - -volume ra3 - type performance/read-ahead - subvolumes wb3 -# option <key> <value> -end-volume - -volume brick4 - type storage/posix # POSIX FS translator - option directory /tmp/export4 # Export this directory -end-volume - -# == Posix-Locks == -volume plocks4 - type features/posix-locks -# option <something> <something> - subvolumes brick4 -end-volume - -volume iot4 - type performance/io-threads - subvolumes plocks4 # change properly if above commented volumes needs to be included -# option <key> <value> -end-volume - -volume wb4 - type performance/write-behind - subvolumes iot4 -# option <key> <value> -end-volume - -volume ra4 - type performance/read-ahead - subvolumes wb4 -# option <key> <value> -end-volume - -volume brick5 - type storage/posix # POSIX FS translator - option directory /tmp/export5 # Export this directory -end-volume - - -# == Posix-Locks == -volume plocks5 - type features/posix-locks -# option <something> <something> - subvolumes brick5 -end-volume - -volume iot5 - type performance/io-threads - subvolumes plocks5 # change properly if above commented volumes needs to be included -# option <key> <value> -end-volume - -volume wb5 - type performance/write-behind - subvolumes iot5 -# option <key> <value> -end-volume - -volume ra5 - type performance/read-ahead - subvolumes wb5 -# option <key> <value> -end-volume - -volume brick6 - type storage/posix # POSIX FS translator - option directory /tmp/export6 # Export this directory -end-volume - -# == Posix-Locks == -volume plocks6 - type features/posix-locks -# option <something> <something> - subvolumes brick6 -end-volume - -volume iot6 - type performance/io-threads - subvolumes plocks6 # change properly if above commented volumes needs to be included -# option <key> <value> -end-volume - -volume wb6 - type performance/write-behind - subvolumes iot6 -# option <key> <value> -end-volume - -volume ra6 - type performance/read-ahead - subvolumes wb6 -# option <key> <value> -end-volume - -volume brick7 - type storage/posix # POSIX FS translator - option directory /tmp/export7 # Export this directory -end-volume - -# == Posix-Locks == -volume plocks7 - type features/posix-locks -# option <something> <something> - subvolumes brick7 -end-volume - -volume iot7 - type performance/io-threads - subvolumes plocks7 # change properly if above commented volumes needs to be included -# option <key> <value> -end-volume - -volume wb7 - type performance/write-behind - subvolumes iot7 -# option <key> <value> -end-volume - -volume ra7 - type performance/read-ahead - subvolumes wb7 -# option <key> <value> -end-volume - -volume brick8 - type storage/posix # POSIX FS translator - option directory /tmp/export8 # Export this directory -end-volume - -# == Posix-Locks == -volume plocks8 - type features/posix-locks -# option <something> <something> - subvolumes brick8 -end-volume - -volume iot8 - type performance/io-threads - subvolumes plocks8 # change properly if above commented volumes needs to be included -# option <key> <value> -end-volume - -volume wb8 - type performance/write-behind - subvolumes iot8 -# option <key> <value> -end-volume - -volume ra8 - type performance/read-ahead - subvolumes wb8 -# option <key> <value> -end-volume - -volume server8 - type protocol/server - subvolumes ra8 ra1 ra2 ra3 ra4 ra5 ra6 ra7 brick-ns - option transport-type tcp # For TCP/IP transport -# option transport-type ib-sdp # For Infiniband transport -# option transport-type ib-verbs # For ib-verbs transport - option client-volume-filename /examples/qa-client.vol - option auth.addr.ra1.allow * # Allow access to "stat8" volume - option auth.addr.ra2.allow * # Allow access to "stat8" volume - option auth.addr.ra3.allow * # Allow access to "stat8" volume - option auth.addr.ra4.allow * # Allow access to "stat8" volume - option auth.addr.ra5.allow * # Allow access to "stat8" volume - option auth.addr.ra6.allow * # Allow access to "stat8" volume - option auth.addr.ra7.allow * # Allow access to "stat8" volume - option auth.addr.ra8.allow * # Allow access to "stat8" volume - option auth.addr.brick-ns.allow * # Allow access to "stat8" volume -end-volume - diff --git a/doc/user-guide/legacy/Makefile.am b/doc/user-guide/legacy/Makefile.am deleted file mode 100644 index b2caabaa2f3..00000000000 --- a/doc/user-guide/legacy/Makefile.am +++ /dev/null @@ -1,3 +0,0 @@ -info_TEXINFOS = user-guide.texi -CLEANFILES = *~ -DISTCLEANFILES = .deps/*.P *.info *vti diff --git a/doc/user-guide/legacy/advanced-stripe.odg b/doc/user-guide/legacy/advanced-stripe.odg Binary files differdeleted file mode 100644 index 7686d7091b2..00000000000 --- a/doc/user-guide/legacy/advanced-stripe.odg +++ /dev/null diff --git a/doc/user-guide/legacy/advanced-stripe.pdf b/doc/user-guide/legacy/advanced-stripe.pdf Binary files differdeleted file mode 100644 index ec8b03dcfbb..00000000000 --- a/doc/user-guide/legacy/advanced-stripe.pdf +++ /dev/null diff --git a/doc/user-guide/legacy/colonO-icon.jpg b/doc/user-guide/legacy/colonO-icon.jpg Binary files differdeleted file mode 100644 index 3e66f7a2775..00000000000 --- a/doc/user-guide/legacy/colonO-icon.jpg +++ /dev/null diff --git a/doc/user-guide/legacy/fdl.texi b/doc/user-guide/legacy/fdl.texi deleted file mode 100644 index e33c687cdfb..00000000000 --- a/doc/user-guide/legacy/fdl.texi +++ /dev/null @@ -1,454 +0,0 @@ - -@c @node GNU Free Documentation License -@c @appendixsec GNU Free Documentation License - -@cindex FDL, GNU Free Documentation License -@center Version 1.2, November 2002 - -@display -Copyright @copyright{} 2000,2001,2002 Free Software Foundation, Inc. -59 Temple Place, Suite 330, Boston, MA 02111-1307, USA - -Everyone is permitted to copy and distribute verbatim copies -of this license document, but changing it is not allowed. -@end display - -@enumerate 0 -@item -PREAMBLE - -The purpose of this License is to make a manual, textbook, or other -functional and useful document @dfn{free} in the sense of freedom: to -assure everyone the effective freedom to copy and redistribute it, -with or without modifying it, either commercially or noncommercially. -Secondarily, this License preserves for the author and publisher a way -to get credit for their work, while not being considered responsible -for modifications made by others. - -This License is a kind of ``copyleft'', which means that derivative -works of the document must themselves be free in the same sense. It -complements the GNU General Public License, which is a copyleft -license designed for free software. - -We have designed this License in order to use it for manuals for free -software, because free software needs free documentation: a free -program should come with manuals providing the same freedoms that the -software does. But this License is not limited to software manuals; -it can be used for any textual work, regardless of subject matter or -whether it is published as a printed book. We recommend this License -principally for works whose purpose is instruction or reference. - -@item -APPLICABILITY AND DEFINITIONS - -This License applies to any manual or other work, in any medium, that -contains a notice placed by the copyright holder saying it can be -distributed under the terms of this License. Such a notice grants a -world-wide, royalty-free license, unlimited in duration, to use that -work under the conditions stated herein. The ``Document'', below, -refers to any such manual or work. Any member of the public is a -licensee, and is addressed as ``you''. You accept the license if you -copy, modify or distribute the work in a way requiring permission -under copyright law. - -A ``Modified Version'' of the Document means any work containing the -Document or a portion of it, either copied verbatim, or with -modifications and/or translated into another language. - -A ``Secondary Section'' is a named appendix or a front-matter section -of the Document that deals exclusively with the relationship of the -publishers or authors of the Document to the Document's overall -subject (or to related matters) and contains nothing that could fall -directly within that overall subject. (Thus, if the Document is in -part a textbook of mathematics, a Secondary Section may not explain -any mathematics.) The relationship could be a matter of historical -connection with the subject or with related matters, or of legal, -commercial, philosophical, ethical or political position regarding -them. - -The ``Invariant Sections'' are certain Secondary Sections whose titles -are designated, as being those of Invariant Sections, in the notice -that says that the Document is released under this License. If a -section does not fit the above definition of Secondary then it is not -allowed to be designated as Invariant. The Document may contain zero -Invariant Sections. If the Document does not identify any Invariant -Sections then there are none. - -The ``Cover Texts'' are certain short passages of text that are listed, -as Front-Cover Texts or Back-Cover Texts, in the notice that says that -the Document is released under this License. A Front-Cover Text may -be at most 5 words, and a Back-Cover Text may be at most 25 words. - -A ``Transparent'' copy of the Document means a machine-readable copy, -represented in a format whose specification is available to the -general public, that is suitable for revising the document -straightforwardly with generic text editors or (for images composed of -pixels) generic paint programs or (for drawings) some widely available -drawing editor, and that is suitable for input to text formatters or -for automatic translation to a variety of formats suitable for input -to text formatters. A copy made in an otherwise Transparent file -format whose markup, or absence of markup, has been arranged to thwart -or discourage subsequent modification by readers is not Transparent. -An image format is not Transparent if used for any substantial amount -of text. A copy that is not ``Transparent'' is called ``Opaque''. - -Examples of suitable formats for Transparent copies include plain -@sc{ascii} without markup, Texinfo input format, La@TeX{} input -format, @acronym{SGML} or @acronym{XML} using a publicly available -@acronym{DTD}, and standard-conforming simple @acronym{HTML}, -PostScript or @acronym{PDF} designed for human modification. Examples -of transparent image formats include @acronym{PNG}, @acronym{XCF} and -@acronym{JPG}. Opaque formats include proprietary formats that can be -read and edited only by proprietary word processors, @acronym{SGML} or -@acronym{XML} for which the @acronym{DTD} and/or processing tools are -not generally available, and the machine-generated @acronym{HTML}, -PostScript or @acronym{PDF} produced by some word processors for -output purposes only. - -The ``Title Page'' means, for a printed book, the title page itself, -plus such following pages as are needed to hold, legibly, the material -this License requires to appear in the title page. For works in -formats which do not have any title page as such, ``Title Page'' means -the text near the most prominent appearance of the work's title, -preceding the beginning of the body of the text. - -A section ``Entitled XYZ'' means a named subunit of the Document whose -title either is precisely XYZ or contains XYZ in parentheses following -text that translates XYZ in another language. (Here XYZ stands for a -specific section name mentioned below, such as ``Acknowledgements'', -``Dedications'', ``Endorsements'', or ``History''.) To ``Preserve the Title'' -of such a section when you modify the Document means that it remains a -section ``Entitled XYZ'' according to this definition. - -The Document may include Warranty Disclaimers next to the notice which -states that this License applies to the Document. These Warranty -Disclaimers are considered to be included by reference in this -License, but only as regards disclaiming warranties: any other -implication that these Warranty Disclaimers may have is void and has -no effect on the meaning of this License. - -@item -VERBATIM COPYING - -You may copy and distribute the Document in any medium, either -commercially or noncommercially, provided that this License, the -copyright notices, and the license notice saying this License applies -to the Document are reproduced in all copies, and that you add no other -conditions whatsoever to those of this License. You may not use -technical measures to obstruct or control the reading or further -copying of the copies you make or distribute. However, you may accept -compensation in exchange for copies. If you distribute a large enough -number of copies you must also follow the conditions in section 3. - -You may also lend copies, under the same conditions stated above, and -you may publicly display copies. - -@item -COPYING IN QUANTITY - -If you publish printed copies (or copies in media that commonly have -printed covers) of the Document, numbering more than 100, and the -Document's license notice requires Cover Texts, you must enclose the -copies in covers that carry, clearly and legibly, all these Cover -Texts: Front-Cover Texts on the front cover, and Back-Cover Texts on -the back cover. Both covers must also clearly and legibly identify -you as the publisher of these copies. The front cover must present -the full title with all words of the title equally prominent and -visible. You may add other material on the covers in addition. -Copying with changes limited to the covers, as long as they preserve -the title of the Document and satisfy these conditions, can be treated -as verbatim copying in other respects. - -If the required texts for either cover are too voluminous to fit -legibly, you should put the first ones listed (as many as fit -reasonably) on the actual cover, and continue the rest onto adjacent -pages. - -If you publish or distribute Opaque copies of the Document numbering -more than 100, you must either include a machine-readable Transparent -copy along with each Opaque copy, or state in or with each Opaque copy -a computer-network location from which the general network-using -public has access to download using public-standard network protocols -a complete Transparent copy of the Document, free of added material. -If you use the latter option, you must take reasonably prudent steps, -when you begin distribution of Opaque copies in quantity, to ensure -that this Transparent copy will remain thus accessible at the stated -location until at least one year after the last time you distribute an -Opaque copy (directly or through your agents or retailers) of that -edition to the public. - -It is requested, but not required, that you contact the authors of the -Document well before redistributing any large number of copies, to give -them a chance to provide you with an updated version of the Document. - -@item -MODIFICATIONS - -You may copy and distribute a Modified Version of the Document under -the conditions of sections 2 and 3 above, provided that you release -the Modified Version under precisely this License, with the Modified -Version filling the role of the Document, thus licensing distribution -and modification of the Modified Version to whoever possesses a copy -of it. In addition, you must do these things in the Modified Version: - -@enumerate A -@item -Use in the Title Page (and on the covers, if any) a title distinct -from that of the Document, and from those of previous versions -(which should, if there were any, be listed in the History section -of the Document). You may use the same title as a previous version -if the original publisher of that version gives permission. - -@item -List on the Title Page, as authors, one or more persons or entities -responsible for authorship of the modifications in the Modified -Version, together with at least five of the principal authors of the -Document (all of its principal authors, if it has fewer than five), -unless they release you from this requirement. - -@item -State on the Title page the name of the publisher of the -Modified Version, as the publisher. - -@item -Preserve all the copyright notices of the Document. - -@item -Add an appropriate copyright notice for your modifications -adjacent to the other copyright notices. - -@item -Include, immediately after the copyright notices, a license notice -giving the public permission to use the Modified Version under the -terms of this License, in the form shown in the Addendum below. - -@item -Preserve in that license notice the full lists of Invariant Sections -and required Cover Texts given in the Document's license notice. - -@item -Include an unaltered copy of this License. - -@item -Preserve the section Entitled ``History'', Preserve its Title, and add -to it an item stating at least the title, year, new authors, and -publisher of the Modified Version as given on the Title Page. If -there is no section Entitled ``History'' in the Document, create one -stating the title, year, authors, and publisher of the Document as -given on its Title Page, then add an item describing the Modified -Version as stated in the previous sentence. - -@item -Preserve the network location, if any, given in the Document for -public access to a Transparent copy of the Document, and likewise -the network locations given in the Document for previous versions -it was based on. These may be placed in the ``History'' section. -You may omit a network location for a work that was published at -least four years before the Document itself, or if the original -publisher of the version it refers to gives permission. - -@item -For any section Entitled ``Acknowledgements'' or ``Dedications'', Preserve -the Title of the section, and preserve in the section all the -substance and tone of each of the contributor acknowledgements and/or -dedications given therein. - -@item -Preserve all the Invariant Sections of the Document, -unaltered in their text and in their titles. Section numbers -or the equivalent are not considered part of the section titles. - -@item -Delete any section Entitled ``Endorsements''. Such a section -may not be included in the Modified Version. - -@item -Do not retitle any existing section to be Entitled ``Endorsements'' or -to conflict in title with any Invariant Section. - -@item -Preserve any Warranty Disclaimers. -@end enumerate - -If the Modified Version includes new front-matter sections or -appendices that qualify as Secondary Sections and contain no material -copied from the Document, you may at your option designate some or all -of these sections as invariant. To do this, add their titles to the -list of Invariant Sections in the Modified Version's license notice. -These titles must be distinct from any other section titles. - -You may add a section Entitled ``Endorsements'', provided it contains -nothing but endorsements of your Modified Version by various -parties---for example, statements of peer review or that the text has -been approved by an organization as the authoritative definition of a -standard. - -You may add a passage of up to five words as a Front-Cover Text, and a -passage of up to 25 words as a Back-Cover Text, to the end of the list -of Cover Texts in the Modified Version. Only one passage of -Front-Cover Text and one of Back-Cover Text may be added by (or -through arrangements made by) any one entity. If the Document already -includes a cover text for the same cover, previously added by you or -by arrangement made by the same entity you are acting on behalf of, -you may not add another; but you may replace the old one, on explicit -permission from the previous publisher that added the old one. - -The author(s) and publisher(s) of the Document do not by this License -give permission to use their names for publicity for or to assert or -imply endorsement of any Modified Version. - -@item -COMBINING DOCUMENTS - -You may combine the Document with other documents released under this -License, under the terms defined in section 4 above for modified -versions, provided that you include in the combination all of the -Invariant Sections of all of the original documents, unmodified, and -list them all as Invariant Sections of your combined work in its -license notice, and that you preserve all their Warranty Disclaimers. - -The combined work need only contain one copy of this License, and -multiple identical Invariant Sections may be replaced with a single -copy. If there are multiple Invariant Sections with the same name but -different contents, make the title of each such section unique by -adding at the end of it, in parentheses, the name of the original -author or publisher of that section if known, or else a unique number. -Make the same adjustment to the section titles in the list of -Invariant Sections in the license notice of the combined work. - -In the combination, you must combine any sections Entitled ``History'' -in the various original documents, forming one section Entitled -``History''; likewise combine any sections Entitled ``Acknowledgements'', -and any sections Entitled ``Dedications''. You must delete all -sections Entitled ``Endorsements.'' - -@item -COLLECTIONS OF DOCUMENTS - -You may make a collection consisting of the Document and other documents -released under this License, and replace the individual copies of this -License in the various documents with a single copy that is included in -the collection, provided that you follow the rules of this License for -verbatim copying of each of the documents in all other respects. - -You may extract a single document from such a collection, and distribute -it individually under this License, provided you insert a copy of this -License into the extracted document, and follow this License in all -other respects regarding verbatim copying of that document. - -@item -AGGREGATION WITH INDEPENDENT WORKS - -A compilation of the Document or its derivatives with other separate -and independent documents or works, in or on a volume of a storage or -distribution medium, is called an ``aggregate'' if the copyright -resulting from the compilation is not used to limit the legal rights -of the compilation's users beyond what the individual works permit. -When the Document is included in an aggregate, this License does not -apply to the other works in the aggregate which are not themselves -derivative works of the Document. - -If the Cover Text requirement of section 3 is applicable to these -copies of the Document, then if the Document is less than one half of -the entire aggregate, the Document's Cover Texts may be placed on -covers that bracket the Document within the aggregate, or the -electronic equivalent of covers if the Document is in electronic form. -Otherwise they must appear on printed covers that bracket the whole -aggregate. - -@item -TRANSLATION - -Translation is considered a kind of modification, so you may -distribute translations of the Document under the terms of section 4. -Replacing Invariant Sections with translations requires special -permission from their copyright holders, but you may include -translations of some or all Invariant Sections in addition to the -original versions of these Invariant Sections. You may include a -translation of this License, and all the license notices in the -Document, and any Warranty Disclaimers, provided that you also include -the original English version of this License and the original versions -of those notices and disclaimers. In case of a disagreement between -the translation and the original version of this License or a notice -or disclaimer, the original version will prevail. - -If a section in the Document is Entitled ``Acknowledgements'', -``Dedications'', or ``History'', the requirement (section 4) to Preserve -its Title (section 1) will typically require changing the actual -title. - -@item -TERMINATION - -You may not copy, modify, sublicense, or distribute the Document except -as expressly provided for under this License. Any other attempt to -copy, modify, sublicense or distribute the Document is void, and will -automatically terminate your rights under this License. However, -parties who have received copies, or rights, from you under this -License will not have their licenses terminated so long as such -parties remain in full compliance. - -@item -FUTURE REVISIONS OF THIS LICENSE - -The Free Software Foundation may publish new, revised versions -of the GNU Free Documentation License from time to time. Such new -versions will be similar in spirit to the present version, but may -differ in detail to address new problems or concerns. See -@uref{http://www.gnu.org/copyleft/}. - -Each version of the License is given a distinguishing version number. -If the Document specifies that a particular numbered version of this -License ``or any later version'' applies to it, you have the option of -following the terms and conditions either of that specified version or -of any later version that has been published (not as a draft) by the -Free Software Foundation. If the Document does not specify a version -number of this License, you may choose any version ever published (not -as a draft) by the Free Software Foundation. -@end enumerate - -@page -@c @appendixsubsec ADDENDUM: How to use this License for your -@c documents -@subsection ADDENDUM: How to use this License for your documents - -To use this License in a document you have written, include a copy of -the License in the document and put the following copyright and -license notices just after the title page: - -@smallexample -@group - Copyright (C) @var{year} @var{your name}. - Permission is granted to copy, distribute and/or modify this document - under the terms of the GNU Free Documentation License, Version 1.2 - or any later version published by the Free Software Foundation; - with no Invariant Sections, no Front-Cover Texts, and no Back-Cover - Texts. A copy of the license is included in the section entitled ``GNU - Free Documentation License''. -@end group -@end smallexample - -If you have Invariant Sections, Front-Cover Texts and Back-Cover Texts, -replace the ``with...Texts.'' line with this: - -@smallexample -@group - with the Invariant Sections being @var{list their titles}, with - the Front-Cover Texts being @var{list}, and with the Back-Cover Texts - being @var{list}. -@end group -@end smallexample - -If you have Invariant Sections without Cover Texts, or some other -combination of the three, merge those two alternatives to suit the -situation. - -If your document contains nontrivial examples of program code, we -recommend releasing these examples in parallel under your choice of -free software license, such as the GNU General Public License, -to permit their use in free software. - -@c Local Variables: -@c ispell-local-pdict: "ispell-dict" -@c End: - diff --git a/doc/user-guide/legacy/fuse.odg b/doc/user-guide/legacy/fuse.odg Binary files differdeleted file mode 100644 index 61bd103c78b..00000000000 --- a/doc/user-guide/legacy/fuse.odg +++ /dev/null diff --git a/doc/user-guide/legacy/fuse.pdf b/doc/user-guide/legacy/fuse.pdf Binary files differdeleted file mode 100644 index a7d13faff56..00000000000 --- a/doc/user-guide/legacy/fuse.pdf +++ /dev/null diff --git a/doc/user-guide/legacy/ha.odg b/doc/user-guide/legacy/ha.odg Binary files differdeleted file mode 100644 index e4b8b72d08b..00000000000 --- a/doc/user-guide/legacy/ha.odg +++ /dev/null diff --git a/doc/user-guide/legacy/ha.pdf b/doc/user-guide/legacy/ha.pdf Binary files differdeleted file mode 100644 index e372c0ab03e..00000000000 --- a/doc/user-guide/legacy/ha.pdf +++ /dev/null diff --git a/doc/user-guide/legacy/stripe.odg b/doc/user-guide/legacy/stripe.odg Binary files differdeleted file mode 100644 index 79441bf1452..00000000000 --- a/doc/user-guide/legacy/stripe.odg +++ /dev/null diff --git a/doc/user-guide/legacy/stripe.pdf b/doc/user-guide/legacy/stripe.pdf Binary files differdeleted file mode 100644 index b94446feb56..00000000000 --- a/doc/user-guide/legacy/stripe.pdf +++ /dev/null diff --git a/doc/user-guide/legacy/unify.odg b/doc/user-guide/legacy/unify.odg Binary files differdeleted file mode 100644 index ccaa9bf16f9..00000000000 --- a/doc/user-guide/legacy/unify.odg +++ /dev/null diff --git a/doc/user-guide/legacy/unify.pdf b/doc/user-guide/legacy/unify.pdf Binary files differdeleted file mode 100644 index c22027f66e7..00000000000 --- a/doc/user-guide/legacy/unify.pdf +++ /dev/null diff --git a/doc/user-guide/legacy/user-guide.info b/doc/user-guide/legacy/user-guide.info deleted file mode 100644 index 6a49d078d64..00000000000 --- a/doc/user-guide/legacy/user-guide.info +++ /dev/null @@ -1,2697 +0,0 @@ -This is ../../../doc/user-guide/user-guide.info, produced by makeinfo version 4.13 from ../../../doc/user-guide/user-guide.texi. - -START-INFO-DIR-ENTRY -* GlusterFS: (user-guide). GlusterFS distributed filesystem user guide -END-INFO-DIR-ENTRY - - This is the user manual for GlusterFS 2.0. - - Copyright (c) 2007-2011 Gluster, Inc. Permission is granted to -copy, distribute and/or modify this document under the terms of the GNU -Free Documentation License, Version 1.2 or any later version published -by the Free Software Foundation; with no Invariant Sections, no -Front-Cover Texts, and no Back-Cover Texts. A copy of the license is -included in the chapter entitled "GNU Free Documentation License". - - -File: user-guide.info, Node: Top, Next: Acknowledgements, Up: (dir) - -GlusterFS 2.0 User Guide -************************ - -This is the user manual for GlusterFS 2.0. - - Copyright (c) 2007-2011 Gluster, Inc. Permission is granted to -copy, distribute and/or modify this document under the terms of the GNU -Free Documentation License, Version 1.2 or any later version published -by the Free Software Foundation; with no Invariant Sections, no -Front-Cover Texts, and no Back-Cover Texts. A copy of the license is -included in the chapter entitled "GNU Free Documentation License". - -* Menu: - -* Acknowledgements:: -* Introduction:: -* Installation and Invocation:: -* Concepts:: -* Translators:: -* Usage Scenarios:: -* Troubleshooting:: -* GNU Free Documentation Licence:: -* Index:: - - --- The Detailed Node Listing --- - -Installation and Invocation - -* Pre requisites:: -* Getting GlusterFS:: -* Building:: -* Running GlusterFS:: -* A Tutorial Introduction:: - -Running GlusterFS - -* Server:: -* Client:: - -Concepts - -* Filesystems in Userspace:: -* Translator:: -* Volume specification file:: - -Translators - -* Storage Translators:: -* Client and Server Translators:: -* Clustering Translators:: -* Performance Translators:: -* Features Translators:: - -Storage Translators - -* POSIX:: - -Client and Server Translators - -* Transport modules:: -* Client protocol:: -* Server protocol:: - -Clustering Translators - -* Unify:: -* Replicate:: -* Stripe:: - -Performance Translators - -* Read Ahead:: -* Write Behind:: -* IO Threads:: -* IO Cache:: - -Features Translators - -* POSIX Locks:: -* Fixed ID:: - -Miscellaneous Translators - -* ROT-13:: -* Trace:: - - -File: user-guide.info, Node: Acknowledgements, Next: Introduction, Prev: Top, Up: Top - -Acknowledgements -**************** - -GlusterFS continues to be a wonderful and enriching experience for all -of us involved. - - GlusterFS development would not have been possible at this pace if -not for our enthusiastic users. People from around the world have -helped us with bug reports, performance numbers, and feature -suggestions. A huge thanks to them all. - - Matthew Paine - for RPMs & general enthu - - Leonardo Rodrigues de Mello - for DEBs - - Julian Perez & Adam D'Auria - for multi-server tutorial - - Paul England - for HA spec - - Brent Nelson - for many bug reports - - Jacques Mattheij - for Europe mirror. - - Patrick Negri - for TCP non-blocking connect. - http://gluster.org/core-team.php (<list-hacking@gluster.com>) - Gluster - - -File: user-guide.info, Node: Introduction, Next: Installation and Invocation, Prev: Acknowledgements, Up: Top - -1 Introduction -************** - -GlusterFS is a distributed filesystem. It works at the file level, not -block level. - - A network filesystem is one which allows us to access remote files. A -distributed filesystem is one that stores data on multiple machines and -makes them all appear to be a part of the same filesystem. - - Need for distributed filesystems - - * Scalability: A distributed filesystem allows us to store more data - than what can be stored on a single machine. - - * Redundancy: We might want to replicate crucial data on to several - machines. - - * Uniform access: One can mount a remote volume (for example your - home directory) from any machine and access the same data. - -1.1 Contacting us -================= - -You can reach us through the mailing list *gluster-devel* -(<gluster-devel@nongnu.org>). - - You can also find many of the developers on IRC, on the `#gluster' -channel on Freenode (<irc.freenode.net>). - - The GlusterFS documentation wiki is also useful: -<http://gluster.org/docs/index.php/GlusterFS> - - For commercial support, you can contact Gluster at: - - 3194 Winding Vista Common - Fremont, CA 94539 - USA. - - Phone: +1 (510) 354 6801 - Toll free: +1 (888) 813 6309 - Fax: +1 (510) 372 0604 - - You can also email us at <support@gluster.com>. - - -File: user-guide.info, Node: Installation and Invocation, Next: Concepts, Prev: Introduction, Up: Top - -2 Installation and Invocation -***************************** - -* Menu: - -* Pre requisites:: -* Getting GlusterFS:: -* Building:: -* Running GlusterFS:: -* A Tutorial Introduction:: - - -File: user-guide.info, Node: Pre requisites, Next: Getting GlusterFS, Up: Installation and Invocation - -2.1 Pre requisites -================== - -Before installing GlusterFS make sure you have the following components -installed. - -2.1.1 FUSE ----------- - -You'll need FUSE version 2.6.0 or higher to use GlusterFS. You can omit -installing FUSE if you want to build _only_ the server. Note that you -won't be able to mount a GlusterFS filesystem on a machine that does -not have FUSE installed. - - FUSE can be downloaded from: <http://fuse.sourceforge.net/> - - To get the best performance from GlusterFS, however, it is -recommended that you use our patched version of FUSE. See Patched FUSE -for details. - -2.1.2 Patched FUSE ------------------- - -The GlusterFS project maintains a patched version of FUSE meant to be -used with GlusterFS. The patches increase GlusterFS performance. It is -recommended that all users use the patched FUSE. - - The patched FUSE tarball can be downloaded from: - - <ftp://ftp.gluster.com/pub/gluster/glusterfs/fuse/> - - The specific changes made to FUSE are: - - * The communication channel size between FUSE kernel module and - GlusterFS has been increased to 1MB, permitting large reads and - writes to be sent in bigger chunks. - - * The kernel's read-ahead boundry has been extended upto 1MB. - - * Block size returned in the `stat()'/`fstat()' calls tuned to 1MB, - to make cp and similar commands perform I/O using that block size. - - * `flock()' locking support has been added (although some rework in - GlusterFS is needed for perfect compliance). - -2.1.3 libibverbs (optional) ---------------------------- - -This is only needed if you want GlusterFS to use InfiniBand as the -interconnect mechanism between server and client. You can get it from: - - <http://www.openfabrics.org/downloads.htm>. - -2.1.4 Bison and Flex --------------------- - -These should be already installed on most Linux systems. If not, use -your distribution's normal software installation procedures to install -them. Make sure you install the relevant developer packages also. - - -File: user-guide.info, Node: Getting GlusterFS, Next: Building, Prev: Pre requisites, Up: Installation and Invocation - -2.2 Getting GlusterFS -===================== - -There are many ways to get hold of GlusterFS. For a production -deployment, the recommended method is to download the latest release -tarball. Release tarballs are available at: -<http://gluster.org/download.php>. - - If you want the bleeding edge development source, you can get them -from the GNU Arch(1) repository. First you must install GNU Arch -itself. Then register the GlusterFS archive by doing: - - $ tla register-archive http://arch.sv.gnu.org/archives/gluster - - Now you can check out the source itself: - - $ tla get -A gluster@sv.gnu.org glusterfs--mainline--3.0 - - ---------- Footnotes ---------- - - (1) <http://www.gnu.org/software/gnu-arch/> - - -File: user-guide.info, Node: Building, Next: Running GlusterFS, Prev: Getting GlusterFS, Up: Installation and Invocation - -2.3 Building -============ - -You can skip this section if you're installing from RPMs or DEBs. - - GlusterFS uses the Autotools mechanism to build. As such, the -procedure is straight-forward. First, change into the GlusterFS source -directory. - - $ cd glusterfs-<version> - - If you checked out the source from the Arch repository, you'll need -to run `./autogen.sh' first. Note that you'll need to have Autoconf and -Automake installed for this. - - Run `configure'. - - $ ./configure - - The configure script accepts the following options: - -`--disable-ibverbs' - Disable the InfiniBand transport mechanism. - -`--disable-fuse-client' - Disable the FUSE client. - -`--disable-server' - Disable building of the GlusterFS server. - -`--disable-bdb' - Disable building of Berkeley DB based storage translator. - -`--disable-mod_glusterfs' - Disable building of Apache/lighttpd glusterfs plugins. - -`--disable-epoll' - Use poll instead of epoll. - -`--disable-libglusterfsclient' - Disable building of libglusterfsclient - - - Build and install GlusterFS. - - # make install - - The binaries (`glusterfsd' and `glusterfs') will be by default -installed in `/usr/local/sbin/'. Translator, scheduler, and transport -shared libraries will be installed in -`/usr/local/lib/glusterfs/<version>/'. Sample volume specification -files will be in `/usr/local/etc/glusterfs/'. This document itself can -be found in `/usr/local/share/doc/glusterfs/'. If you passed the -`--prefix' argument to the configure script, then replace `/usr/local' -in the preceding paths with the prefix. - - -File: user-guide.info, Node: Running GlusterFS, Next: A Tutorial Introduction, Prev: Building, Up: Installation and Invocation - -2.4 Running GlusterFS -===================== - -* Menu: - -* Server:: -* Client:: - - -File: user-guide.info, Node: Server, Next: Client, Up: Running GlusterFS - -2.4.1 Server ------------- - -The GlusterFS server is necessary to export storage volumes to remote -clients (See *note Server protocol:: for more info). This section -documents the invocation of the GlusterFS server program and all the -command-line options accepted by it. - - Basic Options - -`-f, --volfile=<path>' - Use the volume file as the volume specification. - -`-s, --volfile-server=<hostname>' - Server to get volume file from. This option overrides -volfile - option. - -`-l, --log-file=<path>' - Specify the path for the log file. - -`-L, --log-level=<level>' - Set the log level for the server. Log level should be one of DEBUG, - WARNING, ERROR, CRITICAL, or NONE. - - Advanced Options - -`--debug' - Run in debug mode. This option sets -no-daemon, -log-level to - DEBUG and -log-file to console. - -`-N, --no-daemon' - Run glusterfsd as a foreground process. - -`-p, --pid-file=<path>' - Path for the PID file. - -`--volfile-id=<key>' - 'key' of the volfile to be fetched from server. - -`--volfile-server-port=<port-number>' - Listening port number of volfile server. - -`--volfile-server-transport=[socket|ib-verbs]' - Transport type to get volfile from server. [default: `socket'] - -`--xlator-options=<volume-name.option=value>' - Add/override a translator option for a volume with specified value. - - Miscellaneous Options - -`-?, --help' - Show this help text. - -`--usage' - Display a short usage message. - -`-V, --version' - Show version information. - - -File: user-guide.info, Node: Client, Prev: Server, Up: Running GlusterFS - -2.4.2 Client ------------- - -The GlusterFS client process is necessary to access remote storage -volumes and mount them locally using FUSE. This section documents the -invocation of the client process and all its command-line arguments. - - # glusterfs [options] <mountpoint> - - The `mountpoint' is the directory where you want the GlusterFS -filesystem to appear. Example: - - # glusterfs -f /usr/local/etc/glusterfs-client.vol /mnt - - The command-line options are detailed below. - - Basic Options - -`-f, --volfile=<path>' - Use the volume file as the volume specification. - -`-s, --volfile-server=<hostname>' - Server to get volume file from. This option overrides -volfile - option. - -`-l, --log-file=<path>' - Specify the path for the log file. - -`-L, --log-level=<level>' - Set the log level for the server. Log level should be one of DEBUG, - WARNING, ERROR, CRITICAL, or NONE. - - Advanced Options - -`--debug' - Run in debug mode. This option sets -no-daemon, -log-level to - DEBUG and -log-file to console. - -`-N, --no-daemon' - Run `glusterfs' as a foreground process. - -`-p, --pid-file=<path>' - Path for the PID file. - -`--volfile-id=<key>' - 'key' of the volfile to be fetched from server. - -`--volfile-server-port=<port-number>' - Listening port number of volfile server. - -`--volfile-server-transport=[socket|ib-verbs]' - Transport type to get volfile from server. [default: `socket'] - -`--xlator-options=<volume-name.option=value>' - Add/override a translator option for a volume with specified value. - -`--volume-name=<volume name>' - Volume name in client spec to use. Defaults to the root volume. - - FUSE Options - -`--attribute-timeout=<n>' - Attribute timeout for inodes in the kernel, in seconds. Defaults - to 1 second. - -`--disable-direct-io-mode' - Disable direct I/O mode in FUSE kernel module. - -`-e, --entry-timeout=<n>' - Entry timeout for directory entries in the kernel, in seconds. - Defaults to 1 second. - - Missellaneous Options - -`-?, --help' - Show this help information. - -`-V, --version' - Show version information. - - -File: user-guide.info, Node: A Tutorial Introduction, Prev: Running GlusterFS, Up: Installation and Invocation - -2.5 A Tutorial Introduction -=========================== - -This section will show you how to quickly get GlusterFS up and running. -We'll configure GlusterFS as a simple network filesystem, with one -server and one client. In this mode of usage, GlusterFS can serve as a -replacement for NFS. - - We'll make use of two machines; call them _server_ and _client_ (If -you don't want to setup two machines, just run everything that follows -on the same machine). In the examples that follow, the shell prompts -will use these names to clarify the machine on which the command is -being run. For example, a command that should be run on the server will -be shown with the prompt: - - [root@server]# - - Our goal is to make a directory on the _server_ (say, `/export') -accessible to the _client_. - - First of all, get GlusterFS installed on both the machines, as -described in the previous sections. Make sure you have the FUSE kernel -module loaded. You can ensure this by running: - - [root@server]# modprobe fuse - - Before we can run the GlusterFS client or server programs, we need -to write two files called _volume specifications_ (equivalently refered -to as _volfiles_). The volfile describes the _translator tree_ on a -node. The next chapter will explain the concepts of `translator' and -`volume specification' in detail. For now, just assume that the volfile -is like an NFS `/etc/export' file. - - On the server, create a text file somewhere (we'll assume the path -`/tmp/glusterfsd.vol') with the following contents. - - volume colon-o - type storage/posix - option directory /export - end-volume - - volume server - type protocol/server - subvolumes colon-o - option transport-type tcp - option auth.addr.colon-o.allow * - end-volume - - A brief explanation of the file's contents. The first section -defines a storage volume, named "colon-o" (the volume names are -arbitrary), which exports the `/export' directory. The second section -defines options for the translator which will make the storage volume -accessible remotely. It specifies `colon-o' as a subvolume. This -defines the _translator tree_, about which more will be said in the -next chapter. The two options specify that the TCP protocol is to be -used (as opposed to InfiniBand, for example), and that access to the -storage volume is to be provided to clients with any IP address at all. -If you wanted to restrict access to this server to only your subnet for -example, you'd specify something like `192.168.1.*' in the second -option line. - - On the client machine, create the following text file (again, we'll -assume the path to be `/tmp/glusterfs-client.vol'). Replace -_server-ip-address_ with the IP address of your server machine. If you -are doing all this on a single machine, use `127.0.0.1'. - - volume client - type protocol/client - option transport-type tcp - option remote-host _server-ip-address_ - option remote-subvolume colon-o - end-volume - - Now we need to start both the server and client programs. To start -the server: - - [root@server]# glusterfsd -f /tmp/glusterfs-server.vol - - To start the client: - - [root@client]# glusterfs -f /tmp/glusterfs-client.vol /mnt/glusterfs - - You should now be able to see the files under the server's `/export' -directory in the `/mnt/glusterfs' directory on the client. That's it; -GlusterFS is now working as a network file system. - - -File: user-guide.info, Node: Concepts, Next: Translators, Prev: Installation and Invocation, Up: Top - -3 Concepts -********** - -* Menu: - -* Filesystems in Userspace:: -* Translator:: -* Volume specification file:: - - -File: user-guide.info, Node: Filesystems in Userspace, Next: Translator, Up: Concepts - -3.1 Filesystems in Userspace -============================ - -A filesystem is usually implemented in kernel space. Kernel space -development is much harder than userspace development. FUSE is a kernel -module/library that allows us to write a filesystem completely in -userspace. - - FUSE consists of a kernel module which interacts with the userspace -implementation using a device file `/dev/fuse'. When a process makes a -syscall on a FUSE filesystem, VFS hands the request to the FUSE module, -which writes the request to `/dev/fuse'. The userspace implementation -polls `/dev/fuse', and when a request arrives, processes it and writes -the result back to `/dev/fuse'. The kernel then reads from the device -file and returns the result to the user process. - - In case of GlusterFS, the userspace program is the GlusterFS client. -The control flow is shown in the diagram below. The GlusterFS client -services the request by sending it to the server, which in turn hands -it to the local POSIX filesystem. - - - Fig 1. Control flow in GlusterFS - - -File: user-guide.info, Node: Translator, Next: Volume specification file, Prev: Filesystems in Userspace, Up: Concepts - -3.2 Translator -============== - -The _translator_ is the most important concept in GlusterFS. In fact, -GlusterFS is nothing but a collection of translators working together, -forming a translator _tree_. - - The idea of a translator is perhaps best understood using an -analogy. Consider the VFS in the Linux kernel. The VFS abstracts the -various filesystem implementations (such as EXT3, ReiserFS, XFS, etc.) -supported by the kernel. When an application calls the kernel to -perform an operation on a file, the kernel passes the request on to the -appropriate filesystem implementation. - - For example, let's say there are two partitions on a Linux machine: -`/', which is an EXT3 partition, and `/usr', which is a ReiserFS -partition. Now if an application wants to open a file called, say, -`/etc/fstab', then the kernel will internally pass the request to the -EXT3 implementation. If on the other hand, an application wants to -read a file called `/usr/src/linux/CREDITS', then the kernel will call -upon the ReiserFS implementation to do the job. - - The "filesystem implementation" objects are analogous to GlusterFS -translators. A GlusterFS translator implements all the filesystem -operations. Whereas in VFS there is a two-level tree (with the kernel -at the root and all the filesystem implementation as its children), in -GlusterFS there exists a more elaborate tree structure. - - We can now define translators more precisely. A GlusterFS translator -is a shared object (`.so') that implements every filesystem call. -GlusterFS translators can be arranged in an arbitrary tree structure -(subject to constraints imposed by the translators). When GlusterFS -receives a filesystem call, it passes it on to the translator at the -root of the translator tree. The root translator may in turn pass it on -to any or all of its children, and so on, until the leaf nodes are -reached. The result of a filesystem call is communicated in the reverse -fashion, from the leaf nodes up to the root node, and then on to the -application. - - So what might a translator tree look like? - - - Fig 2. A sample translator tree - - The diagram depicts three servers and one GlusterFS client. It is -important to note that conceptually, the translator tree spans machine -boundaries. Thus, the client machine in the diagram, `10.0.0.1', can -access the aggregated storage of the filesystems on the server machines -`10.0.0.2', `10.0.0.3', and `10.0.0.4'. The translator diagram will -make more sense once you've read the next chapter and understood the -functions of the various translators. - - -File: user-guide.info, Node: Volume specification file, Prev: Translator, Up: Concepts - -3.3 Volume specification file -============================= - -The volume specification file describes the translator tree for both the -server and client programs. - - A volume specification file is a sequence of volume definitions. -The syntax of a volume definition is explained below: - - *volume* _volume-name_ - *type* _translator-name_ - *option* _option-name_ _option-value_ - ... - *subvolumes* _subvolume1_ _subvolume2_ ... - *end-volume* - - ... - -_volume-name_ - An identifier for the volume. This is just a human-readable name, - and can contain any alphanumeric character. For instance, - "storage-1", "colon-o", or "forty-two". - -_translator-name_ - Name of one of the available translators. Example: - `protocol/client', `cluster/unify'. - -_option-name_ - Name of a valid option for the translator. - -_option-value_ - Value for the option. Everything following the "option" keyword to - the end of the line is considered the value; it is up to the - translator to parse it. - -_subvolume1_, _subvolume2_, ... - Volume names of sub-volumes. The sub-volumes must already have - been defined earlier in the file. - - There are a few rules you must follow when writing a volume -specification file: - - * Everything following a ``#'' is considered a comment and is - ignored. Blank lines are also ignored. - - * All names and keywords are case-sensitive. - - * The order of options inside a volume definition does not matter. - - * An option value may not span multiple lines. - - * If an option is not specified, it will assume its default value. - - * A sub-volume must have already been defined before it can be - referenced. This means you have to write the specification file - "bottom-up", starting from the leaf nodes of the translator tree - and moving up to the root. - - A simple example volume specification file is shown below: - - # This is a comment line - volume client - type protocol/client - option transport-type tcp - option remote-host localhost # Also a comment - option remote-subvolume brick - # The subvolumes line may be absent - end-volume - - volume iot - type performance/io-threads - option thread-count 4 - subvolumes client - end-volume - - volume wb - type performance/write-behind - subvolumes iot - end-volume - - -File: user-guide.info, Node: Translators, Next: Usage Scenarios, Prev: Concepts, Up: Top - -4 Translators -************* - -* Menu: - -* Storage Translators:: -* Client and Server Translators:: -* Clustering Translators:: -* Performance Translators:: -* Features Translators:: -* Miscellaneous Translators:: - - This chapter documents all the available GlusterFS translators in -detail. Each translator section will show its name (for example, -`cluster/unify'), briefly describe its purpose and workings, and list -every option accepted by that translator and their meaning. - - -File: user-guide.info, Node: Storage Translators, Next: Client and Server Translators, Up: Translators - -4.1 Storage Translators -======================= - -The storage translators form the "backend" for GlusterFS. Currently, -the only available storage translator is the POSIX translator, which -stores files on a normal POSIX filesystem. A pleasant consequence of -this is that your data will still be accessible if GlusterFS crashes or -cannot be started. - - Other storage backends are planned for the future. One of the -possibilities is an Amazon S3 translator. Amazon S3 is an unlimited -online storage service accessible through a web services API. The S3 -translator will allow you to access the storage as a normal POSIX -filesystem. (1) - -* Menu: - -* POSIX:: -* BDB:: - - ---------- Footnotes ---------- - - (1) Some more discussion about this can be found at: - -http://developer.amazonwebservices.com/connect/message.jspa?messageID=52873 - - -File: user-guide.info, Node: POSIX, Next: BDB, Up: Storage Translators - -4.1.1 POSIX ------------ - - type storage/posix - - The `posix' translator uses a normal POSIX filesystem as its -"backend" to actually store files and directories. This can be any -filesystem that supports extended attributes (EXT3, ReiserFS, XFS, -...). Extended attributes are used by some translators to store -metadata, for example, by the replicate and stripe translators. See -*note Replicate:: and *note Stripe::, respectively for details. - -`directory <path>' - The directory on the local filesystem which is to be used for - storage. - - -File: user-guide.info, Node: BDB, Prev: POSIX, Up: Storage Translators - -4.1.2 BDB ---------- - - type storage/bdb - - The `BDB' translator uses a Berkeley DB database as its "backend" to -actually store files as key-value pair in the database and directories -as regular POSIX directories. Note that BDB does not provide extended -attribute support for regular files. Do not use BDB as storage -translator while using any translator that demands extended attributes -on "backend". - -`directory <path>' - The directory on the local filesystem which is to be used for - storage. - -`mode [cache|persistent] (cache)' - When BDB is run in `cache' mode, recovery of back-end is not - completely guaranteed. `persistent' guarantees that BDB can - recover back-end from Berkeley DB even if GlusterFS crashes. - -`errfile <path>' - The path of the file to be used as `errfile' for Berkeley DB to - report detailed error messages, if any. Note that all the contents - of this file will be written by Berkeley DB, not GlusterFS. - -`logdir <path>' - - -File: user-guide.info, Node: Client and Server Translators, Next: Clustering Translators, Prev: Storage Translators, Up: Translators - -4.2 Client and Server Translators -================================= - -The client and server translator enable GlusterFS to export a -translator tree over the network or access a remote GlusterFS server. -These two translators implement GlusterFS's network protocol. - -* Menu: - -* Transport modules:: -* Client protocol:: -* Server protocol:: - - -File: user-guide.info, Node: Transport modules, Next: Client protocol, Up: Client and Server Translators - -4.2.1 Transport modules ------------------------ - -The client and server translators are capable of using any of the -pluggable transport modules. Currently available transport modules are -`tcp', which uses a TCP connection between client and server to -communicate; `ib-sdp', which uses a TCP connection over InfiniBand, and -`ibverbs', which uses high-speed InfiniBand connections. - - Each transport module comes in two different versions, one to be -used on the server side and the other on the client side. - -4.2.1.1 TCP -........... - -The TCP transport module uses a TCP/IP connection between the server -and the client. - - option transport-type tcp - - The TCP client module accepts the following options: - -`non-blocking-connect [no|off|on|yes] (on)' - Whether to make the connection attempt asynchronous. - -`remote-port <n> (24007)' - Server port to connect to. - -`remote-host <hostname> *' - Hostname or IP address of the server. If the host name resolves to - multiple IP addresses, all of them will be tried in a round-robin - fashion. This feature can be used to implement fail-over. - - The TCP server module accepts the following options: - -`bind-address <address> (0.0.0.0)' - The local interface on which the server should listen to requests. - Default is to listen on all interfaces. - -`listen-port <n> (24007)' - The local port to listen on. - -4.2.1.2 IB-SDP -.............. - - option transport-type ib-sdp - - kernel implements socket interface for ib hardware. SDP is over -ib-verbs. This module accepts the same options as `tcp' - -4.2.1.3 ibverbs -............... - - option transport-type tcp - - InfiniBand is a scalable switched fabric interconnect mechanism -primarily used in high-performance computing. InfiniBand can deliver -data throughput of the order of 10 Gbit/s, with latencies of 4-5 ms. - - The `ib-verbs' transport accesses the InfiniBand hardware through -the "verbs" API, which is the lowest level of software access possible -and which gives the highest performance. On InfiniBand hardware, it is -always best to use `ib-verbs'. Use `ib-sdp' only if you cannot get -`ib-verbs' working for some reason. - - The `ib-verbs' client module accepts the following options: - -`non-blocking-connect [no|off|on|yes] (on)' - Whether to make the connection attempt asynchronous. - -`remote-port <n> (24007)' - Server port to connect to. - -`remote-host <hostname> *' - Hostname or IP address of the server. If the host name resolves to - multiple IP addresses, all of them will be tried in a round-robin - fashion. This feature can be used to implement fail-over. - - The `ib-verbs' server module accepts the following options: - -`bind-address <address> (0.0.0.0)' - The local interface on which the server should listen to requests. - Default is to listen on all interfaces. - -`listen-port <n> (24007)' - The local port to listen on. - - The following options are common to both the client and server -modules: - - If you are familiar with InfiniBand jargon, the mode is used by -GlusterFS is "reliable connection-oriented channel transfer". - -`ib-verbs-work-request-send-count <n> (64)' - Length of the send queue in datagrams. [Reason to - increase/decrease?] - -`ib-verbs-work-request-recv-count <n> (64)' - Length of the receive queue in datagrams. [Reason to - increase/decrease?] - -`ib-verbs-work-request-send-size <size> (128KB)' - Size of each datagram that is sent. [Reason to increase/decrease?] - -`ib-verbs-work-request-recv-size <size> (128KB)' - Size of each datagram that is received. [Reason to - increase/decrease?] - -`ib-verbs-port <n> (1)' - Port number for ib-verbs. - -`ib-verbs-mtu [256|512|1024|2048|4096] (2048)' - The Maximum Transmission Unit [Reason to increase/decrease?] - -`ib-verbs-device-name <device-name> (first device in the list)' - InfiniBand device to be used. - - For maximum performance, you should ensure that the send/receive -counts on both the client and server are the same. - - ib-verbs is preferred over ib-sdp. - - -File: user-guide.info, Node: Client protocol, Next: Server protocol, Prev: Transport modules, Up: Client and Server Translators - -4.2.2 Client ------------- - - type procotol/client - - The client translator enables the GlusterFS client to access a -remote server's translator tree. - -`transport-type [tcp,ib-sdp,ib-verbs] (tcp)' - The transport type to use. You should use the client versions of - all the transport modules (`tcp', `ib-sdp', `ib-verbs'). - -`remote-subvolume <volume_name> *' - The name of the volume on the remote host to attach to. Note that - this is _not_ the name of the `protocol/server' volume on the - server. It should be any volume under the server. - -`transport-timeout <n> (120- seconds)' - Inactivity timeout. If a reply is expected and no activity takes - place on the connection within this time, the transport connection - will be broken, and a new connection will be attempted. - - -File: user-guide.info, Node: Server protocol, Prev: Client protocol, Up: Client and Server Translators - -4.2.3 Server ------------- - - type protocol/server - - The server translator exports a translator tree and makes it -accessible to remote GlusterFS clients. - -`client-volume-filename <path> (<CONFDIR>/glusterfs-client.vol)' - The volume specification file to use for the client. This is the - file the client will receive when it is invoked with the - `--server' option (*note Client::). - -`transport-type [tcp,ib-verbs,ib-sdp] (tcp)' - The transport to use. You should use the server versions of all - the transport modules (`tcp', `ib-sdp', `ib-verbs'). - -`auth.addr.<volume name>.allow <IP address wildcard pattern>' - IP addresses of the clients that are allowed to attach to the - specified volume. This can be a wildcard. For example, a wildcard - of the form `192.168.*.*' allows any host in the `192.168.x.x' - subnet to connect to the server. - - - -File: user-guide.info, Node: Clustering Translators, Next: Performance Translators, Prev: Client and Server Translators, Up: Translators - -4.3 Clustering Translators -========================== - -The clustering translators are the most important GlusterFS -translators, since it is these that make GlusterFS a cluster -filesystem. These translators together enable GlusterFS to access an -arbitrarily large amount of storage, and provide RAID-like redundancy -and distribution over the entire cluster. - - There are three clustering translators: *unify*, *replicate*, and -*stripe*. The unify translator aggregates storage from many server -nodes. The replicate translator provides file replication. The stripe -translator allows a file to be spread across many server nodes. The -following sections look at each of these translators in detail. - -* Menu: - -* Unify:: -* Replicate:: -* Stripe:: - - -File: user-guide.info, Node: Unify, Next: Replicate, Up: Clustering Translators - -4.3.1 Unify ------------ - - type cluster/unify - - The unify translator presents a `unified' view of all its -sub-volumes. That is, it makes the union of all its sub-volumes appear -as a single volume. It is the unify translator that gives GlusterFS the -ability to access an arbitrarily large amount of storage. - - For unify to work correctly, certain invariants need to be -maintained across the entire network. These are: - - * The directory structure of all the sub-volumes must be identical. - - * A particular file can exist on only one of the sub-volumes. - Phrasing it in another way, a pathname such as - `/home/calvin/homework.txt') is unique across the entire cluster. - - - -Looking at the second requirement, you might wonder how one can -accomplish storing redundant copies of a file, if no file can exist -multiple times. To answer, we must remember that these invariants are -from _unify's perspective_. A translator such as replicate at a lower -level in the translator tree than unify may subvert this picture. - - The first invariant might seem quite tedious to ensure. We shall see -later that this is not so, since unify's _self-heal_ mechanism takes -care of maintaining it. - - The second invariant implies that unify needs some way to decide -which file goes where. Unify makes use of _scheduler_ modules for this -purpose. - - When a file needs to be created, unify's scheduler decides upon the -sub-volume to be used to store the file. There are many schedulers -available, each using a different algorithm and suitable for different -purposes. - - The various schedulers are described in detail in the sections that -follow. - -4.3.1.1 ALU -........... - - option scheduler alu - - ALU stands for "Adaptive Least Usage". It is the most advanced -scheduler available in GlusterFS. It balances the load across volumes -taking several factors in account. It adapts itself to changing I/O -patterns according to its configuration. When properly configured, it -can eliminate the need for regular tuning of the filesystem to keep -volume load nicely balanced. - - The ALU scheduler is composed of multiple least-usage -sub-schedulers. Each sub-scheduler keeps track of a certain type of -load, for each of the sub-volumes, getting statistics from the -sub-volumes themselves. The sub-schedulers are these: - - * disk-usage: The used and free disk space on the volume. - - * read-usage: The amount of reading done from this volume. - - * write-usage: The amount of writing done to this volume. - - * open-files-usage: The number of files currently open from this - volume. - - * disk-speed-usage: The speed at which the disks are spinning. This - is a constant value and therefore not very useful. - - The ALU scheduler needs to know which of these sub-schedulers to use, -and in which order to evaluate them. This is done through the `option -alu.order' configuration directive. - - Each sub-scheduler needs to know two things: when to kick in (the -entry-threshold), and how long to stay in control (the exit-threshold). -For example: when unifying three disks of 100GB, keeping an exact -balance of disk-usage is not necesary. Instead, there could be a 1GB -margin, which can be used to nicely balance other factors, such as -read-usage. The disk-usage scheduler can be told to kick in only when a -certain threshold of discrepancy is passed, such as 1GB. When it -assumes control under this condition, it will write all subsequent data -to the least-used volume. If it is doing so, it is unwise to stop right -after the values are below the entry-threshold again, since that would -make it very likely that the situation will occur again very soon. Such -a situation would cause the ALU to spend most of its time disk-usage -scheduling, which is unfair to the other sub-schedulers. The -exit-threshold therefore defines the amount of data that needs to be -written to the least-used disk, before control is relinquished again. - - In addition to the sub-schedulers, the ALU scheduler also has -"limits" options. These can stop the creation of new files on a volume -once values drop below a certain threshold. For example, setting -`option alu.limits.min-free-disk 5GB' will stop the scheduling of files -to volumes that have less than 5GB of free disk space, leaving the -files on that disk some room to grow. - - The actual values you assign to the thresholds for sub-schedulers and -limits depend on your situation. If you have fast-growing files, you'll -want to stop file-creation on a disk much earlier than when hardly any -of your files are growing. If you care less about disk-usage balance -than about read-usage balance, you'll want a bigger disk-usage -scheduler entry-threshold and a smaller read-usage scheduler -entry-threshold. - - For thresholds defining a size, values specifying "KB", "MB" and "GB" -are allowed. For example: `option alu.limits.min-free-disk 5GB'. - -`alu.order <order> * ("disk-usage:write-usage:read-usage:open-files-usage:disk-speed")' - -`alu.disk-usage.entry-threshold <size> (1GB)' - -`alu.disk-usage.exit-threshold <size> (512MB)' - -`alu.write-usage.entry-threshold <%> (25)' - -`alu.write-usage.exit-threshold <%> (5)' - -`alu.read-usage.entry-threshold <%> (25)' - -`alu.read-usage.exit-threshold <%> (5)' - -`alu.open-files-usage.entry-threshold <n> (1000)' - -`alu.open-files-usage.exit-threshold <n> (100)' - -`alu.limits.min-free-disk <%>' - -`alu.limits.max-open-files <n>' - -4.3.1.2 Round Robin (RR) -........................ - - option scheduler rr - - Round-Robin (RR) scheduler creates files in a round-robin fashion. -Each client will have its own round-robin loop. When your files are -mostly similar in size and I/O access pattern, this scheduler is a good -choice. RR scheduler checks for free disk space on the server before -scheduling, so you can know when to add another server node. The -default value of min-free-disk is 5% and is checked on file creation -calls, with atleast 10 seconds (by default) elapsing between two checks. - - Options: -`rr.limits.min-free-disk <%> (5)' - Minimum free disk space a node must have for RR to schedule a file - to it. - -`rr.refresh-interval <t> (10 seconds)' - Time between two successive free disk space checks. - -4.3.1.3 Random -.............. - - option scheduler random - - The random scheduler schedules file creation randomly among its -child nodes. Like the round-robin scheduler, it also checks for a -minimum amount of free disk space before scheduling a file to a node. - -`random.limits.min-free-disk <%> (5)' - Minimum free disk space a node must have for random to schedule a - file to it. - -`random.refresh-interval <t> (10 seconds)' - Time between two successive free disk space checks. - -4.3.1.4 NUFA -............ - - option scheduler nufa - - It is common in many GlusterFS computing environments for all -deployed machines to act as both servers and clients. For example, a -research lab may have 40 workstations each with its own storage. All of -these workstations might act as servers exporting a volume as well as -clients accessing the entire cluster's storage. In such a situation, -it makes sense to store locally created files on the local workstation -itself (assuming files are accessed most by the workstation that -created them). The Non-Uniform File Allocation (NUFA) scheduler -accomplishes that. - - NUFA gives the local system first priority for file creation over -other nodes. If the local volume does not have more free disk space -than a specified amount (5% by default) then NUFA schedules files among -the other child volumes in a round-robin fashion. - - NUFA is named after the similar strategy used for memory access, -NUMA(1). - -`nufa.limits.min-free-disk <%> (5)' - Minimum disk space that must be free (local or remote) for NUFA to - schedule a file to it. - -`nufa.refresh-interval <t> (10 seconds)' - Time between two successive free disk space checks. - -`nufa.local-volume-name <volume>' - The name of the volume corresponding to the local system. This - volume must be one of the children of the unify volume. This - option is mandatory. - -4.3.1.5 Namespace -................. - -Namespace volume needed because: - persistent inode numbers. - file -exists even when node is down. - - namespace files are simply touched. on every lookup it is checked. - -`namespace <volume> *' - Name of the namespace volume (which should be one of the unify - volume's children). - -`self-heal [on|off] (on)' - Enable/disable self-heal. Unless you know what you are doing, do - not disable self-heal. - -4.3.1.6 Self Heal -................. - -* When a 'lookup()/stat()' call is made on directory for the first -time, a self-heal call is made, which checks for the consistancy of its -child nodes. If an entry is present in storage node, but not in -namespace, that entry is created in namespace, and vica-versa. There is -an writedir() API introduced which is used for the same. It also checks -for permissions, and uid/gid consistencies. - - * This check is also done when an server goes down and comes up. - - * If one starts with an empty namespace export, but has data in -storage nodes, a 'find .>/dev/null' or 'ls -lR >/dev/null' should help -to build namespace in one shot. Even otherwise, namespace is built on -demand when a file is looked up for the first time. - - NOTE: There are some issues (Kernel 'Oops' msgs) seen with -fuse-2.6.3, when someone deletes namespace in backend, when glusterfs is -running. But with fuse-2.6.5, this issue is not there. - - ---------- Footnotes ---------- - - (1) Non-Uniform Memory Access: -<http://en.wikipedia.org/wiki/Non-Uniform_Memory_Access> - - -File: user-guide.info, Node: Replicate, Next: Stripe, Prev: Unify, Up: Clustering Translators - -4.3.2 Replicate (formerly AFR) ------------------------------- - - type cluster/replicate - - Replicate provides RAID-1 like functionality for GlusterFS. -Replicate replicates files and directories across the subvolumes. Hence -if Replicate has four subvolumes, there will be four copies of all -files and directories. Replicate provides high-availability, i.e., in -case one of the subvolumes go down (e. g. server crash, network -disconnection) Replicate will still service the requests using the -redundant copies. - - Replicate also provides self-heal functionality, i.e., in case the -crashed servers come up, the outdated files and directories will be -updated with the latest versions. Replicate uses extended attributes of -the backend file system to track the versioning of files and -directories and provide the self-heal feature. - - volume replicate-example - type cluster/replicate - subvolumes brick1 brick2 brick3 - end-volume - - This sample configuration will replicate all directories and files on -brick1, brick2 and brick3. - - All the read operations happen from the first alive child. If all the -three sub-volumes are up, reads will be done from brick1; if brick1 is -down read will be done from brick2. In case read() was being done on -brick1 and it goes down, replicate transparently falls back to brick2. - - The next release of GlusterFS will add the following features: - * Ability to specify the sub-volume from which read operations are - to be done (this will help users who have one of the sub-volumes - as a local storage volume). - - * Allow scheduling of read operations amongst the sub-volumes in a - round-robin fashion. - - The order of the subvolumes list should be same across all the -'replicate's as they will be used for locking purposes. - -4.3.2.1 Self Heal -................. - -Replicate has self-heal feature, which updates the outdated file and -directory copies by the most recent versions. For example consider the -following config: - - volume replicate-example - type cluster/replicate - subvolumes brick1 brick2 - end-volume - -4.3.2.2 File self-heal -...................... - -Now if we create a file foo.txt on replicate-example, the file will be -created on brick1 and brick2. The file will have two extended -attributes associated with it in the backend filesystem. One is -trusted.afr.createtime and the other is trusted.afr.version. The -trusted.afr.createtime xattr has the create time (in terms of seconds -since epoch) and trusted.afr.version is a number that is incremented -each time a file is modified. This increment happens during close -(incase any write was done before close). - - If brick1 goes down, we edit foo.txt the version gets incremented. -Now the brick1 comes back up, when we open() on foo.txt replicate will -check if their versions are same. If they are not same, the outdated -copy is replaced by the latest copy and its version is updated. After -the sync the open() proceeds in the usual manner and the application -calling open() can continue on its access to the file. - - If brick1 goes down, we delete foo.txt and create a file with the -same name again i.e foo.txt. Now brick1 comes back up, clearly there is -a chance that the version on brick1 being more than the version on -brick2, this is where createtime extended attribute helps in deciding -which the outdated copy is. Hence we need to consider both createtime -and version to decide on the latest copy. - - The version attribute is incremented during the close() call. Version -will not be incremented in case there was no write() done. In case the -fd that the close() gets was got by create() call, we also create the -createtime extended attribute. - -4.3.2.3 Directory self-heal -........................... - -Suppose brick1 goes down, we delete foo.txt, brick1 comes back up, now -we should not create foo.txt on brick2 but we should delete foo.txt on -brick1. We handle this situation by having the createtime and version -attribute on the directory similar to the file. when lookup() is done -on the directory, we compare the createtime/version attributes of the -copies and see which files needs to be deleted and delete those files -and update the extended attributes of the outdated directory copy. -Each time a directory is modified (a file or a subdirectory is created -or deleted inside the directory) and one of the subvols is down, we -increment the directory's version. - - lookup() is a call initiated by the kernel on a file or directory -just before any access to that file or directory. In glusterfs, by -default, lookup() will not be called in case it was called in the past -one second on that particular file or directory. - - The extended attributes can be seen in the backend filesystem using -the `getfattr' command. (`getfattr -n trusted.afr.version <file>') - -`debug [on|off] (off)' - -`self-heal [on|off] (on)' - -`replicate <pattern> (*:1)' - -`lock-node <child_volume> (first child is used by default)' - - -File: user-guide.info, Node: Stripe, Prev: Replicate, Up: Clustering Translators - -4.3.3 Stripe ------------- - - type cluster/stripe - - The stripe translator distributes the contents of a file over its -sub-volumes. It does this by creating a file equal in size to the -total size of the file on each of its sub-volumes. It then writes only -a part of the file to each sub-volume, leaving the rest of it empty. -These empty regions are called `holes' in Unix terminology. The holes -do not consume any disk space. - - The diagram below makes this clear. - - - -You can configure stripe so that only filenames matching a pattern are -striped. You can also configure the size of the data to be stored on -each sub-volume. - -`block-size <pattern>:<size> (*:0 no striping)' - Distribute files matching `<pattern>' over the sub-volumes, - storing at least `<size>' on each sub-volume. For example, - - option block-size *.mpg:1M - - distributes all files ending in `.mpg', storing at least 1 MB on - each sub-volume. - - Any number of `block-size' option lines may be present, specifying - different sizes for different file name patterns. - - -File: user-guide.info, Node: Performance Translators, Next: Features Translators, Prev: Clustering Translators, Up: Translators - -4.4 Performance Translators -=========================== - -* Menu: - -* Read Ahead:: -* Write Behind:: -* IO Threads:: -* IO Cache:: -* Booster:: - - -File: user-guide.info, Node: Read Ahead, Next: Write Behind, Up: Performance Translators - -4.4.1 Read Ahead ----------------- - - type performance/read-ahead - - The read-ahead translator pre-fetches data in advance on every read. -This benefits applications that mostly process files in sequential -order, since the next block of data will already be available by the -time the application is done with the current one. - - Additionally, the read-ahead translator also behaves as a -read-aggregator. Many small read operations are combined and issued as -fewer, larger read requests to the server. - - Read-ahead deals in "pages" as the unit of data fetched. The page -size is configurable, as is the "page count", which is the number of -pages that are pre-fetched. - - Read-ahead is best used with InfiniBand (using the ib-verbs -transport). On FastEthernet and Gigabit Ethernet networks, GlusterFS -can achieve the link-maximum throughput even without read-ahead, making -it quite superflous. - - Note that read-ahead only happens if the reads are perfectly -sequential. If your application accesses data in a random fashion, -using read-ahead might actually lead to a performance loss, since -read-ahead will pointlessly fetch pages which won't be used by the -application. - - Options: -`page-size <n> (256KB)' - The unit of data that is pre-fetched. - -`page-count <n> (2)' - The number of pages that are pre-fetched. - -`force-atime-update [on|off|yes|no] (off|no)' - Whether to force an access time (atime) update on the file on - every read. Without this, the atime will be slightly imprecise, as - it will reflect the time when the read-ahead translator read the - data, not when the application actually read it. - - -File: user-guide.info, Node: Write Behind, Next: IO Threads, Prev: Read Ahead, Up: Performance Translators - -4.4.2 Write Behind ------------------- - - type performance/write-behind - - The write-behind translator improves the latency of a write -operation. It does this by relegating the write operation to the -background and returning to the application even as the write is in -progress. Using the write-behind translator, successive write requests -can be pipelined. This mode of write-behind operation is best used on -the client side, to enable decreased write latency for the application. - - The write-behind translator can also aggregate write requests. If the -`aggregate-size' option is specified, then successive writes upto that -size are accumulated and written in a single operation. This mode of -operation is best used on the server side, as this will decrease the -disk's head movement when multiple files are being written to in -parallel. - - The `aggregate-size' option has a default value of 128KB. Although -this works well for most users, you should always experiment with -different values to determine the one that will deliver maximum -performance. This is because the performance of write-behind depends on -your interconnect, size of RAM, and the work load. - -`aggregate-size <n> (128KB)' - Amount of data to accumulate before doing a write - -`flush-behind [on|yes|off|no] (off|no)' - - -File: user-guide.info, Node: IO Threads, Next: IO Cache, Prev: Write Behind, Up: Performance Translators - -4.4.3 IO Threads ----------------- - - type performance/io-threads - - The IO threads translator is intended to increase the responsiveness -of the server to metadata operations by doing file I/O (read, write) in -a background thread. Since the GlusterFS server is single-threaded, -using the IO threads translator can significantly improve performance. -This translator is best used on the server side, loaded just below the -server protocol translator. - - IO threads operates by handing out read and write requests to a -separate thread. The total number of threads in existence at a time is -constant, and configurable. - -`thread-count <n> (1)' - Number of threads to use. - - -File: user-guide.info, Node: IO Cache, Next: Booster, Prev: IO Threads, Up: Performance Translators - -4.4.4 IO Cache --------------- - - type performance/io-cache - - The IO cache translator caches data that has been read. This is -useful if many applications read the same data multiple times, and if -reads are much more frequent than writes (for example, IO caching may be -useful in a web hosting environment, where most clients will simply -read some files and only a few will write to them). - - The IO cache translator reads data from its child in `page-size' -chunks. It caches data upto `cache-size' bytes. The cache is -maintained as a prioritized least-recently-used (LRU) list, with -priorities determined by user-specified patterns to match filenames. - - When the IO cache translator detects a write operation, the cache -for that file is flushed. - - The IO cache translator periodically verifies the consistency of -cached data, using the modification times on the files. The -verification timeout is configurable. - -`page-size <n> (128KB)' - Size of a page. - -`cache-size (n) (32MB)' - Total amount of data to be cached. - -`force-revalidate-timeout <n> (1)' - Timeout to force a cache consistency verification, in seconds. - -`priority <pattern> (*:0)' - Filename patterns listed in order of priority. - - -File: user-guide.info, Node: Booster, Prev: IO Cache, Up: Performance Translators - -4.4.5 Booster -------------- - - type performance/booster - - The booster translator gives applications a faster path to -communicate read and write requests to GlusterFS. Normally, all -requests to GlusterFS from applications go through FUSE, as indicated -in *note Filesystems in Userspace::. Using the booster translator in -conjunction with the GlusterFS booster shared library, an application -can bypass the FUSE path and send read/write requests directly to the -GlusterFS client process. - - The booster mechanism consists of two parts: the booster translator, -and the booster shared library. The booster translator is meant to be -loaded on the client side, usually at the root of the translator tree. -The booster shared library should be `LD_PRELOAD'ed with the -application. - - The booster translator when loaded opens a Unix domain socket and -listens for read/write requests on it. The booster shared library -intercepts read and write system calls and sends the requests to the -GlusterFS process directly using the Unix domain socket, bypassing FUSE. -This leads to superior performance. - - Once you've loaded the booster translator in your volume -specification file, you can start your application as: - - $ LD_PRELOAD=/usr/local/bin/glusterfs-booster.so your_app - - The booster translator accepts no options. - - -File: user-guide.info, Node: Features Translators, Next: Miscellaneous Translators, Prev: Performance Translators, Up: Translators - -4.5 Features Translators -======================== - -* Menu: - -* POSIX Locks:: -* Fixed ID:: - - -File: user-guide.info, Node: POSIX Locks, Next: Fixed ID, Up: Features Translators - -4.5.1 POSIX Locks ------------------ - - type features/posix-locks - - This translator provides storage independent POSIX record locking -support (`fcntl' locking). Typically you'll want to load this on the -server side, just above the POSIX storage translator. Using this -translator you can get both advisory locking and mandatory locking -support. It also handles `flock()' locks properly. - - Caveat: Consider a file that does not have its mandatory locking bits -(+setgid, -group execution) turned on. Assume that this file is now -opened by a process on a client that has the write-behind xlator -loaded. The write-behind xlator does not cache anything for files which -have mandatory locking enabled, to avoid incoherence. Let's say that -mandatory locking is now enabled on this file through another client. -The former client will not know about this change, and write-behind may -erroneously report a write as being successful when in fact it would -fail due to the region it is writing to being locked. - - There seems to be no easy way to fix this. To work around this -problem, it is recommended that you never enable the mandatory bits on -a file while it is open. - -`mandatory [on|off] (on)' - Turns mandatory locking on. - - -File: user-guide.info, Node: Fixed ID, Prev: POSIX Locks, Up: Features Translators - -4.5.2 Fixed ID --------------- - - type features/fixed-id - - The fixed ID translator makes all filesystem requests from the client -to appear to be coming from a fixed, specified UID/GID, regardless of -which user actually initiated the request. - -`fixed-uid <n> [if not set, not used]' - The UID to send to the server - -`fixed-gid <n> [if not set, not used]' - The GID to send to the server - - -File: user-guide.info, Node: Miscellaneous Translators, Prev: Features Translators, Up: Translators - -4.6 Miscellaneous Translators -============================= - -* Menu: - -* ROT-13:: -* Trace:: - - -File: user-guide.info, Node: ROT-13, Next: Trace, Up: Miscellaneous Translators - -4.6.1 ROT-13 ------------- - - type encryption/rot-13 - - ROT-13 is a toy translator that can "encrypt" and "decrypt" file -contents using the ROT-13 algorithm. ROT-13 is a trivial algorithm that -rotates each alphabet by thirteen places. Thus, 'A' becomes 'N', 'B' -becomes 'O', and 'Z' becomes 'M'. - - It goes without saying that you shouldn't use this translator if you -need _real_ encryption (a future release of GlusterFS will have real -encryption translators). - -`encrypt-write [on|off] (on)' - Whether to encrypt on write - -`decrypt-read [on|off] (on)' - Whether to decrypt on read - - -File: user-guide.info, Node: Trace, Prev: ROT-13, Up: Miscellaneous Translators - -4.6.2 Trace ------------ - - type debug/trace - - The trace translator is intended for debugging purposes. When -loaded, it logs all the system calls received by the server or client -(wherever trace is loaded), their arguments, and the results. You must -use a GlusterFS log level of DEBUG (See *note Running GlusterFS::) for -trace to work. - - Sample trace output (lines have been wrapped for readability): - 2007-10-30 00:08:58 D [trace.c:1579:trace_opendir] trace: callid: 68 - (*this=0x8059e40, loc=0x8091984 {path=/iozone3_283, inode=0x8091f00}, - fd=0x8091d50) - - 2007-10-30 00:08:58 D [trace.c:630:trace_opendir_cbk] trace: - (*this=0x8059e40, op_ret=4, op_errno=1, fd=0x8091d50) - - 2007-10-30 00:08:58 D [trace.c:1602:trace_readdir] trace: callid: 69 - (*this=0x8059e40, size=4096, offset=0 fd=0x8091d50) - - 2007-10-30 00:08:58 D [trace.c:215:trace_readdir_cbk] trace: - (*this=0x8059e40, op_ret=0, op_errno=0, count=4) - - 2007-10-30 00:08:58 D [trace.c:1624:trace_closedir] trace: callid: 71 - (*this=0x8059e40, *fd=0x8091d50) - - 2007-10-30 00:08:58 D [trace.c:809:trace_closedir_cbk] trace: - (*this=0x8059e40, op_ret=0, op_errno=1) - - -File: user-guide.info, Node: Usage Scenarios, Next: Troubleshooting, Prev: Translators, Up: Top - -5 Usage Scenarios -***************** - -5.1 Advanced Striping -===================== - -This section is based on the Advanced Striping tutorial written by -Anand Avati on the GlusterFS wiki (1). - -5.1.1 Mixed Storage Requirements --------------------------------- - -There are two ways of scheduling the I/O. One at file level (using -unify translator) and other at block level (using stripe translator). -Striped I/O is good for files that are potentially large and require -high parallel throughput (for example, a single file of 400GB being -accessed by 100s and 1000s of systems simultaneously and randomly). For -most of the cases, file level scheduling works best. - - In the real world, it is desirable to mix file level and block level -scheduling on a single storage volume. Alternatively users can choose -to have two separate volumes and hence two mount points, but the -applications may demand a single storage system to host both. - - This document explains how to mix file level scheduling with stripe. - -5.1.2 Configuration Brief -------------------------- - -This setup demonstrates how users can configure unify translator with -appropriate I/O scheduler for file level scheduling and strip for only -matching patterns. This way, GlusterFS chooses appropriate I/O profile -and knows how to efficiently handle both the types of data. - - A simple technique to achieve this effect is to create a stripe set -of unify and stripe blocks, where unify is the first sub-volume. Files -that do not match the stripe policy passed on to first unify sub-volume -and inturn scheduled arcoss the cluster using its file level I/O -scheduler. - - 5.1.3 Preparing GlusterFS Envoronment -------------------------------------- - -Create the directories /export/namespace, /export/unify and -/export/stripe on all the storage bricks. - - Place the following server and client volume spec file under -/etc/glusterfs (or appropriate installed path) and replace the IP -addresses / access control fields to match your environment. - - ## file: /etc/glusterfs/glusterfsd.vol - volume posix-unify - type storage/posix - option directory /export/for-unify - end-volume - - volume posix-stripe - type storage/posix - option directory /export/for-stripe - end-volume - - volume posix-namespace - type storage/posix - option directory /export/for-namespace - end-volume - - volume server - type protocol/server - option transport-type tcp - option auth.addr.posix-unify.allow 192.168.1.* - option auth.addr.posix-stripe.allow 192.168.1.* - option auth.addr.posix-namespace.allow 192.168.1.* - subvolumes posix-unify posix-stripe posix-namespace - end-volume - - ## file: /etc/glusterfs/glusterfs.vol - volume client-namespace - type protocol/client - option transport-type tcp - option remote-host 192.168.1.1 - option remote-subvolume posix-namespace - end-volume - - volume client-unify-1 - type protocol/client - option transport-type tcp - option remote-host 192.168.1.1 - option remote-subvolume posix-unify - end-volume - - volume client-unify-2 - type protocol/client - option transport-type tcp - option remote-host 192.168.1.2 - option remote-subvolume posix-unify - end-volume - - volume client-unify-3 - type protocol/client - option transport-type tcp - option remote-host 192.168.1.3 - option remote-subvolume posix-unify - end-volume - - volume client-unify-4 - type protocol/client - option transport-type tcp - option remote-host 192.168.1.4 - option remote-subvolume posix-unify - end-volume - - volume client-stripe-1 - type protocol/client - option transport-type tcp - option remote-host 192.168.1.1 - option remote-subvolume posix-stripe - end-volume - - volume client-stripe-2 - type protocol/client - option transport-type tcp - option remote-host 192.168.1.2 - option remote-subvolume posix-stripe - end-volume - - volume client-stripe-3 - type protocol/client - option transport-type tcp - option remote-host 192.168.1.3 - option remote-subvolume posix-stripe - end-volume - - volume client-stripe-4 - type protocol/client - option transport-type tcp - option remote-host 192.168.1.4 - option remote-subvolume posix-stripe - end-volume - - volume unify - type cluster/unify - option scheduler rr - subvolumes cluster-unify-1 cluster-unify-2 cluster-unify-3 cluster-unify-4 - end-volume - - volume stripe - type cluster/stripe - option block-size *.img:2MB # All files ending with .img are striped with 2MB stripe block size. - subvolumes unify cluster-stripe-1 cluster-stripe-2 cluster-stripe-3 cluster-stripe-4 - end-volume - - Bring up the Storage - - Starting GlusterFS Server: If you have installed through binary -package, you can start the service through init.d startup script. If -not: - - [root@server]# glusterfsd - - Mounting GlusterFS Volumes: - - [root@client]# glusterfs -s [BRICK-IP-ADDRESS] /mnt/cluster - - Improving upon this Setup - - Infiniband Verbs RDMA transport is much faster than TCP/IP GigE -transport. - - Use of performance translators such as read-ahead, write-behind, -io-cache, io-threads, booster is recommended. - - Replace round-robin (rr) scheduler with ALU to handle more dynamic -storage environments. - - ---------- Footnotes ---------- - - (1) -http://gluster.org/docs/index.php/Mixing_Striped_and_Regular_Files - - -File: user-guide.info, Node: Troubleshooting, Next: GNU Free Documentation Licence, Prev: Usage Scenarios, Up: Top - -6 Troubleshooting -***************** - -This chapter is a general troubleshooting guide to GlusterFS. It lists -common GlusterFS server and client error messages, debugging hints, and -concludes with the suggested procedure to report bugs in GlusterFS. - -6.1 GlusterFS error messages -============================ - -6.1.1 Server errors -------------------- - - glusterfsd: FATAL: could not open specfile: - '/etc/glusterfs/glusterfsd.vol' - - The GlusterFS server expects the volume specification file to be at -`/etc/glusterfs/glusterfsd.vol'. The example specification file will be -installed as `/etc/glusterfs/glusterfsd.vol.sample'. You need to edit -it and rename it, or provide a different specification file using the -`--spec-file' command line option (See *note Server::). - - gf_log_init: failed to open logfile "/usr/var/log/glusterfs/glusterfsd.log" - (Permission denied) - - You don't have permission to create files in the -`/usr/var/log/glusterfs' directory. Make sure you are running GlusterFS -as root. Alternatively, specify a different path for the log file using -the `--log-file' option (See *note Server::). - -6.1.2 Client errors -------------------- - - fusermount: failed to access mountpoint /mnt: - Transport endpoint is not connected - - A previous failed (or hung) mount of GlusterFS is preventing it from -being mounted again in the same location. The fix is to do: - - # umount /mnt - - and try mounting again. - - *"Transport endpoint is not connected".* - - If you get this error when you try a command such as `ls' or `cat', -it means the GlusterFS mount did not succeed. Try running GlusterFS in -`DEBUG' logging level and study the log messages to discover the cause. - - *"Connect to server failed", "SERVER-ADDRESS: Connection refused".* - - GluserFS Server is not running or dead. Check your network -connections and firewall settings. To check if the server is reachable, -try: - - telnet IP-ADDRESS 24007 - - If the server is accessible, your `telnet' command should connect and -block. If not you will see an error message such as `telnet: Unable to -connect to remote host: Connection refused'. 24007 is the default -GlusterFS port. If you have changed it, then use the corresponding port -instead. - - gf_log_init: failed to open logfile "/usr/var/log/glusterfs/glusterfs.log" - (Permission denied) - - You don't have permission to create files in the -`/usr/var/log/glusterfs' directory. Make sure you are running GlusterFS -as root. Alternatively, specify a different path for the log file using -the `--log-file' option (See *note Client::). - -6.2 FUSE error messages -======================= - -`modprobe fuse' fails with: "Unknown symbol in module, or unknown -parameter". - - If you are using fuse-2.6.x on Redhat Enterprise Linux Work Station 4 -and Advanced Server 4 with 2.6.9-42.ELlargesmp, 2.6.9-42.ELsmp, -2.6.9-42.EL kernels and get this error while loading FUSE kernel -module, you need to apply the following patch. - - For fuse-2.6.2: - -<http://ftp.gluster.com/pub/gluster/glusterfs/fuse/fuse-2.6.2-rhel-build.patch> - - For fuse-2.6.3: - -<http://ftp.gluster.com/pub/gluster/glusterfs/fuse/fuse-2.6.3-rhel-build.patch> - -6.3 AppArmour and GlusterFS -=========================== - -Under OpenSuSE GNU/Linux, the AppArmour security feature does not allow -GlusterFS to create temporary files or network socket connections even -while running as root. You will see error messages like `Unable to open -log file: Operation not permitted' or `Connection refused'. Disabling -AppArmour using YaST or properly configuring AppArmour to recognize -`glusterfsd' or `glusterfs'/`fusermount' should solve the problem. - -6.4 Reporting a bug -=================== - -If you encounter a bug in GlusterFS, please follow the below guidelines -when you report it to the mailing list. Be sure to report it! User -feedback is crucial to the health of the project and we value it highly. - -6.4.1 General instructions --------------------------- - -When running GlusterFS in a non-production environment, be sure to -build it with the following command: - - $ make CFLAGS='-g -O0 -DDEBUG' - - This includes debugging information which will be helpful in getting -backtraces (see below) and also disable optimization. Enabling -optimization can result in incorrect line numbers being reported to gdb. - -6.4.2 Volume specification files --------------------------------- - -Attach all relevant server and client spec files you were using when -you encountered the bug. Also tell us details of your setup, i.e., how -many clients and how many servers. - -6.4.3 Log files ---------------- - -Set the loglevel of your client and server programs to DEBUG (by -passing the -L DEBUG option) and attach the log files with your bug -report. Obviously, if only the client is failing (for example), you -only need to send us the client log file. - -6.4.4 Backtrace ---------------- - -If GlusterFS has encountered a segmentation fault or has crashed for -some other reason, include the backtrace with the bug report. You can -get the backtrace using the following procedure. - - Run the GlusterFS client or server inside gdb. - - $ gdb ./glusterfs - (gdb) set args -f client.spec -N -l/path/to/log/file -LDEBUG /mnt/point - (gdb) run - - Now when the process segfaults, you can get the backtrace by typing: - - (gdb) bt - - If the GlusterFS process has crashed and dumped a core file (you can -find this in / if running as a daemon and in the current directory -otherwise), you can do: - - $ gdb /path/to/glusterfs /path/to/core.<pid> - - and then get the backtrace. - - If the GlusterFS server or client seems to be hung, then you can get -the backtrace by attaching gdb to the process. First get the `PID' of -the process (using ps), and then do: - - $ gdb ./glusterfs <pid> - - Press Ctrl-C to interrupt the process and then generate the -backtrace. - -6.4.5 Reproducing the bug -------------------------- - -If the bug is reproducible, please include the steps necessary to do -so. If the bug is not reproducible, send us the bug report anyway. - -6.4.6 Other information ------------------------ - -If you think it is relevant, send us also the version of FUSE you're -using, the kernel version, platform. - - -File: user-guide.info, Node: GNU Free Documentation Licence, Next: Index, Prev: Troubleshooting, Up: Top - -Appendix A GNU Free Documentation Licence -***************************************** - - Version 1.2, November 2002 - - Copyright (C) 2000,2001,2002 Free Software Foundation, Inc. - 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA - - Everyone is permitted to copy and distribute verbatim copies - of this license document, but changing it is not allowed. - - 0. PREAMBLE - - The purpose of this License is to make a manual, textbook, or other - functional and useful document "free" in the sense of freedom: to - assure everyone the effective freedom to copy and redistribute it, - with or without modifying it, either commercially or - noncommercially. Secondarily, this License preserves for the - author and publisher a way to get credit for their work, while not - being considered responsible for modifications made by others. - - This License is a kind of "copyleft", which means that derivative - works of the document must themselves be free in the same sense. - It complements the GNU General Public License, which is a copyleft - license designed for free software. - - We have designed this License in order to use it for manuals for - free software, because free software needs free documentation: a - free program should come with manuals providing the same freedoms - that the software does. But this License is not limited to - software manuals; it can be used for any textual work, regardless - of subject matter or whether it is published as a printed book. - We recommend this License principally for works whose purpose is - instruction or reference. - - 1. APPLICABILITY AND DEFINITIONS - - This License applies to any manual or other work, in any medium, - that contains a notice placed by the copyright holder saying it - can be distributed under the terms of this License. Such a notice - grants a world-wide, royalty-free license, unlimited in duration, - to use that work under the conditions stated herein. The - "Document", below, refers to any such manual or work. Any member - of the public is a licensee, and is addressed as "you". You - accept the license if you copy, modify or distribute the work in a - way requiring permission under copyright law. - - A "Modified Version" of the Document means any work containing the - Document or a portion of it, either copied verbatim, or with - modifications and/or translated into another language. - - A "Secondary Section" is a named appendix or a front-matter section - of the Document that deals exclusively with the relationship of the - publishers or authors of the Document to the Document's overall - subject (or to related matters) and contains nothing that could - fall directly within that overall subject. (Thus, if the Document - is in part a textbook of mathematics, a Secondary Section may not - explain any mathematics.) The relationship could be a matter of - historical connection with the subject or with related matters, or - of legal, commercial, philosophical, ethical or political position - regarding them. - - The "Invariant Sections" are certain Secondary Sections whose - titles are designated, as being those of Invariant Sections, in - the notice that says that the Document is released under this - License. If a section does not fit the above definition of - Secondary then it is not allowed to be designated as Invariant. - The Document may contain zero Invariant Sections. If the Document - does not identify any Invariant Sections then there are none. - - The "Cover Texts" are certain short passages of text that are - listed, as Front-Cover Texts or Back-Cover Texts, in the notice - that says that the Document is released under this License. A - Front-Cover Text may be at most 5 words, and a Back-Cover Text may - be at most 25 words. - - A "Transparent" copy of the Document means a machine-readable copy, - represented in a format whose specification is available to the - general public, that is suitable for revising the document - straightforwardly with generic text editors or (for images - composed of pixels) generic paint programs or (for drawings) some - widely available drawing editor, and that is suitable for input to - text formatters or for automatic translation to a variety of - formats suitable for input to text formatters. A copy made in an - otherwise Transparent file format whose markup, or absence of - markup, has been arranged to thwart or discourage subsequent - modification by readers is not Transparent. An image format is - not Transparent if used for any substantial amount of text. A - copy that is not "Transparent" is called "Opaque". - - Examples of suitable formats for Transparent copies include plain - ASCII without markup, Texinfo input format, LaTeX input format, - SGML or XML using a publicly available DTD, and - standard-conforming simple HTML, PostScript or PDF designed for - human modification. Examples of transparent image formats include - PNG, XCF and JPG. Opaque formats include proprietary formats that - can be read and edited only by proprietary word processors, SGML or - XML for which the DTD and/or processing tools are not generally - available, and the machine-generated HTML, PostScript or PDF - produced by some word processors for output purposes only. - - The "Title Page" means, for a printed book, the title page itself, - plus such following pages as are needed to hold, legibly, the - material this License requires to appear in the title page. For - works in formats which do not have any title page as such, "Title - Page" means the text near the most prominent appearance of the - work's title, preceding the beginning of the body of the text. - - A section "Entitled XYZ" means a named subunit of the Document - whose title either is precisely XYZ or contains XYZ in parentheses - following text that translates XYZ in another language. (Here XYZ - stands for a specific section name mentioned below, such as - "Acknowledgements", "Dedications", "Endorsements", or "History".) - To "Preserve the Title" of such a section when you modify the - Document means that it remains a section "Entitled XYZ" according - to this definition. - - The Document may include Warranty Disclaimers next to the notice - which states that this License applies to the Document. These - Warranty Disclaimers are considered to be included by reference in - this License, but only as regards disclaiming warranties: any other - implication that these Warranty Disclaimers may have is void and - has no effect on the meaning of this License. - - 2. VERBATIM COPYING - - You may copy and distribute the Document in any medium, either - commercially or noncommercially, provided that this License, the - copyright notices, and the license notice saying this License - applies to the Document are reproduced in all copies, and that you - add no other conditions whatsoever to those of this License. You - may not use technical measures to obstruct or control the reading - or further copying of the copies you make or distribute. However, - you may accept compensation in exchange for copies. If you - distribute a large enough number of copies you must also follow - the conditions in section 3. - - You may also lend copies, under the same conditions stated above, - and you may publicly display copies. - - 3. COPYING IN QUANTITY - - If you publish printed copies (or copies in media that commonly - have printed covers) of the Document, numbering more than 100, and - the Document's license notice requires Cover Texts, you must - enclose the copies in covers that carry, clearly and legibly, all - these Cover Texts: Front-Cover Texts on the front cover, and - Back-Cover Texts on the back cover. Both covers must also clearly - and legibly identify you as the publisher of these copies. The - front cover must present the full title with all words of the - title equally prominent and visible. You may add other material - on the covers in addition. Copying with changes limited to the - covers, as long as they preserve the title of the Document and - satisfy these conditions, can be treated as verbatim copying in - other respects. - - If the required texts for either cover are too voluminous to fit - legibly, you should put the first ones listed (as many as fit - reasonably) on the actual cover, and continue the rest onto - adjacent pages. - - If you publish or distribute Opaque copies of the Document - numbering more than 100, you must either include a - machine-readable Transparent copy along with each Opaque copy, or - state in or with each Opaque copy a computer-network location from - which the general network-using public has access to download - using public-standard network protocols a complete Transparent - copy of the Document, free of added material. If you use the - latter option, you must take reasonably prudent steps, when you - begin distribution of Opaque copies in quantity, to ensure that - this Transparent copy will remain thus accessible at the stated - location until at least one year after the last time you - distribute an Opaque copy (directly or through your agents or - retailers) of that edition to the public. - - It is requested, but not required, that you contact the authors of - the Document well before redistributing any large number of - copies, to give them a chance to provide you with an updated - version of the Document. - - 4. MODIFICATIONS - - You may copy and distribute a Modified Version of the Document - under the conditions of sections 2 and 3 above, provided that you - release the Modified Version under precisely this License, with - the Modified Version filling the role of the Document, thus - licensing distribution and modification of the Modified Version to - whoever possesses a copy of it. In addition, you must do these - things in the Modified Version: - - A. Use in the Title Page (and on the covers, if any) a title - distinct from that of the Document, and from those of - previous versions (which should, if there were any, be listed - in the History section of the Document). You may use the - same title as a previous version if the original publisher of - that version gives permission. - - B. List on the Title Page, as authors, one or more persons or - entities responsible for authorship of the modifications in - the Modified Version, together with at least five of the - principal authors of the Document (all of its principal - authors, if it has fewer than five), unless they release you - from this requirement. - - C. State on the Title page the name of the publisher of the - Modified Version, as the publisher. - - D. Preserve all the copyright notices of the Document. - - E. Add an appropriate copyright notice for your modifications - adjacent to the other copyright notices. - - F. Include, immediately after the copyright notices, a license - notice giving the public permission to use the Modified - Version under the terms of this License, in the form shown in - the Addendum below. - - G. Preserve in that license notice the full lists of Invariant - Sections and required Cover Texts given in the Document's - license notice. - - H. Include an unaltered copy of this License. - - I. Preserve the section Entitled "History", Preserve its Title, - and add to it an item stating at least the title, year, new - authors, and publisher of the Modified Version as given on - the Title Page. If there is no section Entitled "History" in - the Document, create one stating the title, year, authors, - and publisher of the Document as given on its Title Page, - then add an item describing the Modified Version as stated in - the previous sentence. - - J. Preserve the network location, if any, given in the Document - for public access to a Transparent copy of the Document, and - likewise the network locations given in the Document for - previous versions it was based on. These may be placed in - the "History" section. You may omit a network location for a - work that was published at least four years before the - Document itself, or if the original publisher of the version - it refers to gives permission. - - K. For any section Entitled "Acknowledgements" or "Dedications", - Preserve the Title of the section, and preserve in the - section all the substance and tone of each of the contributor - acknowledgements and/or dedications given therein. - - L. Preserve all the Invariant Sections of the Document, - unaltered in their text and in their titles. Section numbers - or the equivalent are not considered part of the section - titles. - - M. Delete any section Entitled "Endorsements". Such a section - may not be included in the Modified Version. - - N. Do not retitle any existing section to be Entitled - "Endorsements" or to conflict in title with any Invariant - Section. - - O. Preserve any Warranty Disclaimers. - - If the Modified Version includes new front-matter sections or - appendices that qualify as Secondary Sections and contain no - material copied from the Document, you may at your option - designate some or all of these sections as invariant. To do this, - add their titles to the list of Invariant Sections in the Modified - Version's license notice. These titles must be distinct from any - other section titles. - - You may add a section Entitled "Endorsements", provided it contains - nothing but endorsements of your Modified Version by various - parties--for example, statements of peer review or that the text - has been approved by an organization as the authoritative - definition of a standard. - - You may add a passage of up to five words as a Front-Cover Text, - and a passage of up to 25 words as a Back-Cover Text, to the end - of the list of Cover Texts in the Modified Version. Only one - passage of Front-Cover Text and one of Back-Cover Text may be - added by (or through arrangements made by) any one entity. If the - Document already includes a cover text for the same cover, - previously added by you or by arrangement made by the same entity - you are acting on behalf of, you may not add another; but you may - replace the old one, on explicit permission from the previous - publisher that added the old one. - - The author(s) and publisher(s) of the Document do not by this - License give permission to use their names for publicity for or to - assert or imply endorsement of any Modified Version. - - 5. COMBINING DOCUMENTS - - You may combine the Document with other documents released under - this License, under the terms defined in section 4 above for - modified versions, provided that you include in the combination - all of the Invariant Sections of all of the original documents, - unmodified, and list them all as Invariant Sections of your - combined work in its license notice, and that you preserve all - their Warranty Disclaimers. - - The combined work need only contain one copy of this License, and - multiple identical Invariant Sections may be replaced with a single - copy. If there are multiple Invariant Sections with the same name - but different contents, make the title of each such section unique - by adding at the end of it, in parentheses, the name of the - original author or publisher of that section if known, or else a - unique number. Make the same adjustment to the section titles in - the list of Invariant Sections in the license notice of the - combined work. - - In the combination, you must combine any sections Entitled - "History" in the various original documents, forming one section - Entitled "History"; likewise combine any sections Entitled - "Acknowledgements", and any sections Entitled "Dedications". You - must delete all sections Entitled "Endorsements." - - 6. COLLECTIONS OF DOCUMENTS - - You may make a collection consisting of the Document and other - documents released under this License, and replace the individual - copies of this License in the various documents with a single copy - that is included in the collection, provided that you follow the - rules of this License for verbatim copying of each of the - documents in all other respects. - - You may extract a single document from such a collection, and - distribute it individually under this License, provided you insert - a copy of this License into the extracted document, and follow - this License in all other respects regarding verbatim copying of - that document. - - 7. AGGREGATION WITH INDEPENDENT WORKS - - A compilation of the Document or its derivatives with other - separate and independent documents or works, in or on a volume of - a storage or distribution medium, is called an "aggregate" if the - copyright resulting from the compilation is not used to limit the - legal rights of the compilation's users beyond what the individual - works permit. When the Document is included in an aggregate, this - License does not apply to the other works in the aggregate which - are not themselves derivative works of the Document. - - If the Cover Text requirement of section 3 is applicable to these - copies of the Document, then if the Document is less than one half - of the entire aggregate, the Document's Cover Texts may be placed - on covers that bracket the Document within the aggregate, or the - electronic equivalent of covers if the Document is in electronic - form. Otherwise they must appear on printed covers that bracket - the whole aggregate. - - 8. TRANSLATION - - Translation is considered a kind of modification, so you may - distribute translations of the Document under the terms of section - 4. Replacing Invariant Sections with translations requires special - permission from their copyright holders, but you may include - translations of some or all Invariant Sections in addition to the - original versions of these Invariant Sections. You may include a - translation of this License, and all the license notices in the - Document, and any Warranty Disclaimers, provided that you also - include the original English version of this License and the - original versions of those notices and disclaimers. In case of a - disagreement between the translation and the original version of - this License or a notice or disclaimer, the original version will - prevail. - - If a section in the Document is Entitled "Acknowledgements", - "Dedications", or "History", the requirement (section 4) to - Preserve its Title (section 1) will typically require changing the - actual title. - - 9. TERMINATION - - You may not copy, modify, sublicense, or distribute the Document - except as expressly provided for under this License. Any other - attempt to copy, modify, sublicense or distribute the Document is - void, and will automatically terminate your rights under this - License. However, parties who have received copies, or rights, - from you under this License will not have their licenses - terminated so long as such parties remain in full compliance. - - 10. FUTURE REVISIONS OF THIS LICENSE - - The Free Software Foundation may publish new, revised versions of - the GNU Free Documentation License from time to time. Such new - versions will be similar in spirit to the present version, but may - differ in detail to address new problems or concerns. See - `http://www.gnu.org/copyleft/'. - - Each version of the License is given a distinguishing version - number. If the Document specifies that a particular numbered - version of this License "or any later version" applies to it, you - have the option of following the terms and conditions either of - that specified version or of any later version that has been - published (not as a draft) by the Free Software Foundation. If - the Document does not specify a version number of this License, - you may choose any version ever published (not as a draft) by the - Free Software Foundation. - -A.0.1 ADDENDUM: How to use this License for your documents ----------------------------------------------------------- - -To use this License in a document you have written, include a copy of -the License in the document and put the following copyright and license -notices just after the title page: - - Copyright (C) YEAR YOUR NAME. - Permission is granted to copy, distribute and/or modify this document - under the terms of the GNU Free Documentation License, Version 1.2 - or any later version published by the Free Software Foundation; - with no Invariant Sections, no Front-Cover Texts, and no Back-Cover - Texts. A copy of the license is included in the section entitled ``GNU - Free Documentation License''. - - If you have Invariant Sections, Front-Cover Texts and Back-Cover -Texts, replace the "with...Texts." line with this: - - with the Invariant Sections being LIST THEIR TITLES, with - the Front-Cover Texts being LIST, and with the Back-Cover Texts - being LIST. - - If you have Invariant Sections without Cover Texts, or some other -combination of the three, merge those two alternatives to suit the -situation. - - If your document contains nontrivial examples of program code, we -recommend releasing these examples in parallel under your choice of -free software license, such as the GNU General Public License, to -permit their use in free software. - - -File: user-guide.info, Node: Index, Prev: GNU Free Documentation Licence, Up: Top - -Index -***** - - -* Menu: - -* alu (scheduler): Unify. (line 49) -* AppArmour: Troubleshooting. (line 96) -* arch: Getting GlusterFS. (line 6) -* booster: Booster. (line 6) -* commercial support: Introduction. (line 36) -* DNS round robin: Transport modules. (line 29) -* fcntl: POSIX Locks. (line 6) -* FDL, GNU Free Documentation License: GNU Free Documentation Licence. - (line 6) -* fixed-id (translator): Fixed ID. (line 6) -* GlusterFS client: Client. (line 6) -* GlusterFS mailing list: Introduction. (line 28) -* GlusterFS server: Server. (line 6) -* infiniband transport: Transport modules. (line 58) -* InfiniBand, installation: Pre requisites. (line 51) -* io-cache (translator): IO Cache. (line 6) -* io-threads (translator): IO Threads. (line 6) -* IRC channel, #gluster: Introduction. (line 31) -* libibverbs: Pre requisites. (line 51) -* namespace: Unify. (line 207) -* nufa (scheduler): Unify. (line 175) -* OpenSuSE: Troubleshooting. (line 96) -* posix-locks (translator): POSIX Locks. (line 6) -* random (scheduler): Unify. (line 159) -* read-ahead (translator): Read Ahead. (line 6) -* record locking: POSIX Locks. (line 6) -* Redhat Enterprise Linux: Troubleshooting. (line 78) -* Replicate: Replicate. (line 6) -* rot-13 (translator): ROT-13. (line 6) -* rr (scheduler): Unify. (line 138) -* scheduler (unify): Unify. (line 6) -* self heal (replicate): Replicate. (line 46) -* self heal (unify): Unify. (line 223) -* stripe (translator): Stripe. (line 6) -* trace (translator): Trace. (line 6) -* unify (translator): Unify. (line 6) -* unify invariants: Unify. (line 16) -* write-behind (translator): Write Behind. (line 6) -* Gluster, Inc.: Introduction. (line 36) - - - -Tag Table: -Node: Top704 -Node: Acknowledgements2304 -Node: Introduction3214 -Node: Installation and Invocation4649 -Node: Pre requisites4933 -Node: Getting GlusterFS7023 -Ref: Getting GlusterFS-Footnote-17809 -Node: Building7857 -Node: Running GlusterFS9559 -Node: Server9770 -Node: Client11358 -Node: A Tutorial Introduction13564 -Node: Concepts17101 -Node: Filesystems in Userspace17316 -Node: Translator18457 -Node: Volume specification file21160 -Node: Translators23632 -Node: Storage Translators24201 -Ref: Storage Translators-Footnote-125008 -Node: POSIX25142 -Node: BDB25765 -Node: Client and Server Translators26822 -Node: Transport modules27298 -Node: Client protocol31445 -Node: Server protocol32384 -Node: Clustering Translators33373 -Node: Unify34260 -Ref: Unify-Footnote-143859 -Node: Replicate43951 -Node: Stripe49006 -Node: Performance Translators50164 -Node: Read Ahead50438 -Node: Write Behind52170 -Node: IO Threads53579 -Node: IO Cache54367 -Node: Booster55691 -Node: Features Translators57105 -Node: POSIX Locks57333 -Node: Fixed ID58650 -Node: Miscellaneous Translators59136 -Node: ROT-1359334 -Node: Trace60013 -Node: Usage Scenarios61282 -Ref: Usage Scenarios-Footnote-167215 -Node: Troubleshooting67290 -Node: GNU Free Documentation Licence73638 -Node: Index96087 - -End Tag Table diff --git a/doc/user-guide/legacy/user-guide.pdf b/doc/user-guide/legacy/user-guide.pdf Binary files differdeleted file mode 100644 index ed7bd2a9907..00000000000 --- a/doc/user-guide/legacy/user-guide.pdf +++ /dev/null diff --git a/doc/user-guide/legacy/user-guide.texi b/doc/user-guide/legacy/user-guide.texi deleted file mode 100644 index 2d51da022cf..00000000000 --- a/doc/user-guide/legacy/user-guide.texi +++ /dev/null @@ -1,2246 +0,0 @@ -\input texinfo -@setfilename user-guide.info -@settitle GlusterFS 2.0 User Guide -@afourpaper - -@direntry -* GlusterFS: (user-guide). GlusterFS distributed filesystem user guide -@end direntry - -@copying -This is the user manual for GlusterFS 2.0. - -Copyright @copyright{} 2007-2011 @email{@b{Gluster}} , Inc. Permission is granted to -copy, distribute and/or modify this document under the terms of the -@acronym{GNU} Free Documentation License, Version 1.2 or any later -version published by the Free Software Foundation; with no Invariant -Sections, no Front-Cover Texts, and no Back-Cover Texts. A copy of the -license is included in the chapter entitled ``@acronym{GNU} Free -Documentation License''. -@end copying - -@titlepage -@title GlusterFS 2.0 User Guide [DRAFT] -@subtitle January 15, 2008 -@author http://gluster.org/core-team.php -@author @email{@b{Gluster}} -@page -@vskip 0pt plus 1filll -@insertcopying -@end titlepage - -@c Info stuff -@ifnottex -@node Top -@top GlusterFS 2.0 User Guide - -@insertcopying -@menu -* Acknowledgements:: -* Introduction:: -* Installation and Invocation:: -* Concepts:: -* Translators:: -* Usage Scenarios:: -* Troubleshooting:: -* GNU Free Documentation Licence:: -* Index:: - -@detailmenu - --- The Detailed Node Listing --- - -Installation and Invocation - -* Pre requisites:: -* Getting GlusterFS:: -* Building:: -* Running GlusterFS:: -* A Tutorial Introduction:: - -Running GlusterFS - -* Server:: -* Client:: - -Concepts - -* Filesystems in Userspace:: -* Translator:: -* Volume specification file:: - -Translators - -* Storage Translators:: -* Client and Server Translators:: -* Clustering Translators:: -* Performance Translators:: -* Features Translators:: - -Storage Translators - -* POSIX:: - -Client and Server Translators - -* Transport modules:: -* Client protocol:: -* Server protocol:: - -Clustering Translators - -* Unify:: -* Replicate:: -* Stripe:: - -Performance Translators - -* Read Ahead:: -* Write Behind:: -* IO Threads:: -* IO Cache:: - -Features Translators - -* POSIX Locks:: -* Fixed ID:: - -Miscellaneous Translators - -* ROT-13:: -* Trace:: - -@end detailmenu -@end menu - -@end ifnottex -@c Info stuff end - -@contents - -@node Acknowledgements -@unnumbered Acknowledgements -GlusterFS continues to be a wonderful and enriching experience for all -of us involved. - -GlusterFS development would not have been possible at this pace if -not for our enthusiastic users. People from around the world have -helped us with bug reports, performance numbers, and feature suggestions. -A huge thanks to them all. - -Matthew Paine - for RPMs & general enthu - -Leonardo Rodrigues de Mello - for DEBs - -Julian Perez & Adam D'Auria - for multi-server tutorial - -Paul England - for HA spec - -Brent Nelson - for many bug reports - -Jacques Mattheij - for Europe mirror. - -Patrick Negri - for TCP non-blocking connect. -@flushright -http://gluster.org/core-team.php (@email{list-hacking@@gluster.com}) -@email{@b{Gluster}} -@end flushright - -@node Introduction -@chapter Introduction - -GlusterFS is a distributed filesystem. It works at the file level, -not block level. - -A network filesystem is one which allows us to access remote files. A -distributed filesystem is one that stores data on multiple machines -and makes them all appear to be a part of the same filesystem. - -Need for distributed filesystems - -@itemize @bullet -@item Scalability: A distributed filesystem allows us to store more data than what can be stored on a single machine. - -@item Redundancy: We might want to replicate crucial data on to several machines. - -@item Uniform access: One can mount a remote volume (for example your home directory) from any machine and access the same data. -@end itemize - -@section Contacting us -You can reach us through the mailing list @strong{gluster-devel} -(@email{gluster-devel@@nongnu.org}). -@cindex GlusterFS mailing list - -You can also find many of the developers on @acronym{IRC}, on the @code{#gluster} -channel on Freenode (@indicateurl{irc.freenode.net}). -@cindex IRC channel, #gluster - -The GlusterFS documentation wiki is also useful: @* -@indicateurl{http://gluster.org/docs/index.php/GlusterFS} - -For commercial support, you can contact @email{@b{Gluster}} at: -@cindex commercial support -@cindex Gluster, Inc. - -@display -3194 Winding Vista Common -Fremont, CA 94539 -USA. - -Phone: +1 (510) 354 6801 -Toll free: +1 (888) 813 6309 -Fax: +1 (510) 372 0604 -@end display - -You can also email us at @email{support@@gluster.com}. - -@node Installation and Invocation -@chapter Installation and Invocation - -@menu -* Pre requisites:: -* Getting GlusterFS:: -* Building:: -* Running GlusterFS:: -* A Tutorial Introduction:: -@end menu - -@node Pre requisites -@section Pre requisites - -Before installing GlusterFS make sure you have the -following components installed. - -@subsection @acronym{FUSE} -GlusterFS has now built-in support for the @acronym{FUSE} protocol. -You need a kernel with @acronym{FUSE} support to mount GlusterFS. -You do not need the @acronym{FUSE} package (library and utilities), -but be aware of the following issues: - -@itemize -@item If you want unprivileged users to be able to mount GlusterFS filesystems, -you need a recent version of the @command{fusermount} utility. You already have -it if you have @acronym{FUSE} version 2.7.0 or higher installed; if that's not -the case, one will be compiled along with GlusterFS if you pass -@command{--enable-fusermount} to the @command{configure} script. @item You -need to ensure @acronym{FUSE} support is configured properly on your system. In -details: -@itemize -@item If your kernel has @acronym{FUSE} as a loadable module, make sure it's -loaded. -@item Create @command{/dev/fuse} (major 10, minor 229) either by means of udev -rules or by hand. -@item Optionally, if you want runtime control over your @acronym{FUSE} mounts, -mount the fusectl auxiliary filesystem: - -@example -# mount -t fusectl none /sys/fs/fuse/connections -@end example -@end itemize - -The @acronym{FUSE} packages shipped by the various distributions usually take care -about these things, so the easiest way to get the above tasks handled is still -installing the @acronym{FUSE} package(s). -@end itemize - -To get the best performance from GlusterFS,it is recommended that you use -our patched version of the @acronym{FUSE} kernel module. See Patched FUSE for details. - -@subsection Patched FUSE - -The GlusterFS project maintains a patched version of @acronym{FUSE} meant to be used -with GlusterFS. The patches increase GlusterFS performance. It is recommended that -all users use the patched @acronym{FUSE}. - -The patched @acronym{FUSE} tarball can be downloaded from: - -@indicateurl{ftp://ftp.gluster.com/pub/gluster/glusterfs/fuse/} - -The specific changes made to @acronym{FUSE} are: - -@itemize -@item The communication channel size between @acronym{FUSE} kernel module and GlusterFS has been increased to 1MB, permitting large reads and writes to be sent in bigger chunks. - -@item The kernel's read-ahead boundry has been extended upto 1MB. - -@item Block size returned in the @command{stat()}/@command{fstat()} calls tuned to 1MB, to make cp and similar commands perform I/O using that block size. - -@item @command{flock()} locking support has been added (although some rework in GlusterFS is needed for perfect compliance). -@end itemize - -@subsection libibverbs (optional) -@cindex InfiniBand, installation -@cindex libibverbs -This is only needed if you want GlusterFS to use InfiniBand as the -interconnect mechanism between server and client. You can get it from: - -@indicateurl{http://www.openfabrics.org/downloads.htm}. - -@subsection Bison and Flex -These should be already installed on most Linux systems. If not, use your distribution's -normal software installation procedures to install them. Make sure you install the -relevant developer packages also. - -@node Getting GlusterFS -@section Getting GlusterFS -@cindex arch -There are many ways to get hold of GlusterFS. For a production deployment, -the recommended method is to download the latest release tarball. -Release tarballs are available at: @indicateurl{http://gluster.org/download.php}. - -If you want the bleeding edge development source, you can get them -from the Git -@footnote{@indicateurl{http://git-scm.com}} -repository. First you must install Git itself. Then -you can check out the source - -@example -$ git clone git://git.sv.gnu.org/gluster.git glusterfs -@end example - -@node Building -@section Building -You can skip this section if you're installing from @acronym{RPM}s -or @acronym{DEB}s. - -GlusterFS uses the Autotools mechanism to build. As such, the procedure -is straight-forward. First, change into the GlusterFS source directory. - -@example -$ cd glusterfs-<version> -@end example - -If you checked out the source from the Arch repository, you'll need -to run @command{./autogen.sh} first. Note that you'll need to have -Autoconf and Automake installed for this. - -Run @command{configure}. - -@example -$ ./configure -@end example - -The configure script accepts the following options: - -@cartouche -@table @code - -@item --disable-ibverbs -Disable the InfiniBand transport mechanism. - -@item --disable-fuse-client -Disable the @acronym{FUSE} client. - -@item --disable-server -Disable building of the GlusterFS server. - -@item --disable-bdb -Disable building of Berkeley DB based storage translator. - -@item --disable-mod_glusterfs -Disable building of Apache/lighttpd glusterfs plugins. - -@item --disable-epoll -Use poll instead of epoll. - -@item --disable-libglusterfsclient -Disable building of libglusterfsclient - -@item --enable-fusermount -Build fusermount - -@end table -@end cartouche - -Build and install GlusterFS. - -@example -# make install -@end example - -The binaries (@command{glusterfsd} and @command{glusterfs}) will be by -default installed in @command{/usr/local/sbin/}. Translator, -scheduler, and transport shared libraries will be installed in -@command{/usr/local/lib/glusterfs/<version>/}. Sample volume -specification files will be in @command{/usr/local/etc/glusterfs/}. -This document itself can be found in -@command{/usr/local/share/doc/glusterfs/}. If you passed the @command{--prefix} -argument to the configure script, then replace @command{/usr/local} in the preceding -paths with the prefix. - -@node Running GlusterFS -@section Running GlusterFS - -@menu -* Server:: -* Client:: -@end menu - -@node Server -@subsection Server -@cindex GlusterFS server - -The GlusterFS server is necessary to export storage volumes to remote clients -(See @ref{Server protocol} for more info). This section documents the invocation -of the GlusterFS server program and all the command-line options accepted by it. - -@cartouche -@table @code -Basic Options -@item -f, --volfile=<path> - Use the volume file as the volume specification. - -@item -s, --volfile-server=<hostname> - Server to get volume file from. This option overrides --volfile option. - -@item -l, --log-file=<path> - Specify the path for the log file. - -@item -L, --log-level=<level> - Set the log level for the server. Log level should be one of @acronym{DEBUG}, -@acronym{WARNING}, @acronym{ERROR}, @acronym{CRITICAL}, or @acronym{NONE}. - -Advanced Options -@item --debug - Run in debug mode. This option sets --no-daemon, --log-level to DEBUG and - --log-file to console. - -@item -N, --no-daemon - Run glusterfsd as a foreground process. - -@item -p, --pid-file=<path> - Path for the @acronym{PID} file. - -@item --volfile-id=<key> - 'key' of the volfile to be fetched from server. - -@item --volfile-server-port=<port-number> - Listening port number of volfile server. - -@item --volfile-server-transport=[socket|ib-verbs] - Transport type to get volfile from server. [default: @command{socket}] - -@item --xlator-options=<volume-name.option=value> - Add/override a translator option for a volume with specified value. - -Miscellaneous Options -@item -?, --help - Show this help text. - -@item --usage - Display a short usage message. - -@item -V, --version - Show version information. -@end table -@end cartouche - -@node Client -@subsection Client -@cindex GlusterFS client - -The GlusterFS client process is necessary to access remote storage volumes and -mount them locally using @acronym{FUSE}. This section documents the invocation of the -client process and all its command-line arguments. - -@example - # glusterfs [options] <mountpoint> -@end example - -The @command{mountpoint} is the directory where you want the GlusterFS -filesystem to appear. Example: - -@example - # glusterfs -f /usr/local/etc/glusterfs-client.vol /mnt -@end example - -The command-line options are detailed below. - -@tex -\vfill -@end tex -@page - -@cartouche -@table @code - -Basic Options -@item -f, --volfile=<path> - Use the volume file as the volume specification. - -@item -s, --volfile-server=<hostname> - Server to get volume file from. This option overrides --volfile option. - -@item -l, --log-file=<path> - Specify the path for the log file. - -@item -L, --log-level=<level> - Set the log level for the server. Log level should be one of @acronym{DEBUG}, -@acronym{WARNING}, @acronym{ERROR}, @acronym{CRITICAL}, or @acronym{NONE}. - -Advanced Options -@item --debug - Run in debug mode. This option sets --no-daemon, --log-level to DEBUG and - --log-file to console. - -@item -N, --no-daemon - Run @command{glusterfs} as a foreground process. - -@item -p, --pid-file=<path> - Path for the @acronym{PID} file. - -@item --volfile-id=<key> - 'key' of the volfile to be fetched from server. - -@item --volfile-server-port=<port-number> - Listening port number of volfile server. - -@item --volfile-server-transport=[socket|ib-verbs] - Transport type to get volfile from server. [default: @command{socket}] - -@item --xlator-options=<volume-name.option=value> - Add/override a translator option for a volume with specified value. - -@item --volume-name=<volume name> - Volume name in client spec to use. Defaults to the root volume. - -@acronym{FUSE} Options -@item --attribute-timeout=<n> - Attribute timeout for inodes in the kernel, in seconds. Defaults to 1 second. - -@item --disable-direct-io-mode - Disable direct @acronym{I/O} mode in @acronym{FUSE} kernel module. This is set - automatically if kernel supports big writes (>= 2.6.26). - -@item -e, --entry-timeout=<n> - Entry timeout for directory entries in the kernel, in seconds. - Defaults to 1 second. - -Missellaneous Options -@item -?, --help - Show this help information. - -@item -V, --version - Show version information. -@end table -@end cartouche - -@node A Tutorial Introduction -@section A Tutorial Introduction - -This section will show you how to quickly get GlusterFS up and running. We'll -configure GlusterFS as a simple network filesystem, with one server and one client. -In this mode of usage, GlusterFS can serve as a replacement for NFS. - -We'll make use of two machines; call them @emph{server} and -@emph{client} (If you don't want to setup two machines, just run -everything that follows on the same machine). In the examples that -follow, the shell prompts will use these names to clarify the machine -on which the command is being run. For example, a command that should -be run on the server will be shown with the prompt: - -@example -[root@@server]# -@end example - -Our goal is to make a directory on the @emph{server} (say, @command{/export}) -accessible to the @emph{client}. - -First of all, get GlusterFS installed on both the machines, as described in the -previous sections. Make sure you have the @acronym{FUSE} kernel module loaded. You -can ensure this by running: - -@example -[root@@server]# modprobe fuse -@end example - -Before we can run the GlusterFS client or server programs, we need to write -two files called @emph{volume specifications} (equivalently refered to as @emph{volfiles}). -The volfile describes the @emph{translator tree} on a node. The next chapter will -explain the concepts of `translator' and `volume specification' in detail. For now, -just assume that the volfile is like an NFS @command{/etc/export} file. - -On the server, create a text file somewhere (we'll assume the path -@command{/tmp/glusterfsd.vol}) with the following contents. - -@cartouche -@example -volume colon-o - type storage/posix - option directory /export -end-volume - -volume server - type protocol/server - subvolumes colon-o - option transport-type tcp - option auth.addr.colon-o.allow * -end-volume -@end example -@end cartouche - -A brief explanation of the file's contents. The first section defines a storage -volume, named ``colon-o'' (the volume names are arbitrary), which exports the -@command{/export} directory. The second section defines options for the translator -which will make the storage volume accessible remotely. It specifies @command{colon-o} as -a subvolume. This defines the @emph{translator tree}, about which more will be said -in the next chapter. The two options specify that the @acronym{TCP} protocol is to be -used (as opposed to InfiniBand, for example), and that access to the storage volume -is to be provided to clients with any @acronym{IP} address at all. If you wanted to -restrict access to this server to only your subnet for example, you'd specify -something like @command{192.168.1.*} in the second option line. - -On the client machine, create the following text file (again, we'll assume -the path to be @command{/tmp/glusterfs-client.vol}). Replace -@emph{server-ip-address} with the @acronym{IP} address of your server machine. If you -are doing all this on a single machine, use @command{127.0.0.1}. - -@cartouche -@example -volume client - type protocol/client - option transport-type tcp - option remote-host @emph{server-ip-address} - option remote-subvolume colon-o -end-volume -@end example -@end cartouche - -Now we need to start both the server and client programs. To start the server: - -@example -[root@@server]# glusterfsd -f /tmp/glusterfs-server.vol -@end example - -To start the client: - -@example -[root@@client]# glusterfs -f /tmp/glusterfs-client.vol /mnt/glusterfs -@end example - -You should now be able to see the files under the server's @command{/export} directory -in the @command{/mnt/glusterfs} directory on the client. That's it; GlusterFS is now -working as a network file system. - -@node Concepts -@chapter Concepts - -@menu -* Filesystems in Userspace:: -* Translator:: -* Volume specification file:: -@end menu - -@node Filesystems in Userspace -@section Filesystems in Userspace - -A filesystem is usually implemented in kernel space. Kernel space -development is much harder than userspace development. @acronym{FUSE} -is a kernel module/library that allows us to write a filesystem -completely in userspace. - -@acronym{FUSE} consists of a kernel module which interacts with the userspace -implementation using a device file @code{/dev/fuse}. When a process -makes a syscall on a @acronym{FUSE} filesystem, @acronym{VFS} hands the request to the -@acronym{FUSE} module, which writes the request to @code{/dev/fuse}. The -userspace implementation polls @code{/dev/fuse}, and when a request arrives, -processes it and writes the result back to @code{/dev/fuse}. The kernel then -reads from the device file and returns the result to the user process. - -In case of GlusterFS, the userspace program is the GlusterFS client. -The control flow is shown in the diagram below. The GlusterFS client -services the request by sending it to the server, which in turn -hands it to the local @acronym{POSIX} filesystem. - -@center @image{fuse,44pc,,,.pdf} -@center Fig 1. Control flow in GlusterFS - -@node Translator -@section Translator - -The @emph{translator} is the most important concept in GlusterFS. In -fact, GlusterFS is nothing but a collection of translators working -together, forming a translator @emph{tree}. - -The idea of a translator is perhaps best understood using an -analogy. Consider the @acronym{VFS} in the Linux kernel. The -@acronym{VFS} abstracts the various filesystem implementations (such -as @acronym{EXT3}, ReiserFS, @acronym{XFS}, etc.) supported by the -kernel. When an application calls the kernel to perform an operation -on a file, the kernel passes the request on to the appropriate -filesystem implementation. - -For example, let's say there are two partitions on a Linux machine: -@command{/}, which is an @acronym{EXT3} partition, and @command{/usr}, -which is a ReiserFS partition. Now if an application wants to open a -file called, say, @command{/etc/fstab}, then the kernel will -internally pass the request to the @acronym{EXT3} implementation. If -on the other hand, an application wants to read a file called -@command{/usr/src/linux/CREDITS}, then the kernel will call upon the -ReiserFS implementation to do the job. - -The ``filesystem implementation'' objects are analogous to GlusterFS -translators. A GlusterFS translator implements all the filesystem -operations. Whereas in @acronym{VFS} there is a two-level tree (with -the kernel at the root and all the filesystem implementation as its -children), in GlusterFS there exists a more elaborate tree structure. - -We can now define translators more precisely. A GlusterFS translator -is a shared object (@command{.so}) that implements every filesystem -call. GlusterFS translators can be arranged in an arbitrary tree -structure (subject to constraints imposed by the translators). When -GlusterFS receives a filesystem call, it passes it on to the -translator at the root of the translator tree. The root translator may -in turn pass it on to any or all of its children, and so on, until the -leaf nodes are reached. The result of a filesystem call is -communicated in the reverse fashion, from the leaf nodes up to the -root node, and then on to the application. - -So what might a translator tree look like? - -@tex -\vfill -@end tex -@page - -@center @image{xlator,44pc,,,.pdf} -@center Fig 2. A sample translator tree - -The diagram depicts three servers and one GlusterFS client. It is important -to note that conceptually, the translator tree spans machine boundaries. -Thus, the client machine in the diagram, @command{10.0.0.1}, can access -the aggregated storage of the filesystems on the server machines @command{10.0.0.2}, -@command{10.0.0.3}, and @command{10.0.0.4}. The translator diagram will make more -sense once you've read the next chapter and understood the functions of the -various translators. - -@node Volume specification file -@section Volume specification file -The volume specification file describes the translator tree for both the -server and client programs. - -A volume specification file is a sequence of volume definitions. -The syntax of a volume definition is explained below: - -@cartouche -@example -@strong{volume} @emph{volume-name} - @strong{type} @emph{translator-name} - @strong{option} @emph{option-name} @emph{option-value} - @dots{} - @strong{subvolumes} @emph{subvolume1} @emph{subvolume2} @dots{} -@strong{end-volume} -@end example - -@dots{} -@end cartouche - -@table @asis -@item @emph{volume-name} - An identifier for the volume. This is just a human-readable name, -and can contain any alphanumeric character. For instance, ``storage-1'', ``colon-o'', -or ``forty-two''. - -@item @emph{translator-name} - Name of one of the available translators. Example: @command{protocol/client}, -@command{cluster/unify}. - -@item @emph{option-name} - Name of a valid option for the translator. - -@item @emph{option-value} - Value for the option. Everything following the ``option'' keyword to the end of the -line is considered the value; it is up to the translator to parse it. - -@item @emph{subvolume1}, @emph{subvolume2}, @dots{} - Volume names of sub-volumes. The sub-volumes must already have been defined earlier -in the file. -@end table - -There are a few rules you must follow when writing a volume specification file: - -@itemize -@item Everything following a `@command{#}' is considered a comment and is ignored. Blank lines are also ignored. -@item All names and keywords are case-sensitive. -@item The order of options inside a volume definition does not matter. -@item An option value may not span multiple lines. -@item If an option is not specified, it will assume its default value. -@item A sub-volume must have already been defined before it can be referenced. This means you have to write the specification file ``bottom-up'', starting from the leaf nodes of the translator tree and moving up to the root. -@end itemize - -A simple example volume specification file is shown below: - -@cartouche -@example -# This is a comment line -volume client - type protocol/client - option transport-type tcp - option remote-host localhost # Also a comment - option remote-subvolume brick -# The subvolumes line may be absent -end-volume - -volume iot - type performance/io-threads - option thread-count 4 - subvolumes client -end-volume - -volume wb - type performance/write-behind - subvolumes iot -end-volume -@end example -@end cartouche - -@node Translators -@chapter Translators - -@menu -* Storage Translators:: -* Client and Server Translators:: -* Clustering Translators:: -* Performance Translators:: -* Features Translators:: -* Miscellaneous Translators:: -@end menu - -This chapter documents all the available GlusterFS translators in detail. -Each translator section will show its name (for example, @command{cluster/unify}), -briefly describe its purpose and workings, and list every option accepted by -that translator and their meaning. - -@node Storage Translators -@section Storage Translators - -The storage translators form the ``backend'' for GlusterFS. Currently, -the only available storage translator is the @acronym{POSIX} -translator, which stores files on a normal @acronym{POSIX} -filesystem. A pleasant consequence of this is that your data will -still be accessible if GlusterFS crashes or cannot be started. - -Other storage backends are planned for the future. One of the possibilities is an -Amazon S3 translator. Amazon S3 is an unlimited online storage service accessible -through a web services @acronym{API}. The S3 translator will allow you to access -the storage as a normal @acronym{POSIX} filesystem. -@footnote{Some more discussion about this can be found at: - -http://developer.amazonwebservices.com/connect/message.jspa?messageID=52873} - -@menu -* POSIX:: -* BDB:: -@end menu - -@node POSIX -@subsection POSIX -@example -type storage/posix -@end example - -The @command{posix} translator uses a normal @acronym{POSIX} -filesystem as its ``backend'' to actually store files and -directories. This can be any filesystem that supports extended -attributes (@acronym{EXT3}, ReiserFS, @acronym{XFS}, ...). Extended -attributes are used by some translators to store metadata, for -example, by the replicate and stripe translators. See -@ref{Replicate} and @ref{Stripe}, respectively for details. - -@cartouche -@table @code -@item directory <path> -The directory on the local filesystem which is to be used for storage. -@end table -@end cartouche - -@node BDB -@subsection BDB -@example -type storage/bdb -@end example - -The @command{BDB} translator uses a @acronym{Berkeley DB} database as its -``backend'' to actually store files as key-value pair in the database and -directories as regular @acronym{POSIX} directories. Note that @acronym{BDB} -does not provide extended attribute support for regular files. Do not use -@acronym{BDB} as storage translator while using any translator that demands -extended attributes on ``backend''. - -@cartouche -@table @code -@item directory <path> -The directory on the local filesystem which is to be used for storage. -@item mode [cache|persistent] (cache) -When @acronym{BDB} is run in @command{cache} mode, recovery of back-end is not completely -guaranteed. @command{persistent} guarantees that @acronym{BDB} can recover back-end from -@acronym{Berkeley DB} even if GlusterFS crashes. -@item errfile <path> -The path of the file to be used as @command{errfile} for @acronym{Berkeley DB} to report -detailed error messages, if any. Note that all the contents of this file will be written -by @acronym{Berkeley DB}, not GlusterFS. -@item logdir <path> - - -@end table -@end cartouche - -@node Client and Server Translators, Clustering Translators, Storage Translators, Translators -@section Client and Server Translators - -The client and server translator enable GlusterFS to export a -translator tree over the network or access a remote GlusterFS -server. These two translators implement GlusterFS's network protocol. - -@menu -* Transport modules:: -* Client protocol:: -* Server protocol:: -@end menu - -@node Transport modules -@subsection Transport modules -The client and server translators are capable of using any of the -pluggable transport modules. Currently available transport modules are -@command{tcp}, which uses a @acronym{TCP} connection between client -and server to communicate; @command{ib-sdp}, which uses a -@acronym{TCP} connection over InfiniBand, and @command{ibverbs}, which -uses high-speed InfiniBand connections. - -Each transport module comes in two different versions, one to be used on -the server side and the other on the client side. - -@subsubsection TCP - -The @acronym{TCP} transport module uses a @acronym{TCP/IP} connection between -the server and the client. - -@example - option transport-type tcp -@end example - -The @acronym{TCP} client module accepts the following options: - -@cartouche -@table @code -@item non-blocking-connect [no|off|on|yes] (on) -Whether to make the connection attempt asynchronous. -@item remote-port <n> (24007) -Server port to connect to. -@cindex DNS round robin -@item remote-host <hostname> * -Hostname or @acronym{IP} address of the server. If the host name resolves to -multiple IP addresses, all of them will be tried in a round-robin fashion. This -feature can be used to implement fail-over. -@end table -@end cartouche - -The @acronym{TCP} server module accepts the following options: - -@cartouche -@table @code -@item bind-address <address> (0.0.0.0) -The local interface on which the server should listen to requests. Default is to -listen on all interfaces. -@item listen-port <n> (24007) -The local port to listen on. -@end table -@end cartouche - -@subsubsection IB-SDP -@example - option transport-type ib-sdp -@end example - -kernel implements socket interface for ib hardware. SDP is over ib-verbs. -This module accepts the same options as @command{tcp} - -@subsubsection ibverbs - -@example - option transport-type tcp -@end example - -@cindex infiniband transport - -InfiniBand is a scalable switched fabric interconnect mechanism -primarily used in high-performance computing. InfiniBand can deliver -data throughput of the order of 10 Gbit/s, with latencies of 4-5 ms. - -The @command{ib-verbs} transport accesses the InfiniBand hardware through -the ``verbs'' @acronym{API}, which is the lowest level of software access possible -and which gives the highest performance. On InfiniBand hardware, it is always -best to use @command{ib-verbs}. Use @command{ib-sdp} only if you cannot get -@command{ib-verbs} working for some reason. - -The @command{ib-verbs} client module accepts the following options: - -@cartouche -@table @code -@item non-blocking-connect [no|off|on|yes] (on) -Whether to make the connection attempt asynchronous. -@item remote-port <n> (24007) -Server port to connect to. -@cindex DNS round robin -@item remote-host <hostname> * -Hostname or @acronym{IP} address of the server. If the host name resolves to -multiple IP addresses, all of them will be tried in a round-robin fashion. This -feature can be used to implement fail-over. -@end table -@end cartouche - -The @command{ib-verbs} server module accepts the following options: - -@cartouche -@table @code -@item bind-address <address> (0.0.0.0) -The local interface on which the server should listen to requests. Default is to -listen on all interfaces. -@item listen-port <n> (24007) -The local port to listen on. -@end table -@end cartouche - -The following options are common to both the client and server modules: - -If you are familiar with InfiniBand jargon, -the mode is used by GlusterFS is ``reliable connection-oriented channel transfer''. - -@cartouche -@table @code -@item ib-verbs-work-request-send-count <n> (64) -Length of the send queue in datagrams. [Reason to increase/decrease?] - -@item ib-verbs-work-request-recv-count <n> (64) -Length of the receive queue in datagrams. [Reason to increase/decrease?] - -@item ib-verbs-work-request-send-size <size> (128KB) -Size of each datagram that is sent. [Reason to increase/decrease?] - -@item ib-verbs-work-request-recv-size <size> (128KB) -Size of each datagram that is received. [Reason to increase/decrease?] - -@item ib-verbs-port <n> (1) -Port number for ib-verbs. - -@item ib-verbs-mtu [256|512|1024|2048|4096] (2048) -The Maximum Transmission Unit [Reason to increase/decrease?] - -@item ib-verbs-device-name <device-name> (first device in the list) -InfiniBand device to be used. -@end table -@end cartouche - -For maximum performance, you should ensure that the send/receive counts on both -the client and server are the same. - -ib-verbs is preferred over ib-sdp. - -@node Client protocol -@subsection Client -@example -type procotol/client -@end example - -The client translator enables the GlusterFS client to access a remote server's -translator tree. - -@cartouche -@table @code - -@item transport-type [tcp,ib-sdp,ib-verbs] (tcp) -The transport type to use. You should use the client versions of all the -transport modules (@command{tcp}, @command{ib-sdp}, -@command{ib-verbs}). -@item remote-subvolume <volume_name> * -The name of the volume on the remote host to attach to. Note that -this is @emph{not} the name of the @command{protocol/server} volume on the -server. It should be any volume under the server. -@item transport-timeout <n> (120- seconds) -Inactivity timeout. If a reply is expected and no activity takes place -on the connection within this time, the transport connection will be -broken, and a new connection will be attempted. -@end table -@end cartouche - -@node Server protocol -@subsection Server -@example -type protocol/server -@end example - -The server translator exports a translator tree and makes it accessible to -remote GlusterFS clients. - -@cartouche -@table @code -@item client-volume-filename <path> (<CONFDIR>/glusterfs-client.vol) -The volume specification file to use for the client. This is the file the -client will receive when it is invoked with the @command{--server} option -(@ref{Client}). - -@item transport-type [tcp,ib-verbs,ib-sdp] (tcp) -The transport to use. You should use the server versions of all the transport -modules (@command{tcp}, @command{ib-sdp}, @command{ib-verbs}). - -@item auth.addr.<volume name>.allow <IP address wildcard pattern> -IP addresses of the clients that are allowed to attach to the specified volume. -This can be a wildcard. For example, a wildcard of the form @command{192.168.*.*} -allows any host in the @command{192.168.x.x} subnet to connect to the server. - -@end table -@end cartouche - -@node Clustering Translators -@section Clustering Translators - -The clustering translators are the most important GlusterFS -translators, since it is these that make GlusterFS a cluster -filesystem. These translators together enable GlusterFS to access an -arbitrarily large amount of storage, and provide @acronym{RAID}-like -redundancy and distribution over the entire cluster. - -There are three clustering translators: @strong{unify}, @strong{replicate}, -and @strong{stripe}. The unify translator aggregates storage from -many server nodes. The replicate translator provides file replication. The stripe -translator allows a file to be spread across many server nodes. The following sections -look at each of these translators in detail. - -@menu -* Unify:: -* Replicate:: -* Stripe:: -@end menu - -@node Unify -@subsection Unify -@cindex unify (translator) -@cindex scheduler (unify) -@example -type cluster/unify -@end example - -The unify translator presents a `unified' view of all its sub-volumes. That is, -it makes the union of all its sub-volumes appear as a single volume. It is the -unify translator that gives GlusterFS the ability to access an arbitrarily -large amount of storage. - -For unify to work correctly, certain invariants need to be maintained across -the entire network. These are: - -@cindex unify invariants -@itemize -@item The directory structure of all the sub-volumes must be identical. -@item A particular file can exist on only one of the sub-volumes. Phrasing it in another way, a pathname such as @command{/home/calvin/homework.txt}) is unique across the entire cluster. -@end itemize - -@tex -\vfill -@end tex -@page - -@center @image{unify,44pc,,,.pdf} - -Looking at the second requirement, you might wonder how one can -accomplish storing redundant copies of a file, if no file can exist -multiple times. To answer, we must remember that these invariants are -from @emph{unify's perspective}. A translator such as replicate at a lower -level in the translator tree than unify may subvert this picture. - -The first invariant might seem quite tedious to ensure. We shall see -later that this is not so, since unify's @emph{self-heal} mechanism -takes care of maintaining it. - -The second invariant implies that unify needs some way to decide which file goes where. -Unify makes use of @emph{scheduler} modules for this purpose. - -When a file needs to be created, unify's scheduler decides upon the -sub-volume to be used to store the file. There are many schedulers -available, each using a different algorithm and suitable for different -purposes. - -The various schedulers are described in detail in the sections that follow. - -@subsubsection ALU -@cindex alu (scheduler) - -@example - option scheduler alu -@end example - -ALU stands for "Adaptive Least Usage". It is the most advanced -scheduler available in GlusterFS. It balances the load across volumes -taking several factors in account. It adapts itself to changing I/O -patterns according to its configuration. When properly configured, it -can eliminate the need for regular tuning of the filesystem to keep -volume load nicely balanced. - -The ALU scheduler is composed of multiple least-usage -sub-schedulers. Each sub-scheduler keeps track of a certain type of -load, for each of the sub-volumes, getting statistics from -the sub-volumes themselves. The sub-schedulers are these: - -@itemize -@item disk-usage: The used and free disk space on the volume. - -@item read-usage: The amount of reading done from this volume. - -@item write-usage: The amount of writing done to this volume. - -@item open-files-usage: The number of files currently open from this volume. - -@item disk-speed-usage: The speed at which the disks are spinning. This is a constant value and therefore not very useful. -@end itemize - -The ALU scheduler needs to know which of these sub-schedulers to use, -and in which order to evaluate them. This is done through the -@command{option alu.order} configuration directive. - -Each sub-scheduler needs to know two things: when to kick in (the -entry-threshold), and how long to stay in control (the -exit-threshold). For example: when unifying three disks of 100GB, -keeping an exact balance of disk-usage is not necesary. Instead, there -could be a 1GB margin, which can be used to nicely balance other -factors, such as read-usage. The disk-usage scheduler can be told to -kick in only when a certain threshold of discrepancy is passed, such -as 1GB. When it assumes control under this condition, it will write -all subsequent data to the least-used volume. If it is doing so, it is -unwise to stop right after the values are below the entry-threshold -again, since that would make it very likely that the situation will -occur again very soon. Such a situation would cause the ALU to spend -most of its time disk-usage scheduling, which is unfair to the other -sub-schedulers. The exit-threshold therefore defines the amount of -data that needs to be written to the least-used disk, before control -is relinquished again. - -In addition to the sub-schedulers, the ALU scheduler also has "limits" -options. These can stop the creation of new files on a volume once -values drop below a certain threshold. For example, setting -@command{option alu.limits.min-free-disk 5GB} will stop the scheduling -of files to volumes that have less than 5GB of free disk space, -leaving the files on that disk some room to grow. - -The actual values you assign to the thresholds for sub-schedulers and -limits depend on your situation. If you have fast-growing files, -you'll want to stop file-creation on a disk much earlier than when -hardly any of your files are growing. If you care less about -disk-usage balance than about read-usage balance, you'll want a bigger -disk-usage scheduler entry-threshold and a smaller read-usage -scheduler entry-threshold. - -For thresholds defining a size, values specifying "KB", "MB" and "GB" -are allowed. For example: @command{option alu.limits.min-free-disk 5GB}. - -@cartouche -@table @code -@item alu.order <order> * ("disk-usage:write-usage:read-usage:open-files-usage:disk-speed") -@item alu.disk-usage.entry-threshold <size> (1GB) -@item alu.disk-usage.exit-threshold <size> (512MB) -@item alu.write-usage.entry-threshold <%> (25) -@item alu.write-usage.exit-threshold <%> (5) -@item alu.read-usage.entry-threshold <%> (25) -@item alu.read-usage.exit-threshold <%> (5) -@item alu.open-files-usage.entry-threshold <n> (1000) -@item alu.open-files-usage.exit-threshold <n> (100) -@item alu.limits.min-free-disk <%> -@item alu.limits.max-open-files <n> -@end table -@end cartouche - -@subsubsection Round Robin (RR) -@cindex rr (scheduler) - -@example - option scheduler rr -@end example - -Round-Robin (RR) scheduler creates files in a round-robin -fashion. Each client will have its own round-robin loop. When your -files are mostly similar in size and I/O access pattern, this -scheduler is a good choice. RR scheduler checks for free disk space -on the server before scheduling, so you can know when to add -another server node. The default value of min-free-disk is 5% and is -checked on file creation calls, with atleast 10 seconds (by default) -elapsing between two checks. - -Options: -@cartouche -@table @code -@item rr.limits.min-free-disk <%> (5) -Minimum free disk space a node must have for RR to schedule a file to it. -@item rr.refresh-interval <t> (10 seconds) -Time between two successive free disk space checks. -@end table -@end cartouche - -@subsubsection Random -@cindex random (scheduler) - -@example - option scheduler random -@end example - -The random scheduler schedules file creation randomly among its child nodes. -Like the round-robin scheduler, it also checks for a minimum amount of free disk -space before scheduling a file to a node. - -@cartouche -@table @code -@item random.limits.min-free-disk <%> (5) -Minimum free disk space a node must have for random to schedule a file to it. -@item random.refresh-interval <t> (10 seconds) -Time between two successive free disk space checks. -@end table -@end cartouche - -@subsubsection NUFA -@cindex nufa (scheduler) - -@example - option scheduler nufa -@end example - -It is common in many GlusterFS computing environments for all deployed -machines to act as both servers and clients. For example, a -research lab may have 40 workstations each with its own storage. All -of these workstations might act as servers exporting a volume as well -as clients accessing the entire cluster's storage. In such a -situation, it makes sense to store locally created files on the local -workstation itself (assuming files are accessed most by the -workstation that created them). The Non-Uniform File Allocation (@acronym{NUFA}) -scheduler accomplishes that. - -@acronym{NUFA} gives the local system first priority for file creation -over other nodes. If the local volume does not have more free disk space -than a specified amount (5% by default) then @acronym{NUFA} schedules files -among the other child volumes in a round-robin fashion. - -@acronym{NUFA} is named after the similar strategy used for memory access, -@acronym{NUMA}@footnote{Non-Uniform Memory Access: -@indicateurl{http://en.wikipedia.org/wiki/Non-Uniform_Memory_Access}}. - -@cartouche -@table @code -@item nufa.limits.min-free-disk <%> (5) -Minimum disk space that must be free (local or remote) for @acronym{NUFA} to schedule a -file to it. -@item nufa.refresh-interval <t> (10 seconds) -Time between two successive free disk space checks. -@item nufa.local-volume-name <volume> -The name of the volume corresponding to the local system. This volume must be -one of the children of the unify volume. This option is mandatory. -@end table -@end cartouche - -@cindex namespace -@subsubsection Namespace -Namespace volume needed because: - - persistent inode numbers. - - file exists even when node is down. - -namespace files are simply touched. on every lookup it is checked. - -@cartouche -@table @code -@item namespace <volume> * -Name of the namespace volume (which should be one of the unify volume's children). -@item self-heal [on|off] (on) -Enable/disable self-heal. Unless you know what you are doing, do not disable self-heal. -@end table -@end cartouche - -@cindex self heal (unify) -@subsubsection Self Heal - * When a 'lookup()/stat()' call is made on directory for the first -time, a self-heal call is made, which checks for the consistancy of -its child nodes. If an entry is present in storage node, but not in -namespace, that entry is created in namespace, and vica-versa. There -is an writedir() API introduced which is used for the same. It also -checks for permissions, and uid/gid consistencies. - - * This check is also done when an server goes down and comes up. - - * If one starts with an empty namespace export, but has data in -storage nodes, a 'find .>/dev/null' or 'ls -lR >/dev/null' should help -to build namespace in one shot. Even otherwise, namespace is built on -demand when a file is looked up for the first time. - -NOTE: There are some issues (Kernel 'Oops' msgs) seen with fuse-2.6.3, -when someone deletes namespace in backend, when glusterfs is -running. But with fuse-2.6.5, this issue is not there. - -@node Replicate -@subsection Replicate (formerly AFR) -@cindex Replicate -@example -type cluster/replicate -@end example - -Replicate provides @acronym{RAID}-1 like functionality for -GlusterFS. Replicate replicates files and directories across the -subvolumes. Hence if Replicate has four subvolumes, there will be -four copies of all files and directories. Replicate provides -high-availability, i.e., in case one of the subvolumes go down -(e. g. server crash, network disconnection) Replicate will still -service the requests using the redundant copies. - -Replicate also provides self-heal functionality, i.e., in case the -crashed servers come up, the outdated files and directories will be -updated with the latest versions. Replicate uses extended -attributes of the backend file system to track the versioning of files -and directories and provide the self-heal feature. - -@example -volume replicate-example - type cluster/replicate - subvolumes brick1 brick2 brick3 -end-volume -@end example - -This sample configuration will replicate all directories and files on -brick1, brick2 and brick3. - -All the read operations happen from the first alive child. If all the -three sub-volumes are up, reads will be done from brick1; if brick1 is -down read will be done from brick2. In case read() was being done on -brick1 and it goes down, replicate transparently falls back to -brick2. - -The next release of GlusterFS will add the following features: -@itemize -@item Ability to specify the sub-volume from which read operations are to be done (this will help users who have one of the sub-volumes as a local storage volume). -@item Allow scheduling of read operations amongst the sub-volumes in a round-robin fashion. -@end itemize - -The order of the subvolumes list should be same across all the 'replicate's as -they will be used for locking purposes. - -@cindex self heal (replicate) -@subsubsection Self Heal -Replicate has self-heal feature, which updates the outdated file and -directory copies by the most recent versions. For example consider the -following config: - -@example -volume replicate-example - type cluster/replicate - subvolumes brick1 brick2 -end-volume -@end example - -@subsubsection File self-heal - -Now if we create a file foo.txt on replicate-example, the file will be created -on brick1 and brick2. The file will have two extended attributes associated -with it in the backend filesystem. One is trusted.afr.createtime and the -other is trusted.afr.version. The trusted.afr.createtime xattr has the -create time (in terms of seconds since epoch) and trusted.afr.version -is a number that is incremented each time a file is modified. This increment -happens during close (incase any write was done before close). - -If brick1 goes down, we edit foo.txt the version gets incremented. Now -the brick1 comes back up, when we open() on foo.txt replicate will check if -their versions are same. If they are not same, the outdated copy is -replaced by the latest copy and its version is updated. After the sync -the open() proceeds in the usual manner and the application calling open() -can continue on its access to the file. - -If brick1 goes down, we delete foo.txt and create a file with the same -name again i.e foo.txt. Now brick1 comes back up, clearly there is a -chance that the version on brick1 being more than the version on brick2, -this is where createtime extended attribute helps in deciding which -the outdated copy is. Hence we need to consider both createtime and -version to decide on the latest copy. - -The version attribute is incremented during the close() call. Version -will not be incremented in case there was no write() done. In case the -fd that the close() gets was got by create() call, we also create -the createtime extended attribute. - -@subsubsection Directory self-heal - -Suppose brick1 goes down, we delete foo.txt, brick1 comes back up, now -we should not create foo.txt on brick2 but we should delete foo.txt -on brick1. We handle this situation by having the createtime and version -attribute on the directory similar to the file. when lookup() is done -on the directory, we compare the createtime/version attributes of the -copies and see which files needs to be deleted and delete those files -and update the extended attributes of the outdated directory copy. -Each time a directory is modified (a file or a subdirectory is created -or deleted inside the directory) and one of the subvols is down, we -increment the directory's version. - -lookup() is a call initiated by the kernel on a file or directory -just before any access to that file or directory. In glusterfs, by -default, lookup() will not be called in case it was called in the -past one second on that particular file or directory. - -The extended attributes can be seen in the backend filesystem using -the @command{getfattr} command. (@command{getfattr -n trusted.afr.version <file>}) - -@cartouche -@table @code -@item debug [on|off] (off) -@item self-heal [on|off] (on) -@item replicate <pattern> (*:1) -@item lock-node <child_volume> (first child is used by default) -@end table -@end cartouche - -@node Stripe -@subsection Stripe -@cindex stripe (translator) -@example -type cluster/stripe -@end example - -The stripe translator distributes the contents of a file over its -sub-volumes. It does this by creating a file equal in size to the -total size of the file on each of its sub-volumes. It then writes only -a part of the file to each sub-volume, leaving the rest of it empty. -These empty regions are called `holes' in Unix terminology. The holes -do not consume any disk space. - -The diagram below makes this clear. - -@center @image{stripe,44pc,,,.pdf} - -You can configure stripe so that only filenames matching a pattern -are striped. You can also configure the size of the data to be stored -on each sub-volume. - -@cartouche -@table @code -@item block-size <pattern>:<size> (*:0 no striping) -Distribute files matching @command{<pattern>} over the sub-volumes, -storing at least @command{<size>} on each sub-volume. For example, - -@example - option block-size *.mpg:1M -@end example - -distributes all files ending in @command{.mpg}, storing at least 1 MB on -each sub-volume. - -Any number of @command{block-size} option lines may be present, specifying -different sizes for different file name patterns. -@end table -@end cartouche - -@node Performance Translators -@section Performance Translators - -@menu -* Read Ahead:: -* Write Behind:: -* IO Threads:: -* IO Cache:: -* Booster:: -@end menu - -@node Read Ahead -@subsection Read Ahead -@cindex read-ahead (translator) -@example -type performance/read-ahead -@end example - -The read-ahead translator pre-fetches data in advance on every read. -This benefits applications that mostly process files in sequential order, -since the next block of data will already be available by the time the -application is done with the current one. - -Additionally, the read-ahead translator also behaves as a read-aggregator. -Many small read operations are combined and issued as fewer, larger read -requests to the server. - -Read-ahead deals in ``pages'' as the unit of data fetched. The page size -is configurable, as is the ``page count'', which is the number of pages -that are pre-fetched. - -Read-ahead is best used with InfiniBand (using the ib-verbs transport). -On FastEthernet and Gigabit Ethernet networks, -GlusterFS can achieve the link-maximum throughput even without -read-ahead, making it quite superflous. - -Note that read-ahead only happens if the reads are perfectly -sequential. If your application accesses data in a random fashion, -using read-ahead might actually lead to a performance loss, since -read-ahead will pointlessly fetch pages which won't be used by the -application. - -@cartouche -Options: -@table @code -@item page-size <n> (256KB) -The unit of data that is pre-fetched. -@item page-count <n> (2) -The number of pages that are pre-fetched. -@item force-atime-update [on|off|yes|no] (off|no) -Whether to force an access time (atime) update on the file on every read. Without -this, the atime will be slightly imprecise, as it will reflect the time when -the read-ahead translator read the data, not when the application actually read it. -@end table -@end cartouche - -@node Write Behind -@subsection Write Behind -@cindex write-behind (translator) -@example -type performance/write-behind -@end example - -The write-behind translator improves the latency of a write operation. -It does this by relegating the write operation to the background and -returning to the application even as the write is in progress. Using the -write-behind translator, successive write requests can be pipelined. -This mode of write-behind operation is best used on the client side, to -enable decreased write latency for the application. - -The write-behind translator can also aggregate write requests. If the -@command{aggregate-size} option is specified, then successive writes upto that -size are accumulated and written in a single operation. This mode of operation -is best used on the server side, as this will decrease the disk's head movement -when multiple files are being written to in parallel. - -The @command{aggregate-size} option has a default value of 128KB. Although -this works well for most users, you should always experiment with different values -to determine the one that will deliver maximum performance. This is because the -performance of write-behind depends on your interconnect, size of RAM, and the -work load. - -@cartouche -@table @code -@item aggregate-size <n> (128KB) -Amount of data to accumulate before doing a write -@item flush-behind [on|yes|off|no] (off|no) - -@end table -@end cartouche - -@node IO Threads -@subsection IO Threads -@cindex io-threads (translator) -@example -type performance/io-threads -@end example - -The IO threads translator is intended to increase the responsiveness -of the server to metadata operations by doing file I/O (read, write) -in a background thread. Since the GlusterFS server is -single-threaded, using the IO threads translator can significantly -improve performance. This translator is best used on the server side, -loaded just below the server protocol translator. - -IO threads operates by handing out read and write requests to a separate thread. -The total number of threads in existence at a time is constant, and configurable. - -@cartouche -@table @code -@item thread-count <n> (1) -Number of threads to use. -@end table -@end cartouche - -@node IO Cache -@subsection IO Cache -@cindex io-cache (translator) -@example -type performance/io-cache -@end example - -The IO cache translator caches data that has been read. This is useful -if many applications read the same data multiple times, and if reads -are much more frequent than writes (for example, IO caching may be -useful in a web hosting environment, where most clients will simply -read some files and only a few will write to them). - -The IO cache translator reads data from its child in @command{page-size} chunks. -It caches data upto @command{cache-size} bytes. The cache is maintained as -a prioritized least-recently-used (@acronym{LRU}) list, with priorities determined -by user-specified patterns to match filenames. - -When the IO cache translator detects a write operation, the -cache for that file is flushed. - -The IO cache translator periodically verifies the consistency of -cached data, using the modification times on the files. The verification timeout -is configurable. - -@cartouche -@table @code -@item page-size <n> (128KB) -Size of a page. -@item cache-size (n) (32MB) -Total amount of data to be cached. -@item force-revalidate-timeout <n> (1) -Timeout to force a cache consistency verification, in seconds. -@item priority <pattern> (*:0) -Filename patterns listed in order of priority. -@end table -@end cartouche - -@node Booster -@subsection Booster -@cindex booster -@example - type performance/booster -@end example - -The booster translator gives applications a faster path to communicate -read and write requests to GlusterFS. Normally, all requests to GlusterFS from -applications go through FUSE, as indicated in @ref{Filesystems in Userspace}. -Using the booster translator in conjunction with the GlusterFS booster shared -library, an application can bypass the FUSE path and send read/write requests -directly to the GlusterFS client process. - -The booster mechanism consists of two parts: the booster translator, -and the booster shared library. The booster translator is meant to be -loaded on the client side, usually at the root of the translator tree. -The booster shared library should be @command{LD_PRELOAD}ed with the -application. - -The booster translator when loaded opens a Unix domain socket and -listens for read/write requests on it. The booster shared library -intercepts read and write system calls and sends the requests to the -GlusterFS process directly using the Unix domain socket, bypassing FUSE. -This leads to superior performance. - -Once you've loaded the booster translator in your volume specification file, you -can start your application as: - -@example - $ LD_PRELOAD=/usr/local/bin/glusterfs-booster.so your_app -@end example - -The booster translator accepts no options. - -@node Features Translators -@section Features Translators - -@menu -* POSIX Locks:: -* Fixed ID:: -@end menu - -@node POSIX Locks -@subsection POSIX Locks -@cindex record locking -@cindex fcntl -@cindex posix-locks (translator) -@example -type features/posix-locks -@end example - -This translator provides storage independent POSIX record locking -support (@command{fcntl} locking). Typically you'll want to load this on the -server side, just above the @acronym{POSIX} storage translator. Using this -translator you can get both advisory locking and mandatory locking -support. It also handles @command{flock()} locks properly. - -Caveat: Consider a file that does not have its mandatory locking bits -(+setgid, -group execution) turned on. Assume that this file is now -opened by a process on a client that has the write-behind xlator -loaded. The write-behind xlator does not cache anything for files -which have mandatory locking enabled, to avoid incoherence. Let's say -that mandatory locking is now enabled on this file through another -client. The former client will not know about this change, and -write-behind may erroneously report a write as being successful when -in fact it would fail due to the region it is writing to being locked. - -There seems to be no easy way to fix this. To work around this -problem, it is recommended that you never enable the mandatory bits on -a file while it is open. - -@cartouche -@table @code -@item mandatory [on|off] (on) -Turns mandatory locking on. -@end table -@end cartouche - -@node Fixed ID -@subsection Fixed ID -@cindex fixed-id (translator) -@example -type features/fixed-id -@end example - -The fixed ID translator makes all filesystem requests from the client -to appear to be coming from a fixed, specified -@acronym{UID}/@acronym{GID}, regardless of which user actually -initiated the request. - -@cartouche -@table @code -@item fixed-uid <n> [if not set, not used] -The @acronym{UID} to send to the server -@item fixed-gid <n> [if not set, not used] -The @acronym{GID} to send to the server -@end table -@end cartouche - -@node Miscellaneous Translators -@section Miscellaneous Translators - -@menu -* ROT-13:: -* Trace:: -@end menu - -@node ROT-13 -@subsection ROT-13 -@cindex rot-13 (translator) -@example -type encryption/rot-13 -@end example - -@acronym{ROT-13} is a toy translator that can ``encrypt'' and ``decrypt'' file -contents using the @acronym{ROT-13} algorithm. @acronym{ROT-13} is a trivial -algorithm that rotates each alphabet by thirteen places. Thus, 'A' becomes 'N', -'B' becomes 'O', and 'Z' becomes 'M'. - -It goes without saying that you shouldn't use this translator if you need -@emph{real} encryption (a future release of GlusterFS will have real encryption -translators). - -@cartouche -@table @code -@item encrypt-write [on|off] (on) -Whether to encrypt on write -@item decrypt-read [on|off] (on) -Whether to decrypt on read -@end table -@end cartouche - -@node Trace -@subsection Trace -@cindex trace (translator) -@example -type debug/trace -@end example - -The trace translator is intended for debugging purposes. When loaded, it -logs all the system calls received by the server or client (wherever -trace is loaded), their arguments, and the results. You must use a GlusterFS log -level of DEBUG (See @ref{Running GlusterFS}) for trace to work. - -Sample trace output (lines have been wrapped for readability): -@cartouche -@example -2007-10-30 00:08:58 D [trace.c:1579:trace_opendir] trace: callid: 68 -(*this=0x8059e40, loc=0x8091984 @{path=/iozone3_283, inode=0x8091f00@}, - fd=0x8091d50) - -2007-10-30 00:08:58 D [trace.c:630:trace_opendir_cbk] trace: -(*this=0x8059e40, op_ret=4, op_errno=1, fd=0x8091d50) - -2007-10-30 00:08:58 D [trace.c:1602:trace_readdir] trace: callid: 69 -(*this=0x8059e40, size=4096, offset=0 fd=0x8091d50) - -2007-10-30 00:08:58 D [trace.c:215:trace_readdir_cbk] trace: -(*this=0x8059e40, op_ret=0, op_errno=0, count=4) - -2007-10-30 00:08:58 D [trace.c:1624:trace_closedir] trace: callid: 71 -(*this=0x8059e40, *fd=0x8091d50) - -2007-10-30 00:08:58 D [trace.c:809:trace_closedir_cbk] trace: -(*this=0x8059e40, op_ret=0, op_errno=1) -@end example -@end cartouche - -@node Usage Scenarios -@chapter Usage Scenarios - -@section Advanced Striping - -This section is based on the Advanced Striping tutorial written by -Anand Avati on the GlusterFS wiki -@footnote{http://gluster.org/docs/index.php/Mixing_Striped_and_Regular_Files}. - -@subsection Mixed Storage Requirements - -There are two ways of scheduling the I/O. One at file level (using -unify translator) and other at block level (using stripe -translator). Striped I/O is good for files that are potentially large -and require high parallel throughput (for example, a single file of -400GB being accessed by 100s and 1000s of systems simultaneously and -randomly). For most of the cases, file level scheduling works best. - -In the real world, it is desirable to mix file level and block level -scheduling on a single storage volume. Alternatively users can choose -to have two separate volumes and hence two mount points, but the -applications may demand a single storage system to host both. - -This document explains how to mix file level scheduling with stripe. - -@subsection Configuration Brief - -This setup demonstrates how users can configure unify translator with -appropriate I/O scheduler for file level scheduling and strip for only -matching patterns. This way, GlusterFS chooses appropriate I/O profile -and knows how to efficiently handle both the types of data. - -A simple technique to achieve this effect is to create a stripe set of -unify and stripe blocks, where unify is the first sub-volume. Files -that do not match the stripe policy passed on to first unify -sub-volume and inturn scheduled arcoss the cluster using its file -level I/O scheduler. - -@image{advanced-stripe,44pc,,,.pdf} - -@subsection Preparing GlusterFS Envoronment - -Create the directories /export/namespace, /export/unify and -/export/stripe on all the storage bricks. - - Place the following server and client volume spec file under -/etc/glusterfs (or appropriate installed path) and replace the IP -addresses / access control fields to match your environment. - -@cartouche -@example - ## file: /etc/glusterfs/glusterfsd.vol - volume posix-unify - type storage/posix - option directory /export/for-unify - end-volume - - volume posix-stripe - type storage/posix - option directory /export/for-stripe - end-volume - - volume posix-namespace - type storage/posix - option directory /export/for-namespace - end-volume - - volume server - type protocol/server - option transport-type tcp - option auth.addr.posix-unify.allow 192.168.1.* - option auth.addr.posix-stripe.allow 192.168.1.* - option auth.addr.posix-namespace.allow 192.168.1.* - subvolumes posix-unify posix-stripe posix-namespace - end-volume -@end example -@end cartouche - -@cartouche -@example - ## file: /etc/glusterfs/glusterfs.vol - volume client-namespace - type protocol/client - option transport-type tcp - option remote-host 192.168.1.1 - option remote-subvolume posix-namespace - end-volume - - volume client-unify-1 - type protocol/client - option transport-type tcp - option remote-host 192.168.1.1 - option remote-subvolume posix-unify - end-volume - - volume client-unify-2 - type protocol/client - option transport-type tcp - option remote-host 192.168.1.2 - option remote-subvolume posix-unify - end-volume - - volume client-unify-3 - type protocol/client - option transport-type tcp - option remote-host 192.168.1.3 - option remote-subvolume posix-unify - end-volume - - volume client-unify-4 - type protocol/client - option transport-type tcp - option remote-host 192.168.1.4 - option remote-subvolume posix-unify - end-volume - - volume client-stripe-1 - type protocol/client - option transport-type tcp - option remote-host 192.168.1.1 - option remote-subvolume posix-stripe - end-volume - - volume client-stripe-2 - type protocol/client - option transport-type tcp - option remote-host 192.168.1.2 - option remote-subvolume posix-stripe - end-volume - - volume client-stripe-3 - type protocol/client - option transport-type tcp - option remote-host 192.168.1.3 - option remote-subvolume posix-stripe - end-volume - - volume client-stripe-4 - type protocol/client - option transport-type tcp - option remote-host 192.168.1.4 - option remote-subvolume posix-stripe - end-volume - - volume unify - type cluster/unify - option scheduler rr - subvolumes cluster-unify-1 cluster-unify-2 cluster-unify-3 cluster-unify-4 - end-volume - - volume stripe - type cluster/stripe - option block-size *.img:2MB # All files ending with .img are striped with 2MB stripe block size. - subvolumes unify cluster-stripe-1 cluster-stripe-2 cluster-stripe-3 cluster-stripe-4 - end-volume -@end example -@end cartouche - - -Bring up the Storage - -Starting GlusterFS Server: If you have installed through binary -package, you can start the service through init.d startup script. If -not: - -@example -[root@@server]# glusterfsd -@end example - -Mounting GlusterFS Volumes: - -@example -[root@@client]# glusterfs -s [BRICK-IP-ADDRESS] /mnt/cluster -@end example - -Improving upon this Setup - -Infiniband Verbs RDMA transport is much faster than TCP/IP GigE -transport. - -Use of performance translators such as read-ahead, write-behind, -io-cache, io-threads, booster is recommended. - -Replace round-robin (rr) scheduler with ALU to handle more dynamic -storage environments. - -@node Troubleshooting -@chapter Troubleshooting - -This chapter is a general troubleshooting guide to GlusterFS. It lists -common GlusterFS server and client error messages, debugging hints, and -concludes with the suggested procedure to report bugs in GlusterFS. - -@section GlusterFS error messages - -@subsection Server errors - -@example -glusterfsd: FATAL: could not open specfile: -'/etc/glusterfs/glusterfsd.vol' -@end example - -The GlusterFS server expects the volume specification file to be -at @command{/etc/glusterfs/glusterfsd.vol}. The example -specification file will be installed as -@command{/etc/glusterfs/glusterfsd.vol.sample}. You need to edit -it and rename it, or provide a different specification file using -the @command{--spec-file} command line option (See @ref{Server}). - -@vskip 4ex - -@example -gf_log_init: failed to open logfile "/usr/var/log/glusterfs/glusterfsd.log" - (Permission denied) -@end example - -You don't have permission to create files in the -@command{/usr/var/log/glusterfs} directory. Make sure you are running -GlusterFS as root. Alternatively, specify a different path for the log -file using the @command{--log-file} option (See @ref{Server}). - -@subsection Client errors - -@example -fusermount: failed to access mountpoint /mnt: - Transport endpoint is not connected -@end example - -A previous failed (or hung) mount of GlusterFS is preventing it from being -mounted again in the same location. The fix is to do: - -@example -# umount /mnt -@end example - -and try mounting again. - -@vskip 4ex - -@strong{``Transport endpoint is not connected''.} - -If you get this error when you try a command such as @command{ls} or @command{cat}, -it means the GlusterFS mount did not succeed. Try running GlusterFS in @command{DEBUG} -logging level and study the log messages to discover the cause. - -@vskip 4ex - -@strong{``Connect to server failed'', ``SERVER-ADDRESS: Connection refused''.} - -GluserFS Server is not running or dead. Check your network -connections and firewall settings. To check if the server is reachable, -try: - -@example -telnet IP-ADDRESS 24007 -@end example - -If the server is accessible, your `telnet' command should connect and -block. If not you will see an error message such as @command{telnet: Unable to -connect to remote host: Connection refused}. 24007 is the default -GlusterFS port. If you have changed it, then use the corresponding -port instead. - -@vskip 4ex - -@example -gf_log_init: failed to open logfile "/usr/var/log/glusterfs/glusterfs.log" - (Permission denied) -@end example - -You don't have permission to create files in the -@command{/usr/var/log/glusterfs} directory. Make sure you are running -GlusterFS as root. Alternatively, specify a different path for the log -file using the @command{--log-file} option (See @ref{Client}). - -@section FUSE error messages -@command{modprobe fuse} fails with: ``Unknown symbol in module, or unknown parameter''. -@cindex Redhat Enterprise Linux - -If you are using fuse-2.6.x on Redhat Enterprise Linux Work Station 4 -and Advanced Server 4 with 2.6.9-42.ELlargesmp, 2.6.9-42.ELsmp, -2.6.9-42.EL kernels and get this error while loading @acronym{FUSE} kernel -module, you need to apply the following patch. - -For fuse-2.6.2: - -@indicateurl{http://ftp.gluster.com/pub/gluster/glusterfs/fuse/fuse-2.6.2-rhel-build.patch} - -For fuse-2.6.3: - -@indicateurl{http://ftp.gluster.com/pub/gluster/glusterfs/fuse/fuse-2.6.3-rhel-build.patch} - -@section AppArmour and GlusterFS -@cindex AppArmour -@cindex OpenSuSE -Under OpenSuSE GNU/Linux, the AppArmour security feature does not -allow GlusterFS to create temporary files or network socket -connections even while running as root. You will see error messages -like `Unable to open log file: Operation not permitted' or `Connection -refused'. Disabling AppArmour using YaST or properly configuring -AppArmour to recognize @command{glusterfsd} or @command{glusterfs}/@command{fusermount} -should solve the problem. - -@section Reporting a bug - -If you encounter a bug in GlusterFS, please follow the below -guidelines when you report it to the mailing list. Be sure to report -it! User feedback is crucial to the health of the project and we value -it highly. - -@subsection General instructions - -When running GlusterFS in a non-production environment, be sure to -build it with the following command: - -@example - $ make CFLAGS='-g -O0 -DDEBUG' -@end example - -This includes debugging information which will be helpful in getting -backtraces (see below) and also disable optimization. Enabling -optimization can result in incorrect line numbers being reported to -gdb. - -@subsection Volume specification files - -Attach all relevant server and client spec files you were using when -you encountered the bug. Also tell us details of your setup, i.e., how -many clients and how many servers. - -@subsection Log files - -Set the loglevel of your client and server programs to @acronym{DEBUG} (by -passing the -L @acronym{DEBUG} option) and attach the log files with your bug -report. Obviously, if only the client is failing (for example), you -only need to send us the client log file. - -@subsection Backtrace - -If GlusterFS has encountered a segmentation fault or has crashed for -some other reason, include the backtrace with the bug report. You can -get the backtrace using the following procedure. - -Run the GlusterFS client or server inside gdb. - -@example - $ gdb ./glusterfs - (gdb) set args -f client.spec -N -l/path/to/log/file -LDEBUG /mnt/point - (gdb) run -@end example - -Now when the process segfaults, you can get the backtrace by typing: - -@example - (gdb) bt -@end example - -If the GlusterFS process has crashed and dumped a core file (you can -find this in / if running as a daemon and in the current directory -otherwise), you can do: - -@example - $ gdb /path/to/glusterfs /path/to/core.<pid> -@end example - -and then get the backtrace. - -If the GlusterFS server or client seems to be hung, then you can get -the backtrace by attaching gdb to the process. First get the @command{PID} of -the process (using ps), and then do: - -@example - $ gdb ./glusterfs <pid> -@end example - -Press Ctrl-C to interrupt the process and then generate the backtrace. - -@subsection Reproducing the bug - -If the bug is reproducible, please include the steps necessary to do -so. If the bug is not reproducible, send us the bug report anyway. - -@subsection Other information - -If you think it is relevant, send us also the version of @acronym{FUSE} you're -using, the kernel version, platform. - -@node GNU Free Documentation Licence -@appendix GNU Free Documentation Licence -@include fdl.texi - -@node Index -@unnumbered Index -@printindex cp - -@bye diff --git a/doc/user-guide/legacy/xlator.odg b/doc/user-guide/legacy/xlator.odg Binary files differdeleted file mode 100644 index 179a65f6e26..00000000000 --- a/doc/user-guide/legacy/xlator.odg +++ /dev/null diff --git a/doc/user-guide/legacy/xlator.pdf b/doc/user-guide/legacy/xlator.pdf Binary files differdeleted file mode 100644 index a07e14d67d2..00000000000 --- a/doc/user-guide/legacy/xlator.pdf +++ /dev/null |
