mgmt/glusterd: Implementation of server-side quorum

Feature-page:
http://www.gluster.org/community/documentation/index.php/Features/Server-quorum

Change-Id: I747b222519e71022462343d2c1bcd3626e1f9c86
BUG: 839595
Signed-off-by: Pranith Kumar K <pranithk@gluster.com>
Reviewed-on: http://review.gluster.org/3811
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Vijay Bellur <vbellur@redhat.com>

1 files changed, 437 insertions, 55 deletions

diff --git a/xlators/mgmt/glusterd/src/glusterd-op-sm.c b/xlators/mgmt/glusterd/src/glusterd-op-sm.c
index 40f38f5bb..d5686bbab 100644
--- a/xlators/mgmt/glusterd/src/glusterd-op-sm.c
+++ b/xlators/mgmt/glusterd/src/glusterd-op-sm.c
@@ -46,6 +46,24 @@
 #include <signal.h>
 #include <sys/wait.h>
 
+#define ALL_VOLUME_OPTION_CHECK(volname, key, ret, op_errstr, label)           \
+        do {                                                                   \
+                gf_boolean_t    _all = !strcmp ("all", volname);               \
+                gf_boolean_t    _ratio = !strcmp (key,                         \
+                                                  GLUSTERD_QUORUM_RATIO_KEY);  \
+                if (_all && !_ratio) {                                         \
+                        ret = -1;                                              \
+                        *op_errstr = gf_strdup ("Not a valid option for all "  \
+                                                "volumes");                    \
+                        goto label;                                            \
+                } else if (!_all && _ratio) {                                  \
+                        ret = -1;                                              \
+                        *op_errstr = gf_strdup ("Not a valid option for "      \
+                                                "single volume");              \
+                        goto label;                                            \
+                }                                                              \
+         } while (0)
+
 static struct list_head gd_op_sm_queue;
 pthread_mutex_t       gd_op_sm_lock;
 glusterd_op_info_t    opinfo = {{0},};
@@ -291,6 +309,24 @@ out:
 }
 
 static int
+glusterd_validate_quorum_options (xlator_t *this, char *fullkey, char *value,
+                                  char **op_errstr)
+{
+        int             ret = 0;
+        char            *key = NULL;
+        volume_option_t *opt = NULL;
+
+        if (!glusterd_is_quorum_option (fullkey))
+                goto out;
+        key = strchr (fullkey, '.');
+        key++;
+        opt = xlator_volume_option_get (this, key);
+        ret = xlator_option_validate (this, key, value, opt, op_errstr);
+out:
+        return ret;
+}
+
+static int
 glusterd_op_stage_set_volume (dict_t *dict, char **op_errstr)
 {
         int                             ret                     = -1;
@@ -315,6 +351,7 @@ glusterd_op_stage_set_volume (dict_t *dict, char **op_errstr)
         uint32_t                        local_key_op_version    = 0;
         gf_boolean_t                    origin_glusterd         = _gf_true;
         gf_boolean_t                    check_op_version        = _gf_true;
+        gf_boolean_t                    all_vol        = _gf_false;
 
         GF_ASSERT (dict);
         this = THIS;
@@ -399,26 +436,30 @@ glusterd_op_stage_set_volume (dict_t *dict, char **op_errstr)
                 goto out;
         }
 
-        exists = glusterd_check_volume_exists (volname);
-        if (!exists) {
-                snprintf (errstr, sizeof (errstr), "Volume %s does not exist",
-                          volname);
-                gf_log (this->name, GF_LOG_ERROR, "%s", errstr);
-                *op_errstr = gf_strdup (errstr);
-                ret = -1;
-                goto out;
-        }
+        if (strcasecmp (volname, "all") != 0) {
+                exists = glusterd_check_volume_exists (volname);
+                if (!exists) {
+                        snprintf (errstr, sizeof (errstr), "Volume %s does "
+                                  "not exist", volname);
+                        gf_log (this->name, GF_LOG_ERROR, "%s", errstr);
+                        *op_errstr = gf_strdup (errstr);
+                        ret = -1;
+                        goto out;
+                }
 
-        ret = glusterd_volinfo_find (volname, &volinfo);
-        if (ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Unable to allocate memory");
-                goto out;
-        }
+                ret = glusterd_volinfo_find (volname, &volinfo);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Unable to allocate memory");
+                        goto out;
+                }
 
-        ret = glusterd_validate_volume_id (dict, volinfo);
-        if (ret)
-                goto out;
+                ret = glusterd_validate_volume_id (dict, volinfo);
+                if (ret)
+                        goto out;
+        } else {
+                all_vol = _gf_true;
+        }
 
         local_new_op_version = priv->op_version;
 
@@ -473,6 +514,7 @@ glusterd_op_stage_set_volume (dict_t *dict, char **op_errstr)
                         ret = -1;
                         goto out;
                 }
+
                 if (!exists) {
                         gf_log (this->name, GF_LOG_ERROR,
                                 "Option with name: %s "
@@ -490,6 +532,11 @@ glusterd_op_stage_set_volume (dict_t *dict, char **op_errstr)
 
                 if (key_fixed)
                         key = key_fixed;
+                ALL_VOLUME_OPTION_CHECK (volname, key, ret, op_errstr, out);
+                ret = glusterd_validate_quorum_options (this, key, value,
+                                                        op_errstr);
+                if (ret)
+                        goto out;
 
                 local_key_op_version = glusterd_get_op_version_for_key (key);
                 if (local_key_op_version > local_new_op_version)
@@ -539,9 +586,9 @@ glusterd_op_stage_set_volume (dict_t *dict, char **op_errstr)
                 }
 
                 *op_errstr = NULL;
-                if (!global_opt)
+                if (!global_opt && !all_vol)
                         ret = glusterd_validate_reconfopts (volinfo, val_dict, op_errstr);
-                else {
+                else if (!all_vol) {
                         voliter = NULL;
                         list_for_each_entry (voliter, &priv->volumes, vol_list) {
                                 ret = glusterd_validate_globalopts (voliter, val_dict, op_errstr);
@@ -626,23 +673,24 @@ glusterd_op_stage_reset_volume (dict_t *dict, char **op_errstr)
                 goto out;
         }
 
-        exists = glusterd_check_volume_exists (volname);
+        if (strcasecmp (volname, "all") != 0) {
+                exists = glusterd_check_volume_exists (volname);
+                if (!exists) {
+                        snprintf (msg, sizeof (msg), "Volume %s does not "
+                                  "exist", volname);
+                        gf_log ("", GF_LOG_ERROR, "%s", msg);
+                        *op_errstr = gf_strdup (msg);
+                        ret = -1;
+                        goto out;
+                }
+                ret = glusterd_volinfo_find (volname, &volinfo);
+                if (ret)
+                        goto out;
 
-        if (!exists) {
-                snprintf (msg, sizeof (msg), "Volume %s does not "
-                          "exist", volname);
-                gf_log ("", GF_LOG_ERROR, "%s", msg);
-                *op_errstr = gf_strdup (msg);
-                ret = -1;
-                goto out;
+                ret = glusterd_validate_volume_id (dict, volinfo);
+                if (ret)
+                        goto out;
         }
-        ret = glusterd_volinfo_find (volname, &volinfo);
-        if (ret)
-                goto out;
-
-        ret = glusterd_validate_volume_id (dict, volinfo);
-        if (ret)
-                goto out;
 
         ret = dict_get_str (dict, "key", &key);
         if (ret) {
@@ -666,6 +714,11 @@ glusterd_op_stage_reset_volume (dict_t *dict, char **op_errstr)
                         *op_errstr = gf_strdup (msg);
                         ret = -1;
                         goto out;
+                } else if (exists > 0) {
+                        if (key_fixed)
+                                key = key_fixed;
+                        ALL_VOLUME_OPTION_CHECK (volname, key, ret,
+                                                 op_errstr, out);
                 }
         }
 
@@ -993,6 +1046,22 @@ out:
 }
 
 static int
+_delete_reconfig_global_opt (dict_t *this, char *key, data_t *value, void *data)
+{
+        int32_t        *is_force = 0;
+
+        GF_ASSERT (data);
+        is_force = (int32_t*)data;
+
+        if (strcmp (GLUSTERD_GLOBAL_OPT_VERSION, key) == 0)
+                goto out;
+
+        _delete_reconfig_opt (this, key, value, data);
+out:
+        return 0;
+}
+
+static int
 glusterd_options_reset (glusterd_volinfo_t *volinfo, char *key,
                         int32_t *is_force)
 {
@@ -1008,15 +1077,6 @@ glusterd_options_reset (glusterd_volinfo_t *volinfo, char *key,
         if (!strncmp(key, "all", 3))
                 dict_foreach (volinfo->dict, _delete_reconfig_opt, is_force);
         else {
-                if (glusterd_check_option_exists (key, &key_fixed) != 1) {
-                        gf_log ("glusterd", GF_LOG_ERROR,
-                                "volinfo dict inconsistency: option %s not found",
-                                key);
-                        ret = -1;
-                        goto out;
-                }
-                if (key_fixed)
-                        key = key_fixed;
                 value = dict_get (volinfo->dict, key);
                 if (!value) {
                         gf_log ("glusterd", GF_LOG_DEBUG,
@@ -1053,6 +1113,91 @@ out:
         return ret;
 }
 
+static int
+glusterd_op_reset_all_volume_options (xlator_t *this, dict_t *dict)
+{
+        char            *key            = NULL;
+        char            *key_fixed      = NULL;
+        int             ret             = -1;
+        int32_t         is_force        = 0;
+        glusterd_conf_t *conf           = NULL;
+        dict_t          *dup_opt        = NULL;
+        gf_boolean_t    all             = _gf_false;
+        char            *next_version   = NULL;
+        gf_boolean_t    quorum_action   = _gf_false;
+
+        conf = this->private;
+        ret = dict_get_str (dict, "key", &key);
+        if (ret)
+                goto out;
+
+        ret = dict_get_int32 (dict, "force", &is_force);
+        if (ret)
+                is_force = 0;
+
+        if (strcmp (key, "all")) {
+                ret = glusterd_check_option_exists (key, &key_fixed);
+                if (ret <= 0) {
+                        gf_log (this->name, GF_LOG_ERROR, "Invalid key %s",
+                                key);
+                        ret = -1;
+                        goto out;
+                }
+        } else {
+                all = _gf_true;
+        }
+
+        if (key_fixed)
+                key = key_fixed;
+
+        ret = -1;
+        dup_opt = dict_new ();
+        if (!dup_opt)
+                goto out;
+        if (!all) {
+                dict_copy (conf->opts, dup_opt);
+                dict_del (dup_opt, key);
+        }
+        ret = glusterd_get_next_global_opt_version_str (conf->opts,
+                                                        &next_version);
+        if (ret)
+                goto out;
+
+        ret = dict_set_str (dup_opt, GLUSTERD_GLOBAL_OPT_VERSION, next_version);
+        if (ret)
+                goto out;
+
+        ret = glusterd_store_options (this, dup_opt);
+        if (ret)
+                goto out;
+
+        if (glusterd_is_quorum_changed (conf->opts, key, NULL))
+                quorum_action = _gf_true;
+
+        ret = dict_set_dynstr (conf->opts, GLUSTERD_GLOBAL_OPT_VERSION,
+                               next_version);
+        if (ret)
+                goto out;
+        else
+                next_version = NULL;
+
+        if (!all) {
+                dict_del (conf->opts, key);
+        } else {
+                dict_foreach (conf->opts, _delete_reconfig_global_opt,
+                              &is_force);
+        }
+out:
+        GF_FREE (key_fixed);
+        if (dup_opt)
+                dict_unref (dup_opt);
+
+        gf_log (this->name, GF_LOG_DEBUG, "returning %d", ret);
+        if (quorum_action)
+                glusterd_do_quorum_action ();
+        GF_FREE (next_version);
+        return ret;
+}
 
 static int
 glusterd_op_reset_volume (dict_t *dict, char **op_errstr)
@@ -1061,14 +1206,23 @@ glusterd_op_reset_volume (dict_t *dict, char **op_errstr)
         int                     ret         = -1;
         char                    *volname    = NULL;
         char                    *key        = NULL;
+        char                    *key_fixed  = NULL;
         int32_t                 is_force    = 0;
+        gf_boolean_t            quorum_action = _gf_false;
+        xlator_t                *this         = NULL;
 
+        this = THIS;
         ret = dict_get_str (dict, "volname", &volname);
         if (ret) {
                 gf_log ("", GF_LOG_ERROR, "Unable to get volume name " );
                 goto out;
         }
 
+        if (strcasecmp (volname, "all") == 0) {
+                ret = glusterd_op_reset_all_volume_options (this, dict);
+                goto out;
+        }
+
         ret = dict_get_int32 (dict, "force", &is_force);
         if (ret)
                 is_force = 0;
@@ -1085,6 +1239,20 @@ glusterd_op_reset_volume (dict_t *dict, char **op_errstr)
                 goto out;
         }
 
+        if (strcmp (key, "all") &&
+            glusterd_check_option_exists (key, &key_fixed) != 1) {
+                gf_log ("glusterd", GF_LOG_ERROR,
+                        "volinfo dict inconsistency: option %s not found",
+                        key);
+                ret = -1;
+                goto out;
+        }
+        if (key_fixed)
+                key = key_fixed;
+
+        if (glusterd_is_quorum_changed (volinfo->dict, key, NULL))
+                quorum_action = _gf_true;
+
         ret = glusterd_options_reset (volinfo, key, &is_force);
         if (is_force == -1) {
                 ret = -1;
@@ -1093,12 +1261,14 @@ glusterd_op_reset_volume (dict_t *dict, char **op_errstr)
         }
 
 out:
+        GF_FREE (key_fixed);
+        if (quorum_action)
+                glusterd_do_quorum_action ();
+
         gf_log ("", GF_LOG_DEBUG, "'volume reset' returning %d", ret);
         return ret;
-
 }
 
-
 int
 glusterd_stop_bricks (glusterd_volinfo_t *volinfo)
 {
@@ -1128,6 +1298,91 @@ glusterd_start_bricks (glusterd_volinfo_t *volinfo)
 }
 
 static int
+glusterd_op_set_all_volume_options (xlator_t *this, dict_t *dict)
+{
+        char            *key            = NULL;
+        char            *key_fixed      = NULL;
+        char            *value          = NULL;
+        char            *dup_value      = NULL;
+        int             ret             = -1;
+        glusterd_conf_t *conf           = NULL;
+        dict_t          *dup_opt        = NULL;
+        char            *next_version   = NULL;
+        gf_boolean_t    quorum_action   = _gf_false;
+
+        conf = this->private;
+        ret = dict_get_str (dict, "key1", &key);
+        if (ret)
+                goto out;
+
+        ret = dict_get_str (dict, "value1", &value);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "invalid key,value pair in 'volume set'");
+                goto out;
+        }
+        ret = glusterd_check_option_exists (key, &key_fixed);
+        if (ret <= 0) {
+                gf_log (this->name, GF_LOG_ERROR, "Invalid key %s", key);
+                ret = -1;
+                goto out;
+        }
+
+        if (key_fixed)
+                key = key_fixed;
+
+        ret = -1;
+        dup_opt = dict_new ();
+        if (!dup_opt)
+                goto out;
+        dict_copy (conf->opts, dup_opt);
+        ret = dict_set_str (dup_opt, key, value);
+        if (ret)
+                goto out;
+
+        ret = glusterd_get_next_global_opt_version_str (conf->opts,
+                                                        &next_version);
+        if (ret)
+                goto out;
+
+        ret = dict_set_str (dup_opt, GLUSTERD_GLOBAL_OPT_VERSION, next_version);
+        if (ret)
+                goto out;
+
+        dup_value = gf_strdup (value);
+        if (!dup_value)
+                goto out;
+
+        ret = glusterd_store_options (this, dup_opt);
+        if (ret)
+                goto out;
+
+        if (glusterd_is_quorum_changed (conf->opts, key, value))
+                quorum_action = _gf_true;
+
+        ret = dict_set_dynstr (conf->opts, GLUSTERD_GLOBAL_OPT_VERSION,
+                               next_version);
+        if (ret)
+                goto out;
+        else
+                next_version = NULL;
+
+        ret = dict_set_dynstr (conf->opts, key, dup_value);
+        if (ret)
+                goto out;
+out:
+        GF_FREE (key_fixed);
+        if (dup_opt)
+                dict_unref (dup_opt);
+
+        gf_log (this->name, GF_LOG_DEBUG, "returning %d", ret);
+        if (quorum_action)
+                glusterd_do_quorum_action ();
+        GF_FREE (next_version);
+        return ret;
+}
+
+static int
 glusterd_op_set_volume (dict_t *dict)
 {
         int                                      ret = 0;
@@ -1146,7 +1401,7 @@ glusterd_op_set_volume (dict_t *dict)
         int32_t                                  dict_count = 0;
         gf_boolean_t                             check_op_version = _gf_false;
         uint32_t                                 new_op_version = 0;
-
+        gf_boolean_t                            quorum_action  = _gf_false;
 
         this = THIS;
         GF_ASSERT (this);
@@ -1176,6 +1431,11 @@ glusterd_op_set_volume (dict_t *dict)
                 goto out;
         }
 
+        if (strcasecmp (volname, "all") == 0) {
+                ret = glusterd_op_set_all_volume_options (this, dict);
+                goto out;
+        }
+
         ret = glusterd_volinfo_find (volname, &volinfo);
         if (ret) {
                 gf_log (this->name, GF_LOG_ERROR, "Unable to allocate memory");
@@ -1242,7 +1502,6 @@ glusterd_op_set_volume (dict_t *dict)
                                 key_fixed = NULL;
                                 goto out;
                         }
-
                 }
 
                 if (glusterd_check_globaloption (key))
@@ -1261,6 +1520,9 @@ glusterd_op_set_volume (dict_t *dict)
                 if (key_fixed)
                         key = key_fixed;
 
+                if (glusterd_is_quorum_changed (volinfo->dict, key, value))
+                        quorum_action = _gf_true;
+
                 if (global_opt) {
                        list_for_each_entry (voliter, &priv->volumes, vol_list) {
                                value = gf_strdup (value);
@@ -1350,6 +1612,8 @@ glusterd_op_set_volume (dict_t *dict)
  out:
         GF_FREE (key_fixed);
         gf_log (this->name, GF_LOG_DEBUG, "returning %d", ret);
+        if (quorum_action)
+                glusterd_do_quorum_action ();
         return ret;
 }
 
@@ -1875,20 +2139,26 @@ glusterd_op_ac_lock (glusterd_op_sm_event_t *event, void *ctx)
 static int
 glusterd_op_ac_unlock (glusterd_op_sm_event_t *event, void *ctx)
 {
-        int ret = 0;
+        int                      ret = 0;
         glusterd_op_lock_ctx_t   *lock_ctx = NULL;
+        xlator_t                 *this = NULL;
+        glusterd_conf_t          *priv = NULL;
 
         GF_ASSERT (event);
         GF_ASSERT (ctx);
 
+        this = THIS;
+        priv = this->private;
         lock_ctx = (glusterd_op_lock_ctx_t *)ctx;
 
         ret = glusterd_unlock (lock_ctx->uuid);
 
-        gf_log ("", GF_LOG_DEBUG, "Unlock Returned %d", ret);
+        gf_log (this->name, GF_LOG_DEBUG, "Unlock Returned %d", ret);
 
         glusterd_op_unlock_send_resp (lock_ctx->req, ret);
 
+        if (priv->pending_quorum_action)
+                glusterd_do_quorum_action ();
         return ret;
 }
 
@@ -2042,7 +2312,8 @@ glusterd_op_build_payload (dict_t **req, char **op_errstr, dict_t *op_ctx)
                                         goto out;
                                 }
                                 if (strcmp (volname, "help") &&
-                                    strcmp (volname, "help-xml")) {
+                                    strcmp (volname, "help-xml") &&
+                                    strcasecmp (volname, "all")) {
                                         ret = glusterd_dict_set_volid
                                                  (dict, volname, op_errstr);
                                         if (ret)
@@ -2095,10 +2366,13 @@ glusterd_op_build_payload (dict_t **req, char **op_errstr, dict_t *op_ctx)
                                         goto out;
                                 }
 
-                                ret = glusterd_dict_set_volid (dict, volname,
-                                                               op_errstr);
-                                if (ret)
-                                        goto out;
+                                if (strcasecmp (volname, "all")) {
+                                        ret = glusterd_dict_set_volid (dict,
+                                                                       volname,
+                                                                     op_errstr);
+                                        if (ret)
+                                                goto out;
+                                }
                                 dict_copy (dict, req_dict);
                         }
                         break;
@@ -2114,6 +2388,105 @@ out:
         return ret;
 }
 
+gf_boolean_t
+glusterd_is_get_op (xlator_t *this, glusterd_op_t op, dict_t *dict)
+{
+        char            *key = NULL;
+        char            *volname = NULL;
+        int             ret = 0;
+
+        if (op == GD_OP_STATUS_VOLUME)
+                return _gf_true;
+
+        if ((op == GD_OP_SET_VOLUME)) {
+                //check for set volume help
+                ret = dict_get_str (dict, "volname", &volname);
+                if (volname &&
+                    ((strcmp (volname, "help") == 0) ||
+                     (strcmp (volname, "help-xml") == 0))) {
+                        ret = dict_get_str (dict, "key1", &key);
+                        if (ret < 0)
+                                return _gf_true;
+                }
+        }
+
+        return _gf_false;
+}
+
+gf_boolean_t
+glusterd_is_op_quorum_validation_required (xlator_t *this, glusterd_op_t op,
+                                           dict_t *dict)
+{
+        gf_boolean_t    required = _gf_true;
+        char            *key = NULL;
+        char            *key_fixed = NULL;
+        int             ret = -1;
+
+        if (glusterd_is_get_op (this, op, dict)) {
+                required = _gf_false;
+                goto out;
+        }
+        if ((op != GD_OP_SET_VOLUME) && (op != GD_OP_RESET_VOLUME))
+                goto out;
+        if (op == GD_OP_SET_VOLUME)
+                ret = dict_get_str (dict, "key1", &key);
+        else if (op == GD_OP_RESET_VOLUME)
+                ret = dict_get_str (dict, "key", &key);
+        if (ret)
+                goto out;
+        ret = glusterd_check_option_exists (key, &key_fixed);
+        if (ret <= 0)
+                goto out;
+        if (key_fixed)
+                key = key_fixed;
+        if (glusterd_is_quorum_option (key))
+                required = _gf_false;
+out:
+        GF_FREE (key_fixed);
+        return required;
+}
+
+static int
+glusterd_op_validate_quorum (xlator_t *this, glusterd_op_t op,
+                             dict_t *dict, char **op_errstr)
+{
+        int                     ret = 0;
+        char                    *volname = NULL;
+        glusterd_volinfo_t      *volinfo = NULL;
+        char                    *errstr = NULL;
+
+
+        errstr = "Quorum not met. Volume operation not allowed.";
+        if (!glusterd_is_op_quorum_validation_required (this, op, dict))
+                goto out;
+
+        ret = dict_get_str (dict, "volname", &volname);
+        if (ret) {
+                ret = 0;
+                goto out;
+        }
+
+        ret = glusterd_volinfo_find (volname, &volinfo);
+        if (ret) {
+                ret = 0;
+                goto out;
+        }
+
+        if (does_gd_meet_server_quorum (this)) {
+                ret = 0;
+                goto out;
+        }
+
+        if (glusterd_is_volume_in_server_quorum (volinfo)) {
+                ret = -1;
+                *op_errstr = gf_strdup (errstr);
+                goto out;
+        }
+        ret = 0;
+out:
+        return ret;
+}
+
 static int
 glusterd_op_ac_send_stage_op (glusterd_op_sm_event_t *event, void *ctx)
 {
@@ -2141,6 +2514,13 @@ glusterd_op_ac_send_stage_op (glusterd_op_sm_event_t *event, void *ctx)
                 goto out;
         }
 
+        ret = glusterd_op_validate_quorum (this, op, dict, &op_errstr);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, op_errstr);
+                opinfo.op_errstr = op_errstr;
+                goto out;
+        }
+
         /* rsp_dict NULL from source */
         ret = glusterd_op_stage_validate (op, dict, &op_errstr, NULL);
         if (ret) {
@@ -2817,6 +3197,8 @@ glusterd_op_txn_complete ()
                 GF_FREE (op_errstr);
 
 
+        if (priv->pending_quorum_action)
+                glusterd_do_quorum_action ();
         gf_log ("glusterd", GF_LOG_DEBUG, "Returning %d", ret);
         return ret;
 }